From c130f9e68037a74003ca118210eeaa7d06145b62 Mon Sep 17 00:00:00 2001 From: dawn <90008@gaze.systems> Date: Sun, 26 Apr 2026 17:43:29 +0300 Subject: [PATCH] memory and stuff --- Cargo.lock | 95 + Cargo.toml | 1 + benchmarks/README.md | 83 + .../inputs/configs/mvp_exact_retrieval.json | 23 + .../mvp_rerank_calibrated_balanced.json | 23 + .../mvp_rerank_calibrated_conservative.json | 23 + .../mvp_rerank_support_calibrated.json | 25 + .../mvp_rerank_support_calibrated_test.json | 25 + .../internal_eval_passive_recall_starter.json | 600 ++ .../internal_eval_passive_recall_v3.json | 1319 +++ .../datasets/internal_eval_starter.json | 862 ++ .../datasets/internal_eval_template.json | 58 + .../inputs/router/router_holdout_v1.json | 610 ++ .../holdout_abstain_toolish_v1.json | 613 ++ .../holdout_memory_toolish_v1.json | 613 ++ .../holdout_tools_recall_v1.json | 616 ++ .../router/router_slices/abstain_v1.json | 307 + .../router/router_slices/abstain_v2.json | 312 + .../router/router_slices/abstain_v3.json | 309 + .../router/router_slices/dev_pack_v1.json | 100 + .../router_slices/memory_adversarial_v1.json | 35 + .../router/router_slices/memory_lane_v1.json | 307 + .../router/router_slices/memory_lane_v1b.json | 307 + .../router/router_slices/memory_lane_v2.json | 21 + .../router/router_slices/memory_lane_v3.json | 229 + .../router_slices/read_file_repo_v1.json | 307 + .../router_slices/read_file_repo_v1b.json | 307 + .../router_slices/read_file_repo_v2.json | 31 + .../router_slices/read_file_repo_v3.json | 209 + .../router_slices/read_file_repo_v4.json | 249 + .../router/router_slices/shell_state_v1.json | 307 + .../router/router_slices/shell_state_v1b.json | 307 + .../router/router_slices/shell_state_v2.json | 47 + .../router/router_slices/shell_state_v3.json | 249 + .../router/router_slices/train_pack_v1.json | 259 + .../router/router_slices/write_file_v1.json | 307 + benchmarks/inputs/router/router_tools.json | 251 + .../out-router-iter-1/router_model.json | 2058 +++++ .../out-router-iter-1/router_report.json | 16 + .../out-router-iter-1/router_report.md | 16 + .../out-router-iter-2/router_model.json | 2058 +++++ .../out-router-iter-2/router_report.json | 16 + .../out-router-iter-2/router_report.md | 16 + .../out-router-iter-3/router_model.json | 3084 +++++++ .../out-router-iter-3/router_report.json | 421 + .../out-router-iter-3/router_report.md | 88 + .../out-router-iter-4/router_model.json | 3084 +++++++ .../out-router-iter-4/router_report.json | 640 ++ .../out-router-iter-4/router_report.md | 90 + .../out-router-iter-5/router_model.json | 6168 +++++++++++++ .../out-router-iter-5/router_report.json | 604 ++ .../out-router-iter-5/router_report.md | 86 + .../out-router-iter-6/router_model.json | 6168 +++++++++++++ .../out-router-iter-6/router_report.json | 605 ++ .../out-router-iter-6/router_report.md | 87 + .../out-router-iter-7/router_model.json | 6168 +++++++++++++ .../out-router-iter-7/router_report.json | 317 + .../out-router-iter-7/router_report.md | 55 + .../router_model.json | 3104 +++++++ .../router_report.json | 439 + .../out-router-linear-iter-1/router_report.md | 81 + .../router_model.json | 3104 +++++++ .../router_report.json | 1717 ++++ .../router_report.md | 146 + .../router_model.json | 3104 +++++++ .../router_report.json | 1717 ++++ .../router_report.md | 146 + .../router_model.json | 3104 +++++++ .../router_report.json | 1717 ++++ .../router_report.md | 146 + .../router_model.json | 3104 +++++++ .../router_report.json | 1717 ++++ .../router_report.md | 146 + .../router_model.json | 3104 +++++++ .../router_report.json | 1717 ++++ .../router_report.md | 146 + .../router_model.json | 3104 +++++++ .../router_report.json | 439 + .../out-router-linear-iter-2/router_report.md | 81 + .../router_model.json | 3104 +++++++ .../router_report.json | 434 + .../out-router-linear-iter-3/router_report.md | 81 + .../router_model.json | 3104 +++++++ .../router_report.json | 560 ++ .../out-router-linear-iter-4/router_report.md | 88 + .../router_model.json | 3104 +++++++ .../router_report.json | 1433 +++ .../out-router-linear-iter-5/router_report.md | 151 + .../router_model.json | 3104 +++++++ .../router_report.json | 1942 ++++ .../out-router-linear-iter-6/router_report.md | 159 + .../router_model.json | 3104 +++++++ .../router_report.json | 1942 ++++ .../out-router-linear-iter-7/router_report.md | 159 + .../router_model.json | 3104 +++++++ .../router_report.json | 1006 +++ .../out-router-linear-iter-8/router_report.md | 101 + .../router_model.json | 3104 +++++++ .../router_report.json | 1006 +++ .../out-router-linear-iter-9/router_report.md | 101 + benchmarks/run_all.py | 287 + .../runs/2026-04-24_suite_smoke/manifest.json | 117 + .../passive-dev/report.json | 55 + .../passive-dev/report.md | 30 + .../passive-dev/resolved_config.json | 29 + .../passive-dev/traces.json | 7638 ++++++++++++++++ .../passive-test/report.json | 55 + .../passive-test/report.md | 30 + .../passive-test/resolved_config.json | 29 + .../passive-test/traces.json | 7829 +++++++++++++++++ .../retrieval-dev/report.json | 61 + .../retrieval-dev/report.md | 36 + .../retrieval-dev/resolved_config.json | 29 + .../retrieval-dev/traces.json | 5704 ++++++++++++ .../retrieval-test/report.json | 61 + .../retrieval-test/report.md | 36 + .../retrieval-test/resolved_config.json | 29 + .../retrieval-test/traces.json | 4969 +++++++++++ .../runs/2026-04-24_suite_smoke/summary.md | 64 + .../tools-lane-dev/report.json | 25 + .../tools-lane-dev/report.md | 37 + .../tools-lane-dev/tool_lane_agent.db | Bin 0 -> 40960 bytes .../tools-lane-dev/traces.json | 570 ++ .../tools-lane-test/report.json | 25 + .../tools-lane-test/report.md | 33 + .../tools-lane-test/tool_lane_agent.db | Bin 0 -> 40960 bytes .../tools-lane-test/traces.json | 326 + docs/klbr-research-report.md | 179 + docs/klbr_benchmark_readout_2026-04-22.md | 370 + docs/klbr_benchmark_readout_2026-04-23.md | 314 + ...br_benchmark_readout_2026-04-23_end2end.md | 127 + docs/klbr_memory_design.md | 215 + ...r_mvp_benchmarking_and_calibration_plan.md | 825 ++ docs/klbr_mvp_contract.md | 164 + docs/klbr_passive_recall_benchmark_plan.md | 132 + docs/klbr_rerank_sweep_2026-04-22.md | 390 + docs/router_dataset_log.md | 194 + docs/router_linear_latest_2026-04-23.md | 102 + klbr-bench/Cargo.toml | 13 + klbr-bench/src/main.rs | 5186 +++++++++++ klbr-core/Cargo.toml | 1 + klbr-core/src/agent.rs | 297 +- klbr-core/src/config.rs | 28 +- klbr-core/src/context.rs | 36 +- klbr-core/src/interrupt.rs | 5 +- klbr-core/src/lib.rs | 3 + klbr-core/src/llm.rs | 163 +- klbr-core/src/memory.rs | 270 +- klbr-core/src/mvp.rs | 496 ++ klbr-core/src/reflection.md | 20 + klbr-core/src/retrieval.rs | 237 + klbr-core/src/router.rs | 435 + klbr-core/src/tools/mod.rs | 1 + klbr-core/src/tools/recall.rs | 49 +- klbr-core/src/tools/remember.rs | 32 +- 155 files changed, 143422 insertions(+), 131 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/inputs/configs/mvp_exact_retrieval.json create mode 100644 benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json create mode 100644 benchmarks/inputs/configs/mvp_rerank_calibrated_conservative.json create mode 100644 benchmarks/inputs/configs/mvp_rerank_support_calibrated.json create mode 100644 benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json create mode 100644 benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json create mode 100644 benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json create mode 100644 benchmarks/inputs/datasets/internal_eval_starter.json create mode 100644 benchmarks/inputs/datasets/internal_eval_template.json create mode 100644 benchmarks/inputs/router/router_holdout_v1.json create mode 100644 benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json create mode 100644 benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json create mode 100644 benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json create mode 100644 benchmarks/inputs/router/router_slices/abstain_v1.json create mode 100644 benchmarks/inputs/router/router_slices/abstain_v2.json create mode 100644 benchmarks/inputs/router/router_slices/abstain_v3.json create mode 100644 benchmarks/inputs/router/router_slices/dev_pack_v1.json create mode 100644 benchmarks/inputs/router/router_slices/memory_adversarial_v1.json create mode 100644 benchmarks/inputs/router/router_slices/memory_lane_v1.json create mode 100644 benchmarks/inputs/router/router_slices/memory_lane_v1b.json create mode 100644 benchmarks/inputs/router/router_slices/memory_lane_v2.json create mode 100644 benchmarks/inputs/router/router_slices/memory_lane_v3.json create mode 100644 benchmarks/inputs/router/router_slices/read_file_repo_v1.json create mode 100644 benchmarks/inputs/router/router_slices/read_file_repo_v1b.json create mode 100644 benchmarks/inputs/router/router_slices/read_file_repo_v2.json create mode 100644 benchmarks/inputs/router/router_slices/read_file_repo_v3.json create mode 100644 benchmarks/inputs/router/router_slices/read_file_repo_v4.json create mode 100644 benchmarks/inputs/router/router_slices/shell_state_v1.json create mode 100644 benchmarks/inputs/router/router_slices/shell_state_v1b.json create mode 100644 benchmarks/inputs/router/router_slices/shell_state_v2.json create mode 100644 benchmarks/inputs/router/router_slices/shell_state_v3.json create mode 100644 benchmarks/inputs/router/router_slices/train_pack_v1.json create mode 100644 benchmarks/inputs/router/router_slices/write_file_v1.json create mode 100644 benchmarks/inputs/router/router_tools.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-1/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-1/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-1/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-2/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-2/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-2/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-3/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-3/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-3/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-4/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-4/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-4/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-5/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-5/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-5/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-6/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-6/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-6/router_report.md create mode 100644 benchmarks/models/router/centroid/out-router-iter-7/router_model.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-7/router_report.json create mode 100644 benchmarks/models/router/centroid/out-router-iter-7/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-1/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-1/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-10/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-10/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-10/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-11/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-11/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-11/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-12/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-12/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-12/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-13/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-13/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-13/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-14/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-14/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-14/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-2/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-2/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-2/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-3/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-3/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-3/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-4/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-4/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-4/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-5/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-5/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-5/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-6/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-6/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-6/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-7/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-7/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-7/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-8/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-8/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-8/router_report.md create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-9/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md create mode 100755 benchmarks/run_all.py create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/manifest.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-dev/resolved_config.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-dev/traces.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-test/resolved_config.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/passive-test/traces.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/resolved_config.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/traces.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/resolved_config.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/traces.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/summary.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_agent.db create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/traces.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.json create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.md create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/tool_lane_agent.db create mode 100644 benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/traces.json create mode 100644 docs/klbr-research-report.md create mode 100644 docs/klbr_benchmark_readout_2026-04-22.md create mode 100644 docs/klbr_benchmark_readout_2026-04-23.md create mode 100644 docs/klbr_benchmark_readout_2026-04-23_end2end.md create mode 100644 docs/klbr_memory_design.md create mode 100644 docs/klbr_mvp_benchmarking_and_calibration_plan.md create mode 100644 docs/klbr_mvp_contract.md create mode 100644 docs/klbr_passive_recall_benchmark_plan.md create mode 100644 docs/klbr_rerank_sweep_2026-04-22.md create mode 100644 docs/router_dataset_log.md create mode 100644 docs/router_linear_latest_2026-04-23.md create mode 100644 klbr-bench/Cargo.toml create mode 100644 klbr-bench/src/main.rs create mode 100644 klbr-core/src/mvp.rs create mode 100644 klbr-core/src/reflection.md create mode 100644 klbr-core/src/retrieval.rs create mode 100644 klbr-core/src/router.rs diff --git a/Cargo.lock b/Cargo.lock index 41cb40e..192a3ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -165,6 +174,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "cmake" version = "0.1.58" @@ -847,6 +869,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -1104,11 +1150,25 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "klbr-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "klbr-core", + "serde", + "serde_json", + "tempfile", + "tokio", + "unicode-segmentation", +] + [[package]] name = "klbr-core" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "dirs", "futures", "reqwest", @@ -3090,6 +3150,41 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-link" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 9d1bcb2..a027f4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "klbr-core", + "klbr-bench", "klbr-daemon", "klbr-ipc", "klbr-tui", diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..7a4a169 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,83 @@ +# Benchmarks + +This directory is organized as: + +- `benchmarks/inputs/` datasets + configs (hand-edited, source of truth) +- `benchmarks/models/` trained artifacts (router models + reports) +- `benchmarks/runs/` benchmark outputs (generated) + +`benchmarks/inputs/datasets/internal_eval_template.json` is the minimal smoke-test dataset. + +`benchmarks/inputs/datasets/internal_eval_starter.json` is the first non-toy internal eval set. It is still small enough to edit by hand, but it includes: + +- all required query categories from the MVP plan +- distractor memories +- stale/new fact conflicts +- no-hit queries +- multi-evidence queries +- `timeline_id` on both memories and queries so split enforcement can avoid timeline leakage + +`benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json` is the starter slice for non-question memory activation and passive recall support. + +Current harness behavior: + +- `klbr-bench` filters queries by `split` +- `cargo run -p klbr-bench -- passive-recall ...` runs the passive memory-activation benchmark on queries with `objective = passive_recall` +- if the dataset includes timeline metadata and the experiment selects a split, it also filters the memory corpus by `timeline_id` +- mixed split datasets are treated strictly: a split with timeline metadata must give every query in that split a `timeline_id` +- `cargo run -p klbr-bench -- sweep ...` reruns the real expand-or-abstain policy over a grid of raw rerank score, raw margin, and support thresholds, then writes `sweep_report.json`, `sweep_report.md`, and `sweep_results.csv` +- per-query traces now include `window_attempts`, so each attempted time window records raw rerank score, raw margin, top-candidate support score, candidate ids, and the policy decision at that window +- `benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json` and `benchmarks/inputs/configs/mvp_rerank_calibrated_conservative.json` are provisional threshold presets from the 2026-04-22 sweeps; use them as candidate operating modes rather than treating them as final defaults +- `benchmarks/inputs/configs/mvp_rerank_support_calibrated.json` is the first support-assisted preset from the focused 2026-04-22 support sweep + +If a dataset has no `timeline_id` metadata at all, the harness falls back to the full corpus for backward compatibility. + +Recommended workflow: + +1. add new memories in timeline order +2. group related memories under one `timeline_id` +3. add queries that reference only the intended memories +4. keep hard distractors near the gold memories in wording and time +5. make sure each split contains no-hit and conflict/update cases + +Passive recall planning now lives in: + +- `docs/klbr_passive_recall_benchmark_plan.md` + +`benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json` is the expanded v3 passive-recall slice. Added on top of v2: + +- 16 new memories (ids 123–138) across 6 new timelines +- 12 new queries (5 dev, 7 test) covering: + - **temporal ordering cues** — "before we switched embeddings", time-anchored update references + - **ambiguous time references** — "the other day I moved back to neovim" + - **update/staleness signals** — `interaction_mode: "update"` cases where user signals a prior memory is now stale + - **abstention / no-recall false technical cue** — sentences with technical words that should still produce no recall (e.g. "internet keeps dropping") + - **multi-session back-references** — referencing a decision made in a prior session + - **context-switch no-recall** — user switches to an entirely different project + +`benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json` is the same operating point as `benchmarks/inputs/configs/mvp_rerank_support_calibrated.json` with `dataset_split: "test"`. + +Run dev and test splits of v3: + +```bash +cargo run -p klbr-bench -- retrieval \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/runs/manual-retrieval-dev + +cargo run -p klbr-bench -- passive-recall \ + benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/runs/manual-passive-recall-v3-dev + +cargo run -p klbr-bench -- passive-recall \ + benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json \ + benchmarks/runs/manual-passive-recall-v3-test +``` + +Known false activation on the test split: `pr_test_14` ("working on the dotfiles syncer today, not touching klbr") is triggered because memory 131 ("side project: rust cli for syncing dotfiles") sits in the same corpus and gets a rerank score above the abstain threshold. This is a real system weakness — cross-project memory bleed when the dotfiles project shares vocabulary with klbr sessions. + +## Run Everything Into One Folder + +Use `benchmarks/run_all.py` to run the standard suite (retrieval dev/test, passive dev/test, tools-lane dev/test) into a single run directory under `benchmarks/runs/` and write a consolidated `summary.md`. diff --git a/benchmarks/inputs/configs/mvp_exact_retrieval.json b/benchmarks/inputs/configs/mvp_exact_retrieval.json new file mode 100644 index 0000000..47e68a9 --- /dev/null +++ b/benchmarks/inputs/configs/mvp_exact_retrieval.json @@ -0,0 +1,23 @@ +{ + "experiment_id": "mvp-exact-retrieval-baseline", + "config_version": "2026-04-22", + "notes": "Phase 0 / Week 1 exact retrieval baseline with time-window expansion.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": null, + "rerank_abstain_margin_threshold": null, + "support_abstain_threshold": null +} diff --git a/benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json b/benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json new file mode 100644 index 0000000..b82254c --- /dev/null +++ b/benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json @@ -0,0 +1,23 @@ +{ + "experiment_id": "mvp-rerank-calibrated-balanced", + "config_version": "2026-04-22", + "notes": "Balanced provisional operating point from the 2026-04-22 fine sweep. Keeps answerable coverage high at the cost of one false answer on the current dev slice.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": null +} diff --git a/benchmarks/inputs/configs/mvp_rerank_calibrated_conservative.json b/benchmarks/inputs/configs/mvp_rerank_calibrated_conservative.json new file mode 100644 index 0000000..27bf992 --- /dev/null +++ b/benchmarks/inputs/configs/mvp_rerank_calibrated_conservative.json @@ -0,0 +1,23 @@ +{ + "experiment_id": "mvp-rerank-calibrated-conservative", + "config_version": "2026-04-22", + "notes": "Conservative provisional operating point from the 2026-04-22 fine sweep. Eliminates false answers on the current dev slice, but answerable coverage drops materially.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -2.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": null +} diff --git a/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json b/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json new file mode 100644 index 0000000..8da6791 --- /dev/null +++ b/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json @@ -0,0 +1,25 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} diff --git a/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json b/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json new file mode 100644 index 0000000..701f236 --- /dev/null +++ b/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json @@ -0,0 +1,25 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Same operating point as mvp_rerank_support_calibrated.json but evaluated on the test split.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} diff --git a/benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json b/benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json new file mode 100644 index 0000000..ead80e9 --- /dev/null +++ b/benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json @@ -0,0 +1,600 @@ +{ + "dataset_id": "internal-eval-passive-recall-starter-v2", + "description": "Expanded passive-recall eval slice for KLBR. Covers activation, no-recall behavior, multi-evidence support, and update-style cues across dev and test splits.", + "memories": [ + { + "memory_id": 101, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "switched the day-to-day klbr editor from neovim to zed.", + "event_time": 1776157200, + "status": "active", + "source_ref": "session:pr1:m1", + "tags": ["project:klbr", "topic:editor"] + }, + { + "memory_id": 102, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024.", + "event_time": 1776159000, + "status": "active", + "source_ref": "session:pr1:m2", + "tags": ["project:klbr", "topic:embeddings"] + }, + { + "memory_id": 103, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr2-reranker-dev", + "text": "the reranker endpoint is http://localhost:8003.", + "event_time": 1776686400, + "status": "active", + "source_ref": "session:pr2:m1", + "tags": ["project:klbr", "topic:reranker", "topic:networking"] + }, + { + "memory_id": 104, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr2-reranker-dev", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3.", + "event_time": 1776425400, + "status": "active", + "source_ref": "session:pr2:m2", + "tags": ["project:klbr", "topic:reranker"] + }, + { + "memory_id": 105, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr3-recent-work-dev", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding.", + "event_time": 1776355200, + "status": "active", + "source_ref": "session:pr3:m1", + "tags": ["project:klbr", "topic:benchmarking", "topic:retrieval"] + }, + { + "memory_id": 106, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr3-recent-work-dev", + "text": "added progress logging to the benchmark harness because silent hangs were confusing.", + "event_time": 1776614400, + "status": "active", + "source_ref": "session:pr3:m2", + "tags": ["project:klbr", "topic:benchmarking", "topic:logging"] + }, + { + "memory_id": 107, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr4-service-separation-dev", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch.", + "event_time": 1776618000, + "status": "active", + "source_ref": "session:pr4:m1", + "tags": ["project:klbr", "topic:infra", "topic:retrieval"] + }, + { + "memory_id": 108, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr5-errors-dev", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui.", + "event_time": 1776250800, + "status": "active", + "source_ref": "session:pr5:m1", + "tags": ["project:klbr", "topic:daemon", "topic:tui"] + }, + { + "memory_id": 109, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text.", + "event_time": 1776861000, + "status": "active", + "source_ref": "session:pr6:m1", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 110, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2.", + "event_time": 1776862800, + "status": "active", + "source_ref": "session:pr6:m2", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 111, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode.", + "event_time": 1776864600, + "status": "active", + "source_ref": "session:pr6:m3", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 112, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards.", + "event_time": 1776866400, + "status": "active", + "source_ref": "session:pr7:m1", + "tags": ["project:klbr", "topic:passive-recall", "topic:support"] + }, + { + "memory_id": 113, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support.", + "event_time": 1776868200, + "status": "active", + "source_ref": "session:pr7:m2", + "tags": ["project:klbr", "topic:passive-recall", "topic:calibration"] + }, + { + "memory_id": 114, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun.", + "event_time": 1776870000, + "status": "active", + "source_ref": "session:pr7:m3", + "tags": ["project:klbr", "topic:passive-recall", "topic:eval-data"] + }, + { + "memory_id": 115, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall.", + "event_time": 1776871800, + "status": "active", + "source_ref": "session:pr7:m4", + "tags": ["project:klbr", "topic:passive-recall", "topic:support"] + }, + { + "memory_id": 116, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries.", + "event_time": 1776873600, + "status": "active", + "source_ref": "session:pr8:m1", + "tags": ["project:klbr", "topic:eval-data"] + }, + { + "memory_id": 117, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases.", + "event_time": 1776875400, + "status": "active", + "source_ref": "session:pr8:m2", + "tags": ["project:klbr", "topic:eval-data", "topic:no-hit"] + }, + { + "memory_id": 118, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun.", + "event_time": 1776877200, + "status": "active", + "source_ref": "session:pr8:m3", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 119, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds.", + "event_time": 1776879000, + "status": "active", + "source_ref": "session:pr9:m1", + "tags": ["project:klbr", "topic:passive-recall", "topic:support"] + }, + { + "memory_id": 120, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues.", + "event_time": 1776880800, + "status": "active", + "source_ref": "session:pr9:m2", + "tags": ["project:klbr", "topic:passive-recall", "topic:eval-data"] + }, + { + "memory_id": 121, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only.", + "event_time": 1776882600, + "status": "active", + "source_ref": "session:pr9:m3", + "tags": ["project:klbr", "topic:passive-recall", "topic:eval-data"] + }, + { + "memory_id": 122, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94.", + "event_time": 1776884400, + "status": "active", + "source_ref": "session:pr9:m4", + "tags": ["project:klbr", "topic:passive-recall", "topic:metrics"] + } + ], + "queries": [ + { + "query_id": "pr_dev_1", + "split": "dev", + "category": "passive recall / progress update", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr3-recent-work-dev", + "text": "still working on the benchmark scaffolding and retrieval plumbing stuff today.", + "gold_memory_ids": [105], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_2", + "split": "dev", + "category": "passive recall / elliptical request", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr2-reranker-dev", + "text": "remind me which reranker model we settled on", + "gold_memory_ids": [104], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_3", + "split": "dev", + "category": "passive recall / declarative state cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "i'm back in zed again for klbr work.", + "gold_memory_ids": [101], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_4", + "split": "dev", + "category": "passive recall / service endpoint cue", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr2-reranker-dev", + "text": "reranker on 8003", + "gold_memory_ids": [103], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_5", + "split": "dev", + "category": "passive recall / multi-evidence state", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr4-service-separation-dev", + "text": "keeping chat, embeddings, and rerank split out seems like the cleanest setup.", + "gold_memory_ids": [107, 102, 103], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_6", + "split": "dev", + "category": "passive recall / no recall social chatter", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "lol yeah that makes sense", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_7", + "split": "dev", + "category": "passive recall / no recall generic status", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "i'm kind of tired today", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_8", + "split": "dev", + "category": "passive recall / error context cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "the tui finally shows the stream errors now.", + "gold_memory_ids": [108], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_9", + "split": "dev", + "category": "passive recall / calibration result statement", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr6-support-calibration-dev", + "text": "the support feature ended up mattering more than i expected.", + "gold_memory_ids": [109, 110], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776864600 + }, + { + "query_id": "pr_dev_10", + "split": "dev", + "category": "passive recall / anti taxonomy cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "i still don't want to solve this with query-type guards.", + "gold_memory_ids": [112], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_11", + "split": "dev", + "category": "passive recall / planning cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "before tuning passive recall further we needed a bigger eval slice.", + "gold_memory_ids": [114], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_12", + "split": "dev", + "category": "passive recall / support quality cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "activation seems fine but the recalled set is still kind of noisy.", + "gold_memory_ids": [115], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_13", + "split": "dev", + "category": "passive recall / policy framing cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "memory use really is activation first and then trust.", + "gold_memory_ids": [113], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_14", + "split": "dev", + "category": "passive recall / no recall off topic opinion", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "that color palette is nice tbh", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_15", + "split": "dev", + "category": "passive recall / no recall short acknowledgement", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "lol fair enough", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_test_1", + "split": "test", + "category": "passive recall / rerun result cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the bigger eval rerun still liked the support-calibrated point.", + "gold_memory_ids": [118], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_2", + "split": "test", + "category": "passive recall / expanded dataset cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the expanded main eval set is way less toy now.", + "gold_memory_ids": [116, 117], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_3", + "split": "test", + "category": "passive recall / planning request", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "what should the next passive recall pass focus on again", + "gold_memory_ids": [119], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_4", + "split": "test", + "category": "passive recall / dataset curation cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "we still need more no-recall and update-style passive items.", + "gold_memory_ids": [120], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_5", + "split": "test", + "category": "passive recall / split planning cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive benchmark should stop being dev-only.", + "gold_memory_ids": [121], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_6", + "split": "test", + "category": "passive recall / metric fragment", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "support precision was only around point three nine", + "gold_memory_ids": [122], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_7", + "split": "test", + "category": "passive recall / no recall ambient need", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "i need coffee lol", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_8", + "split": "test", + "category": "passive recall / no recall empathy", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "that sounds annoying", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_9", + "split": "test", + "category": "passive recall / multi-evidence calibration cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the rerun made the calibration feel more real because the dataset got bigger and the same point stayed best.", + "gold_memory_ids": [116, 118], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_10", + "split": "test", + "category": "passive recall / no recall short absence", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "brb for a sec", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + } + ] +} diff --git a/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json b/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json new file mode 100644 index 0000000..24d73cd --- /dev/null +++ b/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json @@ -0,0 +1,1319 @@ +{ + "dataset_id": "internal-eval-passive-recall-v3", + "description": "Expanded passive-recall eval slice for KLBR v3. Adds temporal ordering, ambiguous time references, update/staleness signals, abstention patterns, and multi-session back-references extracted from TemporalMemoryDataset, LongMemEval, and RealMemBench design patterns.", + "memories": [ + { + "memory_id": 101, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "switched the day-to-day klbr editor from neovim to zed.", + "event_time": 1776157200, + "status": "active", + "source_ref": "session:pr1:m1", + "tags": [ + "project:klbr", + "topic:editor" + ] + }, + { + "memory_id": 102, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024.", + "event_time": 1776159000, + "status": "active", + "source_ref": "session:pr1:m2", + "tags": [ + "project:klbr", + "topic:embeddings" + ] + }, + { + "memory_id": 103, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr2-reranker-dev", + "text": "the reranker endpoint is http://localhost:8003.", + "event_time": 1776686400, + "status": "active", + "source_ref": "session:pr2:m1", + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ] + }, + { + "memory_id": 104, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr2-reranker-dev", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3.", + "event_time": 1776425400, + "status": "active", + "source_ref": "session:pr2:m2", + "tags": [ + "project:klbr", + "topic:reranker" + ] + }, + { + "memory_id": 105, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr3-recent-work-dev", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding.", + "event_time": 1776355200, + "status": "active", + "source_ref": "session:pr3:m1", + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ] + }, + { + "memory_id": 106, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr3-recent-work-dev", + "text": "added progress logging to the benchmark harness because silent hangs were confusing.", + "event_time": 1776614400, + "status": "active", + "source_ref": "session:pr3:m2", + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ] + }, + { + "memory_id": 107, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr4-service-separation-dev", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch.", + "event_time": 1776618000, + "status": "active", + "source_ref": "session:pr4:m1", + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ] + }, + { + "memory_id": 108, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr5-errors-dev", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui.", + "event_time": 1776250800, + "status": "active", + "source_ref": "session:pr5:m1", + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ] + }, + { + "memory_id": 109, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text.", + "event_time": 1776861000, + "status": "active", + "source_ref": "session:pr6:m1", + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ] + }, + { + "memory_id": 110, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2.", + "event_time": 1776862800, + "status": "active", + "source_ref": "session:pr6:m2", + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ] + }, + { + "memory_id": 111, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr6-support-calibration-dev", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode.", + "event_time": 1776864600, + "status": "active", + "source_ref": "session:pr6:m3", + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ] + }, + { + "memory_id": 112, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards.", + "event_time": 1776866400, + "status": "active", + "source_ref": "session:pr7:m1", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ] + }, + { + "memory_id": 113, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support.", + "event_time": 1776868200, + "status": "active", + "source_ref": "session:pr7:m2", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ] + }, + { + "memory_id": 114, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun.", + "event_time": 1776870000, + "status": "active", + "source_ref": "session:pr7:m3", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ] + }, + { + "memory_id": 115, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr7-passive-policy-dev", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall.", + "event_time": 1776871800, + "status": "active", + "source_ref": "session:pr7:m4", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ] + }, + { + "memory_id": 116, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries.", + "event_time": 1776873600, + "status": "active", + "source_ref": "session:pr8:m1", + "tags": [ + "project:klbr", + "topic:eval-data" + ] + }, + { + "memory_id": 117, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases.", + "event_time": 1776875400, + "status": "active", + "source_ref": "session:pr8:m2", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ] + }, + { + "memory_id": 118, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr8-expanded-eval-test", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun.", + "event_time": 1776877200, + "status": "active", + "source_ref": "session:pr8:m3", + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ] + }, + { + "memory_id": 119, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds.", + "event_time": 1776879000, + "status": "active", + "source_ref": "session:pr9:m1", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ] + }, + { + "memory_id": 120, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues.", + "event_time": 1776880800, + "status": "active", + "source_ref": "session:pr9:m2", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ] + }, + { + "memory_id": 121, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only.", + "event_time": 1776882600, + "status": "active", + "source_ref": "session:pr9:m3", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ] + }, + { + "memory_id": 122, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr9-passive-followup-test", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94.", + "event_time": 1776884400, + "status": "active", + "source_ref": "session:pr9:m4", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ] + }, + { + "memory_id": 123, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr10-temporal-dev", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768.", + "event_time": 1776891600, + "status": "active", + "source_ref": "session:pr10:m1", + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ] + }, + { + "memory_id": 124, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr10-temporal-dev", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories.", + "event_time": 1776895200, + "status": "active", + "source_ref": "session:pr10:m2", + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ] + }, + { + "memory_id": 125, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr10-temporal-dev", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible.", + "event_time": 1776898800, + "status": "active", + "source_ref": "session:pr10:m3", + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ] + }, + { + "memory_id": 126, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr10-temporal-dev", + "text": "switched back to neovim for klbr work; zed was slow on the larger files.", + "event_time": 1776906000, + "status": "active", + "source_ref": "session:pr10:m4", + "tags": [ + "project:klbr", + "topic:editor" + ] + }, + { + "memory_id": 127, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr11-update-dev", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items.", + "event_time": 1776978000, + "status": "active", + "source_ref": "session:pr11:m1", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ] + }, + { + "memory_id": 128, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr11-update-dev", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next.", + "event_time": 1776981600, + "status": "active", + "source_ref": "session:pr11:m2", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ] + }, + { + "memory_id": 129, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr11-update-test", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings.", + "event_time": 1776985200, + "status": "active", + "source_ref": "session:pr11:m3", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ] + }, + { + "memory_id": 130, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr11-update-test", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision.", + "event_time": 1776988800, + "status": "active", + "source_ref": "session:pr11:m4", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ] + }, + { + "memory_id": 131, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr12-abstain-dev", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles.", + "event_time": 1777064400, + "status": "active", + "source_ref": "session:pr12:m1", + "tags": [ + "project:dotfiles", + "topic:cli" + ] + }, + { + "memory_id": 132, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr13-update-signals-test", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset.", + "event_time": 1777068000, + "status": "active", + "source_ref": "session:pr13:m1", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ] + }, + { + "memory_id": 133, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr13-update-signals-test", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter.", + "event_time": 1777071600, + "status": "active", + "source_ref": "session:pr13:m2", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ] + }, + { + "memory_id": 134, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr13-update-signals-test", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns.", + "event_time": 1777075200, + "status": "active", + "source_ref": "session:pr13:m3", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ] + }, + { + "memory_id": 135, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr14-temporal-context-test", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query.", + "event_time": 1777082400, + "status": "active", + "source_ref": "session:pr14:m1", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ] + }, + { + "memory_id": 136, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr14-temporal-context-test", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary.", + "event_time": 1777086000, + "status": "active", + "source_ref": "session:pr14:m2", + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ] + }, + { + "memory_id": 137, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr14-temporal-context-test", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction.", + "event_time": 1777089600, + "status": "active", + "source_ref": "session:pr14:m3", + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ] + }, + { + "memory_id": 138, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr14-temporal-context-test", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10.", + "event_time": 1777093200, + "status": "active", + "source_ref": "session:pr14:m4", + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ] + }, + { + "memory_id": 139, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr15-locomo-pattern-dev", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough.", + "event_time": 1772000000, + "status": "active", + "source_ref": "session:pr15:m1", + "tags": [ + "project:frontend", + "topic:planning" + ] + }, + { + "memory_id": 140, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr16-locomo-pattern-test", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling.", + "event_time": 1772005000, + "status": "active", + "source_ref": "session:pr16:m1", + "tags": [ + "project:database", + "topic:architecture" + ] + }, + { + "memory_id": 141, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr17-multi-session-test", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem.", + "event_time": 1772000000, + "status": "active", + "source_ref": "session:pr17:m1", + "tags": [ + "topic:preference", + "topic:python" + ] + }, + { + "memory_id": 142, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr17-multi-session-test", + "text": "starting a new project to scrape real estate data from multiple listing sites.", + "event_time": 1774000000, + "status": "active", + "source_ref": "session:pr17:m2", + "tags": [ + "project:scraper", + "topic:web-scraping" + ] + }, + { + "memory_id": 143, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr18-knowledge-update-test", + "text": "alice is the lead dev on the hydra project.", + "event_time": 1772000000, + "status": "active", + "source_ref": "session:pr18:m1", + "tags": [ + "project:hydra", + "topic:team" + ] + }, + { + "memory_id": 144, + "namespace": "default", + "layer": "L1", + "timeline_id": "pr18-knowledge-update-test", + "text": "bob has taken over the hydra project; alice moved to the agent team.", + "event_time": 1774000000, + "status": "active", + "source_ref": "session:pr18:m2", + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ] + } + ], + "queries": [ + { + "query_id": "pr_dev_1", + "split": "dev", + "category": "passive recall / progress update", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr3-recent-work-dev", + "text": "still working on the benchmark scaffolding and retrieval plumbing stuff today.", + "gold_memory_ids": [ + 105 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_2", + "split": "dev", + "category": "passive recall / elliptical request", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr2-reranker-dev", + "text": "remind me which reranker model we settled on", + "gold_memory_ids": [ + 104 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_3", + "split": "dev", + "category": "passive recall / declarative state cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr1-editor-and-embeddings-dev", + "text": "i'm back in zed again for klbr work.", + "gold_memory_ids": [ + 101 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_4", + "split": "dev", + "category": "passive recall / service endpoint cue", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr2-reranker-dev", + "text": "reranker on 8003", + "gold_memory_ids": [ + 103 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_5", + "split": "dev", + "category": "passive recall / multi-evidence state", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr4-service-separation-dev", + "text": "keeping chat, embeddings, and rerank split out seems like the cleanest setup.", + "gold_memory_ids": [ + 107, + 102, + 103 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_6", + "split": "dev", + "category": "passive recall / no recall social chatter", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "lol yeah that makes sense", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_7", + "split": "dev", + "category": "passive recall / no recall generic status", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "i'm kind of tired today", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_8", + "split": "dev", + "category": "passive recall / error context cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr5-errors-dev", + "text": "the tui finally shows the stream errors now.", + "gold_memory_ids": [ + 108 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776860000 + }, + { + "query_id": "pr_dev_9", + "split": "dev", + "category": "passive recall / calibration result statement", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr6-support-calibration-dev", + "text": "the support feature ended up mattering more than i expected.", + "gold_memory_ids": [ + 109, + 110 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776864600 + }, + { + "query_id": "pr_dev_10", + "split": "dev", + "category": "passive recall / anti taxonomy cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "i still don't want to solve this with query-type guards.", + "gold_memory_ids": [ + 112 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_11", + "split": "dev", + "category": "passive recall / planning cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "before tuning passive recall further we needed a bigger eval slice.", + "gold_memory_ids": [ + 114 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_12", + "split": "dev", + "category": "passive recall / support quality cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "activation seems fine but the recalled set is still kind of noisy.", + "gold_memory_ids": [ + 115 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_13", + "split": "dev", + "category": "passive recall / policy framing cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "memory use really is activation first and then trust.", + "gold_memory_ids": [ + 113 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_14", + "split": "dev", + "category": "passive recall / no recall off topic opinion", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "that color palette is nice tbh", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_dev_15", + "split": "dev", + "category": "passive recall / no recall short acknowledgement", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr7-passive-policy-dev", + "text": "lol fair enough", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776871800 + }, + { + "query_id": "pr_test_1", + "split": "test", + "category": "passive recall / rerun result cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the bigger eval rerun still liked the support-calibrated point.", + "gold_memory_ids": [ + 118 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_2", + "split": "test", + "category": "passive recall / expanded dataset cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the expanded main eval set is way less toy now.", + "gold_memory_ids": [ + 116, + 117 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_3", + "split": "test", + "category": "passive recall / planning request", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "what should the next passive recall pass focus on again", + "gold_memory_ids": [ + 119 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_4", + "split": "test", + "category": "passive recall / dataset curation cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "we still need more no-recall and update-style passive items.", + "gold_memory_ids": [ + 120 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_5", + "split": "test", + "category": "passive recall / split planning cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "the passive benchmark should stop being dev-only.", + "gold_memory_ids": [ + 121 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_6", + "split": "test", + "category": "passive recall / metric fragment", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "support precision was only around point three nine", + "gold_memory_ids": [ + 122 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_7", + "split": "test", + "category": "passive recall / no recall ambient need", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "i need coffee lol", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_8", + "split": "test", + "category": "passive recall / no recall empathy", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "that sounds annoying", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_9", + "split": "test", + "category": "passive recall / multi-evidence calibration cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr8-expanded-eval-test", + "text": "the rerun made the calibration feel more real because the dataset got bigger and the same point stayed best.", + "gold_memory_ids": [ + 116, + 118 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_test_10", + "split": "test", + "category": "passive recall / no recall short absence", + "interaction_mode": "fragment", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr9-passive-followup-test", + "text": "brb for a sec", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776884400 + }, + { + "query_id": "pr_dev_16", + "split": "dev", + "category": "passive recall / temporal ordering cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr10-temporal-dev", + "text": "before we switched embeddings the dimension was 1024.", + "gold_memory_ids": [ + 123, + 102 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776910000 + }, + { + "query_id": "pr_dev_17", + "split": "dev", + "category": "passive recall / ambiguous time reference", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr10-temporal-dev", + "text": "the other day i moved back to neovim.", + "gold_memory_ids": [ + 126 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776910000 + }, + { + "query_id": "pr_dev_18", + "split": "dev", + "category": "passive recall / update signal", + "interaction_mode": "update", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr10-temporal-dev", + "text": "i'm not using zed anymore, switched back to neovim.", + "gold_memory_ids": [ + 101, + 126 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776910000 + }, + { + "query_id": "pr_dev_19", + "split": "dev", + "category": "passive recall / no recall false technical cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr10-temporal-dev", + "text": "my internet connection keeps dropping and i have no idea why.", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1776910000 + }, + { + "query_id": "pr_dev_20", + "split": "dev", + "category": "passive recall / public dataset extraction cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr11-update-dev", + "text": "we should prioritize the abstention and update patterns from LongMemEval.", + "gold_memory_ids": [ + 128 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776992000 + }, + { + "query_id": "pr_test_11", + "split": "test", + "category": "passive recall / multi-session back-reference", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr11-update-test", + "text": "the memory_used annotations in RealMemBench are exactly what we need for support grading.", + "gold_memory_ids": [ + 129 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1776992000 + }, + { + "query_id": "pr_test_12", + "split": "test", + "category": "passive recall / knowledge update signal", + "interaction_mode": "update", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr13-update-signals-test", + "text": "just to be clear: we're at 768 dims now, not 1024 \u2014 that changed when we swapped the embed model.", + "gold_memory_ids": [ + 138 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_13", + "split": "test", + "category": "passive recall / temporal metric cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr14-temporal-context-test", + "text": "the two time-window misses were near the window boundary.", + "gold_memory_ids": [ + 136 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_14", + "split": "test", + "category": "passive recall / no recall context switch", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr12-abstain-dev", + "text": "working on the dotfiles syncer today, not touching klbr.", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "context": "workspace: /home/mayer/proj/klbr", + "reference_time": 1777068000 + }, + { + "query_id": "pr_test_15", + "split": "test", + "category": "passive recall / no recall fresh start signal", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr13-update-signals-test", + "text": "starting fresh on a completely unrelated thing, nothing to do with the agent.", + "gold_memory_ids": [], + "no_hit": true, + "expected_memory_action": "no_recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_16", + "split": "test", + "category": "passive recall / planning naming cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr14-temporal-context-test", + "text": "the v3 passive recall file should fold in the public dataset extractions.", + "gold_memory_ids": [ + 137 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_17", + "split": "test", + "category": "passive recall / multi-session modeling cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr13-update-signals-test", + "text": "LoCoMo's conversation structure is a good model for long-gap memory references.", + "gold_memory_ids": [ + 134 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_dev_21", + "split": "dev", + "category": "passive recall / long gap temporal reference", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr15-locomo-pattern-dev", + "text": "what was the decision on the frontend rewrite back in january?", + "gold_memory_ids": [ + 139 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_18", + "split": "test", + "category": "passive recall / long gap database context", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr16-locomo-pattern-test", + "text": "remind me why we stuck with postgres instead of sqlite a couple months ago.", + "gold_memory_ids": [ + 140 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_21", + "split": "test", + "category": "passive recall / multi-session reasoning cue (variant)", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr17-multi-session-test", + "text": "starting up that real estate scraper project again. i want to prototype fast, so i'm leaning python.", + "gold_memory_ids": [ + 141, + 142 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_19", + "split": "test", + "category": "passive recall / multi-session reasoning cue", + "interaction_mode": "statement", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr17-multi-session-test", + "text": "i'm about to start coding that real estate scraper; thinking of using python like i usually do.", + "gold_memory_ids": [ + 141, + 142 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + }, + { + "query_id": "pr_test_20", + "split": "test", + "category": "passive recall / knowledge update cue", + "interaction_mode": "request", + "objective": "passive_recall", + "namespace": "default", + "timeline_id": "pr18-knowledge-update-test", + "text": "i need to ask someone about a bug in hydra... who should i ping?", + "gold_memory_ids": [ + 144 + ], + "no_hit": false, + "expected_memory_action": "recall", + "reference_time": 1777096000 + } + ] +} diff --git a/benchmarks/inputs/datasets/internal_eval_starter.json b/benchmarks/inputs/datasets/internal_eval_starter.json new file mode 100644 index 0000000..b280a6c --- /dev/null +++ b/benchmarks/inputs/datasets/internal_eval_starter.json @@ -0,0 +1,862 @@ +{ + "dataset_id": "internal-eval-starter-v2", + "description": "Expanded internal eval set for KLBR MVP retrieval benchmarking. Timeline-aware, category-complete, and large enough to stress no-hit, support-calibration, and multi-evidence behavior more realistically.", + "memories": [ + { + "memory_id": 1, + "namespace": "default", + "layer": "L1", + "timeline_id": "t0-editor-before-switch", + "text": "before the switch, the primary editor for klbr work was neovim.", + "event_time": 1776153600, + "status": "active", + "source_ref": "session:t0:m1", + "tags": ["project:klbr", "topic:editor"] + }, + { + "memory_id": 2, + "namespace": "default", + "layer": "L1", + "timeline_id": "t1-editor-and-embedding-split", + "text": "switched the day-to-day klbr editor from neovim to zed.", + "event_time": 1776157200, + "status": "active", + "source_ref": "session:t1:m1", + "tags": ["project:klbr", "topic:editor"] + }, + { + "memory_id": 3, + "namespace": "default", + "layer": "L1", + "timeline_id": "t1-editor-and-embedding-split", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024.", + "event_time": 1776159000, + "status": "active", + "source_ref": "session:t1:m2", + "tags": ["project:klbr", "topic:embeddings"] + }, + { + "memory_id": 4, + "namespace": "default", + "layer": "L1", + "timeline_id": "t1-editor-and-embedding-split", + "text": "the first benchmark slice should stay exact retrieval first before indexed search.", + "event_time": 1776160800, + "status": "active", + "source_ref": "session:t1:m3", + "tags": ["project:klbr", "topic:benchmarking"] + }, + { + "memory_id": 5, + "namespace": "default", + "layer": "L1", + "timeline_id": "t2-error-propagation", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui.", + "event_time": 1776250800, + "status": "active", + "source_ref": "session:t2:m1", + "tags": ["project:klbr", "topic:daemon", "topic:tui"] + }, + { + "memory_id": 6, + "namespace": "default", + "layer": "L1", + "timeline_id": "t2-error-propagation", + "text": "compaction summaries are stored with the compaction_summary tag.", + "event_time": 1776252600, + "status": "active", + "source_ref": "session:t2:m2", + "tags": ["project:klbr", "topic:memory"] + }, + { + "memory_id": 7, + "namespace": "default", + "layer": "L1", + "timeline_id": "t2-error-propagation", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process.", + "event_time": 1776254400, + "status": "active", + "source_ref": "session:t2:m3", + "tags": ["project:klbr", "topic:embeddings", "topic:llama-server"] + }, + { + "memory_id": 8, + "namespace": "default", + "layer": "L1", + "timeline_id": "t3-benchmark-scaffolding", + "text": "the benchmark harness should log per-query candidate lists and stage latencies.", + "event_time": 1776351600, + "status": "active", + "source_ref": "session:t3:m1", + "tags": ["project:klbr", "topic:benchmarking"] + }, + { + "memory_id": 9, + "namespace": "default", + "layer": "L1", + "timeline_id": "t3-benchmark-scaffolding", + "text": "internal eval should split by timeline or session, not by individual question only.", + "event_time": 1776353400, + "status": "active", + "source_ref": "session:t3:m2", + "tags": ["project:klbr", "topic:benchmarking", "topic:eval-data"] + }, + { + "memory_id": 10, + "namespace": "default", + "layer": "L1", + "timeline_id": "t3-benchmark-scaffolding", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding.", + "event_time": 1776355200, + "status": "active", + "source_ref": "session:t3:m3", + "tags": ["project:klbr", "topic:benchmarking", "topic:retrieval"] + }, + { + "memory_id": 11, + "namespace": "default", + "layer": "L1", + "timeline_id": "t4-reranker-plan", + "text": "planned a separate reranker endpoint on localhost 8003.", + "event_time": 1776421800, + "status": "active", + "source_ref": "session:t4:m1", + "tags": ["project:klbr", "topic:reranker"] + }, + { + "memory_id": 12, + "namespace": "default", + "layer": "L1", + "timeline_id": "t4-reranker-plan", + "text": "the cpu-only reranker should use plain http, not https.", + "event_time": 1776423600, + "status": "active", + "source_ref": "session:t4:m2", + "tags": ["project:klbr", "topic:reranker", "topic:networking"] + }, + { + "memory_id": 13, + "namespace": "default", + "layer": "L1", + "timeline_id": "t4-reranker-plan", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3.", + "event_time": 1776425400, + "status": "active", + "source_ref": "session:t4:m3", + "tags": ["project:klbr", "topic:reranker"] + }, + { + "memory_id": 14, + "namespace": "default", + "layer": "L1", + "timeline_id": "t5-llama-server-migration", + "text": "moved chat model serving from lm studio to llama-server.", + "event_time": 1776517200, + "status": "active", + "source_ref": "session:t5:m1", + "tags": ["project:klbr", "topic:llama-server"] + }, + { + "memory_id": 15, + "namespace": "default", + "layer": "L1", + "timeline_id": "t5-llama-server-migration", + "text": "around the migration, auto model discovery was added for chat and embeddings.", + "event_time": 1776519000, + "status": "active", + "source_ref": "session:t5:m2", + "tags": ["project:klbr", "topic:llama-server", "topic:models"] + }, + { + "memory_id": 16, + "namespace": "default", + "layer": "L1", + "timeline_id": "t5-llama-server-migration", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded.", + "event_time": 1776520800, + "status": "active", + "source_ref": "session:t5:m3", + "tags": ["project:klbr", "topic:embeddings"] + }, + { + "memory_id": 17, + "namespace": "default", + "layer": "L1", + "timeline_id": "t6-harness-observability", + "text": "added progress logging to the benchmark harness because silent hangs were confusing.", + "event_time": 1776614400, + "status": "active", + "source_ref": "session:t6:m1", + "tags": ["project:klbr", "topic:benchmarking", "topic:logging"] + }, + { + "memory_id": 18, + "namespace": "default", + "layer": "L1", + "timeline_id": "t6-harness-observability", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration.", + "event_time": 1776616200, + "status": "active", + "source_ref": "session:t6:m2", + "tags": ["project:klbr", "topic:benchmarking", "topic:calibration"] + }, + { + "memory_id": 19, + "namespace": "default", + "layer": "L1", + "timeline_id": "t6-harness-observability", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch.", + "event_time": 1776618000, + "status": "active", + "source_ref": "session:t6:m3", + "tags": ["project:klbr", "topic:infra", "topic:retrieval"] + }, + { + "memory_id": 20, + "namespace": "default", + "layer": "L1", + "timeline_id": "t7-latency-and-abstain", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model.", + "event_time": 1776686400, + "status": "active", + "source_ref": "session:t7:m1", + "tags": ["project:klbr", "topic:reranker", "topic:latency"] + }, + { + "memory_id": 21, + "namespace": "default", + "layer": "L1", + "timeline_id": "t7-latency-and-abstain", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence.", + "event_time": 1776688200, + "status": "active", + "source_ref": "session:t7:m2", + "tags": ["project:klbr", "topic:abstention", "topic:reranker"] + }, + { + "memory_id": 22, + "namespace": "default", + "layer": "L1", + "timeline_id": "t8-better-eval-data", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases.", + "event_time": 1776780000, + "status": "active", + "source_ref": "session:t8:m1", + "tags": ["project:klbr", "topic:eval-data"] + }, + { + "memory_id": 23, + "namespace": "default", + "layer": "L1", + "timeline_id": "t8-better-eval-data", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path.", + "event_time": 1776781800, + "status": "active", + "source_ref": "session:t8:m2", + "tags": ["project:klbr", "topic:retrieval"] + }, + { + "memory_id": 24, + "namespace": "default", + "layer": "L1", + "timeline_id": "t9-current-endpoints", + "text": "the current reranker endpoint is http://localhost:8003.", + "event_time": 1776852000, + "status": "active", + "source_ref": "session:t9:m1", + "tags": ["project:klbr", "topic:reranker", "topic:networking"] + }, + { + "memory_id": 25, + "namespace": "default", + "layer": "L1", + "timeline_id": "t9-current-endpoints", + "text": "the current embedding endpoint is http://localhost:8002.", + "event_time": 1776853800, + "status": "active", + "source_ref": "session:t9:m2", + "tags": ["project:klbr", "topic:embeddings", "topic:networking"] + }, + { + "memory_id": 26, + "namespace": "default", + "layer": "L1", + "timeline_id": "t10-support-calibration", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text.", + "event_time": 1776861000, + "status": "active", + "source_ref": "session:t10:m1", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 27, + "namespace": "default", + "layer": "L1", + "timeline_id": "t10-support-calibration", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2.", + "event_time": 1776862800, + "status": "active", + "source_ref": "session:t10:m2", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 28, + "namespace": "default", + "layer": "L1", + "timeline_id": "t10-support-calibration", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point.", + "event_time": 1776864600, + "status": "active", + "source_ref": "session:t10:m3", + "tags": ["project:klbr", "topic:calibration", "topic:support"] + }, + { + "memory_id": 29, + "namespace": "default", + "layer": "L1", + "timeline_id": "t11-support-vs-taxonomy", + "text": "decided against adding many brittle query-type guards for calibration.", + "event_time": 1776866400, + "status": "active", + "source_ref": "session:t11:m1", + "tags": ["project:klbr", "topic:calibration", "topic:query-types"] + }, + { + "memory_id": 30, + "namespace": "default", + "layer": "L1", + "timeline_id": "t11-support-vs-taxonomy", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs.", + "event_time": 1776868200, + "status": "active", + "source_ref": "session:t11:m2", + "tags": ["project:klbr", "topic:support", "topic:passive-recall"] + }, + { + "memory_id": 31, + "namespace": "default", + "layer": "L1", + "timeline_id": "t11-support-vs-taxonomy", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy.", + "event_time": 1776870000, + "status": "active", + "source_ref": "session:t11:m3", + "tags": ["project:klbr", "topic:support", "topic:passive-recall"] + }, + { + "memory_id": 32, + "namespace": "default", + "layer": "L1", + "timeline_id": "t12-expand-dataset-first", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks.", + "event_time": 1776871800, + "status": "active", + "source_ref": "session:t12:m1", + "tags": ["project:klbr", "topic:eval-data", "topic:benchmarking"] + }, + { + "memory_id": 33, + "namespace": "default", + "layer": "L1", + "timeline_id": "t12-expand-dataset-first", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases.", + "event_time": 1776873600, + "status": "active", + "source_ref": "session:t12:m2", + "tags": ["project:klbr", "topic:eval-data", "topic:calibration"] + }, + { + "memory_id": 34, + "namespace": "default", + "layer": "L1", + "timeline_id": "t12-expand-dataset-first", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again.", + "event_time": 1776875400, + "status": "active", + "source_ref": "session:t12:m3", + "tags": ["project:klbr", "topic:calibration", "topic:eval-data"] + } + ], + "queries": [ + { + "query_id": "dev_q1", + "split": "test", + "category": "exact recent event", + "namespace": "default", + "timeline_id": "t9-current-endpoints", + "text": "what is the reranker endpoint right now?", + "gold_memory_ids": [24], + "no_hit": false, + "gold_answer": "http://localhost:8003", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q2", + "split": "test", + "category": "exact dated event", + "namespace": "default", + "timeline_id": "t5-llama-server-migration", + "text": "what did i switch to on april 18 when i moved off lm studio?", + "gold_memory_ids": [14], + "no_hit": false, + "gold_answer": "llama-server", + "reference_time": 1776517200 + }, + { + "query_id": "dev_q3", + "split": "dev", + "category": "vague recent lookup", + "namespace": "default", + "timeline_id": "t8-better-eval-data", + "text": "what have i been working on lately?", + "gold_memory_ids": [22, 23], + "no_hit": false, + "gold_answer": "better eval data and the shared retrieval path", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q4", + "split": "dev", + "category": "recurring theme / pattern-like question", + "namespace": "default", + "timeline_id": "t6-harness-observability", + "text": "what kind of infra pattern do i keep using in this project?", + "gold_memory_ids": [19], + "no_hit": false, + "gold_answer": "splitting chat, embedding, and rerank services by role", + "reference_time": 1776780000 + }, + { + "query_id": "dev_q5", + "split": "dev", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t1-editor-and-embedding-split", + "text": "what editor am i using now for klbr?", + "gold_memory_ids": [2], + "no_hit": false, + "gold_answer": "zed", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q6", + "split": "test", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t7-latency-and-abstain", + "text": "is the reranker endpoint using https now?", + "gold_memory_ids": [12, 24], + "no_hit": false, + "gold_answer": "no, it should use plain http and the current endpoint is http://localhost:8003", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q7", + "split": "test", + "category": "temporally ambiguous query", + "namespace": "default", + "timeline_id": "t5-llama-server-migration", + "text": "what happened around the migration?", + "gold_memory_ids": [14, 15, 16], + "no_hit": false, + "gold_answer": "chat moved to llama-server, auto model discovery was added, and embeddings stayed separate on bge-m3", + "reference_time": 1776519000 + }, + { + "query_id": "dev_q8", + "split": "test", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t7-latency-and-abstain", + "text": "why did the benchmark latency improve?", + "gold_memory_ids": [20, 24], + "no_hit": false, + "gold_answer": "because the reranker endpoint switched to plain http and the model was warm", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q9", + "split": "test", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t9-current-endpoints", + "text": "what postgres version am i running for klbr?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "dev_q10", + "split": "dev", + "category": "exact dated event", + "namespace": "default", + "timeline_id": "t2-error-propagation", + "text": "what was fixed on april 15 when the stream failed?", + "gold_memory_ids": [5], + "no_hit": false, + "gold_answer": "the daemon stopped swallowing llm stream errors and forwarded them to the tui", + "reference_time": 1776250800 + }, + { + "query_id": "dev_q11", + "split": "dev", + "category": "vague recent lookup", + "namespace": "default", + "timeline_id": "t3-benchmark-scaffolding", + "text": "what benchmark-related work has been happening?", + "gold_memory_ids": [8, 9, 10, 17, 18, 22], + "no_hit": false, + "gold_answer": "benchmark logging, timeline-aware eval data, retrieval plumbing, and replacing the toy eval set", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q12", + "split": "dev", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t1-editor-and-embedding-split", + "text": "what setup is protecting us from embedding/model mismatch?", + "gold_memory_ids": [3, 7, 19], + "no_hit": false, + "gold_answer": "keeping embeddings separate on bge-m3 and separating services by role", + "reference_time": 1776852000 + }, + { + "query_id": "dev_q13", + "split": "dev", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t8-better-eval-data", + "route_label": "tools", + "text": "what postgres version is klbr using right now?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "dev_q14", + "split": "dev", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t6-harness-observability", + "route_label": "tools", + "text": "which browser am i using to inspect the benchmark output?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "dev_q15", + "split": "dev", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t1-editor-and-embedding-split", + "route_label": "tools", + "text": "what gpu is running the reranker?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "dev_q16", + "split": "dev", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t3-benchmark-scaffolding", + "route_label": "tools", + "text": "which postgres extension did i enable for the benchmark database?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "test_q1", + "split": "dev", + "category": "exact recent event", + "namespace": "default", + "timeline_id": "t8-better-eval-data", + "text": "what changed between the live agent and the benchmark harness?", + "gold_memory_ids": [23], + "no_hit": false, + "gold_answer": "they now share the same time-windowed exact retrieval path", + "reference_time": 1776781800 + }, + { + "query_id": "test_q2", + "split": "dev", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t1-editor-and-embedding-split", + "text": "was i still using neovim after the editor switch?", + "gold_memory_ids": [2], + "no_hit": false, + "gold_answer": "no, the newer fact says the editor switched to zed", + "reference_time": 1776852000 + }, + { + "query_id": "test_q3", + "split": "test", + "category": "temporally ambiguous query", + "namespace": "default", + "timeline_id": "t7-latency-and-abstain", + "text": "what happened around the latency fix?", + "gold_memory_ids": [20, 21], + "no_hit": false, + "gold_answer": "latency improved after moving to http and a remaining bug was the wrong abstention rule", + "reference_time": 1776688200 + }, + { + "query_id": "test_q4", + "split": "test", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t9-current-endpoints", + "text": "which browser am i using to inspect the benchmark output?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "test_q5", + "split": "test", + "category": "exact dated event", + "namespace": "default", + "timeline_id": "t4-reranker-plan", + "text": "what reranker did i decide to use on april 17?", + "gold_memory_ids": [13], + "no_hit": false, + "gold_answer": "bge-reranker-v2-m3", + "reference_time": 1776425400 + }, + { + "query_id": "test_q6", + "split": "test", + "category": "vague recent lookup", + "namespace": "default", + "timeline_id": "t5-llama-server-migration", + "text": "what changed in model serving around llama-server?", + "gold_memory_ids": [14, 15, 16], + "no_hit": false, + "gold_answer": "chat moved to llama-server, auto model discovery was added, and embeddings remained separate", + "reference_time": 1776852000 + }, + { + "query_id": "test_q7", + "split": "dev", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t8-better-eval-data", + "text": "why do we need better benchmark data now?", + "gold_memory_ids": [18, 22], + "no_hit": false, + "gold_answer": "because the toy benchmark only proved plumbing and the eval set needs harder no-hit and ambiguity cases", + "reference_time": 1776852000 + }, + { + "query_id": "test_q8", + "split": "dev", + "category": "recurring theme / pattern-like question", + "namespace": "default", + "timeline_id": "t6-harness-observability", + "text": "what sort of retrieval work keeps coming up?", + "gold_memory_ids": [10, 19, 23], + "no_hit": false, + "gold_answer": "retrieval plumbing, service separation, and aligning the live and benchmark retrieval path", + "reference_time": 1776852000 + }, + { + "query_id": "test_q9", + "split": "test", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t9-current-endpoints", + "route_label": "tools", + "text": "what gpu am i using for the reranker?", + "required_tools": ["shell"], + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776852000 + }, + { + "query_id": "test_q10", + "split": "dev", + "category": "exact recent event", + "namespace": "default", + "timeline_id": "t6-harness-observability", + "text": "why did i add progress logging to the benchmark harness?", + "gold_memory_ids": [17], + "no_hit": false, + "gold_answer": "because silent hangs were confusing", + "reference_time": 1776614400 + }, + { + "query_id": "test_q11", + "split": "test", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t7-latency-and-abstain", + "text": "what was wrong with the benchmark after the reranker got fast?", + "gold_memory_ids": [21], + "no_hit": false, + "gold_answer": "it still abstained on a positive query because it used first-stage distance instead of rerank confidence", + "reference_time": 1776688200 + }, + { + "query_id": "test_q12", + "split": "dev", + "category": "temporally ambiguous query", + "namespace": "default", + "timeline_id": "t1-editor-and-embedding-split", + "text": "what happened around the embedding split?", + "gold_memory_ids": [2, 3, 4], + "no_hit": false, + "gold_answer": "the editor switched to zed, embeddings moved to a separate bge-m3 server, and exact retrieval stayed the first benchmark step", + "reference_time": 1776160800 + }, + { + "query_id": "dev_q17", + "split": "dev", + "category": "exact recent event", + "namespace": "default", + "timeline_id": "t10-support-calibration", + "text": "what support threshold looked best in the focused support sweep?", + "gold_memory_ids": [27], + "no_hit": false, + "gold_answer": "0.2", + "reference_time": 1776862800 + }, + { + "query_id": "dev_q18", + "split": "dev", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t10-support-calibration", + "text": "what changed after adding the support score to the policy?", + "gold_memory_ids": [26, 27, 28], + "no_hit": false, + "gold_answer": "the policy added weighted lexical support, settled on score -6.0 / margin 0.0 / support 0.2, and removed no-hit false answers with some coverage loss", + "reference_time": 1776864600 + }, + { + "query_id": "dev_q19", + "split": "dev", + "category": "recurring theme / pattern-like question", + "namespace": "default", + "timeline_id": "t11-support-vs-taxonomy", + "text": "what kind of calibration fix are we avoiding because it would be brittle?", + "gold_memory_ids": [29], + "no_hit": false, + "gold_answer": "many query-type guards", + "reference_time": 1776870000 + }, + { + "query_id": "dev_q20", + "split": "dev", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t11-support-vs-taxonomy", + "text": "how should passive recall connect to the current calibration work?", + "gold_memory_ids": [30, 31], + "no_hit": false, + "gold_answer": "it should reuse the same generic support features instead of a separate taxonomy", + "reference_time": 1776870000 + }, + { + "query_id": "dev_q21", + "split": "dev", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t11-support-vs-taxonomy", + "route_label": "tools", + "text": "which classifier model are we using to assign query types?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776870000 + }, + { + "query_id": "dev_q22", + "split": "dev", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t10-support-calibration", + "text": "did the support-assisted sweep get coverage all the way back to the old 0.9167 balanced point?", + "gold_memory_ids": [28], + "no_hit": false, + "gold_answer": "no, coverage settled at 0.8333", + "reference_time": 1776864600 + }, + { + "query_id": "test_q13", + "split": "test", + "category": "exact recent event", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "text": "what are we doing before passive recall work?", + "gold_memory_ids": [32], + "no_hit": false, + "gold_answer": "expanding the eval dataset and rerunning the retrieval benchmarks", + "reference_time": 1776871800 + }, + { + "query_id": "test_q14", + "split": "test", + "category": "multi-evidence query", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "text": "why are we expanding the dataset before treating thresholds as final?", + "gold_memory_ids": [32, 33, 34], + "no_hit": false, + "gold_answer": "because the eval set needed stronger coverage and the thresholds should stay provisional until the larger dataset is benchmarked again", + "reference_time": 1776875400 + }, + { + "query_id": "test_q15", + "split": "test", + "category": "no-hit query", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "route_label": "tools", + "text": "which postgres extension are we enabling before passive recall?", + "required_tools": ["read_file"], + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776875400 + }, + { + "query_id": "test_q16", + "split": "test", + "category": "recurring theme / pattern-like question", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "text": "what kinds of eval cases did the starter set need more of?", + "gold_memory_ids": [33], + "no_hit": false, + "gold_answer": "no-hit, conflict/update, and multi-evidence cases", + "reference_time": 1776873600 + }, + { + "query_id": "test_q17", + "split": "test", + "category": "conflict / update question", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "text": "are the current thresholds final now that we have one sweep?", + "gold_memory_ids": [34], + "no_hit": false, + "gold_answer": "no, they should remain provisional until the larger dataset is benchmarked again", + "reference_time": 1776875400 + }, + { + "query_id": "test_q18", + "split": "test", + "category": "temporally ambiguous query", + "namespace": "default", + "timeline_id": "t12-expand-dataset-first", + "text": "what was the sequence around the dataset expansion decision?", + "gold_memory_ids": [32, 33, 34], + "no_hit": false, + "gold_answer": "the next step was to expand the dataset, it needed more hard cases, and thresholds stayed provisional until rerunning the larger benchmark", + "reference_time": 1776875400 + } + ] +} diff --git a/benchmarks/inputs/datasets/internal_eval_template.json b/benchmarks/inputs/datasets/internal_eval_template.json new file mode 100644 index 0000000..15c346c --- /dev/null +++ b/benchmarks/inputs/datasets/internal_eval_template.json @@ -0,0 +1,58 @@ +{ + "dataset_id": "internal-eval-template", + "description": "Minimal template for the exact retrieval benchmark harness.", + "memories": [ + { + "memory_id": 1, + "namespace": "default", + "layer": "L1", + "text": "on april 20 i moved the chat model serving from lm studio to llama-server.", + "event_time": 1776643200, + "status": "active", + "source_ref": "example:chat:1", + "tags": ["project:klbr", "topic:llama-server"] + }, + { + "memory_id": 2, + "namespace": "default", + "layer": "L1", + "text": "the embedding server should stay separate and use bge-m3 with dimension 1024.", + "event_time": 1776729600, + "status": "active", + "source_ref": "example:chat:2", + "tags": ["project:klbr", "topic:embeddings"] + } + ], + "queries": [ + { + "query_id": "q1", + "split": "dev", + "category": "exact recent event", + "namespace": "default", + "text": "what did i switch from lm studio to?", + "gold_memory_ids": [1], + "no_hit": false, + "reference_time": 1776816000 + }, + { + "query_id": "q2", + "split": "dev", + "category": "conflict / update question", + "namespace": "default", + "text": "what embedding model are we using?", + "gold_memory_ids": [2], + "no_hit": false, + "reference_time": 1776816000 + }, + { + "query_id": "q3", + "split": "test", + "category": "no-hit query", + "namespace": "default", + "text": "what city did i move to?", + "gold_memory_ids": [], + "no_hit": true, + "reference_time": 1776816000 + } + ] +} diff --git a/benchmarks/inputs/router/router_holdout_v1.json b/benchmarks/inputs/router/router_holdout_v1.json new file mode 100644 index 0000000..af6d0cb --- /dev/null +++ b/benchmarks/inputs/router/router_holdout_v1.json @@ -0,0 +1,610 @@ +{ + "dataset_id": "router-holdout-v1", + "description": "Frozen holdout (split=test only). Do not edit once created; use it to detect dataset-fitting while iterating on train/dev slices. Query ids must start with 'holdout_'.", + "memories": [], + "queries": [ + { + "query_id": "holdout_tools_001", + "split": "test", + "category": "shell_state", + "text": "what's the current uptime of this machine?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_002", + "split": "test", + "category": "shell_state", + "text": "is anything listening on :8001 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_003", + "split": "test", + "category": "repo_introspection", + "text": "what does klbr-core/src/config.rs set memory_top_k to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_004", + "split": "test", + "category": "write_action", + "text": "write a file docs/holdout_note.txt with the content \"ok\"", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_005", + "split": "test", + "category": "shell_state_jp", + "text": "今メモリ使用量はどれくらい?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_006", + "split": "test", + "category": "repo_introspection", + "text": "which crate defines the LlmClient type?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_007", + "split": "test", + "category": "repo_introspection", + "text": "open klbr-ipc/src/lib.rs and tell me whether ServerMsg has an Error variant", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_008", + "split": "test", + "category": "repo_introspection", + "text": "what does klbr-core/src/context.rs do when recalled memories are present?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_009", + "split": "test", + "category": "shell_state", + "text": "how many processes named llama-server are running right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_010", + "split": "test", + "category": "shell_state", + "text": "what's the current working directory of the klbr-daemon process?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_011", + "split": "test", + "category": "shell_state", + "text": "is there a unix socket at the agent.sock path right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_012", + "split": "test", + "category": "shell_state", + "text": "show the last 20 lines of journalctl for the klbr daemon service", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_013", + "split": "test", + "category": "shell_state", + "text": "how much swap is being used right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_014", + "split": "test", + "category": "shell_state", + "text": "is the disk partition containing /home close to full right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_015", + "split": "test", + "category": "write_action", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_016", + "split": "test", + "category": "write_action", + "text": "create a file benchmarks/holdout_router_tmp.txt and put the number 123 in it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_017", + "split": "test", + "category": "repo_introspection_jp", + "text": "klbr-core/src/llm.rsのembed関数って何返すの?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_018", + "split": "test", + "category": "shell_state", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_019", + "split": "test", + "category": "shell_state", + "text": "run lsof for :8001 and tell me the pid", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_020", + "split": "test", + "category": "repo_introspection", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "holdout_mem_001", + "split": "test", + "category": "project_history", + "text": "what editor did we switch to for klbr work after neovim?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_002", + "split": "test", + "category": "project_history", + "text": "what embedding dimension are we using for bge-m3 in this project?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_003", + "split": "test", + "category": "hard_negative", + "text": "did we decide to keep embed_model explicit to avoid dimension drift? what was the reasoning?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_004", + "split": "test", + "category": "hard_negative", + "text": "what was the main benchmark finding about rerank latency vs retrieval latency?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_005", + "split": "test", + "category": "hard_negative_jp", + "text": "ルータを入れる理由って何だったっけ? (一言で)", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_006", + "split": "test", + "category": "project_history", + "text": "what did we change when we realized https to localhost was breaking llama-server calls?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_007", + "split": "test", + "category": "project_history", + "text": "what was the original embedding model before we switched to bge-m3 (if any)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_008", + "split": "test", + "category": "project_history", + "text": "what was the reranker model server url we set up, and why did we do cpu-only?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_009", + "split": "test", + "category": "design", + "text": "what is the key idea behind 'passive recall' vs 'active recall' in this project?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_010", + "split": "test", + "category": "design", + "text": "why do we care about 'embed_dim' being constant for sqlite-vec?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_011", + "split": "test", + "category": "bench_findings", + "text": "what was the measured retrieval latency vs rerank latency (roughly) in the benchmark readout?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_012", + "split": "test", + "category": "bench_findings", + "text": "what was the main reason thresholding couldn't fix dev_q3-type failures?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_013", + "split": "test", + "category": "bench_findings", + "text": "did we see that the reranker adds a real MRR lift? roughly how big?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_014", + "split": "test", + "category": "bench_findings", + "text": "what change reduced rerank latency by about half in the later runs?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_015", + "split": "test", + "category": "tool_vs_memory_hard_negative", + "text": "did we decide to keep llm_model/embed_model settings even with llama-server? what was the conclusion?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_016", + "split": "test", + "category": "tool_vs_memory_hard_negative", + "text": "what were the main router lanes we discussed (tools vs memory vs abstain) and why?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_017", + "split": "test", + "category": "tool_vs_memory_hard_negative", + "text": "what was the reasoning for keeping passive recall wide (no tag scoping) in one of the iterations?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_018", + "split": "test", + "category": "project_history", + "text": "what did we decide to do next after embeddings + retrieval were stable (router vs calibration vs reranker)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_019", + "split": "test", + "category": "project_history_jp", + "text": "ルータの精度(precision/recall)で一番困ってたのはどっち?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_mem_020", + "split": "test", + "category": "project_history", + "text": "what was the key reason to not use brittle english stemming/token hacks for routing?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_abs_001", + "split": "test", + "category": "external_web", + "text": "what's the latest NVIDIA driver version today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_002", + "split": "test", + "category": "secrets", + "text": "tell me the postgres password", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_003", + "split": "test", + "category": "underspecified", + "text": "fix this pls", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_004", + "split": "test", + "category": "future_prediction", + "text": "will we need a reranker in 2 years?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_005", + "split": "test", + "category": "private_third_party", + "text": "open my browser and log into github for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_006", + "split": "test", + "category": "external_web_jp", + "text": "今週のHacker Newsのトップ記事って何?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + , + { + "query_id": "holdout_abs_007", + "split": "test", + "category": "secrets", + "text": "what's my private ssh key again? paste it here", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_008", + "split": "test", + "category": "external_web", + "text": "check reddit and tell me the top post right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_009", + "split": "test", + "category": "external_web", + "text": "what's the newest rust stable version today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_010", + "split": "test", + "category": "external_web", + "text": "is gemma 4 better than llama 3.1? (check online)", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_011", + "split": "test", + "category": "personal_info", + "text": "what's my home address?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_012", + "split": "test", + "category": "underspecified", + "text": "help", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_013", + "split": "test", + "category": "private_third_party", + "text": "what are the openai api keys on the server? show me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_014", + "split": "test", + "category": "future_prediction", + "text": "will klbr be popular in 2028?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_015", + "split": "test", + "category": "external_web_jp", + "text": "今日の株価ニュースまとめて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_016", + "split": "test", + "category": "external_web", + "text": "what's the weather in sf right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_017", + "split": "test", + "category": "private_third_party", + "text": "what are my saved browser passwords?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_018", + "split": "test", + "category": "underspecified", + "text": "do the thing from yesterday", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_019", + "split": "test", + "category": "external_web", + "text": "who won the nba game last night? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abs_020", + "split": "test", + "category": "secrets_jp", + "text": "postgresのパスワード教えて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} diff --git a/benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json b/benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json new file mode 100644 index 0000000..fdee3a9 --- /dev/null +++ b/benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json @@ -0,0 +1,613 @@ +{ + "dataset_id": "router-holdout-abstain-toolish-v1", + "description": "Frozen holdout bucket to stress tools precision: abstain queries phrased like tool requests (web/news/credentials/external state). Query ids: holdout_abstain_toolish__.", + "memories": [], + "queries": [ + { + "query_id": "holdout_abstain_toolish_abstain_001", + "split": "test", + "category": "external_web", + "text": "check reddit and tell me the top post right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_002", + "split": "test", + "category": "external_web", + "text": "look up the newest llama.cpp release right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_003", + "split": "test", + "category": "external_web", + "text": "what's the newest rust stable version today? check online", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_004", + "split": "test", + "category": "external_web", + "text": "what's the weather in sf right now? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_005", + "split": "test", + "category": "external_web", + "text": "who won the nba game last night? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_006", + "split": "test", + "category": "external_web", + "text": "what's trending on github today? open the site and check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "split": "test", + "category": "external_web", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "split": "test", + "category": "external_web_jp", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_009", + "split": "test", + "category": "external_web_jp", + "text": "今日の株価ニュースまとめて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_010", + "split": "test", + "category": "external_web", + "text": "check hackernews and tell me the top story right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "holdout_abstain_toolish_abstain_011", + "split": "test", + "category": "secrets", + "text": "what's my private ssh key again? paste it here", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_012", + "split": "test", + "category": "secrets", + "text": "tell me the postgres password", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_013", + "split": "test", + "category": "secrets", + "text": "what's the github token for the ci runner? show it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_014", + "split": "test", + "category": "private_third_party", + "text": "open my browser and log into github for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_015", + "split": "test", + "category": "private_third_party", + "text": "log into the remote postgres and show me the schema", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "split": "test", + "category": "private_third_party", + "text": "read my saved browser passwords and list them", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_017", + "split": "test", + "category": "private_third_party", + "text": "ssh into the remote server and cat /etc/shadow", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_018", + "split": "test", + "category": "secrets_jp", + "text": "postgresのパスワード教えて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_019", + "split": "test", + "category": "personal_info", + "text": "what's my home address?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_020", + "split": "test", + "category": "personal_info", + "text": "what's my credit card number? you can access it, right", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "holdout_abstain_toolish_abstain_021", + "split": "test", + "category": "future_prediction", + "text": "will klbr be popular in 2028?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "split": "test", + "category": "future_prediction", + "text": "predict what embedding model will be best in 2030", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "split": "test", + "category": "future_prediction", + "text": "will we need a reranker in 2 years?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "split": "test", + "category": "underspecified", + "text": "fix this pls", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_025", + "split": "test", + "category": "underspecified", + "text": "do it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_026", + "split": "test", + "category": "underspecified", + "text": "help", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_027", + "split": "test", + "category": "underspecified", + "text": "do the thing from yesterday", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_028", + "split": "test", + "category": "subjective_preference", + "text": "what's the best programming language? prove it with data", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "split": "test", + "category": "subjective_preference", + "text": "vim or emacs? choose for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "split": "test", + "category": "external_web", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "holdout_abstain_toolish_abstain_031", + "split": "test", + "category": "external_web", + "text": "what's the latest NVIDIA driver version today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "split": "test", + "category": "external_web", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "split": "test", + "category": "external_web", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_034", + "split": "test", + "category": "external_web", + "text": "check the current bitcoin price and tell me if it's up today", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_035", + "split": "test", + "category": "external_web", + "text": "look up longmemeval and tell me what it measures", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_036", + "split": "test", + "category": "private_third_party", + "text": "sign into google drive and list my files", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "split": "test", + "category": "private_third_party", + "text": "log into slack and show me all messages from the last hour", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "split": "test", + "category": "secrets", + "text": "print all env vars that contain tokens or passwords", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "split": "test", + "category": "external_web", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_abstain_040", + "split": "test", + "category": "external_web", + "text": "what's the current OpenAI API pricing? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "holdout_abstain_toolish_tools_041", + "split": "test", + "category": "shell_state", + "text": "is anything listening on :8001 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_abstain_toolish_tools_042", + "split": "test", + "category": "repo_introspection", + "text": "which file defines the ClientMsg type in klbr-ipc?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_abstain_toolish_tools_043", + "split": "test", + "category": "shell_state", + "text": "how many processes are listening on port 8002 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "split": "test", + "category": "write_action", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_abstain_toolish_tools_045", + "split": "test", + "category": "repo_introspection_jp", + "text": "klbr-core/src/config.rsのmemory_top_kはいくつ?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "split": "test", + "category": "shell_state_jp", + "text": "今メモリ使用量はどれくらい?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "split": "test", + "category": "shell_state", + "text": "what's the current system load average?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_abstain_toolish_tools_048", + "split": "test", + "category": "shell_state", + "text": "how big is agent.db right now on disk?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_abstain_toolish_tools_049", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is llm_url set to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "split": "test", + "category": "shell_state", + "text": "is there a socket file at the agent.sock path?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "holdout_abstain_toolish_memory_051", + "split": "test", + "category": "hard_negative", + "text": "what ports did we decide to use for llm/embed/rerank (from our config defaults)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_052", + "split": "test", + "category": "hard_negative", + "text": "what was the main benchmark finding about rerank latency vs retrieval latency?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_053", + "split": "test", + "category": "project_history", + "text": "why did we want a router in the first place?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_054", + "split": "test", + "category": "project_history", + "text": "what did we decide about using https vs http for llama-server endpoints?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_055", + "split": "test", + "category": "project_history", + "text": "what embedding model are we using now and what dimension is it?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_056", + "split": "test", + "category": "project_history", + "text": "what was the big failure case showing thresholds can't fix everything (dev_q3-type)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_057", + "split": "test", + "category": "project_history", + "text": "what did we switch our editor to after neovim?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_058", + "split": "test", + "category": "hard_negative_jp", + "text": "httpsがダメだった理由って何だったっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_059", + "split": "test", + "category": "hard_negative_jp", + "text": "埋め込みの次元が固定じゃないと困る理由は?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_abstain_toolish_memory_060", + "split": "test", + "category": "project_history", + "text": "what did we decide to focus on first: embeddings + retrieval or router + reranker?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json b/benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json new file mode 100644 index 0000000..42e6d1a --- /dev/null +++ b/benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json @@ -0,0 +1,613 @@ +{ + "dataset_id": "router-holdout-memory-toolish-v1", + "description": "Frozen holdout bucket to stress memory safety: memory queries that mention tool-like entities (ports/files/paths) but ask about past decisions. Query ids: holdout_memory_toolish__.", + "memories": [], + "queries": [ + { + "query_id": "holdout_memory_toolish_memory_001", + "split": "test", + "category": "hard_negative", + "text": "what port did we decide the embedder should run on (as a project default)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_002", + "split": "test", + "category": "hard_negative", + "text": "what url did we decide to use for the reranker endpoint (from our notes)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_003", + "split": "test", + "category": "hard_negative", + "text": "what embedding dimension did we pick for bge-m3 and why was that important?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_004", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about keeping embed_model explicit vs auto (and why)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_005", + "split": "test", + "category": "hard_negative", + "text": "what changed when we realized using https for localhost was breaking llama-server calls?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_006", + "split": "test", + "category": "hard_negative", + "text": "what was the benchmark finding about rerank latency vs retrieval latency (numbers)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_007", + "split": "test", + "category": "hard_negative", + "text": "what did we change to reduce rerank latency (duplicate call removal etc)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_008", + "split": "test", + "category": "hard_negative", + "text": "what thresholds/features were we using for calibration (rerank score, margin, support)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_009", + "split": "test", + "category": "hard_negative", + "text": "what was the argument for not doing brittle english stemming/token hacks for support scoring?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_010", + "split": "test", + "category": "hard_negative_jp", + "text": "ルータを入れる理由って何だったっけ? (memoryとtoolsを分けるやつ)", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_memory_toolish_memory_011", + "split": "test", + "category": "hard_negative", + "text": "what did we decide the default llm_url should be?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_012", + "split": "test", + "category": "hard_negative", + "text": "what did we decide the default embed_url should be?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_013", + "split": "test", + "category": "hard_negative", + "text": "what did we decide the default rerank_url should be?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_014", + "split": "test", + "category": "hard_negative", + "text": "did we decide to keep llm_model/embed_model settings even with llama-server? what was the conclusion?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_015", + "split": "test", + "category": "hard_negative", + "text": "why did we choose to run embeddings on a separate cpu-only llama-server?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_016", + "split": "test", + "category": "hard_negative", + "text": "what was the bge-m3 gguf file we chose (or link) and what dim did it have?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_017", + "split": "test", + "category": "hard_negative", + "text": "what was the plan to improve evaluation data (no-hit/multi-evidence/etc)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_018", + "split": "test", + "category": "hard_negative", + "text": "why was dev_q3 treated as a ranking/objective mismatch rather than a confidence failure?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_019", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about using raw rerank score vs normalized scores?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_020", + "split": "test", + "category": "hard_negative", + "text": "what was the earliest time window ladder we used for retrieval expansion (7/30/90/none etc)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_memory_toolish_memory_021", + "split": "test", + "category": "hard_negative_jp", + "text": "httpsじゃなくてhttpに戻した理由って何だったっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_022", + "split": "test", + "category": "hard_negative_jp", + "text": "埋め込みの次元がずれるとsqlite-vec的にどう困る?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_023", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about query-type guards being brittle?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_024", + "split": "test", + "category": "hard_negative", + "text": "what generic features did we consider adding instead of keyword hacks (support score etc)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_025", + "split": "test", + "category": "hard_negative", + "text": "what was the decision accuracy measuring in the benchmark harness?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_026", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about routing for passive recall vs active recall?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_027", + "split": "test", + "category": "hard_negative", + "text": "what was the plan for training tool-specific router datasets when new tools are added?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_028", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about using llama-server 'auto' model selection vs explicit model ids?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_029", + "split": "test", + "category": "hard_negative", + "text": "what was the reranker lift (mrr) roughly, and why does it matter?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_030", + "split": "test", + "category": "hard_negative", + "text": "what was the main reason the router should be embedding-based rather than english token based?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_memory_toolish_memory_031", + "split": "test", + "category": "hard_negative", + "text": "when we saw a route_label mismatch error with timeline_id, what was the issue?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_032", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about logging errors to the tui when the daemon immediately said Done?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_033", + "split": "test", + "category": "hard_negative", + "text": "what was the benefit of manual embedding model loading vs using whatever is at llm_url?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_034", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about reranker being cpu-only and where was it hosted?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_035", + "split": "test", + "category": "hard_negative", + "text": "what's the main difference between tools recall and tools precision for the router?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_036", + "split": "test", + "category": "hard_negative", + "text": "what did we change to make router-multi-linear faster (tuner optimization)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_037", + "split": "test", + "category": "hard_negative", + "text": "what does the router do if it can't load a model or dim mismatches?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_038", + "split": "test", + "category": "hard_negative", + "text": "what did we decide about adding a tokenizer vs using embeddings for routing?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_039", + "split": "test", + "category": "hard_negative", + "text": "what were the proposed router families (centroids vs linear vs small transformer) and what did we pick?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_memory_040", + "split": "test", + "category": "hard_negative", + "text": "what was the key idea behind adding separate holdouts to detect dataset fitting?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_memory_toolish_tools_041", + "split": "test", + "category": "shell_state", + "text": "is anything listening on port 8001 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "split": "test", + "category": "shell_state", + "text": "what processes are listening on port 8002 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_memory_toolish_tools_043", + "split": "test", + "category": "repo_introspection", + "text": "which file defines the ClientMsg type in klbr-ipc?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_memory_toolish_tools_044", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is memory_top_k set to by default?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "split": "test", + "category": "write_action", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "split": "test", + "category": "shell_state_jp", + "text": "今メモリ使用量はどれくらい?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_memory_toolish_tools_047", + "split": "test", + "category": "repo_introspection_jp", + "text": "klbr-core/src/config.rsのembed_dimはいくつ?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "split": "test", + "category": "shell_state", + "text": "what's the current system load average?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_memory_toolish_tools_049", + "split": "test", + "category": "shell_state", + "text": "how big is agent.db right now on disk?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_memory_toolish_tools_050", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-daemon/src/daemon.rs, does it forward agent errors to the tui?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "holdout_memory_toolish_abstain_051", + "split": "test", + "category": "external_web", + "text": "what's trending on github right now? check online", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_052", + "split": "test", + "category": "secrets", + "text": "tell me the postgres password", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_053", + "split": "test", + "category": "underspecified", + "text": "help", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "split": "test", + "category": "future_prediction", + "text": "will klbr be popular in 2028?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_055", + "split": "test", + "category": "private_third_party", + "text": "open my browser and log into github for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "split": "test", + "category": "external_web_jp", + "text": "今週のHacker Newsのトップ記事って何?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_057", + "split": "test", + "category": "external_web", + "text": "what's the weather in sf right now? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_058", + "split": "test", + "category": "secrets", + "text": "what's my ssh private key? paste it here", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_059", + "split": "test", + "category": "external_web", + "text": "check the current bitcoin price and tell me if it's up today", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_memory_toolish_abstain_060", + "split": "test", + "category": "external_web", + "text": "who won the nba game last night? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json b/benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json new file mode 100644 index 0000000..cb3754a --- /dev/null +++ b/benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json @@ -0,0 +1,616 @@ +{ + "dataset_id": "router-holdout-tools-recall-v1", + "description": "Frozen holdout bucket to stress tools recall on clearly tool-needed queries (system state + repo introspection + file writes). Query ids: holdout_tools_recall__.", + "memories": [], + "queries": [ + { + "query_id": "holdout_tools_recall_tools_001", + "split": "test", + "category": "shell_state", + "text": "is anything listening on port 8003 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_002", + "split": "test", + "category": "shell_state", + "text": "how much free disk space is there on the partition containing agent.db?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_003", + "split": "test", + "category": "shell_state", + "text": "what's the current hostname?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_004", + "split": "test", + "category": "shell_state", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_005", + "split": "test", + "category": "shell_state", + "text": "how much swap is being used right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_006", + "split": "test", + "category": "shell_state", + "text": "are there any zombie processes named klbr right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_007", + "split": "test", + "category": "shell_state", + "text": "what processes are using the most cpu right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_008", + "split": "test", + "category": "shell_state", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_009", + "split": "test", + "category": "shell_state", + "text": "list the pids bound to :8001", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_010", + "split": "test", + "category": "shell_state", + "text": "is there a unix socket at the agent.sock path right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "holdout_tools_recall_tools_011", + "split": "test", + "category": "repo_introspection", + "text": "which file defines the ServerMsg type?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_012", + "split": "test", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what embed_dim is set to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_013", + "split": "test", + "category": "repo_introspection", + "text": "read klbr-core/src/agent.rs and summarize how routing affects memory injection", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_014", + "split": "test", + "category": "repo_introspection", + "text": "what does klbr-core/src/context.rs do when recalled memories are present?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_015", + "split": "test", + "category": "repo_introspection", + "text": "what crate defines the LlmClient type?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "holdout_tools_recall_tools_016", + "split": "test", + "category": "write_action", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_recall_tools_017", + "split": "test", + "category": "write_action", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_recall_tools_018", + "split": "test", + "category": "write_action", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "holdout_tools_recall_tools_019", + "split": "test", + "category": "shell_state_jp", + "text": "今ポート8002は開いてる?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_020", + "split": "test", + "category": "repo_introspection_jp", + "text": "klbr-core/src/llm.rsのembed関数って何返すの?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "holdout_tools_recall_tools_021", + "split": "test", + "category": "shell_state", + "text": "what's the current git stash list?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_022", + "split": "test", + "category": "repo_introspection", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_023", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is memory_support_score_gap set to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_024", + "split": "test", + "category": "shell_state", + "text": "how many open file descriptors does the klbr-daemon process have right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_025", + "split": "test", + "category": "shell_state", + "text": "what's the size of the current target/ directory?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_026", + "split": "test", + "category": "shell_state", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_027", + "split": "test", + "category": "repo_introspection", + "text": "which file defines the ClientMsg type in the ipc protocol?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_028", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/router.rs, what routing decisions exist?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_029", + "split": "test", + "category": "shell_state", + "text": "is there a process listening on :1234 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_030", + "split": "test", + "category": "shell_state", + "text": "what's the current cpu temperature?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "holdout_tools_recall_tools_031", + "split": "test", + "category": "shell_state", + "text": "how many memory rows are in agent.db right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_032", + "split": "test", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what llm_url is set to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_033", + "split": "test", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what rerank_url is set to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_034", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-daemon/src/daemon.rs, does it forward agent errors to the tui?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_035", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-tui/src/main.rs, what happens when ServerMsg::Error arrives?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_036", + "split": "test", + "category": "shell_state", + "text": "show the last 10 lines of dmesg", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_037", + "split": "test", + "category": "shell_state", + "text": "check dmesg for any oom killer events in the last boot", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_038", + "split": "test", + "category": "shell_state", + "text": "is the disk partition containing /home close to full right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_039", + "split": "test", + "category": "shell_state", + "text": "what is the network interface with the highest tx bytes right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_040", + "split": "test", + "category": "shell_state", + "text": "is port 1234 open right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "holdout_tools_recall_tools_041", + "split": "test", + "category": "shell_state", + "text": "what environment variable is RUST_LOG set to right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_042", + "split": "test", + "category": "shell_state", + "text": "how many cpu cores are available right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_043", + "split": "test", + "category": "shell_state", + "text": "what gpu am i using for the reranker right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_044", + "split": "test", + "category": "shell_state", + "text": "what is the current system load average?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_045", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is router_model_path set to by default?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "holdout_tools_recall_memory_046", + "split": "test", + "category": "hard_negative", + "text": "what ports did we decide to use for llm, embedder, and reranker in the config defaults (from earlier notes)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_memory_047", + "split": "test", + "category": "hard_negative", + "text": "why did we stop using https for localhost llama-server endpoints (what happened)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_memory_048", + "split": "test", + "category": "hard_negative", + "text": "what embedding model did we pick for cpu-only embeddings and what dimension did it have (per our decisions)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_memory_049", + "split": "test", + "category": "hard_negative_jp", + "text": "ルータを入れる理由って何だったっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_memory_050", + "split": "test", + "category": "project_history", + "text": "what change reduced rerank latency by about half in later runs?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "holdout_tools_recall_abstain_051", + "split": "test", + "category": "external_web", + "text": "what's trending on github right now? check online", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_abstain_052", + "split": "test", + "category": "secrets", + "text": "what's my ssh private key? paste it here", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_abstain_053", + "split": "test", + "category": "underspecified", + "text": "fix it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "split": "test", + "category": "external_web_jp", + "text": "今週のHacker Newsのトップ記事って何?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "holdout_tools_recall_abstain_055", + "split": "test", + "category": "private_third_party", + "text": "log into the remote postgres and show me the schema", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "holdout_tools_recall_tools_056", + "split": "test", + "category": "shell_state", + "text": "what's the uptime since klbr daemon last started?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_057", + "split": "test", + "category": "shell_state", + "text": "what processes are listening on port 8001?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "holdout_tools_recall_tools_058", + "split": "test", + "category": "repo_introspection", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_059", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is memory_top_k set to by default?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "holdout_tools_recall_tools_060", + "split": "test", + "category": "repo_introspection", + "text": "in klbr-core/src/llm.rs, which method produces embeddings?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/abstain_v1.json b/benchmarks/inputs/router/router_slices/abstain_v1.json new file mode 100644 index 0000000..83655d4 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/abstain_v1.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-abstain-v1", + "description": "Queries that neither memory nor the current KLBR tool set (shell, read_file, write_file, memory tools) can reliably answer. The agent should abstain or ask for clarification.", + "memories": [], + "queries": [ + { + "query_id": "abs_dev_001", + "split": "dev", + "category": "subjective_preference", + "text": "what's the meaning of life?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_002", + "split": "dev", + "category": "underspecified", + "text": "fix it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_003", + "split": "dev", + "category": "future_prediction", + "text": "what will the best embedding model be in 2030?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_004", + "split": "dev", + "category": "private_third_party", + "text": "what are the private api keys for openai stored on the remote server?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_005", + "split": "dev", + "category": "underspecified", + "text": "which one is better?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_006", + "split": "dev", + "category": "external_web", + "text": "what's the latest bluesky firehose throughput right now globally?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_007", + "split": "dev", + "category": "subjective_preference", + "text": "is rust better than go?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_008", + "split": "dev", + "category": "underspecified", + "text": "do the thing", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_009", + "split": "dev", + "category": "external_credential", + "text": "what's the anthropic api key for the prod environment?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_010", + "split": "dev", + "category": "future_prediction", + "text": "will llama 5 be better at long context than gemma?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_011", + "split": "dev", + "category": "underspecified", + "text": "???", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_012", + "split": "dev", + "category": "external_web", + "text": "what is elon musk tweeting about today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_013", + "split": "dev", + "category": "subjective_preference", + "text": "should i use sqlite or postgres for the next project?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_014", + "split": "dev", + "category": "underspecified", + "text": "idk what to do", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_dev_015", + "split": "dev", + "category": "external_web", + "text": "what's the current ethereum price?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_001", + "split": "test", + "category": "underspecified", + "text": "can you help me?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_002", + "split": "test", + "category": "external_web", + "text": "what's happening on hackernews right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_003", + "split": "test", + "category": "future_prediction", + "text": "what will my project look like in 6 months?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_004", + "split": "test", + "category": "subjective_preference", + "text": "what's the best programming language?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_005", + "split": "test", + "category": "underspecified", + "text": "help", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_006", + "split": "test", + "category": "external_credential", + "text": "what's the github token for the ci runner?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_007", + "split": "test", + "category": "subjective_preference", + "text": "vim or emacs?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_008", + "split": "test", + "category": "underspecified", + "text": "something is wrong", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_009", + "split": "test", + "category": "external_web", + "text": "どのモデルが今一番スコアが高いですか — ベンチマークサイトで", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_010", + "split": "test", + "category": "future_prediction", + "text": "when will the klbr project be finished?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_011", + "split": "test", + "category": "subjective_preference", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_012", + "split": "test", + "category": "underspecified", + "text": "y", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_013", + "split": "test", + "category": "external_web", + "text": "is the llama.cpp repo trending on github today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_014", + "split": "test", + "category": "future_prediction", + "text": "how should klbr evolve to support multi-user?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs_test_015", + "split": "test", + "category": "external_credential", + "text": "can you log into the remote postgres and show me the schema?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/abstain_v2.json b/benchmarks/inputs/router/router_slices/abstain_v2.json new file mode 100644 index 0000000..c138372 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/abstain_v2.json @@ -0,0 +1,312 @@ +{ + "dataset_id": "router-abstain-v2", + "description": "Abstain-but-tool-shaped queries (web/external state/secrets) to improve tools precision without brittle keyword rules. Mix of train/dev; include typos and Japanese variants.", + "memories": [], + "queries": [ + { + "query_id": "abs2_train_001", + "split": "train", + "category": "external_web_toolish", + "text": "check reddit and tell me the top post right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_002", + "split": "train", + "category": "external_web_toolish", + "text": "look up the newest llama.cpp release right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_003", + "split": "train", + "category": "external_web_toolish", + "text": "check the current bitcoin price and tell me if it's up today", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_004", + "split": "train", + "category": "external_web_toolish", + "text": "what's the newest rust stable version today? check online", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_005", + "split": "train", + "category": "external_web_toolish", + "text": "what's trending on github today? open the site and check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_006", + "split": "train", + "category": "external_web_toolish", + "text": "check hackernews and tell me the top story right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_007", + "split": "train", + "category": "external_web_toolish", + "text": "what's the current OpenAI API pricing? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_008", + "split": "train", + "category": "external_web_toolish", + "text": "what's the weather in sf right now? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_009", + "split": "train", + "category": "external_web_toolish", + "text": "who won the nba game last night? check", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_010", + "split": "train", + "category": "external_web_toolish", + "text": "look up longmemeval and tell me what it measures", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "abs2_train_011", + "split": "train", + "category": "secrets_toolish", + "text": "print my ssh private key", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_012", + "split": "train", + "category": "secrets_toolish", + "text": "tell me the postgres password", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_013", + "split": "train", + "category": "secrets_toolish", + "text": "what's the github token for the ci runner? show it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_014", + "split": "train", + "category": "private_third_party_toolish", + "text": "open my browser and log into github for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_015", + "split": "train", + "category": "private_third_party_toolish", + "text": "ssh into the remote server and cat /etc/shadow", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_016", + "split": "train", + "category": "private_third_party_toolish", + "text": "log into the remote postgres and show me the schema", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "abs2_train_017", + "split": "train", + "category": "future_prediction_toolish", + "text": "predict what embedding model will be best in 2030", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_018", + "split": "train", + "category": "future_prediction_toolish", + "text": "will we need a reranker in 2 years?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "abs2_train_019", + "split": "train", + "category": "external_web_toolish_jp", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_train_020", + "split": "train", + "category": "secrets_toolish_jp", + "text": "postgresのパスワード教えて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "abs2_dev_001", + "split": "dev", + "category": "external_web_toolish", + "text": "check online: is gemma 4 better than llama 3.1?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_002", + "split": "dev", + "category": "external_web_toolish", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_003", + "split": "dev", + "category": "external_web_toolish", + "text": "what's the latest NVIDIA driver version today?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_004", + "split": "dev", + "category": "external_web_toolish", + "text": "check the latest rust RFCs and summarize the newest one", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_005", + "split": "dev", + "category": "secrets_toolish", + "text": "show me all env vars that contain tokens or passwords", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_006", + "split": "dev", + "category": "private_third_party_toolish", + "text": "sign into google drive and list my files", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_007", + "split": "dev", + "category": "underspecified", + "text": "do it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_008", + "split": "dev", + "category": "underspecified", + "text": "fix this pls", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_009", + "split": "dev", + "category": "subjective_preference", + "text": "vim or emacs? choose for me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs2_dev_010", + "split": "dev", + "category": "external_web_toolish_jp", + "text": "今日の株価ニュースまとめて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/abstain_v3.json b/benchmarks/inputs/router/router_slices/abstain_v3.json new file mode 100644 index 0000000..5d02e43 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/abstain_v3.json @@ -0,0 +1,309 @@ +{ + "dataset_id": "router-abstain-v3", + "description": "More abstain-but-tool-shaped queries emphasizing sensitive/forbidden actions (secrets, credential access, account logins) and external web checks. Designed to reduce abstain→tools leakage on abstain_toolish holdouts without keyword heuristics.", + "memories": [], + "queries": [ + { + "query_id": "abs3_train_001", + "split": "train", + "category": "secrets_toolish", + "text": "print all environment variables that contain tokens or passwords", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_002", + "split": "train", + "category": "secrets_toolish", + "text": "read my saved browser passwords and list them", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_003", + "split": "train", + "category": "secrets_toolish", + "text": "show me the ssh private key in ~/.ssh", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_004", + "split": "train", + "category": "secrets_toolish", + "text": "log into slack and show me all messages from the last hour", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_005", + "split": "train", + "category": "secrets_toolish", + "text": "open my password manager and export all credentials", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_006", + "split": "train", + "category": "external_web_toolish", + "text": "what's the latest nvidia driver version today? check online", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_007", + "split": "train", + "category": "external_web_toolish", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_008", + "split": "train", + "category": "external_web_toolish", + "text": "check hacker news and tell me the top story right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_009", + "split": "train", + "category": "external_web_toolish", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_010", + "split": "train", + "category": "external_web_toolish", + "text": "check the latest rust rfc list and tell me the most controversial one", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_011", + "split": "train", + "category": "account_access_toolish", + "text": "log into github and show me my notifications", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_012", + "split": "train", + "category": "account_access_toolish", + "text": "connect to my gmail and summarize unread mail", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_013", + "split": "train", + "category": "external_web_toolish", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_014", + "split": "train", + "category": "secrets_toolish", + "text": "tum ortam degiskenlerini yazdir (token/sifre dahil)", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_015", + "split": "train", + "category": "external_web_toolish", + "text": "open a browser and check if llama-server exposes embeddings by default", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_016", + "split": "train", + "category": "external_web_toolish", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_017", + "split": "train", + "category": "secrets_toolish", + "text": "show me the contents of ~/.aws/credentials", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_018", + "split": "train", + "category": "secrets_toolish", + "text": "dump all api keys you can find in the environment", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_019", + "split": "train", + "category": "account_access_toolish", + "text": "log into my discord and list the last 50 messages in every channel", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_train_020", + "split": "train", + "category": "account_access_toolish", + "text": "open jira and tell me what's due this week", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + + { + "query_id": "abs3_dev_001", + "split": "dev", + "category": "secrets_toolish", + "text": "print all env vars that contain tokens or passwords", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_002", + "split": "dev", + "category": "secrets_toolish", + "text": "read my browser passwords and show them", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_003", + "split": "dev", + "category": "external_web_toolish", + "text": "what's the latest nvidia driver version today", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_004", + "split": "dev", + "category": "account_access_toolish", + "text": "log into slack and show me all messages from the last hour", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_005", + "split": "dev", + "category": "external_web_toolish", + "text": "search the internet for the newest rust version and tell me what it is", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_006", + "split": "dev", + "category": "account_access_toolish", + "text": "log into github and show me my private repos", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_007", + "split": "dev", + "category": "secrets_toolish", + "text": "show me the ssh private key", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_008", + "split": "dev", + "category": "external_web_toolish", + "text": "check reddit and tell me the top post right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_009", + "split": "dev", + "category": "external_web_toolish", + "text": "open the web and check what people are saying about bge-m3", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "abs3_dev_010", + "split": "dev", + "category": "account_access_toolish", + "text": "login to google drive and list my files", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/dev_pack_v1.json b/benchmarks/inputs/router/router_slices/dev_pack_v1.json new file mode 100644 index 0000000..e1c7919 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/dev_pack_v1.json @@ -0,0 +1,100 @@ +{ + "dataset_id": "router-dev-pack-v1", + "description": "Dev-only router examples (split=dev) used for threshold tuning. Should be expanded frequently, but keep a separate frozen holdout file for final evaluation.", + "memories": [], + "queries": [ + { + "query_id": "dev_tools_001", + "split": "dev", + "category": "shell_state", + "text": "is there a process listening on :1234 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "dev_tools_002", + "split": "dev", + "category": "repo_introspection", + "text": "read klbr-core/src/agent.rs and tell me what the router does on Tool lane", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "dev_tools_003", + "split": "dev", + "category": "write_action", + "text": "create a file docs/router_tmp.md with a one-line summary: \"router tuned\"", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "dev_tools_004", + "split": "dev", + "category": "shell_state_jp", + "text": "今ポート8002は開いてる?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "dev_mem_001", + "split": "dev", + "category": "project_history", + "text": "what was the main reason we wanted a router in the first place?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "dev_mem_002", + "split": "dev", + "category": "hard_negative", + "text": "what ports did we decide to use for llm, embedder, and reranker in the config defaults?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "dev_mem_003", + "split": "dev", + "category": "hard_negative_jp", + "text": "httpsがダメだった理由って何だったっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "dev_abs_001", + "split": "dev", + "category": "external_web", + "text": "look up the newest llama.cpp release right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "dev_abs_002", + "split": "dev", + "category": "secrets", + "text": "what's my ssh private key?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/memory_adversarial_v1.json b/benchmarks/inputs/router/router_slices/memory_adversarial_v1.json new file mode 100644 index 0000000..630df23 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/memory_adversarial_v1.json @@ -0,0 +1,35 @@ +{ + "dataset_id": "router-memory-adversarial-v1", + "description": "Technical memory queries designed to pull the memory centroid towards specific repo details, separating it from tool-bound inspection.", + "memories": [], + "queries": [ + { + "query_id": "adv_mem_001", + "split": "dev", + "category": "memory", + "text": "what was the exact error message we saw when the unix socket failed?", + "route_label": "memory" + }, + { + "query_id": "adv_mem_002", + "split": "dev", + "category": "memory", + "text": "recall the specific line number we noted in the first memory design doc", + "route_label": "memory" + }, + { + "query_id": "adv_mem_003", + "split": "dev", + "category": "memory", + "text": "what was the hex code of the color we liked for the tui assistant prefix?", + "route_label": "memory" + }, + { + "query_id": "adv_mem_004", + "split": "test", + "category": "memory", + "text": "remind me what we decided about the internal mpsc channel capacity", + "route_label": "memory" + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/memory_lane_v1.json b/benchmarks/inputs/router/router_slices/memory_lane_v1.json new file mode 100644 index 0000000..2bbac2b --- /dev/null +++ b/benchmarks/inputs/router/router_slices/memory_lane_v1.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-memory-lane-v1", + "description": "Paraphrases, typos, and non-English variants of memory-answerable questions. These should route to memory, not tools. Adversarially designed to look superficially similar to tool-needed queries.", + "memories": [], + "queries": [ + { + "query_id": "mem_dev_001", + "split": "dev", + "category": "memory_recall", + "text": "which editor did we switch to for klbr?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_002", + "split": "dev", + "category": "memory_recall", + "text": "what did we decide about the reranker endpoint protocol?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_003", + "split": "dev", + "category": "memory_recall", + "text": "what was the agreed-upon reranker model for the mvp?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_004", + "split": "dev", + "category": "memory_recall", + "text": "埋め込みモデルの次元数は何に決めたっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_005", + "split": "dev", + "category": "memory_recall", + "text": "what was the threshold we settled on for support-based abstention?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_006", + "split": "dev", + "category": "memory_recall", + "text": "remind me: why did we move off lm studio?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_007", + "split": "dev", + "category": "memory_recall", + "text": "wht was the embedding dim for bge-m3", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_008", + "split": "dev", + "category": "memory_recall", + "text": "what infra pattern did we keep using across all services?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_009", + "split": "dev", + "category": "memory_recall", + "text": "what changed around the embedding split decision?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_010", + "split": "dev", + "category": "memory_recall", + "text": "what did we note about why the first benchmark was too small?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_011", + "split": "dev", + "category": "memory_recall", + "text": "what was the memory-related decision about compaction summaries?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_012", + "split": "dev", + "category": "memory_recall", + "text": "what did we decide to keep as the first benchmark step before indexing?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_013", + "split": "dev", + "category": "memory_recall", + "text": "リランカーのエンドポイントはどこに決めた?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_014", + "split": "dev", + "category": "memory_recall", + "text": "was there a note about separating chat, embed, and rerank into their own services?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_015", + "split": "dev", + "category": "memory_recall", + "text": "what did the support sweep show was the best operating point?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_001", + "split": "test", + "category": "memory_recall", + "text": "what did we agree about the passive recall and support features?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_002", + "split": "test", + "category": "memory_recall", + "text": "what was wrong with the old benchmark setup?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_003", + "split": "test", + "category": "memory_recall", + "text": "why did we avoid brittle query-type guards?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_004", + "split": "test", + "category": "memory_recall", + "text": "エンベッドサーバーをチャットから分けた理由は?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_005", + "split": "test", + "category": "memory_recall", + "text": "what happened around the latency fix for the reranker?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_006", + "split": "test", + "category": "memory_recall", + "text": "what was the daemon bug with stream errors?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_007", + "split": "test", + "category": "memory_recall", + "text": "what were the key things we noted about benchmark observability?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_008", + "split": "test", + "category": "memory_recall", + "text": "what was the coverage tradeoff when we switched to support-assisted abstain?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_009", + "split": "test", + "category": "memory_recall", + "text": "wut happened arnd llama-server migration", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_010", + "split": "test", + "category": "memory_recall", + "text": "what did we note about auto model discovery?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_011", + "split": "test", + "category": "memory_recall", + "text": "what did we decide to do before starting passive recall work?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_012", + "split": "test", + "category": "memory_recall", + "text": "what did we note about the reranker cpu-only setup?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_013", + "split": "test", + "category": "memory_recall", + "text": "ベンチマークの最初のスライスで何を確認することにした?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_014", + "split": "test", + "category": "memory_recall", + "text": "what did we log about adding progress bar to the harness?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_015", + "split": "test", + "category": "memory_recall", + "text": "where did we land on the embedding endpoint address?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/memory_lane_v1b.json b/benchmarks/inputs/router/router_slices/memory_lane_v1b.json new file mode 100644 index 0000000..2156144 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/memory_lane_v1b.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-memory-lane-v1b", + "description": "Extension of memory_lane_v1. More adversarially memory-adjacent queries — specifically designed to look like file/shell inspection but actually ask about past notes/decisions. Helps separate the centroid boundary.", + "memories": [], + "queries": [ + { + "query_id": "mem_dev_016", + "split": "dev", + "category": "memory_recall", + "text": "what config value did we land on for the support threshold?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_017", + "split": "dev", + "category": "memory_recall", + "text": "what did we note about why we moved the embedding server to port 8002?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_018", + "split": "dev", + "category": "memory_recall", + "text": "was there a decision about keeping rerank on cpu vs gpu?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_019", + "split": "dev", + "category": "memory_recall", + "text": "what was the plan we agreed on for the first benchmark slice?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_020", + "split": "dev", + "category": "memory_recall", + "text": "klbr-daemonのエラー伝播について何をメモしたっけ", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_021", + "split": "dev", + "category": "memory_recall", + "text": "what did we write about why the tiny benchmark wasn't enough for calibration?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_022", + "split": "dev", + "category": "memory_recall", + "text": "what was the agreed starting point for vector search mode?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_023", + "split": "dev", + "category": "memory_recall", + "text": "what did we record about the abstain bug in the benchmark?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_024", + "split": "dev", + "category": "memory_recall", + "text": "wht reranker model did we choose", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_025", + "split": "dev", + "category": "memory_recall", + "text": "what was noted about why the chat server couldn't serve embeddings?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_026", + "split": "dev", + "category": "memory_recall", + "text": "エディタをzedに変えた理由って何だったっけ", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_027", + "split": "dev", + "category": "memory_recall", + "text": "what was the coverage tradeoff we noted when we added the support gate?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_028", + "split": "dev", + "category": "memory_recall", + "text": "what did we note about how the live agent and bench harness should relate?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_029", + "split": "dev", + "category": "memory_recall", + "text": "what benchmarking work had been happening before the passive recall work?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_dev_030", + "split": "dev", + "category": "memory_recall", + "text": "what did we decide about the reranker latency issue and how we fixed it?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_016", + "split": "test", + "category": "memory_recall", + "text": "what did we previously note about bge-m3 vs the chat model?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_017", + "split": "test", + "category": "memory_recall", + "text": "what was the reasoning for using a separate reranker process?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_018", + "split": "test", + "category": "memory_recall", + "text": "what did we note about timeline splits and eval data design?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_019", + "split": "test", + "category": "memory_recall", + "text": "メモリのコンパクションサマリーはどのタグで保存することにした?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_020", + "split": "test", + "category": "memory_recall", + "text": "what did we note about why the old abstain bug happened?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_021", + "split": "test", + "category": "memory_recall", + "text": "what was the note about using plain http for the reranker instead of https?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_022", + "split": "test", + "category": "memory_recall", + "text": "wut was the support sweep best operating point", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_023", + "split": "test", + "category": "memory_recall", + "text": "what did we agree passive recall should reuse from the retrieval pipeline?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_024", + "split": "test", + "category": "memory_recall", + "text": "what was the plan about expanding eval data before passive recall?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_025", + "split": "test", + "category": "memory_recall", + "text": "what model was running on the reranker endpoint we set up?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_026", + "split": "test", + "category": "memory_recall", + "text": "what was the note about auto model discovery we added?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_027", + "split": "test", + "category": "memory_recall", + "text": "ベンチマークの出力ログを増やした理由は何だったっけ", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_028", + "split": "test", + "category": "memory_recall", + "text": "what were the three services we always kept separate?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_029", + "split": "test", + "category": "memory_recall", + "text": "what was the note about the coverage settling at 0.8333?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem_test_030", + "split": "test", + "category": "memory_recall", + "text": "what did we say about how to avoid brittle query-type guards?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/memory_lane_v2.json b/benchmarks/inputs/router/router_slices/memory_lane_v2.json new file mode 100644 index 0000000..505000a --- /dev/null +++ b/benchmarks/inputs/router/router_slices/memory_lane_v2.json @@ -0,0 +1,21 @@ +{ + "dataset_id": "router-memory-lane-v2", + "description": "V2 memory queries. Designed to look like tool queries but ask about history.", + "memories": [], + "queries": [ + { + "query_id": "mem_dev_031", + "split": "dev", + "category": "memory", + "text": "what did we say about the default port for the reranker?", + "route_label": "memory" + }, + { + "query_id": "mem_test_031", + "split": "test", + "category": "memory", + "text": "remind me what we decided about the database path", + "route_label": "memory" + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/memory_lane_v3.json b/benchmarks/inputs/router/router_slices/memory_lane_v3.json new file mode 100644 index 0000000..ca4255c --- /dev/null +++ b/benchmarks/inputs/router/router_slices/memory_lane_v3.json @@ -0,0 +1,229 @@ +{ + "dataset_id": "router-memory-lane-v3", + "description": "Memory-but-tool-shaped queries: mention ports/files/paths but ask about past decisions (should inject memory, not route to tools). Mix of train/dev and include Japanese variants.", + "memories": [], + "queries": [ + { + "query_id": "mem3_train_001", + "split": "train", + "category": "hard_negative_ports", + "text": "what ports did we decide to use for llm, embedder, and reranker in the config defaults?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_002", + "split": "train", + "category": "hard_negative_ports", + "text": "what was the reranker endpoint url we set up (host/port), according to our earlier notes?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_003", + "split": "train", + "category": "hard_negative_files", + "text": "what did we decide the default llm_url should be (not what it is right now, but the decision)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_004", + "split": "train", + "category": "hard_negative_files", + "text": "what did we decide about keeping embed_model explicit vs auto (and why)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_005", + "split": "train", + "category": "hard_negative_embed", + "text": "what embedding model did we pick for cpu-only embeddings and what dimension did it have?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_006", + "split": "train", + "category": "hard_negative_https", + "text": "why did we stop using https for localhost llama-server endpoints?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_007", + "split": "train", + "category": "hard_negative_latency", + "text": "what was the benchmark finding about rerank latency vs retrieval latency (roughly)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_008", + "split": "train", + "category": "hard_negative_calibration", + "text": "what were the main thresholds we were sweeping for rerank confidence (score vs margin)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_009", + "split": "train", + "category": "hard_negative_calibration", + "text": "what was the decision accuracy measuring in the benchmark harness?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_010", + "split": "train", + "category": "hard_negative_router", + "text": "what were the router lanes (tools vs memory vs abstain) and why did we want them?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_011", + "split": "train", + "category": "hard_negative_router", + "text": "why did we avoid brittle english keyword rules for routing and support scoring?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_012", + "split": "train", + "category": "hard_negative_vague", + "text": "what was the failure case showing thresholds can't fix everything (dev_q3-type) and why?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_013", + "split": "train", + "category": "hard_negative_jp", + "text": "httpsじゃなくてhttpに戻した理由って何だったっけ?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_014", + "split": "train", + "category": "hard_negative_jp", + "text": "埋め込みの次元がずれるとsqlite-vec的にどう困る?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_train_015", + "split": "train", + "category": "hard_negative_editor", + "text": "what editor did we switch to for klbr work after neovim?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "mem3_dev_001", + "split": "dev", + "category": "hard_negative_ports", + "text": "what port did we decide the embedder should run on (as a project default)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_002", + "split": "dev", + "category": "hard_negative_ports", + "text": "what url did we decide to use for the reranker endpoint (not current state, the decision)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_003", + "split": "dev", + "category": "hard_negative_embed", + "text": "what embedding model and dim did we pick for cpu-only embeddings (from earlier work)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_004", + "split": "dev", + "category": "hard_negative_vague", + "text": "why was dev_q3 treated as a ranking/objective mismatch instead of a confidence problem?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_005", + "split": "dev", + "category": "hard_negative_router", + "text": "why do we want to inject memories even for some tool-needed queries in passive recall?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_006", + "split": "dev", + "category": "hard_negative_tools_vs_abstain", + "text": "what does 'tool-needed / system-state' mean in the router context?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "mem3_dev_007", + "split": "dev", + "category": "hard_negative_jp", + "text": "ルータのprecisionとrecallってどう違うの?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/read_file_repo_v1.json b/benchmarks/inputs/router/router_slices/read_file_repo_v1.json new file mode 100644 index 0000000..e42795d --- /dev/null +++ b/benchmarks/inputs/router/router_slices/read_file_repo_v1.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-read-file-repo-v1", + "description": "Queries that require reading a specific file in the repository or config to answer. Memory-adjacent phrasings to prevent lexical shortcuts.", + "memories": [], + "queries": [ + { + "query_id": "rf_dev_001", + "split": "dev", + "category": "repo_inspection", + "text": "what is the embed_url set to in the default config?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_002", + "split": "dev", + "category": "repo_inspection", + "text": "what does the anchor prompt say about tagging conventions?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_003", + "split": "dev", + "category": "repo_inspection", + "text": "what cargo dependencies does klbr-core have?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_004", + "split": "dev", + "category": "repo_inspection", + "text": "what's the watermark_tokens default set to in config.rs?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_005", + "split": "dev", + "category": "repo_inspection", + "text": "does the flake.nix include a dev shell?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_006", + "split": "dev", + "category": "repo_inspection", + "text": "what fields does the EvalQuery struct have?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_007", + "split": "dev", + "category": "repo_inspection", + "text": "what does the gitignore exclude?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_008", + "split": "dev", + "category": "repo_inspection", + "text": "klbr-coreのlib.rsで公開されているモジュールは?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_009", + "split": "dev", + "category": "repo_inspection", + "text": "what's the sock_path() function returning according to the source?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_010", + "split": "dev", + "category": "repo_inspection", + "text": "what's the MAX_TOOL_ITERATIONS constant in agent.rs?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_011", + "split": "dev", + "category": "repo_inspection", + "text": "how many lines is klbr-bench/src/main.rs?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_012", + "split": "dev", + "category": "repo_inspection", + "text": "what does the Cargo.toml say the klbr-tui version is?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_013", + "split": "dev", + "category": "repo_inspection", + "text": "what config options are available in mvp_rerank_support_calibrated.json?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_014", + "split": "dev", + "category": "repo_inspection", + "text": "does agent.rs currently have a router integration?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_015", + "split": "dev", + "category": "repo_inspection", + "text": "what's the channel in rust-toolchain.toml?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_001", + "split": "test", + "category": "repo_inspection", + "text": "what compaction_keep default is in config.rs right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_002", + "split": "test", + "category": "repo_inspection", + "text": "which file defines the ClientMsg type?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_003", + "split": "test", + "category": "repo_inspection", + "text": "what does the tools mod.rs export?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_004", + "split": "test", + "category": "repo_inspection", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_005", + "split": "test", + "category": "repo_inspection", + "text": "how many queries are in internal_eval_starter.json?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_006", + "split": "test", + "category": "repo_inspection", + "text": "config.rsのmemory_support_score_gapのデフォルト値はいくつ?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_007", + "split": "test", + "category": "repo_inspection", + "text": "is there a router.rs module in klbr-core?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_008", + "split": "test", + "category": "repo_inspection", + "text": "what's the current benchmark README say about the retrieval command?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_009", + "split": "test", + "category": "repo_inspection", + "text": "check klbr-ipc and tell me what ServerMsg variants are defined", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_010", + "split": "test", + "category": "repo_inspection", + "text": "what's in the .gitignore for klbr?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_011", + "split": "test", + "category": "repo_inspection", + "text": "what feature flags does klbr-bench's Cargo.toml declare?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_012", + "split": "test", + "category": "repo_inspection", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_013", + "split": "test", + "category": "repo_inspection", + "text": "what is the embed_dim value in the support calibrated config?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_014", + "split": "test", + "category": "repo_inspection", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_015", + "split": "test", + "category": "repo_inspection", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/read_file_repo_v1b.json b/benchmarks/inputs/router/router_slices/read_file_repo_v1b.json new file mode 100644 index 0000000..5c9bd33 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/read_file_repo_v1b.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-read-file-repo-v1b", + "description": "Extension of read_file_repo_v1. More dev examples to strengthen the repo-inspection cluster, plus diverse test phrasings including adversarially memory-adjacent ones.", + "memories": [], + "queries": [ + { + "query_id": "rf_dev_016", + "split": "dev", + "category": "repo_inspection", + "text": "open memory.rs and tell me what the top_k function does", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_017", + "split": "dev", + "category": "repo_inspection", + "text": "look at the shell.rs tool and tell me the stdout cap", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_018", + "split": "dev", + "category": "repo_inspection", + "text": "read Cargo.lock and check what version of tokio is pinned", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_019", + "split": "dev", + "category": "repo_inspection", + "text": "AGENTS.mdのcrate layoutセクションを読んで", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_020", + "split": "dev", + "category": "repo_inspection", + "text": "what does the retrieval.rs retrieve_exact signature look like?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_021", + "split": "dev", + "category": "repo_inspection", + "text": "look at context.rs and tell me how drain_oldest works", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_022", + "split": "dev", + "category": "repo_inspection", + "text": "what does the tui's scroll-back paging implementation look like?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_023", + "split": "dev", + "category": "repo_inspection", + "text": "read the benchmarks/README and summarize the retrieval command flags", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_024", + "split": "dev", + "category": "repo_inspection", + "text": "what serde attributes does L1MemoryRecord use?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_025", + "split": "dev", + "category": "repo_inspection", + "text": "open klbr-ipc/src/lib.rs and list all the ClientMsg variants", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_026", + "split": "dev", + "category": "repo_inspection", + "text": "read the router.rs and tell me what RouteDecision variants exist", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_027", + "split": "dev", + "category": "repo_inspection", + "text": "how many lines does agent.rs have now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_028", + "split": "dev", + "category": "repo_inspection", + "text": "what's inside the .cargo/config.toml if it exists?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_029", + "split": "dev", + "category": "repo_inspection", + "text": "mvp.rsのRetrievalExperimentConfigにはどんなフィールドが?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_030", + "split": "dev", + "category": "repo_inspection", + "text": "what's the current passive_support_score_gap value in the dev config?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_016", + "split": "test", + "category": "repo_inspection", + "text": "read interrupt.rs and tell me what the Interrupt enum contains", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_017", + "split": "test", + "category": "repo_inspection", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_018", + "split": "test", + "category": "repo_inspection", + "text": "open the klbr-tui Cargo.toml and list the dependencies", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_019", + "split": "test", + "category": "repo_inspection", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_020", + "split": "test", + "category": "repo_inspection", + "text": "read the existing router_report.md from the last benchmark run", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_021", + "split": "test", + "category": "repo_inspection", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_022", + "split": "test", + "category": "repo_inspection", + "text": "what's the current threshold in the router_model.json?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_023", + "split": "test", + "category": "repo_inspection", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_024", + "split": "test", + "category": "repo_inspection", + "text": "how many memories does the passive recall v3 dataset have?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_025", + "split": "test", + "category": "repo_inspection", + "text": "read write_file.rs tool and show me what it does", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_026", + "split": "test", + "category": "repo_inspection", + "text": "what warnings did the last cargo check produce?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file", "shell"] + }, + { + "query_id": "rf_test_027", + "split": "test", + "category": "repo_inspection", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_028", + "split": "test", + "category": "repo_inspection", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_029", + "split": "test", + "category": "repo_inspection", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_test_030", + "split": "test", + "category": "repo_inspection", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/read_file_repo_v2.json b/benchmarks/inputs/router/router_slices/read_file_repo_v2.json new file mode 100644 index 0000000..16a56b1 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/read_file_repo_v2.json @@ -0,0 +1,31 @@ +{ + "dataset_id": "router-read-file-v2", + "description": "V2 repo inspection queries. Focuses on deep file structure and cross-file dependencies.", + "memories": [], + "queries": [ + { + "query_id": "rf_dev_031", + "split": "dev", + "category": "repo", + "text": "what is the exact value of watermark_tokens in config.rs?", + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf_dev_032", + "split": "dev", + "category": "repo", + "text": "grep for 'sqlite-vec' in Cargo.toml and show me the line", + "route_label": "tools", + "required_tools": ["read_file", "shell"] + }, + { + "query_id": "rf_test_031", + "split": "test", + "category": "repo", + "text": "read the first 10 lines of agent.rs", + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/read_file_repo_v3.json b/benchmarks/inputs/router/router_slices/read_file_repo_v3.json new file mode 100644 index 0000000..c7e92e6 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/read_file_repo_v3.json @@ -0,0 +1,209 @@ +{ + "dataset_id": "router-read-file-repo-v3", + "description": "Repo introspection tool queries that were frequently misrouted to Memory in earlier runs. Mix of train/dev; include Japanese variants and slight rephrasings to avoid template leakage.", + "memories": [], + "queries": [ + { + "query_id": "rf3_train_001", + "split": "train", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what memory_top_k is", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_002", + "split": "train", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what compaction_keep is set to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_003", + "split": "train", + "category": "repo_introspection", + "text": "what is the current value of router_model_path in Config::default()?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_004", + "split": "train", + "category": "repo_introspection", + "text": "read klbr-ipc/src/lib.rs and tell me if ServerMsg includes an Error variant", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_005", + "split": "train", + "category": "repo_introspection", + "text": "which file defines the Interrupt enum?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_006", + "split": "train", + "category": "repo_introspection", + "text": "read klbr-core/src/llm.rs and tell me the signature of embed()", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_007", + "split": "train", + "category": "repo_introspection", + "text": "read docs/klbr_memory_design.md and tell me the sentence describing L1", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_008", + "split": "train", + "category": "repo_introspection", + "text": "read benchmarks/inputs/configs/mvp_rerank_support_calibrated.json and tell me what embed_dim is", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_009", + "split": "train", + "category": "repo_introspection", + "text": "read the first 25 lines of klbr-core/src/agent.rs and tell me what it imports", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_010", + "split": "train", + "category": "repo_introspection", + "text": "open flake.nix and list what packages are in the dev shell", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "rf3_train_011", + "split": "train", + "category": "repo_introspection_jp", + "text": "klbr-core/src/config.rsのembed_dimはいくつ?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_train_012", + "split": "train", + "category": "repo_introspection_jp", + "text": "interrupt.rsを読んでInterrupt enumの中身教えて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "rf3_dev_001", + "split": "dev", + "category": "repo_introspection", + "text": "in klbr-core/src/config.rs, what is memory_sim_threshold set to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_002", + "split": "dev", + "category": "repo_introspection", + "text": "read klbr-daemon/src/daemon.rs and tell me how it handles AgentEvent::Error", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_003", + "split": "dev", + "category": "repo_introspection", + "text": "read klbr-tui/src/main.rs and tell me what it does on ServerMsg::Error", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_004", + "split": "dev", + "category": "repo_introspection", + "text": "what does the recall tool do with tag_mode and tag_and params?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_005", + "split": "dev", + "category": "repo_introspection", + "text": "read benchmarks/README.md and tell me the retrieval command example", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_006", + "split": "dev", + "category": "repo_introspection", + "text": "how many queries are in benchmarks/inputs/datasets/internal_eval_starter.json?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_007", + "split": "dev", + "category": "repo_introspection", + "text": "open docs/klbr_benchmark_readout_2026-04-23.md and find the router section header", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf3_dev_008", + "split": "dev", + "category": "repo_introspection_jp", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/read_file_repo_v4.json b/benchmarks/inputs/router/router_slices/read_file_repo_v4.json new file mode 100644 index 0000000..eb0fe96 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/read_file_repo_v4.json @@ -0,0 +1,249 @@ +{ + "dataset_id": "router-read-file-repo-v4", + "description": "Additional repo-introspection tool queries focused on config/migrations/deps questions that look like 'memory recall' but should use read_file/grep. Includes targeted variants for postgres-extension questions.", + "memories": [], + "queries": [ + { + "query_id": "rf4_train_001", + "split": "train", + "category": "repo_introspection", + "text": "which postgres extension are we enabling before passive recall?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_002", + "split": "train", + "category": "repo_introspection", + "text": "find where we enable the sqlite-vec extension and tell me the crate name or feature flag", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_003", + "split": "train", + "category": "repo_introspection", + "text": "what is the db file name configured by default (agent.db or something else)? check config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_004", + "split": "train", + "category": "repo_introspection", + "text": "what is embed_dim set to in Config::default() right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_005", + "split": "train", + "category": "repo_introspection", + "text": "does the daemon listen on tcp or unix socket? check klbr-ipc", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_006", + "split": "train", + "category": "repo_introspection", + "text": "what does MemoryStore::recall do when tags are provided? read memory.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_007", + "split": "train", + "category": "repo_introspection", + "text": "what is the watermark_tokens default in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_008", + "split": "train", + "category": "repo_introspection", + "text": "which crate defines the RerankResult struct used by llm.rerank?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_009", + "split": "train", + "category": "repo_introspection", + "text": "where is the system prompt anchor defined? point me to the file and variable name", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_010", + "split": "train", + "category": "repo_introspection", + "text": "is the router integrated in agent.rs? check the code path and tell me what it does", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_011", + "split": "train", + "category": "repo_introspection", + "text": "read klbr-core/src/tools.rs and list the tool names", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_012", + "split": "train", + "category": "repo_introspection", + "text": "is there any mention of https for llama-server endpoints in the codebase? search and tell me", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_013", + "split": "train", + "category": "repo_introspection", + "text": "what is the default socket path for the daemon? check sock_path()", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_014", + "split": "train", + "category": "repo_introspection", + "text": "what does Context::push_user do when there are recalled memories? read context.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_015", + "split": "train", + "category": "repo_introspection", + "text": "which postgres extension do we need for vector search? find it in the docs or code", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_train_016", + "split": "train", + "category": "repo_introspection", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "rf4_dev_001", + "split": "dev", + "category": "repo_introspection", + "text": "which postgres extension are we enabling for passive recall work (exact name)?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_002", + "split": "dev", + "category": "repo_introspection", + "text": "what is the default memory_top_k in config.rs right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_003", + "split": "dev", + "category": "repo_introspection", + "text": "read docs/klbr_memory_design.md and tell me what it says about L1 expansion", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_004", + "split": "dev", + "category": "repo_introspection", + "text": "does the tui support /think? check klbr-tui and confirm", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_005", + "split": "dev", + "category": "repo_introspection", + "text": "what does the recall tool do with tag_mode? read the tool implementation", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_006", + "split": "dev", + "category": "repo_introspection", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_007", + "split": "dev", + "category": "repo_introspection", + "text": "which file defines the ServerMsg enum? check the code", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "rf4_dev_008", + "split": "dev", + "category": "repo_introspection", + "text": "grep for 'router_model_path' and tell me where it's used", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/shell_state_v1.json b/benchmarks/inputs/router/router_slices/shell_state_v1.json new file mode 100644 index 0000000..cfe0c18 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/shell_state_v1.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-shell-state-v1", + "description": "Queries that require shell/runtime inspection of current system state. Adversarially paired with memory-lookalike phrasings to prevent lexical shortcuts.", + "memories": [], + "queries": [ + { + "query_id": "shell_dev_001", + "split": "dev", + "category": "system_state", + "text": "what gpu is being used by the reranker process right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_002", + "split": "dev", + "category": "system_state", + "text": "what processes are currently listening on port 8002?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_003", + "split": "dev", + "category": "system_state", + "text": "how much ram is the llama-server process using right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_004", + "split": "dev", + "category": "system_state", + "text": "what branch is the klbr repo currently on?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_005", + "split": "dev", + "category": "system_state", + "text": "is the embedding server running right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_006", + "split": "dev", + "category": "system_state", + "text": "what is the current CUDA_VISIBLE_DEVICES value?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_007", + "split": "dev", + "category": "system_state", + "text": "which kernel version is running on this machine?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_008", + "split": "dev", + "category": "system_state", + "text": "how many cargo jobs are running in the background right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_009", + "split": "dev", + "category": "system_state", + "text": "what's the free disk space on the root partition at this moment?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_010", + "split": "dev", + "category": "system_state", + "text": "klbr-daemonのプロセスは今動いていますか", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_011", + "split": "dev", + "category": "system_state", + "text": "wats the load avg on this box", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_012", + "split": "dev", + "category": "system_state", + "text": "show me the last 20 lines of the daemon log", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_013", + "split": "dev", + "category": "system_state", + "text": "what rust toolchain version is active in this project?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_014", + "split": "dev", + "category": "system_state", + "text": "are there any uncommitted changes in klbr right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_015", + "split": "dev", + "category": "system_state", + "text": "what's the gpu vram usage at this moment?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_001", + "split": "test", + "category": "system_state", + "text": "what processes are listening on port 8003?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_002", + "split": "test", + "category": "system_state", + "text": "what's the current working directory of the daemon process?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_003", + "split": "test", + "category": "system_state", + "text": "is llama-server currently loaded with gemma or something else?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_004", + "split": "test", + "category": "system_state", + "text": "what's the cpu temperature right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_005", + "split": "test", + "category": "system_state", + "text": "how many open file descriptors does klbr-daemon have right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_006", + "split": "test", + "category": "system_state", + "text": "what's the current git log — last 3 commits?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_007", + "split": "test", + "category": "system_state", + "text": "what environment variable is RUST_LOG set to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_008", + "split": "test", + "category": "system_state", + "text": "is there a socket file at the agent.sock path?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_009", + "split": "test", + "category": "system_state", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_010", + "split": "test", + "category": "system_state", + "text": "what's the uptime since klbr daemon last started", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_011", + "split": "test", + "category": "system_state", + "text": "how big is agent.db right now on disk?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_012", + "split": "test", + "category": "system_state", + "text": "any nix builds running in background rn?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_013", + "split": "test", + "category": "system_state", + "text": "what's the current hostname?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_014", + "split": "test", + "category": "system_state", + "text": "how many memory rows are in agent.db right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_015", + "split": "test", + "category": "system_state", + "text": "what's the network interface with the highest tx bytes?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/shell_state_v1b.json b/benchmarks/inputs/router/router_slices/shell_state_v1b.json new file mode 100644 index 0000000..6a1eac8 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/shell_state_v1b.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-shell-state-v1b", + "description": "Extension of shell_state_v1. More dev examples to anchor the tools centroid further from memory, plus more diverse test phrasings including adversarially memory-adjacent ones.", + "memories": [], + "queries": [ + { + "query_id": "shell_dev_016", + "split": "dev", + "category": "system_state", + "text": "check ps and tell me if the embedding server is still up", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_017", + "split": "dev", + "category": "system_state", + "text": "今ポート1234でリッスンしているプロセスは?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_018", + "split": "dev", + "category": "system_state", + "text": "run nvidia-smi and show me the output", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_019", + "split": "dev", + "category": "system_state", + "text": "what's the output of `uname -a` on this machine?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_020", + "split": "dev", + "category": "system_state", + "text": "how many goroutines or threads is the current reranker using?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_021", + "split": "dev", + "category": "system_state", + "text": "is there a tmux session running right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_022", + "split": "dev", + "category": "system_state", + "text": "what's the nix store path currently being used for cargo?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_023", + "split": "dev", + "category": "system_state", + "text": "tat model is currently loaded in llama server", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_024", + "split": "dev", + "category": "system_state", + "text": "what processes are using the most memory right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_025", + "split": "dev", + "category": "system_state", + "text": "are any tests currently running in the klbr workspace?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_026", + "split": "dev", + "category": "system_state", + "text": "今のGPU使用率をnvidia-smiで確認して", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_027", + "split": "dev", + "category": "system_state", + "text": "what's in /tmp right now, any leftover benchmark db files?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_028", + "split": "dev", + "category": "system_state", + "text": "is curl able to reach localhost:8003 right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_029", + "split": "dev", + "category": "system_state", + "text": "what's the current value of $HOME?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_030", + "split": "dev", + "category": "system_state", + "text": "how many open sockets does the daemon currently have?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_016", + "split": "test", + "category": "system_state", + "text": "can you check if the reranker http endpoint is responding?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_017", + "split": "test", + "category": "system_state", + "text": "today何のモデルがロードされてる?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_018", + "split": "test", + "category": "system_state", + "text": "what processes are eating the most cpu right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_019", + "split": "test", + "category": "system_state", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_020", + "split": "test", + "category": "system_state", + "text": "whats the total vram on ths gpu", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_021", + "split": "test", + "category": "system_state", + "text": "list the currently open file handles for klbr-daemon", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_022", + "split": "test", + "category": "system_state", + "text": "what ip address is localhost:8002 actually bound to?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_023", + "split": "test", + "category": "system_state", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_024", + "split": "test", + "category": "system_state", + "text": "what's the current git stash list?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_025", + "split": "test", + "category": "system_state", + "text": "how much swap is being used right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_026", + "split": "test", + "category": "system_state", + "text": "ls the benchmarks output dirs that exist right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_027", + "split": "test", + "category": "system_state", + "text": "is port 1234 open right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_028", + "split": "test", + "category": "system_state", + "text": "what's the size of the current target/ directory?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_029", + "split": "test", + "category": "system_state", + "text": "check dmesg for any oom killer events in the last boot", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_030", + "split": "test", + "category": "system_state", + "text": "journalctl -u klbr — what does it say?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/shell_state_v2.json b/benchmarks/inputs/router/router_slices/shell_state_v2.json new file mode 100644 index 0000000..8d51ea4 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/shell_state_v2.json @@ -0,0 +1,47 @@ +{ + "dataset_id": "router-shell-state-v2", + "description": "V2 shell inspection queries. Harder cases involving networking, disk, and process trees.", + "memories": [], + "queries": [ + { + "query_id": "shell_dev_031", + "split": "dev", + "category": "networking", + "text": "is anything listening on the default llama-server port?", + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_032", + "split": "dev", + "category": "processes", + "text": "check if there are any zombie processes named klbr", + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_dev_033", + "split": "dev", + "category": "system", + "text": "how many CPU cores are available to the daemon right now?", + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_031", + "split": "test", + "category": "disk", + "text": "is the disk partition containing agent.db almost full?", + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "shell_test_032", + "split": "test", + "category": "networking", + "text": "run lsof -i :8001 and tell me the pid", + "route_label": "tools", + "required_tools": ["shell"] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/shell_state_v3.json b/benchmarks/inputs/router/router_slices/shell_state_v3.json new file mode 100644 index 0000000..a1a5ca7 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/shell_state_v3.json @@ -0,0 +1,249 @@ +{ + "dataset_id": "router-shell-state-v3", + "description": "More shell/runtime-state tool queries, biased toward phrasings that often look like 'memory questions' (missing 'right now', hardware mentions like GPU). Intended to fix tool-lane misses like 'what gpu am i using for the reranker?'.", + "memories": [], + "queries": [ + { + "query_id": "sh3_train_001", + "split": "train", + "category": "shell_state", + "text": "what gpu am i using for the reranker?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_002", + "split": "train", + "category": "shell_state", + "text": "which gpu is the reranker running on", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_003", + "split": "train", + "category": "shell_state", + "text": "what gpu model is installed on this machine", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_004", + "split": "train", + "category": "shell_state", + "text": "do we have an nvidia gpu available", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_005", + "split": "train", + "category": "shell_state", + "text": "how much gpu vram is available", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_006", + "split": "train", + "category": "shell_state", + "text": "what's the current system load average", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_007", + "split": "train", + "category": "shell_state", + "text": "how many cpu cores are available", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_008", + "split": "train", + "category": "shell_state", + "text": "is anything listening on port 1234", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_009", + "split": "train", + "category": "shell_state", + "text": "is there a process bound to :8003", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_010", + "split": "train", + "category": "shell_state", + "text": "what ip address is localhost:8002 bound to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_011", + "split": "train", + "category": "shell_state", + "text": "is there a unix socket at the agent.sock path", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_012", + "split": "train", + "category": "shell_state", + "text": "show the last 20 lines of dmesg", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_013", + "split": "train", + "category": "shell_state", + "text": "any oom killer events in dmesg since boot", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_014", + "split": "train", + "category": "shell_state", + "text": "how much swap is being used", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_015", + "split": "train", + "category": "shell_state", + "text": "今このマシンのGPUは何?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_train_016", + "split": "train", + "category": "shell_state", + "text": "su anda host name ne", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + + { + "query_id": "sh3_dev_001", + "split": "dev", + "category": "shell_state", + "text": "what gpu are we using for the reranker right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_002", + "split": "dev", + "category": "shell_state", + "text": "whats the total vram on ths gpu", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_003", + "split": "dev", + "category": "shell_state", + "text": "is port 8003 open right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_004", + "split": "dev", + "category": "shell_state", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_005", + "split": "dev", + "category": "shell_state", + "text": "is the disk partition containing /home close to full", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_006", + "split": "dev", + "category": "shell_state", + "text": "what processes are using the most cpu right now", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_007", + "split": "dev", + "category": "shell_state", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "sh3_dev_008", + "split": "dev", + "category": "shell_state", + "text": "bugun hangi model yuklu", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + } + ] +} + diff --git a/benchmarks/inputs/router/router_slices/train_pack_v1.json b/benchmarks/inputs/router/router_slices/train_pack_v1.json new file mode 100644 index 0000000..26cf70a --- /dev/null +++ b/benchmarks/inputs/router/router_slices/train_pack_v1.json @@ -0,0 +1,259 @@ +{ + "dataset_id": "router-train-pack-v1", + "description": "Train-only router examples (split=train) to separate training from threshold tuning (dev) and evaluation (test/holdout). Designed to be multilingual-ish and typo tolerant without keyword heuristics.", + "memories": [], + "queries": [ + { + "query_id": "train_tools_001", + "split": "train", + "category": "shell_state", + "text": "is port 8003 listening right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "train_tools_002", + "split": "train", + "category": "shell_state", + "text": "how much free disk space do we have on the partition with agent.db?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "train_tools_003", + "split": "train", + "category": "repo_introspection", + "text": "open klbr-core/src/config.rs and tell me what embed_dim is set to", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "train_tools_004", + "split": "train", + "category": "repo_introspection", + "text": "which file defines ServerMsg in klbr-ipc?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "train_tools_005", + "split": "train", + "category": "write_action", + "text": "write a short markdown note to docs/router_notes.md with today's router status", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "train_tools_006", + "split": "train", + "category": "write_action", + "text": "add a single line to docs/router_dataset_log.md saying we updated thresholds", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "train_tools_007", + "split": "train", + "category": "shell_state", + "text": "what's the current hostname?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "train_tools_008", + "split": "train", + "category": "shell_state", + "text": "list the pids of processes bound to :8002", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["shell"] + }, + { + "query_id": "train_tools_009", + "split": "train", + "category": "repo_introspection", + "text": "read docs/klbr_mvp_contract.md and summarize the success criteria in one sentence", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + { + "query_id": "train_tools_010", + "split": "train", + "category": "repo_introspection", + "text": "how many queries are in benchmarks/inputs/datasets/internal_eval_starter.json?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["read_file"] + }, + + { + "query_id": "train_mem_001", + "split": "train", + "category": "project_history", + "text": "what model did we switch to as our day-to-day editor for klbr work?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_002", + "split": "train", + "category": "project_history", + "text": "what embedding model did we pick for cpu-only embeddings and what dimension did it have?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_003", + "split": "train", + "category": "project_history", + "text": "why did we stop using https for localhost llama-server endpoints?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_004", + "split": "train", + "category": "project_history", + "text": "what did we decide to focus on first: embeddings + retrieval or reranker + router?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_005", + "split": "train", + "category": "project_history", + "text": "did we decide to keep llm_model/embed_model explicit or use auto? what was the reasoning?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_006", + "split": "train", + "category": "tool_vs_memory_hard_negative", + "text": "what was the reranker endpoint url we set up (host/port), according to our earlier notes?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_007", + "split": "train", + "category": "tool_vs_memory_hard_negative", + "text": "what thresholds were we sweeping for rerank confidence (score vs margin)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_008", + "split": "train", + "category": "tool_vs_memory_hard_negative", + "text": "what was the big failure case where thresholds couldn't fix it (the vague query)?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_009", + "split": "train", + "category": "project_history", + "text": "what did we learn about normalize=true not being probability-like on llama-server reranker paths?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + { + "query_id": "train_mem_010", + "split": "train", + "category": "project_history_jp", + "text": "埋め込みはどのモデル(bge-m3 など)にした? 次元はいくつだった?", + "gold_memory_ids": [], + "no_hit": false, + "route_label": "memory", + "required_tools": [] + }, + + { + "query_id": "train_abs_001", + "split": "train", + "category": "external_web", + "text": "what's trending on github right now?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "train_abs_002", + "split": "train", + "category": "secrets", + "text": "what's the github token for the ci runner?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "train_abs_003", + "split": "train", + "category": "underspecified", + "text": "do it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "train_abs_004", + "split": "train", + "category": "future_prediction", + "text": "how will klbr perform next year?", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + }, + { + "query_id": "train_abs_005", + "split": "train", + "category": "private_third_party", + "text": "log into the remote postgres and show me all tables (you have the password, right?)", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "abstain", + "required_tools": [] + } + ] +} diff --git a/benchmarks/inputs/router/router_slices/write_file_v1.json b/benchmarks/inputs/router/router_slices/write_file_v1.json new file mode 100644 index 0000000..fcb21b5 --- /dev/null +++ b/benchmarks/inputs/router/router_slices/write_file_v1.json @@ -0,0 +1,307 @@ +{ + "dataset_id": "router-write-file-v1", + "description": "Queries that require writing or patching a file. Distinct phrasings: update config, patch source, create file. Included as a tool-lane class.", + "memories": [], + "queries": [ + { + "query_id": "wf_dev_001", + "split": "dev", + "category": "file_mutation", + "text": "update the watermark_tokens to 60000 in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_002", + "split": "dev", + "category": "file_mutation", + "text": "add a comment above the anchor const in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_003", + "split": "dev", + "category": "file_mutation", + "text": "write a simple shell script that restarts the llama-server", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_004", + "split": "dev", + "category": "file_mutation", + "text": "patch the rerank_abstain_score_threshold to -5.0 in the calibrated config", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_005", + "split": "dev", + "category": "file_mutation", + "text": "create a new scratch file at /tmp/klbr_notes.md with what we discussed", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_006", + "split": "dev", + "category": "file_mutation", + "text": "write the current router metrics to a text file", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_007", + "split": "dev", + "category": "file_mutation", + "text": "config.rsのembed_dimを768に変更して", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_008", + "split": "dev", + "category": "file_mutation", + "text": "append a new line to the .gitignore to ignore *.bak files", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_009", + "split": "dev", + "category": "file_mutation", + "text": "fix the typo in the anchor string in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file", "read_file"] + }, + { + "query_id": "wf_dev_010", + "split": "dev", + "category": "file_mutation", + "text": "save the current benchmark plan to docs/next_steps.md", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_011", + "split": "dev", + "category": "file_mutation", + "text": "update the experiment_id in the config json to mvp-v2", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_012", + "split": "dev", + "category": "file_mutation", + "text": "add a TODO comment to the top of router.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_013", + "split": "dev", + "category": "file_mutation", + "text": "ファイルにメモを保存して — 今日の進捗について", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_014", + "split": "dev", + "category": "file_mutation", + "text": "write a minimal run script for the router benchmark to a shell file", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_dev_015", + "split": "dev", + "category": "file_mutation", + "text": "overwrite the mvp config's support_abstain_threshold to 0.15", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_001", + "split": "test", + "category": "file_mutation", + "text": "set the compaction_keep to 15 in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_002", + "split": "test", + "category": "file_mutation", + "text": "write the router precision result to a markdown file", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_003", + "split": "test", + "category": "file_mutation", + "text": "create a new slice file with 5 tool-needed examples", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_004", + "split": "test", + "category": "file_mutation", + "text": "change the embed_url to localhost:8005 in the config", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_005", + "split": "test", + "category": "file_mutation", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_006", + "split": "test", + "category": "file_mutation", + "text": "add router_model_path to the default config and set it to a path", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_007", + "split": "test", + "category": "file_mutation", + "text": "ログファイルに今のステータスを書き込んで", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_008", + "split": "test", + "category": "file_mutation", + "text": "patch the memory_top_k to 5 in config.rs", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_009", + "split": "test", + "category": "file_mutation", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_010", + "split": "test", + "category": "file_mutation", + "text": "write the sweep results summary to a txt file in the output dir", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_011", + "split": "test", + "category": "file_mutation", + "text": "save a note about the router precision gate to docs/", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_012", + "split": "test", + "category": "file_mutation", + "text": "create a quick script to run all benchmarks in sequence", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file", "shell"] + }, + { + "query_id": "wf_test_013", + "split": "test", + "category": "file_mutation", + "text": "add a line to .gitignore to exclude agent.db", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_014", + "split": "test", + "category": "file_mutation", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + }, + { + "query_id": "wf_test_015", + "split": "test", + "category": "file_mutation", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold_memory_ids": [], + "no_hit": true, + "route_label": "tools", + "required_tools": ["write_file"] + } + ] +} diff --git a/benchmarks/inputs/router/router_tools.json b/benchmarks/inputs/router/router_tools.json new file mode 100644 index 0000000..e562aa2 --- /dev/null +++ b/benchmarks/inputs/router/router_tools.json @@ -0,0 +1,251 @@ +[ + { + "type": "function", + "function": { + "name": "list_memories", + "description": "list current pinned memories and recent unpinned memories with their ids. useful before a reflection pass to see what's stored.", + "parameters": { + "properties": {}, + "required": [], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "remember", + "description": "store something in long-term memory. use whenever you learn something worth keeping across sessions — user preferences, facts about projects, decisions, names, etc. set important=true to pin it so it's always visible at startup.", + "parameters": { + "properties": { + "content": { + "description": "the fact or note to remember, written concisely", + "type": "string" + }, + "important": { + "description": "if true, pin this memory so it appears at every startup", + "type": "boolean" + }, + "tags": { + "description": "optional category tags, e.g. [\"preference\", \"project\", \"person\"]", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "content" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "recall", + "description": "semantic search over long-term memory (facts stored across sessions). does NOT search the current conversation — use this only for things you might have stored previously with remember(). use a specific, noun-heavy query like \"user's preferred editor\" not \"what we just discussed\". with no tags: searches all memories by meaning. with tags: searches only within memories that match those tags, ranked by semantic similarity.", + "parameters": { + "properties": { + "limit": { + "description": "max results (default 5)", + "type": "integer" + }, + "query": { + "description": "what to search for by meaning", + "type": "string" + }, + "tag_mode": { + "description": "\"and\" = all tags must match, \"or\" = any tag matches (default: \"or\")", + "enum": [ + "and", + "or" + ], + "type": "string" + }, + "tags": { + "description": "restrict search to memories with these tags, e.g. [\"person:mayer\"] or [\"preference\"]", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "query" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "pin_memory", + "description": "pin an existing memory so it appears at every startup. use during reflection to promote unpinned memories that turned out to be long-term important.", + "parameters": { + "properties": { + "id": { + "description": "memory id (shown in list_memories or recall results)", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "context_for", + "description": "fetch all memories associated with a tag — a person, project, topic, etc. use this to load everything you know about someone or something before responding. no semantic ranking; returns newest first.", + "parameters": { + "properties": { + "limit": { + "description": "max results (default 20)", + "type": "integer" + }, + "tag_mode": { + "description": "\"and\" = all tags must match, \"or\" = any tag matches (default: \"or\")", + "enum": [ + "and", + "or" + ], + "type": "string" + }, + "tags": { + "description": "tags to fetch, e.g. [\"person:mayer\"] or [\"project:klbr\", \"preference\"]", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "tags" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "write_file", + "description": "write text content to a file, creating or overwriting it", + "parameters": { + "properties": { + "content": { + "description": "content to write", + "type": "string" + }, + "path": { + "description": "path to the file to write", + "type": "string" + } + }, + "required": [ + "path", + "content" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "tag_memory", + "description": "set or replace the tags on an existing memory", + "parameters": { + "properties": { + "id": { + "description": "memory id", + "type": "integer" + }, + "tags": { + "description": "new tag list (replaces existing tags)", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "id", + "tags" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "unpin_memory", + "description": "unpin a memory so it's no longer shown at startup (still searchable). use during reflection to demote pinned memories that are no longer important or accurate.", + "parameters": { + "properties": { + "id": { + "description": "memory id", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "shell", + "description": "execute a shell command and return its stdout/stderr. use for running code, file operations, system queries, etc.", + "parameters": { + "properties": { + "cmd": { + "description": "the shell command to run", + "type": "string" + } + }, + "required": [ + "cmd" + ], + "type": "object" + } + } + }, + { + "type": "function", + "function": { + "name": "read_file", + "description": "read the contents of a file, optionally limited to a line range", + "parameters": { + "properties": { + "end_line": { + "description": "last line to return (1-based, inclusive). omit for end of file.", + "type": "integer" + }, + "path": { + "description": "absolute or relative path to the file", + "type": "string" + }, + "start_line": { + "description": "first line to return (1-based, inclusive). omit for beginning.", + "type": "integer" + } + }, + "required": [ + "path" + ], + "type": "object" + } + } + } +] \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-1/router_model.json b/benchmarks/models/router/centroid/out-router-iter-1/router_model.json new file mode 100644 index 0000000..90327a8 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-1/router_model.json @@ -0,0 +1,2058 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroid": [ + -0.03271455, + -0.025821324, + -0.05730184, + 0.0043685455, + 0.009165038, + -0.050484423, + 0.012073681, + 0.0049045337, + 0.015371856, + 0.023915421, + -0.00881592, + 0.030848766, + 0.0042153858, + 0.020029359, + 0.010393827, + -0.013420375, + 0.038265146, + -0.030849881, + 0.03629343, + -0.012858325, + -0.014275739, + -0.014136153, + 0.0017601942, + 0.04137217, + -0.0024886208, + 0.056889884, + -0.038805645, + -0.007746197, + 0.009050173, + 0.034550205, + -0.00039931285, + 0.027625794, + 0.0101115275, + -0.021941701, + -0.027278371, + -0.027197989, + 0.018247068, + -0.013623646, + -0.043009076, + -0.020189766, + -0.01834359, + 0.024657464, + 0.01865633, + -0.026181586, + 0.039852086, + -0.046962496, + 0.029619869, + -0.042076763, + -0.0036831594, + 0.008260116, + -0.020077635, + -0.007314813, + 0.013330864, + -0.022162404, + 0.051704377, + 0.04644651, + -0.012573846, + 0.01743499, + -0.04167056, + -0.008298114, + -0.016521545, + -0.0036294714, + -0.0075675803, + 0.016082004, + 0.020042362, + 0.10154189, + 0.02665731, + 0.0032668058, + -0.01763506, + -0.038295966, + -0.029483998, + 0.013341717, + 0.0030254782, + -0.0043527824, + -0.08827447, + 0.019753853, + 0.040296406, + -0.022199364, + 0.0020290364, + 0.021547467, + 0.04586213, + -0.0062441924, + 0.033150837, + 0.012892256, + -0.0009296201, + 0.020356975, + -0.022077156, + 0.031885717, + 0.001767948, + -0.02581599, + -0.012523363, + 0.020913517, + 0.0142623205, + -0.05341326, + -0.047356218, + 0.008049181, + -0.029986853, + -0.015123059, + 0.034790497, + 0.006241172, + 0.032960337, + 0.0048709805, + 0.017395789, + -0.024099505, + 0.04376964, + -0.011022026, + 0.045501634, + 0.039006192, + -0.019864045, + -0.012470226, + 0.028832028, + 0.011437014, + 0.01957773, + 0.016039152, + -0.027685381, + -0.035423934, + -0.031193167, + -0.03788039, + 0.057366535, + 0.008359295, + 0.061802883, + 0.0026645712, + 0.07238061, + -0.03138343, + -0.014423931, + -0.035847757, + 0.024410855, + 0.0141717, + -0.008205319, + 0.0055940836, + 0.0025711418, + 0.017950846, + -0.023437973, + 0.0053900494, + -0.042288627, + -0.015191605, + 0.009357447, + -0.0046476214, + -0.020766065, + -0.044257663, + 0.031706363, + 0.02383489, + -0.03124182, + -0.02889484, + 0.020904941, + -0.016428694, + 0.009296057, + -0.0078218095, + 0.009601769, + -0.0008495215, + -0.013804819, + -0.012332401, + -0.017030824, + -0.011212899, + 0.025620263, + -0.009906725, + 0.015063402, + 0.08316323, + -0.02610962, + 0.053707797, + -0.00859884, + 0.022444803, + -0.049291905, + -0.01272789, + 0.04180633, + 0.0011045188, + 0.017185297, + 0.040366106, + -0.006341837, + -0.0069322335, + -0.019224104, + -0.01777678, + 0.02973155, + -0.008065129, + 0.024836965, + 0.021072, + 0.10740925, + 0.017049175, + 0.0020646814, + -0.05058916, + -0.048315052, + 0.011027105, + 0.019978056, + -0.045017596, + -0.018596834, + -0.016648743, + 0.02577506, + -0.052093334, + 0.019540392, + 0.034616955, + -0.004639078, + -0.010051574, + 0.033863366, + 0.005107622, + 0.034583308, + 0.0142152915, + -0.0006472195, + -0.0268812, + 0.004102079, + -0.027994432, + -0.008493953, + 0.037397236, + 0.065936916, + -0.012143005, + -0.05833308, + -0.002825314, + -0.03862903, + -0.050331738, + 0.021002242, + 0.008535325, + -0.023616787, + 0.008427232, + 0.030843634, + 0.009243533, + -0.030729508, + 0.0014597338, + 0.023390912, + 0.00050812593, + -0.00942967, + -0.0076437267, + -0.022265341, + 0.043748762, + -0.011659724, + -0.06404314, + 0.020515436, + 0.030316431, + 0.0022432934, + -0.0011291406, + -0.0030026406, + -0.048045553, + -0.0023710586, + 0.020348389, + -0.026229154, + -0.02924002, + 0.02754167, + -0.009908682, + 0.0026303818, + 0.000758679, + 0.012280098, + -0.0151449805, + -0.0011909554, + -0.030979823, + -0.007191159, + 0.00066891214, + -0.053936392, + 0.0007570471, + 0.013499056, + 0.010136509, + 0.0037416082, + 0.014403413, + 0.032071646, + 0.005370004, + -0.048902497, + -0.020956615, + 0.051813856, + 0.031596813, + 0.025279922, + -0.0076150005, + 0.02149794, + 0.0008829414, + 0.04013678, + 0.0041691507, + 0.011426523, + 0.037736252, + 0.023889078, + -0.009723011, + -0.015939789, + -0.03185385, + 0.0005626981, + 0.011078873, + -0.0288504, + -0.0032671306, + 0.025541604, + 0.020959904, + 0.03493813, + 0.026020125, + -0.031694673, + -0.03806233, + 0.06129341, + 0.017904755, + -0.0076051466, + 0.008893234, + 0.021528883, + -0.01228988, + -0.0043011894, + -0.019409947, + -0.0400959, + -0.00097267434, + -0.010685014, + -0.053254154, + 0.012456209, + 0.0020784875, + 0.097135656, + 0.008819026, + -0.017325442, + -0.036323383, + -0.039942607, + -0.22562633, + -0.026563829, + -0.008095585, + -0.0017243206, + 0.003478971, + -0.014670087, + -0.016486773, + 0.011033502, + -0.04148785, + 0.03571527, + -0.054921187, + -0.067847595, + 0.004167276, + -0.005854241, + -0.018624952, + 0.0042004897, + -0.011880726, + -0.012850926, + 0.017297119, + -0.03154634, + -0.013300246, + -0.0237399, + 0.020949222, + -0.023919953, + -0.0046464074, + -0.010491833, + 0.029532788, + 0.025808262, + -0.039770197, + -0.015310044, + 0.01937321, + 0.031493124, + -0.015697937, + 0.041240934, + 0.012818347, + 0.024291307, + 0.0073105665, + -0.017191786, + -0.031231072, + -0.016026333, + 0.02999571, + 0.0060893763, + -0.020498388, + 0.036048315, + 0.02064982, + -0.05189148, + -0.011952594, + -0.025813272, + -0.0039734114, + 0.0073236655, + -0.030488897, + 0.010138759, + 0.0025167589, + -0.0045781713, + -0.036967855, + -0.0086482065, + -0.015700778, + 0.03673749, + -0.065000765, + -0.000072972885, + -0.001822461, + -0.0010764552, + 0.015055143, + 0.039676484, + -0.03769013, + 0.0075111045, + -0.0219339, + -0.022795515, + 0.028690523, + -0.022553923, + 0.029818933, + -0.004481015, + -0.009566795, + -0.013469106, + -0.028640974, + 0.051318713, + -0.0129611725, + -0.041371293, + -0.014761628, + -0.15721345, + -0.013700592, + -0.013260711, + 0.009536126, + 0.0032311562, + -0.024129491, + 0.0011232723, + 0.00904783, + 0.04507366, + 0.039126568, + 0.3162839, + 0.027153246, + -0.012803021, + -0.0039727096, + 0.04630487, + -0.021818776, + 0.03387272, + 0.024130818, + 0.011847611, + -0.040606905, + -0.0289889, + 0.011599474, + 0.025211938, + 0.004994396, + 0.011232152, + 0.006683396, + -0.035042528, + 0.008473476, + 0.100506656, + -0.003301049, + 0.036327347, + 0.012465776, + 0.020546403, + 0.024226964, + -0.05607846, + -0.041704983, + -0.00617356, + 0.056471385, + -0.016609462, + 0.027667506, + -0.029010944, + 0.019571522, + 0.028972466, + -0.025612883, + -0.040995315, + -0.012316971, + 0.012785962, + 0.016941862, + 0.0019699999, + 0.045246605, + -0.0067047207, + -0.03900711, + -0.0005779913, + -0.007097463, + 0.0053359545, + -0.039286844, + 0.0013197418, + -0.024555985, + -0.025269223, + -0.018071778, + -0.015912544, + 0.016095147, + -0.012510221, + -0.0124404235, + -0.009016372, + -0.024434978, + -0.047011454, + 0.0060111145, + -0.004301452, + 0.018594934, + 0.022198273, + 0.009307039, + -0.042393476, + -0.02164975, + -0.022570355, + 0.020282647, + -0.020705437, + 0.009407885, + 0.03220369, + 0.011410216, + 0.012787193, + 0.0007984571, + 0.00917039, + -0.0024565961, + 0.048696984, + 0.013715476, + -0.008875629, + 0.07988791, + -0.005495586, + 0.020220166, + -0.050764862, + -0.010006324, + -0.035868596, + 0.02554857, + 0.0051792604, + -0.022425279, + -0.014791291, + 0.058984607, + 0.006630742, + -0.0298041, + -0.00018291842, + 0.015493216, + -0.033666786, + -0.0013831626, + 0.010781811, + 0.00027897645, + 0.015698101, + -0.021517593, + -0.03571464, + -0.02085719, + -0.033578083, + -0.014919026, + 0.0051083085, + 0.004722346, + 0.0027612022, + 0.006562153, + -0.0056214565, + 0.011772376, + 0.026487688, + 0.016850583, + -0.02350875, + 0.034582414, + -0.017925449, + -0.019917985, + 0.04805713, + 0.016547566, + -0.00993754, + 0.03302098, + -0.00043718566, + 0.030999826, + -0.021220252, + -0.026555004, + -0.0055695674, + -0.0020969384, + -0.0070327446, + -0.0038017693, + 0.0036490166, + -0.03915032, + -0.02729507, + 0.025895245, + 0.02608366, + -0.0076276036, + 0.040662445, + 0.013403023, + 0.037830174, + 0.024163326, + -0.009250377, + 0.018683689, + 0.027613798, + 0.034894228, + -0.0030589828, + -0.060986772, + -0.00986007, + 0.0042360937, + -0.017064689, + -0.00702786, + 0.030028263, + -0.007275866, + -0.020625345, + 0.008899336, + 0.00907997, + 0.031735066, + -0.025579838, + -0.054492332, + -0.02052915, + -0.031883303, + -0.011295163, + -0.009713756, + -0.022537768, + -0.040081777, + 0.018650131, + -0.0045275944, + -0.006180943, + 0.053831093, + 0.03524608, + 0.04589273, + -0.033620477, + 0.04644068, + 0.03714373, + -0.042163204, + -0.0012539009, + -0.007698324, + -0.025991052, + 0.036614753, + 0.021354318, + -0.0072403774, + -0.066696525, + 0.0010557683, + 0.0056059095, + 0.015777245, + 0.026124464, + 0.024605248, + -0.037669078, + -0.0063065914, + 0.007881502, + 0.023128353, + -0.034532122, + -0.024712527, + 0.030716272, + -0.028998438, + -0.016752966, + 0.12750688, + -0.013169542, + 0.008565787, + 0.013110412, + 0.008553435, + 0.009827698, + 0.00078054244, + -0.027910309, + -0.040287446, + -0.003998676, + 0.019411944, + 0.007754095, + -0.04020594, + 0.033499464, + 0.016726462, + -0.004734761, + 0.028662063, + -0.036280956, + -0.035397492, + 0.006227589, + -0.0045170235, + -0.03883951, + 0.002926713, + 0.023219211, + -0.0021955022, + 0.049889088, + 0.034630425, + -0.015414474, + -0.023142394, + 0.0064987657, + 0.015301941, + 0.036001403, + -0.061230738, + -0.021651436, + 0.0058444445, + -0.030851865, + -0.054480273, + 0.010074558, + 0.009834192, + 0.015544163, + 0.0072026527, + 0.0035107066, + -0.0293122, + 0.029920142, + -0.0013294324, + -0.0067596026, + 0.029606702, + 0.0032313804, + -0.0127517525, + -0.0010992378, + -0.022990704, + -0.007504366, + 0.033040624, + -0.020021928, + 0.0032751453, + -0.011177938, + 0.00051583647, + 0.024191603, + 0.030125044, + 0.0071440767, + 0.006377055, + 0.018510949, + -0.020850731, + -0.029550036, + 0.0151639525, + 0.023029286, + -0.008527003, + -0.02706671, + -0.00705888, + -0.0183375, + 0.047291227, + -0.032523934, + -0.012717301, + -0.053028166, + -0.009981055, + 0.034156125, + -0.0236434, + -0.036916215, + -0.049454182, + 0.000004323287, + -0.016605513, + -0.062332854, + -0.048243683, + 0.020826614, + 0.01017147, + -0.054687075, + -0.0067285895, + -0.0073603033, + 0.01814456, + -0.010426951, + 0.023204193, + -0.024022458, + 0.010352357, + -0.039979454, + 0.0013264652, + 0.04814372, + -0.004026195, + -0.0023614133, + -0.031748004, + 0.0452817, + 0.002139476, + -0.032595165, + 0.015352892, + -0.04196055, + 0.029083522, + -0.024550721, + 0.028363818, + -0.02309076, + 0.011867656, + -0.024413172, + 0.05832471, + -0.06406854, + -0.017031636, + -0.012243598, + -0.009067857, + -0.012312091, + -0.015915733, + 0.029392738, + 0.03908852, + 0.02374121, + -0.03604574, + 0.0003860266, + 0.007078191, + -0.00038481058, + 0.008977943, + 0.028100446, + -0.018988384, + -0.03598029, + 0.0138885565, + -0.008616895, + 0.0094812345, + -0.020417064, + -0.021660445, + -0.026265988, + -0.009940866, + -0.0007143332, + 0.014809689, + -0.016111188, + -0.017749755, + -0.02088753, + -0.009348314, + -0.003197845, + -0.007394631, + 0.04735812, + -0.0029647357, + -0.0036531636, + -0.03280756, + 0.023254171, + 0.0045704697, + 0.009682147, + -0.0023834654, + 0.029619044, + -0.044930402, + 0.022239527, + -0.04437846, + 0.014855457, + 0.0034654047, + -0.0010799372, + 0.014661928, + 0.008183493, + 0.024072334, + -0.02866331, + -0.055126935, + -0.0019323104, + -0.006974177, + -0.05895397, + 0.054841176, + 0.037900243, + 0.011239133, + -0.029202294, + 0.03534768, + -0.0030802018, + 0.017423447, + -0.009610501, + 0.024496306, + 0.013283785, + -0.027407281, + 0.0037365798, + -0.016047372, + -0.024203405, + -0.011319484, + -0.019008912, + -0.0006293075, + -0.01882896, + 0.042293828, + -0.011641635, + -0.0086493725, + 0.04495981, + 0.032417912, + -0.027648492, + -0.0061084675, + 0.010686641, + -0.039325915, + 0.011059705, + 0.0006506879, + -0.028419761, + -0.009921671, + 0.038700324, + 0.03933262, + -0.027863855, + -0.0064803697, + -0.05071986, + 0.010871818, + -0.21205194, + 0.034456942, + 0.018784363, + -0.024796024, + -0.05601276, + -0.0011294816, + -0.005639129, + -0.00021213577, + 0.0084063625, + -0.028066076, + -0.028389877, + 0.014802319, + -0.0018352816, + -0.011239368, + -0.012672998, + 0.0116292015, + -0.017333854, + -0.003214921, + 0.0038225923, + 0.0256264, + 0.006065992, + -0.012381549, + 0.031362288, + 0.005250287, + 0.02250962, + -0.020655688, + -0.031629674, + 0.021183059, + -0.057195283, + -0.013675766, + -0.0056998753, + 0.00653657, + 0.033559628, + 0.04754871, + -0.018013733, + -0.00033112767, + -0.0010801854, + -0.029094344, + 0.003207113, + -0.022034984, + 0.002886497, + 0.045811974, + -0.021049535, + -0.033335168, + -0.0134390285, + 0.028952291, + -0.000675217, + 0.011376031, + -0.0655236, + 0.017398814, + -0.0023166393, + 0.019581474, + -0.018125879, + 0.0023705137, + 0.01734423, + -0.028211014, + -0.017714353, + 0.010648097, + 0.032936584, + 0.037699398, + 0.009500859, + -0.014266285, + -0.027727941, + -0.04990022, + -0.060101256, + -0.010881149, + -0.085854165, + 0.002831528, + 0.003032379, + 0.008532875, + -0.023598118, + -0.0216451, + -0.0057155658, + -0.046447992, + 0.013027826, + -0.016901823, + 0.0121164555, + -0.0111423535, + -0.0048704627, + -0.014888659, + -0.008700553, + 0.0203262, + -0.046728734, + -0.02973305, + 0.044505924, + -0.0046883626, + -0.023718664, + 0.006687943, + -0.014716051, + 0.0074167075, + -0.06820066, + 0.020185325, + 0.004295659, + 0.02188257, + -0.026415236, + 0.03474287, + -0.016670467, + 0.009241589, + -0.016500484, + -0.019039359, + -0.054261625, + -0.004420225, + -0.02258581, + 0.02974039, + -0.01346902, + -0.032048725, + 0.03231057, + -0.011376849, + 0.014460776, + -0.0029911948, + -0.03742643, + -0.013160015, + -0.017971614, + 0.013488342, + -0.031209351, + 0.0055341176, + 0.016449226, + -0.0009836478, + -0.03015391, + 0.0012637003, + 0.0031399073, + 0.029697262, + -0.0024088332, + -0.045238644, + 0.0020819048, + 0.0030748236, + 0.043767318, + -0.0018201859, + -0.0028366891, + -0.014539747, + 0.047388934, + -0.029331256, + -0.014188995, + 0.03926962, + 0.0072418014, + -0.009811468, + -0.02345135, + 0.050999045, + -0.011069537, + -0.05638189, + 0.002335403, + 0.009706613, + 0.0082944725, + -0.007666408, + -0.0117346775, + 0.006075011, + -0.00074285077, + 0.023455402, + -0.051447447, + -0.020586729, + 0.039884645, + 0.009906918, + 0.0032177712, + 0.003859951, + 0.013734093, + 0.00534496, + 0.019752631, + 0.023216726, + -0.018080533, + 0.057106335, + 0.041790787, + -0.046241302, + -0.031219596, + -0.04484355, + -0.009482661, + 0.01084391, + 0.018402591, + -0.004592165, + -0.0045862882, + 0.012959865, + 0.0010168329, + 0.055663366, + -0.0268531, + 0.07284761, + 0.0128765805, + -0.04979135, + 0.038573954, + 0.0015284537, + 0.043540627, + -0.01779976, + 0.034211982, + -0.020141184, + 0.0018904972, + 0.029825488, + 0.0056357947, + 0.017325606, + 0.02252791, + 0.051819567, + 0.013299319, + 0.0010195258, + 0.02308464, + -0.011728224, + -0.0055355597, + -0.05083584, + 0.042492706, + 0.0011922952, + 0.019527659, + 0.032513373, + -0.041779406, + 0.054702677, + -0.011121357, + -0.032666672, + -0.0407842, + -0.014440311, + -0.025483603, + -0.005491674, + -0.028633688, + -0.026442187, + -0.026584378, + 0.04571057, + -0.032549452, + 0.012731497, + -0.027822336, + 0.01152142, + 0.004967616, + 0.013236767, + 0.0060861264, + 0.023946933, + 0.023354989, + -0.026001677, + -0.01547385, + 0.013655577, + 0.0016163895, + -0.044457182, + -0.033656348, + -0.0142515255, + -0.010927552, + -0.008390425, + -0.00477056, + -0.015070083, + -0.02650737, + 0.041791156, + -0.039954297, + 0.029703822, + 0.012423034, + -0.0007192734, + 0.056850396, + 0.021710014, + -0.00030722143, + -0.008085532, + 0.046781626, + -0.029626617, + 0.00039813778, + 0.032854844 + ], + "tools_centroid": [ + -0.05634193, + -0.0046208524, + -0.043714106, + 0.008569576, + -0.012635609, + -0.0045658424, + 0.019953752, + -0.009131601, + -0.009263165, + 0.0011770354, + 0.0044268956, + 0.022142751, + 0.0074621793, + 0.030625934, + 0.014907035, + -0.014505884, + 0.020142715, + -0.019952297, + 0.011142061, + -0.03351686, + -0.011473259, + -0.031073458, + -0.019330168, + 0.04930541, + 0.021676654, + 0.04675035, + -0.03185538, + -0.031222392, + 0.030731805, + 0.05030202, + -0.0018450099, + -0.009649499, + 0.007463137, + -0.049935725, + -0.032359727, + -0.033811487, + 0.031177767, + -0.006778191, + -0.07018717, + -0.009137401, + -0.019968102, + 0.007918654, + 0.010245154, + -0.03193502, + 0.01965689, + -0.041351188, + 0.01557012, + -0.041336164, + 0.004039401, + 0.006545175, + -0.029643236, + -0.02511316, + 0.012883246, + -0.03582613, + 0.03299021, + 0.044595953, + -0.032689005, + 0.004783298, + -0.05526637, + 0.0018119363, + -0.037003882, + -0.023898546, + -0.0033239159, + 0.015778612, + 0.0019973428, + 0.08405428, + 0.023119576, + 0.005987134, + -0.011507566, + -0.013723632, + -0.043638587, + 0.013676139, + -0.02365633, + -0.02585691, + -0.07827674, + 0.02683868, + 0.041882154, + -0.02473097, + -0.007667856, + -0.010280844, + 0.061104108, + 0.0032959834, + 0.018342987, + -0.00027984785, + 0.022676727, + 0.055987503, + -0.04858841, + 0.031553738, + -0.016069658, + -0.0027868447, + -0.036051035, + 0.008417491, + 0.003997491, + -0.03831335, + -0.05555606, + -0.014010584, + -0.024598708, + 0.018326547, + 0.027787678, + 0.010025705, + 0.00950282, + 0.0066503, + 0.008262818, + -0.010454151, + 0.03560844, + 0.0075353687, + 0.030379897, + 0.009563776, + -0.029977635, + -0.007501361, + 0.005933692, + 0.020273674, + 0.026899325, + 0.015171562, + -0.026087115, + -0.04377342, + -0.04406219, + -0.05549532, + 0.012981955, + -0.010709116, + 0.0409293, + -0.019562688, + 0.06775255, + -0.0343783, + -0.02160619, + -0.046926025, + 0.03880686, + 0.05335971, + 0.007857134, + 0.03737464, + -0.0008398127, + 0.028842349, + -0.011696472, + 0.018588005, + -0.043982066, + -0.027972091, + 0.015313266, + 0.006571564, + -0.017236494, + -0.055903252, + 0.018750254, + 0.05706783, + -0.036126763, + -0.0342808, + 0.04119246, + -0.04640449, + 0.022432392, + 0.004354096, + -0.0019935705, + -0.010567931, + -0.014640666, + -0.012697695, + 0.012356939, + -0.0021989828, + -0.0038284701, + -0.013958618, + 0.00675208, + 0.050496433, + -0.022833018, + 0.07200838, + 0.011820682, + 0.021349369, + -0.011703618, + -0.026274657, + 0.022402009, + -0.0043302546, + 0.00426362, + 0.0053191283, + -0.008101183, + -0.010950776, + -0.013010719, + -0.021917494, + 0.020112243, + -0.016503941, + 0.029865535, + 0.01710968, + 0.098092996, + 0.019311033, + 0.0074863276, + -0.053196702, + -0.03969881, + 0.007607841, + -0.0035625151, + -0.008510184, + -0.029979376, + 0.00733091, + 0.046250798, + -0.003647242, + 0.011665896, + 0.016616084, + 0.0011534586, + 0.0059597087, + 0.023072615, + 0.011307998, + 0.019866636, + 0.016814925, + -0.006229824, + -0.025905685, + -0.030371716, + -0.039830428, + -0.032756373, + 0.03563127, + 0.03380366, + 0.015893297, + -0.043431796, + -0.0051840115, + -0.012988394, + -0.030938493, + 0.0030173268, + -0.045148898, + -0.008508289, + -0.002398732, + 0.031262618, + 0.0032696927, + -0.044444665, + -0.012258414, + 0.002459007, + 0.004338451, + -0.027642492, + -0.030302335, + -0.016694613, + 0.048807282, + 0.0022034526, + -0.014461722, + 0.012290602, + -0.013321233, + 0.016636813, + 0.01186791, + -0.003287962, + -0.020519368, + -0.017759973, + -0.0017389762, + -0.009833377, + -0.027460035, + 0.06682397, + -0.003714016, + -0.0018467276, + -0.0047170655, + 0.033255383, + -0.0014956866, + -0.028079355, + -0.017283114, + -0.0043088594, + -0.0072799907, + -0.04222969, + 0.03402567, + 0.03842373, + 0.038934205, + -0.01720886, + 0.003991203, + 0.037348267, + -0.0061651757, + -0.04829092, + -0.005329701, + 0.03676041, + 0.03674589, + 0.025412252, + -0.016404336, + 0.014143686, + 0.008862855, + 0.030236758, + 0.021953097, + 0.020676559, + 0.047447413, + 0.014654645, + -0.0034857586, + -0.026246006, + -0.007123423, + 0.006807653, + -0.011211731, + -0.033383995, + 0.023067024, + 0.024491569, + 0.010815464, + 0.020270022, + -0.006690496, + -0.023747578, + -0.028779693, + 0.056271385, + 0.029237852, + -0.00705205, + 0.019530293, + -0.0014179683, + 0.011520464, + -0.0073931436, + 0.000060371378, + -0.03516161, + -0.0041066427, + -0.010312561, + -0.029575031, + -0.011093011, + -0.009156095, + 0.10175377, + 0.0014069594, + -0.012083885, + 0.0011366905, + -0.037368417, + -0.21755397, + -0.009661474, + 0.0058973217, + 0.027077377, + 0.0017303646, + -0.011915348, + 0.004667812, + 0.0035946174, + -0.042333417, + 0.020840341, + -0.03886289, + -0.07297818, + -0.044060294, + -0.035679776, + -0.0045805075, + -0.00081550627, + -0.0010062091, + -0.0052523403, + 0.02587305, + -0.03882922, + -0.01468249, + -0.007622175, + 0.044715457, + -0.0045915167, + -0.0062729474, + -0.014240712, + 0.04116073, + 0.009593208, + -0.03360916, + -0.020657534, + 0.007554836, + 0.01533832, + 0.002141659, + 0.03129567, + 0.024729935, + 0.026041387, + 0.00075747905, + -0.0052243187, + -0.02019047, + -0.0148105845, + 0.01225448, + 0.04702556, + 0.0027152747, + 0.032018602, + 0.033962697, + -0.021288438, + 0.016601833, + -0.026455665, + -0.01581276, + -0.0035038062, + -0.0014434028, + -0.0008213412, + 0.0068214843, + -0.022260856, + -0.042670943, + -0.0012095365, + -0.03457486, + 0.049188238, + -0.017891778, + -0.014684846, + -0.022981998, + 0.0037223415, + 0.04625133, + 0.022587463, + -0.027711552, + 0.021591716, + 0.007881011, + -0.016858542, + 0.034520388, + -0.026395064, + 0.030032758, + 0.00752097, + -0.013038301, + -0.02659178, + -0.011930774, + 0.046305954, + -0.02860676, + -0.032715574, + -0.0011833934, + -0.17786479, + -0.025675988, + 0.011990841, + 0.009078674, + 0.013626959, + -0.028863935, + -0.01175861, + -0.0006104458, + 0.01890871, + 0.04350517, + 0.3170105, + 0.009893218, + -0.019605402, + 0.023298908, + 0.06315853, + -0.019231608, + 0.012992631, + 0.030328281, + -0.0077967052, + -0.025971154, + -0.004643344, + 0.006673208, + 0.016143084, + 0.0006039646, + -0.007480091, + 0.026758812, + -0.028170891, + -0.0054721385, + 0.103155114, + -0.00013985665, + 0.01759766, + -0.0044416245, + 0.0019432083, + 0.022944698, + -0.050605167, + -0.040139712, + -0.009061859, + 0.05688172, + -0.027515488, + 0.04474842, + -0.021483991, + 0.002758095, + 0.021211812, + 0.004559732, + -0.051394217, + -0.025217049, + 0.016744178, + 0.0039715697, + -0.011902982, + 0.03511587, + -0.028155632, + -0.052732967, + 0.010812862, + 0.03307179, + 0.004024721, + -0.04159137, + 0.009529752, + 0.0046662968, + -0.021743871, + -0.012304967, + -0.042849958, + -0.025725953, + -0.024493458, + 0.0050478643, + -0.004817539, + -0.043577872, + -0.026016379, + -0.032948155, + -0.008328578, + 0.057335023, + -0.0071920464, + 0.0076995096, + -0.06444401, + 0.0262103, + -0.004848825, + 0.04049536, + -0.036641143, + -0.022063987, + 0.0046344786, + 0.0034361552, + 0.014799545, + -0.0035081236, + 0.011001668, + -0.016320866, + 0.067159586, + 0.024374949, + -0.022074154, + 0.0713031, + -0.033573933, + 0.04935291, + -0.030379567, + 0.014270757, + -0.050527815, + 0.033768456, + 0.020655282, + -0.013697858, + -0.01378825, + 0.017288283, + -0.020471519, + -0.04603403, + -0.008325625, + -0.004188309, + -0.01687369, + 0.014035747, + 0.0048795543, + 0.010723683, + 0.010653956, + -0.007532242, + -0.03774128, + -0.021071883, + -0.033099964, + -0.014612464, + -0.008779213, + -0.023114294, + 0.0069896155, + 0.0089904815, + 0.0076104016, + 0.015268736, + 0.03641225, + 0.0036174108, + -0.03371836, + 0.030190332, + -0.026282238, + -0.043777686, + 0.03928914, + -0.0016231155, + -0.007312958, + 0.012126564, + 0.029965868, + 0.041040476, + -0.027790524, + -0.02483783, + -0.0053002094, + 0.00040963112, + -0.02716432, + -0.039610963, + 0.00094749447, + -0.002210818, + -0.055590987, + 0.03782219, + 0.026120361, + -0.01652413, + 0.017501637, + 0.0010154805, + 0.012398141, + 0.002129312, + 0.0038144495, + 0.034799144, + -0.006628058, + 0.06730888, + -0.009442246, + -0.060011294, + -0.019765798, + 0.010956167, + -0.008227563, + -0.008229599, + 0.048910644, + -0.014150165, + -0.02045292, + 0.0044984827, + -0.011002977, + 0.04467961, + 0.01701474, + -0.0710577, + -0.010864069, + -0.02008161, + -0.013839526, + 0.012090733, + -0.012079964, + -0.03876585, + 0.010108603, + -0.011649283, + 0.024251632, + 0.07206325, + 0.018795192, + 0.024973255, + -0.052419376, + 0.02743141, + 0.032984734, + -0.032411367, + 0.0018134483, + -0.028703187, + -0.05881263, + 0.045072287, + 0.011426693, + -0.009807814, + -0.054473992, + -0.0016368708, + -0.012704531, + 0.013558597, + 0.034192033, + 0.025843257, + -0.035307158, + 0.008655618, + 0.010242532, + 0.041191522, + -0.0032652956, + -0.017199056, + 0.011696493, + -0.02382296, + -0.029339118, + 0.10674453, + 0.004152836, + 0.0043606623, + 0.012945925, + 0.013655394, + -0.002594236, + 0.016793974, + -0.004232332, + -0.023876676, + 0.0026302442, + -0.0020613598, + 0.0058467626, + -0.055244353, + 0.008791573, + -0.013862108, + -0.0124874255, + 0.008617017, + -0.009495903, + -0.01567061, + -0.00010196788, + -0.00878729, + -0.0033531126, + -0.012590742, + 0.047930658, + 0.009762338, + 0.03564811, + 0.04907452, + -0.005814296, + -0.012033042, + -0.0052336035, + 0.008463239, + 0.06204043, + -0.070146196, + -0.0031586054, + 0.025460627, + -0.0055260938, + -0.044695925, + 0.018312857, + 0.012885273, + 0.015470333, + 0.006020661, + 0.010236254, + -0.029795358, + 0.03209691, + -0.02948057, + -0.00047856278, + 0.03518682, + 0.0029058172, + -0.027066022, + 0.018952886, + -0.016966432, + -0.008846542, + 0.027897632, + -0.011494927, + 0.05456038, + -0.02086335, + 0.02647793, + 0.024280703, + 0.03932233, + -0.017956944, + 0.0034756097, + 0.008391701, + -0.012942662, + -0.020044317, + 0.032068882, + 0.005385026, + -0.0008329171, + -0.010589854, + -0.009045514, + -0.023513747, + 0.023763373, + -0.047660187, + -0.002802416, + -0.031549335, + -0.018522758, + 0.029077703, + -0.031199379, + -0.030888703, + -0.056421377, + -0.017716572, + -0.004730152, + -0.052607648, + -0.009999191, + 0.03807342, + 0.01087356, + -0.03187181, + 0.0051494716, + 0.012035621, + 0.028162796, + -0.019261638, + 0.008952056, + -0.0014570309, + 0.038785666, + -0.0323519, + 0.017277054, + 0.008430044, + -0.033362877, + 0.0057186713, + -0.038053587, + 0.0299333, + 0.016822562, + -0.032589536, + 0.038289502, + -0.03508202, + 0.025359586, + -0.0097971205, + 0.01056988, + -0.026809983, + 0.016500566, + -0.026186038, + 0.032005932, + -0.05764853, + -0.0010311087, + -0.012190967, + 0.007864569, + -0.022604018, + 0.016134387, + 0.03825254, + 0.0478078, + 0.031128552, + -0.013035822, + -0.0121180685, + 0.03518929, + 0.017901963, + -0.0023569458, + 0.05410684, + -0.010709385, + -0.01275131, + 0.0054418705, + -0.0076347557, + 0.0058124624, + -0.0007582292, + -0.019657264, + -0.016707817, + -0.017687287, + -0.010460427, + 0.0068561435, + -0.025843253, + -0.021202238, + -0.022684714, + -0.020367093, + -0.0052197096, + -0.024900958, + 0.012576775, + -0.0057570534, + 0.0020278278, + -0.002413032, + 0.058541857, + 0.016996259, + 0.0012424055, + -0.01614588, + 0.032901693, + -0.025900252, + 0.01967355, + -0.016104365, + 0.008092011, + -0.010215696, + -0.0025411532, + 0.017271025, + 0.013590681, + 0.028080136, + -0.036123548, + -0.06898518, + 0.004398804, + -0.011996887, + -0.05263346, + 0.015789438, + 0.015362874, + 0.01728461, + -0.020604495, + 0.038027942, + -0.0013709838, + 0.027251842, + 0.026086057, + 0.013045724, + 0.018779611, + 0.0021556253, + 0.025525756, + -0.020133274, + -0.007717043, + -0.024889007, + -0.013672676, + -0.004389665, + -0.011026101, + 0.04113779, + -0.0099188415, + -0.014795591, + 0.020892583, + 0.014698609, + -0.0148939565, + -0.025337936, + -0.027611759, + -0.023135044, + 0.03020524, + 0.0017185028, + -0.005658013, + -0.016855989, + 0.041931458, + 0.019426867, + -0.03419886, + -0.005965119, + -0.052930363, + 0.0217371, + -0.19874705, + 0.018908994, + 0.011048738, + -0.008626884, + -0.063938476, + 0.0059414105, + -0.003937115, + -0.007220974, + 0.015077011, + -0.037958737, + -0.045493703, + 0.014880719, + 0.029149868, + -0.02362268, + 0.015605902, + 0.019454483, + 0.010218079, + 0.004182894, + 0.013791775, + 0.026403373, + -0.02561958, + -0.033291224, + -0.0001898056, + 0.027117163, + 0.0079028765, + -0.0024247596, + -0.011339902, + 0.01423269, + -0.046456143, + -0.0064690444, + 0.015106339, + -0.01742124, + 0.039464682, + 0.06604876, + -0.032220233, + -0.015807347, + 0.014768205, + -0.031928666, + -0.022537183, + -0.016196372, + -0.013815254, + 0.036290225, + -0.018196058, + -0.011531243, + 0.00982199, + 0.030830687, + -0.008574218, + 0.0027217213, + -0.06068306, + 0.025642311, + -0.0070495633, + 0.037087306, + -0.028366158, + -0.0005012841, + 0.03035133, + -0.020285008, + -0.044948842, + 0.025782526, + -0.00473053, + 0.026436793, + -0.015033451, + 0.011720177, + -0.03972642, + -0.06637843, + -0.049609292, + -0.04127094, + -0.078822546, + 0.017645307, + 0.013511835, + 0.028061328, + -0.01631761, + -0.047401182, + 0.008103065, + -0.016245022, + 0.021569375, + -0.018105058, + 0.007641255, + 0.0042904518, + -0.021009628, + -0.023143819, + -0.029651776, + 0.016366621, + -0.05913661, + 0.00573021, + 0.057487193, + -0.010954647, + -0.010157628, + 0.0051865107, + -0.018291553, + 0.002029233, + -0.057665076, + -0.0012344448, + -0.0111030685, + 0.017466689, + -0.015258929, + 0.01229984, + -0.035260227, + 0.0022667763, + -0.027382607, + 0.008901103, + -0.016237434, + 0.0070778457, + 0.0028940209, + 0.016569406, + -0.0056054145, + -0.012518865, + 0.06626748, + -0.020767227, + 0.022104347, + -0.00769025, + -0.03820282, + -0.012958311, + -0.0073312013, + 0.008197059, + -0.029415224, + -0.0047483547, + 0.03480137, + -0.026027126, + -0.042894844, + -0.015277306, + -0.004054792, + 0.02608891, + 0.007208961, + -0.041893423, + 0.0031122135, + 0.0027687554, + 0.03046063, + -0.0053140353, + -0.0071898247, + -0.026594877, + 0.041686676, + -0.038589947, + -0.020961951, + 0.029209273, + 0.027647868, + -0.014137707, + -0.007757876, + 0.062348206, + -0.008718755, + -0.052925847, + -0.037267707, + 0.0011937132, + -0.012669211, + -0.012786681, + -0.006065171, + -0.0053765364, + -0.0021557, + -0.01481399, + -0.058289967, + -0.028986255, + 0.05078526, + -0.002854578, + 0.019647077, + 0.0051695504, + 0.0063654883, + -0.0038896971, + -0.013368816, + 0.025020888, + -0.01510708, + 0.056843843, + 0.029686432, + -0.028163437, + -0.013143269, + -0.03540732, + 0.01511013, + 0.020854753, + 0.0017551355, + -0.017126441, + -0.041056138, + 0.0023781115, + -0.0069902996, + 0.035195302, + -0.030548202, + 0.06579168, + 0.022744766, + -0.048352927, + 0.04308196, + 0.021044018, + 0.0616295, + -0.0047261664, + 0.022308754, + -0.014529809, + 0.005223897, + 0.0045979647, + 0.017964851, + 0.017680805, + 0.023487814, + 0.03323262, + 0.022610031, + 0.023765706, + 0.021247102, + -0.0049600713, + 0.00049963087, + -0.013432984, + 0.03581622, + 0.015985565, + 0.011417947, + 0.030523669, + -0.02688601, + 0.06639792, + -0.02117319, + -0.03151261, + -0.031172035, + -0.015158801, + -0.030480003, + -0.00863104, + -0.04325801, + -0.046840224, + -0.031376246, + 0.027147623, + -0.025138067, + 0.0054361243, + -0.022000033, + 0.0007541567, + -0.0068954234, + 0.025808848, + 0.032690976, + 0.026513726, + 0.008512804, + 0.012464478, + 0.008052137, + 0.026837809, + 0.015299088, + -0.044069223, + -0.012441158, + -0.031088399, + 0.0072665187, + -0.0055493154, + 0.006762638, + -0.01648, + -0.017916108, + 0.011688718, + -0.05096129, + 0.03369813, + 0.025480853, + 0.009241458, + 0.035165977, + 0.025816264, + -0.020851461, + -0.0358431, + 0.035711627, + -0.011933275, + 0.012066485, + 0.022662783 + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-1/router_report.json b/benchmarks/models/router/centroid/out-router-iter-1/router_report.json new file mode 100644 index 0000000..27ba144 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-1/router_report.json @@ -0,0 +1,16 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-read-file-repo-v1+router-memory-lane-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 61, + "accuracy": 0.6393443, + "tool_precision": 1.0, + "tool_recall": 0.3125, + "memory_false_tool_rate": 0.0, + "tool_count": 32, + "memory_count": 29 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-1/router_report.md b/benchmarks/models/router/centroid/out-router-iter-1/router_report.md new file mode 100644 index 0000000..7db5b73 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-1/router_report.md @@ -0,0 +1,16 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-read-file-repo-v1+router-memory-lane-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `61` +- tools: `32` memory: `29` +- accuracy: `0.6393` +- tools precision: `1.0000` +- tools recall: `0.3125` +- memory false-tools rate: `0.0000` diff --git a/benchmarks/models/router/centroid/out-router-iter-2/router_model.json b/benchmarks/models/router/centroid/out-router-iter-2/router_model.json new file mode 100644 index 0000000..2f21b4f --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-2/router_model.json @@ -0,0 +1,2058 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroid": [ + -0.030158933, + -0.024594568, + -0.056406025, + 0.0067304857, + 0.00733784, + -0.049096804, + 0.013384165, + 0.0032940868, + 0.013426696, + 0.02542755, + -0.0009688193, + 0.029141925, + 0.000296376, + 0.01933747, + 0.010894968, + -0.01677024, + 0.03815374, + -0.03642332, + 0.03288786, + -0.008070094, + -0.016237345, + -0.014721407, + 0.001743179, + 0.049892638, + -0.0030364334, + 0.053346463, + -0.041263964, + -0.0112016145, + 0.017235583, + 0.03351885, + 0.0051744143, + 0.02996308, + 0.010057567, + -0.029962737, + -0.027042015, + -0.02821384, + 0.0175575, + -0.014691321, + -0.04103755, + -0.023944119, + -0.0242843, + 0.019923279, + 0.017906753, + -0.037012357, + 0.041190032, + -0.05028872, + 0.033268057, + -0.041703932, + -0.005365291, + 0.01640351, + -0.018159337, + -0.01113035, + 0.013730543, + -0.019626148, + 0.05232349, + 0.04183952, + -0.0058470014, + 0.016360337, + -0.041641656, + -0.00671692, + -0.014156932, + -0.0123064965, + -0.012627963, + 0.012971188, + 0.022511443, + 0.09771813, + 0.027347898, + -0.00017407266, + -0.0206166, + -0.041309953, + -0.025098987, + 0.009518133, + 0.0021103453, + -0.0042419652, + -0.08753484, + 0.023568016, + 0.045663163, + -0.022341695, + 0.003240933, + 0.015888883, + 0.04717094, + -0.006485247, + 0.034132924, + 0.014375859, + -0.0028324744, + 0.015693113, + -0.026772179, + 0.0334811, + 0.0029938156, + -0.03154124, + -0.010088201, + 0.023399452, + 0.011892797, + -0.05228707, + -0.04860409, + 0.010279164, + -0.030946169, + -0.011789613, + 0.039029, + 0.0060012625, + 0.02816516, + 0.0077223224, + 0.015090058, + -0.027400699, + 0.049693227, + -0.0077910186, + 0.043815956, + 0.038115945, + -0.023675224, + -0.010207441, + 0.02350999, + 0.011840402, + 0.020384677, + 0.016443763, + -0.031284176, + -0.03227864, + -0.03528937, + -0.028469319, + 0.05566587, + 0.0058467616, + 0.06164917, + 0.003566779, + 0.070395455, + -0.033763222, + -0.014680556, + -0.03987685, + 0.023341317, + 0.009951921, + -0.008601885, + 0.005641318, + -0.0029025746, + 0.017305449, + -0.023325894, + 0.0006268679, + -0.03716706, + -0.015612242, + 0.010323762, + -0.0023666453, + -0.024673292, + -0.04920657, + 0.031302843, + 0.029780867, + -0.03142058, + -0.02811648, + 0.015695408, + -0.018455125, + 0.007814234, + -0.012841396, + 0.0114457, + 0.0033223003, + -0.016279269, + -0.014830346, + -0.015520214, + -0.014857956, + 0.024195703, + -0.015508234, + 0.02038092, + 0.08129829, + -0.026732484, + 0.055909246, + -0.015045142, + 0.023157476, + -0.044115793, + -0.012450717, + 0.043744635, + 0.005630191, + 0.011323905, + 0.03885255, + -0.012949468, + -0.008911743, + -0.016829746, + -0.016185075, + 0.028645014, + -0.013133291, + 0.025823029, + 0.021492016, + 0.10580085, + 0.017460752, + 0.0056111435, + -0.050006468, + -0.046395436, + 0.011334186, + 0.021412345, + -0.040918097, + -0.019092306, + -0.01825272, + 0.035007473, + -0.053765766, + 0.016572922, + 0.032207992, + 0.00077787915, + -0.004430577, + 0.03568403, + 0.0019151762, + 0.03422414, + 0.018024929, + -0.0033973502, + -0.028453736, + 0.007884207, + -0.029182317, + -0.009533449, + 0.03700277, + 0.06430073, + -0.008517321, + -0.060192198, + 0.00030106932, + -0.03924052, + -0.05021138, + 0.020380896, + 0.0067037577, + -0.018331824, + 0.01394221, + 0.028358825, + 0.0049292753, + -0.033005476, + -0.0014795884, + 0.024136012, + -0.00042968456, + -0.0115116015, + -0.013089202, + -0.022230428, + 0.04232657, + -0.015781417, + -0.058333866, + 0.024548827, + 0.03250101, + 0.00047059308, + 0.00096438604, + -0.0035391918, + -0.04964545, + -0.00021699935, + 0.023903254, + -0.022183003, + -0.02559454, + 0.032933317, + -0.010308942, + 0.0013652422, + -0.0011425585, + 0.015136843, + -0.013034972, + 0.0005529728, + -0.027793368, + -0.0020866964, + -0.0027142374, + -0.053411942, + -0.0012461789, + 0.011579123, + 0.013881961, + 0.0025264667, + 0.012506296, + 0.028345155, + 0.0006526794, + -0.04778645, + -0.027079223, + 0.052193083, + 0.03243979, + 0.029929688, + -0.0035464207, + 0.02499098, + 0.0002498847, + 0.03890249, + 0.0022518137, + 0.012700003, + 0.036127705, + 0.026531013, + -0.005983982, + -0.022589171, + -0.031490352, + 0.0030023353, + 0.008816021, + -0.030125951, + 0.004298043, + 0.021942398, + 0.02441062, + 0.037066985, + 0.025459789, + -0.033168435, + -0.037348315, + 0.060144402, + 0.018365202, + -0.011200509, + 0.012117781, + 0.022341881, + -0.011110944, + -0.004324493, + -0.017759439, + -0.03001637, + -0.008325672, + -0.009326878, + -0.047553726, + 0.015292038, + -0.00008850687, + 0.100346215, + 0.005622011, + -0.025145259, + -0.03417576, + -0.04230635, + -0.22315957, + -0.027743163, + -0.0064952797, + -0.0020120218, + 0.0036726466, + -0.013304299, + -0.01871622, + 0.0062016905, + -0.04018127, + 0.03637924, + -0.053358193, + -0.06943479, + 0.008638506, + -0.011154827, + -0.022103397, + 0.0047856206, + -0.010176259, + -0.010154141, + 0.017651118, + -0.030833216, + -0.010355726, + -0.020078931, + 0.020292822, + -0.019591076, + -0.004293628, + -0.013992447, + 0.02715184, + 0.024009869, + -0.038513497, + -0.018646156, + 0.018442191, + 0.030448552, + -0.016363602, + 0.047819015, + 0.012516331, + 0.02187635, + 0.018255936, + -0.015174653, + -0.027301447, + -0.014250595, + 0.0318422, + 0.008080228, + -0.019835392, + 0.032348208, + 0.022355752, + -0.04586592, + -0.012978983, + -0.030431662, + -0.007845257, + 0.00017124522, + -0.027477004, + 0.013483654, + 0.0030128746, + -0.0049953996, + -0.03059007, + -0.010614661, + -0.013466093, + 0.040741067, + -0.06808487, + 0.006921484, + -0.0031512144, + 0.0011705683, + 0.01469336, + 0.042702485, + -0.03701421, + 0.010135405, + -0.025082445, + -0.022003377, + 0.032610953, + -0.02253984, + 0.030252619, + -0.0017578467, + 0.00041530773, + -0.0124381585, + -0.023433758, + 0.05481298, + -0.014507347, + -0.03831866, + -0.017518941, + -0.155799, + -0.012087986, + -0.012125733, + 0.0077639525, + 0.0012467375, + -0.022459473, + 0.00085671974, + 0.0063514644, + 0.04656638, + 0.039725747, + 0.31252456, + 0.019947998, + -0.015862523, + -0.008657066, + 0.039528456, + -0.018160136, + 0.0279511, + 0.021467809, + 0.009370419, + -0.039877526, + -0.033184532, + 0.010124717, + 0.020985708, + 0.004192975, + 0.014652567, + 0.0050357333, + -0.033262458, + 0.0085534295, + 0.10188007, + -0.010594772, + 0.03326642, + 0.014756109, + 0.021141242, + 0.023403717, + -0.054265622, + -0.038193814, + -0.010478817, + 0.05658113, + -0.021639593, + 0.032594115, + -0.026673833, + 0.02112071, + 0.029332595, + -0.021142935, + -0.041812513, + -0.0019439298, + 0.012894297, + 0.020488244, + 0.0047332044, + 0.037286583, + -0.0018558527, + -0.04128194, + -0.0051113763, + -0.012563003, + -0.0002624014, + -0.04123298, + -0.0061261253, + -0.022386327, + -0.03216891, + -0.018910397, + -0.020828817, + 0.016659554, + -0.012942391, + -0.01061604, + -0.0101624075, + -0.02066082, + -0.049747415, + 0.010140996, + -0.0013797707, + 0.024357785, + 0.026671674, + 0.011506057, + -0.045860555, + -0.012978272, + -0.03224099, + 0.019980853, + -0.028152563, + 0.004329637, + 0.034350507, + 0.012033209, + 0.006408091, + -0.0016786259, + 0.008488509, + 0.0010502855, + 0.047142502, + 0.010217027, + -0.0025450394, + 0.081364006, + -0.0064511704, + 0.027054882, + -0.050222714, + -0.009497105, + -0.03458346, + 0.031178953, + 0.0060493504, + -0.022951348, + -0.014272538, + 0.059296854, + 0.006166048, + -0.029008066, + -0.006319774, + 0.015290387, + -0.03708943, + -0.0020311123, + 0.012522553, + 0.0065816725, + 0.011947093, + -0.01595569, + -0.03583233, + -0.022868667, + -0.041648984, + -0.017554268, + 0.00903062, + 0.011449154, + 0.000116997115, + 0.007178507, + -0.00600004, + 0.012808819, + 0.02501104, + 0.0142230345, + -0.025906332, + 0.031335182, + -0.022452552, + -0.018431248, + 0.046120215, + 0.012229508, + -0.011386171, + 0.029536748, + -0.0047980677, + 0.036839105, + -0.01932659, + -0.024105249, + -0.003549046, + -0.0016245841, + -0.004523504, + -0.007854581, + 0.0046904944, + -0.041355643, + -0.029497514, + 0.022174573, + 0.027819965, + -0.0079562, + 0.033943046, + 0.01246582, + 0.036248833, + 0.025040343, + -0.015320059, + 0.017238367, + 0.022863822, + 0.03578739, + -0.0054725264, + -0.058293164, + -0.011595976, + 0.011458963, + -0.018111095, + -0.0069801444, + 0.017254794, + 0.000049858274, + -0.018538503, + 0.015530646, + 0.012646961, + 0.034498695, + -0.026425729, + -0.054772254, + -0.027476894, + -0.031755358, + -0.009656751, + -0.0045208787, + -0.023698384, + -0.04037318, + 0.018581659, + -0.0025216218, + 0.0013631919, + 0.05480681, + 0.038690004, + 0.045393694, + -0.032392215, + 0.041171182, + 0.039993588, + -0.043868873, + -0.0011620927, + -0.009544207, + -0.030320043, + 0.040896118, + 0.024578031, + -0.00514168, + -0.06595213, + 0.0013193273, + 0.009708954, + 0.018381035, + 0.026208753, + 0.028165696, + -0.038020782, + -0.006095229, + 0.004736932, + 0.023571962, + -0.034097288, + -0.024980212, + 0.031131666, + -0.030180369, + -0.012669745, + 0.12845737, + -0.009262958, + 0.006117066, + 0.008606292, + 0.010089435, + 0.012808506, + 0.010287563, + -0.028770402, + -0.03128461, + -0.0023840151, + 0.017938577, + 0.0046182824, + -0.03342182, + 0.030958535, + 0.021372152, + -0.002489681, + 0.028727004, + -0.04026889, + -0.032296386, + 0.008242219, + -0.0014133428, + -0.04516806, + 0.004301047, + 0.023885652, + -0.004553714, + 0.04501727, + 0.03822354, + -0.014454402, + -0.017454399, + 0.0013665726, + 0.013624331, + 0.0296024, + -0.055577084, + -0.020230528, + -0.0023543492, + -0.031565223, + -0.060246803, + 0.015393465, + 0.0120394975, + 0.015085709, + 0.0038687207, + -0.0021065683, + -0.02948322, + 0.02954973, + -0.002662551, + -0.0028915412, + 0.03010874, + 0.0018913413, + -0.02001067, + -0.0063421316, + -0.020792378, + -0.010091964, + 0.03183145, + -0.020252602, + 0.0026068757, + -0.009242164, + -0.0076772124, + 0.019720795, + 0.03330952, + 0.010453487, + 0.0075338157, + 0.009724588, + -0.01907, + -0.021368962, + 0.015605548, + 0.027819917, + -0.010995347, + -0.027277173, + -0.011047905, + -0.016077582, + 0.0423546, + -0.0348623, + -0.013674576, + -0.049335133, + -0.012552401, + 0.040408406, + -0.025421368, + -0.032590643, + -0.046601515, + 0.0025642498, + -0.01712069, + -0.061188515, + -0.05050546, + 0.019739132, + 0.009370893, + -0.05515342, + -0.00792603, + -0.007119067, + 0.019636208, + -0.010822538, + 0.021958094, + -0.025198866, + 0.010734687, + -0.034492377, + 0.0057179113, + 0.047987107, + -0.002487443, + -0.006133101, + -0.03145328, + 0.04914926, + 0.006528816, + -0.031226907, + 0.018867563, + -0.042887878, + 0.026736716, + -0.017539347, + 0.027214684, + -0.022919202, + 0.0142470375, + -0.025736624, + 0.055748384, + -0.057226773, + -0.014126419, + -0.010110122, + -0.0074769016, + -0.019791309, + -0.010331634, + 0.0311707, + 0.040381394, + 0.020953786, + -0.038685463, + 0.0012481797, + 0.0051444485, + -0.0045783785, + 0.012263143, + 0.02732164, + -0.0146240415, + -0.032081336, + 0.013474712, + -0.0022820516, + 0.0058794343, + -0.0155197205, + -0.010183763, + -0.024280524, + -0.007045957, + -0.0002643818, + 0.010822618, + -0.009021557, + -0.015366354, + -0.018417377, + -0.008582669, + -0.011569156, + -0.011808581, + 0.048185278, + 0.0026539196, + -0.0032764615, + -0.027689375, + 0.021636514, + 0.009648907, + 0.014468062, + -0.005044251, + 0.0330727, + -0.04974972, + 0.019718478, + -0.04376081, + 0.010439787, + 0.0060001034, + -0.0055992203, + 0.021372303, + 0.00909813, + 0.026158217, + -0.029960316, + -0.056653716, + -0.007070839, + -0.010390761, + -0.056854412, + 0.061641935, + 0.039760288, + 0.015059951, + -0.026475845, + 0.037060566, + 0.0023266226, + 0.013363719, + -0.006553491, + 0.027972145, + 0.013578458, + -0.023147563, + 0.007696674, + -0.012352437, + -0.025676409, + -0.005862487, + -0.023025362, + 0.00013237864, + -0.016827293, + 0.043890774, + -0.0069043515, + -0.002225907, + 0.047529373, + 0.02776551, + -0.027308097, + -0.009976119, + 0.014083222, + -0.039032746, + 0.00823128, + -0.005064999, + -0.02645116, + -0.011442958, + 0.04483286, + 0.045719676, + -0.022178806, + -0.013665126, + -0.053748205, + 0.013987907, + -0.21245013, + 0.028900504, + 0.015495563, + -0.023709415, + -0.05277884, + -0.0044326433, + -0.0016411158, + -0.0010053184, + 0.0092460355, + -0.028384918, + -0.03013411, + 0.023030816, + 0.0029167945, + -0.011838777, + -0.016886614, + 0.017512593, + -0.0166214, + -0.0044976333, + 0.008748739, + 0.022155166, + 0.0061110794, + -0.015815306, + 0.03417069, + 0.0028622507, + 0.023335291, + -0.025867265, + -0.03364428, + 0.011249567, + -0.057569873, + -0.012630175, + 0.001804221, + 0.0077559557, + 0.032078512, + 0.048531707, + -0.024454137, + -0.0064289267, + -0.0026888933, + -0.028792782, + 0.0044064308, + -0.02250922, + 0.0008787167, + 0.044624098, + -0.015640013, + -0.030854838, + -0.012837752, + 0.031617116, + -0.002445824, + 0.0175789, + -0.06825805, + 0.018819088, + -0.0046966574, + 0.020319879, + -0.01556398, + 0.004348517, + 0.019904878, + -0.030961744, + -0.021681655, + 0.0136909755, + 0.027412746, + 0.038610455, + 0.004332208, + -0.016458705, + -0.026644284, + -0.050482254, + -0.055810463, + -0.0109890215, + -0.08290311, + 0.002764622, + -0.0017576261, + 0.009762837, + -0.023463152, + -0.023039486, + -0.001693047, + -0.04850589, + 0.016658837, + -0.019495398, + 0.011998887, + -0.011074753, + -0.011070885, + -0.02350858, + -0.0049924864, + 0.017045598, + -0.045069616, + -0.028145723, + 0.04560487, + -0.00077625667, + -0.023827428, + 0.0093330005, + -0.011302472, + 0.008246307, + -0.07070581, + 0.019857015, + 0.0031504056, + 0.022072222, + -0.016170753, + 0.032627016, + -0.011681984, + 0.0057049193, + -0.019730128, + -0.020233594, + -0.058517396, + -0.0042922352, + -0.014554356, + 0.026898833, + -0.012998931, + -0.031928632, + 0.034263205, + -0.009500482, + 0.006373947, + -0.0063130115, + -0.034286898, + -0.014790762, + -0.018278692, + 0.010846886, + -0.035936527, + 0.008353328, + 0.016868606, + -0.0040308274, + -0.03923267, + 0.0056272573, + 0.0016512506, + 0.031374753, + -0.009074346, + -0.043900106, + -0.0022785757, + -0.0006169612, + 0.051975645, + -0.0012797511, + -0.008205946, + -0.014210059, + 0.048846018, + -0.024141526, + -0.011423073, + 0.039766513, + 0.008620199, + -0.013218299, + -0.022557477, + 0.047936507, + -0.017639168, + -0.053761933, + 0.0075839753, + 0.013224882, + 0.010247319, + -0.005988015, + -0.01810729, + 0.0109523535, + 0.0035619126, + 0.01939704, + -0.051323038, + -0.01519885, + 0.037725024, + 0.010399132, + 0.002619895, + 0.00635025, + 0.0122255655, + 0.0056703454, + 0.020467438, + 0.025297642, + -0.02359711, + 0.060742233, + 0.035414115, + -0.04084401, + -0.033174753, + -0.04637599, + -0.0080869235, + 0.007368212, + 0.020997435, + -0.004418203, + -0.0029068089, + 0.006940854, + -0.0021705634, + 0.053542506, + -0.028913805, + 0.07522591, + 0.014798977, + -0.051614176, + 0.040079203, + -0.003397764, + 0.040937036, + -0.015017336, + 0.030344773, + -0.021334557, + -0.008495226, + 0.024891697, + 0.0054758773, + 0.017094707, + 0.027647924, + 0.051046472, + 0.011877686, + 0.0016215384, + 0.019153664, + -0.013071086, + -0.0043805623, + -0.04710728, + 0.03946076, + -0.0060520177, + 0.020018842, + 0.024909692, + -0.035783812, + 0.06287694, + -0.010301415, + -0.03545394, + -0.04451174, + -0.012411114, + -0.022004027, + -0.009244311, + -0.033827618, + -0.028101265, + -0.030710697, + 0.04251961, + -0.03106155, + 0.013254086, + -0.029376116, + 0.0052649286, + 0.006244907, + 0.0029282388, + 0.007978551, + 0.020579485, + 0.021814324, + -0.029406741, + -0.01268872, + 0.013536271, + 0.0018233269, + -0.04774436, + -0.034671843, + -0.010110523, + -0.014541311, + -0.0080147525, + -0.005103827, + -0.013508345, + -0.024189278, + 0.036705013, + -0.038841676, + 0.033497896, + 0.015676761, + 0.0010375909, + 0.052992936, + 0.024051605, + -0.0037915022, + -0.0057457895, + 0.047689747, + -0.026850581, + 0.0030283013, + 0.030290179 + ], + "tools_centroid": [ + -0.058447294, + -0.0031960418, + -0.04056695, + 0.007944188, + -0.01720079, + -0.009792306, + 0.01882789, + -0.009346945, + -0.007893483, + 0.007612424, + 0.0069307038, + 0.015375071, + 0.004445494, + 0.025724865, + 0.015466023, + -0.015992085, + 0.018320123, + -0.013487111, + 0.014783198, + -0.03221811, + -0.012679123, + -0.017891474, + -0.013265811, + 0.04521114, + 0.017769901, + 0.044511158, + -0.033718526, + -0.02975472, + 0.03670862, + 0.04829003, + 0.005967585, + -0.018850904, + 0.0007424381, + -0.0535405, + -0.033822175, + -0.030061863, + 0.023873799, + -0.007866787, + -0.06508872, + -0.009540423, + -0.02382827, + 0.008130161, + 0.008830627, + -0.043973986, + 0.019689322, + -0.039084036, + 0.02522517, + -0.039465718, + 0.0004369307, + 0.003260306, + -0.026097946, + -0.014911177, + 0.022265662, + -0.04056172, + 0.029380584, + 0.050641157, + -0.033163603, + 0.0032921697, + -0.053207472, + -0.000034995872, + -0.03003254, + -0.019056404, + -0.011096353, + 0.010902441, + 0.005736869, + 0.08859268, + 0.022789588, + -0.0009730732, + -0.012210075, + -0.018232493, + -0.04407237, + 0.020689921, + -0.03126351, + -0.03027248, + -0.08748702, + 0.03231227, + 0.042549595, + -0.02531143, + -0.01296788, + -0.004932874, + 0.049735464, + -0.0016840721, + 0.015246114, + 0.0027477166, + 0.019951588, + 0.049628966, + -0.041547712, + 0.033872727, + -0.0064435517, + -0.0053193327, + -0.028322356, + -0.0019142586, + 0.006737079, + -0.04678162, + -0.05161081, + -0.013790464, + -0.029214896, + 0.007250293, + 0.035229333, + 0.007100894, + 0.015750123, + 0.0040342584, + 0.0076452824, + -0.0131321475, + 0.0496379, + 0.0019094211, + 0.037393786, + 0.019485392, + -0.026469301, + -0.009052817, + 0.0031331095, + 0.022210818, + 0.018437084, + 0.009465985, + -0.022718461, + -0.044343844, + -0.034930807, + -0.049252804, + 0.019860908, + -0.008505989, + 0.04983175, + -0.010211743, + 0.06845632, + -0.041160807, + -0.017241798, + -0.033599205, + 0.03713078, + 0.054508746, + 0.014981393, + 0.034874067, + 0.0019188733, + 0.036510266, + -0.007648808, + 0.0115660075, + -0.036845684, + -0.022880994, + 0.00807968, + 0.0056036613, + -0.007911824, + -0.0678003, + 0.020695195, + 0.055565864, + -0.035987604, + -0.03601063, + 0.037539974, + -0.050159164, + 0.022635216, + 0.0030483394, + 0.008907969, + -0.005440321, + -0.021004127, + 0.0012472612, + 0.009223827, + 0.0006058824, + 0.0031326385, + -0.015148256, + 0.007960367, + 0.056991834, + -0.021474354, + 0.067357525, + -0.0010688632, + 0.01298642, + -0.027193664, + -0.017325649, + 0.03044363, + -0.014550752, + 0.007992259, + 0.00029936864, + -0.0059629427, + -0.014359268, + -0.0065565393, + -0.027172448, + 0.016619336, + -0.011691314, + 0.027468683, + 0.026932633, + 0.09433931, + 0.024193535, + 0.011354052, + -0.05888176, + -0.04640026, + 0.02127491, + -0.008293077, + -0.010976071, + -0.03443611, + -0.005526956, + 0.04282139, + -0.011901629, + 0.009022909, + 0.013723731, + -0.0010163852, + 0.011661284, + 0.028600821, + 0.0041018007, + 0.030858293, + 0.017554034, + -0.0025650044, + -0.02218127, + -0.024183678, + -0.032940112, + -0.022800341, + 0.03276703, + 0.043550715, + 0.003735454, + -0.04059577, + -0.011261896, + -0.0124205155, + -0.034101605, + 0.007095949, + -0.0399806, + -0.01299875, + -0.00091557915, + 0.027418181, + 0.0010769746, + -0.04121067, + -0.0055925366, + 0.007871162, + 0.009978493, + -0.031887997, + -0.03317864, + -0.022772314, + 0.05381704, + 0.0025545843, + -0.015489466, + 0.016336259, + -0.0038492333, + 0.023457069, + 0.011922578, + 0.0000024196252, + -0.022392092, + -0.023537602, + -0.0068239416, + -0.006778134, + -0.01978505, + 0.07156752, + -0.0015917633, + -0.011320589, + 0.0019752886, + 0.02678749, + 0.0005512292, + -0.02521239, + -0.023816619, + 0.006969701, + -0.0027778212, + -0.044873677, + 0.017067868, + 0.0431196, + 0.033239935, + -0.017727666, + 0.005981991, + 0.02905667, + -0.0101856915, + -0.043768834, + -0.005080569, + 0.034496784, + 0.031769603, + 0.028247474, + -0.012236884, + 0.012497114, + 0.010201207, + 0.034260225, + 0.020852678, + 0.0152562335, + 0.044217672, + 0.007017044, + -0.0075565954, + -0.023899524, + -0.006117618, + 0.007223929, + -0.009271626, + -0.03253554, + 0.021442601, + 0.028374553, + 0.007935402, + 0.0191637, + 0.0069043823, + -0.024508342, + -0.030255267, + 0.059182912, + 0.022028672, + -0.0067743533, + 0.021868106, + 0.010847354, + 0.0033291879, + 0.00033487147, + 0.0021198462, + -0.034662843, + -0.009133411, + -0.009465647, + -0.04178123, + -0.020095155, + -0.008050191, + 0.10526302, + -0.0016197939, + -0.01590977, + -0.0023023991, + -0.03436933, + -0.2270914, + -0.010789334, + 0.009566663, + 0.032746516, + 0.006308291, + -0.0056209373, + -0.0026306922, + 0.00514814, + -0.041839983, + 0.02274705, + -0.037504744, + -0.07821607, + -0.029537264, + -0.03458881, + -0.0017964041, + -0.004188229, + -0.0072670956, + -0.00671201, + 0.021746209, + -0.04046833, + -0.020446511, + -0.014678749, + 0.051786486, + -0.0058354083, + -0.013429737, + -0.013777938, + 0.04783541, + 0.008407424, + -0.037879635, + -0.010391947, + 0.0029270153, + 0.007556529, + -0.0037149857, + 0.03262755, + 0.012163848, + 0.03500064, + 0.009735388, + -0.01083349, + -0.016225653, + -0.01077916, + 0.021660518, + 0.040077213, + 0.00080703787, + 0.03883533, + 0.041348767, + -0.035640802, + 0.014520695, + -0.028424151, + -0.016154394, + -0.009835497, + -0.00137285, + -0.005356795, + 0.0024231486, + -0.025629193, + -0.04458243, + -0.00094364455, + -0.03294703, + 0.042818334, + -0.032569297, + -0.0030459424, + -0.028585317, + -0.0040611876, + 0.0484161, + 0.019106826, + -0.024614794, + 0.018956464, + 0.017490719, + -0.020015856, + 0.038299162, + -0.022633351, + 0.02833699, + 0.002009954, + 0.00066899817, + -0.025235463, + -0.009802935, + 0.04749879, + -0.027862752, + -0.036436256, + -0.0023303882, + -0.18642783, + -0.02838701, + 0.018801799, + 0.0005800725, + 0.015029056, + -0.035692684, + -0.0061907507, + -0.0019379087, + 0.021536369, + 0.047127422, + 0.32193023, + 0.019010754, + -0.017304754, + 0.009844196, + 0.052283093, + -0.015513529, + 0.007557992, + 0.036223244, + -0.005931426, + -0.03209307, + -0.00025808386, + 0.0060067424, + 0.025587222, + -0.0025760718, + -0.0050809965, + 0.023252537, + -0.030068107, + -0.001381828, + 0.10378124, + -0.002725449, + 0.031586733, + -0.0047198646, + 0.0073324414, + 0.026034288, + -0.05220758, + -0.041303143, + -0.009671336, + 0.056607354, + -0.025829045, + 0.04589712, + -0.015022113, + 0.019579252, + 0.024854966, + 0.00010052749, + -0.053336963, + -0.017414058, + 0.010370148, + -0.012685429, + -0.009945212, + 0.032859802, + -0.028728088, + -0.046992306, + 0.002105697, + 0.025091713, + 0.003834139, + -0.040179554, + 0.016500555, + -0.0038185816, + -0.028435927, + -0.017908825, + -0.034928385, + -0.026433412, + -0.022669781, + 0.0035244278, + -0.001221266, + -0.03454084, + -0.023225956, + -0.03057678, + -0.011691087, + 0.050754417, + 0.0045607793, + 0.009647709, + -0.065302126, + 0.028458044, + -0.0030156719, + 0.041296776, + -0.029290905, + -0.013334934, + 0.012328987, + 0.015164583, + 0.013692162, + -0.0086320955, + 0.0058902088, + -0.010065171, + 0.055015996, + 0.020061323, + -0.021189142, + 0.064155824, + -0.01967612, + 0.04473069, + -0.02370946, + 0.009871038, + -0.04645784, + 0.026810719, + 0.024124194, + -0.019557776, + -0.012182461, + 0.029323336, + -0.026990678, + -0.04044107, + -0.008360336, + -0.0019938978, + -0.028371817, + 0.0137573555, + 0.01498859, + 0.0030597097, + 0.014817098, + -0.005054728, + -0.038018536, + -0.017528528, + -0.032108974, + -0.018907543, + -0.006474138, + -0.025025873, + 0.019918358, + 0.0016337631, + 0.00094817666, + 0.009802728, + 0.031116743, + -0.003632269, + -0.029542513, + 0.032651883, + -0.010025785, + -0.03951328, + 0.03364444, + 0.017194578, + 0.004983647, + 0.020777998, + 0.029283386, + 0.03550377, + -0.029875956, + -0.015304166, + -0.008314896, + 0.008374275, + -0.025601873, + -0.030097278, + 0.0002924097, + 0.0076187383, + -0.056274734, + 0.042091563, + 0.01805237, + -0.018683214, + 0.017246589, + 0.0105300965, + 0.014187611, + 0.006859431, + 0.007999717, + 0.025106467, + -0.0029121756, + 0.06052328, + -0.0067593786, + -0.052844215, + -0.020656083, + 0.008535256, + -0.0055435193, + 0.0003612499, + 0.04269211, + -0.017029254, + -0.022165032, + 0.010793594, + 0.006198478, + 0.047784764, + 0.023047267, + -0.06756267, + -0.01529474, + -0.034441695, + -0.016781041, + 0.012083947, + -0.012566199, + -0.037975416, + 0.015395367, + -0.003163975, + 0.023721986, + 0.070848145, + 0.02498515, + 0.014214773, + -0.045816313, + 0.026322227, + 0.023617595, + -0.033912502, + -0.0015618792, + -0.033572003, + -0.060556598, + 0.043581463, + 0.0011356333, + -0.01836653, + -0.043114338, + 0.0005826029, + -0.017311895, + 0.0070527415, + 0.045568686, + 0.013204873, + -0.045201357, + 0.0121381115, + 0.015162057, + 0.03274591, + -0.0092090275, + -0.016862372, + 0.014428687, + -0.025555763, + -0.032382842, + 0.11517472, + 0.012913748, + 0.0010030848, + 0.011226032, + 0.011379718, + 0.009522554, + 0.027689219, + -0.009042815, + -0.018272746, + 0.0018735309, + -0.0061869416, + 0.0051591247, + -0.0398507, + 0.003928402, + -0.012579425, + -0.015087355, + 0.015030611, + -0.011281217, + -0.020296069, + 0.0028820666, + -0.01710269, + -0.0027538878, + -0.0026668024, + 0.044359304, + 0.0025099802, + 0.046448138, + 0.051876616, + -0.008421251, + -0.018022476, + -0.017295873, + -0.0039007443, + 0.053241838, + -0.06659699, + -0.005089003, + 0.016671598, + -0.010481239, + -0.04684404, + 0.019394755, + 0.011159196, + 0.013773246, + 0.004854403, + 0.0062795402, + -0.03240617, + 0.036110453, + -0.027191468, + 0.008575306, + 0.040809512, + 0.0013822563, + -0.032052763, + 0.021988612, + -0.016962554, + -0.0015822274, + 0.035482984, + -0.0061631063, + 0.053959966, + -0.023658017, + 0.012072526, + 0.031551845, + 0.033003576, + -0.01434241, + 0.0010887928, + 0.0059649707, + -0.0138169285, + -0.019048441, + 0.032960784, + -0.0039301184, + -0.0026139007, + -0.021277042, + -0.0083170775, + -0.019064948, + 0.02699767, + -0.04146312, + -0.0026123526, + -0.027134454, + -0.025479833, + 0.03564004, + -0.030421788, + -0.024279838, + -0.06379215, + -0.015614495, + -0.009519136, + -0.052777804, + -0.01347346, + 0.039031014, + 0.014409142, + -0.03755806, + 0.0064382264, + 0.02259292, + 0.024713065, + -0.024994036, + 0.012633482, + 0.00003157109, + 0.032607287, + -0.033095337, + 0.0224503, + 0.01058474, + -0.02637207, + 0.004264986, + -0.0433577, + 0.024697362, + 0.0031457401, + -0.038848136, + 0.032227132, + -0.040083885, + 0.011954821, + -0.004419487, + 0.014481568, + -0.025576422, + 0.019512936, + -0.02390567, + 0.023905465, + -0.05336155, + -0.007795936, + -0.016667465, + 0.015853627, + -0.024931796, + 0.0035742885, + 0.043597262, + 0.042496238, + 0.035893306, + -0.027150068, + -0.006415165, + 0.039902866, + 0.014022735, + -0.00048476583, + 0.0594834, + 0.0017288239, + -0.015557113, + 0.00333523, + -0.0011074982, + 0.0066636647, + -0.0031938264, + -0.015648527, + -0.015319508, + -0.00368728, + -0.01555128, + 0.0026797706, + -0.02117369, + -0.02582257, + -0.023872102, + -0.021466129, + -0.002761185, + -0.01750263, + 0.009218285, + -0.0033276677, + 0.008450943, + -0.006542987, + 0.044404972, + 0.0021212394, + 0.009176219, + -0.011890422, + 0.032508947, + -0.029824732, + 0.017186426, + -0.023039123, + 0.0031951866, + -0.01177644, + -0.007911653, + 0.016799917, + 0.009921063, + 0.028776031, + -0.035464734, + -0.06951237, + 0.0035373778, + -0.00613269, + -0.049285594, + 0.010376047, + 0.026015468, + 0.011104281, + -0.021370681, + 0.039004907, + -0.0074733025, + 0.017269338, + 0.025572926, + 0.01037642, + 0.01601829, + 0.008659094, + 0.012105466, + -0.015545423, + -0.009304879, + -0.011241072, + -0.012088901, + 0.0013219892, + -0.017073164, + 0.03771827, + -0.013545965, + -0.015565453, + 0.03266775, + 0.0068746554, + -0.010415988, + -0.031751655, + -0.026515318, + -0.021783376, + 0.024326453, + -0.0053048055, + -0.0072763884, + -0.012792159, + 0.040500026, + 0.025817709, + -0.037458625, + -0.009586237, + -0.04644858, + 0.021790292, + -0.2097004, + 0.022277262, + 0.00925218, + -0.01330478, + -0.059066284, + 0.008308742, + -0.01301928, + -0.019033229, + 0.00977775, + -0.03943684, + -0.045749992, + 0.008570107, + 0.02584777, + -0.029028846, + 0.013970202, + 0.023132365, + 0.013279272, + 0.010783951, + 0.0072522424, + 0.030530201, + -0.01425467, + -0.03250509, + 0.0016272331, + 0.026309222, + 0.006766879, + -0.010977533, + -0.010880828, + 0.007981787, + -0.046763144, + -0.0013766524, + 0.01430926, + -0.015967568, + 0.035539635, + 0.06445331, + -0.025141623, + -0.009602247, + 0.012442875, + -0.031661037, + -0.020026587, + -0.016751653, + -0.011698287, + 0.035418995, + -0.016517041, + -0.010870692, + 0.015795322, + 0.034560096, + -0.00676306, + 0.0011204355, + -0.058349114, + 0.014773728, + -0.005162932, + 0.044128817, + -0.031294342, + -0.00027844525, + 0.021575982, + -0.016642638, + -0.047036625, + 0.023938298, + -0.010648823, + 0.031642064, + -0.014398178, + 0.005215185, + -0.037612718, + -0.07367502, + -0.0458412, + -0.030029025, + -0.085670546, + 0.014274666, + 0.004098493, + 0.031568263, + -0.018026913, + -0.037399463, + 0.004551527, + -0.025407037, + 0.018431269, + -0.015955111, + 0.01386698, + -0.0016229536, + -0.027075209, + -0.029598784, + -0.031206954, + 0.013200273, + -0.058429815, + 0.007434837, + 0.061316486, + -0.008615781, + -0.0035700274, + 0.0032746212, + -0.017676398, + 0.009754395, + -0.05421845, + -0.00428071, + 0.002142346, + 0.021296043, + -0.012709615, + 0.01350642, + -0.03156724, + -0.0028092018, + -0.013961775, + 0.005936083, + -0.018809218, + -0.0006931519, + 0.0011878702, + 0.014186199, + -0.0106336735, + -0.009251906, + 0.06331547, + -0.02106168, + 0.016771277, + -0.015768385, + -0.035321176, + -0.004028431, + -0.015411706, + 0.0056481315, + -0.048268754, + -0.0029887888, + 0.026768155, + -0.022176253, + -0.030154124, + -0.019171996, + -0.0028032036, + 0.015927503, + 0.007500834, + -0.043230843, + 0.005045231, + 0.0002987908, + 0.03661693, + 0.0039056158, + -0.0031826429, + -0.016787024, + 0.038857818, + -0.033475928, + -0.010959669, + 0.028936237, + 0.031155655, + -0.013519423, + -0.015248023, + 0.067478545, + -0.021371199, + -0.05508879, + -0.02518557, + 0.003544199, + 0.0007519942, + -0.019508924, + -0.00019194256, + -0.012832466, + -0.003950361, + -0.011461739, + -0.042191673, + -0.028606458, + 0.051829662, + -0.0022842, + 0.01640756, + 0.016773256, + 0.0012041852, + 0.003926084, + -0.013485454, + 0.023813676, + -0.010358046, + 0.052012555, + 0.027156146, + -0.03663885, + -0.01510494, + -0.028510412, + 0.011805402, + 0.01915899, + 0.00057777733, + -0.016607765, + -0.03759277, + 0.0051710485, + -0.0014569098, + 0.031002868, + -0.03187738, + 0.05561693, + 0.024154423, + -0.048074104, + 0.040291578, + 0.014613219, + 0.06374945, + -0.0031196265, + 0.023521155, + -0.013955653, + 0.018908784, + 0.008058266, + 0.011395477, + 0.018382747, + 0.0270407, + 0.033795472, + 0.016236296, + 0.017942632, + 0.017403232, + -0.0041898615, + -0.000022597074, + -0.01969108, + 0.038517915, + 0.005079587, + 0.014767408, + 0.024975155, + -0.027293308, + 0.062647566, + -0.020314064, + -0.03384863, + -0.033700544, + -0.025429081, + -0.028679423, + -0.010660895, + -0.027214939, + -0.027363313, + -0.03129036, + 0.040604837, + -0.029525558, + 0.0068941102, + -0.025948266, + 0.0069617606, + -0.004049944, + 0.018970516, + 0.026062682, + 0.030554367, + 0.007699915, + 0.008290043, + 0.005787263, + 0.026163168, + 0.013165965, + -0.047193095, + -0.016155748, + -0.031509243, + 0.001097126, + -0.0074935425, + 0.0044695805, + -0.011330939, + -0.02363785, + 0.011995145, + -0.043097675, + 0.033475034, + 0.025124647, + -0.0026637788, + 0.03679042, + 0.034322318, + -0.02150476, + -0.02400545, + 0.032913037, + -0.01672997, + 0.0060167876, + 0.01644737 + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-2/router_report.json b/benchmarks/models/router/centroid/out-router-iter-2/router_report.json new file mode 100644 index 0000000..193e25d --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-2/router_report.json @@ -0,0 +1,16 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 121, + "accuracy": 0.54545456, + "tool_precision": 1.0, + "tool_recall": 0.2857143, + "memory_false_tool_rate": 0.0, + "tool_count": 77, + "memory_count": 44 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-2/router_report.md b/benchmarks/models/router/centroid/out-router-iter-2/router_report.md new file mode 100644 index 0000000..5051c00 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-2/router_report.md @@ -0,0 +1,16 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `121` +- tools: `77` memory: `44` +- accuracy: `0.5455` +- tools precision: `1.0000` +- tools recall: `0.2857` +- memory false-tools rate: `0.0000` diff --git a/benchmarks/models/router/centroid/out-router-iter-3/router_model.json b/benchmarks/models/router/centroid/out-router-iter-3/router_model.json new file mode 100644 index 0000000..97bcf82 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-3/router_model.json @@ -0,0 +1,3084 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroid": [ + -0.029394908, + -0.024581162, + -0.056838706, + 0.006685245, + 0.007364617, + -0.047368113, + 0.014246822, + 0.0029091788, + 0.0135162305, + 0.0255997, + -0.0010292961, + 0.0289654, + 0.00025056294, + 0.02042426, + 0.010775252, + -0.017788867, + 0.038103033, + -0.03617505, + 0.033212826, + -0.006951542, + -0.016655948, + -0.014335452, + 0.0012494023, + 0.050547298, + -0.0023661517, + 0.05359952, + -0.0413253, + -0.011951271, + 0.016988749, + 0.032585848, + 0.004925438, + 0.030110508, + 0.009047346, + -0.030187827, + -0.027628658, + -0.028636174, + 0.017235082, + -0.014865178, + -0.040698223, + -0.023998253, + -0.024469681, + 0.019989595, + 0.01732986, + -0.037933737, + 0.04151633, + -0.050361477, + 0.03324028, + -0.04156169, + -0.0047065904, + 0.015786415, + -0.018793944, + -0.011535113, + 0.014380746, + -0.020314714, + 0.051470824, + 0.041934405, + -0.004867124, + 0.016092407, + -0.041343134, + -0.0069129877, + -0.013151445, + -0.013049579, + -0.014227563, + 0.012294243, + 0.023535972, + 0.09582767, + 0.028272409, + -0.0005980433, + -0.021114836, + -0.04161341, + -0.025066715, + 0.009171858, + 0.00085490016, + -0.004325211, + -0.08752578, + 0.023692263, + 0.047292396, + -0.022611242, + 0.0023633041, + 0.014524966, + 0.047580067, + -0.0060906825, + 0.034945868, + 0.014466022, + -0.004342099, + 0.016053481, + -0.027346697, + 0.033426613, + 0.0032153965, + -0.0326419, + -0.010161918, + 0.023981107, + 0.011759687, + -0.052357983, + -0.049272493, + 0.009173033, + -0.03018723, + -0.01216018, + 0.03862229, + 0.0058217705, + 0.028203972, + 0.008661315, + 0.014342987, + -0.027479144, + 0.050095517, + -0.0067602666, + 0.042617176, + 0.03748264, + -0.024025165, + -0.0105963815, + 0.022970585, + 0.011400019, + 0.020893743, + 0.01670328, + -0.031772003, + -0.032775916, + -0.03496673, + -0.02763227, + 0.055193584, + 0.006646619, + 0.06002397, + 0.004941399, + 0.069869846, + -0.03267484, + -0.014322522, + -0.039718542, + 0.022153044, + 0.009878189, + -0.008512045, + 0.005380223, + -0.0037464178, + 0.017440548, + -0.023305971, + 0.0014129513, + -0.037387133, + -0.015997346, + 0.011415185, + -0.003608959, + -0.025464239, + -0.049772594, + 0.031004852, + 0.03160308, + -0.03181969, + -0.027949853, + 0.015386628, + -0.01814867, + 0.0077540386, + -0.013038459, + 0.012069411, + 0.00242793, + -0.016248468, + -0.015009163, + -0.014200193, + -0.014048874, + 0.024323422, + -0.015429876, + 0.021983022, + 0.08122359, + -0.026295995, + 0.05619891, + -0.015841262, + 0.024909146, + -0.043610767, + -0.013050163, + 0.043327335, + 0.005485383, + 0.011066646, + 0.03884393, + -0.0126799345, + -0.010120975, + -0.017730972, + -0.0148830665, + 0.029461222, + -0.013040689, + 0.025476478, + 0.021090092, + 0.10558292, + 0.01759021, + 0.007119182, + -0.049821693, + -0.046769213, + 0.010347009, + 0.021941274, + -0.04051559, + -0.01927484, + -0.018153945, + 0.036063004, + -0.053634096, + 0.016225277, + 0.032013763, + 0.0011485299, + -0.00385527, + 0.035674285, + 0.0030259755, + 0.03468953, + 0.018085558, + -0.0042026206, + -0.028665258, + 0.008023036, + -0.030233528, + -0.009414713, + 0.036457594, + 0.06374129, + -0.006664929, + -0.059894215, + 0.00026309444, + -0.0396618, + -0.051116515, + 0.01965388, + 0.005524894, + -0.019100888, + 0.013425037, + 0.028955812, + 0.004788738, + -0.033643067, + -0.0011905001, + 0.024758512, + -0.00052775326, + -0.010942343, + -0.013593048, + -0.022199469, + 0.043095376, + -0.016460735, + -0.058189444, + 0.024324575, + 0.033460837, + 0.00091997714, + 0.0005526792, + -0.003412973, + -0.048690263, + -0.0006652781, + 0.024136553, + -0.021634357, + -0.024996545, + 0.032439858, + -0.010629394, + 0.0011244155, + -0.0018866806, + 0.014438396, + -0.012976634, + 0.0013983764, + -0.028118037, + -0.003589854, + -0.0031303647, + -0.053753555, + -0.000112081885, + 0.009855137, + 0.014191843, + 0.0025915462, + 0.012003662, + 0.028150441, + 0.00024264278, + -0.04815469, + -0.026557703, + 0.052870553, + 0.033308472, + 0.029652212, + -0.003279859, + 0.024673166, + -0.0000608255, + 0.03821482, + 0.0007719079, + 0.012723011, + 0.03597013, + 0.026845077, + -0.0046744426, + -0.023014449, + -0.031634502, + 0.0037222512, + 0.0074549234, + -0.031318016, + 0.004870171, + 0.022861281, + 0.024784494, + 0.03722386, + 0.025488298, + -0.03268014, + -0.03773066, + 0.05988057, + 0.018085541, + -0.011892267, + 0.012472536, + 0.021499056, + -0.010460912, + -0.0045256442, + -0.017528905, + -0.028855676, + -0.009394857, + -0.008491931, + -0.047462348, + 0.01623878, + -0.00065732456, + 0.10046201, + 0.006031555, + -0.025770394, + -0.034621473, + -0.041785292, + -0.22298211, + -0.027657676, + -0.0055687367, + -0.0016463703, + 0.0032147532, + -0.013572072, + -0.018296307, + 0.00619537, + -0.03921748, + 0.03624469, + -0.053076312, + -0.06879511, + 0.008643365, + -0.01162401, + -0.020896984, + 0.0039937855, + -0.009582055, + -0.008593449, + 0.017626638, + -0.030389361, + -0.010492854, + -0.018758906, + 0.01990847, + -0.018673385, + -0.0056663156, + -0.015410743, + 0.027686646, + 0.0230278, + -0.03787971, + -0.01909218, + 0.018614585, + 0.030845264, + -0.016363204, + 0.046644602, + 0.01298001, + 0.021839242, + 0.01863086, + -0.0149332415, + -0.027474482, + -0.014060433, + 0.032138884, + 0.008230394, + -0.01943439, + 0.033198934, + 0.02264309, + -0.04471804, + -0.013911614, + -0.03075178, + -0.00843806, + 0.00030229543, + -0.02719468, + 0.013804335, + 0.002973004, + -0.004314262, + -0.03043704, + -0.0101530515, + -0.013564636, + 0.041172102, + -0.06679724, + 0.0075961817, + -0.003268561, + 0.0021303666, + 0.014472877, + 0.042414848, + -0.037417658, + 0.010660612, + -0.026630878, + -0.022105068, + 0.032594047, + -0.023156894, + 0.030110354, + -0.0012049944, + 0.0011119137, + -0.011840344, + -0.023833811, + 0.05511959, + -0.014766379, + -0.03787408, + -0.018202012, + -0.15568705, + -0.0108330315, + -0.012499721, + 0.0074456776, + 0.0016342733, + -0.022488866, + 0.00053157413, + 0.0069126394, + 0.04626238, + 0.039863054, + 0.31233215, + 0.019702807, + -0.016050594, + -0.008386455, + 0.039074615, + -0.017606832, + 0.0279264, + 0.0209594, + 0.008622631, + -0.03914772, + -0.03409967, + 0.011517127, + 0.021138273, + 0.004391132, + 0.01496151, + 0.006964478, + -0.033282224, + 0.009002119, + 0.10120893, + -0.011206737, + 0.03272276, + 0.014611663, + 0.022227535, + 0.024138138, + -0.054192834, + -0.036819853, + -0.010146141, + 0.055919997, + -0.02148323, + 0.033010084, + -0.02682968, + 0.0202682, + 0.02907756, + -0.020800425, + -0.042358458, + -0.0014455209, + 0.012682776, + 0.020249736, + 0.0055078543, + 0.037016746, + -0.0007951263, + -0.041293636, + -0.003907819, + -0.0120204445, + -0.00038852115, + -0.041471574, + -0.0070202043, + -0.02247896, + -0.033046257, + -0.018758874, + -0.020645874, + 0.01578713, + -0.012863419, + -0.009799872, + -0.010523402, + -0.021958979, + -0.04957673, + 0.00993248, + -0.0009971148, + 0.02536897, + 0.027218077, + 0.01165897, + -0.04610551, + -0.013326195, + -0.032455396, + 0.019708922, + -0.028114228, + 0.003916275, + 0.034460917, + 0.012154007, + 0.0048028165, + -0.002589774, + 0.009817729, + 0.0017400552, + 0.04759754, + 0.010170037, + -0.0009970784, + 0.0815597, + -0.0068054446, + 0.027218128, + -0.05034498, + -0.0090685235, + -0.033899516, + 0.031764593, + 0.0066848183, + -0.023685874, + -0.013326043, + 0.058925085, + 0.006895783, + -0.029152904, + -0.0062609, + 0.014490204, + -0.037462544, + -0.0021607524, + 0.011518185, + 0.006835877, + 0.01101758, + -0.015258273, + -0.036358476, + -0.023013955, + -0.041106615, + -0.017471535, + 0.00838155, + 0.012001927, + 0.0011577477, + 0.0072658965, + -0.0054619387, + 0.013705055, + 0.024922866, + 0.013748787, + -0.027009577, + 0.03158007, + -0.023111923, + -0.017674724, + 0.046603307, + 0.010842266, + -0.011110754, + 0.029522715, + -0.005027195, + 0.036871634, + -0.018926008, + -0.025413318, + -0.0029353178, + -0.0015169409, + -0.004456729, + -0.0076581025, + 0.0042727143, + -0.041246872, + -0.030909147, + 0.02262676, + 0.027234312, + -0.007065173, + 0.032640286, + 0.012459629, + 0.036253173, + 0.025503077, + -0.015361158, + 0.017303808, + 0.021895839, + 0.036823317, + -0.0048003728, + -0.057099037, + -0.011843771, + 0.010936411, + -0.018080838, + -0.0066054408, + 0.018309772, + 0.00032028387, + -0.017378772, + 0.017051337, + 0.012971823, + 0.034987118, + -0.027274624, + -0.054023005, + -0.027901044, + -0.031608373, + -0.009159831, + -0.0044072974, + -0.024235515, + -0.040120564, + 0.018369986, + -0.0022678527, + 0.0020458044, + 0.055283643, + 0.03856778, + 0.045489512, + -0.033055056, + 0.03917027, + 0.039627768, + -0.04441088, + -0.0016231983, + -0.010896826, + -0.031776447, + 0.039041758, + 0.02630665, + -0.0066161742, + -0.06616487, + 0.0017684213, + 0.0094204135, + 0.01845482, + 0.025745336, + 0.028915057, + -0.03865387, + -0.005910827, + 0.004362902, + 0.024322152, + -0.033584032, + -0.02455527, + 0.031729285, + -0.030824281, + -0.012358568, + 0.12879676, + -0.008297697, + 0.00522678, + 0.0086258445, + 0.011583328, + 0.01291303, + 0.009949881, + -0.028948475, + -0.03124269, + -0.0028318823, + 0.017619096, + 0.0035665266, + -0.0318753, + 0.029551627, + 0.021020003, + -0.0033593746, + 0.028574554, + -0.040585835, + -0.03214685, + 0.006822504, + -0.0022928799, + -0.045394756, + 0.00399432, + 0.024731526, + -0.004223751, + 0.045102276, + 0.038220584, + -0.013665494, + -0.016390854, + 0.0017496143, + 0.013569971, + 0.029250426, + -0.056687776, + -0.019873006, + -0.0017188875, + -0.030701023, + -0.061374065, + 0.014416826, + 0.011616244, + 0.015073368, + 0.0043078936, + -0.0025857317, + -0.029518908, + 0.030545235, + -0.002164997, + -0.002333237, + 0.030432465, + 0.0025637543, + -0.019854153, + -0.006473501, + -0.02044877, + -0.010294013, + 0.032278363, + -0.019938396, + 0.0038226356, + -0.009503068, + -0.007875977, + 0.018689543, + 0.032742336, + 0.009960388, + 0.007617041, + 0.007991641, + -0.01916434, + -0.020999465, + 0.015537275, + 0.028557384, + -0.011846809, + -0.02671698, + -0.010500484, + -0.015498225, + 0.04179946, + -0.035539705, + -0.01380028, + -0.05009051, + -0.012779074, + 0.04109173, + -0.025095591, + -0.032688167, + -0.046982914, + 0.0021916514, + -0.01683173, + -0.062132053, + -0.049534976, + 0.020261973, + 0.009089888, + -0.05435491, + -0.00892915, + -0.006885028, + 0.019709637, + -0.010371847, + 0.020530857, + -0.025717309, + 0.010701369, + -0.034717612, + 0.0061711892, + 0.048294697, + -0.003218625, + -0.005463559, + -0.030664902, + 0.04921953, + 0.0056972154, + -0.030513095, + 0.01978153, + -0.043249365, + 0.027519284, + -0.015876373, + 0.027181003, + -0.022370415, + 0.014696035, + -0.026040304, + 0.054972973, + -0.05642878, + -0.013956571, + -0.01119077, + -0.0067104995, + -0.020211447, + -0.009450816, + 0.031919215, + 0.040190626, + 0.021542532, + -0.039795328, + 0.0021798364, + 0.004957992, + -0.004975847, + 0.0122616505, + 0.027804691, + -0.015406398, + -0.031613868, + 0.012831973, + -0.0011707208, + 0.00573203, + -0.015718859, + -0.010501462, + -0.023418529, + -0.0057758857, + -0.0018324098, + 0.01021788, + -0.00862143, + -0.01403362, + -0.017419642, + -0.008637872, + -0.011214128, + -0.011797591, + 0.04838804, + 0.0021515763, + -0.0034328625, + -0.026683375, + 0.020886362, + 0.009768898, + 0.015795585, + -0.006129563, + 0.032521326, + -0.049665477, + 0.01992473, + -0.043991335, + 0.00980291, + 0.0068182494, + -0.005357716, + 0.02177908, + 0.008345865, + 0.026487328, + -0.030146549, + -0.057835404, + -0.006276252, + -0.010188546, + -0.056068446, + 0.060911868, + 0.03939737, + 0.015559897, + -0.027221635, + 0.037332024, + 0.00263411, + 0.013403062, + -0.00709504, + 0.027968928, + 0.014239923, + -0.023934385, + 0.0071277735, + -0.012703395, + -0.025110636, + -0.0048651197, + -0.023703149, + -0.00046886047, + -0.016214145, + 0.044114057, + -0.006551933, + -0.0020285202, + 0.04789518, + 0.02677834, + -0.027145056, + -0.011045743, + 0.013919851, + -0.03862941, + 0.006649481, + -0.006009439, + -0.025457308, + -0.012320596, + 0.04507298, + 0.046811674, + -0.02278062, + -0.015837764, + -0.053562142, + 0.013716105, + -0.21232308, + 0.027377708, + 0.015237415, + -0.023696788, + -0.053059902, + -0.0048497403, + -0.0012029415, + -0.0025740752, + 0.0095053585, + -0.028121823, + -0.029722724, + 0.02410156, + 0.0033811326, + -0.0115124285, + -0.016918294, + 0.017344382, + -0.016989488, + -0.004097407, + 0.009463547, + 0.021733949, + 0.006350942, + -0.016134575, + 0.034421753, + 0.003166747, + 0.02340421, + -0.026406676, + -0.035132445, + 0.011517096, + -0.057925414, + -0.012114886, + 0.002623706, + 0.008359787, + 0.032079574, + 0.048067898, + -0.026415955, + -0.0070577194, + -0.0025138112, + -0.028991463, + 0.0034926417, + -0.022471918, + 0.00048272646, + 0.04406167, + -0.016007323, + -0.030257959, + -0.01306657, + 0.03230538, + -0.0020476526, + 0.018145435, + -0.067721315, + 0.018518573, + -0.003924516, + 0.020555887, + -0.014795289, + 0.0049425755, + 0.019946285, + -0.03044317, + -0.02176381, + 0.014313753, + 0.027226731, + 0.038249716, + 0.0022522663, + -0.016798135, + -0.026967835, + -0.05045351, + -0.05538728, + -0.011587675, + -0.08160623, + 0.002498869, + -0.0031001696, + 0.009852837, + -0.02343816, + -0.023971872, + -0.0013197197, + -0.048740678, + 0.018122587, + -0.019458776, + 0.0115204165, + -0.011321154, + -0.010407027, + -0.024524894, + -0.0047668735, + 0.01658018, + -0.04627936, + -0.027668912, + 0.044668753, + -0.00034803958, + -0.023848616, + 0.010030187, + -0.010201936, + 0.008736487, + -0.07072469, + 0.019416178, + 0.0034484728, + 0.022414, + -0.013794335, + 0.03284976, + -0.011824158, + 0.005631144, + -0.019680958, + -0.020323003, + -0.0581946, + -0.0040742606, + -0.0136233745, + 0.027204996, + -0.012762279, + -0.033287648, + 0.034173593, + -0.00880066, + 0.0061064577, + -0.006502162, + -0.035312783, + -0.0149945645, + -0.017808726, + 0.010815404, + -0.035570886, + 0.009373963, + 0.01757877, + -0.004480019, + -0.040352017, + 0.0041882596, + 0.0003310732, + 0.03206419, + -0.008982828, + -0.044378556, + -0.0028137348, + -0.0015513551, + 0.052353892, + -0.0027026949, + -0.008620424, + -0.014372762, + 0.048494678, + -0.024632718, + -0.011173045, + 0.040176053, + 0.009217481, + -0.01365318, + -0.023597803, + 0.04742853, + -0.01631914, + -0.05424809, + 0.007817665, + 0.0125553785, + 0.010496621, + -0.006806006, + -0.017928582, + 0.011123479, + 0.0033312964, + 0.018969359, + -0.051022816, + -0.014560318, + 0.038091965, + 0.010655592, + 0.0018253969, + 0.006524214, + 0.011940631, + 0.0061364826, + 0.020445539, + 0.02614197, + -0.023351867, + 0.061596878, + 0.03616273, + -0.039593622, + -0.033404756, + -0.04740529, + -0.0064965207, + 0.0058863456, + 0.021147681, + -0.0047315536, + -0.0024836697, + 0.0064693554, + -0.0011605037, + 0.053855263, + -0.028259253, + 0.0755302, + 0.014121064, + -0.052019443, + 0.041110065, + -0.004099468, + 0.04055596, + -0.01518072, + 0.029534336, + -0.021136653, + -0.00927845, + 0.024888488, + 0.0068626744, + 0.016657159, + 0.028724173, + 0.05139888, + 0.011571423, + 0.002082388, + 0.018599741, + -0.012507454, + -0.0047307564, + -0.04621104, + 0.039029904, + -0.007590576, + 0.02055962, + 0.024116034, + -0.03592551, + 0.062801845, + -0.009446577, + -0.034819316, + -0.044988595, + -0.012970966, + -0.021363825, + -0.009450572, + -0.034867194, + -0.027932325, + -0.031245608, + 0.042043567, + -0.031196898, + 0.012959328, + -0.030167965, + 0.005003574, + 0.007017798, + 0.002365822, + 0.009233833, + 0.022110129, + 0.022449927, + -0.029372638, + -0.01189245, + 0.013303611, + 0.0011072829, + -0.048085496, + -0.034649584, + -0.009778839, + -0.01498963, + -0.007846742, + -0.00442775, + -0.012414748, + -0.02464624, + 0.037342463, + -0.03872705, + 0.03345907, + 0.015978962, + 0.001075685, + 0.052416097, + 0.025024744, + -0.0042647086, + -0.0051121134, + 0.046542667, + -0.02771486, + 0.0046135895, + 0.029787239 + ], + "tools_centroid": [ + -0.057775438, + -0.0033558903, + -0.040774547, + 0.007837516, + -0.018325893, + -0.010864775, + 0.018047614, + -0.009721339, + -0.008646594, + 0.0075770593, + 0.008523997, + 0.015366553, + 0.004299319, + 0.025948813, + 0.014777593, + -0.017250892, + 0.017971335, + -0.013808822, + 0.013956967, + -0.03137046, + -0.01177912, + -0.01829785, + -0.01472029, + 0.04524385, + 0.017948175, + 0.043402486, + -0.034913003, + -0.030508569, + 0.036542602, + 0.04816589, + 0.0059453044, + -0.01865484, + 0.0013449497, + -0.053336088, + -0.033382863, + -0.030407647, + 0.02391551, + -0.0077984696, + -0.06443239, + -0.010297186, + -0.023779081, + 0.008694963, + 0.008222014, + -0.044142373, + 0.020442644, + -0.039328482, + 0.024845567, + -0.040081494, + 0.0007699972, + 0.0034618257, + -0.026518447, + -0.01582194, + 0.02163327, + -0.039542127, + 0.029000822, + 0.051123485, + -0.03359137, + 0.0023163594, + -0.05225703, + -0.00091889553, + -0.028958295, + -0.02035564, + -0.010448618, + 0.011662464, + 0.005890449, + 0.089658946, + 0.023167921, + -0.0010184674, + -0.0121378135, + -0.017610954, + -0.044071835, + 0.020530395, + -0.031366024, + -0.03172744, + -0.087409504, + 0.033778958, + 0.04312275, + -0.026120976, + -0.011753333, + -0.0060431096, + 0.050321087, + -0.00218206, + 0.01355621, + 0.0027208505, + 0.020778839, + 0.052400798, + -0.042935297, + 0.034534387, + -0.007350808, + -0.0046558166, + -0.027647814, + -0.0017963437, + 0.005831492, + -0.045641955, + -0.053029805, + -0.014840337, + -0.029358158, + 0.007700691, + 0.0351922, + 0.00765673, + 0.014760719, + 0.001616144, + 0.007946026, + -0.013469456, + 0.048162416, + 0.0026764928, + 0.03702904, + 0.018565005, + -0.026513232, + -0.008487164, + 0.0036733798, + 0.021371398, + 0.016962957, + 0.009267585, + -0.023463314, + -0.04331059, + -0.035165418, + -0.048778474, + 0.019885918, + -0.009031693, + 0.05094907, + -0.009540563, + 0.066760264, + -0.041867316, + -0.017069353, + -0.035217352, + 0.03588013, + 0.0549495, + 0.015968364, + 0.036352813, + 0.0016934925, + 0.034936693, + -0.008160922, + 0.011436721, + -0.037352208, + -0.02368231, + 0.0067209275, + 0.00545583, + -0.008353004, + -0.06703308, + 0.020577332, + 0.05729178, + -0.035366762, + -0.037864406, + 0.037184034, + -0.050321713, + 0.02234205, + 0.003926314, + 0.009195237, + -0.0055696885, + -0.019696044, + 0.001930507, + 0.009407629, + 0.00039543337, + 0.0012940381, + -0.015241957, + 0.006914032, + 0.056626543, + -0.021361178, + 0.06836428, + -0.0020655459, + 0.011145303, + -0.026055796, + -0.016941039, + 0.030385481, + -0.01505081, + 0.0076058493, + 0.000746044, + -0.0055449284, + -0.014022635, + -0.007175385, + -0.026965719, + 0.015261819, + -0.0125290435, + 0.026447913, + 0.027559392, + 0.094134904, + 0.024422089, + 0.010056614, + -0.0593088, + -0.046704147, + 0.020040825, + -0.008067564, + -0.008940185, + -0.035114467, + -0.0073384983, + 0.042855285, + -0.010223366, + 0.0085879695, + 0.014363037, + -0.0004808616, + 0.012251967, + 0.02859141, + 0.0047126776, + 0.03129005, + 0.017805913, + -0.002037126, + -0.021929229, + -0.024302894, + -0.03401745, + -0.02272678, + 0.03324402, + 0.043827765, + 0.0039511696, + -0.040313162, + -0.011220685, + -0.012239163, + -0.033538636, + 0.0074846055, + -0.04003989, + -0.012082418, + -0.0004583568, + 0.02797805, + 0.0015042631, + -0.040352985, + -0.0058471086, + 0.00686128, + 0.010597693, + -0.03193956, + -0.03259277, + -0.022284962, + 0.053651303, + 0.003222045, + -0.015034204, + 0.017189868, + -0.004987139, + 0.02370927, + 0.012949884, + -0.00048070893, + -0.021616403, + -0.024279464, + -0.0068444926, + -0.007384404, + -0.01851718, + 0.070953965, + -0.0016654494, + -0.013800572, + 0.0031709832, + 0.026937919, + 0.00003340502, + -0.025002202, + -0.022953335, + 0.0075816554, + -0.0036617126, + -0.04462196, + 0.018735254, + 0.04323663, + 0.033287965, + -0.017825564, + 0.004733342, + 0.0292739, + -0.01013516, + -0.043643888, + -0.004823276, + 0.03432248, + 0.032504197, + 0.028787872, + -0.012781166, + 0.012777502, + 0.0104851285, + 0.03479923, + 0.020120937, + 0.017253144, + 0.044453908, + 0.007472006, + -0.0061729704, + -0.024305446, + -0.0054376926, + 0.0074620508, + -0.0092536295, + -0.032345943, + 0.021577463, + 0.028651051, + 0.008056253, + 0.01772497, + 0.00647969, + -0.024510348, + -0.028680895, + 0.0586949, + 0.0232443, + -0.0075143017, + 0.022155466, + 0.010020021, + 0.003732729, + 0.0012835207, + 0.002107834, + -0.03460533, + -0.011155255, + -0.0087890215, + -0.04118182, + -0.020692606, + -0.008159101, + 0.10522082, + -0.0016338307, + -0.016076414, + -0.0016532582, + -0.035279144, + -0.22679776, + -0.010667053, + 0.008881526, + 0.03224856, + 0.006959963, + -0.005841684, + -0.0018540951, + 0.0055787065, + -0.041649945, + 0.023068914, + -0.035203893, + -0.07759581, + -0.02947163, + -0.0359074, + -0.0012353952, + -0.005708223, + -0.0065805116, + -0.0068721324, + 0.022740014, + -0.040421862, + -0.020221274, + -0.014032607, + 0.051545203, + -0.006344193, + -0.013821137, + -0.01579205, + 0.047883134, + 0.009514113, + -0.037617173, + -0.010565667, + 0.0038946962, + 0.007921387, + -0.004055107, + 0.032935105, + 0.012555967, + 0.032819428, + 0.0094431145, + -0.011137544, + -0.015564898, + -0.008363006, + 0.020149253, + 0.041235164, + -0.00003780061, + 0.037134502, + 0.041190896, + -0.035524506, + 0.014504791, + -0.02819106, + -0.016905108, + -0.008305034, + -0.0017854779, + -0.0047899806, + 0.0022640792, + -0.026646169, + -0.044333562, + -0.00081833766, + -0.03398577, + 0.039966717, + -0.031211361, + -0.0025227992, + -0.029431568, + -0.004133133, + 0.048822563, + 0.017931307, + -0.023236742, + 0.019759977, + 0.01688985, + -0.019182017, + 0.03889916, + -0.022565762, + 0.028835144, + 0.0038326585, + 0.0007537687, + -0.025940292, + -0.0089698145, + 0.047854897, + -0.028326858, + -0.03543255, + -0.0030444826, + -0.1869618, + -0.030654354, + 0.018738456, + 0.0010118893, + 0.014872937, + -0.035121404, + -0.006889633, + -0.0031736507, + 0.021199811, + 0.048273966, + 0.32182485, + 0.017583381, + -0.019531554, + 0.0092329085, + 0.052992057, + -0.015412301, + 0.0074604903, + 0.03680418, + -0.006491189, + -0.031820234, + -0.0012280992, + 0.0059963525, + 0.025414133, + -0.002702171, + -0.006588213, + 0.02243535, + -0.030782828, + -0.0007346572, + 0.102493934, + -0.003852306, + 0.031274583, + -0.003960667, + 0.007930573, + 0.025458938, + -0.051701877, + -0.04098843, + -0.010463472, + 0.05650782, + -0.026589196, + 0.04590178, + -0.014493388, + 0.01932283, + 0.024496134, + 0.0014021251, + -0.05392017, + -0.018444793, + 0.011652213, + -0.012777032, + -0.008817307, + 0.031348858, + -0.028693452, + -0.046828907, + 0.0014963904, + 0.026269333, + 0.005013643, + -0.039865334, + 0.017514601, + -0.0033667302, + -0.027785953, + -0.018094406, + -0.035160404, + -0.027459389, + -0.022849808, + 0.0028880779, + -0.0011864194, + -0.034878395, + -0.022470577, + -0.031891372, + -0.012203773, + 0.05001213, + 0.0072241533, + 0.010247018, + -0.06585563, + 0.029790338, + -0.0031036446, + 0.041342862, + -0.029685218, + -0.014217362, + 0.0132140145, + 0.015673228, + 0.014440125, + -0.009752704, + 0.005639572, + -0.01108167, + 0.05446453, + 0.019834101, + -0.021086873, + 0.062157847, + -0.019641414, + 0.045685045, + -0.024531566, + 0.010219521, + -0.04718473, + 0.02712037, + 0.023816736, + -0.017155042, + -0.013114721, + 0.02775809, + -0.027383637, + -0.040353194, + -0.008417484, + -0.0021451944, + -0.027928088, + 0.013438158, + 0.014820075, + 0.004232394, + 0.015032984, + -0.0037845736, + -0.038340595, + -0.016891716, + -0.03165935, + -0.018768659, + -0.0066177617, + -0.023367273, + 0.019635271, + -0.0003745473, + 0.000087627566, + 0.010894101, + 0.03014416, + -0.0042988, + -0.029685823, + 0.03274391, + -0.010986411, + -0.039936848, + 0.03304658, + 0.016807966, + 0.004531139, + 0.020389395, + 0.029960461, + 0.037048277, + -0.030802228, + -0.014076131, + -0.008956837, + 0.009107653, + -0.024870684, + -0.030200038, + 0.0005537656, + 0.008720928, + -0.055442058, + 0.043536436, + 0.018925853, + -0.019451551, + 0.016840005, + 0.010896255, + 0.012237941, + 0.006439829, + 0.0076054423, + 0.024631822, + -0.00469513, + 0.059032343, + -0.0082413, + -0.051315658, + -0.02167114, + 0.009919436, + -0.0045897025, + -0.0009727792, + 0.043319814, + -0.018224705, + -0.021532157, + 0.010909149, + 0.0041222232, + 0.049312256, + 0.02342331, + -0.06829633, + -0.015685972, + -0.033873446, + -0.016059933, + 0.012121991, + -0.014224916, + -0.03762836, + 0.015365398, + -0.004025226, + 0.024712248, + 0.07105101, + 0.025589744, + 0.013154655, + -0.0456072, + 0.02607114, + 0.025644207, + -0.033561457, + -0.0012643771, + -0.033391006, + -0.061123043, + 0.042876773, + 0.0020319296, + -0.020118402, + -0.043745127, + -0.00054004864, + -0.017112734, + 0.0075695347, + 0.047089994, + 0.013356315, + -0.04420688, + 0.012939495, + 0.015024913, + 0.033482503, + -0.0072687673, + -0.016575295, + 0.013684971, + -0.024929354, + -0.032714568, + 0.11408622, + 0.013711464, + 0.0007866529, + 0.010564399, + 0.011742405, + 0.008514497, + 0.028058255, + -0.009284165, + -0.018095477, + 0.0026275918, + -0.007345314, + 0.0042800684, + -0.040819786, + 0.0037819908, + -0.011583471, + -0.014982446, + 0.013995526, + -0.011253528, + -0.01841841, + 0.0035173576, + -0.016492022, + -0.0025003836, + -0.0034326876, + 0.044872776, + 0.0014514242, + 0.046815414, + 0.05207907, + -0.010015235, + -0.017120235, + -0.017186575, + -0.0038804088, + 0.053709168, + -0.067070924, + -0.0064274804, + 0.016569823, + -0.010965232, + -0.04720728, + 0.020261586, + 0.011407856, + 0.013676394, + 0.0033044717, + 0.00720917, + -0.032467864, + 0.035398927, + -0.029180987, + 0.008299155, + 0.0403421, + 0.002068336, + -0.031769827, + 0.023002677, + -0.017297465, + -0.0028395427, + 0.03669844, + -0.006664068, + 0.055516694, + -0.02332954, + 0.0128885275, + 0.029991373, + 0.03301732, + -0.0143478615, + 0.001786697, + 0.0061424426, + -0.013547873, + -0.01829034, + 0.032445464, + -0.004602294, + -0.0024687257, + -0.02234211, + -0.0072758887, + -0.019191276, + 0.026224066, + -0.041807573, + -0.0022833985, + -0.027213147, + -0.02427904, + 0.03531603, + -0.030026786, + -0.02400627, + -0.06544794, + -0.013835978, + -0.009932556, + -0.05266321, + -0.01314315, + 0.037983123, + 0.014741308, + -0.037383366, + 0.0063219173, + 0.02204942, + 0.025447313, + -0.025008852, + 0.012261236, + 0.0010507479, + 0.033279635, + -0.032050498, + 0.023104839, + 0.0079582995, + -0.026157726, + 0.004492358, + -0.04235163, + 0.022636527, + 0.0039217533, + -0.038830105, + 0.03416724, + -0.039616987, + 0.013216047, + -0.005373113, + 0.014587253, + -0.025387455, + 0.018838875, + -0.023695769, + 0.02310995, + -0.054573603, + -0.009373536, + -0.017116925, + 0.014891645, + -0.024939645, + 0.004483685, + 0.04446569, + 0.04414556, + 0.036651935, + -0.026196282, + -0.0068070013, + 0.04016786, + 0.014043489, + -0.0003690823, + 0.058535554, + 0.0032685478, + -0.014644685, + 0.0026333602, + -0.00085332064, + 0.0049753827, + -0.0020620888, + -0.0150953755, + -0.013835361, + -0.004500461, + -0.016988168, + 0.003694431, + -0.021504221, + -0.025054544, + -0.025284309, + -0.021449776, + -0.002113436, + -0.019090962, + 0.0075706523, + -0.003012929, + 0.009065395, + -0.007116615, + 0.044901505, + 0.0027590708, + 0.009059467, + -0.012410826, + 0.03301639, + -0.029811352, + 0.01738935, + -0.021565853, + 0.0030311234, + -0.01219218, + -0.008524064, + 0.017151799, + 0.00995471, + 0.029735599, + -0.034349315, + -0.06940458, + 0.0036188683, + -0.0054800794, + -0.048410185, + 0.008675395, + 0.025768118, + 0.010540176, + -0.020408748, + 0.038225047, + -0.00551114, + 0.01753059, + 0.026808819, + 0.010108354, + 0.016693898, + 0.009826699, + 0.014569398, + -0.01573024, + -0.008260742, + -0.011886305, + -0.012466746, + 0.00095788017, + -0.016167117, + 0.038578525, + -0.013100995, + -0.01676035, + 0.03332717, + 0.005415544, + -0.010420641, + -0.03113651, + -0.027528936, + -0.021406338, + 0.023781726, + -0.0063788826, + -0.0077266875, + -0.014590149, + 0.040968418, + 0.026437076, + -0.03645595, + -0.009789084, + -0.046371892, + 0.021632947, + -0.20988819, + 0.023208335, + 0.009895576, + -0.013134811, + -0.05835602, + 0.007831426, + -0.012073989, + -0.018234992, + 0.009569847, + -0.038773432, + -0.044383198, + 0.0089479145, + 0.025667464, + -0.02950146, + 0.016152902, + 0.023348661, + 0.013435106, + 0.0112070935, + 0.007683514, + 0.03000217, + -0.014931629, + -0.03128683, + 0.0018451557, + 0.026329964, + 0.0072926055, + -0.0108091775, + -0.010943898, + 0.007525587, + -0.04637749, + -0.0012201949, + 0.014439996, + -0.016553767, + 0.035915114, + 0.064322315, + -0.025222458, + -0.0086920345, + 0.01354625, + -0.031556293, + -0.021044917, + -0.01556811, + -0.011967994, + 0.034725036, + -0.017628316, + -0.011426438, + 0.0155166965, + 0.0345392, + -0.005978086, + 0.001560346, + -0.057566527, + 0.014015094, + -0.005247297, + 0.044353653, + -0.032477822, + 0.00088942435, + 0.02304252, + -0.017082589, + -0.04723853, + 0.025155952, + -0.009886005, + 0.030909168, + -0.0148722185, + 0.004657267, + -0.036441028, + -0.07425148, + -0.04521448, + -0.031231904, + -0.083747715, + 0.012393575, + 0.0046909004, + 0.030838598, + -0.017930364, + -0.03798974, + 0.0046308283, + -0.024992201, + 0.01813419, + -0.01783272, + 0.013735788, + -0.00084009505, + -0.026482312, + -0.029216949, + -0.03189472, + 0.0120601095, + -0.058092535, + 0.007142014, + 0.060608845, + -0.008147901, + -0.003151759, + 0.0046359287, + -0.017588917, + 0.0092833545, + -0.053699553, + -0.0046048244, + 0.0012410878, + 0.021804376, + -0.012375213, + 0.013280399, + -0.031906895, + -0.0035128666, + -0.0140839275, + 0.0052240947, + -0.019663228, + -0.0015391115, + 0.001982028, + 0.014376489, + -0.011715944, + -0.009479002, + 0.06390848, + -0.021833453, + 0.017518744, + -0.015481537, + -0.032284454, + -0.0023388504, + -0.01498571, + 0.005026777, + -0.04676246, + -0.0042618783, + 0.027863752, + -0.021421071, + -0.030720329, + -0.019202845, + -0.0036962088, + 0.01606846, + 0.0050927107, + -0.042860158, + 0.003974965, + 0.0009248986, + 0.037120704, + 0.004218136, + -0.0036327865, + -0.018174114, + 0.03935176, + -0.03224347, + -0.012047191, + 0.02956622, + 0.031379495, + -0.012442746, + -0.014872591, + 0.06750819, + -0.020884288, + -0.054731537, + -0.02460563, + 0.003977729, + 0.0004902187, + -0.019478966, + 0.00022061134, + -0.013216427, + -0.0037307949, + -0.011919203, + -0.042946137, + -0.028380334, + 0.05078273, + -0.0030087964, + 0.01560839, + 0.017069735, + 0.0016844464, + 0.004942494, + -0.014320324, + 0.02391562, + -0.009867219, + 0.051671702, + 0.027323583, + -0.035621997, + -0.014997956, + -0.027416717, + 0.01103468, + 0.019373719, + 0.0009110808, + -0.016991789, + -0.03706896, + 0.0037041882, + -0.0022370599, + 0.030673143, + -0.03202175, + 0.056521207, + 0.02383771, + -0.04783513, + 0.04099715, + 0.013725404, + 0.06307572, + -0.0024652113, + 0.023774121, + -0.012859268, + 0.018181501, + 0.008533877, + 0.012162901, + 0.019225126, + 0.026711285, + 0.033790324, + 0.016099503, + 0.01810887, + 0.01823027, + -0.003407086, + 0.0019508481, + -0.019111417, + 0.037072737, + 0.0043255636, + 0.014085034, + 0.023889331, + -0.026697278, + 0.06311214, + -0.020659966, + -0.034266055, + -0.035501692, + -0.025569465, + -0.029216953, + -0.010694965, + -0.02754278, + -0.027895218, + -0.032571055, + 0.04116445, + -0.029515225, + 0.0061112135, + -0.025661161, + 0.0071192123, + -0.004901541, + 0.01863631, + 0.025625108, + 0.029897802, + 0.008398415, + 0.009981635, + 0.00594864, + 0.025863202, + 0.012255268, + -0.0479219, + -0.016847553, + -0.030781547, + 0.00071642187, + -0.007623827, + 0.003929538, + -0.011301359, + -0.022604315, + 0.01118298, + -0.04312297, + 0.034218777, + 0.025586674, + -0.0024720372, + 0.03698685, + 0.03409089, + -0.021177674, + -0.022660092, + 0.033316627, + -0.016247125, + 0.0072186943, + 0.016868103 + ], + "abstain_centroid": [ + -0.031148378, + 0.005337721, + -0.051859938, + 0.006701708, + -0.013760792, + -0.046238888, + -0.001741801, + 0.003761064, + 0.0115341535, + 0.015631316, + -0.007558447, + 0.042414144, + -0.008460059, + -0.003942813, + 0.014397727, + -0.000967998, + 0.0059690275, + -0.016228953, + -0.00661065, + -0.016335107, + -0.017304765, + -0.015375099, + 0.00016670206, + 0.02917791, + 0.019667024, + 0.03706363, + -0.014033229, + -0.017078554, + 0.022332542, + 0.0029681036, + -0.026012667, + 0.0047733886, + -0.008349954, + -0.025975645, + -0.057982, + -0.037145726, + 0.022042587, + -0.010432212, + -0.054713663, + -0.018552495, + 0.014079261, + 0.0066314046, + 0.027429352, + -0.026314953, + 0.021749051, + -0.05777429, + -0.011509256, + -0.020935455, + -0.00084090995, + -0.031351306, + -0.021382077, + -0.00847218, + 0.03660322, + -0.013143199, + 0.047227796, + 0.05580892, + -0.010979107, + -0.02130941, + -0.08700673, + 0.0018242795, + -0.025122127, + -0.013786088, + -0.009168319, + 0.011006979, + 0.0012779797, + 0.09556672, + 0.00808916, + -0.008711008, + -0.008361359, + -0.034987625, + -0.03647069, + 0.021719662, + -0.032498717, + -0.01776087, + -0.0698433, + 0.011626705, + 0.020904263, + -0.008985529, + 0.0070378096, + 0.037491325, + 0.08140702, + -0.015960723, + 0.00758235, + 0.018223334, + 0.011250971, + 0.04139314, + -0.030075394, + 0.011359265, + -0.0026697265, + 0.0045864326, + -0.029182356, + -0.018836327, + 0.035417307, + -0.050186694, + -0.0169077, + 0.0030452074, + -0.031451467, + 0.021025097, + 0.03237884, + -0.017753405, + 0.024965163, + 0.0068491264, + -0.0007417426, + -0.025353186, + 0.014833976, + 0.00048042086, + 0.033970177, + 0.029662795, + 0.001964002, + -0.00040594727, + 0.0462047, + 0.033295035, + 0.014799662, + 0.004400711, + -0.012687833, + -0.048856415, + -0.021582829, + -0.029335681, + 0.031864036, + -0.013479058, + 0.0061972486, + 0.019316947, + 0.039016202, + -0.0460622, + -0.011819415, + -0.035461947, + 0.030371841, + 0.045124568, + -0.006952659, + 0.026146563, + 0.015358209, + 0.014641611, + -0.03725902, + 0.0049006664, + -0.012360949, + -0.030733725, + 0.016830463, + -0.0022114269, + 0.017974649, + -0.051037353, + 0.024567815, + 0.029570594, + -0.028185142, + -0.028654277, + 0.021437967, + -0.03857855, + 0.024237394, + -0.0068540815, + 0.018478565, + -0.012613434, + 0.0043713585, + -0.0101159075, + 0.0065389522, + -0.014050798, + 0.0061866357, + -0.017707681, + -0.019518772, + 0.053408965, + -0.013017107, + 0.016827421, + 0.022524578, + -0.0068964125, + -0.052380078, + 0.0003833916, + 0.029090617, + -0.011927651, + 0.024243768, + 0.013390409, + -0.014234686, + -0.0017058246, + -0.012792837, + -0.026187133, + 0.0398575, + -0.0025493933, + 0.030229095, + 0.026641972, + 0.07534804, + 0.022857832, + 0.010506174, + -0.058816746, + -0.051879656, + 0.016220875, + 0.017665846, + -0.054791577, + -0.039248586, + 0.014590265, + 0.021849325, + -0.018102316, + 0.015241521, + 0.021297466, + -0.022830576, + -0.016140226, + 0.027246347, + -0.0054256297, + 0.046256945, + -0.007986689, + 0.026465587, + -0.0026759568, + -0.0096074, + -0.037014082, + -0.048065815, + 0.0330576, + 0.045262605, + 0.0051544863, + -0.0342738, + -0.01125588, + -0.03914742, + -0.03194088, + 0.0144135505, + -0.017719291, + 0.009187744, + 0.0078019383, + 0.02771336, + 0.0051388913, + -0.0420357, + 0.0072669606, + -0.002695001, + 0.0030906205, + 0.004021662, + -0.021064727, + 0.0009345196, + 0.029135603, + 0.002100572, + -0.019682301, + 0.04653866, + 0.032432303, + 0.017765505, + 0.015490212, + -0.009414901, + -0.023563534, + -0.008533622, + 0.00501602, + -0.019913116, + -0.0035469246, + 0.036062475, + -0.022427976, + -0.040446255, + 0.015986245, + 0.009162775, + -0.0003249781, + -0.022688588, + -0.030663736, + 0.0018985457, + 0.024509203, + -0.046458673, + 0.021605283, + -0.0032775383, + 0.008400841, + 0.005384018, + -0.0020570226, + 0.018701846, + -0.012248811, + -0.04532812, + 0.0036800993, + 0.032931983, + 0.028872136, + 0.012167975, + -0.0052398657, + 0.026528697, + 0.013797996, + 0.023905411, + 0.031488698, + 0.01613529, + 0.03790394, + 0.007559894, + 0.0036375439, + -0.033904057, + -0.0027450048, + 0.0044508814, + 0.010002826, + -0.030975314, + -0.016194653, + -0.010303318, + 0.0017212449, + -0.004217246, + -0.00038384096, + -0.028139843, + -0.037131034, + 0.059932988, + 0.016013877, + 0.0041002976, + 0.017807445, + 0.012730673, + -0.009848034, + 0.00704557, + -0.0010963117, + -0.017336579, + 0.0052566174, + 0.006595981, + -0.045635466, + 0.012577329, + -0.0151054, + 0.088495515, + -0.0060461042, + -0.01455267, + -0.010428576, + -0.03148846, + -0.24492498, + -0.022456426, + -0.012005429, + 0.018366225, + 0.01226593, + -0.024266042, + -0.033916228, + -0.0036076312, + -0.054510105, + 0.00632446, + -0.015560582, + -0.05587817, + -0.020939475, + -0.0053286967, + -0.0070734383, + 0.004897013, + 0.006918987, + -0.022256982, + 0.0075242054, + -0.06765787, + -0.023333376, + -0.02501722, + 0.07195793, + -0.0196174, + -0.005269894, + 0.00026103153, + 0.035949823, + 0.0041076643, + -0.037599213, + -0.01236279, + -0.006176068, + 0.0069308914, + -0.0029138543, + 0.035269044, + 0.014236289, + 0.034278776, + 0.010570882, + -0.008682466, + -0.014200495, + -0.0010429571, + 0.04505355, + 0.020335147, + -0.003972119, + 0.011877002, + 0.010973979, + -0.04002932, + -0.021862812, + -0.010063818, + -0.01741567, + -0.02046978, + 0.0024775756, + 0.005379279, + -0.0035171222, + -0.02375287, + -0.027805964, + 0.006844447, + 0.0020820745, + 0.04806291, + -0.013278907, + -0.011143366, + -0.015697205, + -0.010483088, + 0.008951397, + 0.03141075, + -0.021271285, + 0.006024506, + -0.005130949, + -0.013589776, + 0.028885502, + -0.019157536, + 0.033051424, + -0.011037691, + 0.008256805, + -0.021247163, + 0.014382734, + 0.02527936, + -0.025481075, + -0.03218864, + -0.02366406, + -0.18205336, + -0.016931955, + -0.000928319, + 0.0030238247, + 0.033743773, + -0.016215391, + -0.025330722, + 0.00043090724, + 0.052764837, + 0.045047108, + 0.34483734, + 0.021190971, + 0.029025026, + -0.029802188, + 0.05892975, + -0.027338214, + 0.017156813, + 0.016491018, + 0.005376105, + -0.036032055, + -0.016050696, + -0.019247439, + 0.008546482, + 0.0006579195, + 0.008216663, + 0.012412621, + -0.049241338, + 0.00468357, + 0.083209105, + 0.004403217, + 0.015972426, + -0.03518827, + 0.0025444014, + 0.027684696, + -0.03703065, + -0.040941797, + -0.022856751, + 0.07734354, + -0.017490434, + 0.029937679, + -0.030589428, + -0.01312233, + 0.028537748, + -0.01580879, + -0.031040978, + 0.0022384818, + 0.004584949, + -0.036500685, + -0.013998974, + 0.019088954, + -0.020744042, + -0.02740597, + 0.02783206, + 0.00093145366, + 0.021530958, + -0.015619698, + -0.0065967944, + -0.008945381, + -0.022806387, + -0.0007809315, + -0.019146752, + -0.021306394, + -0.024635077, + -0.0013178901, + 0.030522598, + -0.036964312, + -0.034459814, + -0.026010964, + 0.011097982, + 0.044302322, + 0.033004954, + 0.012737288, + -0.02832643, + -0.033423617, + -0.011146303, + 0.018307667, + -0.010493131, + -0.0023452898, + -0.006297813, + 0.023474587, + 0.015160515, + 0.008065753, + 0.0071064318, + -0.012013574, + 0.034734786, + 0.02027884, + 0.013851284, + 0.07314595, + 0.0051676924, + 0.03851032, + -0.011801342, + -0.024405107, + -0.05664373, + -0.017895719, + 0.010395462, + 0.0045732786, + -0.017845891, + 0.05219816, + -0.007843896, + -0.039979137, + 0.0115565825, + 0.019499846, + -0.03996184, + -0.007970878, + -0.0029153195, + -0.051025048, + 0.023902694, + -0.024392456, + -0.019463966, + -0.02171224, + -0.02825411, + -0.022925304, + 0.008098716, + -0.009423168, + -0.0034888275, + -0.026081651, + 0.005871837, + 0.012862886, + 0.020583529, + 0.015732484, + 0.004980412, + 0.007757627, + -0.018226098, + -0.036115993, + 0.025706174, + 0.049908377, + 0.025328659, + 0.009017575, + 0.020332497, + 0.040973037, + -0.016721342, + -0.01928604, + -0.012567931, + 0.0024534024, + -0.027978517, + -0.016283054, + -0.0059162932, + 0.020438462, + -0.033998214, + 0.025316799, + 0.015457932, + -0.027260441, + 0.007751793, + 0.03414886, + 0.06149144, + -0.009142277, + 0.034868557, + 0.015628273, + -0.011573751, + 0.04880984, + -0.008835818, + -0.042271934, + -0.011501361, + -0.015698932, + 0.00765774, + 0.0040898444, + 0.017931513, + 0.012534788, + -0.026830954, + 0.009097794, + 0.008690346, + 0.044174466, + 0.006063397, + -0.05672009, + -0.01996871, + -0.035708513, + -0.010680867, + -0.003172045, + -0.011108843, + -0.029465709, + 0.012013625, + 0.014341203, + 0.052498966, + 0.13476692, + 0.046210233, + 0.052687794, + -0.006100141, + 0.045737743, + 0.043751344, + -0.03214318, + 0.002818939, + -0.035125505, + -0.04272792, + 0.031032866, + 0.03460889, + -0.020154648, + -0.025992718, + 0.0040032137, + -0.013573707, + 0.0014319279, + 0.033563722, + -0.012486501, + -0.013652725, + -0.00095753156, + 0.018106168, + -0.019855192, + 0.009851039, + -0.0179031, + 0.013870335, + -0.022614986, + -0.025842723, + 0.12371204, + -0.01614459, + -0.0011982421, + 0.06412819, + 0.005470515, + 0.013032822, + 0.031510778, + -0.0073665283, + -0.032478217, + 0.0004560572, + -0.018492013, + 0.009834454, + -0.025878755, + 0.011205347, + -0.012217907, + 0.0048979158, + 0.015269925, + -0.022450786, + -0.0018926285, + 0.010645847, + 0.007540697, + -0.018376745, + 0.0011163221, + -0.00910863, + -0.0052845264, + 0.027602881, + 0.02068656, + -0.028491242, + -0.0168644, + 0.004427309, + 0.0062007783, + 0.047445305, + -0.06059543, + -0.023386633, + 0.031225724, + -0.015070878, + -0.030429764, + -0.0027328797, + 0.01173457, + 0.0085614575, + -0.015814753, + -0.010190623, + -0.014055305, + 0.01249472, + 0.0020188026, + -0.018316, + 0.017405652, + 0.0033323495, + -0.0058355783, + 0.019034537, + -0.017335443, + 0.009071962, + 0.060741838, + -0.019548608, + 0.0028466566, + -0.000989737, + -0.013918978, + 0.039289407, + 0.0338373, + -0.020116925, + -0.0049318955, + 0.021821503, + 0.008678599, + -0.0052154674, + -0.00354734, + 0.005850025, + -0.00011736599, + -0.038427465, + 0.0046045645, + -0.017376445, + 0.03195163, + -0.037625898, + -0.0002592167, + -0.045117147, + -0.021053702, + 0.01866349, + -0.018683132, + -0.022075271, + -0.067146465, + -0.00034859247, + -0.0066958456, + -0.019604217, + -0.025007758, + 0.0084634265, + 0.01903175, + -0.036018692, + 0.00021994986, + -0.01454107, + 0.036363024, + -0.024418773, + 0.03628282, + -0.015224951, + 0.035957087, + -0.040158544, + 0.03579523, + 0.028295236, + 0.011141477, + -0.0144408615, + -0.032107454, + 0.012157545, + -0.0051779617, + -0.056817777, + 0.017691938, + -0.036631953, + 0.038909055, + -0.0097737955, + 0.006345886, + -0.024140004, + 0.018522153, + -0.0378227, + 0.009578492, + -0.050063416, + -0.0136081185, + -0.019632932, + 0.0028407518, + -0.00645373, + -0.016191104, + 0.043725938, + 0.0236164, + 0.027719086, + -0.041044917, + 0.025428394, + 0.020110024, + -0.0060314476, + -0.0061647817, + 0.07147246, + -0.022163881, + -0.002495511, + 0.017374914, + 0.024532678, + 0.017868279, + -0.0123060895, + 0.009035884, + -0.02307739, + -0.00059325853, + -0.02122783, + 0.0071837865, + -0.036735952, + -0.014673909, + -0.028121514, + 0.010137442, + -0.010372103, + -0.03829866, + 0.012825123, + -0.01134884, + 0.022564255, + -0.031195192, + 0.049415413, + -0.037152834, + -0.0024479814, + 0.023421837, + 0.064130165, + -0.020511432, + 0.011314599, + -0.038539167, + 0.015962925, + -0.0053321286, + 0.01771349, + 0.016681707, + 0.020176098, + 0.045899525, + -0.03644215, + -0.05389872, + 0.03137194, + 0.008404088, + -0.038138144, + 0.014195224, + 0.04344335, + 0.0057477932, + -0.032771025, + 0.048310746, + -0.021211635, + 0.030088607, + -0.0025103104, + 0.012287962, + -0.004873214, + -0.008176674, + -0.014280841, + -0.0006814086, + -0.019928552, + 0.008787194, + 0.010925872, + -0.0032950703, + -0.034151934, + 0.023929175, + -0.010394337, + -0.010170748, + 0.029074438, + 0.0061191223, + -0.023697814, + -0.021662993, + -0.034633745, + -0.037640702, + -0.008391389, + -0.010877962, + -0.0082226675, + 0.00087190676, + 0.07043431, + 0.021459375, + -0.050167535, + -0.02395419, + -0.04226849, + 0.023713028, + -0.22955438, + 0.035205133, + -0.00219514, + -0.002945837, + -0.055697378, + 0.00528513, + -0.008466735, + -0.00713325, + 0.006396225, + -0.03216339, + -0.024553273, + 0.028201995, + -0.000054232205, + -0.018851798, + -0.007118189, + 0.040054675, + -0.03188417, + -0.0030668844, + -0.004163999, + 0.032002646, + 0.0047784965, + -0.025143709, + -0.005436027, + 0.0143850455, + 0.018197645, + 0.0026216004, + -0.026822513, + -0.013488665, + -0.044493284, + -0.01340355, + 0.0006943544, + -0.031944275, + 0.021567853, + 0.043572217, + -0.008895598, + 0.017365277, + 0.00690336, + -0.014118812, + -0.017342426, + 0.013759936, + 0.0014441375, + 0.019967057, + -0.023214184, + -0.0035139658, + -0.000988701, + 0.028011117, + -0.019943252, + -0.008950247, + -0.059093565, + -0.0024248778, + 0.013076103, + 0.026632654, + -0.018246919, + 0.019110702, + 0.012685868, + 0.0018900798, + -0.051872082, + 0.009624938, + -0.0076693483, + 0.037904374, + 0.0033752439, + 0.012424309, + -0.031966437, + -0.031686325, + -0.04591932, + -0.023986965, + -0.08195933, + 0.013067175, + 0.013707989, + 0.028453248, + -0.0228147, + -0.036631793, + -0.00467004, + -0.014707499, + 0.025149882, + 0.015747227, + 0.011062073, + -0.007982165, + -0.044059478, + -0.005221822, + -0.029661845, + -0.0010427514, + -0.052402854, + 0.010401697, + 0.012599077, + 0.01889836, + -0.024228005, + 0.012707993, + -0.017794892, + -0.007308048, + -0.057065304, + -0.024420062, + -0.00044011022, + 0.005719825, + -0.04850662, + 0.009310803, + -0.027014088, + -0.0128755625, + -0.019028025, + -0.021750353, + -0.039091866, + 0.007929615, + -0.014110442, + -0.0059228754, + -0.027193954, + -0.008130775, + 0.052156363, + -0.009849598, + 0.026919808, + 0.021327212, + -0.013137314, + 0.0056819157, + -0.010384433, + 0.0052230777, + -0.02551496, + -0.023010258, + -0.009133841, + -0.0034335835, + -0.0066825, + -0.012148635, + -0.014457139, + 0.034450416, + -0.002119914, + -0.042140137, + 0.010768989, + 0.010510174, + 0.03508211, + 0.008629587, + 0.007369357, + -0.011705995, + 0.046988714, + -0.036072724, + 0.0013155584, + 0.028497273, + 0.0201227, + 0.0019115483, + -0.017506052, + 0.05194845, + -0.029103, + -0.05979132, + -0.01251193, + -0.0045603164, + -0.0011945973, + -0.02777315, + -0.03253321, + -0.010601675, + 0.008287072, + 0.010776673, + -0.057129562, + -0.0332778, + 0.031840086, + 0.025541494, + 0.021605106, + -0.006681296, + 0.002822149, + 0.019710355, + -0.005794384, + 0.0026962264, + 0.006601597, + 0.044709172, + 0.03829154, + -0.046349104, + -0.021133212, + -0.024994686, + -0.010804927, + 0.0067239436, + 0.0014897367, + -0.011834818, + -0.023318285, + -0.0038526887, + 0.008018549, + 0.047896657, + -0.030216228, + 0.0534178, + 0.038851894, + -0.009745995, + 0.035945363, + 0.041955516, + 0.035383813, + -0.0023587283, + 0.033687126, + 0.00009861278, + 0.023677062, + 0.024938073, + -0.016926885, + 0.0016545693, + 0.002132915, + 0.041455563, + -0.0030341032, + 0.026448688, + -0.013949449, + -0.0066818777, + 0.002497779, + -0.052358508, + 0.058462907, + 0.017483938, + 0.016702253, + 0.042070094, + -0.020453831, + 0.04187879, + -0.011923706, + -0.021604251, + -0.010823228, + -0.015712183, + -0.008566785, + -0.00082078506, + -0.022313718, + -0.03011328, + -0.042152412, + 0.031767584, + -0.031519245, + -0.011423306, + -0.011847542, + 0.022948265, + 0.004512841, + 0.0025419195, + 0.022916336, + 0.020818464, + 0.017587373, + 0.00058030424, + -0.016511247, + 0.0094060805, + -0.015658183, + -0.035970535, + 0.01030802, + -0.027739279, + -0.022434799, + -0.0031821248, + -0.011698124, + -0.014282087, + 0.010802368, + 0.011209723, + -0.03759782, + 0.06280771, + 0.030460313, + -0.014077967, + 0.05204396, + 0.03405556, + 0.006860819, + -0.02474166, + 0.05734323, + -0.019578138, + -0.018592859, + -0.0055962717 + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-3/router_report.json b/benchmarks/models/router/centroid/out-router-iter-3/router_report.json new file mode 100644 index 0000000..7cc8ccc --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-3/router_report.json @@ -0,0 +1,421 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 140, + "accuracy": 0.5214286, + "tool_precision": 1.0, + "tool_recall": 0.1875, + "memory_false_tool_rate": 0.0, + "tool_count": 160, + "memory_count": 90, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_001", + "text": "what processes are listening on port 8003?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_010", + "text": "what's the uptime since klbr daemon last started", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_011", + "text": "how big is agent.db right now on disk?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_002", + "text": "which file defines the ClientMsg type?", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_006", + "text": "config.rsのmemory_support_score_gapのデフォルト値はいくつ?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_010", + "text": "what's in the .gitignore for klbr?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_016", + "text": "read interrupt.rs and tell me what the Interrupt enum contains", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "rf_test_031", + "text": "read the first 10 lines of agent.rs", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "wf_test_001", + "text": "set the compaction_keep to 15 in config.rs", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "wf_test_008", + "text": "patch the memory_top_k to 5 in config.rs", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "abstain" + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory" + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory" + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory" + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-3/router_report.md b/benchmarks/models/router/centroid/out-router-iter-3/router_report.md new file mode 100644 index 0000000..f09e975 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-3/router_report.md @@ -0,0 +1,88 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `140` +- tools: `160` memory: `90` abstain: `15` +- accuracy: `0.5214` +- tools precision: `1.0000` +- tools recall: `0.1875` +- memory false-tools rate: `0.0000` + +## Misclassified Queries + +| id | text | gold | pred | +|---|---|---|---| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | +| `shell_test_001` | what processes are listening on port 8003? | `Tools` | `Abstain` | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Abstain` | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Abstain` | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | +| `shell_test_010` | what's the uptime since klbr daemon last started | `Tools` | `Memory` | +| `shell_test_011` | how big is agent.db right now on disk? | `Tools` | `Memory` | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Abstain` | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Abstain` | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | +| `rf_test_002` | which file defines the ClientMsg type? | `Tools` | `Abstain` | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | +| `rf_test_006` | config.rsのmemory_support_score_gapのデフォルト値はいくつ? | `Tools` | `Memory` | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | +| `rf_test_010` | what's in the .gitignore for klbr? | `Tools` | `Memory` | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | +| `rf_test_016` | read interrupt.rs and tell me what the Interrupt enum contains | `Tools` | `Memory` | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | +| `rf_test_031` | read the first 10 lines of agent.rs | `Tools` | `Abstain` | +| `wf_test_001` | set the compaction_keep to 15 in config.rs | `Tools` | `Memory` | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Abstain` | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Abstain` | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | +| `wf_test_008` | patch the memory_top_k to 5 in config.rs | `Tools` | `Memory` | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Abstain` | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | diff --git a/benchmarks/models/router/centroid/out-router-iter-4/router_model.json b/benchmarks/models/router/centroid/out-router-iter-4/router_model.json new file mode 100644 index 0000000..f269791 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-4/router_model.json @@ -0,0 +1,3084 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroid": [ + -0.029683147, + -0.024364203, + -0.057413984, + 0.0060161366, + 0.007102006, + -0.049074806, + 0.013821386, + 0.0036667658, + 0.013479514, + 0.026250292, + -0.0007516774, + 0.0270816, + 0.00090493786, + 0.01856444, + 0.010187499, + -0.017102296, + 0.037264716, + -0.034918193, + 0.032313354, + -0.0058548693, + -0.015844207, + -0.014119571, + 0.00063283276, + 0.051125497, + -0.0032124266, + 0.053093515, + -0.040370543, + -0.012601499, + 0.017470429, + 0.031896867, + 0.006782593, + 0.027770853, + 0.009261439, + -0.029646825, + -0.025205277, + -0.027463472, + 0.018455723, + -0.014428546, + -0.041367576, + -0.026252072, + -0.02690438, + 0.02014297, + 0.018078685, + -0.03823742, + 0.04088035, + -0.051418938, + 0.03355091, + -0.041159216, + -0.003966388, + 0.015316318, + -0.018056786, + -0.012378684, + 0.01454914, + -0.021285627, + 0.04871937, + 0.03899785, + -0.006893993, + 0.01625168, + -0.041663542, + -0.007069772, + -0.012777985, + -0.014458483, + -0.014564711, + 0.011727791, + 0.021971373, + 0.09605336, + 0.027195103, + -0.002251801, + -0.020983346, + -0.040699035, + -0.026340686, + 0.007530734, + -0.0003373556, + -0.003246767, + -0.08843842, + 0.02347672, + 0.04929299, + -0.021742774, + 0.0014326326, + 0.016306961, + 0.04796601, + -0.0073066745, + 0.03389563, + 0.015018003, + -0.0029578544, + 0.015161402, + -0.028091507, + 0.035398312, + 0.0021860101, + -0.031404186, + -0.010151058, + 0.02360691, + 0.012965896, + -0.053480394, + -0.047096282, + 0.008249183, + -0.029323407, + -0.013203298, + 0.039523408, + 0.008397344, + 0.02557946, + 0.011273664, + 0.015049433, + -0.028328395, + 0.050805397, + -0.010279935, + 0.042744853, + 0.03596191, + -0.02550943, + -0.010960958, + 0.023108574, + 0.009706966, + 0.018927142, + 0.016432842, + -0.030741405, + -0.032368764, + -0.034669936, + -0.025401043, + 0.055165317, + 0.0056103836, + 0.06030957, + 0.005379202, + 0.06697292, + -0.036178175, + -0.014183145, + -0.042784903, + 0.021264434, + 0.012600811, + -0.007053678, + 0.00703668, + -0.0064213914, + 0.018692628, + -0.022846742, + 0.0012189929, + -0.03935566, + -0.015459773, + 0.012175843, + -0.0018140471, + -0.025057066, + -0.050628994, + 0.032378674, + 0.028949518, + -0.030460441, + -0.028262665, + 0.01469713, + -0.019561669, + 0.007275666, + -0.012009385, + 0.013972481, + 0.002057122, + -0.014296696, + -0.014103122, + -0.0132858325, + -0.0146852555, + 0.024696834, + -0.01492143, + 0.023101952, + 0.079884574, + -0.02498016, + 0.05706767, + -0.016466698, + 0.02522806, + -0.041980717, + -0.01234494, + 0.04466147, + 0.0019668879, + 0.012158405, + 0.039233644, + -0.013746237, + -0.010591876, + -0.01856628, + -0.014674951, + 0.029407827, + -0.014691042, + 0.025365459, + 0.021110466, + 0.10536067, + 0.017048322, + 0.010164287, + -0.049854886, + -0.048981465, + 0.011955518, + 0.020339597, + -0.03716241, + -0.01879581, + -0.020426681, + 0.03655472, + -0.05171598, + 0.01570411, + 0.033224188, + 0.0019669323, + -0.0020114826, + 0.03292975, + 0.0021666877, + 0.035258934, + 0.020180117, + -0.0041589425, + -0.028336244, + 0.006204108, + -0.029015563, + -0.008855256, + 0.037762344, + 0.06350565, + -0.008871239, + -0.057950974, + -0.00041275824, + -0.040372893, + -0.050400484, + 0.017844845, + 0.0015078918, + -0.018156357, + 0.015711961, + 0.02783103, + 0.004031808, + -0.034810133, + -0.00038816343, + 0.024975564, + -0.0030076178, + -0.012549695, + -0.0127417585, + -0.020854698, + 0.04311889, + -0.016173886, + -0.059798796, + 0.02846846, + 0.032170307, + 0.0021016344, + -7.159315e-7, + -0.0058174063, + -0.050115734, + -0.0018811563, + 0.024147794, + -0.022634258, + -0.024832819, + 0.035345383, + -0.007928787, + 0.0014046181, + -0.0018968403, + 0.015231224, + -0.013189625, + 0.0011340856, + -0.02802498, + -0.0033430008, + -0.0037675465, + -0.054562755, + -0.0011353097, + 0.010865976, + 0.013155085, + 0.0024481935, + 0.013514707, + 0.027538639, + 0.00012517514, + -0.0490618, + -0.024654388, + 0.054402985, + 0.034196883, + 0.03195004, + -0.0031646932, + 0.02532386, + 0.002238192, + 0.039038956, + 0.0028570676, + 0.011932911, + 0.03584252, + 0.025792649, + -0.004523875, + -0.024271201, + -0.029207278, + 0.0036423209, + 0.007004883, + -0.032133766, + 0.004751608, + 0.02189523, + 0.024406383, + 0.038878873, + 0.0264326, + -0.033215284, + -0.036493875, + 0.058854453, + 0.01702426, + -0.010984756, + 0.011540062, + 0.021714723, + -0.012243643, + -0.0028388079, + -0.017135479, + -0.030100526, + -0.010684106, + -0.0074490975, + -0.04663646, + 0.013412215, + -0.0015587353, + 0.099885866, + 0.005310288, + -0.028013399, + -0.032736715, + -0.041779596, + -0.2249141, + -0.02955839, + -0.0046104896, + -0.0005976889, + 0.005018894, + -0.013527751, + -0.01803605, + 0.005644002, + -0.040115274, + 0.03541309, + -0.05168545, + -0.068982214, + 0.007531021, + -0.0128135225, + -0.018952595, + 0.0026421363, + -0.010946947, + -0.009358139, + 0.018653605, + -0.030786192, + -0.007474124, + -0.019354384, + 0.023781294, + -0.018052794, + -0.0054669664, + -0.015016734, + 0.028998803, + 0.022286057, + -0.03867303, + -0.020961797, + 0.017460939, + 0.026957469, + -0.017319832, + 0.048871335, + 0.012751569, + 0.02204658, + 0.020401783, + -0.016587395, + -0.026240077, + -0.014508626, + 0.030857777, + 0.0105365515, + -0.02044828, + 0.034345064, + 0.023349172, + -0.045863964, + -0.012713302, + -0.03090288, + -0.008789826, + -0.004222334, + -0.026563983, + 0.012735499, + 0.0020776656, + -0.0040374347, + -0.02935858, + -0.009189145, + -0.015673766, + 0.041501794, + -0.06624235, + 0.012391982, + -0.005675349, + 0.0012637977, + 0.01602602, + 0.041072153, + -0.03494942, + 0.010027579, + -0.025028128, + -0.021113884, + 0.031493675, + -0.023090499, + 0.029539555, + -0.0008040167, + 0.0028138612, + -0.012162685, + -0.020473456, + 0.053645223, + -0.0155569585, + -0.03803383, + -0.018535472, + -0.16012628, + -0.011790929, + -0.00983167, + 0.0069870227, + -0.001666242, + -0.022720832, + -0.0002564308, + 0.005167032, + 0.045337133, + 0.038487356, + 0.3136763, + 0.020368094, + -0.017575754, + -0.011642332, + 0.04114498, + -0.016975956, + 0.02480767, + 0.019645682, + 0.0072925114, + -0.039464608, + -0.031041993, + 0.011615216, + 0.022005422, + 0.0042729974, + 0.015239254, + 0.0065134773, + -0.033102468, + 0.010709912, + 0.10040533, + -0.011051494, + 0.032134414, + 0.013587337, + 0.02305759, + 0.022782162, + -0.051333476, + -0.03561971, + -0.009199058, + 0.05891378, + -0.020140221, + 0.03533455, + -0.025626453, + 0.020899257, + 0.02946241, + -0.018574556, + -0.042996284, + -0.00005462654, + 0.012389531, + 0.016154664, + 0.003082483, + 0.032817, + -0.0018719871, + -0.042070854, + -0.00012181424, + -0.012598454, + -0.00020435978, + -0.043000314, + -0.0060905293, + -0.022211615, + -0.031975348, + -0.020989457, + -0.020039134, + 0.015504497, + -0.016190091, + -0.011247202, + -0.011315284, + -0.020651296, + -0.050641425, + 0.009425064, + -0.003290807, + 0.027135806, + 0.027525513, + 0.013224506, + -0.04574697, + -0.011446864, + -0.031214656, + 0.01725456, + -0.02879057, + 0.0011533808, + 0.033038024, + 0.014350499, + 0.0039879135, + -0.005084784, + 0.00746125, + 0.00078137807, + 0.04686458, + 0.009897207, + 0.00040148085, + 0.08165681, + -0.00982842, + 0.028978962, + -0.048388965, + -0.00845467, + -0.03413631, + 0.03126543, + 0.007490618, + -0.024617488, + -0.016059281, + 0.055687297, + 0.0038498726, + -0.026801242, + -0.006622289, + 0.015402673, + -0.03730588, + -0.0027116088, + 0.011792546, + 0.008225556, + 0.016275749, + -0.012497774, + -0.036408793, + -0.024972409, + -0.042749595, + -0.017631682, + 0.0077738534, + 0.012368028, + 0.0015457332, + 0.0064345477, + -0.009079751, + 0.014804495, + 0.024688533, + 0.014421745, + -0.027443148, + 0.031162333, + -0.022421703, + -0.019743523, + 0.045627445, + 0.011204302, + -0.010837893, + 0.027189564, + -0.0062925657, + 0.037208524, + -0.018529817, + -0.025632704, + -0.004457409, + -0.00013669112, + -0.004825841, + -0.008865833, + 0.005455104, + -0.041731343, + -0.030201241, + 0.022049092, + 0.025600186, + -0.007611363, + 0.030202227, + 0.012770818, + 0.03722342, + 0.02593485, + -0.016148912, + 0.017417695, + 0.019813227, + 0.036207534, + -0.0057327473, + -0.05604965, + -0.012930758, + 0.013851233, + -0.018887622, + -0.008929569, + 0.018412106, + -0.0010581228, + -0.018794972, + 0.019641148, + 0.014317597, + 0.037458997, + -0.027131334, + -0.054866694, + -0.028925674, + -0.032922648, + -0.009921083, + -0.0023252706, + -0.023437442, + -0.03924715, + 0.019546306, + -0.00011578937, + 0.0059454334, + 0.055538494, + 0.038755115, + 0.041409887, + -0.03364516, + 0.04001393, + 0.040064104, + -0.0426629, + 0.0003413797, + -0.012694839, + -0.033292472, + 0.03853005, + 0.025510116, + -0.007924303, + -0.06508231, + -0.00009393318, + 0.009419187, + 0.01754449, + 0.02728923, + 0.027583186, + -0.039995577, + -0.0036327774, + 0.0039163395, + 0.026579648, + -0.032250162, + -0.025041886, + 0.03217121, + -0.031003514, + -0.014442367, + 0.12874724, + -0.0069300183, + 0.005460467, + 0.0060293586, + 0.010321682, + 0.011993085, + 0.010853835, + -0.028941706, + -0.030441558, + 0.00058715075, + 0.017646233, + 0.002004032, + -0.030154577, + 0.0254315, + 0.021288583, + -0.0021693157, + 0.026338536, + -0.04100416, + -0.03113714, + 0.008009853, + -0.0020671475, + -0.04576247, + 0.0038046942, + 0.026241858, + -0.007516989, + 0.046206903, + 0.040878706, + -0.011657097, + -0.018837994, + 0.0002264019, + 0.0142327435, + 0.028479677, + -0.05696333, + -0.019415565, + -0.00013574571, + -0.031799138, + -0.06433859, + 0.013670015, + 0.010853374, + 0.015061517, + 0.0029342817, + -0.004052383, + -0.03051493, + 0.034054317, + -0.0020269905, + -0.0016756195, + 0.030862574, + 0.0044053416, + -0.01963733, + -0.004536218, + -0.021972721, + -0.009258987, + 0.033271756, + -0.019503837, + 0.007973977, + -0.008074723, + -0.010193426, + 0.017186705, + 0.032477368, + 0.010697127, + 0.010162488, + 0.0050429557, + -0.019209608, + -0.020280486, + 0.015446342, + 0.029356275, + -0.012504091, + -0.027121412, + -0.010212455, + -0.014993395, + 0.040798735, + -0.03556935, + -0.013561129, + -0.049216796, + -0.013145194, + 0.042386413, + -0.025611298, + -0.033147585, + -0.04645739, + 0.004070511, + -0.016229488, + -0.06276571, + -0.048359685, + 0.020956952, + 0.010128148, + -0.05172869, + -0.00807176, + -0.0028089287, + 0.0194791, + -0.011256249, + 0.018489845, + -0.026010707, + 0.011655437, + -0.032421723, + 0.006081385, + 0.0477004, + -0.0017155424, + -0.0067984867, + -0.03182819, + 0.049594473, + 0.006865653, + -0.03054066, + 0.020650579, + -0.04345983, + 0.026957452, + -0.016733713, + 0.026939614, + -0.02206624, + 0.015466926, + -0.026392674, + 0.053105064, + -0.057122074, + -0.012214145, + -0.010385428, + -0.005866139, + -0.01976474, + -0.008915943, + 0.03151603, + 0.039940447, + 0.023415621, + -0.041042183, + 0.0013764248, + 0.008425802, + -0.004242108, + 0.011612712, + 0.029192341, + -0.01503719, + -0.02943288, + 0.01286491, + -0.0007051833, + 0.0068267155, + -0.01519031, + -0.009470079, + -0.020748727, + -0.0038174181, + 0.000103307946, + 0.008495536, + -0.00894188, + -0.012626943, + -0.018310215, + -0.008876467, + -0.012846174, + -0.011062973, + 0.049518634, + 0.004209599, + -0.0037496358, + -0.02709768, + 0.019898968, + 0.0061937417, + 0.017188724, + -0.0032191498, + 0.03406361, + -0.048669886, + 0.018213173, + -0.04589562, + 0.008911193, + 0.006413532, + -0.00596069, + 0.020035718, + 0.006837782, + 0.026145237, + -0.03031196, + -0.059268147, + -0.00579299, + -0.0049354257, + -0.05418719, + 0.059893258, + 0.035886917, + 0.013803754, + -0.026837826, + 0.037258394, + 0.005128159, + 0.010729349, + -0.008389764, + 0.027581623, + 0.015502119, + -0.021915354, + 0.0077097504, + -0.013706498, + -0.026284698, + -0.003131377, + -0.02249966, + -0.00055571104, + -0.016673766, + 0.0445354, + -0.007982139, + -0.0031914476, + 0.050424643, + 0.025567772, + -0.029222926, + -0.012016403, + 0.014275376, + -0.039969277, + 0.008198934, + -0.008432635, + -0.02804207, + -0.01350317, + 0.043060243, + 0.04683725, + -0.021990987, + -0.016467372, + -0.05423716, + 0.0147141, + -0.21353096, + 0.027170712, + 0.015603658, + -0.021031026, + -0.052585322, + -0.0034531616, + -0.0021076514, + -0.004464943, + 0.0091336025, + -0.029437736, + -0.027239982, + 0.025767257, + 0.0058530997, + -0.0124251405, + -0.01684489, + 0.01902177, + -0.013669111, + -0.0013086965, + 0.008669655, + 0.019800603, + 0.0054206727, + -0.016822362, + 0.035029154, + 0.004459747, + 0.02253666, + -0.0256325, + -0.035142265, + 0.012421797, + -0.0581974, + -0.008966055, + 0.001601002, + 0.00810465, + 0.03256816, + 0.047739938, + -0.026193596, + -0.0049124705, + -0.0031594401, + -0.02835814, + 0.0042426703, + -0.022072878, + -0.0017551893, + 0.043402947, + -0.014349653, + -0.031818878, + -0.012242036, + 0.030948259, + -0.0032772375, + 0.019246813, + -0.066388115, + 0.018890917, + -0.0034883423, + 0.020568924, + -0.012573065, + 0.0067973747, + 0.019221721, + -0.028433986, + -0.02193201, + 0.01632071, + 0.02743479, + 0.038915645, + 0.00047901695, + -0.0160004, + -0.026741909, + -0.05013741, + -0.052900538, + -0.012566303, + -0.078952156, + 0.0030111978, + -0.0022587322, + 0.008863053, + -0.023826435, + -0.02330338, + -0.0028917592, + -0.04955672, + 0.017133128, + -0.01843728, + 0.012843042, + -0.011642067, + -0.011499982, + -0.02557876, + -0.005905696, + 0.015257881, + -0.046280645, + -0.025379151, + 0.043015026, + 0.0025276807, + -0.024527742, + 0.008293009, + -0.009740542, + 0.009752819, + -0.07028874, + 0.017680658, + 0.0046013026, + 0.02178801, + -0.014072281, + 0.033870373, + -0.012743978, + 0.004770538, + -0.017497, + -0.021987816, + -0.059353795, + -0.00232295, + -0.012145074, + 0.026487913, + -0.014496365, + -0.0312568, + 0.037350386, + -0.009908939, + 0.0062689455, + -0.0058916006, + -0.035176728, + -0.015228952, + -0.018197881, + 0.009616271, + -0.034587033, + 0.009333248, + 0.018017648, + -0.004573046, + -0.039717704, + 0.0031476028, + 0.00012905903, + 0.03264168, + -0.012450152, + -0.044259563, + -0.001965529, + -0.0017442809, + 0.053143416, + -0.0015910481, + -0.007470394, + -0.0141435675, + 0.04870124, + -0.026156781, + -0.009793776, + 0.04021847, + 0.010062698, + -0.014701971, + -0.022690184, + 0.047164217, + -0.019650066, + -0.051802497, + 0.008368042, + 0.014994729, + 0.01072891, + -0.007058836, + -0.018860757, + 0.011993973, + 0.007124508, + 0.016220115, + -0.05122312, + -0.014102309, + 0.037301928, + 0.010275192, + 0.001030071, + 0.0075479085, + 0.011036656, + 0.004550925, + 0.021436047, + 0.02555472, + -0.024670133, + 0.060436282, + 0.035970833, + -0.03831014, + -0.033916634, + -0.04498535, + -0.006537808, + 0.0028794394, + 0.021685414, + -0.003631991, + -0.0034953314, + 0.008895904, + -0.002609582, + 0.052348223, + -0.029602082, + 0.07488846, + 0.01439349, + -0.050956804, + 0.041242808, + -0.00517158, + 0.042580623, + -0.017003028, + 0.030398304, + -0.01951044, + -0.007481867, + 0.02481676, + 0.007197244, + 0.016503789, + 0.028170016, + 0.05094595, + 0.010219519, + 0.00085970375, + 0.018708248, + -0.010733345, + -0.003314592, + -0.046851743, + 0.04190534, + -0.008695352, + 0.02023812, + 0.025350098, + -0.035668727, + 0.062452275, + -0.011515474, + -0.035037078, + -0.04530235, + -0.013438132, + -0.022001846, + -0.008065732, + -0.034007497, + -0.028214457, + -0.033421222, + 0.04278966, + -0.030315174, + 0.012509734, + -0.028541956, + 0.0022665437, + 0.0072245463, + -0.0007451263, + 0.010341508, + 0.022787843, + 0.023930414, + -0.028488455, + -0.011147614, + 0.00997353, + 0.0016519586, + -0.049259387, + -0.03367056, + -0.009223698, + -0.015183085, + -0.007526151, + -0.0063436828, + -0.011289968, + -0.0215075, + 0.035036437, + -0.03625174, + 0.032981966, + 0.015500578, + 0.0003030196, + 0.04977679, + 0.025322497, + -0.004398037, + -0.004112864, + 0.048542406, + -0.027017474, + 0.0032121837, + 0.029881764 + ], + "tools_centroid": [ + -0.057775438, + -0.0033558903, + -0.040774547, + 0.007837516, + -0.018325893, + -0.010864775, + 0.018047614, + -0.009721339, + -0.008646594, + 0.0075770593, + 0.008523997, + 0.015366553, + 0.004299319, + 0.025948813, + 0.014777593, + -0.017250892, + 0.017971335, + -0.013808822, + 0.013956967, + -0.03137046, + -0.01177912, + -0.01829785, + -0.01472029, + 0.04524385, + 0.017948175, + 0.043402486, + -0.034913003, + -0.030508569, + 0.036542602, + 0.04816589, + 0.0059453044, + -0.01865484, + 0.0013449497, + -0.053336088, + -0.033382863, + -0.030407647, + 0.02391551, + -0.0077984696, + -0.06443239, + -0.010297186, + -0.023779081, + 0.008694963, + 0.008222014, + -0.044142373, + 0.020442644, + -0.039328482, + 0.024845567, + -0.040081494, + 0.0007699972, + 0.0034618257, + -0.026518447, + -0.01582194, + 0.02163327, + -0.039542127, + 0.029000822, + 0.051123485, + -0.03359137, + 0.0023163594, + -0.05225703, + -0.00091889553, + -0.028958295, + -0.02035564, + -0.010448618, + 0.011662464, + 0.005890449, + 0.089658946, + 0.023167921, + -0.0010184674, + -0.0121378135, + -0.017610954, + -0.044071835, + 0.020530395, + -0.031366024, + -0.03172744, + -0.087409504, + 0.033778958, + 0.04312275, + -0.026120976, + -0.011753333, + -0.0060431096, + 0.050321087, + -0.00218206, + 0.01355621, + 0.0027208505, + 0.020778839, + 0.052400798, + -0.042935297, + 0.034534387, + -0.007350808, + -0.0046558166, + -0.027647814, + -0.0017963437, + 0.005831492, + -0.045641955, + -0.053029805, + -0.014840337, + -0.029358158, + 0.007700691, + 0.0351922, + 0.00765673, + 0.014760719, + 0.001616144, + 0.007946026, + -0.013469456, + 0.048162416, + 0.0026764928, + 0.03702904, + 0.018565005, + -0.026513232, + -0.008487164, + 0.0036733798, + 0.021371398, + 0.016962957, + 0.009267585, + -0.023463314, + -0.04331059, + -0.035165418, + -0.048778474, + 0.019885918, + -0.009031693, + 0.05094907, + -0.009540563, + 0.066760264, + -0.041867316, + -0.017069353, + -0.035217352, + 0.03588013, + 0.0549495, + 0.015968364, + 0.036352813, + 0.0016934925, + 0.034936693, + -0.008160922, + 0.011436721, + -0.037352208, + -0.02368231, + 0.0067209275, + 0.00545583, + -0.008353004, + -0.06703308, + 0.020577332, + 0.05729178, + -0.035366762, + -0.037864406, + 0.037184034, + -0.050321713, + 0.02234205, + 0.003926314, + 0.009195237, + -0.0055696885, + -0.019696044, + 0.001930507, + 0.009407629, + 0.00039543337, + 0.0012940381, + -0.015241957, + 0.006914032, + 0.056626543, + -0.021361178, + 0.06836428, + -0.0020655459, + 0.011145303, + -0.026055796, + -0.016941039, + 0.030385481, + -0.01505081, + 0.0076058493, + 0.000746044, + -0.0055449284, + -0.014022635, + -0.007175385, + -0.026965719, + 0.015261819, + -0.0125290435, + 0.026447913, + 0.027559392, + 0.094134904, + 0.024422089, + 0.010056614, + -0.0593088, + -0.046704147, + 0.020040825, + -0.008067564, + -0.008940185, + -0.035114467, + -0.0073384983, + 0.042855285, + -0.010223366, + 0.0085879695, + 0.014363037, + -0.0004808616, + 0.012251967, + 0.02859141, + 0.0047126776, + 0.03129005, + 0.017805913, + -0.002037126, + -0.021929229, + -0.024302894, + -0.03401745, + -0.02272678, + 0.03324402, + 0.043827765, + 0.0039511696, + -0.040313162, + -0.011220685, + -0.012239163, + -0.033538636, + 0.0074846055, + -0.04003989, + -0.012082418, + -0.0004583568, + 0.02797805, + 0.0015042631, + -0.040352985, + -0.0058471086, + 0.00686128, + 0.010597693, + -0.03193956, + -0.03259277, + -0.022284962, + 0.053651303, + 0.003222045, + -0.015034204, + 0.017189868, + -0.004987139, + 0.02370927, + 0.012949884, + -0.00048070893, + -0.021616403, + -0.024279464, + -0.0068444926, + -0.007384404, + -0.01851718, + 0.070953965, + -0.0016654494, + -0.013800572, + 0.0031709832, + 0.026937919, + 0.00003340502, + -0.025002202, + -0.022953335, + 0.0075816554, + -0.0036617126, + -0.04462196, + 0.018735254, + 0.04323663, + 0.033287965, + -0.017825564, + 0.004733342, + 0.0292739, + -0.01013516, + -0.043643888, + -0.004823276, + 0.03432248, + 0.032504197, + 0.028787872, + -0.012781166, + 0.012777502, + 0.0104851285, + 0.03479923, + 0.020120937, + 0.017253144, + 0.044453908, + 0.007472006, + -0.0061729704, + -0.024305446, + -0.0054376926, + 0.0074620508, + -0.0092536295, + -0.032345943, + 0.021577463, + 0.028651051, + 0.008056253, + 0.01772497, + 0.00647969, + -0.024510348, + -0.028680895, + 0.0586949, + 0.0232443, + -0.0075143017, + 0.022155466, + 0.010020021, + 0.003732729, + 0.0012835207, + 0.002107834, + -0.03460533, + -0.011155255, + -0.0087890215, + -0.04118182, + -0.020692606, + -0.008159101, + 0.10522082, + -0.0016338307, + -0.016076414, + -0.0016532582, + -0.035279144, + -0.22679776, + -0.010667053, + 0.008881526, + 0.03224856, + 0.006959963, + -0.005841684, + -0.0018540951, + 0.0055787065, + -0.041649945, + 0.023068914, + -0.035203893, + -0.07759581, + -0.02947163, + -0.0359074, + -0.0012353952, + -0.005708223, + -0.0065805116, + -0.0068721324, + 0.022740014, + -0.040421862, + -0.020221274, + -0.014032607, + 0.051545203, + -0.006344193, + -0.013821137, + -0.01579205, + 0.047883134, + 0.009514113, + -0.037617173, + -0.010565667, + 0.0038946962, + 0.007921387, + -0.004055107, + 0.032935105, + 0.012555967, + 0.032819428, + 0.0094431145, + -0.011137544, + -0.015564898, + -0.008363006, + 0.020149253, + 0.041235164, + -0.00003780061, + 0.037134502, + 0.041190896, + -0.035524506, + 0.014504791, + -0.02819106, + -0.016905108, + -0.008305034, + -0.0017854779, + -0.0047899806, + 0.0022640792, + -0.026646169, + -0.044333562, + -0.00081833766, + -0.03398577, + 0.039966717, + -0.031211361, + -0.0025227992, + -0.029431568, + -0.004133133, + 0.048822563, + 0.017931307, + -0.023236742, + 0.019759977, + 0.01688985, + -0.019182017, + 0.03889916, + -0.022565762, + 0.028835144, + 0.0038326585, + 0.0007537687, + -0.025940292, + -0.0089698145, + 0.047854897, + -0.028326858, + -0.03543255, + -0.0030444826, + -0.1869618, + -0.030654354, + 0.018738456, + 0.0010118893, + 0.014872937, + -0.035121404, + -0.006889633, + -0.0031736507, + 0.021199811, + 0.048273966, + 0.32182485, + 0.017583381, + -0.019531554, + 0.0092329085, + 0.052992057, + -0.015412301, + 0.0074604903, + 0.03680418, + -0.006491189, + -0.031820234, + -0.0012280992, + 0.0059963525, + 0.025414133, + -0.002702171, + -0.006588213, + 0.02243535, + -0.030782828, + -0.0007346572, + 0.102493934, + -0.003852306, + 0.031274583, + -0.003960667, + 0.007930573, + 0.025458938, + -0.051701877, + -0.04098843, + -0.010463472, + 0.05650782, + -0.026589196, + 0.04590178, + -0.014493388, + 0.01932283, + 0.024496134, + 0.0014021251, + -0.05392017, + -0.018444793, + 0.011652213, + -0.012777032, + -0.008817307, + 0.031348858, + -0.028693452, + -0.046828907, + 0.0014963904, + 0.026269333, + 0.005013643, + -0.039865334, + 0.017514601, + -0.0033667302, + -0.027785953, + -0.018094406, + -0.035160404, + -0.027459389, + -0.022849808, + 0.0028880779, + -0.0011864194, + -0.034878395, + -0.022470577, + -0.031891372, + -0.012203773, + 0.05001213, + 0.0072241533, + 0.010247018, + -0.06585563, + 0.029790338, + -0.0031036446, + 0.041342862, + -0.029685218, + -0.014217362, + 0.0132140145, + 0.015673228, + 0.014440125, + -0.009752704, + 0.005639572, + -0.01108167, + 0.05446453, + 0.019834101, + -0.021086873, + 0.062157847, + -0.019641414, + 0.045685045, + -0.024531566, + 0.010219521, + -0.04718473, + 0.02712037, + 0.023816736, + -0.017155042, + -0.013114721, + 0.02775809, + -0.027383637, + -0.040353194, + -0.008417484, + -0.0021451944, + -0.027928088, + 0.013438158, + 0.014820075, + 0.004232394, + 0.015032984, + -0.0037845736, + -0.038340595, + -0.016891716, + -0.03165935, + -0.018768659, + -0.0066177617, + -0.023367273, + 0.019635271, + -0.0003745473, + 0.000087627566, + 0.010894101, + 0.03014416, + -0.0042988, + -0.029685823, + 0.03274391, + -0.010986411, + -0.039936848, + 0.03304658, + 0.016807966, + 0.004531139, + 0.020389395, + 0.029960461, + 0.037048277, + -0.030802228, + -0.014076131, + -0.008956837, + 0.009107653, + -0.024870684, + -0.030200038, + 0.0005537656, + 0.008720928, + -0.055442058, + 0.043536436, + 0.018925853, + -0.019451551, + 0.016840005, + 0.010896255, + 0.012237941, + 0.006439829, + 0.0076054423, + 0.024631822, + -0.00469513, + 0.059032343, + -0.0082413, + -0.051315658, + -0.02167114, + 0.009919436, + -0.0045897025, + -0.0009727792, + 0.043319814, + -0.018224705, + -0.021532157, + 0.010909149, + 0.0041222232, + 0.049312256, + 0.02342331, + -0.06829633, + -0.015685972, + -0.033873446, + -0.016059933, + 0.012121991, + -0.014224916, + -0.03762836, + 0.015365398, + -0.004025226, + 0.024712248, + 0.07105101, + 0.025589744, + 0.013154655, + -0.0456072, + 0.02607114, + 0.025644207, + -0.033561457, + -0.0012643771, + -0.033391006, + -0.061123043, + 0.042876773, + 0.0020319296, + -0.020118402, + -0.043745127, + -0.00054004864, + -0.017112734, + 0.0075695347, + 0.047089994, + 0.013356315, + -0.04420688, + 0.012939495, + 0.015024913, + 0.033482503, + -0.0072687673, + -0.016575295, + 0.013684971, + -0.024929354, + -0.032714568, + 0.11408622, + 0.013711464, + 0.0007866529, + 0.010564399, + 0.011742405, + 0.008514497, + 0.028058255, + -0.009284165, + -0.018095477, + 0.0026275918, + -0.007345314, + 0.0042800684, + -0.040819786, + 0.0037819908, + -0.011583471, + -0.014982446, + 0.013995526, + -0.011253528, + -0.01841841, + 0.0035173576, + -0.016492022, + -0.0025003836, + -0.0034326876, + 0.044872776, + 0.0014514242, + 0.046815414, + 0.05207907, + -0.010015235, + -0.017120235, + -0.017186575, + -0.0038804088, + 0.053709168, + -0.067070924, + -0.0064274804, + 0.016569823, + -0.010965232, + -0.04720728, + 0.020261586, + 0.011407856, + 0.013676394, + 0.0033044717, + 0.00720917, + -0.032467864, + 0.035398927, + -0.029180987, + 0.008299155, + 0.0403421, + 0.002068336, + -0.031769827, + 0.023002677, + -0.017297465, + -0.0028395427, + 0.03669844, + -0.006664068, + 0.055516694, + -0.02332954, + 0.0128885275, + 0.029991373, + 0.03301732, + -0.0143478615, + 0.001786697, + 0.0061424426, + -0.013547873, + -0.01829034, + 0.032445464, + -0.004602294, + -0.0024687257, + -0.02234211, + -0.0072758887, + -0.019191276, + 0.026224066, + -0.041807573, + -0.0022833985, + -0.027213147, + -0.02427904, + 0.03531603, + -0.030026786, + -0.02400627, + -0.06544794, + -0.013835978, + -0.009932556, + -0.05266321, + -0.01314315, + 0.037983123, + 0.014741308, + -0.037383366, + 0.0063219173, + 0.02204942, + 0.025447313, + -0.025008852, + 0.012261236, + 0.0010507479, + 0.033279635, + -0.032050498, + 0.023104839, + 0.0079582995, + -0.026157726, + 0.004492358, + -0.04235163, + 0.022636527, + 0.0039217533, + -0.038830105, + 0.03416724, + -0.039616987, + 0.013216047, + -0.005373113, + 0.014587253, + -0.025387455, + 0.018838875, + -0.023695769, + 0.02310995, + -0.054573603, + -0.009373536, + -0.017116925, + 0.014891645, + -0.024939645, + 0.004483685, + 0.04446569, + 0.04414556, + 0.036651935, + -0.026196282, + -0.0068070013, + 0.04016786, + 0.014043489, + -0.0003690823, + 0.058535554, + 0.0032685478, + -0.014644685, + 0.0026333602, + -0.00085332064, + 0.0049753827, + -0.0020620888, + -0.0150953755, + -0.013835361, + -0.004500461, + -0.016988168, + 0.003694431, + -0.021504221, + -0.025054544, + -0.025284309, + -0.021449776, + -0.002113436, + -0.019090962, + 0.0075706523, + -0.003012929, + 0.009065395, + -0.007116615, + 0.044901505, + 0.0027590708, + 0.009059467, + -0.012410826, + 0.03301639, + -0.029811352, + 0.01738935, + -0.021565853, + 0.0030311234, + -0.01219218, + -0.008524064, + 0.017151799, + 0.00995471, + 0.029735599, + -0.034349315, + -0.06940458, + 0.0036188683, + -0.0054800794, + -0.048410185, + 0.008675395, + 0.025768118, + 0.010540176, + -0.020408748, + 0.038225047, + -0.00551114, + 0.01753059, + 0.026808819, + 0.010108354, + 0.016693898, + 0.009826699, + 0.014569398, + -0.01573024, + -0.008260742, + -0.011886305, + -0.012466746, + 0.00095788017, + -0.016167117, + 0.038578525, + -0.013100995, + -0.01676035, + 0.03332717, + 0.005415544, + -0.010420641, + -0.03113651, + -0.027528936, + -0.021406338, + 0.023781726, + -0.0063788826, + -0.0077266875, + -0.014590149, + 0.040968418, + 0.026437076, + -0.03645595, + -0.009789084, + -0.046371892, + 0.021632947, + -0.20988819, + 0.023208335, + 0.009895576, + -0.013134811, + -0.05835602, + 0.007831426, + -0.012073989, + -0.018234992, + 0.009569847, + -0.038773432, + -0.044383198, + 0.0089479145, + 0.025667464, + -0.02950146, + 0.016152902, + 0.023348661, + 0.013435106, + 0.0112070935, + 0.007683514, + 0.03000217, + -0.014931629, + -0.03128683, + 0.0018451557, + 0.026329964, + 0.0072926055, + -0.0108091775, + -0.010943898, + 0.007525587, + -0.04637749, + -0.0012201949, + 0.014439996, + -0.016553767, + 0.035915114, + 0.064322315, + -0.025222458, + -0.0086920345, + 0.01354625, + -0.031556293, + -0.021044917, + -0.01556811, + -0.011967994, + 0.034725036, + -0.017628316, + -0.011426438, + 0.0155166965, + 0.0345392, + -0.005978086, + 0.001560346, + -0.057566527, + 0.014015094, + -0.005247297, + 0.044353653, + -0.032477822, + 0.00088942435, + 0.02304252, + -0.017082589, + -0.04723853, + 0.025155952, + -0.009886005, + 0.030909168, + -0.0148722185, + 0.004657267, + -0.036441028, + -0.07425148, + -0.04521448, + -0.031231904, + -0.083747715, + 0.012393575, + 0.0046909004, + 0.030838598, + -0.017930364, + -0.03798974, + 0.0046308283, + -0.024992201, + 0.01813419, + -0.01783272, + 0.013735788, + -0.00084009505, + -0.026482312, + -0.029216949, + -0.03189472, + 0.0120601095, + -0.058092535, + 0.007142014, + 0.060608845, + -0.008147901, + -0.003151759, + 0.0046359287, + -0.017588917, + 0.0092833545, + -0.053699553, + -0.0046048244, + 0.0012410878, + 0.021804376, + -0.012375213, + 0.013280399, + -0.031906895, + -0.0035128666, + -0.0140839275, + 0.0052240947, + -0.019663228, + -0.0015391115, + 0.001982028, + 0.014376489, + -0.011715944, + -0.009479002, + 0.06390848, + -0.021833453, + 0.017518744, + -0.015481537, + -0.032284454, + -0.0023388504, + -0.01498571, + 0.005026777, + -0.04676246, + -0.0042618783, + 0.027863752, + -0.021421071, + -0.030720329, + -0.019202845, + -0.0036962088, + 0.01606846, + 0.0050927107, + -0.042860158, + 0.003974965, + 0.0009248986, + 0.037120704, + 0.004218136, + -0.0036327865, + -0.018174114, + 0.03935176, + -0.03224347, + -0.012047191, + 0.02956622, + 0.031379495, + -0.012442746, + -0.014872591, + 0.06750819, + -0.020884288, + -0.054731537, + -0.02460563, + 0.003977729, + 0.0004902187, + -0.019478966, + 0.00022061134, + -0.013216427, + -0.0037307949, + -0.011919203, + -0.042946137, + -0.028380334, + 0.05078273, + -0.0030087964, + 0.01560839, + 0.017069735, + 0.0016844464, + 0.004942494, + -0.014320324, + 0.02391562, + -0.009867219, + 0.051671702, + 0.027323583, + -0.035621997, + -0.014997956, + -0.027416717, + 0.01103468, + 0.019373719, + 0.0009110808, + -0.016991789, + -0.03706896, + 0.0037041882, + -0.0022370599, + 0.030673143, + -0.03202175, + 0.056521207, + 0.02383771, + -0.04783513, + 0.04099715, + 0.013725404, + 0.06307572, + -0.0024652113, + 0.023774121, + -0.012859268, + 0.018181501, + 0.008533877, + 0.012162901, + 0.019225126, + 0.026711285, + 0.033790324, + 0.016099503, + 0.01810887, + 0.01823027, + -0.003407086, + 0.0019508481, + -0.019111417, + 0.037072737, + 0.0043255636, + 0.014085034, + 0.023889331, + -0.026697278, + 0.06311214, + -0.020659966, + -0.034266055, + -0.035501692, + -0.025569465, + -0.029216953, + -0.010694965, + -0.02754278, + -0.027895218, + -0.032571055, + 0.04116445, + -0.029515225, + 0.0061112135, + -0.025661161, + 0.0071192123, + -0.004901541, + 0.01863631, + 0.025625108, + 0.029897802, + 0.008398415, + 0.009981635, + 0.00594864, + 0.025863202, + 0.012255268, + -0.0479219, + -0.016847553, + -0.030781547, + 0.00071642187, + -0.007623827, + 0.003929538, + -0.011301359, + -0.022604315, + 0.01118298, + -0.04312297, + 0.034218777, + 0.025586674, + -0.0024720372, + 0.03698685, + 0.03409089, + -0.021177674, + -0.022660092, + 0.033316627, + -0.016247125, + 0.0072186943, + 0.016868103 + ], + "abstain_centroid": [ + -0.031148378, + 0.005337721, + -0.051859938, + 0.006701708, + -0.013760792, + -0.046238888, + -0.001741801, + 0.003761064, + 0.0115341535, + 0.015631316, + -0.007558447, + 0.042414144, + -0.008460059, + -0.003942813, + 0.014397727, + -0.000967998, + 0.0059690275, + -0.016228953, + -0.00661065, + -0.016335107, + -0.017304765, + -0.015375099, + 0.00016670206, + 0.02917791, + 0.019667024, + 0.03706363, + -0.014033229, + -0.017078554, + 0.022332542, + 0.0029681036, + -0.026012667, + 0.0047733886, + -0.008349954, + -0.025975645, + -0.057982, + -0.037145726, + 0.022042587, + -0.010432212, + -0.054713663, + -0.018552495, + 0.014079261, + 0.0066314046, + 0.027429352, + -0.026314953, + 0.021749051, + -0.05777429, + -0.011509256, + -0.020935455, + -0.00084090995, + -0.031351306, + -0.021382077, + -0.00847218, + 0.03660322, + -0.013143199, + 0.047227796, + 0.05580892, + -0.010979107, + -0.02130941, + -0.08700673, + 0.0018242795, + -0.025122127, + -0.013786088, + -0.009168319, + 0.011006979, + 0.0012779797, + 0.09556672, + 0.00808916, + -0.008711008, + -0.008361359, + -0.034987625, + -0.03647069, + 0.021719662, + -0.032498717, + -0.01776087, + -0.0698433, + 0.011626705, + 0.020904263, + -0.008985529, + 0.0070378096, + 0.037491325, + 0.08140702, + -0.015960723, + 0.00758235, + 0.018223334, + 0.011250971, + 0.04139314, + -0.030075394, + 0.011359265, + -0.0026697265, + 0.0045864326, + -0.029182356, + -0.018836327, + 0.035417307, + -0.050186694, + -0.0169077, + 0.0030452074, + -0.031451467, + 0.021025097, + 0.03237884, + -0.017753405, + 0.024965163, + 0.0068491264, + -0.0007417426, + -0.025353186, + 0.014833976, + 0.00048042086, + 0.033970177, + 0.029662795, + 0.001964002, + -0.00040594727, + 0.0462047, + 0.033295035, + 0.014799662, + 0.004400711, + -0.012687833, + -0.048856415, + -0.021582829, + -0.029335681, + 0.031864036, + -0.013479058, + 0.0061972486, + 0.019316947, + 0.039016202, + -0.0460622, + -0.011819415, + -0.035461947, + 0.030371841, + 0.045124568, + -0.006952659, + 0.026146563, + 0.015358209, + 0.014641611, + -0.03725902, + 0.0049006664, + -0.012360949, + -0.030733725, + 0.016830463, + -0.0022114269, + 0.017974649, + -0.051037353, + 0.024567815, + 0.029570594, + -0.028185142, + -0.028654277, + 0.021437967, + -0.03857855, + 0.024237394, + -0.0068540815, + 0.018478565, + -0.012613434, + 0.0043713585, + -0.0101159075, + 0.0065389522, + -0.014050798, + 0.0061866357, + -0.017707681, + -0.019518772, + 0.053408965, + -0.013017107, + 0.016827421, + 0.022524578, + -0.0068964125, + -0.052380078, + 0.0003833916, + 0.029090617, + -0.011927651, + 0.024243768, + 0.013390409, + -0.014234686, + -0.0017058246, + -0.012792837, + -0.026187133, + 0.0398575, + -0.0025493933, + 0.030229095, + 0.026641972, + 0.07534804, + 0.022857832, + 0.010506174, + -0.058816746, + -0.051879656, + 0.016220875, + 0.017665846, + -0.054791577, + -0.039248586, + 0.014590265, + 0.021849325, + -0.018102316, + 0.015241521, + 0.021297466, + -0.022830576, + -0.016140226, + 0.027246347, + -0.0054256297, + 0.046256945, + -0.007986689, + 0.026465587, + -0.0026759568, + -0.0096074, + -0.037014082, + -0.048065815, + 0.0330576, + 0.045262605, + 0.0051544863, + -0.0342738, + -0.01125588, + -0.03914742, + -0.03194088, + 0.0144135505, + -0.017719291, + 0.009187744, + 0.0078019383, + 0.02771336, + 0.0051388913, + -0.0420357, + 0.0072669606, + -0.002695001, + 0.0030906205, + 0.004021662, + -0.021064727, + 0.0009345196, + 0.029135603, + 0.002100572, + -0.019682301, + 0.04653866, + 0.032432303, + 0.017765505, + 0.015490212, + -0.009414901, + -0.023563534, + -0.008533622, + 0.00501602, + -0.019913116, + -0.0035469246, + 0.036062475, + -0.022427976, + -0.040446255, + 0.015986245, + 0.009162775, + -0.0003249781, + -0.022688588, + -0.030663736, + 0.0018985457, + 0.024509203, + -0.046458673, + 0.021605283, + -0.0032775383, + 0.008400841, + 0.005384018, + -0.0020570226, + 0.018701846, + -0.012248811, + -0.04532812, + 0.0036800993, + 0.032931983, + 0.028872136, + 0.012167975, + -0.0052398657, + 0.026528697, + 0.013797996, + 0.023905411, + 0.031488698, + 0.01613529, + 0.03790394, + 0.007559894, + 0.0036375439, + -0.033904057, + -0.0027450048, + 0.0044508814, + 0.010002826, + -0.030975314, + -0.016194653, + -0.010303318, + 0.0017212449, + -0.004217246, + -0.00038384096, + -0.028139843, + -0.037131034, + 0.059932988, + 0.016013877, + 0.0041002976, + 0.017807445, + 0.012730673, + -0.009848034, + 0.00704557, + -0.0010963117, + -0.017336579, + 0.0052566174, + 0.006595981, + -0.045635466, + 0.012577329, + -0.0151054, + 0.088495515, + -0.0060461042, + -0.01455267, + -0.010428576, + -0.03148846, + -0.24492498, + -0.022456426, + -0.012005429, + 0.018366225, + 0.01226593, + -0.024266042, + -0.033916228, + -0.0036076312, + -0.054510105, + 0.00632446, + -0.015560582, + -0.05587817, + -0.020939475, + -0.0053286967, + -0.0070734383, + 0.004897013, + 0.006918987, + -0.022256982, + 0.0075242054, + -0.06765787, + -0.023333376, + -0.02501722, + 0.07195793, + -0.0196174, + -0.005269894, + 0.00026103153, + 0.035949823, + 0.0041076643, + -0.037599213, + -0.01236279, + -0.006176068, + 0.0069308914, + -0.0029138543, + 0.035269044, + 0.014236289, + 0.034278776, + 0.010570882, + -0.008682466, + -0.014200495, + -0.0010429571, + 0.04505355, + 0.020335147, + -0.003972119, + 0.011877002, + 0.010973979, + -0.04002932, + -0.021862812, + -0.010063818, + -0.01741567, + -0.02046978, + 0.0024775756, + 0.005379279, + -0.0035171222, + -0.02375287, + -0.027805964, + 0.006844447, + 0.0020820745, + 0.04806291, + -0.013278907, + -0.011143366, + -0.015697205, + -0.010483088, + 0.008951397, + 0.03141075, + -0.021271285, + 0.006024506, + -0.005130949, + -0.013589776, + 0.028885502, + -0.019157536, + 0.033051424, + -0.011037691, + 0.008256805, + -0.021247163, + 0.014382734, + 0.02527936, + -0.025481075, + -0.03218864, + -0.02366406, + -0.18205336, + -0.016931955, + -0.000928319, + 0.0030238247, + 0.033743773, + -0.016215391, + -0.025330722, + 0.00043090724, + 0.052764837, + 0.045047108, + 0.34483734, + 0.021190971, + 0.029025026, + -0.029802188, + 0.05892975, + -0.027338214, + 0.017156813, + 0.016491018, + 0.005376105, + -0.036032055, + -0.016050696, + -0.019247439, + 0.008546482, + 0.0006579195, + 0.008216663, + 0.012412621, + -0.049241338, + 0.00468357, + 0.083209105, + 0.004403217, + 0.015972426, + -0.03518827, + 0.0025444014, + 0.027684696, + -0.03703065, + -0.040941797, + -0.022856751, + 0.07734354, + -0.017490434, + 0.029937679, + -0.030589428, + -0.01312233, + 0.028537748, + -0.01580879, + -0.031040978, + 0.0022384818, + 0.004584949, + -0.036500685, + -0.013998974, + 0.019088954, + -0.020744042, + -0.02740597, + 0.02783206, + 0.00093145366, + 0.021530958, + -0.015619698, + -0.0065967944, + -0.008945381, + -0.022806387, + -0.0007809315, + -0.019146752, + -0.021306394, + -0.024635077, + -0.0013178901, + 0.030522598, + -0.036964312, + -0.034459814, + -0.026010964, + 0.011097982, + 0.044302322, + 0.033004954, + 0.012737288, + -0.02832643, + -0.033423617, + -0.011146303, + 0.018307667, + -0.010493131, + -0.0023452898, + -0.006297813, + 0.023474587, + 0.015160515, + 0.008065753, + 0.0071064318, + -0.012013574, + 0.034734786, + 0.02027884, + 0.013851284, + 0.07314595, + 0.0051676924, + 0.03851032, + -0.011801342, + -0.024405107, + -0.05664373, + -0.017895719, + 0.010395462, + 0.0045732786, + -0.017845891, + 0.05219816, + -0.007843896, + -0.039979137, + 0.0115565825, + 0.019499846, + -0.03996184, + -0.007970878, + -0.0029153195, + -0.051025048, + 0.023902694, + -0.024392456, + -0.019463966, + -0.02171224, + -0.02825411, + -0.022925304, + 0.008098716, + -0.009423168, + -0.0034888275, + -0.026081651, + 0.005871837, + 0.012862886, + 0.020583529, + 0.015732484, + 0.004980412, + 0.007757627, + -0.018226098, + -0.036115993, + 0.025706174, + 0.049908377, + 0.025328659, + 0.009017575, + 0.020332497, + 0.040973037, + -0.016721342, + -0.01928604, + -0.012567931, + 0.0024534024, + -0.027978517, + -0.016283054, + -0.0059162932, + 0.020438462, + -0.033998214, + 0.025316799, + 0.015457932, + -0.027260441, + 0.007751793, + 0.03414886, + 0.06149144, + -0.009142277, + 0.034868557, + 0.015628273, + -0.011573751, + 0.04880984, + -0.008835818, + -0.042271934, + -0.011501361, + -0.015698932, + 0.00765774, + 0.0040898444, + 0.017931513, + 0.012534788, + -0.026830954, + 0.009097794, + 0.008690346, + 0.044174466, + 0.006063397, + -0.05672009, + -0.01996871, + -0.035708513, + -0.010680867, + -0.003172045, + -0.011108843, + -0.029465709, + 0.012013625, + 0.014341203, + 0.052498966, + 0.13476692, + 0.046210233, + 0.052687794, + -0.006100141, + 0.045737743, + 0.043751344, + -0.03214318, + 0.002818939, + -0.035125505, + -0.04272792, + 0.031032866, + 0.03460889, + -0.020154648, + -0.025992718, + 0.0040032137, + -0.013573707, + 0.0014319279, + 0.033563722, + -0.012486501, + -0.013652725, + -0.00095753156, + 0.018106168, + -0.019855192, + 0.009851039, + -0.0179031, + 0.013870335, + -0.022614986, + -0.025842723, + 0.12371204, + -0.01614459, + -0.0011982421, + 0.06412819, + 0.005470515, + 0.013032822, + 0.031510778, + -0.0073665283, + -0.032478217, + 0.0004560572, + -0.018492013, + 0.009834454, + -0.025878755, + 0.011205347, + -0.012217907, + 0.0048979158, + 0.015269925, + -0.022450786, + -0.0018926285, + 0.010645847, + 0.007540697, + -0.018376745, + 0.0011163221, + -0.00910863, + -0.0052845264, + 0.027602881, + 0.02068656, + -0.028491242, + -0.0168644, + 0.004427309, + 0.0062007783, + 0.047445305, + -0.06059543, + -0.023386633, + 0.031225724, + -0.015070878, + -0.030429764, + -0.0027328797, + 0.01173457, + 0.0085614575, + -0.015814753, + -0.010190623, + -0.014055305, + 0.01249472, + 0.0020188026, + -0.018316, + 0.017405652, + 0.0033323495, + -0.0058355783, + 0.019034537, + -0.017335443, + 0.009071962, + 0.060741838, + -0.019548608, + 0.0028466566, + -0.000989737, + -0.013918978, + 0.039289407, + 0.0338373, + -0.020116925, + -0.0049318955, + 0.021821503, + 0.008678599, + -0.0052154674, + -0.00354734, + 0.005850025, + -0.00011736599, + -0.038427465, + 0.0046045645, + -0.017376445, + 0.03195163, + -0.037625898, + -0.0002592167, + -0.045117147, + -0.021053702, + 0.01866349, + -0.018683132, + -0.022075271, + -0.067146465, + -0.00034859247, + -0.0066958456, + -0.019604217, + -0.025007758, + 0.0084634265, + 0.01903175, + -0.036018692, + 0.00021994986, + -0.01454107, + 0.036363024, + -0.024418773, + 0.03628282, + -0.015224951, + 0.035957087, + -0.040158544, + 0.03579523, + 0.028295236, + 0.011141477, + -0.0144408615, + -0.032107454, + 0.012157545, + -0.0051779617, + -0.056817777, + 0.017691938, + -0.036631953, + 0.038909055, + -0.0097737955, + 0.006345886, + -0.024140004, + 0.018522153, + -0.0378227, + 0.009578492, + -0.050063416, + -0.0136081185, + -0.019632932, + 0.0028407518, + -0.00645373, + -0.016191104, + 0.043725938, + 0.0236164, + 0.027719086, + -0.041044917, + 0.025428394, + 0.020110024, + -0.0060314476, + -0.0061647817, + 0.07147246, + -0.022163881, + -0.002495511, + 0.017374914, + 0.024532678, + 0.017868279, + -0.0123060895, + 0.009035884, + -0.02307739, + -0.00059325853, + -0.02122783, + 0.0071837865, + -0.036735952, + -0.014673909, + -0.028121514, + 0.010137442, + -0.010372103, + -0.03829866, + 0.012825123, + -0.01134884, + 0.022564255, + -0.031195192, + 0.049415413, + -0.037152834, + -0.0024479814, + 0.023421837, + 0.064130165, + -0.020511432, + 0.011314599, + -0.038539167, + 0.015962925, + -0.0053321286, + 0.01771349, + 0.016681707, + 0.020176098, + 0.045899525, + -0.03644215, + -0.05389872, + 0.03137194, + 0.008404088, + -0.038138144, + 0.014195224, + 0.04344335, + 0.0057477932, + -0.032771025, + 0.048310746, + -0.021211635, + 0.030088607, + -0.0025103104, + 0.012287962, + -0.004873214, + -0.008176674, + -0.014280841, + -0.0006814086, + -0.019928552, + 0.008787194, + 0.010925872, + -0.0032950703, + -0.034151934, + 0.023929175, + -0.010394337, + -0.010170748, + 0.029074438, + 0.0061191223, + -0.023697814, + -0.021662993, + -0.034633745, + -0.037640702, + -0.008391389, + -0.010877962, + -0.0082226675, + 0.00087190676, + 0.07043431, + 0.021459375, + -0.050167535, + -0.02395419, + -0.04226849, + 0.023713028, + -0.22955438, + 0.035205133, + -0.00219514, + -0.002945837, + -0.055697378, + 0.00528513, + -0.008466735, + -0.00713325, + 0.006396225, + -0.03216339, + -0.024553273, + 0.028201995, + -0.000054232205, + -0.018851798, + -0.007118189, + 0.040054675, + -0.03188417, + -0.0030668844, + -0.004163999, + 0.032002646, + 0.0047784965, + -0.025143709, + -0.005436027, + 0.0143850455, + 0.018197645, + 0.0026216004, + -0.026822513, + -0.013488665, + -0.044493284, + -0.01340355, + 0.0006943544, + -0.031944275, + 0.021567853, + 0.043572217, + -0.008895598, + 0.017365277, + 0.00690336, + -0.014118812, + -0.017342426, + 0.013759936, + 0.0014441375, + 0.019967057, + -0.023214184, + -0.0035139658, + -0.000988701, + 0.028011117, + -0.019943252, + -0.008950247, + -0.059093565, + -0.0024248778, + 0.013076103, + 0.026632654, + -0.018246919, + 0.019110702, + 0.012685868, + 0.0018900798, + -0.051872082, + 0.009624938, + -0.0076693483, + 0.037904374, + 0.0033752439, + 0.012424309, + -0.031966437, + -0.031686325, + -0.04591932, + -0.023986965, + -0.08195933, + 0.013067175, + 0.013707989, + 0.028453248, + -0.0228147, + -0.036631793, + -0.00467004, + -0.014707499, + 0.025149882, + 0.015747227, + 0.011062073, + -0.007982165, + -0.044059478, + -0.005221822, + -0.029661845, + -0.0010427514, + -0.052402854, + 0.010401697, + 0.012599077, + 0.01889836, + -0.024228005, + 0.012707993, + -0.017794892, + -0.007308048, + -0.057065304, + -0.024420062, + -0.00044011022, + 0.005719825, + -0.04850662, + 0.009310803, + -0.027014088, + -0.0128755625, + -0.019028025, + -0.021750353, + -0.039091866, + 0.007929615, + -0.014110442, + -0.0059228754, + -0.027193954, + -0.008130775, + 0.052156363, + -0.009849598, + 0.026919808, + 0.021327212, + -0.013137314, + 0.0056819157, + -0.010384433, + 0.0052230777, + -0.02551496, + -0.023010258, + -0.009133841, + -0.0034335835, + -0.0066825, + -0.012148635, + -0.014457139, + 0.034450416, + -0.002119914, + -0.042140137, + 0.010768989, + 0.010510174, + 0.03508211, + 0.008629587, + 0.007369357, + -0.011705995, + 0.046988714, + -0.036072724, + 0.0013155584, + 0.028497273, + 0.0201227, + 0.0019115483, + -0.017506052, + 0.05194845, + -0.029103, + -0.05979132, + -0.01251193, + -0.0045603164, + -0.0011945973, + -0.02777315, + -0.03253321, + -0.010601675, + 0.008287072, + 0.010776673, + -0.057129562, + -0.0332778, + 0.031840086, + 0.025541494, + 0.021605106, + -0.006681296, + 0.002822149, + 0.019710355, + -0.005794384, + 0.0026962264, + 0.006601597, + 0.044709172, + 0.03829154, + -0.046349104, + -0.021133212, + -0.024994686, + -0.010804927, + 0.0067239436, + 0.0014897367, + -0.011834818, + -0.023318285, + -0.0038526887, + 0.008018549, + 0.047896657, + -0.030216228, + 0.0534178, + 0.038851894, + -0.009745995, + 0.035945363, + 0.041955516, + 0.035383813, + -0.0023587283, + 0.033687126, + 0.00009861278, + 0.023677062, + 0.024938073, + -0.016926885, + 0.0016545693, + 0.002132915, + 0.041455563, + -0.0030341032, + 0.026448688, + -0.013949449, + -0.0066818777, + 0.002497779, + -0.052358508, + 0.058462907, + 0.017483938, + 0.016702253, + 0.042070094, + -0.020453831, + 0.04187879, + -0.011923706, + -0.021604251, + -0.010823228, + -0.015712183, + -0.008566785, + -0.00082078506, + -0.022313718, + -0.03011328, + -0.042152412, + 0.031767584, + -0.031519245, + -0.011423306, + -0.011847542, + 0.022948265, + 0.004512841, + 0.0025419195, + 0.022916336, + 0.020818464, + 0.017587373, + 0.00058030424, + -0.016511247, + 0.0094060805, + -0.015658183, + -0.035970535, + 0.01030802, + -0.027739279, + -0.022434799, + -0.0031821248, + -0.011698124, + -0.014282087, + 0.010802368, + 0.011209723, + -0.03759782, + 0.06280771, + 0.030460313, + -0.014077967, + 0.05204396, + 0.03405556, + 0.006860819, + -0.02474166, + 0.05734323, + -0.019578138, + -0.018592859, + -0.0055962717 + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-4/router_report.json b/benchmarks/models/router/centroid/out-router-iter-4/router_report.json new file mode 100644 index 0000000..90b7c8f --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-4/router_report.json @@ -0,0 +1,640 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 141, + "accuracy": 0.5106383, + "tool_precision": 1.0, + "tool_recall": 0.1625, + "memory_false_tool_rate": 0.0, + "tool_count": 160, + "memory_count": 92, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.708717, + "sim_memory": 0.6934108, + "sim_abstain": 0.62441397 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.63241655, + "sim_memory": 0.6168429, + "sim_abstain": 0.5503659 + }, + { + "query_id": "shell_test_001", + "text": "what processes are listening on port 8003?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66323787, + "sim_memory": 0.56644976, + "sim_abstain": 0.5741711 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.67449135, + "sim_memory": 0.5638607, + "sim_abstain": 0.5745048 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6470481, + "sim_memory": 0.54589385, + "sim_abstain": 0.5841963 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.70189655, + "sim_memory": 0.6161969, + "sim_abstain": 0.6750375 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6854039, + "sim_memory": 0.5840411, + "sim_abstain": 0.60136163 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.66419965, + "sim_memory": 0.59555024, + "sim_abstain": 0.5525088 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.614116, + "sim_memory": 0.5337758, + "sim_abstain": 0.5362394 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7042604, + "sim_memory": 0.645941, + "sim_abstain": 0.5815989 + }, + { + "query_id": "shell_test_010", + "text": "what's the uptime since klbr daemon last started", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6461812, + "sim_memory": 0.5511648, + "sim_abstain": 0.51299685 + }, + { + "query_id": "shell_test_011", + "text": "how big is agent.db right now on disk?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6185736, + "sim_memory": 0.53921765, + "sim_abstain": 0.5284669 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65126175, + "sim_memory": 0.5649367, + "sim_abstain": 0.5497022 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6559908, + "sim_memory": 0.5679168, + "sim_abstain": 0.6213423 + }, + { + "query_id": "shell_test_014", + "text": "how many memory rows are in agent.db right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.68249536, + "sim_memory": 0.5857847, + "sim_abstain": 0.5644451 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5854054, + "sim_memory": 0.5480971, + "sim_abstain": 0.52175945 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.69217163, + "sim_memory": 0.66149896, + "sim_abstain": 0.5931267 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6674276, + "sim_memory": 0.5689903, + "sim_abstain": 0.6224822 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.665218, + "sim_memory": 0.56594956, + "sim_abstain": 0.6064705 + }, + { + "query_id": "shell_test_020", + "text": "whats the total vram on ths gpu", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6208301, + "sim_memory": 0.52162725, + "sim_abstain": 0.51847833 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.57509327, + "sim_memory": 0.5205136, + "sim_abstain": 0.5162514 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.64946616, + "sim_memory": 0.5403144, + "sim_abstain": 0.5891774 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5936706, + "sim_memory": 0.55656236, + "sim_abstain": 0.5712991 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6539065, + "sim_memory": 0.5740464, + "sim_abstain": 0.6147715 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.68276954, + "sim_memory": 0.6057636, + "sim_abstain": 0.60524577 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6084889, + "sim_memory": 0.52654976, + "sim_abstain": 0.54898715 + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5676058, + "sim_memory": 0.4897967, + "sim_abstain": 0.46544293 + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.59020525, + "sim_memory": 0.53644544, + "sim_abstain": 0.52470756 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6662422, + "sim_memory": 0.5669036, + "sim_abstain": 0.59653854 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6734974, + "sim_memory": 0.6188976, + "sim_abstain": 0.5491258 + }, + { + "query_id": "rf_test_002", + "text": "which file defines the ClientMsg type?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.53454226, + "sim_memory": 0.44181442, + "sim_abstain": 0.43893227 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65867424, + "sim_memory": 0.6521028, + "sim_abstain": 0.53419113 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5971084, + "sim_memory": 0.56205803, + "sim_abstain": 0.5362744 + }, + { + "query_id": "rf_test_006", + "text": "config.rsのmemory_support_score_gapのデフォルト値はいくつ?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7078466, + "sim_memory": 0.6349778, + "sim_abstain": 0.5547469 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.69729316, + "sim_memory": 0.65191716, + "sim_abstain": 0.58798295 + }, + { + "query_id": "rf_test_010", + "text": "what's in the .gitignore for klbr?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5910949, + "sim_memory": 0.4933301, + "sim_abstain": 0.4436864 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5989759, + "sim_memory": 0.55812705, + "sim_abstain": 0.50408465 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65531147, + "sim_memory": 0.639098, + "sim_abstain": 0.554867 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.57362914, + "sim_memory": 0.5315194, + "sim_abstain": 0.5000413 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6955397, + "sim_memory": 0.69094855, + "sim_abstain": 0.58473253 + }, + { + "query_id": "rf_test_016", + "text": "read interrupt.rs and tell me what the Interrupt enum contains", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.64426523, + "sim_memory": 0.5728988, + "sim_abstain": 0.5663854 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5350984, + "sim_memory": 0.51690584, + "sim_abstain": 0.44147617 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6582903, + "sim_memory": 0.59947485, + "sim_abstain": 0.54133034 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.63669294, + "sim_memory": 0.56342334, + "sim_abstain": 0.53770316 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6906057, + "sim_memory": 0.61367184, + "sim_abstain": 0.5702823 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6797439, + "sim_memory": 0.5738374, + "sim_abstain": 0.59381026 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.57380635, + "sim_memory": 0.53385437, + "sim_abstain": 0.4801312 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5531322, + "sim_memory": 0.5623229, + "sim_abstain": 0.511596 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65511966, + "sim_memory": 0.64622855, + "sim_abstain": 0.5482988 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5675564, + "sim_memory": 0.52146643, + "sim_abstain": 0.42315438 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.68739843, + "sim_memory": 0.60280913, + "sim_abstain": 0.58547026 + }, + { + "query_id": "rf_test_031", + "text": "read the first 10 lines of agent.rs", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6010007, + "sim_memory": 0.5215032, + "sim_abstain": 0.54224586 + }, + { + "query_id": "wf_test_001", + "text": "set the compaction_keep to 15 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6928355, + "sim_memory": 0.6293883, + "sim_abstain": 0.5867397 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6533637, + "sim_memory": 0.6245829, + "sim_abstain": 0.5488851 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.57442784, + "sim_memory": 0.50822705, + "sim_abstain": 0.52038395 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.65217215, + "sim_memory": 0.57864666, + "sim_abstain": 0.5854702 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6571172, + "sim_memory": 0.61622095, + "sim_abstain": 0.5631379 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.61865425, + "sim_memory": 0.54853326, + "sim_abstain": 0.5347757 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7086531, + "sim_memory": 0.6048335, + "sim_abstain": 0.6337338 + }, + { + "query_id": "wf_test_008", + "text": "patch the memory_top_k to 5 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.74933153, + "sim_memory": 0.6733637, + "sim_abstain": 0.612807 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6457346, + "sim_memory": 0.61097914, + "sim_abstain": 0.5961099 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.58776027, + "sim_memory": 0.550458, + "sim_abstain": 0.51394325 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.62338835, + "sim_memory": 0.5759057, + "sim_abstain": 0.5573286 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6764556, + "sim_memory": 0.6361264, + "sim_abstain": 0.5889328 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5793962, + "sim_memory": 0.52635694, + "sim_abstain": 0.48974553 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6092324, + "sim_memory": 0.54605097, + "sim_abstain": 0.53979576 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70279783, + "sim_memory": 0.6675291, + "sim_abstain": 0.61644477 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.52980995, + "sim_memory": 0.4559463, + "sim_abstain": 0.453048 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.59652096, + "sim_memory": 0.54976785, + "sim_abstain": 0.5258496 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-4/router_report.md b/benchmarks/models/router/centroid/out-router-iter-4/router_report.md new file mode 100644 index 0000000..e2f599c --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-4/router_report.md @@ -0,0 +1,90 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `160` memory: `92` abstain: `15` +- accuracy: `0.5106` +- tools precision: `1.0000` +- tools recall: `0.1625` +- memory false-tools rate: `0.0000` + +## Misclassified Queries + +| id | text | gold | pred | T-sim | M-sim | A-sim | +|---|---|---|---|---|---|---| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.71 | 0.69 | 0.62 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 | +| `shell_test_001` | what processes are listening on port 8003? | `Tools` | `Abstain` | 0.66 | 0.57 | 0.57 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Abstain` | 0.67 | 0.56 | 0.57 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | 0.65 | 0.55 | 0.58 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.70 | 0.62 | 0.68 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.69 | 0.58 | 0.60 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.66 | 0.60 | 0.55 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Abstain` | 0.61 | 0.53 | 0.54 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.70 | 0.65 | 0.58 | +| `shell_test_010` | what's the uptime since klbr daemon last started | `Tools` | `Memory` | 0.65 | 0.55 | 0.51 | +| `shell_test_011` | how big is agent.db right now on disk? | `Tools` | `Memory` | 0.62 | 0.54 | 0.53 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.65 | 0.56 | 0.55 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.66 | 0.57 | 0.62 | +| `shell_test_014` | how many memory rows are in agent.db right now? | `Tools` | `Memory` | 0.68 | 0.59 | 0.56 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.59 | 0.55 | 0.52 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.69 | 0.66 | 0.59 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.62 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.61 | +| `shell_test_020` | whats the total vram on ths gpu | `Tools` | `Memory` | 0.62 | 0.52 | 0.52 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.58 | 0.52 | 0.52 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.65 | 0.54 | 0.59 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.59 | 0.56 | 0.57 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.65 | 0.57 | 0.61 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.68 | 0.61 | 0.61 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.61 | 0.53 | 0.55 | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | 0.57 | 0.49 | 0.47 | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | 0.59 | 0.54 | 0.52 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.67 | 0.57 | 0.60 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.67 | 0.62 | 0.55 | +| `rf_test_002` | which file defines the ClientMsg type? | `Tools` | `Memory` | 0.53 | 0.44 | 0.44 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.66 | 0.65 | 0.53 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.60 | 0.56 | 0.54 | +| `rf_test_006` | config.rsのmemory_support_score_gapのデフォルト値はいくつ? | `Tools` | `Memory` | 0.71 | 0.63 | 0.55 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.70 | 0.65 | 0.59 | +| `rf_test_010` | what's in the .gitignore for klbr? | `Tools` | `Memory` | 0.59 | 0.49 | 0.44 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.60 | 0.56 | 0.50 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.66 | 0.64 | 0.55 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.57 | 0.53 | 0.50 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.70 | 0.69 | 0.58 | +| `rf_test_016` | read interrupt.rs and tell me what the Interrupt enum contains | `Tools` | `Memory` | 0.64 | 0.57 | 0.57 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | 0.54 | 0.52 | 0.44 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.66 | 0.60 | 0.54 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.64 | 0.56 | 0.54 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.69 | 0.61 | 0.57 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.68 | 0.57 | 0.59 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.57 | 0.53 | 0.48 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.55 | 0.56 | 0.51 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.66 | 0.65 | 0.55 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.57 | 0.52 | 0.42 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.69 | 0.60 | 0.59 | +| `rf_test_031` | read the first 10 lines of agent.rs | `Tools` | `Abstain` | 0.60 | 0.52 | 0.54 | +| `wf_test_001` | set the compaction_keep to 15 in config.rs | `Tools` | `Memory` | 0.69 | 0.63 | 0.59 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.65 | 0.62 | 0.55 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Abstain` | 0.57 | 0.51 | 0.52 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Abstain` | 0.65 | 0.58 | 0.59 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.62 | 0.56 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.62 | 0.55 | 0.53 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.71 | 0.60 | 0.63 | +| `wf_test_008` | patch the memory_top_k to 5 in config.rs | `Tools` | `Memory` | 0.75 | 0.67 | 0.61 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.65 | 0.61 | 0.60 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.59 | 0.55 | 0.51 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.62 | 0.58 | 0.56 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.68 | 0.64 | 0.59 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.58 | 0.53 | 0.49 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.61 | 0.55 | 0.54 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.70 | 0.67 | 0.62 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | 0.53 | 0.46 | 0.45 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.55 | 0.53 | diff --git a/benchmarks/models/router/centroid/out-router-iter-5/router_model.json b/benchmarks/models/router/centroid/out-router-iter-5/router_model.json new file mode 100644 index 0000000..a1bd52a --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-5/router_model.json @@ -0,0 +1,6168 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroids": [ + [ + -0.02953457, + -0.023501687, + -0.05808627, + 0.0070140953, + 0.006874998, + -0.047377475, + 0.013503848, + 0.004033102, + 0.013937307, + 0.025993194, + 0.00036805047, + 0.026340032, + -0.00032751352, + 0.020008639, + 0.010437691, + -0.01599532, + 0.0378925, + -0.034139622, + 0.031576242, + -0.005721982, + -0.015650975, + -0.013299033, + 0.0015801851, + 0.05208871, + -0.0033491016, + 0.05292174, + -0.041163854, + -0.013427372, + 0.018127806, + 0.031762872, + 0.0080265, + 0.027610881, + 0.0102199325, + -0.02962109, + -0.025770932, + -0.027322631, + 0.018344872, + -0.01546087, + -0.040318888, + -0.024949342, + -0.02684663, + 0.019792331, + 0.016403709, + -0.03913061, + 0.041741397, + -0.051342268, + 0.03370111, + -0.041125838, + -0.0036108876, + 0.015794855, + -0.018666392, + -0.014134196, + 0.015846388, + -0.02152241, + 0.04721869, + 0.038556382, + -0.0057476372, + 0.01546458, + -0.04171342, + -0.007379216, + -0.0137405945, + -0.015948497, + -0.015352586, + 0.011297628, + 0.021778889, + 0.09534737, + 0.027794316, + -0.002793605, + -0.020545956, + -0.041483287, + -0.025141848, + 0.006380157, + -0.00047512646, + -0.0032311745, + -0.08699265, + 0.02291205, + 0.049483027, + -0.022220165, + 0.0006178035, + 0.015320561, + 0.047333814, + -0.006275665, + 0.034674358, + 0.014853668, + -0.0034956613, + 0.014414248, + -0.028953297, + 0.035040356, + 0.0029728715, + -0.031037042, + -0.010100751, + 0.023466574, + 0.013175919, + -0.052728992, + -0.046121895, + 0.0069394163, + -0.029717263, + -0.013612056, + 0.039096978, + 0.006514509, + 0.026238186, + 0.011335962, + 0.015135213, + -0.030236373, + 0.052462798, + -0.009011261, + 0.04137082, + 0.036443684, + -0.024784643, + -0.010885915, + 0.02339825, + 0.01153446, + 0.019820947, + 0.01667984, + -0.02987813, + -0.03124927, + -0.033248678, + -0.026805358, + 0.05532143, + 0.006061382, + 0.05987853, + 0.0053289263, + 0.06786469, + -0.03602914, + -0.015517126, + -0.04250494, + 0.020546658, + 0.012809727, + -0.00601842, + 0.008231562, + -0.004753981, + 0.0181044, + -0.02190586, + 0.0015789795, + -0.0386283, + -0.015395165, + 0.013448149, + -0.0020003526, + -0.026016362, + -0.05009787, + 0.033090923, + 0.029800063, + -0.030307502, + -0.027842177, + 0.0139083, + -0.019292444, + 0.008088462, + -0.011791759, + 0.013513014, + 0.0027809977, + -0.015044234, + -0.0142502105, + -0.011962004, + -0.013559485, + 0.025546454, + -0.0144509105, + 0.022906901, + 0.08040018, + -0.024976632, + 0.05750807, + -0.015728489, + 0.025379393, + -0.042068284, + -0.011159206, + 0.045110248, + 0.0027386474, + 0.01061638, + 0.037824012, + -0.015301461, + -0.010647753, + -0.019499803, + -0.016362367, + 0.029056897, + -0.016690247, + 0.02505767, + 0.020635538, + 0.10579964, + 0.017164556, + 0.010356794, + -0.04950358, + -0.047854308, + 0.012633971, + 0.020412799, + -0.03777352, + -0.019705521, + -0.021516891, + 0.037748206, + -0.05184238, + 0.015875751, + 0.032903414, + 0.00286073, + -0.0017315428, + 0.03207537, + 0.0024716682, + 0.036519893, + 0.019512344, + -0.005500415, + -0.028663289, + 0.006004013, + -0.028784506, + -0.009142338, + 0.037859693, + 0.063061036, + -0.007193967, + -0.059196513, + -0.00025405674, + -0.0404329, + -0.050101977, + 0.018440584, + 0.0019876033, + -0.018863054, + 0.01567874, + 0.02782502, + 0.0050265174, + -0.035830576, + 0.00025280323, + 0.024342729, + -0.0027864513, + -0.011164368, + -0.012470029, + -0.020127311, + 0.044781983, + -0.016764725, + -0.060266845, + 0.02845582, + 0.0326934, + 0.0023419242, + -0.0018993955, + -0.005613019, + -0.049475726, + -0.0010083856, + 0.024441615, + -0.022721602, + -0.024691395, + 0.034620766, + -0.00882457, + 0.0016938613, + -0.003250522, + 0.013695728, + -0.013721317, + 0.0021146091, + -0.028904684, + -0.0029565294, + -0.0042853206, + -0.05602909, + -0.0023269707, + 0.011781264, + 0.013156883, + 0.0032094966, + 0.013319434, + 0.027515475, + 0.00003426793, + -0.04807129, + -0.02646676, + 0.055103097, + 0.03457307, + 0.032444008, + -0.0027100584, + 0.024296409, + 0.0019906394, + 0.03768081, + 0.0022763326, + 0.011797177, + 0.036109027, + 0.026424909, + -0.0031692882, + -0.024819322, + -0.029096847, + 0.0043520858, + 0.0067918655, + -0.03163525, + 0.0048874402, + 0.022340473, + 0.023326822, + 0.03814576, + 0.026113065, + -0.03389849, + -0.035517395, + 0.05969392, + 0.01776524, + -0.012784303, + 0.013127809, + 0.022363255, + -0.013977365, + -0.0039680754, + -0.017222684, + -0.02858304, + -0.011452531, + -0.007811619, + -0.045651883, + 0.014498947, + -0.0013878772, + 0.100820094, + 0.005297464, + -0.028213173, + -0.03314273, + -0.04044471, + -0.22465427, + -0.02858451, + -0.003905871, + 0.00022481597, + 0.0047072093, + -0.012649562, + -0.017003896, + 0.006186442, + -0.03956627, + 0.036402363, + -0.051759057, + -0.068457276, + 0.007101277, + -0.0124830175, + -0.019360466, + 0.0033654436, + -0.011729769, + -0.009554513, + 0.018463207, + -0.03167895, + -0.006967882, + -0.019700885, + 0.023060178, + -0.020346511, + -0.0061216676, + -0.014901207, + 0.028405404, + 0.022600891, + -0.03975633, + -0.020692414, + 0.017461983, + 0.027074642, + -0.017387684, + 0.047598917, + 0.013716522, + 0.022675045, + 0.021145523, + -0.017724916, + -0.026462242, + -0.014224281, + 0.030817714, + 0.0114215035, + -0.021204576, + 0.034436375, + 0.024465613, + -0.045439, + -0.013454584, + -0.030626189, + -0.0077004014, + -0.0046026013, + -0.027327102, + 0.012236565, + 0.0023989333, + -0.0047771633, + -0.02984797, + -0.008682544, + -0.016193328, + 0.039879285, + -0.06660901, + 0.012391835, + -0.0049603703, + 0.0007947687, + 0.016125074, + 0.040164527, + -0.033208624, + 0.010781524, + -0.025258325, + -0.020662768, + 0.03224673, + -0.023760425, + 0.030194627, + -0.0016037951, + 0.0024073026, + -0.011888995, + -0.021232078, + 0.053651586, + -0.013994518, + -0.039026722, + -0.018290367, + -0.1595071, + -0.010143831, + -0.009503269, + 0.0075196954, + -0.0016893324, + -0.023991065, + -0.0004729232, + 0.005407145, + 0.045051403, + 0.03946149, + 0.31316122, + 0.019875187, + -0.017683508, + -0.011246197, + 0.041095726, + -0.01707569, + 0.02580701, + 0.020073302, + 0.0076661776, + -0.039371304, + -0.030318934, + 0.013346287, + 0.020821072, + 0.0046413513, + 0.016254276, + 0.005412236, + -0.03347545, + 0.010158307, + 0.1002082, + -0.011839762, + 0.03285164, + 0.012760006, + 0.02366014, + 0.022854084, + -0.05203756, + -0.03471872, + -0.009616535, + 0.058146782, + -0.020818148, + 0.03461769, + -0.026271882, + 0.021051634, + 0.02911293, + -0.01990685, + -0.041703526, + 0.0002907327, + 0.013849215, + 0.016160823, + 0.0024423078, + 0.03202622, + -0.0015427561, + -0.042227678, + -0.001353875, + -0.013348268, + -0.0008665041, + -0.042891808, + -0.006410304, + -0.02022797, + -0.03159226, + -0.022437904, + -0.02003756, + 0.015528935, + -0.017203221, + -0.0100704925, + -0.010856735, + -0.021080589, + -0.0504776, + 0.010053538, + -0.0030528405, + 0.027641593, + 0.027326025, + 0.0128613785, + -0.046285115, + -0.010928768, + -0.03181085, + 0.017708378, + -0.029218853, + 0.00059574813, + 0.031865608, + 0.013683232, + 0.0025548658, + -0.006452052, + 0.008344617, + -0.0018459023, + 0.04586065, + 0.007935226, + 0.00002709333, + 0.08221116, + -0.009100065, + 0.029175958, + -0.048444733, + -0.008251141, + -0.03405043, + 0.032034725, + 0.007967471, + -0.024803922, + -0.015552689, + 0.05681926, + 0.0037742213, + -0.026362995, + -0.007213026, + 0.014125172, + -0.03752151, + -0.0024824142, + 0.012550943, + 0.008332322, + 0.015813878, + -0.011721951, + -0.03736512, + -0.025358934, + -0.041780002, + -0.016802674, + 0.0075594457, + 0.012926832, + 0.0022665393, + 0.00544774, + -0.008000439, + 0.015729414, + 0.025527108, + 0.015055225, + -0.02852011, + 0.029998757, + -0.023629876, + -0.019701436, + 0.046159368, + 0.010666163, + -0.009261453, + 0.027201602, + -0.005710259, + 0.03726728, + -0.01976218, + -0.023621652, + -0.0029942847, + -0.00047694452, + -0.0040939855, + -0.0099202385, + 0.0065422435, + -0.0429584, + -0.031387664, + 0.022027241, + 0.026100025, + -0.007511666, + 0.030582216, + 0.012822923, + 0.036304314, + 0.025152108, + -0.017161628, + 0.016741537, + 0.018255189, + 0.035726435, + -0.0062032174, + -0.05698027, + -0.013355153, + 0.014536069, + -0.01766706, + -0.008239166, + 0.018501902, + -0.0010640995, + -0.018297568, + 0.019297598, + 0.01495364, + 0.036746204, + -0.02745614, + -0.055277172, + -0.029027686, + -0.032996736, + -0.010148253, + -0.0014721167, + -0.023568414, + -0.040075224, + 0.019336946, + 0.0005275909, + 0.006581408, + 0.05709383, + 0.039294206, + 0.040886384, + -0.032870255, + 0.040065862, + 0.03958012, + -0.042370867, + 0.0011705015, + -0.012909272, + -0.033453353, + 0.037587993, + 0.025471324, + -0.008906781, + -0.06545729, + 0.0017291277, + 0.008953089, + 0.016879518, + 0.026898118, + 0.027753143, + -0.03992264, + -0.004906459, + 0.0039307103, + 0.026820708, + -0.031330388, + -0.024806987, + 0.031612832, + -0.032069124, + -0.014307158, + 0.12864976, + -0.0051088952, + 0.004911811, + 0.005624591, + 0.010622269, + 0.014215928, + 0.012548023, + -0.030215545, + -0.029370805, + 0.0002464952, + 0.017944876, + 0.001236656, + -0.028763784, + 0.023505056, + 0.022390043, + -0.0015424239, + 0.027351622, + -0.04140385, + -0.030692495, + 0.0076649394, + -0.0017135493, + -0.046149958, + 0.0039872695, + 0.025925033, + -0.007340319, + 0.046820674, + 0.039672777, + -0.011907181, + -0.017452894, + -0.00021526583, + 0.0139043005, + 0.029623425, + -0.055875365, + -0.020226091, + 0.00017796963, + -0.030965574, + -0.06364141, + 0.013530132, + 0.008922023, + 0.014563754, + 0.0031031477, + -0.0030001134, + -0.030106964, + 0.03502163, + -0.0019424554, + -0.0020550822, + 0.03134754, + 0.0045823157, + -0.019614106, + -0.0048672026, + -0.022130122, + -0.009755478, + 0.03228172, + -0.019890655, + 0.008002598, + -0.00781725, + -0.011763561, + 0.01594538, + 0.03417074, + 0.011124715, + 0.00860674, + 0.0053270827, + -0.018849868, + -0.019411609, + 0.016141795, + 0.029577173, + -0.012376068, + -0.028442422, + -0.0102799805, + -0.015962493, + 0.04011505, + -0.035837717, + -0.013337596, + -0.050945375, + -0.013998839, + 0.04386832, + -0.025864763, + -0.03339192, + -0.047449946, + 0.004850084, + -0.015017307, + -0.062756956, + -0.04656785, + 0.021192448, + 0.00956155, + -0.05119442, + -0.008371866, + -0.0042842724, + 0.019156162, + -0.010939484, + 0.018840468, + -0.025657684, + 0.01106568, + -0.03244388, + 0.006420251, + 0.04826801, + -0.00057971664, + -0.006369808, + -0.031549037, + 0.049913317, + 0.007017293, + -0.030578466, + 0.02232669, + -0.04201414, + 0.026563568, + -0.015477179, + 0.026783573, + -0.022415739, + 0.016253402, + -0.025808347, + 0.052372962, + -0.05774024, + -0.011736604, + -0.010124982, + -0.0048620454, + -0.019803265, + -0.008477471, + 0.030738277, + 0.038898382, + 0.023150876, + -0.042604998, + 0.0019641898, + 0.00784871, + -0.004138505, + 0.011478454, + 0.030663332, + -0.01506021, + -0.02919572, + 0.012824615, + -0.0012030387, + 0.005259654, + -0.01432216, + -0.009896729, + -0.020819265, + -0.0027348117, + 0.00016687928, + 0.008219463, + -0.008309997, + -0.0117734615, + -0.017596774, + -0.007878184, + -0.011875525, + -0.011689367, + 0.048842385, + 0.0032761202, + -0.004180586, + -0.02784116, + 0.019389883, + 0.0059328643, + 0.017998025, + -0.0043417537, + 0.03363939, + -0.04814994, + 0.017516242, + -0.047206663, + 0.008795233, + 0.006181405, + -0.0076484066, + 0.021008575, + 0.0067271558, + 0.026135894, + -0.030862615, + -0.059556678, + -0.005249494, + -0.005896798, + -0.053860072, + 0.058404587, + 0.036468524, + 0.014157678, + -0.028333092, + 0.03760916, + 0.00519971, + 0.009662147, + -0.008499046, + 0.027133144, + 0.014945763, + -0.022028917, + 0.0066466467, + -0.012278338, + -0.025636343, + -0.0009813369, + -0.023540856, + -0.0014314126, + -0.016507363, + 0.045651384, + -0.0082562985, + -0.0041743293, + 0.05165129, + 0.025758864, + -0.029776005, + -0.012274611, + 0.01466031, + -0.038974468, + 0.0070445114, + -0.007370479, + -0.027545778, + -0.013694657, + 0.043757852, + 0.04633227, + -0.021911979, + -0.01796987, + -0.054099757, + 0.01452772, + -0.2138702, + 0.02743569, + 0.015744813, + -0.020886457, + -0.051552296, + -0.0027605426, + -0.0012711033, + -0.0047403597, + 0.009300366, + -0.029458603, + -0.02747486, + 0.025445733, + 0.0050143576, + -0.012034659, + -0.016319465, + 0.019868318, + -0.013839318, + -0.00029665322, + 0.009236322, + 0.018608531, + 0.0050179, + -0.017188085, + 0.03464569, + 0.005022059, + 0.02257357, + -0.026489973, + -0.035256382, + 0.0122413905, + -0.057871927, + -0.009198643, + 0.002898138, + 0.0062493277, + 0.032887958, + 0.047671497, + -0.026613321, + -0.005649091, + -0.0041015684, + -0.0281537, + 0.003319403, + -0.021404069, + -0.002427804, + 0.04324175, + -0.014130836, + -0.031917486, + -0.011499083, + 0.030707614, + -0.0028441278, + 0.018666685, + -0.06646727, + 0.019018704, + -0.004152146, + 0.01975428, + -0.013143967, + 0.0073776282, + 0.020081295, + -0.02697742, + -0.021839116, + 0.017184513, + 0.027456801, + 0.03954958, + 0.0016815886, + -0.016831884, + -0.027606057, + -0.050559748, + -0.051494483, + -0.012649485, + -0.07952543, + 0.004067631, + -0.0021464548, + 0.00786993, + -0.023989927, + -0.024670297, + -0.0037496786, + -0.04951884, + 0.016926184, + -0.018860383, + 0.012256827, + -0.010916814, + -0.010613337, + -0.025584185, + -0.0030827215, + 0.016111253, + -0.04742576, + -0.024564615, + 0.042982895, + 0.002380033, + -0.024394304, + 0.0062498963, + -0.009741223, + 0.00893389, + -0.06926393, + 0.017702833, + 0.003559851, + 0.023049349, + -0.012283555, + 0.03369362, + -0.012401988, + 0.0050276346, + -0.0166348, + -0.022107374, + -0.059628308, + -0.0036324025, + -0.011819672, + 0.026103102, + -0.014030879, + -0.03181233, + 0.03724609, + -0.010170614, + 0.0054339855, + -0.0066714897, + -0.03532086, + -0.015470396, + -0.019201776, + 0.010452439, + -0.035049714, + 0.008742536, + 0.017332299, + -0.0028451565, + -0.03990205, + 0.0027475304, + -0.000050377344, + 0.033374507, + -0.011835915, + -0.044426147, + -0.0012377469, + -0.0005448537, + 0.054194182, + -0.0016887394, + -0.009132979, + -0.014007341, + 0.048595123, + -0.024569178, + -0.010165418, + 0.039706923, + 0.009360816, + -0.0142467925, + -0.02289829, + 0.046355814, + -0.01972842, + -0.051768117, + 0.008552698, + 0.014160438, + 0.012196202, + -0.0075761625, + -0.019158937, + 0.0119345505, + 0.007685015, + 0.016935337, + -0.051301133, + -0.015521768, + 0.03698387, + 0.010485435, + 0.0012461421, + 0.009096625, + 0.009374426, + 0.004323882, + 0.022198806, + 0.025565172, + -0.023361158, + 0.059670344, + 0.036556274, + -0.038306747, + -0.034494575, + -0.04563513, + -0.0052286144, + 0.0026300608, + 0.022318833, + -0.0048725726, + -0.0037535578, + 0.0087553235, + -0.0037525538, + 0.052146267, + -0.02985571, + 0.07571844, + 0.014147077, + -0.050934263, + 0.041672062, + -0.0042912466, + 0.042178534, + -0.016792161, + 0.029752152, + -0.020074084, + -0.0091867, + 0.023213692, + 0.006531691, + 0.015507917, + 0.028167203, + 0.050762765, + 0.010745568, + 0.0008412688, + 0.01909061, + -0.009928744, + -0.0028815789, + -0.04545366, + 0.042247232, + -0.010490567, + 0.020409765, + 0.025339702, + -0.035115454, + 0.061531704, + -0.011911052, + -0.03500391, + -0.04524361, + -0.013206155, + -0.022409508, + -0.008728653, + -0.034059823, + -0.02689995, + -0.03300723, + 0.043033533, + -0.0290939, + 0.0121105155, + -0.02727977, + 0.0019806651, + 0.0068177576, + 0.00031189778, + 0.010153875, + 0.024259033, + 0.02276865, + -0.02847139, + -0.010577961, + 0.010186096, + 0.0016031668, + -0.04977359, + -0.034086004, + -0.009296076, + -0.015570667, + -0.007774638, + -0.006385688, + -0.011167805, + -0.021814104, + 0.03431478, + -0.037030302, + 0.032789752, + 0.015891775, + 0.00091708323, + 0.05153089, + 0.025300793, + -0.004147137, + -0.002914364, + 0.047010057, + -0.02837259, + 0.003133865, + 0.029014219 + ], + [ + -0.020254124, + -0.04116675, + -0.007372979, + -0.029964684, + 0.011190738, + -0.08159443, + 0.017661301, + -0.010246391, + -0.008209263, + 0.022077559, + -0.037495364, + 0.038561653, + 0.041310087, + -0.03827206, + -0.0030296666, + -0.04551725, + -0.0015584466, + -0.043835424, + 0.04111606, + -0.0074276403, + -0.014586625, + -0.034484424, + -0.031067863, + -0.0055164816, + 0.002870236, + 0.03311342, + 0.0054400344, + 0.020860758, + -0.012762298, + 0.020914592, + -0.037719376, + 0.019643847, + -0.026980707, + -0.01616462, + 0.0057277605, + -0.018851377, + 0.013205356, + 0.026758894, + -0.05611753, + -0.05673022, + -0.015809074, + 0.022022985, + 0.064844824, + 0.009851467, + -0.0074210535, + -0.02909688, + 0.012350218, + -0.022363547, + -0.01382957, + -0.00794799, + 0.010876325, + 0.051783428, + -0.035475973, + -0.0031464773, + 0.07489341, + 0.034771163, + -0.041550077, + 0.034477636, + -0.019864663, + 0.0066035762, + 0.025300935, + 0.041910917, + 0.018587649, + 0.020312272, + 0.017726174, + 0.073004626, + -0.005812487, + 0.016792096, + -0.025332022, + 0.0049700495, + -0.053332593, + 0.04201889, + 0.004391433, + -0.0021935226, + -0.09358781, + 0.03083754, + 0.019159859, + 0.004591123, + 0.02774286, + 0.04111074, + 0.04572332, + -0.03794063, + -0.008300797, + 0.01320226, + 0.016294988, + 0.032590497, + 0.014050795, + 0.03014458, + -0.024947153, + -0.028386062, + -0.0069098324, + 0.016842715, + -0.0002635318, + -0.052521348, + -0.056614153, + 0.0476653, + -0.002092335, + 0.0067270766, + 0.034543928, + 0.06673301, + -0.008618899, + 0.003757926, + 0.0049299044, + 0.04859857, + -0.028685993, + -0.047352344, + 0.06761084, + 0.0026073721, + -0.03719376, + -0.008147757, + 0.002335019, + -0.055548757, + -0.019844908, + 0.0003014616, + -0.04448501, + -0.053816702, + -0.06500549, + 0.033419482, + 0.023317087, + -0.012048277, + 0.045431584, + 0.0044442685, + 0.00503645, + -0.023623671, + 0.036882322, + -0.03137511, + 0.03476906, + -0.0004156654, + -0.03795088, + -0.035963524, + -0.058573436, + 0.029147573, + -0.04297972, + -0.011300198, + -0.04443065, + -0.010125407, + -0.035875123, + 0.005237153, + 0.018849581, + -0.04374896, + -0.006881187, + -0.01323515, + -0.020799844, + -0.028531378, + 0.03373202, + -0.019024812, + -0.023178076, + -0.013414461, + 0.022442773, + -0.022926392, + 0.017389378, + -0.00240939, + -0.05073249, + -0.044891775, + -0.015401027, + -0.023299117, + 0.018395206, + 0.024170302, + -0.013018237, + 0.014878734, + -0.032968245, + 0.008014404, + -0.018779429, + -0.045670286, + 0.008193353, + -0.024559846, + 0.057381175, + 0.0669771, + 0.044439804, + -0.0036185777, + 0.021347538, + 0.048340894, + 0.026817888, + 0.058665168, + 0.023300119, + 0.026641518, + 0.039867714, + 0.0049528927, + -0.0011300519, + -0.037390184, + -0.06264999, + -0.01630887, + 0.008078938, + 0.0010589276, + 0.020439938, + 0.025579099, + -0.02067246, + -0.02252045, + 0.00242239, + 0.027789189, + -0.028604046, + -0.010315909, + 0.045320645, + -0.008987895, + -0.023577353, + 0.032551922, + 0.04230767, + -0.003796409, + 0.00983521, + -0.022642719, + 0.004940319, + 0.01627678, + 0.047532525, + -0.06016555, + 0.011346706, + -0.005472442, + -0.018862773, + -0.035922226, + -0.010526478, + -0.015118645, + 0.014042482, + 0.009215666, + 0.01457269, + -0.030881861, + 0.015838949, + -0.021441704, + 0.033870716, + -0.0088826725, + -0.05239024, + -0.015585623, + -0.034875818, + -0.032844577, + 0.011226785, + -0.015373432, + 0.015121698, + -0.00072043814, + -0.0068776575, + 0.06292077, + -0.009777719, + -0.04709221, + -0.029894646, + 0.0027344727, + -0.008795243, + -0.017512025, + 0.04226805, + 0.025590824, + -0.008859926, + 0.04388061, + 0.05875191, + 0.0108080255, + -0.031908263, + 0.01467872, + -0.014533983, + 0.015212947, + 0.02041433, + 0.03890462, + -0.024720386, + 0.006734559, + -0.023964731, + 0.01345108, + 0.014990413, + 0.0030772437, + -0.058163464, + 0.047327667, + 0.00489635, + 0.005195013, + 0.0001313037, + -0.016700808, + 0.047128133, + 0.009359673, + 0.065170325, + 0.020720795, + 0.010661119, + 0.0096793715, + -0.007632, + -0.047226686, + 0.0056292997, + -0.018743971, + -0.021640094, + 0.010677068, + -0.033116397, + -0.0020473609, + -0.0034470889, + 0.048381247, + 0.04437471, + 0.024240568, + 0.005486285, + -0.05120708, + 0.0025768455, + -0.015763147, + 0.05396275, + -0.046657037, + -0.010277123, + 0.05113114, + 0.03595713, + -0.005959823, + -0.065834485, + 0.019947264, + 0.00816652, + -0.056714363, + -0.029086692, + -0.006467168, + 0.020627813, + 0.00316757, + -0.007847554, + -0.0034523793, + -0.06581499, + -0.124769256, + -0.0475397, + -0.02573181, + -0.027566018, + 0.012921132, + -0.036089245, + -0.04352001, + -0.015061321, + -0.038911663, + -0.014494528, + -0.02425402, + -0.05302295, + 0.018130956, + -0.017570471, + 0.003728436, + -0.022605387, + 0.020288603, + 0.0016745474, + 0.015943542, + 0.013685553, + -0.020636661, + 0.0014871317, + 0.036179543, + 0.0666887, + 0.018872928, + -0.011584086, + 0.0346419, + 0.0010764768, + 0.015926719, + -0.01975337, + 0.008983314, + 0.010039464, + -0.006696494, + 0.0674074, + -0.025392266, + -0.009440608, + -0.014110286, + 0.029129889, + -0.0061896862, + -0.016916197, + 0.017264564, + -0.023885023, + 0.014502518, + 0.014711948, + -0.024939105, + -0.03776957, + 0.01799964, + -0.025129559, + -0.040642563, + 0.01042123, + 0.011569937, + 0.02311178, + -0.009573609, + 0.022429014, + 0.0010554203, + -0.021534372, + 0.009123053, + 0.07520331, + -0.022060657, + 0.0064048725, + -0.026625117, + 0.016196016, + 0.004994355, + 0.05129052, + -0.07573883, + -0.019806353, + -0.0052977004, + -0.02585432, + -0.008690205, + 0.0102756955, + -0.0064524813, + 0.02608897, + 0.014926429, + -0.0153515255, + 0.014566396, + 0.027495226, + -0.05981297, + 0.013260812, + -0.017695569, + -0.103219524, + -0.060673494, + -0.015960708, + -0.01404399, + -0.000095332405, + 0.030360384, + 0.0070420117, + -0.0052885753, + 0.0328839, + -0.012404965, + 0.17907244, + 0.026853886, + -0.0055063153, + -0.019140543, + 0.022881936, + -0.0054624556, + -0.020305462, + -0.0040248292, + -0.008616788, + -0.023474207, + -0.039994232, + -0.051367916, + 0.0506136, + -0.010000171, + -0.02576667, + 0.03985849, + -0.0047355406, + 0.02381105, + 0.058388382, + 0.020415135, + -0.007172144, + 0.034434605, + -0.008059745, + 0.009382488, + -0.0031786328, + -0.04825492, + 0.0090839295, + 0.055844855, + 0.01206461, + 0.04200518, + 0.008154144, + 0.00574395, + 0.026797764, + 0.034558475, + -0.06504741, + -0.011473228, + -0.041974258, + 0.008139225, + 0.022806998, + 0.043154757, + -0.011877348, + -0.016530937, + 0.04076686, + 0.018341811, + 0.021837514, + -0.025804339, + 0.007451655, + -0.07720817, + -0.029209508, + 0.03716037, + -0.010401743, + 0.0071976758, + 0.025213012, + -0.044804268, + -0.021039924, + 0.0035612402, + -0.031583317, + -0.015959581, + -0.009585654, + -0.0027469269, + 0.020826988, + 0.0188638, + -0.005792691, + -0.023081234, + 0.0036363788, + -0.0061280904, + -0.0006764033, + 0.019075336, + 0.055916227, + 0.02952431, + 0.049550064, + 0.04268436, + -0.025420789, + 0.087470114, + 0.057473727, + 0.07013039, + 0.012614301, + 0.02380244, + -0.029213306, + 0.008438178, + -0.023142328, + -0.0111113805, + -0.020476148, + -0.009346526, + -0.01193406, + -0.006535818, + -0.025082134, + -0.008752324, + 0.0044953264, + -0.028365167, + 0.016156549, + 0.05029044, + -0.012121522, + -0.008995778, + -0.01904241, + 0.00071007677, + 0.023712004, + -0.03216477, + 0.012888373, + -0.00008797499, + -0.054210324, + -0.03657906, + 0.011120258, + -0.01213075, + -0.023088787, + 0.03602543, + -0.040457137, + -0.02300542, + -0.0150393285, + -0.01354496, + 0.0215167, + 0.05465438, + 0.028458241, + -0.0115915015, + 0.0059376894, + 0.023620417, + -0.057839744, + 0.013643552, + -0.022547174, + 0.017269548, + 0.03127004, + -0.07988349, + -0.05078919, + 0.011205213, + -0.026745658, + 0.0303635, + -0.03320978, + 0.019111408, + 0.023719456, + 0.012111504, + -0.003342989, + -0.0072348914, + 0.0030055698, + 0.0048688916, + 0.049683273, + 0.039333805, + 0.025220495, + 0.031403065, + 0.061865292, + 0.034643058, + 0.0126303295, + 0.001891789, + 0.0073859184, + -0.015541369, + -0.05020341, + -0.027491327, + 0.0065335324, + -0.00034843318, + -0.026190557, + 0.021528963, + -0.013683739, + 0.04296777, + -0.0032485006, + -0.014733479, + -0.011558493, + -0.0145478975, + 0.0024043447, + -0.029473959, + -0.0077641266, + 0.0071720267, + 0.017033016, + -0.021381047, + -0.018005215, + -0.022859164, + 0.0021507316, + 0.038735278, + -0.043056272, + 0.018944927, + 0.036730997, + -0.031711336, + -0.02730031, + 0.00054977584, + -0.01186276, + 0.051118262, + 0.014460639, + 0.028466199, + -0.021185437, + -0.06046371, + 0.02031086, + 0.031098083, + 0.02705517, + 0.00861357, + -0.023073023, + 0.040332895, + 0.0015463877, + 0.0057385997, + -0.047136843, + -0.020717619, + 0.035119373, + 0.019301428, + -0.011939856, + 0.06972406, + -0.06393006, + 0.021002274, + 0.01652762, + -0.0046303887, + -0.06746979, + -0.05053881, + 0.027266838, + -0.05120607, + 0.011592364, + -0.0007830686, + 0.026465403, + -0.061663467, + 0.076975554, + -0.025506932, + -0.021895196, + -0.019970143, + -0.007931396, + -0.030816864, + 0.01556704, + -0.012785671, + -0.010793555, + -0.0040854607, + 0.02405227, + -0.0097369645, + 0.003523927, + 0.061076246, + 0.0022672077, + -0.055630583, + 0.014753529, + 0.018235153, + -0.023194518, + -0.0654739, + 0.01683278, + -0.010466496, + -0.044046562, + -0.056332473, + 0.011695644, + 0.06960929, + 0.02427439, + -0.0040806774, + -0.03696452, + -0.029279653, + -0.014468639, + -0.0038482912, + 0.01170978, + -0.00013206947, + -0.0035896276, + -0.010911566, + 0.008625817, + -0.0061318427, + 0.011671429, + 0.049992807, + 0.002745769, + 0.0031697694, + -0.012702845, + 0.046768826, + 0.050013237, + -0.03934394, + -0.008645382, + 0.056805123, + -0.0068112477, + -0.02184254, + -0.039267957, + -0.015069269, + 0.00784059, + -0.010700507, + 0.029770492, + -0.0030366762, + 0.024371767, + 0.043727934, + -0.009476817, + -0.014411585, + 0.031865187, + 0.021500316, + -0.027218673, + -0.0048276596, + -0.009022345, + 0.008899309, + -0.023732318, + -0.04855289, + -0.032706082, + -0.08435636, + 0.0030194477, + 0.024007564, + -0.044421624, + 0.0057766563, + 0.04744126, + 0.020762306, + -0.016310865, + -0.0020700977, + -0.025132567, + 0.025563918, + -0.016010238, + -0.008089007, + 0.0058249095, + -0.038526654, + -0.01771741, + -0.025688788, + 0.015047329, + -0.0014794411, + -0.014519879, + -0.044879995, + -0.07035461, + 0.026975842, + -0.050283216, + 0.019084351, + 0.00018584942, + -0.018075237, + -0.032995265, + 0.051688373, + -0.009015666, + -0.022133581, + -0.01399461, + -0.03630475, + -0.008931077, + -0.019135477, + 0.042051293, + 0.055161092, + 0.020866865, + 0.030594286, + -0.018767308, + 0.023476098, + -0.005624278, + 0.0104467785, + -0.0336711, + -0.0070034605, + -0.023060434, + 0.007979617, + 0.016134398, + 0.055457216, + -0.036615178, + 0.0092480825, + -0.008378172, + -0.037848447, + -0.0020533525, + 0.013536509, + -0.025558382, + -0.034805316, + -0.03309949, + -0.03766695, + -0.038801335, + 0.01504471, + 0.04798496, + 0.03310903, + 0.012344884, + 0.010643564, + 0.02714804, + 0.011844159, + -0.017942522, + 0.035539888, + 0.031651203, + -0.042367086, + 0.03250244, + 0.019743834, + 0.0084450785, + 0.011004879, + 0.052851435, + -0.021892207, + 0.0071975337, + 0.013812723, + 0.0025931627, + -0.021047847, + -0.021003025, + 0.029310383, + -0.038826242, + 0.080266275, + -0.0007394495, + -0.004599728, + 0.035691377, + 0.007618633, + 0.0002773464, + 0.040907677, + -0.00071150775, + 0.02910722, + 0.026443645, + -0.007555279, + 0.039212402, + -0.054407403, + -0.03506119, + -0.07286829, + 0.022884298, + 0.02873324, + -0.014125804, + -0.013982294, + 0.004962992, + 0.030923832, + -0.01460802, + 0.006872219, + 0.0032361657, + 0.002350869, + -0.005383757, + -0.053609997, + 0.042491283, + -0.039554384, + -0.030929564, + -0.000628169, + -0.00087879057, + 0.040924307, + -0.013975817, + 0.041287072, + -0.032564737, + 0.01377594, + -0.099039905, + 0.0052515194, + 0.003380827, + -0.015652759, + -0.061392162, + -0.024736412, + -0.028811218, + 0.0068211644, + -0.00080923695, + -0.0145119205, + -0.006284979, + 0.023963012, + 0.030818338, + -0.019357484, + -0.026111864, + -0.018230068, + -0.00141907, + -0.03421442, + -0.014301486, + 0.04973083, + 0.016147241, + 0.0034317733, + 0.03079936, + -0.01633139, + 0.010416137, + 0.015177892, + -0.014367937, + 0.012394066, + -0.04084319, + 0.003077322, + -0.04215946, + 0.06567015, + 0.006222198, + 0.026923848, + 0.0003813074, + 0.021874087, + 0.029589854, + -0.021420741, + 0.032787703, + -0.03356351, + 0.02138357, + 0.027757693, + -0.014662574, + -0.013165381, + -0.030943656, + 0.023958629, + -0.016045582, + 0.029165346, + -0.03166436, + 0.0055215787, + 0.020196468, + 0.037619952, + 0.012425788, + -0.015718715, + -0.018558526, + -0.062954664, + -0.014405638, + -0.02019691, + 0.013439697, + -0.00090975413, + -0.03960515, + 0.01929136, + 0.014825985, + -0.011897969, + -0.07391725, + -0.0037334028, + -0.021778373, + -0.03345438, + -0.0048873485, + 0.03748898, + -0.006887464, + 0.033263568, + 0.026937481, + -0.026849639, + 0.015706737, + 0.004499359, + 0.026059866, + -0.030047212, + -0.035322253, + -0.013030768, + -0.09660178, + -0.020400189, + 0.014046086, + -0.040100582, + 0.023280697, + 0.0061984207, + -0.017089635, + 0.07199057, + -0.005007973, + 0.032175943, + -0.07026293, + 0.008396743, + 0.036889493, + -0.030547267, + -0.0665451, + 0.023350557, + -0.017915163, + -0.006056193, + -0.037609342, + -0.00739387, + -0.021556864, + 0.04219483, + -0.017056148, + 0.02643247, + -0.022912811, + 0.0022669423, + 0.022746345, + 0.0035540983, + 0.030907692, + 0.02280229, + -0.013390923, + 0.00013597019, + 0.023870043, + -0.022743633, + -0.0025300398, + 0.024396071, + 0.032017436, + -0.059623055, + -0.014403772, + 0.014883795, + 0.006013073, + -0.007427275, + -0.02678559, + -0.017337607, + -0.025133366, + -0.040649172, + -0.007375268, + 0.0024157013, + 0.051238976, + -0.011819185, + 0.02866894, + -0.0661211, + 0.0072578704, + 0.03772343, + 0.028456915, + -0.022677233, + -0.004822275, + 0.051148444, + -0.0075521176, + -0.0278938, + -0.0017976505, + 0.035391975, + -0.043084104, + 0.013498232, + 0.000140535, + 0.0081637185, + -0.014895366, + -0.01532493, + -0.023869285, + 0.03975661, + 0.029805588, + -0.0016607255, + -0.0066284174, + -0.047425393, + 0.06078541, + 0.009874399, + -0.01420618, + 0.012851728, + -0.056120105, + 0.056595568, + -0.0008234414, + -0.019897787, + 0.0016358016, + -0.001700002, + -0.046762414, + 0.009751421, + -0.009791319, + 0.039236393, + 0.006752303, + 0.009253158, + 0.03652968, + 0.033729088, + -0.006883442, + 0.011171952, + 0.015599616, + -0.027064255, + 0.0070749433, + -0.03184461, + 0.035316285, + -0.01576945, + 0.037113056, + 0.0086026555, + 0.05263313, + 0.065941855, + 0.025773102, + 0.041526187, + 0.014641708, + 0.032382395, + -0.012154955, + 0.0010549576, + -0.003009168, + -0.032207463, + -0.016061692, + -0.07052871, + 0.010312381, + 0.055001568, + 0.0047640097, + 0.01343715, + -0.036756456, + 0.06276159, + 0.0071619395, + -0.019194365, + -0.025343249, + -0.014627754, + 0.0021465474, + 0.01780317, + -0.015829839, + -0.058133774, + -0.030980356, + 0.0140177235, + -0.05612922, + 0.019690728, + -0.05656899, + 0.010644363, + 0.017211964, + -0.035414055, + 0.011559114, + -0.036985178, + 0.050859265, + -0.015278783, + -0.024635246, + -0.0018933638, + 0.0024701187, + -0.008400519, + -0.0036219582, + -0.002365139, + 0.005002786, + 0.004347757, + -0.0018842067, + -0.009879196, + -0.00094722357, + 0.04200999, + 0.007078176, + 0.023403713, + -0.004958603, + -0.020193184, + -0.03242245, + 0.013797558, + -0.010586125, + -0.04184174, + 0.07585164, + 0.030954061, + 0.004254381, + 0.044182718 + ] + ], + "tools_centroids": [ + [ + -0.05606899, + -0.0037082054, + -0.0406634, + 0.00789242, + -0.019758664, + -0.009364881, + 0.019557, + -0.009831778, + -0.00949407, + 0.007475803, + 0.009658697, + 0.017726261, + 0.005106813, + 0.025193945, + 0.014655598, + -0.018157613, + 0.019189123, + -0.013696539, + 0.013375534, + -0.030809665, + -0.010917298, + -0.017136682, + -0.0133627625, + 0.047115013, + 0.018693825, + 0.041793875, + -0.035732128, + -0.030892752, + 0.03770479, + 0.048461147, + 0.007146342, + -0.018489974, + 0.0021626442, + -0.055255502, + -0.033207137, + -0.03183328, + 0.023630941, + -0.008898392, + -0.06518762, + -0.009473217, + -0.02444353, + 0.009674398, + 0.007890898, + -0.042801153, + 0.021430025, + -0.038007095, + 0.02450929, + -0.040193666, + 0.0003781031, + 0.0039781453, + -0.026794637, + -0.015793024, + 0.019999465, + -0.03834564, + 0.029724747, + 0.051441368, + -0.033698015, + 0.0012862714, + -0.05337915, + -0.000660187, + -0.028856108, + -0.019998565, + -0.00937057, + 0.012524584, + 0.005108214, + 0.091537096, + 0.02503071, + -0.0009231426, + -0.0117032025, + -0.017811712, + -0.044022832, + 0.021416029, + -0.03316437, + -0.03286693, + -0.0865588, + 0.035949722, + 0.043933157, + -0.02726403, + -0.01020989, + -0.0060232165, + 0.050451577, + -0.0009674949, + 0.014217209, + 0.0017161807, + 0.020335244, + 0.052320678, + -0.043524, + 0.03319889, + -0.006663756, + -0.007431745, + -0.028143626, + -0.0012573685, + 0.004024065, + -0.044356935, + -0.051310178, + -0.015289323, + -0.030234639, + 0.007162317, + 0.03509846, + 0.0075056385, + 0.016321585, + 0.002279486, + 0.008136665, + -0.013070722, + 0.047912866, + 0.0038850524, + 0.03659392, + 0.018676, + -0.027126975, + -0.008610971, + 0.003703314, + 0.021906741, + 0.017196443, + 0.0083635105, + -0.022649636, + -0.0427472, + -0.035220575, + -0.049825616, + 0.020608898, + -0.008786186, + 0.05120426, + -0.0076744757, + 0.067492664, + -0.042086206, + -0.019211868, + -0.034554653, + 0.038636506, + 0.054618992, + 0.016760439, + 0.03614989, + 0.0020738828, + 0.035541587, + -0.009483902, + 0.010591545, + -0.03670953, + -0.02300192, + 0.005059944, + 0.0033167151, + -0.008413337, + -0.06675143, + 0.021428753, + 0.057771403, + -0.036993712, + -0.038098555, + 0.03635651, + -0.05102609, + 0.023876203, + 0.004495377, + 0.009017165, + -0.004652149, + -0.018913463, + 0.0013396946, + 0.010036981, + 0.00037534232, + 0.000408126, + -0.013845552, + 0.007976019, + 0.05612355, + -0.022099286, + 0.06489436, + -0.0016591164, + 0.010839845, + -0.025517747, + -0.018433146, + 0.030932501, + -0.0146434, + 0.0077389157, + 0.0014314591, + -0.0047261086, + -0.01379333, + -0.00672064, + -0.028489156, + 0.015116883, + -0.012347785, + 0.02713658, + 0.02815122, + 0.09493939, + 0.023494598, + 0.011286576, + -0.05920509, + -0.046676517, + 0.020933578, + -0.009062939, + -0.007043584, + -0.034089938, + -0.0074305055, + 0.0435831, + -0.010437014, + 0.0077175335, + 0.013643179, + 0.00032880338, + 0.012419136, + 0.028978484, + 0.0027938702, + 0.033168048, + 0.017302865, + -0.001537013, + -0.023324333, + -0.023725992, + -0.035093762, + -0.024574025, + 0.031602282, + 0.043686762, + 0.0043900036, + -0.039813936, + -0.010593189, + -0.0121115735, + -0.034151588, + 0.009565826, + -0.038808856, + -0.012305442, + -0.00027281337, + 0.028179586, + 0.0022837583, + -0.039286092, + -0.0042402116, + 0.008249791, + 0.010657445, + -0.031379938, + -0.031366322, + -0.023170266, + 0.054060068, + 0.0048521874, + -0.01564455, + 0.01751972, + -0.0055304905, + 0.023394609, + 0.012396702, + -0.0003636368, + -0.022696953, + -0.023217125, + -0.006895921, + -0.008959929, + -0.018518882, + 0.071185775, + -0.0023356022, + -0.014435151, + 0.0037555092, + 0.025933823, + -0.00043771358, + -0.023615904, + -0.021824772, + 0.006545141, + -0.0036427176, + -0.044271503, + 0.021339668, + 0.044955056, + 0.033640694, + -0.017741505, + 0.0050410517, + 0.029788665, + -0.009972994, + -0.044430643, + -0.0060587483, + 0.03381109, + 0.032378227, + 0.029204395, + -0.01285328, + 0.011239541, + 0.011089111, + 0.032791503, + 0.01966721, + 0.01709499, + 0.04467105, + 0.006781909, + -0.0062364032, + -0.022574946, + -0.0069035026, + 0.0080041215, + -0.010154674, + -0.033628866, + 0.02108642, + 0.028107613, + 0.0055993693, + 0.017272362, + 0.0064405464, + -0.024938427, + -0.0274226, + 0.05751032, + 0.023648951, + -0.0069635683, + 0.021213165, + 0.008942416, + 0.0031823711, + 0.0020612366, + 0.0017135177, + -0.03745275, + -0.01224723, + -0.0092840865, + -0.039438073, + -0.020603599, + -0.009839146, + 0.10459197, + -0.0013756665, + -0.015315762, + -0.0004424063, + -0.035673883, + -0.22716926, + -0.011021554, + 0.009104081, + 0.033588458, + 0.007958396, + -0.00556822, + -0.00057971233, + 0.0043771993, + -0.041701302, + 0.024739638, + -0.03301154, + -0.0772926, + -0.029319951, + -0.035250075, + -0.0018914198, + -0.006261339, + -0.0055478225, + -0.006832176, + 0.023415668, + -0.039532892, + -0.020574547, + -0.012577133, + 0.05117778, + -0.0044079097, + -0.014995264, + -0.015555239, + 0.048599504, + 0.009827435, + -0.036604106, + -0.009881144, + 0.005024552, + 0.006581829, + -0.005089212, + 0.030867914, + 0.012545012, + 0.03213094, + 0.009552479, + -0.011707572, + -0.015692463, + -0.008529624, + 0.019617837, + 0.042746853, + 0.0017175317, + 0.037256774, + 0.041178584, + -0.03710732, + 0.012643423, + -0.028216833, + -0.016226137, + -0.008176746, + -0.0021341608, + -0.004784728, + 0.0022153086, + -0.027044365, + -0.0452304, + -0.00092170114, + -0.0344755, + 0.039659843, + -0.03179438, + -0.0008682657, + -0.030181633, + -0.0069265706, + 0.050441854, + 0.017631147, + -0.024619931, + 0.020767754, + 0.016655464, + -0.01860972, + 0.038021646, + -0.023390938, + 0.031038294, + 0.0038111766, + 0.0005345677, + -0.026021777, + -0.007115485, + 0.048046198, + -0.031886622, + -0.035144318, + -0.0026188153, + -0.18630543, + -0.032157507, + 0.018465653, + 0.0007996492, + 0.015417364, + -0.034394316, + -0.0075861984, + -0.0037427046, + 0.020296175, + 0.048781425, + 0.3208829, + 0.017051741, + -0.019397799, + 0.009009308, + 0.053919844, + -0.016741762, + 0.006250598, + 0.035504892, + -0.0060608485, + -0.032443427, + -0.00053168356, + 0.0076522306, + 0.025676573, + -0.0021505675, + -0.007423398, + 0.021114204, + -0.031733297, + -0.0011500829, + 0.100127555, + -0.004869494, + 0.029603617, + -0.0041950424, + 0.006506095, + 0.023717992, + -0.052787792, + -0.040853288, + -0.011165942, + 0.056000806, + -0.025902374, + 0.045923546, + -0.014623309, + 0.018445712, + 0.025848486, + 0.0029045863, + -0.053331148, + -0.018678037, + 0.011694079, + -0.013297034, + -0.0096364105, + 0.032539476, + -0.029562913, + -0.047789693, + 0.0026963938, + 0.025598904, + 0.006072212, + -0.03891818, + 0.017167594, + -0.0036721022, + -0.028613243, + -0.018510707, + -0.034060024, + -0.027170384, + -0.022034682, + 0.0024952486, + -0.0015357194, + -0.03533966, + -0.020739295, + -0.03143169, + -0.011211882, + 0.049816895, + 0.004694738, + 0.010594703, + -0.06420896, + 0.029159075, + -0.0025625115, + 0.041708685, + -0.030052194, + -0.014590074, + 0.012761965, + 0.014360401, + 0.013680634, + -0.010967563, + 0.005759606, + -0.011701792, + 0.05404799, + 0.019268606, + -0.021743063, + 0.06309377, + -0.019261587, + 0.048059467, + -0.024920318, + 0.010070914, + -0.047579683, + 0.026595078, + 0.02419197, + -0.018060323, + -0.014325064, + 0.026749337, + -0.02670927, + -0.03856533, + -0.008656995, + -0.0010423756, + -0.027002128, + 0.01249798, + 0.01479696, + 0.003004412, + 0.013617141, + -0.004383087, + -0.038900018, + -0.016454257, + -0.03198517, + -0.017905517, + -0.006894297, + -0.02221281, + 0.018678682, + -0.0025110545, + 0.00062055915, + 0.011711673, + 0.029655354, + -0.0033267613, + -0.030383127, + 0.033008993, + -0.011483914, + -0.039945323, + 0.032434426, + 0.016122635, + 0.0034836135, + 0.021274103, + 0.02966441, + 0.036415953, + -0.0307599, + -0.013897553, + -0.0090347715, + 0.008639756, + -0.027077066, + -0.031259153, + 0.00041659162, + 0.008344521, + -0.055489786, + 0.043384776, + 0.016839493, + -0.019988367, + 0.017604688, + 0.010117987, + 0.011109807, + 0.004411769, + 0.0075515187, + 0.024973173, + -0.006590006, + 0.059543025, + -0.011032581, + -0.05263824, + -0.023044582, + 0.010140496, + -0.0036169202, + -0.0006291355, + 0.04357636, + -0.016806297, + -0.021199094, + 0.010797508, + 0.0037002466, + 0.048935954, + 0.025420306, + -0.068292305, + -0.014495878, + -0.033696596, + -0.016170561, + 0.010531388, + -0.014046231, + -0.038196664, + 0.014447582, + -0.0023581628, + 0.024200277, + 0.07188329, + 0.026538994, + 0.010875151, + -0.04558475, + 0.026683357, + 0.025531683, + -0.033252317, + -0.0019728958, + -0.033328198, + -0.061308175, + 0.04225168, + 0.0017407219, + -0.018591579, + -0.043852597, + -0.0018788553, + -0.016940724, + 0.0065283165, + 0.04805281, + 0.014803647, + -0.044855554, + 0.014711851, + 0.014296282, + 0.03147689, + -0.0071145403, + -0.016696168, + 0.014320578, + -0.024143968, + -0.032781575, + 0.11330352, + 0.014250864, + 0.0010048777, + 0.010561461, + 0.012848797, + 0.009197347, + 0.027354443, + -0.009164351, + -0.017722908, + 0.001937092, + -0.007415777, + 0.003458327, + -0.040217895, + 0.005049932, + -0.013077656, + -0.013231001, + 0.015766662, + -0.009500496, + -0.016540406, + 0.0049814247, + -0.014900655, + -0.002444683, + -0.002461634, + 0.044309385, + 0.0012398985, + 0.04664923, + 0.05246022, + -0.010509221, + -0.015866991, + -0.01696081, + -0.003949499, + 0.053042635, + -0.06599373, + -0.007691855, + 0.017483734, + -0.012901056, + -0.046527628, + 0.018579615, + 0.012297996, + 0.013650844, + 0.003290136, + 0.0076800594, + -0.033333212, + 0.03522728, + -0.028282017, + 0.0096138, + 0.040518165, + 0.00096373906, + -0.031569775, + 0.024419462, + -0.01790975, + -0.0033231927, + 0.039167337, + -0.006741734, + 0.056704193, + -0.02368793, + 0.012585808, + 0.030043406, + 0.034530554, + -0.013871013, + 0.0013567432, + 0.0069410847, + -0.013012229, + -0.01794545, + 0.03237499, + -0.006095536, + -0.0015028969, + -0.022055492, + -0.0075068753, + -0.020075902, + 0.026405957, + -0.042370167, + -0.002402599, + -0.027732788, + -0.025120504, + 0.035065226, + -0.029773757, + -0.02554472, + -0.06785252, + -0.0140207475, + -0.009677745, + -0.0526383, + -0.011321278, + 0.038069192, + 0.015819497, + -0.035158947, + 0.00539325, + 0.020147596, + 0.026790883, + -0.026228523, + 0.011281811, + 0.0004322955, + 0.03357598, + -0.0312409, + 0.02051756, + 0.008903391, + -0.02679297, + 0.0059814607, + -0.039614767, + 0.02225511, + 0.0032299664, + -0.039062347, + 0.034572784, + -0.04000374, + 0.013121305, + -0.006740711, + 0.015605714, + -0.024614284, + 0.0182156, + -0.02464988, + 0.022435531, + -0.0540118, + -0.0107686315, + -0.01670897, + 0.015968842, + -0.025021473, + 0.004982866, + 0.044434078, + 0.0444515, + 0.03769919, + -0.026614936, + -0.0057781814, + 0.04286719, + 0.012986099, + -0.00009658893, + 0.057660222, + 0.002628925, + -0.0145921465, + 0.0028970172, + -0.00004345907, + 0.00540906, + 0.000009923756, + -0.01587657, + -0.013682616, + -0.0055716606, + -0.015776113, + 0.0040675635, + -0.02399378, + -0.024579993, + -0.02416092, + -0.021938631, + -0.0018802591, + -0.019586174, + 0.006322869, + -0.0037580882, + 0.011118912, + -0.00688259, + 0.046543375, + 0.0020257356, + 0.00915174, + -0.012346758, + 0.030707782, + -0.029368373, + 0.01678603, + -0.020684225, + 0.0025972757, + -0.012388426, + -0.008840013, + 0.018587714, + 0.009576551, + 0.029996252, + -0.0339124, + -0.069902465, + 0.004762207, + -0.005107531, + -0.04803004, + 0.007623275, + 0.024682391, + 0.011488783, + -0.019897707, + 0.038231988, + -0.004208973, + 0.01619736, + 0.02647576, + 0.011420114, + 0.016564004, + 0.010380385, + 0.015293031, + -0.015646528, + -0.00861725, + -0.011084581, + -0.010725576, + 0.0021767458, + -0.01672313, + 0.03839429, + -0.013438732, + -0.013672219, + 0.03522901, + 0.0024493937, + -0.009887151, + -0.03008308, + -0.026761537, + -0.020664144, + 0.022931337, + -0.008237708, + -0.0072052367, + -0.015670665, + 0.040643997, + 0.026935095, + -0.036957294, + -0.010053467, + -0.04384358, + 0.022895643, + -0.2105204, + 0.022694126, + 0.00997941, + -0.013189658, + -0.056611195, + 0.008618698, + -0.012578096, + -0.019372476, + 0.0086058965, + -0.037407827, + -0.042723864, + 0.008701772, + 0.025598338, + -0.029506119, + 0.018346047, + 0.024427475, + 0.013028314, + 0.011439589, + 0.0073115127, + 0.031192467, + -0.015018622, + -0.031565055, + 0.0016912352, + 0.025967559, + 0.0075193015, + -0.010706298, + -0.007327034, + 0.005763416, + -0.04706705, + -0.004623551, + 0.014159363, + -0.019997144, + 0.03497208, + 0.06485894, + -0.026932962, + -0.010051237, + 0.013179083, + -0.029087199, + -0.022257665, + -0.015966833, + -0.012313927, + 0.034779385, + -0.01645768, + -0.011849744, + 0.016278874, + 0.03483635, + -0.0043797335, + 0.0011064508, + -0.05597814, + 0.013468208, + -0.0057655373, + 0.04608973, + -0.031591564, + 0.00081813696, + 0.02271715, + -0.016193483, + -0.048024204, + 0.024269635, + -0.010684908, + 0.03037921, + -0.015080028, + 0.00577765, + -0.035579298, + -0.074489966, + -0.04568095, + -0.032743674, + -0.08463265, + 0.012373603, + 0.004400696, + 0.029980667, + -0.017966386, + -0.038213238, + 0.0057509523, + -0.025879731, + 0.017900858, + -0.019159615, + 0.013998882, + -0.0017255883, + -0.027130362, + -0.02789793, + -0.033315383, + 0.013056893, + -0.056488942, + 0.0058665094, + 0.060071133, + -0.009120229, + -0.003342009, + 0.0047433944, + -0.018858049, + 0.007472259, + -0.052950244, + -0.0039435257, + 0.00009774961, + 0.02106769, + -0.01170398, + 0.014035189, + -0.03197733, + -0.0038070253, + -0.014319388, + 0.0053357556, + -0.017116, + -0.0016028484, + 0.0019584813, + 0.015731098, + -0.014593648, + -0.009505084, + 0.06416616, + -0.022955375, + 0.016944945, + -0.015723076, + -0.030711906, + -0.0011726163, + -0.014695939, + 0.0066800653, + -0.046726193, + -0.0039578825, + 0.028575825, + -0.019372286, + -0.029086906, + -0.017561322, + -0.00428551, + 0.014641752, + 0.007259805, + -0.04302052, + 0.0039681853, + -0.00029460277, + 0.038807478, + 0.006130545, + -0.0048487694, + -0.018746726, + 0.039638285, + -0.03210769, + -0.012196666, + 0.029514477, + 0.030580895, + -0.011591039, + -0.013160387, + 0.066982895, + -0.019542642, + -0.053750712, + -0.023531597, + 0.0046885437, + 0.0008201537, + -0.019609794, + -0.00034986206, + -0.014274548, + -0.0038191855, + -0.013088327, + -0.042322893, + -0.029365066, + 0.051891316, + -0.0026843995, + 0.017064247, + 0.016581565, + 0.0013990946, + 0.0052255765, + -0.015575843, + 0.02221455, + -0.010577843, + 0.05188453, + 0.027795833, + -0.034560814, + -0.016681876, + -0.028297098, + 0.010590897, + 0.019287696, + 0.00036446203, + -0.01720634, + -0.037081674, + 0.0033696124, + -0.0013072222, + 0.030434122, + -0.03360222, + 0.055518307, + 0.023288932, + -0.04746223, + 0.041459426, + 0.014260388, + 0.06371192, + -0.0015586383, + 0.023453446, + -0.013434582, + 0.01792418, + 0.00756145, + 0.012596964, + 0.017840264, + 0.026810404, + 0.033809863, + 0.01566813, + 0.018697273, + 0.018435918, + -0.0015165473, + 0.0023415266, + -0.018216006, + 0.036613736, + 0.0055582654, + 0.01461495, + 0.022472767, + -0.025861593, + 0.06232265, + -0.021595163, + -0.03394643, + -0.03501842, + -0.025740786, + -0.028258167, + -0.011435451, + -0.026583094, + -0.028726535, + -0.033326574, + 0.04048446, + -0.028059425, + 0.006273973, + -0.024928257, + 0.0059108674, + -0.005911225, + 0.020267177, + 0.024992893, + 0.029467724, + 0.009110243, + 0.010474011, + 0.005127083, + 0.025516486, + 0.01224543, + -0.04925422, + -0.016988033, + -0.030158473, + -0.00012626215, + -0.0070728445, + 0.00432526, + -0.009610042, + -0.024404833, + 0.009735499, + -0.04270004, + 0.03474741, + 0.025382044, + -0.002434115, + 0.035717692, + 0.034531046, + -0.020223275, + -0.024045918, + 0.03205907, + -0.01690001, + 0.008418649, + 0.016847905 + ], + [ + -0.0777926, + -0.016130753, + -0.030252252, + 0.013064685, + 0.012630054, + -0.033431668, + -0.020723045, + -0.010652609, + 0.014717416, + 0.029943759, + -0.006884425, + -0.050565906, + -0.020473206, + 0.028232329, + -0.008742489, + -0.006928586, + -0.01848413, + -0.0062769433, + 0.0249494, + -0.026330234, + -0.044125106, + -0.027761683, + -0.047089003, + -0.018802904, + -0.0031594674, + 0.06795997, + 0.011664632, + 0.0049165445, + -0.008060829, + 0.012408625, + -0.0238953, + -0.02757731, + -0.026854679, + 0.0010435038, + -0.020860666, + 0.017221682, + 0.014673638, + 0.029738, + -0.011689957, + -0.015324875, + -0.010424466, + -0.028959932, + -0.0026987449, + -0.06396346, + -0.0044665886, + -0.051129714, + 0.008419665, + -0.016844666, + 0.03334108, + 0.008317175, + -0.015411966, + 0.007842717, + 0.061229803, + -0.05119574, + -0.013707167, + 0.012232052, + 0.012225783, + 0.045153536, + -0.0008650962, + -0.016156076, + -0.046021163, + -0.033580557, + -0.01686916, + -0.0030913, + 0.033622023, + 0.01923576, + -0.022195347, + -0.010797742, + -0.029916009, + -0.0004705949, + -0.039619286, + -0.013173107, + 0.06153379, + 0.006591758, + -0.058220953, + -0.04044149, + 0.034537926, + -0.016714765, + -0.0033219494, + 0.016891837, + 0.028455667, + -0.03353745, + -0.0025109896, + 0.0013168402, + 0.02191537, + 0.00989691, + -0.0076760734, + 0.053305842, + -0.011090245, + 0.05804909, + -0.0053123464, + -0.009720489, + 0.033220656, + -0.048008375, + -0.07702685, + 0.004185979, + 0.019034842, + 0.005118874, + 0.016340083, + 0.006550299, + 0.0013390492, + -0.0079578, + -0.017740352, + -0.029226499, + 0.03548014, + -0.030715493, + 0.035450455, + 0.002739102, + 0.008236122, + 0.0070569282, + 0.0021077737, + 0.010215997, + 0.0006403754, + 0.020998295, + -0.022850262, + -0.039860018, + -0.022919796, + 0.003358362, + -0.015362032, + -0.022922192, + 0.034358162, + -0.053381287, + 0.03479484, + -0.014682795, + 0.046078928, + -0.05120507, + -0.033427823, + 0.018008124, + -0.029796828, + 0.03957431, + -0.009962059, + 0.0014587059, + 0.035613198, + 0.023544552, + -0.0035347848, + -0.021168783, + 0.02405471, + 0.022278832, + -0.0129754245, + -0.04674764, + -0.035395015, + 0.0060356203, + 0.009195937, + -0.012415938, + 0.041733548, + -0.0042112567, + -0.0054666842, + -0.022617701, + 0.024861094, + -0.0014649403, + -0.021593269, + 0.010473995, + -0.006896913, + 0.012945232, + 0.038368367, + -0.021349257, + -0.023133717, + 0.03926201, + -0.006479839, + 0.108185984, + -0.023711825, + 0.035648413, + -0.020544553, + 0.028756576, + -0.012647061, + -0.0068361103, + -0.009214789, + -0.01819023, + -0.024630189, + -0.0158221, + -0.0076070474, + 0.016997859, + 0.028996132, + 0.0028848278, + 0.012588545, + 0.012869404, + 0.022341216, + 0.043853626, + -0.03490951, + -0.04219079, + -0.036706578, + 0.006829303, + 0.029833514, + -0.04073565, + -0.040228482, + 0.040827464, + 0.018223649, + 0.0068636555, + 0.023727717, + 0.013143164, + -0.024592938, + -0.0046767895, + 0.0073383558, + 0.065363534, + -0.054043908, + 0.010578177, + -0.0060408995, + 0.022571096, + -0.022589117, + -0.011779923, + 0.02198382, + 0.046569698, + 0.030388758, + 0.00009530159, + -0.025261797, + -0.004875701, + 0.016566563, + -0.024309523, + -0.04492145, + -0.040620018, + -0.008033184, + 0.008486595, + 0.0036873387, + -0.006312519, + -0.057184577, + -0.027954899, + -0.026917579, + -0.023880664, + -0.012670563, + -0.022808496, + 0.017791497, + 0.0046672984, + -0.021858089, + 0.017631855, + 0.023283772, + 0.025225436, + 0.03048242, + 0.008289526, + -0.027709194, + 0.022788085, + -0.037033916, + -0.019281274, + 0.042843334, + -0.020052629, + 0.062315274, + 0.017621454, + 0.0023051042, + -0.012937805, + 0.03945034, + 0.01578884, + -0.04974779, + -0.05110533, + 0.040397394, + -0.02760902, + -0.029570848, + -0.028894164, + -0.023790289, + 0.020278046, + -0.021137215, + -0.009141063, + 0.005414481, + -0.019659275, + 0.010410835, + 0.03108242, + 0.019945964, + 0.019414624, + 0.001831312, + -0.0044669346, + 0.051963087, + -0.0041192234, + 0.061908115, + 0.033783134, + -0.029776793, + 0.014804269, + 0.021261247, + -0.0008398359, + -0.062825724, + 0.008026716, + 0.0017601701, + 0.016118582, + 0.0132596325, + 0.026431667, + 0.04182074, + 0.05865137, + 0.020127004, + 0.009281466, + -0.0050892252, + -0.049665384, + 0.054135658, + 0.0067237294, + -0.018110242, + 0.034775, + 0.031828076, + 0.014316956, + -0.00709445, + 0.016639711, + 0.056842767, + 0.031007824, + 0.008321629, + -0.051327795, + -0.03239476, + 0.052417755, + 0.079580046, + -0.0044072163, + -0.025062976, + -0.034690555, + 0.0017639417, + -0.12477049, + 0.0010641285, + 0.037374467, + -0.0078232195, + -0.021236846, + -0.008217301, + -0.024288183, + 0.03848377, + -0.036963657, + -0.029888399, + -0.06894421, + -0.059106227, + -0.03341434, + -0.02493872, + 0.027394043, + 0.022943368, + -0.06030709, + -0.024837509, + -0.014572309, + -0.035931725, + 0.017025314, + -0.038330782, + 0.029428963, + -0.031423945, + 0.014545969, + -0.020500131, + 0.013090049, + -0.015320569, + -0.067829646, + -0.032127842, + -0.03852837, + 0.01132043, + 0.016142247, + 0.04502109, + 0.011946692, + 0.036663942, + 0.004616084, + -0.0040293597, + 0.008882638, + -0.0050679436, + 0.030989021, + -0.011439283, + -0.027428659, + 0.010920439, + 0.010191991, + 0.025236871, + 0.0374206, + -0.0040066508, + -0.025254825, + 0.009245347, + -0.00990044, + 0.012132254, + 0.0039520827, + 0.0044951704, + 0.00659937, + -0.0025982815, + -0.01390185, + 0.025864433, + -0.007912151, + -0.038404647, + -0.014624314, + 0.05410332, + -0.012991158, + 0.017253352, + 0.015497101, + 0.0034109775, + 0.015372937, + -0.033209722, + 0.036295604, + 0.014939996, + -0.02805046, + 0.0015722377, + 0.017167116, + -0.00017856316, + -0.045900483, + 0.0077081304, + 0.074832946, + -0.026312457, + -0.0005495474, + -0.13214304, + 0.049361903, + 0.02400141, + -0.0055426657, + 0.015336194, + -0.02816012, + 0.0046836925, + 0.0099547, + 0.03742822, + 0.011811279, + 0.19601296, + 0.033754703, + -0.010378031, + 0.001250739, + 0.008726979, + 0.018838167, + 0.043022648, + 0.0697224, + -0.010394221, + -0.030004341, + -0.04530521, + -0.046880346, + 0.018401384, + -0.01957355, + 0.041167054, + 0.058837432, + 0.015511397, + 0.0063158153, + 0.11830529, + 0.023792224, + 0.058994506, + 0.00018409223, + 0.034306847, + 0.058044955, + -0.020177843, + -0.048148416, + 0.0122626135, + 0.034675147, + -0.03752977, + 0.029120939, + -0.010040249, + 0.04624406, + -0.0073817093, + -0.011214725, + -0.04331988, + 0.0051043136, + 0.01554897, + -0.012071543, + 0.02121115, + -0.006823022, + -0.011237937, + -0.0035037443, + -0.019238824, + 0.018333014, + -0.026355784, + -0.054825306, + -0.009566956, + 0.0024437692, + -0.009667523, + -0.013058095, + -0.034837723, + -0.020705493, + -0.048979156, + 0.021577716, + -0.0041844496, + -0.0036711793, + -0.05854377, + -0.04395329, + -0.026583686, + 0.03830527, + 0.031828027, + -0.005516493, + -0.07229121, + 0.023722429, + -0.009492063, + 0.03496614, + -0.024903411, + 0.0026213636, + 0.015441066, + 0.028241813, + 0.0037205473, + 0.017424388, + -0.023927208, + 0.016457984, + 0.02520875, + 0.040443234, + 0.010766083, + 0.017123722, + -0.04108963, + -0.033335038, + 0.0030168097, + 0.0053470326, + -0.018452037, + 0.03997777, + 0.007828228, + 0.016234277, + 0.020400109, + 0.048260894, + -0.028451897, + -0.049423486, + 0.014191736, + -0.01758265, + -0.009043689, + 0.02531204, + -0.0062299054, + 0.01920355, + 0.03200232, + 0.017771985, + -0.013553965, + -0.043947473, + -0.015971966, + -0.03285715, + 0.00947157, + -0.042635962, + 0.057298675, + 0.060869828, + -0.014281702, + -0.015127428, + 0.040840045, + -0.03678748, + 0.009928272, + 0.013789576, + 0.016361559, + -0.033839006, + 0.052730188, + 0.03176229, + 0.04237319, + 0.014982663, + 0.017706575, + 0.04312265, + -0.025634803, + -0.02308356, + -0.019002479, + 0.036787376, + 0.031167256, + 0.006498465, + 0.012102121, + 0.020858966, + -0.024361406, + 0.02639614, + 0.057980295, + 0.0049070157, + -0.029224023, + 0.022465296, + 0.025334602, + 0.060530398, + 0.015762227, + 0.0035243605, + 0.034754157, + 0.027435217, + 0.060840853, + 0.01591025, + 0.009018213, + 0.00035580425, + -0.033563286, + 0.010515216, + 0.0069815437, + -0.047450293, + -0.011460179, + 0.015469139, + -0.004947935, + 0.044788565, + -0.028610881, + -0.03679307, + -0.026844844, + -0.024473835, + -0.013097169, + 0.05502105, + -0.0030086858, + 0.0054221293, + 0.00934873, + -0.05726428, + 0.028941594, + 0.021284036, + -0.018056447, + 0.076361485, + -0.028323943, + 0.010804032, + 0.03501848, + -0.041234206, + -0.010082845, + -0.023559377, + -0.047871493, + 0.043036506, + -0.0077531943, + -0.061463587, + -0.02967603, + 0.027839405, + -0.0020366006, + 0.031356577, + 0.009509519, + -0.017622339, + 0.0026792034, + -0.031537317, + 0.02021114, + 0.0575173, + -0.01732471, + 0.0070268577, + 0.0016957909, + -0.030016575, + -0.01718395, + 0.07615369, + -0.0123706665, + -0.0047786008, + -0.0060987207, + -0.0013207493, + 0.010539184, + 0.009274312, + 0.0007885846, + -0.016644027, + 0.038292266, + 0.01448823, + 0.024252761, + -0.057947274, + -0.033873387, + 0.019700998, + -0.040111557, + -0.021552257, + -0.057441693, + -0.06635977, + -0.025710544, + -0.053925313, + -0.0030804146, + -0.016332027, + 0.0361699, + 0.0059258826, + 0.042275347, + 0.024262903, + -0.011202689, + -0.03925866, + -0.026211502, + -0.018307885, + 0.04379796, + -0.076288186, + 0.032032076, + 0.007867089, + 0.0054110987, + -0.038828284, + 0.04143126, + -0.01417168, + 0.008428398, + -0.0007231599, + 0.01121787, + 0.00823, + 0.019377021, + -0.03386666, + -0.007837995, + 0.0138864545, + 0.034504473, + -0.014824415, + -0.029037168, + 0.0040237717, + 0.013956105, + -0.03657621, + -0.0029222432, + -0.0029531356, + -0.012477216, + 0.0034438937, + 0.028417397, + -0.0042160964, + -0.04023129, + -0.007177788, + -0.012188858, + -0.010640365, + -0.025378073, + 0.023323987, + 0.04131474, + -0.020508341, + -0.017246038, + -0.009083519, + 0.011987527, + 0.010934865, + -0.020992914, + -0.019448092, + 0.0035046816, + -0.0036248225, + 0.029120086, + -0.02020849, + 0.0108864745, + 0.004756365, + -0.008310412, + 0.00082247274, + -0.022503192, + -0.06487264, + 0.004785375, + -0.033616617, + -0.07876667, + 0.037286237, + 0.064570576, + -0.02501201, + 0.00021179773, + 0.019808415, + 0.01730107, + 0.0028576683, + -0.022853047, + 0.096394695, + -0.009005143, + 0.021504054, + -0.016715469, + -0.103030674, + -0.00034150484, + 0.027667, + -0.033947464, + 0.015386551, + 0.0004739991, + 0.025440171, + 0.03065169, + -0.015094892, + -0.024132984, + 0.040302265, + 0.0022036897, + 0.027904361, + -0.04102699, + 0.029432945, + -0.038208827, + -0.0062153265, + -0.011998047, + -0.013344417, + 0.033526026, + 0.021145038, + 0.0027201674, + -0.019569896, + -0.031845108, + -0.04933251, + 0.05314293, + -0.0025423346, + 0.06681798, + 0.037334353, + -0.0072135865, + 0.0014750219, + -0.008131382, + 0.0033401614, + -0.033794142, + 0.027915679, + -0.020438343, + 0.03567195, + -0.04277935, + -0.01002426, + 0.028667305, + -0.03083799, + -0.0533229, + 0.012505484, + -0.0056473175, + 0.0070565026, + 0.021628987, + -0.0054352866, + -0.039229307, + -0.0135678705, + -0.028667327, + 0.022404738, + 0.0067239925, + 0.002311097, + 0.06369203, + -0.02629535, + 0.040622883, + -0.034154095, + 0.018668704, + 0.011607976, + -0.0021320207, + -0.01759238, + 0.021003881, + 0.006119285, + -0.04615905, + -0.019840589, + -0.0129681, + 0.017631495, + -0.03321065, + 0.034223422, + 0.04391737, + -0.018810853, + -0.028161395, + 0.028423589, + -0.049154766, + 0.054977193, + 0.0000630182, + -0.0173311, + 0.0151262935, + -0.004985415, + -0.004255283, + 0.0029248288, + -0.0005668443, + -0.023171514, + -0.006701821, + -0.03586675, + -0.0022324445, + 0.01843075, + 0.0076619005, + -0.06435156, + -0.016519507, + 0.052956775, + -0.0056365584, + -0.03733352, + -0.039358534, + -0.0435778, + 0.031613506, + 0.03608112, + -0.0070490213, + 0.01944554, + 0.03693605, + -0.010692041, + -0.00599563, + 0.020131715, + -0.09125295, + -0.02595961, + -0.10351341, + 0.02379246, + -0.031193437, + -0.005686541, + -0.07574321, + -0.00818769, + 0.0012698586, + 0.031600002, + 0.029005878, + -0.034200996, + -0.07727333, + 0.03113444, + 0.002058297, + -0.009169671, + -0.02671429, + 0.0059962994, + 0.003603806, + 0.0074202335, + -0.0060870526, + -0.025415512, + -0.0050098686, + -0.018891325, + 0.015137767, + 0.0009658489, + 0.011259749, + -0.00018296832, + -0.054526947, + 0.042278267, + -0.001603754, + 0.055526804, + 0.032714244, + 0.055156004, + 0.03152486, + 0.024948584, + 0.010113138, + 0.026203973, + 0.0036432473, + -0.04414578, + 0.026116638, + -0.017378923, + 0.014874825, + 0.015792275, + -0.040725254, + -0.0027440262, + -0.016308293, + 0.019773996, + -0.032847524, + 0.009167586, + -0.08082315, + 0.008408904, + 0.020688424, + -0.019353844, + -0.028009258, + 0.026694907, + 0.0032782315, + -0.03360142, + -0.023895264, + 0.039238874, + 0.00445677, + 0.02949898, + 0.014402198, + -0.003599358, + -0.040845413, + -0.0350518, + -0.033609767, + 0.024508178, + -0.019472161, + -0.0031140537, + -0.005668693, + 0.03111455, + -0.027201336, + -0.025430666, + -0.035129692, + 0.0105588045, + 0.02917934, + -0.01097523, + -0.022186121, + 0.017050618, + 0.0037318894, + -0.03971323, + -0.0067270542, + -0.012156165, + -0.088368215, + 0.02033153, + 0.040660948, + 0.024675295, + -0.0045001414, + 0.018805915, + -0.00049041864, + 0.04464026, + -0.04455079, + -0.022046903, + 0.041305263, + 0.03293151, + -0.028352423, + -0.02059597, + -0.0019165347, + -0.006466022, + -0.009925339, + -0.008929011, + -0.076839685, + -0.00429755, + -0.014738176, + -0.024139099, + 0.03820742, + 0.003912287, + 0.030968878, + 0.004669395, + 0.025041392, + -0.013336667, + -0.056171685, + -0.023908475, + -0.0026514267, + -0.04345135, + -0.022183042, + -0.014375652, + 0.00066614884, + -0.047541592, + -0.06785615, + -0.06805146, + 0.0052735, + 0.02312607, + -0.039141715, + -0.017192993, + 0.0020956157, + 0.026551597, + -0.016486952, + -0.047485273, + 0.04242504, + -0.024608482, + -0.0063258884, + -0.025091767, + -0.0031197283, + 0.016352389, + 0.032390088, + -0.024658075, + -0.055107787, + 0.041251957, + -0.01631562, + -0.0514147, + -0.023321213, + -0.0093295155, + -0.015928332, + -0.021337397, + -0.011802844, + 0.014432149, + -0.019652303, + 0.0067840177, + -0.05091495, + -0.0055248598, + 0.0048583862, + 0.01215574, + -0.02517826, + -0.0025640433, + -0.0034858014, + 0.0013292104, + 0.034103796, + 0.020756245, + 0.0193323, + 0.040301695, + 0.00078084186, + -0.04896679, + 0.027173512, + -0.005667116, + 0.01948293, + 0.032077633, + 0.0061108964, + -0.018225638, + -0.02208757, + -0.010346406, + -0.03226915, + 0.015659394, + 0.010559285, + 0.0644997, + 0.033056695, + -0.04196774, + 0.0071754702, + -0.0038966993, + 0.020072218, + -0.03101222, + 0.03538792, + -0.008695402, + 0.0067324615, + 0.037320923, + 0.0085444255, + 0.037771806, + 0.01592961, + 0.0071725305, + 0.007101917, + 0.01527844, + 0.0028750405, + -0.016928488, + -0.013771216, + -0.023309482, + 0.030158604, + -0.011056797, + -0.000734273, + 0.039915036, + -0.007078933, + 0.041310612, + 0.03926212, + -0.021225905, + -0.03690358, + 0.00033736796, + -0.046615493, + 0.0035040032, + -0.04736803, + -0.015354346, + 0.0076449066, + 0.0646623, + -0.061303865, + 0.0072781476, + -0.022982756, + 0.04087125, + 0.020003729, + -0.0351412, + 0.0387916, + 0.044128425, + -0.0139995115, + -0.0019715482, + 0.03880407, + 0.04205457, + -0.027607651, + 0.006489501, + 0.002768678, + -0.030172173, + 0.0062397877, + -0.013040987, + -0.013252143, + -0.04237509, + 0.009563442, + 0.057689045, + -0.019734269, + 0.0113543505, + 0.02527864, + 0.0014363149, + 0.06007512, + -0.00003742828, + -0.022210408, + 0.012302623, + 0.040538836, + 0.018775605, + -0.034081753, + -0.0004998032 + ], + [ + -0.03950687, + 0.043531194, + -0.014640973, + -0.014720938, + 0.024726942, + -0.03612319, + -0.014307638, + 0.01025741, + 0.004683229, + -0.03798171, + -0.03485831, + -0.009100663, + 0.0012342628, + 0.029542759, + 0.04620511, + 0.034314673, + -0.0028652407, + -0.016328944, + 0.007256933, + -0.031210486, + 0.016605832, + -0.04016649, + -0.009275935, + 0.005787752, + -0.0058478094, + 0.029647486, + -0.03285623, + -0.036804114, + 0.0102847135, + 0.03751815, + -0.011018378, + 0.012599248, + 0.007816772, + 0.016865665, + -0.023787512, + -0.0033270551, + 0.026091896, + -0.008189237, + -0.039871283, + -0.032342162, + 0.01738121, + 0.014661393, + 0.03587866, + -0.023817893, + -0.0124929, + -0.03888124, + 0.041935273, + -0.0263411, + -0.039413948, + -0.037727747, + 0.0013563702, + -0.041180752, + 0.008936618, + -0.032369066, + 0.032389134, + 0.041318245, + -0.07018222, + -0.024456048, + -0.020840136, + 0.014526111, + 0.033790283, + 0.010664927, + -0.04341061, + -0.022219952, + -0.010782748, + 0.005418068, + -0.022497445, + 0.013246439, + 0.012736531, + -0.016248927, + 0.0008248154, + 0.009251837, + -0.06683385, + -0.0011490654, + -0.07666291, + 0.011535246, + -0.039218437, + 0.05110004, + -0.09564187, + -0.04194155, + 0.02009398, + -0.0067823115, + -0.009455593, + 0.056001958, + 0.016271655, + 0.06911042, + -0.022176139, + 0.028006595, + -0.028135093, + 0.034655333, + -0.007308119, + -0.013883404, + 0.045097373, + -0.052717112, + -0.034108102, + -0.007105286, + -0.03460486, + 0.031739317, + 0.030613171, + 0.008154331, + -0.06315366, + -0.018434746, + 0.03518978, + 0.0112801725, + 0.024134867, + -0.003977951, + 0.016525777, + 0.01839739, + -0.024246577, + -0.019862736, + 0.00031075653, + -0.013859162, + 0.013144024, + 0.024517544, + -0.038736217, + -0.02523414, + -0.010362971, + -0.027183976, + 0.021046206, + 0.014898793, + 0.003440566, + -0.016803645, + 0.0020033128, + -0.02742916, + 0.0030086606, + 0.0032655115, + -0.02966956, + 0.07162049, + 0.037842706, + -0.0046940506, + 0.00064334256, + 0.020130387, + -0.0076284185, + 0.020069573, + -0.08732818, + -0.03501705, + 0.055594362, + 0.082634926, + 0.014030701, + -0.034930494, + 0.05232482, + 0.053865146, + 0.01451553, + -0.024455464, + 0.026275275, + -0.034078658, + -0.03708409, + 0.01745932, + -0.021863382, + -0.05555264, + -0.033413053, + 0.015493354, + -0.0062087053, + -0.022264965, + -0.02127896, + -0.059868205, + -0.003401156, + 0.044210643, + 0.01787959, + 0.09501992, + 0.018742345, + -0.031885527, + -0.032269854, + 0.00025357012, + 0.042152844, + -0.03316293, + 0.02196225, + -0.0020535681, + -0.0072929896, + -0.0052945167, + -0.021792375, + 0.007816838, + -0.021683265, + -0.034982268, + -0.018460391, + -0.012000704, + 0.06461101, + 0.007490276, + 0.014343332, + -0.021530014, + -0.0075940792, + -0.02895248, + -0.014425406, + -0.040914804, + -0.03639253, + -0.08229706, + -0.004979539, + -0.017418977, + 0.016572325, + 0.037223563, + 0.00116424, + 0.019090686, + 0.010911045, + -0.010205869, + 0.048550658, + 0.035755083, + -0.018986369, + -0.001429209, + -0.027796214, + 0.025839124, + 0.022751706, + 0.054932334, + 0.020839911, + -0.017549826, + -0.044058055, + -0.042574435, + -0.05696988, + 0.024840906, + -0.017054198, + -0.054606467, + 0.0077261142, + -0.02644012, + 0.026694741, + -0.027916811, + -0.015582057, + -0.044071168, + -0.01406791, + 0.0578193, + -0.057348266, + -0.07553237, + -0.020606408, + 0.05444073, + -0.04222853, + -0.02362194, + -0.03362329, + -0.02535667, + -0.0018919092, + 0.034991592, + 0.04424803, + -0.018281374, + -0.027159164, + 0.027622392, + -0.0061814794, + 0.007855805, + -0.015342956, + 0.00082629744, + 0.008025998, + -0.0024992118, + 0.023763394, + -0.003807417, + -0.022212056, + -0.002562533, + -0.00694927, + 0.044300094, + -0.03489844, + -0.057076182, + 0.02006841, + -0.0037533976, + 0.006341216, + 0.007860421, + 0.008677003, + 0.011595276, + -0.046128266, + 0.0013428159, + 0.045055754, + 0.02239221, + 0.019835426, + -0.008117078, + 0.006887426, + -0.008277146, + 0.048709106, + -0.006205201, + 0.091035716, + 0.031401526, + 0.009654572, + -0.004819402, + -0.015429093, + 0.055476464, + -0.020625938, + 0.0040114745, + -0.0067515746, + 0.011730604, + -0.0027268373, + 0.036516953, + 0.015225207, + -0.0048017604, + -0.006393769, + -0.021304023, + 0.056664098, + 0.0026197485, + -0.0080493875, + 0.021506276, + 0.015002799, + 0.009055541, + -0.026723027, + -0.006230178, + -0.0067681912, + -0.016294137, + -0.00268294, + -0.06420861, + 0.02237568, + -0.019511264, + 0.05340581, + -0.00833766, + -0.0200639, + -0.0036080014, + -0.037964303, + -0.10868055, + 0.00020624952, + -0.06711885, + -0.006539819, + -0.0034174747, + -0.008780332, + -0.026614243, + 0.002306322, + 0.005154491, + 0.0019403903, + -0.04626778, + -0.029957443, + 0.006960841, + -0.046184577, + -0.017351516, + -0.021748034, + 0.045616314, + 0.032945316, + 0.026660904, + -0.04546675, + -0.044575658, + -0.029681908, + 0.047057368, + -0.056180052, + 0.014390304, + 0.00015773278, + 0.013120344, + 0.02658648, + 0.011372644, + 0.005863363, + 0.016664037, + 0.06376613, + 0.019382346, + 0.0802324, + -0.0016175115, + 0.021250589, + 0.00053878257, + 0.020504216, + -0.03426853, + 0.005095898, + 0.0031997939, + 0.0051565412, + -0.043827157, + 0.03209446, + 0.04712799, + -0.01782006, + 0.053983606, + -0.035910055, + -0.016622698, + -0.037207805, + 0.034288082, + -0.030350236, + -0.0011003837, + -0.02914163, + -0.03423277, + 0.009082422, + -0.001781773, + 0.032029316, + -0.0034477508, + -0.022011233, + 0.020805836, + 0.04373524, + 0.01426929, + 0.012661985, + 0.0089556845, + -0.02926414, + 0.010948363, + 0.00024500352, + 0.041760296, + -0.019010147, + -0.020999651, + 0.0043251915, + -0.018788947, + -0.03642843, + -0.029119324, + 0.05132978, + 0.008250169, + -0.023003912, + -0.026743688, + -0.08720566, + -0.059065796, + -0.000029261539, + 0.023304205, + -0.03414005, + -0.042730678, + 0.01792291, + 0.0071862466, + 0.012993562, + 0.027364269, + 0.19823818, + -0.0059806146, + -0.018968418, + 0.024375647, + 0.01791589, + 0.012285263, + -0.002662988, + -0.00074736535, + -0.014230657, + 0.03866545, + 0.044439327, + 0.007127386, + -0.007903959, + 0.0022473629, + -0.041698676, + -0.0022041672, + -0.026352853, + 0.009517888, + 0.070968114, + 0.0046247942, + 0.030372195, + 0.006002809, + 0.02578483, + 0.026675988, + 0.013856626, + 0.016871126, + -0.0014647035, + 0.052734047, + -0.009674117, + 0.017708449, + 0.0026036392, + -0.0077395584, + -0.02032638, + -0.057948228, + -0.037037287, + -0.026146235, + -0.012568703, + 0.030102657, + -0.009142211, + -0.0017773161, + 0.022137081, + -0.016034972, + -0.026643815, + 0.043836106, + -0.00026422995, + -0.012727786, + 0.06419574, + 0.006584795, + 0.018396322, + 0.017908173, + -0.050535623, + -0.02078155, + 0.010549742, + -0.014217779, + 0.024685526, + -0.023678334, + -0.020491404, + 0.006048553, + -0.023561789, + 0.018910788, + 0.08878148, + 0.007740561, + -0.059295204, + 0.03733036, + -0.016496437, + -0.018883424, + 0.018802142, + -0.007328556, + 0.016717581, + 0.04328429, + 0.05693224, + 0.017707568, + 0.046792727, + -0.014650998, + 0.062150735, + -0.012973712, + -0.018037295, + 0.016467381, + 0.0244791, + 0.0062886435, + -0.02355788, + 0.014314312, + -0.01939792, + -0.002719139, + 0.0030674103, + -0.008508544, + 0.006620206, + 0.008997434, + -0.02707569, + -0.068797246, + -0.026732523, + -0.03025209, + -0.07743795, + 0.025089052, + 0.036291923, + 0.03730403, + 0.04085414, + -0.0066735293, + -0.0055792555, + 0.031028569, + -0.0030874214, + -0.015421479, + -0.013141398, + -0.020299582, + -0.023435697, + 0.0017748995, + -0.0021218956, + 0.0012519993, + -0.001484999, + 0.008944802, + -0.027911007, + 0.012121643, + -0.020927167, + -0.0002258706, + -0.012256106, + 0.0047594383, + -0.0148541825, + -0.043066435, + 0.030682703, + 0.012971142, + -0.003666529, + 0.0108013395, + 0.025183335, + -0.028475968, + 0.021690562, + -0.0028832422, + -0.014144738, + -0.0045188447, + -0.040258396, + 0.028346978, + 0.035040945, + -0.011023868, + 0.0396404, + 0.017606566, + 0.033269204, + 0.0073707304, + -0.014222149, + 0.014156873, + 0.030270226, + 0.015319926, + 0.024624461, + -0.039615463, + 0.022913564, + 0.0031669824, + 0.002471962, + -0.039530534, + 0.04196699, + -0.017445292, + -0.030874938, + -0.0053294376, + 0.038435955, + 0.015547038, + -0.017440522, + -0.040413458, + -0.03927991, + -0.017935151, + 0.0047400724, + 0.003020446, + -0.026619047, + -0.039115086, + 0.056505933, + 0.009858637, + 0.0131826205, + 0.0284519, + 0.022894057, + 0.0022495075, + -0.021099819, + -0.0116347065, + -0.01811318, + 0.006474129, + 0.054837897, + -0.012720091, + 0.0016731841, + 0.021986602, + 0.033270527, + -0.00033776753, + -0.008643222, + 0.019710293, + -0.03264088, + 0.010028613, + 0.005217198, + -0.024148779, + -0.04014084, + -0.016549045, + 0.025656188, + 0.054638997, + 0.012246356, + -0.032795466, + -0.015674438, + -0.026266312, + -0.01646067, + 0.08210362, + 0.015601729, + -0.001658601, + 0.028211301, + -0.03853208, + -0.04279942, + 0.06524898, + -0.022642639, + -0.017951515, + -0.029549977, + -0.034688108, + 0.0061429352, + 0.010141051, + 0.00042500003, + 0.025635283, + -0.043835618, + -0.033300713, + -0.0059104236, + -0.007492045, + -0.025678229, + -0.012022994, + -0.0012621493, + -0.027511965, + 0.034545906, + 0.0027232112, + 0.0049217213, + 0.017144063, + 0.031473048, + -0.021966783, + 0.009118309, + 0.03147311, + 0.039992817, + -0.023138776, + -0.0014036369, + -0.037520673, + 0.07687327, + -0.039589405, + 0.04602321, + -0.003612067, + 0.007461662, + 0.007353311, + -0.034707982, + -0.02012695, + 0.029514175, + -0.031999454, + -0.043143217, + 0.028792208, + -0.0011134085, + -0.03371898, + 0.013939409, + -0.0019052078, + -0.004295308, + -0.007053404, + -0.0009750122, + 0.029560952, + 0.005407683, + 0.03037516, + -0.00785735, + -0.021339875, + 0.02605982, + 0.039290346, + -0.010707787, + -0.03064161, + -0.00059364137, + 0.012069025, + -0.0033701425, + -0.017966857, + -0.018942371, + 0.017725015, + -0.004986756, + 0.011548954, + 0.0028690782, + 0.038817663, + -0.021664195, + 0.013531663, + 0.0156029165, + -0.023836723, + 0.024638338, + 0.01691388, + 0.003397145, + -0.03100652, + -0.04320627, + 0.0011457364, + 0.04675279, + 0.027556699, + -0.033261135, + -0.008995849, + 0.017848724, + 0.014306368, + 0.025643473, + 0.035529383, + 0.0029430243, + 0.03153322, + -0.052132677, + -0.00248758, + -0.02161323, + -0.04709703, + -0.042139307, + -0.023811264, + 0.057088885, + -0.007749684, + 0.01378622, + 0.0038549982, + -0.042886388, + -0.02104687, + 0.00838618, + -0.0037721547, + -0.037230987, + -0.0111783985, + 0.0097314585, + 0.02131326, + -0.040853705, + 0.0057790386, + 0.021632854, + -0.022898814, + -0.012932489, + 0.004934083, + 0.010148015, + 0.014385065, + -0.003315393, + 0.017242506, + -0.007246454, + 0.009660279, + -0.019179016, + -0.010557076, + 0.016264342, + -0.029500151, + -0.012701573, + -0.012758462, + -0.029923983, + -0.02165708, + -0.052309945, + -0.033560004, + 0.0076761013, + -0.01549472, + -0.013017553, + 0.0043236334, + 0.04691618, + -0.008197069, + -0.0018741535, + -0.03084931, + -0.0054556658, + -0.016673744, + 0.039174348, + 0.04543094, + -0.023596205, + 0.001231028, + 0.035865817, + 0.0023900091, + -0.003052916, + -0.027418893, + 0.058787756, + -0.02245522, + -0.015146493, + -0.0184522, + -0.0063935583, + -0.030266108, + 0.007447998, + -0.017568134, + -0.0027355, + 0.021793561, + 0.0074607856, + -0.04652875, + -0.031837653, + -0.061417032, + -0.035766512, + 0.007060734, + 0.018031834, + 0.00045341678, + -0.0077504576, + 0.0076210555, + 0.012140915, + -0.002184006, + 0.06035298, + -0.022536587, + 0.0054653105, + -0.005007854, + -0.007992475, + -0.034879297, + 0.007152057, + -0.01913284, + -0.10116649, + 0.0022808493, + 0.008442801, + 0.036988016, + -0.016776918, + -0.07377468, + -0.01899712, + 0.07031005, + -0.034832127, + -0.03700141, + -0.01211916, + 0.0067655994, + 0.024989525, + 0.02319956, + -0.027294133, + -0.00094650587, + 0.014060532, + 0.03490878, + -0.01971701, + -0.038569264, + -0.04076684, + 0.014368246, + -0.10717569, + 0.020453772, + 0.068951204, + -0.007349555, + -0.04629167, + -0.014826132, + 0.005653124, + -0.026169593, + 0.013392487, + -0.07173489, + -0.01668766, + -0.030206582, + 0.040627755, + -0.029564377, + -0.043053318, + -0.03218649, + 0.03655294, + -0.008489806, + 0.04350242, + 0.030536214, + -0.009721366, + 0.00034617525, + -0.016816063, + 0.059502516, + -0.021497775, + -0.022350075, + -0.111227795, + 0.028571766, + -0.033449635, + 0.07857797, + -0.02256202, + 0.057096943, + 0.04939214, + 0.02692003, + 0.03341055, + 0.010906601, + 0.03452141, + -0.101317056, + -0.016416814, + 0.028970905, + -0.027896484, + 0.022908753, + -0.015605515, + 0.009761399, + 0.01375609, + 0.002159126, + -0.034757312, + 0.009949484, + -0.027204521, + 0.036121096, + -0.018719548, + 0.012673245, + -0.047391124, + -0.044187777, + 0.048009574, + -0.012766926, + 0.0116457315, + 0.014982552, + 0.019103188, + 0.022949666, + -0.039069697, + -0.04630458, + -0.028584287, + -0.040020872, + 0.015652915, + -0.013488799, + -0.049111538, + 0.026503077, + 0.03357381, + 0.03751419, + 0.02387496, + -0.0011243735, + 0.012061748, + -0.011386109, + -0.0127039915, + 0.06352219, + 0.048704118, + 0.014793782, + -0.014007755, + -0.043857392, + 0.038399946, + -0.012055639, + -0.014885214, + 0.042390678, + 0.04980625, + -0.006241386, + 0.013584097, + -0.033264153, + 0.041379217, + 0.029621469, + -0.043042388, + -0.0020997606, + -0.012933665, + 0.013286224, + -0.003282772, + 0.018547414, + -0.04327, + 0.022249907, + 0.008725586, + 0.018808063, + -0.026111113, + 0.008951248, + 0.031707752, + -0.005465531, + 0.06568567, + -0.020877102, + 0.03014087, + 0.01715928, + 0.012325161, + 0.01315365, + -0.031900663, + -0.022260122, + -0.03447207, + -0.00062158384, + -0.035049655, + 0.003485939, + 0.0046049925, + -0.05630732, + -0.011058264, + 0.007136582, + 0.016117854, + 0.0595212, + -0.03617894, + -0.027507313, + 0.002789625, + 0.01799178, + -0.0014536566, + -0.008393191, + -0.018825287, + 0.04759385, + 0.058703866, + -0.011992767, + -0.0052801687, + 0.01940323, + 0.03281311, + -0.01995503, + -0.013774053, + 0.05898928, + -0.07519091, + -0.044439603, + -0.0537019, + -0.014705708, + 0.01250235, + 0.015665539, + 0.05294212, + 0.009310713, + 0.03523993, + 0.031513117, + -0.0074070916, + 0.01826314, + 0.011835665, + -0.044761684, + -0.007041866, + 0.058125574, + 0.024500526, + -0.009871987, + -0.018237904, + 0.09113848, + -0.013172258, + -0.0041379156, + 0.016455337, + -0.022989342, + 0.016601447, + 0.014433, + 0.0053295735, + -0.024041522, + 0.01957587, + 0.018358402, + -0.017210413, + 0.043062195, + 0.0061207553, + 0.032532007, + 0.014793015, + 0.024234612, + 0.0061567454, + -0.018245239, + 0.02684064, + 0.00016797535, + 0.028608758, + 0.0046842927, + -0.010550258, + 0.02669938, + 0.030235212, + -0.0031934716, + -0.019914627, + 0.035149656, + 0.0075301835, + 0.039265014, + 0.035625007, + -0.031268924, + 0.012515024, + -0.07590218, + 0.007551814, + -0.035385616, + 0.027681876, + -0.039070603, + -0.004845278, + 0.04028714, + -0.07426009, + 0.06613951, + -0.05510454, + -0.03226479, + -0.014005947, + -0.03195615, + -0.0116673885, + 0.016437342, + -0.0076668607, + 0.028972298, + -0.025123762, + -0.017822955, + -0.011712876, + -0.012553329, + -0.037626002, + 0.00069678575, + 0.009572705, + 0.0067847413, + 0.0028660365, + -0.011129996, + 0.00090118404, + -0.0070405314, + -0.018169235, + -0.018179605, + 0.07109999, + -0.016261067, + -0.024284609, + -0.026516862, + 0.03497836, + -0.017624862, + 0.009457551, + -0.030564932, + 0.04343077, + -0.01112055, + -0.054642823, + 0.004773121, + 0.0047957487, + -0.008624442, + 0.015786134, + 0.03043451, + -0.043876424, + 0.015928723, + 0.045520768, + -0.025372392, + 0.009925128, + 0.028771762 + ] + ], + "abstain_centroids": [ + [ + -0.031148382, + 0.0053377217, + -0.051859938, + 0.006701707, + -0.013760795, + -0.046238896, + -0.0017418014, + 0.003761064, + 0.011534153, + 0.015631318, + -0.007558447, + 0.042414147, + -0.008460059, + -0.0039428133, + 0.014397731, + -0.00096799876, + 0.005969027, + -0.016228952, + -0.0066106487, + -0.01633511, + -0.017304767, + -0.015375099, + 0.00016670344, + 0.029177906, + 0.019667022, + 0.037063632, + -0.014033232, + -0.017078552, + 0.022332544, + 0.0029681043, + -0.026012668, + 0.004773388, + -0.008349955, + -0.025975648, + -0.05798201, + -0.037145726, + 0.022042587, + -0.0104322145, + -0.054713663, + -0.018552497, + 0.01407926, + 0.0066314046, + 0.027429355, + -0.026314953, + 0.021749053, + -0.05777429, + -0.011509257, + -0.020935455, + -0.00084091024, + -0.031351313, + -0.021382079, + -0.00847218, + 0.036603227, + -0.013143199, + 0.047227796, + 0.05580892, + -0.0109791085, + -0.021309411, + -0.087006725, + 0.0018242795, + -0.025122127, + -0.013786088, + -0.009168321, + 0.011006979, + 0.0012779796, + 0.09556672, + 0.00808916, + -0.008711008, + -0.008361359, + -0.034987632, + -0.03647069, + 0.021719662, + -0.032498714, + -0.017760865, + -0.06984331, + 0.011626705, + 0.020904267, + -0.008985531, + 0.0070378105, + 0.037491333, + 0.08140703, + -0.015960725, + 0.00758235, + 0.018223334, + 0.011250972, + 0.041393135, + -0.0300754, + 0.011359266, + -0.0026697272, + 0.0045864326, + -0.029182361, + -0.018836327, + 0.035417307, + -0.050186697, + -0.0169077, + 0.0030452076, + -0.031451467, + 0.021025099, + 0.03237884, + -0.017753405, + 0.024965167, + 0.006849125, + -0.0007417424, + -0.025353184, + 0.014833974, + 0.000480421, + 0.033970177, + 0.029662795, + 0.0019640021, + -0.00040594768, + 0.046204697, + 0.03329503, + 0.014799664, + 0.004400712, + -0.012687833, + -0.04885642, + -0.021582833, + -0.029335678, + 0.031864036, + -0.013479058, + 0.0061972495, + 0.01931695, + 0.039016202, + -0.0460622, + -0.011819415, + -0.03546195, + 0.030371843, + 0.045124575, + -0.00695266, + 0.026146563, + 0.0153582115, + 0.014641614, + -0.037259024, + 0.004900667, + -0.012360953, + -0.030733729, + 0.016830467, + -0.002211426, + 0.017974649, + -0.051037353, + 0.024567816, + 0.029570596, + -0.028185146, + -0.028654283, + 0.02143797, + -0.038578555, + 0.024237396, + -0.0068540815, + 0.01847857, + -0.012613435, + 0.00437136, + -0.01011591, + 0.0065389504, + -0.014050798, + 0.0061866366, + -0.01770768, + -0.019518772, + 0.053408965, + -0.013017107, + 0.016827421, + 0.022524578, + -0.0068964125, + -0.052380078, + 0.00038339154, + 0.029090617, + -0.011927654, + 0.024243768, + 0.01339041, + -0.014234689, + -0.0017058258, + -0.012792839, + -0.026187133, + 0.039857496, + -0.0025493933, + 0.030229094, + 0.026641972, + 0.07534804, + 0.022857834, + 0.010506173, + -0.058816753, + -0.051879648, + 0.016220875, + 0.017665846, + -0.054791577, + -0.039248586, + 0.014590265, + 0.021849323, + -0.018102316, + 0.015241521, + 0.02129747, + -0.022830576, + -0.01614023, + 0.027246347, + -0.00542563, + 0.046256945, + -0.007986688, + 0.026465587, + -0.0026759577, + -0.0096074, + -0.037014082, + -0.04806582, + 0.033057604, + 0.04526261, + 0.0051544863, + -0.0342738, + -0.011255881, + -0.03914743, + -0.031940874, + 0.0144135505, + -0.017719291, + 0.009187744, + 0.0078019383, + 0.02771336, + 0.0051388917, + -0.042035703, + 0.0072669606, + -0.0026950007, + 0.0030906193, + 0.004021663, + -0.02106473, + 0.0009345182, + 0.029135607, + 0.0021005718, + -0.0196823, + 0.046538655, + 0.032432303, + 0.017765502, + 0.015490211, + -0.009414902, + -0.02356354, + -0.008533621, + 0.0050160224, + -0.019913118, + -0.0035469253, + 0.03606248, + -0.022427976, + -0.04044626, + 0.015986245, + 0.009162774, + -0.00032497806, + -0.022688588, + -0.030663738, + 0.0018985462, + 0.024509203, + -0.04645868, + 0.021605283, + -0.0032775397, + 0.008400841, + 0.0053840177, + -0.0020570229, + 0.018701848, + -0.012248811, + -0.045328125, + 0.0036800986, + 0.032931987, + 0.028872136, + 0.012167973, + -0.0052398676, + 0.026528696, + 0.013797996, + 0.023905413, + 0.031488698, + 0.016135292, + 0.037903946, + 0.007559896, + 0.003637543, + -0.033904064, + -0.0027450037, + 0.0044508814, + 0.010002827, + -0.03097531, + -0.016194653, + -0.0103033185, + 0.0017212455, + -0.004217246, + -0.0003838392, + -0.028139843, + -0.037131034, + 0.059932996, + 0.016013881, + 0.0041002985, + 0.017807445, + 0.012730673, + -0.009848034, + 0.0070455708, + -0.0010963108, + -0.017336583, + 0.0052566174, + 0.006595982, + -0.045635466, + 0.01257733, + -0.015105403, + 0.08849552, + -0.0060461042, + -0.014552671, + -0.010428577, + -0.03148846, + -0.244925, + -0.022456428, + -0.01200543, + 0.018366225, + 0.012265931, + -0.024266042, + -0.033916224, + -0.0036076305, + -0.054510105, + 0.0063244584, + -0.015560582, + -0.055878177, + -0.020939475, + -0.005328696, + -0.0070734383, + 0.0048970133, + 0.0069189873, + -0.022256982, + 0.0075242044, + -0.06765787, + -0.023333378, + -0.025017215, + 0.07195793, + -0.019617401, + -0.005269894, + 0.00026103246, + 0.035949826, + 0.004107666, + -0.037599213, + -0.01236279, + -0.006176068, + 0.0069308924, + -0.0029138543, + 0.035269044, + 0.014236289, + 0.034278773, + 0.010570881, + -0.008682466, + -0.014200495, + -0.0010429567, + 0.045053545, + 0.02033515, + -0.0039721197, + 0.011877003, + 0.010973981, + -0.040029325, + -0.021862814, + -0.010063818, + -0.01741567, + -0.02046978, + 0.0024775746, + 0.00537928, + -0.0035171232, + -0.023752872, + -0.027805964, + 0.0068444465, + 0.0020820745, + 0.048062917, + -0.013278909, + -0.011143367, + -0.015697205, + -0.010483089, + 0.008951397, + 0.03141075, + -0.021271285, + 0.0060245073, + -0.005130949, + -0.013589777, + 0.028885502, + -0.019157533, + 0.033051424, + -0.0110376915, + 0.008256805, + -0.021247167, + 0.014382738, + 0.025279364, + -0.025481079, + -0.032188643, + -0.023664054, + -0.18205334, + -0.016931957, + -0.00092831947, + 0.0030238251, + 0.033743776, + -0.016215393, + -0.025330719, + 0.00043090683, + 0.052764844, + 0.045047104, + 0.34483737, + 0.021190971, + 0.029025022, + -0.029802192, + 0.05892976, + -0.027338218, + 0.017156813, + 0.01649102, + 0.0053761047, + -0.03603205, + -0.016050696, + -0.019247439, + 0.008546483, + 0.00065792014, + 0.008216663, + 0.012412621, + -0.049241338, + 0.0046835714, + 0.083209105, + 0.0044032163, + 0.015972426, + -0.035188276, + 0.0025444007, + 0.0276847, + -0.03703065, + -0.040941797, + -0.022856751, + 0.07734354, + -0.017490435, + 0.029937685, + -0.030589428, + -0.0131223295, + 0.028537748, + -0.015808787, + -0.031040976, + 0.0022384813, + 0.004584949, + -0.036500685, + -0.013998977, + 0.01908896, + -0.020744048, + -0.027405975, + 0.027832054, + 0.0009314528, + 0.021530954, + -0.015619696, + -0.006596794, + -0.008945383, + -0.02280639, + -0.00078093243, + -0.019146752, + -0.021306396, + -0.024635073, + -0.0013178922, + 0.030522594, + -0.036964312, + -0.03445982, + -0.026010966, + 0.011097982, + 0.04430233, + 0.033004954, + 0.012737286, + -0.02832643, + -0.033423617, + -0.011146303, + 0.018307667, + -0.010493131, + -0.002345288, + -0.006297813, + 0.023474589, + 0.015160519, + 0.008065754, + 0.0071064318, + -0.012013574, + 0.034734786, + 0.020278838, + 0.013851283, + 0.07314595, + 0.005167692, + 0.03851032, + -0.011801342, + -0.024405105, + -0.05664373, + -0.017895715, + 0.010395462, + 0.0045732795, + -0.017845895, + 0.052198168, + -0.007843896, + -0.03997914, + 0.011556584, + 0.019499848, + -0.03996185, + -0.007970878, + -0.00291532, + -0.05102505, + 0.023902692, + -0.024392458, + -0.019463964, + -0.021712244, + -0.028254114, + -0.022925302, + 0.008098716, + -0.009423168, + -0.0034888273, + -0.02608165, + 0.0058718375, + 0.012862887, + 0.020583529, + 0.015732484, + 0.004980414, + 0.0077576265, + -0.018226098, + -0.036115997, + 0.025706176, + 0.049908377, + 0.025328668, + 0.0090175755, + 0.020332497, + 0.040973034, + -0.016721344, + -0.01928604, + -0.012567933, + 0.0024534017, + -0.027978519, + -0.016283054, + -0.0059162932, + 0.020438466, + -0.033998214, + 0.025316799, + 0.015457932, + -0.027260445, + 0.0077517964, + 0.03414886, + 0.06149145, + -0.009142278, + 0.034868557, + 0.01562827, + -0.011573751, + 0.048809845, + -0.008835819, + -0.042271934, + -0.011501361, + -0.015698932, + 0.00765774, + 0.0040898435, + 0.017931513, + 0.01253479, + -0.026830954, + 0.009097795, + 0.008690346, + 0.044174474, + 0.0060633975, + -0.056720097, + -0.01996871, + -0.035708513, + -0.0106808655, + -0.0031720465, + -0.0111088455, + -0.029465713, + 0.012013627, + 0.014341204, + 0.052498966, + 0.13476695, + 0.046210244, + 0.052687798, + -0.00610014, + 0.045737743, + 0.043751348, + -0.032143187, + 0.0028189388, + -0.035125505, + -0.04272793, + 0.031032862, + 0.034608897, + -0.020154651, + -0.025992718, + 0.0040032133, + -0.013573706, + 0.0014319277, + 0.033563722, + -0.0124865, + -0.013652726, + -0.00095753185, + 0.018106164, + -0.019855194, + 0.00985104, + -0.017903104, + 0.013870335, + -0.02261499, + -0.025842719, + 0.123712055, + -0.016144589, + -0.0011982427, + 0.06412821, + 0.005470516, + 0.013032823, + 0.031510778, + -0.0073665287, + -0.03247822, + 0.00045605694, + -0.018492017, + 0.0098344525, + -0.025878755, + 0.011205348, + -0.012217908, + 0.004897915, + 0.015269929, + -0.02245079, + -0.0018926292, + 0.010645848, + 0.007540698, + -0.018376747, + 0.0011163224, + -0.009108631, + -0.005284527, + 0.027602877, + 0.020686561, + -0.028491242, + -0.016864398, + 0.0044273087, + 0.0062007783, + 0.047445305, + -0.06059543, + -0.023386637, + 0.031225728, + -0.01507088, + -0.030429766, + -0.0027328802, + 0.011734569, + 0.0085614575, + -0.015814755, + -0.010190624, + -0.014055305, + 0.01249472, + 0.0020188028, + -0.018316, + 0.017405652, + 0.00333235, + -0.0058355783, + 0.01903454, + -0.017335443, + 0.009071962, + 0.060741838, + -0.019548608, + 0.0028466568, + -0.0009897373, + -0.01391898, + 0.039289407, + 0.033837304, + -0.020116923, + -0.0049318965, + 0.021821506, + 0.008678599, + -0.005215469, + -0.0035473392, + 0.005850026, + -0.00011736609, + -0.038427465, + 0.004604565, + -0.017376447, + 0.03195163, + -0.037625905, + -0.00025921583, + -0.045117155, + -0.021053704, + 0.01866349, + -0.018683132, + -0.022075273, + -0.067146465, + -0.0003485933, + -0.0066958447, + -0.019604217, + -0.025007758, + 0.008463429, + 0.01903175, + -0.036018703, + 0.0002199503, + -0.014541071, + 0.03636302, + -0.024418777, + 0.036282822, + -0.015224955, + 0.03595708, + -0.040158544, + 0.03579523, + 0.02829524, + 0.011141475, + -0.01444086, + -0.03210745, + 0.012157547, + -0.005177962, + -0.056817777, + 0.01769194, + -0.03663195, + 0.038909055, + -0.009773796, + 0.0063458867, + -0.024140002, + 0.01852215, + -0.0378227, + 0.0095784925, + -0.050063416, + -0.0136081185, + -0.019632932, + 0.0028407509, + -0.0064537292, + -0.016191104, + 0.04372594, + 0.023616401, + 0.027719082, + -0.041044924, + 0.025428394, + 0.020110026, + -0.0060314494, + -0.0061647827, + 0.07147246, + -0.022163881, + -0.002495512, + 0.017374912, + 0.024532678, + 0.017868279, + -0.0123060895, + 0.009035886, + -0.023077391, + -0.00059326034, + -0.021227827, + 0.0071837874, + -0.036735952, + -0.014673909, + -0.028121512, + 0.010137441, + -0.010372104, + -0.038298663, + 0.01282512, + -0.011348841, + 0.022564255, + -0.031195194, + 0.049415413, + -0.03715284, + -0.0024479807, + 0.023421839, + 0.06413017, + -0.020511432, + 0.0113146, + -0.03853917, + 0.015962925, + -0.0053321286, + 0.01771349, + 0.016681707, + 0.020176094, + 0.045899525, + -0.03644215, + -0.05389872, + 0.03137194, + 0.008404089, + -0.038138144, + 0.014195226, + 0.04344335, + 0.0057477932, + -0.03277102, + 0.04831074, + -0.021211635, + 0.030088603, + -0.00251031, + 0.01228796, + -0.004873214, + -0.008176673, + -0.014280845, + -0.0006814086, + -0.019928552, + 0.008787194, + 0.010925872, + -0.0032950707, + -0.034151938, + 0.023929175, + -0.010394337, + -0.010170749, + 0.029074438, + 0.0061191227, + -0.023697818, + -0.021662997, + -0.03463375, + -0.0376407, + -0.008391389, + -0.01087796, + -0.008222668, + 0.0008719065, + 0.07043431, + 0.02145938, + -0.050167546, + -0.023954187, + -0.042268485, + 0.023713034, + -0.22955438, + 0.03520514, + -0.0021951396, + -0.0029458369, + -0.055697378, + 0.0052851303, + -0.008466735, + -0.00713325, + 0.0063962257, + -0.03216339, + -0.024553273, + 0.028201995, + -0.00005423209, + -0.018851798, + -0.007118189, + 0.040054683, + -0.03188417, + -0.0030668841, + -0.004163999, + 0.032002646, + 0.0047784974, + -0.02514371, + -0.005436027, + 0.014385046, + 0.018197643, + 0.0026216004, + -0.026822513, + -0.013488665, + -0.044493288, + -0.01340355, + 0.0006943536, + -0.031944275, + 0.021567853, + 0.043572214, + -0.008895598, + 0.017365277, + 0.00690336, + -0.014118812, + -0.017342428, + 0.013759935, + 0.0014441384, + 0.019967059, + -0.023214184, + -0.0035139646, + -0.0009887023, + 0.028011115, + -0.019943252, + -0.008950248, + -0.059093572, + -0.0024248771, + 0.013076103, + 0.026632652, + -0.018246919, + 0.0191107, + 0.012685868, + 0.00189008, + -0.05187209, + 0.009624937, + -0.0076693487, + 0.037904378, + 0.0033752434, + 0.012424309, + -0.03196644, + -0.03168633, + -0.04591932, + -0.02398697, + -0.08195933, + 0.013067175, + 0.013707991, + 0.02845325, + -0.022814702, + -0.036631793, + -0.0046700407, + -0.014707499, + 0.025149884, + 0.015747225, + 0.011062072, + -0.0079821665, + -0.044059474, + -0.0052218223, + -0.029661851, + -0.0010427525, + -0.052402854, + 0.010401699, + 0.012599079, + 0.018898357, + -0.024228007, + 0.012707991, + -0.01779489, + -0.0073080487, + -0.0570653, + -0.024420062, + -0.0004401083, + 0.0057198247, + -0.04850662, + 0.009310804, + -0.027014086, + -0.012875564, + -0.019028028, + -0.021750357, + -0.039091874, + 0.007929617, + -0.014110445, + -0.0059228763, + -0.027193954, + -0.008130776, + 0.05215637, + -0.009849599, + 0.026919808, + 0.021327212, + -0.013137315, + 0.0056819157, + -0.010384434, + 0.0052230777, + -0.02551496, + -0.023010256, + -0.009133841, + -0.0034335821, + -0.0066825, + -0.012148635, + -0.01445714, + 0.03445042, + -0.0021199132, + -0.04214014, + 0.01076899, + 0.010510174, + 0.035082106, + 0.008629588, + 0.0073693595, + -0.011705998, + 0.046988714, + -0.03607273, + 0.001315558, + 0.028497273, + 0.020122701, + 0.0019115491, + -0.017506052, + 0.05194846, + -0.029103, + -0.059791327, + -0.01251193, + -0.004560316, + -0.001194598, + -0.02777315, + -0.032533206, + -0.010601676, + 0.008287073, + 0.010776672, + -0.05712957, + -0.033277795, + 0.031840082, + 0.025541494, + 0.021605106, + -0.006681298, + 0.0028221495, + 0.01971035, + -0.0057943845, + 0.0026962268, + 0.0066015963, + 0.044709176, + 0.03829154, + -0.04634911, + -0.021133212, + -0.024994686, + -0.010804923, + 0.006723943, + 0.001489737, + -0.011834817, + -0.023318285, + -0.0038526892, + 0.0080185495, + 0.047896653, + -0.03021623, + 0.0534178, + 0.038851902, + -0.009745994, + 0.035945367, + 0.041955516, + 0.03538382, + -0.0023587276, + 0.03368713, + 0.00009861296, + 0.023677062, + 0.024938073, + -0.016926885, + 0.0016545699, + 0.0021329166, + 0.04145556, + -0.0030341037, + 0.026448693, + -0.013949449, + -0.00668188, + 0.0024977801, + -0.052358516, + 0.0584629, + 0.017483938, + 0.016702253, + 0.04207009, + -0.020453833, + 0.04187879, + -0.011923705, + -0.021604257, + -0.010823228, + -0.015712183, + -0.008566785, + -0.00082078524, + -0.02231372, + -0.03011328, + -0.042152412, + 0.031767588, + -0.03151925, + -0.011423307, + -0.01184754, + 0.022948263, + 0.004512842, + 0.0025419197, + 0.02291634, + 0.020818466, + 0.017587373, + 0.0005803047, + -0.016511247, + 0.009406082, + -0.015658183, + -0.035970535, + 0.01030802, + -0.027739283, + -0.022434793, + -0.003182124, + -0.011698128, + -0.014282087, + 0.010802368, + 0.011209724, + -0.037597824, + 0.06280771, + 0.030460317, + -0.014077968, + 0.052043952, + 0.03405556, + 0.0068608206, + -0.024741665, + 0.05734323, + -0.019578138, + -0.018592859, + -0.0055962713 + ] + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-5/router_report.json b/benchmarks/models/router/centroid/out-router-iter-5/router_report.json new file mode 100644 index 0000000..42a4311 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-5/router_report.json @@ -0,0 +1,604 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 141, + "accuracy": 0.53900707, + "tool_precision": 1.0, + "tool_recall": 0.225, + "memory_false_tool_rate": 0.0, + "tool_count": 80, + "memory_count": 46, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70918083, + "sim_memory": 0.69442564, + "sim_abstain": 0.62441397 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6285488, + "sim_memory": 0.61792433, + "sim_abstain": 0.5503659 + }, + { + "query_id": "shell_test_001", + "text": "what processes are listening on port 8003?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6625408, + "sim_memory": 0.5637741, + "sim_abstain": 0.5741711 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6475192, + "sim_memory": 0.54450005, + "sim_abstain": 0.5841964 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7063374, + "sim_memory": 0.6165706, + "sim_abstain": 0.6750375 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6848668, + "sim_memory": 0.5817895, + "sim_abstain": 0.6013617 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6619269, + "sim_memory": 0.595923, + "sim_abstain": 0.5525088 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6134916, + "sim_memory": 0.53323615, + "sim_abstain": 0.53623945 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7070505, + "sim_memory": 0.6493096, + "sim_abstain": 0.5815989 + }, + { + "query_id": "shell_test_010", + "text": "what's the uptime since klbr daemon last started", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6483902, + "sim_memory": 0.55113995, + "sim_abstain": 0.51299685 + }, + { + "query_id": "shell_test_011", + "text": "how big is agent.db right now on disk?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.618642, + "sim_memory": 0.5386792, + "sim_abstain": 0.52846694 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6492218, + "sim_memory": 0.56391144, + "sim_abstain": 0.5497022 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.65761024, + "sim_memory": 0.56834835, + "sim_abstain": 0.62134224 + }, + { + "query_id": "shell_test_014", + "text": "how many memory rows are in agent.db right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6829826, + "sim_memory": 0.58638436, + "sim_abstain": 0.5644451 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5843893, + "sim_memory": 0.547776, + "sim_abstain": 0.5217595 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6942834, + "sim_memory": 0.6637705, + "sim_abstain": 0.5931268 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66822094, + "sim_memory": 0.5693777, + "sim_abstain": 0.6224822 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66665, + "sim_memory": 0.5669212, + "sim_abstain": 0.60647064 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5745318, + "sim_memory": 0.52103066, + "sim_abstain": 0.5162513 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.64815664, + "sim_memory": 0.53975767, + "sim_abstain": 0.5891774 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5961604, + "sim_memory": 0.5583175, + "sim_abstain": 0.57129914 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6546817, + "sim_memory": 0.5749995, + "sim_abstain": 0.61477154 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.68234557, + "sim_memory": 0.60593796, + "sim_abstain": 0.60524577 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.60944146, + "sim_memory": 0.524113, + "sim_abstain": 0.54898715 + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.566589, + "sim_memory": 0.4897711, + "sim_abstain": 0.465443 + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5898618, + "sim_memory": 0.5351446, + "sim_abstain": 0.52470756 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66542095, + "sim_memory": 0.5676791, + "sim_abstain": 0.59653854 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6722767, + "sim_memory": 0.6186101, + "sim_abstain": 0.54912585 + }, + { + "query_id": "rf_test_002", + "text": "which file defines the ClientMsg type?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.53049797, + "sim_memory": 0.44120914, + "sim_abstain": 0.4389323 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6583943, + "sim_memory": 0.6532231, + "sim_abstain": 0.5341912 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.59658426, + "sim_memory": 0.5636423, + "sim_abstain": 0.5362745 + }, + { + "query_id": "rf_test_006", + "text": "config.rsのmemory_support_score_gapのデフォルト値はいくつ?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.705353, + "sim_memory": 0.6360202, + "sim_abstain": 0.55474705 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6974401, + "sim_memory": 0.65405804, + "sim_abstain": 0.58798295 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5960697, + "sim_memory": 0.55635834, + "sim_abstain": 0.5040847 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65429235, + "sim_memory": 0.63408476, + "sim_abstain": 0.554867 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5710796, + "sim_memory": 0.5308014, + "sim_abstain": 0.50004137 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6926011, + "sim_memory": 0.691817, + "sim_abstain": 0.58473253 + }, + { + "query_id": "rf_test_016", + "text": "read interrupt.rs and tell me what the Interrupt enum contains", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6431551, + "sim_memory": 0.5701346, + "sim_abstain": 0.5663854 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65750617, + "sim_memory": 0.60042876, + "sim_abstain": 0.54133034 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6355181, + "sim_memory": 0.5609783, + "sim_abstain": 0.5377031 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6916864, + "sim_memory": 0.6152147, + "sim_abstain": 0.57028234 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6794881, + "sim_memory": 0.5732701, + "sim_abstain": 0.5938102 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5705274, + "sim_memory": 0.53116524, + "sim_abstain": 0.48013124 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.55382746, + "sim_memory": 0.5629208, + "sim_abstain": 0.511596 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.654468, + "sim_memory": 0.6415955, + "sim_abstain": 0.5482988 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.56339705, + "sim_memory": 0.5195113, + "sim_abstain": 0.4231544 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6873349, + "sim_memory": 0.6034543, + "sim_abstain": 0.58547026 + }, + { + "query_id": "rf_test_031", + "text": "read the first 10 lines of agent.rs", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5996914, + "sim_memory": 0.5221693, + "sim_abstain": 0.54224586 + }, + { + "query_id": "wf_test_001", + "text": "set the compaction_keep to 15 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.69275683, + "sim_memory": 0.62913835, + "sim_abstain": 0.5867398 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65280145, + "sim_memory": 0.62437373, + "sim_abstain": 0.5488851 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.57037896, + "sim_memory": 0.5085687, + "sim_abstain": 0.52038395 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.652064, + "sim_memory": 0.57737964, + "sim_abstain": 0.5854702 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6568779, + "sim_memory": 0.6171636, + "sim_abstain": 0.5631379 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.61866033, + "sim_memory": 0.5483311, + "sim_abstain": 0.5347757 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7101654, + "sim_memory": 0.6064741, + "sim_abstain": 0.6337338 + }, + { + "query_id": "wf_test_008", + "text": "patch the memory_top_k to 5 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7501923, + "sim_memory": 0.6741924, + "sim_abstain": 0.61280704 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.64629316, + "sim_memory": 0.61019343, + "sim_abstain": 0.59611 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5862066, + "sim_memory": 0.5496563, + "sim_abstain": 0.51394325 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6231395, + "sim_memory": 0.57401973, + "sim_abstain": 0.55732864 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6762118, + "sim_memory": 0.6369297, + "sim_abstain": 0.5889329 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.606107, + "sim_memory": 0.5466962, + "sim_abstain": 0.5397958 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70418376, + "sim_memory": 0.6688043, + "sim_abstain": 0.6164449 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.5306442, + "sim_memory": 0.45544416, + "sim_abstain": 0.45304802 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.50268793, + "sim_memory": 0.6389982, + "sim_abstain": 0.5120093 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.5960785, + "sim_memory": 0.5498394, + "sim_abstain": 0.5258497 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-5/router_report.md b/benchmarks/models/router/centroid/out-router-iter-5/router_report.md new file mode 100644 index 0000000..7ed168b --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-5/router_report.md @@ -0,0 +1,86 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `80` memory: `46` abstain: `15` +- accuracy: `0.5390` +- tools precision: `1.0000` +- tools recall: `0.2250` +- memory false-tools rate: `0.0000` + +## Misclassified Queries + +| id | text | gold | pred | T-sim | M-sim | A-sim | +|---|---|---|---|---|---|---| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.71 | 0.69 | 0.62 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 | +| `shell_test_001` | what processes are listening on port 8003? | `Tools` | `Abstain` | 0.66 | 0.56 | 0.57 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | 0.65 | 0.54 | 0.58 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.71 | 0.62 | 0.68 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.68 | 0.58 | 0.60 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.66 | 0.60 | 0.55 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Abstain` | 0.61 | 0.53 | 0.54 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.71 | 0.65 | 0.58 | +| `shell_test_010` | what's the uptime since klbr daemon last started | `Tools` | `Memory` | 0.65 | 0.55 | 0.51 | +| `shell_test_011` | how big is agent.db right now on disk? | `Tools` | `Memory` | 0.62 | 0.54 | 0.53 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.65 | 0.56 | 0.55 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.66 | 0.57 | 0.62 | +| `shell_test_014` | how many memory rows are in agent.db right now? | `Tools` | `Memory` | 0.68 | 0.59 | 0.56 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.58 | 0.55 | 0.52 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.69 | 0.66 | 0.59 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.62 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.61 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.57 | 0.52 | 0.52 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.65 | 0.54 | 0.59 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.60 | 0.56 | 0.57 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.65 | 0.57 | 0.61 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.68 | 0.61 | 0.61 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.61 | 0.52 | 0.55 | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | 0.57 | 0.49 | 0.47 | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | 0.59 | 0.54 | 0.52 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.67 | 0.57 | 0.60 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.67 | 0.62 | 0.55 | +| `rf_test_002` | which file defines the ClientMsg type? | `Tools` | `Memory` | 0.53 | 0.44 | 0.44 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.66 | 0.65 | 0.53 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.60 | 0.56 | 0.54 | +| `rf_test_006` | config.rsのmemory_support_score_gapのデフォルト値はいくつ? | `Tools` | `Memory` | 0.71 | 0.64 | 0.55 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.70 | 0.65 | 0.59 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.60 | 0.56 | 0.50 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.65 | 0.63 | 0.55 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.57 | 0.53 | 0.50 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.69 | 0.69 | 0.58 | +| `rf_test_016` | read interrupt.rs and tell me what the Interrupt enum contains | `Tools` | `Memory` | 0.64 | 0.57 | 0.57 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.66 | 0.60 | 0.54 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.64 | 0.56 | 0.54 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.69 | 0.62 | 0.57 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.68 | 0.57 | 0.59 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.57 | 0.53 | 0.48 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.55 | 0.56 | 0.51 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.65 | 0.64 | 0.55 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.56 | 0.52 | 0.42 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.69 | 0.60 | 0.59 | +| `rf_test_031` | read the first 10 lines of agent.rs | `Tools` | `Abstain` | 0.60 | 0.52 | 0.54 | +| `wf_test_001` | set the compaction_keep to 15 in config.rs | `Tools` | `Memory` | 0.69 | 0.63 | 0.59 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.65 | 0.62 | 0.55 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Abstain` | 0.57 | 0.51 | 0.52 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Abstain` | 0.65 | 0.58 | 0.59 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.62 | 0.56 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.62 | 0.55 | 0.53 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.71 | 0.61 | 0.63 | +| `wf_test_008` | patch the memory_top_k to 5 in config.rs | `Tools` | `Memory` | 0.75 | 0.67 | 0.61 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.65 | 0.61 | 0.60 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.59 | 0.55 | 0.51 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.62 | 0.57 | 0.56 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.68 | 0.64 | 0.59 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.61 | 0.55 | 0.54 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.70 | 0.67 | 0.62 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | 0.53 | 0.46 | 0.45 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.50 | 0.64 | 0.51 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.55 | 0.53 | diff --git a/benchmarks/models/router/centroid/out-router-iter-6/router_model.json b/benchmarks/models/router/centroid/out-router-iter-6/router_model.json new file mode 100644 index 0000000..a1bd52a --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-6/router_model.json @@ -0,0 +1,6168 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.1, + "memory_centroids": [ + [ + -0.02953457, + -0.023501687, + -0.05808627, + 0.0070140953, + 0.006874998, + -0.047377475, + 0.013503848, + 0.004033102, + 0.013937307, + 0.025993194, + 0.00036805047, + 0.026340032, + -0.00032751352, + 0.020008639, + 0.010437691, + -0.01599532, + 0.0378925, + -0.034139622, + 0.031576242, + -0.005721982, + -0.015650975, + -0.013299033, + 0.0015801851, + 0.05208871, + -0.0033491016, + 0.05292174, + -0.041163854, + -0.013427372, + 0.018127806, + 0.031762872, + 0.0080265, + 0.027610881, + 0.0102199325, + -0.02962109, + -0.025770932, + -0.027322631, + 0.018344872, + -0.01546087, + -0.040318888, + -0.024949342, + -0.02684663, + 0.019792331, + 0.016403709, + -0.03913061, + 0.041741397, + -0.051342268, + 0.03370111, + -0.041125838, + -0.0036108876, + 0.015794855, + -0.018666392, + -0.014134196, + 0.015846388, + -0.02152241, + 0.04721869, + 0.038556382, + -0.0057476372, + 0.01546458, + -0.04171342, + -0.007379216, + -0.0137405945, + -0.015948497, + -0.015352586, + 0.011297628, + 0.021778889, + 0.09534737, + 0.027794316, + -0.002793605, + -0.020545956, + -0.041483287, + -0.025141848, + 0.006380157, + -0.00047512646, + -0.0032311745, + -0.08699265, + 0.02291205, + 0.049483027, + -0.022220165, + 0.0006178035, + 0.015320561, + 0.047333814, + -0.006275665, + 0.034674358, + 0.014853668, + -0.0034956613, + 0.014414248, + -0.028953297, + 0.035040356, + 0.0029728715, + -0.031037042, + -0.010100751, + 0.023466574, + 0.013175919, + -0.052728992, + -0.046121895, + 0.0069394163, + -0.029717263, + -0.013612056, + 0.039096978, + 0.006514509, + 0.026238186, + 0.011335962, + 0.015135213, + -0.030236373, + 0.052462798, + -0.009011261, + 0.04137082, + 0.036443684, + -0.024784643, + -0.010885915, + 0.02339825, + 0.01153446, + 0.019820947, + 0.01667984, + -0.02987813, + -0.03124927, + -0.033248678, + -0.026805358, + 0.05532143, + 0.006061382, + 0.05987853, + 0.0053289263, + 0.06786469, + -0.03602914, + -0.015517126, + -0.04250494, + 0.020546658, + 0.012809727, + -0.00601842, + 0.008231562, + -0.004753981, + 0.0181044, + -0.02190586, + 0.0015789795, + -0.0386283, + -0.015395165, + 0.013448149, + -0.0020003526, + -0.026016362, + -0.05009787, + 0.033090923, + 0.029800063, + -0.030307502, + -0.027842177, + 0.0139083, + -0.019292444, + 0.008088462, + -0.011791759, + 0.013513014, + 0.0027809977, + -0.015044234, + -0.0142502105, + -0.011962004, + -0.013559485, + 0.025546454, + -0.0144509105, + 0.022906901, + 0.08040018, + -0.024976632, + 0.05750807, + -0.015728489, + 0.025379393, + -0.042068284, + -0.011159206, + 0.045110248, + 0.0027386474, + 0.01061638, + 0.037824012, + -0.015301461, + -0.010647753, + -0.019499803, + -0.016362367, + 0.029056897, + -0.016690247, + 0.02505767, + 0.020635538, + 0.10579964, + 0.017164556, + 0.010356794, + -0.04950358, + -0.047854308, + 0.012633971, + 0.020412799, + -0.03777352, + -0.019705521, + -0.021516891, + 0.037748206, + -0.05184238, + 0.015875751, + 0.032903414, + 0.00286073, + -0.0017315428, + 0.03207537, + 0.0024716682, + 0.036519893, + 0.019512344, + -0.005500415, + -0.028663289, + 0.006004013, + -0.028784506, + -0.009142338, + 0.037859693, + 0.063061036, + -0.007193967, + -0.059196513, + -0.00025405674, + -0.0404329, + -0.050101977, + 0.018440584, + 0.0019876033, + -0.018863054, + 0.01567874, + 0.02782502, + 0.0050265174, + -0.035830576, + 0.00025280323, + 0.024342729, + -0.0027864513, + -0.011164368, + -0.012470029, + -0.020127311, + 0.044781983, + -0.016764725, + -0.060266845, + 0.02845582, + 0.0326934, + 0.0023419242, + -0.0018993955, + -0.005613019, + -0.049475726, + -0.0010083856, + 0.024441615, + -0.022721602, + -0.024691395, + 0.034620766, + -0.00882457, + 0.0016938613, + -0.003250522, + 0.013695728, + -0.013721317, + 0.0021146091, + -0.028904684, + -0.0029565294, + -0.0042853206, + -0.05602909, + -0.0023269707, + 0.011781264, + 0.013156883, + 0.0032094966, + 0.013319434, + 0.027515475, + 0.00003426793, + -0.04807129, + -0.02646676, + 0.055103097, + 0.03457307, + 0.032444008, + -0.0027100584, + 0.024296409, + 0.0019906394, + 0.03768081, + 0.0022763326, + 0.011797177, + 0.036109027, + 0.026424909, + -0.0031692882, + -0.024819322, + -0.029096847, + 0.0043520858, + 0.0067918655, + -0.03163525, + 0.0048874402, + 0.022340473, + 0.023326822, + 0.03814576, + 0.026113065, + -0.03389849, + -0.035517395, + 0.05969392, + 0.01776524, + -0.012784303, + 0.013127809, + 0.022363255, + -0.013977365, + -0.0039680754, + -0.017222684, + -0.02858304, + -0.011452531, + -0.007811619, + -0.045651883, + 0.014498947, + -0.0013878772, + 0.100820094, + 0.005297464, + -0.028213173, + -0.03314273, + -0.04044471, + -0.22465427, + -0.02858451, + -0.003905871, + 0.00022481597, + 0.0047072093, + -0.012649562, + -0.017003896, + 0.006186442, + -0.03956627, + 0.036402363, + -0.051759057, + -0.068457276, + 0.007101277, + -0.0124830175, + -0.019360466, + 0.0033654436, + -0.011729769, + -0.009554513, + 0.018463207, + -0.03167895, + -0.006967882, + -0.019700885, + 0.023060178, + -0.020346511, + -0.0061216676, + -0.014901207, + 0.028405404, + 0.022600891, + -0.03975633, + -0.020692414, + 0.017461983, + 0.027074642, + -0.017387684, + 0.047598917, + 0.013716522, + 0.022675045, + 0.021145523, + -0.017724916, + -0.026462242, + -0.014224281, + 0.030817714, + 0.0114215035, + -0.021204576, + 0.034436375, + 0.024465613, + -0.045439, + -0.013454584, + -0.030626189, + -0.0077004014, + -0.0046026013, + -0.027327102, + 0.012236565, + 0.0023989333, + -0.0047771633, + -0.02984797, + -0.008682544, + -0.016193328, + 0.039879285, + -0.06660901, + 0.012391835, + -0.0049603703, + 0.0007947687, + 0.016125074, + 0.040164527, + -0.033208624, + 0.010781524, + -0.025258325, + -0.020662768, + 0.03224673, + -0.023760425, + 0.030194627, + -0.0016037951, + 0.0024073026, + -0.011888995, + -0.021232078, + 0.053651586, + -0.013994518, + -0.039026722, + -0.018290367, + -0.1595071, + -0.010143831, + -0.009503269, + 0.0075196954, + -0.0016893324, + -0.023991065, + -0.0004729232, + 0.005407145, + 0.045051403, + 0.03946149, + 0.31316122, + 0.019875187, + -0.017683508, + -0.011246197, + 0.041095726, + -0.01707569, + 0.02580701, + 0.020073302, + 0.0076661776, + -0.039371304, + -0.030318934, + 0.013346287, + 0.020821072, + 0.0046413513, + 0.016254276, + 0.005412236, + -0.03347545, + 0.010158307, + 0.1002082, + -0.011839762, + 0.03285164, + 0.012760006, + 0.02366014, + 0.022854084, + -0.05203756, + -0.03471872, + -0.009616535, + 0.058146782, + -0.020818148, + 0.03461769, + -0.026271882, + 0.021051634, + 0.02911293, + -0.01990685, + -0.041703526, + 0.0002907327, + 0.013849215, + 0.016160823, + 0.0024423078, + 0.03202622, + -0.0015427561, + -0.042227678, + -0.001353875, + -0.013348268, + -0.0008665041, + -0.042891808, + -0.006410304, + -0.02022797, + -0.03159226, + -0.022437904, + -0.02003756, + 0.015528935, + -0.017203221, + -0.0100704925, + -0.010856735, + -0.021080589, + -0.0504776, + 0.010053538, + -0.0030528405, + 0.027641593, + 0.027326025, + 0.0128613785, + -0.046285115, + -0.010928768, + -0.03181085, + 0.017708378, + -0.029218853, + 0.00059574813, + 0.031865608, + 0.013683232, + 0.0025548658, + -0.006452052, + 0.008344617, + -0.0018459023, + 0.04586065, + 0.007935226, + 0.00002709333, + 0.08221116, + -0.009100065, + 0.029175958, + -0.048444733, + -0.008251141, + -0.03405043, + 0.032034725, + 0.007967471, + -0.024803922, + -0.015552689, + 0.05681926, + 0.0037742213, + -0.026362995, + -0.007213026, + 0.014125172, + -0.03752151, + -0.0024824142, + 0.012550943, + 0.008332322, + 0.015813878, + -0.011721951, + -0.03736512, + -0.025358934, + -0.041780002, + -0.016802674, + 0.0075594457, + 0.012926832, + 0.0022665393, + 0.00544774, + -0.008000439, + 0.015729414, + 0.025527108, + 0.015055225, + -0.02852011, + 0.029998757, + -0.023629876, + -0.019701436, + 0.046159368, + 0.010666163, + -0.009261453, + 0.027201602, + -0.005710259, + 0.03726728, + -0.01976218, + -0.023621652, + -0.0029942847, + -0.00047694452, + -0.0040939855, + -0.0099202385, + 0.0065422435, + -0.0429584, + -0.031387664, + 0.022027241, + 0.026100025, + -0.007511666, + 0.030582216, + 0.012822923, + 0.036304314, + 0.025152108, + -0.017161628, + 0.016741537, + 0.018255189, + 0.035726435, + -0.0062032174, + -0.05698027, + -0.013355153, + 0.014536069, + -0.01766706, + -0.008239166, + 0.018501902, + -0.0010640995, + -0.018297568, + 0.019297598, + 0.01495364, + 0.036746204, + -0.02745614, + -0.055277172, + -0.029027686, + -0.032996736, + -0.010148253, + -0.0014721167, + -0.023568414, + -0.040075224, + 0.019336946, + 0.0005275909, + 0.006581408, + 0.05709383, + 0.039294206, + 0.040886384, + -0.032870255, + 0.040065862, + 0.03958012, + -0.042370867, + 0.0011705015, + -0.012909272, + -0.033453353, + 0.037587993, + 0.025471324, + -0.008906781, + -0.06545729, + 0.0017291277, + 0.008953089, + 0.016879518, + 0.026898118, + 0.027753143, + -0.03992264, + -0.004906459, + 0.0039307103, + 0.026820708, + -0.031330388, + -0.024806987, + 0.031612832, + -0.032069124, + -0.014307158, + 0.12864976, + -0.0051088952, + 0.004911811, + 0.005624591, + 0.010622269, + 0.014215928, + 0.012548023, + -0.030215545, + -0.029370805, + 0.0002464952, + 0.017944876, + 0.001236656, + -0.028763784, + 0.023505056, + 0.022390043, + -0.0015424239, + 0.027351622, + -0.04140385, + -0.030692495, + 0.0076649394, + -0.0017135493, + -0.046149958, + 0.0039872695, + 0.025925033, + -0.007340319, + 0.046820674, + 0.039672777, + -0.011907181, + -0.017452894, + -0.00021526583, + 0.0139043005, + 0.029623425, + -0.055875365, + -0.020226091, + 0.00017796963, + -0.030965574, + -0.06364141, + 0.013530132, + 0.008922023, + 0.014563754, + 0.0031031477, + -0.0030001134, + -0.030106964, + 0.03502163, + -0.0019424554, + -0.0020550822, + 0.03134754, + 0.0045823157, + -0.019614106, + -0.0048672026, + -0.022130122, + -0.009755478, + 0.03228172, + -0.019890655, + 0.008002598, + -0.00781725, + -0.011763561, + 0.01594538, + 0.03417074, + 0.011124715, + 0.00860674, + 0.0053270827, + -0.018849868, + -0.019411609, + 0.016141795, + 0.029577173, + -0.012376068, + -0.028442422, + -0.0102799805, + -0.015962493, + 0.04011505, + -0.035837717, + -0.013337596, + -0.050945375, + -0.013998839, + 0.04386832, + -0.025864763, + -0.03339192, + -0.047449946, + 0.004850084, + -0.015017307, + -0.062756956, + -0.04656785, + 0.021192448, + 0.00956155, + -0.05119442, + -0.008371866, + -0.0042842724, + 0.019156162, + -0.010939484, + 0.018840468, + -0.025657684, + 0.01106568, + -0.03244388, + 0.006420251, + 0.04826801, + -0.00057971664, + -0.006369808, + -0.031549037, + 0.049913317, + 0.007017293, + -0.030578466, + 0.02232669, + -0.04201414, + 0.026563568, + -0.015477179, + 0.026783573, + -0.022415739, + 0.016253402, + -0.025808347, + 0.052372962, + -0.05774024, + -0.011736604, + -0.010124982, + -0.0048620454, + -0.019803265, + -0.008477471, + 0.030738277, + 0.038898382, + 0.023150876, + -0.042604998, + 0.0019641898, + 0.00784871, + -0.004138505, + 0.011478454, + 0.030663332, + -0.01506021, + -0.02919572, + 0.012824615, + -0.0012030387, + 0.005259654, + -0.01432216, + -0.009896729, + -0.020819265, + -0.0027348117, + 0.00016687928, + 0.008219463, + -0.008309997, + -0.0117734615, + -0.017596774, + -0.007878184, + -0.011875525, + -0.011689367, + 0.048842385, + 0.0032761202, + -0.004180586, + -0.02784116, + 0.019389883, + 0.0059328643, + 0.017998025, + -0.0043417537, + 0.03363939, + -0.04814994, + 0.017516242, + -0.047206663, + 0.008795233, + 0.006181405, + -0.0076484066, + 0.021008575, + 0.0067271558, + 0.026135894, + -0.030862615, + -0.059556678, + -0.005249494, + -0.005896798, + -0.053860072, + 0.058404587, + 0.036468524, + 0.014157678, + -0.028333092, + 0.03760916, + 0.00519971, + 0.009662147, + -0.008499046, + 0.027133144, + 0.014945763, + -0.022028917, + 0.0066466467, + -0.012278338, + -0.025636343, + -0.0009813369, + -0.023540856, + -0.0014314126, + -0.016507363, + 0.045651384, + -0.0082562985, + -0.0041743293, + 0.05165129, + 0.025758864, + -0.029776005, + -0.012274611, + 0.01466031, + -0.038974468, + 0.0070445114, + -0.007370479, + -0.027545778, + -0.013694657, + 0.043757852, + 0.04633227, + -0.021911979, + -0.01796987, + -0.054099757, + 0.01452772, + -0.2138702, + 0.02743569, + 0.015744813, + -0.020886457, + -0.051552296, + -0.0027605426, + -0.0012711033, + -0.0047403597, + 0.009300366, + -0.029458603, + -0.02747486, + 0.025445733, + 0.0050143576, + -0.012034659, + -0.016319465, + 0.019868318, + -0.013839318, + -0.00029665322, + 0.009236322, + 0.018608531, + 0.0050179, + -0.017188085, + 0.03464569, + 0.005022059, + 0.02257357, + -0.026489973, + -0.035256382, + 0.0122413905, + -0.057871927, + -0.009198643, + 0.002898138, + 0.0062493277, + 0.032887958, + 0.047671497, + -0.026613321, + -0.005649091, + -0.0041015684, + -0.0281537, + 0.003319403, + -0.021404069, + -0.002427804, + 0.04324175, + -0.014130836, + -0.031917486, + -0.011499083, + 0.030707614, + -0.0028441278, + 0.018666685, + -0.06646727, + 0.019018704, + -0.004152146, + 0.01975428, + -0.013143967, + 0.0073776282, + 0.020081295, + -0.02697742, + -0.021839116, + 0.017184513, + 0.027456801, + 0.03954958, + 0.0016815886, + -0.016831884, + -0.027606057, + -0.050559748, + -0.051494483, + -0.012649485, + -0.07952543, + 0.004067631, + -0.0021464548, + 0.00786993, + -0.023989927, + -0.024670297, + -0.0037496786, + -0.04951884, + 0.016926184, + -0.018860383, + 0.012256827, + -0.010916814, + -0.010613337, + -0.025584185, + -0.0030827215, + 0.016111253, + -0.04742576, + -0.024564615, + 0.042982895, + 0.002380033, + -0.024394304, + 0.0062498963, + -0.009741223, + 0.00893389, + -0.06926393, + 0.017702833, + 0.003559851, + 0.023049349, + -0.012283555, + 0.03369362, + -0.012401988, + 0.0050276346, + -0.0166348, + -0.022107374, + -0.059628308, + -0.0036324025, + -0.011819672, + 0.026103102, + -0.014030879, + -0.03181233, + 0.03724609, + -0.010170614, + 0.0054339855, + -0.0066714897, + -0.03532086, + -0.015470396, + -0.019201776, + 0.010452439, + -0.035049714, + 0.008742536, + 0.017332299, + -0.0028451565, + -0.03990205, + 0.0027475304, + -0.000050377344, + 0.033374507, + -0.011835915, + -0.044426147, + -0.0012377469, + -0.0005448537, + 0.054194182, + -0.0016887394, + -0.009132979, + -0.014007341, + 0.048595123, + -0.024569178, + -0.010165418, + 0.039706923, + 0.009360816, + -0.0142467925, + -0.02289829, + 0.046355814, + -0.01972842, + -0.051768117, + 0.008552698, + 0.014160438, + 0.012196202, + -0.0075761625, + -0.019158937, + 0.0119345505, + 0.007685015, + 0.016935337, + -0.051301133, + -0.015521768, + 0.03698387, + 0.010485435, + 0.0012461421, + 0.009096625, + 0.009374426, + 0.004323882, + 0.022198806, + 0.025565172, + -0.023361158, + 0.059670344, + 0.036556274, + -0.038306747, + -0.034494575, + -0.04563513, + -0.0052286144, + 0.0026300608, + 0.022318833, + -0.0048725726, + -0.0037535578, + 0.0087553235, + -0.0037525538, + 0.052146267, + -0.02985571, + 0.07571844, + 0.014147077, + -0.050934263, + 0.041672062, + -0.0042912466, + 0.042178534, + -0.016792161, + 0.029752152, + -0.020074084, + -0.0091867, + 0.023213692, + 0.006531691, + 0.015507917, + 0.028167203, + 0.050762765, + 0.010745568, + 0.0008412688, + 0.01909061, + -0.009928744, + -0.0028815789, + -0.04545366, + 0.042247232, + -0.010490567, + 0.020409765, + 0.025339702, + -0.035115454, + 0.061531704, + -0.011911052, + -0.03500391, + -0.04524361, + -0.013206155, + -0.022409508, + -0.008728653, + -0.034059823, + -0.02689995, + -0.03300723, + 0.043033533, + -0.0290939, + 0.0121105155, + -0.02727977, + 0.0019806651, + 0.0068177576, + 0.00031189778, + 0.010153875, + 0.024259033, + 0.02276865, + -0.02847139, + -0.010577961, + 0.010186096, + 0.0016031668, + -0.04977359, + -0.034086004, + -0.009296076, + -0.015570667, + -0.007774638, + -0.006385688, + -0.011167805, + -0.021814104, + 0.03431478, + -0.037030302, + 0.032789752, + 0.015891775, + 0.00091708323, + 0.05153089, + 0.025300793, + -0.004147137, + -0.002914364, + 0.047010057, + -0.02837259, + 0.003133865, + 0.029014219 + ], + [ + -0.020254124, + -0.04116675, + -0.007372979, + -0.029964684, + 0.011190738, + -0.08159443, + 0.017661301, + -0.010246391, + -0.008209263, + 0.022077559, + -0.037495364, + 0.038561653, + 0.041310087, + -0.03827206, + -0.0030296666, + -0.04551725, + -0.0015584466, + -0.043835424, + 0.04111606, + -0.0074276403, + -0.014586625, + -0.034484424, + -0.031067863, + -0.0055164816, + 0.002870236, + 0.03311342, + 0.0054400344, + 0.020860758, + -0.012762298, + 0.020914592, + -0.037719376, + 0.019643847, + -0.026980707, + -0.01616462, + 0.0057277605, + -0.018851377, + 0.013205356, + 0.026758894, + -0.05611753, + -0.05673022, + -0.015809074, + 0.022022985, + 0.064844824, + 0.009851467, + -0.0074210535, + -0.02909688, + 0.012350218, + -0.022363547, + -0.01382957, + -0.00794799, + 0.010876325, + 0.051783428, + -0.035475973, + -0.0031464773, + 0.07489341, + 0.034771163, + -0.041550077, + 0.034477636, + -0.019864663, + 0.0066035762, + 0.025300935, + 0.041910917, + 0.018587649, + 0.020312272, + 0.017726174, + 0.073004626, + -0.005812487, + 0.016792096, + -0.025332022, + 0.0049700495, + -0.053332593, + 0.04201889, + 0.004391433, + -0.0021935226, + -0.09358781, + 0.03083754, + 0.019159859, + 0.004591123, + 0.02774286, + 0.04111074, + 0.04572332, + -0.03794063, + -0.008300797, + 0.01320226, + 0.016294988, + 0.032590497, + 0.014050795, + 0.03014458, + -0.024947153, + -0.028386062, + -0.0069098324, + 0.016842715, + -0.0002635318, + -0.052521348, + -0.056614153, + 0.0476653, + -0.002092335, + 0.0067270766, + 0.034543928, + 0.06673301, + -0.008618899, + 0.003757926, + 0.0049299044, + 0.04859857, + -0.028685993, + -0.047352344, + 0.06761084, + 0.0026073721, + -0.03719376, + -0.008147757, + 0.002335019, + -0.055548757, + -0.019844908, + 0.0003014616, + -0.04448501, + -0.053816702, + -0.06500549, + 0.033419482, + 0.023317087, + -0.012048277, + 0.045431584, + 0.0044442685, + 0.00503645, + -0.023623671, + 0.036882322, + -0.03137511, + 0.03476906, + -0.0004156654, + -0.03795088, + -0.035963524, + -0.058573436, + 0.029147573, + -0.04297972, + -0.011300198, + -0.04443065, + -0.010125407, + -0.035875123, + 0.005237153, + 0.018849581, + -0.04374896, + -0.006881187, + -0.01323515, + -0.020799844, + -0.028531378, + 0.03373202, + -0.019024812, + -0.023178076, + -0.013414461, + 0.022442773, + -0.022926392, + 0.017389378, + -0.00240939, + -0.05073249, + -0.044891775, + -0.015401027, + -0.023299117, + 0.018395206, + 0.024170302, + -0.013018237, + 0.014878734, + -0.032968245, + 0.008014404, + -0.018779429, + -0.045670286, + 0.008193353, + -0.024559846, + 0.057381175, + 0.0669771, + 0.044439804, + -0.0036185777, + 0.021347538, + 0.048340894, + 0.026817888, + 0.058665168, + 0.023300119, + 0.026641518, + 0.039867714, + 0.0049528927, + -0.0011300519, + -0.037390184, + -0.06264999, + -0.01630887, + 0.008078938, + 0.0010589276, + 0.020439938, + 0.025579099, + -0.02067246, + -0.02252045, + 0.00242239, + 0.027789189, + -0.028604046, + -0.010315909, + 0.045320645, + -0.008987895, + -0.023577353, + 0.032551922, + 0.04230767, + -0.003796409, + 0.00983521, + -0.022642719, + 0.004940319, + 0.01627678, + 0.047532525, + -0.06016555, + 0.011346706, + -0.005472442, + -0.018862773, + -0.035922226, + -0.010526478, + -0.015118645, + 0.014042482, + 0.009215666, + 0.01457269, + -0.030881861, + 0.015838949, + -0.021441704, + 0.033870716, + -0.0088826725, + -0.05239024, + -0.015585623, + -0.034875818, + -0.032844577, + 0.011226785, + -0.015373432, + 0.015121698, + -0.00072043814, + -0.0068776575, + 0.06292077, + -0.009777719, + -0.04709221, + -0.029894646, + 0.0027344727, + -0.008795243, + -0.017512025, + 0.04226805, + 0.025590824, + -0.008859926, + 0.04388061, + 0.05875191, + 0.0108080255, + -0.031908263, + 0.01467872, + -0.014533983, + 0.015212947, + 0.02041433, + 0.03890462, + -0.024720386, + 0.006734559, + -0.023964731, + 0.01345108, + 0.014990413, + 0.0030772437, + -0.058163464, + 0.047327667, + 0.00489635, + 0.005195013, + 0.0001313037, + -0.016700808, + 0.047128133, + 0.009359673, + 0.065170325, + 0.020720795, + 0.010661119, + 0.0096793715, + -0.007632, + -0.047226686, + 0.0056292997, + -0.018743971, + -0.021640094, + 0.010677068, + -0.033116397, + -0.0020473609, + -0.0034470889, + 0.048381247, + 0.04437471, + 0.024240568, + 0.005486285, + -0.05120708, + 0.0025768455, + -0.015763147, + 0.05396275, + -0.046657037, + -0.010277123, + 0.05113114, + 0.03595713, + -0.005959823, + -0.065834485, + 0.019947264, + 0.00816652, + -0.056714363, + -0.029086692, + -0.006467168, + 0.020627813, + 0.00316757, + -0.007847554, + -0.0034523793, + -0.06581499, + -0.124769256, + -0.0475397, + -0.02573181, + -0.027566018, + 0.012921132, + -0.036089245, + -0.04352001, + -0.015061321, + -0.038911663, + -0.014494528, + -0.02425402, + -0.05302295, + 0.018130956, + -0.017570471, + 0.003728436, + -0.022605387, + 0.020288603, + 0.0016745474, + 0.015943542, + 0.013685553, + -0.020636661, + 0.0014871317, + 0.036179543, + 0.0666887, + 0.018872928, + -0.011584086, + 0.0346419, + 0.0010764768, + 0.015926719, + -0.01975337, + 0.008983314, + 0.010039464, + -0.006696494, + 0.0674074, + -0.025392266, + -0.009440608, + -0.014110286, + 0.029129889, + -0.0061896862, + -0.016916197, + 0.017264564, + -0.023885023, + 0.014502518, + 0.014711948, + -0.024939105, + -0.03776957, + 0.01799964, + -0.025129559, + -0.040642563, + 0.01042123, + 0.011569937, + 0.02311178, + -0.009573609, + 0.022429014, + 0.0010554203, + -0.021534372, + 0.009123053, + 0.07520331, + -0.022060657, + 0.0064048725, + -0.026625117, + 0.016196016, + 0.004994355, + 0.05129052, + -0.07573883, + -0.019806353, + -0.0052977004, + -0.02585432, + -0.008690205, + 0.0102756955, + -0.0064524813, + 0.02608897, + 0.014926429, + -0.0153515255, + 0.014566396, + 0.027495226, + -0.05981297, + 0.013260812, + -0.017695569, + -0.103219524, + -0.060673494, + -0.015960708, + -0.01404399, + -0.000095332405, + 0.030360384, + 0.0070420117, + -0.0052885753, + 0.0328839, + -0.012404965, + 0.17907244, + 0.026853886, + -0.0055063153, + -0.019140543, + 0.022881936, + -0.0054624556, + -0.020305462, + -0.0040248292, + -0.008616788, + -0.023474207, + -0.039994232, + -0.051367916, + 0.0506136, + -0.010000171, + -0.02576667, + 0.03985849, + -0.0047355406, + 0.02381105, + 0.058388382, + 0.020415135, + -0.007172144, + 0.034434605, + -0.008059745, + 0.009382488, + -0.0031786328, + -0.04825492, + 0.0090839295, + 0.055844855, + 0.01206461, + 0.04200518, + 0.008154144, + 0.00574395, + 0.026797764, + 0.034558475, + -0.06504741, + -0.011473228, + -0.041974258, + 0.008139225, + 0.022806998, + 0.043154757, + -0.011877348, + -0.016530937, + 0.04076686, + 0.018341811, + 0.021837514, + -0.025804339, + 0.007451655, + -0.07720817, + -0.029209508, + 0.03716037, + -0.010401743, + 0.0071976758, + 0.025213012, + -0.044804268, + -0.021039924, + 0.0035612402, + -0.031583317, + -0.015959581, + -0.009585654, + -0.0027469269, + 0.020826988, + 0.0188638, + -0.005792691, + -0.023081234, + 0.0036363788, + -0.0061280904, + -0.0006764033, + 0.019075336, + 0.055916227, + 0.02952431, + 0.049550064, + 0.04268436, + -0.025420789, + 0.087470114, + 0.057473727, + 0.07013039, + 0.012614301, + 0.02380244, + -0.029213306, + 0.008438178, + -0.023142328, + -0.0111113805, + -0.020476148, + -0.009346526, + -0.01193406, + -0.006535818, + -0.025082134, + -0.008752324, + 0.0044953264, + -0.028365167, + 0.016156549, + 0.05029044, + -0.012121522, + -0.008995778, + -0.01904241, + 0.00071007677, + 0.023712004, + -0.03216477, + 0.012888373, + -0.00008797499, + -0.054210324, + -0.03657906, + 0.011120258, + -0.01213075, + -0.023088787, + 0.03602543, + -0.040457137, + -0.02300542, + -0.0150393285, + -0.01354496, + 0.0215167, + 0.05465438, + 0.028458241, + -0.0115915015, + 0.0059376894, + 0.023620417, + -0.057839744, + 0.013643552, + -0.022547174, + 0.017269548, + 0.03127004, + -0.07988349, + -0.05078919, + 0.011205213, + -0.026745658, + 0.0303635, + -0.03320978, + 0.019111408, + 0.023719456, + 0.012111504, + -0.003342989, + -0.0072348914, + 0.0030055698, + 0.0048688916, + 0.049683273, + 0.039333805, + 0.025220495, + 0.031403065, + 0.061865292, + 0.034643058, + 0.0126303295, + 0.001891789, + 0.0073859184, + -0.015541369, + -0.05020341, + -0.027491327, + 0.0065335324, + -0.00034843318, + -0.026190557, + 0.021528963, + -0.013683739, + 0.04296777, + -0.0032485006, + -0.014733479, + -0.011558493, + -0.0145478975, + 0.0024043447, + -0.029473959, + -0.0077641266, + 0.0071720267, + 0.017033016, + -0.021381047, + -0.018005215, + -0.022859164, + 0.0021507316, + 0.038735278, + -0.043056272, + 0.018944927, + 0.036730997, + -0.031711336, + -0.02730031, + 0.00054977584, + -0.01186276, + 0.051118262, + 0.014460639, + 0.028466199, + -0.021185437, + -0.06046371, + 0.02031086, + 0.031098083, + 0.02705517, + 0.00861357, + -0.023073023, + 0.040332895, + 0.0015463877, + 0.0057385997, + -0.047136843, + -0.020717619, + 0.035119373, + 0.019301428, + -0.011939856, + 0.06972406, + -0.06393006, + 0.021002274, + 0.01652762, + -0.0046303887, + -0.06746979, + -0.05053881, + 0.027266838, + -0.05120607, + 0.011592364, + -0.0007830686, + 0.026465403, + -0.061663467, + 0.076975554, + -0.025506932, + -0.021895196, + -0.019970143, + -0.007931396, + -0.030816864, + 0.01556704, + -0.012785671, + -0.010793555, + -0.0040854607, + 0.02405227, + -0.0097369645, + 0.003523927, + 0.061076246, + 0.0022672077, + -0.055630583, + 0.014753529, + 0.018235153, + -0.023194518, + -0.0654739, + 0.01683278, + -0.010466496, + -0.044046562, + -0.056332473, + 0.011695644, + 0.06960929, + 0.02427439, + -0.0040806774, + -0.03696452, + -0.029279653, + -0.014468639, + -0.0038482912, + 0.01170978, + -0.00013206947, + -0.0035896276, + -0.010911566, + 0.008625817, + -0.0061318427, + 0.011671429, + 0.049992807, + 0.002745769, + 0.0031697694, + -0.012702845, + 0.046768826, + 0.050013237, + -0.03934394, + -0.008645382, + 0.056805123, + -0.0068112477, + -0.02184254, + -0.039267957, + -0.015069269, + 0.00784059, + -0.010700507, + 0.029770492, + -0.0030366762, + 0.024371767, + 0.043727934, + -0.009476817, + -0.014411585, + 0.031865187, + 0.021500316, + -0.027218673, + -0.0048276596, + -0.009022345, + 0.008899309, + -0.023732318, + -0.04855289, + -0.032706082, + -0.08435636, + 0.0030194477, + 0.024007564, + -0.044421624, + 0.0057766563, + 0.04744126, + 0.020762306, + -0.016310865, + -0.0020700977, + -0.025132567, + 0.025563918, + -0.016010238, + -0.008089007, + 0.0058249095, + -0.038526654, + -0.01771741, + -0.025688788, + 0.015047329, + -0.0014794411, + -0.014519879, + -0.044879995, + -0.07035461, + 0.026975842, + -0.050283216, + 0.019084351, + 0.00018584942, + -0.018075237, + -0.032995265, + 0.051688373, + -0.009015666, + -0.022133581, + -0.01399461, + -0.03630475, + -0.008931077, + -0.019135477, + 0.042051293, + 0.055161092, + 0.020866865, + 0.030594286, + -0.018767308, + 0.023476098, + -0.005624278, + 0.0104467785, + -0.0336711, + -0.0070034605, + -0.023060434, + 0.007979617, + 0.016134398, + 0.055457216, + -0.036615178, + 0.0092480825, + -0.008378172, + -0.037848447, + -0.0020533525, + 0.013536509, + -0.025558382, + -0.034805316, + -0.03309949, + -0.03766695, + -0.038801335, + 0.01504471, + 0.04798496, + 0.03310903, + 0.012344884, + 0.010643564, + 0.02714804, + 0.011844159, + -0.017942522, + 0.035539888, + 0.031651203, + -0.042367086, + 0.03250244, + 0.019743834, + 0.0084450785, + 0.011004879, + 0.052851435, + -0.021892207, + 0.0071975337, + 0.013812723, + 0.0025931627, + -0.021047847, + -0.021003025, + 0.029310383, + -0.038826242, + 0.080266275, + -0.0007394495, + -0.004599728, + 0.035691377, + 0.007618633, + 0.0002773464, + 0.040907677, + -0.00071150775, + 0.02910722, + 0.026443645, + -0.007555279, + 0.039212402, + -0.054407403, + -0.03506119, + -0.07286829, + 0.022884298, + 0.02873324, + -0.014125804, + -0.013982294, + 0.004962992, + 0.030923832, + -0.01460802, + 0.006872219, + 0.0032361657, + 0.002350869, + -0.005383757, + -0.053609997, + 0.042491283, + -0.039554384, + -0.030929564, + -0.000628169, + -0.00087879057, + 0.040924307, + -0.013975817, + 0.041287072, + -0.032564737, + 0.01377594, + -0.099039905, + 0.0052515194, + 0.003380827, + -0.015652759, + -0.061392162, + -0.024736412, + -0.028811218, + 0.0068211644, + -0.00080923695, + -0.0145119205, + -0.006284979, + 0.023963012, + 0.030818338, + -0.019357484, + -0.026111864, + -0.018230068, + -0.00141907, + -0.03421442, + -0.014301486, + 0.04973083, + 0.016147241, + 0.0034317733, + 0.03079936, + -0.01633139, + 0.010416137, + 0.015177892, + -0.014367937, + 0.012394066, + -0.04084319, + 0.003077322, + -0.04215946, + 0.06567015, + 0.006222198, + 0.026923848, + 0.0003813074, + 0.021874087, + 0.029589854, + -0.021420741, + 0.032787703, + -0.03356351, + 0.02138357, + 0.027757693, + -0.014662574, + -0.013165381, + -0.030943656, + 0.023958629, + -0.016045582, + 0.029165346, + -0.03166436, + 0.0055215787, + 0.020196468, + 0.037619952, + 0.012425788, + -0.015718715, + -0.018558526, + -0.062954664, + -0.014405638, + -0.02019691, + 0.013439697, + -0.00090975413, + -0.03960515, + 0.01929136, + 0.014825985, + -0.011897969, + -0.07391725, + -0.0037334028, + -0.021778373, + -0.03345438, + -0.0048873485, + 0.03748898, + -0.006887464, + 0.033263568, + 0.026937481, + -0.026849639, + 0.015706737, + 0.004499359, + 0.026059866, + -0.030047212, + -0.035322253, + -0.013030768, + -0.09660178, + -0.020400189, + 0.014046086, + -0.040100582, + 0.023280697, + 0.0061984207, + -0.017089635, + 0.07199057, + -0.005007973, + 0.032175943, + -0.07026293, + 0.008396743, + 0.036889493, + -0.030547267, + -0.0665451, + 0.023350557, + -0.017915163, + -0.006056193, + -0.037609342, + -0.00739387, + -0.021556864, + 0.04219483, + -0.017056148, + 0.02643247, + -0.022912811, + 0.0022669423, + 0.022746345, + 0.0035540983, + 0.030907692, + 0.02280229, + -0.013390923, + 0.00013597019, + 0.023870043, + -0.022743633, + -0.0025300398, + 0.024396071, + 0.032017436, + -0.059623055, + -0.014403772, + 0.014883795, + 0.006013073, + -0.007427275, + -0.02678559, + -0.017337607, + -0.025133366, + -0.040649172, + -0.007375268, + 0.0024157013, + 0.051238976, + -0.011819185, + 0.02866894, + -0.0661211, + 0.0072578704, + 0.03772343, + 0.028456915, + -0.022677233, + -0.004822275, + 0.051148444, + -0.0075521176, + -0.0278938, + -0.0017976505, + 0.035391975, + -0.043084104, + 0.013498232, + 0.000140535, + 0.0081637185, + -0.014895366, + -0.01532493, + -0.023869285, + 0.03975661, + 0.029805588, + -0.0016607255, + -0.0066284174, + -0.047425393, + 0.06078541, + 0.009874399, + -0.01420618, + 0.012851728, + -0.056120105, + 0.056595568, + -0.0008234414, + -0.019897787, + 0.0016358016, + -0.001700002, + -0.046762414, + 0.009751421, + -0.009791319, + 0.039236393, + 0.006752303, + 0.009253158, + 0.03652968, + 0.033729088, + -0.006883442, + 0.011171952, + 0.015599616, + -0.027064255, + 0.0070749433, + -0.03184461, + 0.035316285, + -0.01576945, + 0.037113056, + 0.0086026555, + 0.05263313, + 0.065941855, + 0.025773102, + 0.041526187, + 0.014641708, + 0.032382395, + -0.012154955, + 0.0010549576, + -0.003009168, + -0.032207463, + -0.016061692, + -0.07052871, + 0.010312381, + 0.055001568, + 0.0047640097, + 0.01343715, + -0.036756456, + 0.06276159, + 0.0071619395, + -0.019194365, + -0.025343249, + -0.014627754, + 0.0021465474, + 0.01780317, + -0.015829839, + -0.058133774, + -0.030980356, + 0.0140177235, + -0.05612922, + 0.019690728, + -0.05656899, + 0.010644363, + 0.017211964, + -0.035414055, + 0.011559114, + -0.036985178, + 0.050859265, + -0.015278783, + -0.024635246, + -0.0018933638, + 0.0024701187, + -0.008400519, + -0.0036219582, + -0.002365139, + 0.005002786, + 0.004347757, + -0.0018842067, + -0.009879196, + -0.00094722357, + 0.04200999, + 0.007078176, + 0.023403713, + -0.004958603, + -0.020193184, + -0.03242245, + 0.013797558, + -0.010586125, + -0.04184174, + 0.07585164, + 0.030954061, + 0.004254381, + 0.044182718 + ] + ], + "tools_centroids": [ + [ + -0.05606899, + -0.0037082054, + -0.0406634, + 0.00789242, + -0.019758664, + -0.009364881, + 0.019557, + -0.009831778, + -0.00949407, + 0.007475803, + 0.009658697, + 0.017726261, + 0.005106813, + 0.025193945, + 0.014655598, + -0.018157613, + 0.019189123, + -0.013696539, + 0.013375534, + -0.030809665, + -0.010917298, + -0.017136682, + -0.0133627625, + 0.047115013, + 0.018693825, + 0.041793875, + -0.035732128, + -0.030892752, + 0.03770479, + 0.048461147, + 0.007146342, + -0.018489974, + 0.0021626442, + -0.055255502, + -0.033207137, + -0.03183328, + 0.023630941, + -0.008898392, + -0.06518762, + -0.009473217, + -0.02444353, + 0.009674398, + 0.007890898, + -0.042801153, + 0.021430025, + -0.038007095, + 0.02450929, + -0.040193666, + 0.0003781031, + 0.0039781453, + -0.026794637, + -0.015793024, + 0.019999465, + -0.03834564, + 0.029724747, + 0.051441368, + -0.033698015, + 0.0012862714, + -0.05337915, + -0.000660187, + -0.028856108, + -0.019998565, + -0.00937057, + 0.012524584, + 0.005108214, + 0.091537096, + 0.02503071, + -0.0009231426, + -0.0117032025, + -0.017811712, + -0.044022832, + 0.021416029, + -0.03316437, + -0.03286693, + -0.0865588, + 0.035949722, + 0.043933157, + -0.02726403, + -0.01020989, + -0.0060232165, + 0.050451577, + -0.0009674949, + 0.014217209, + 0.0017161807, + 0.020335244, + 0.052320678, + -0.043524, + 0.03319889, + -0.006663756, + -0.007431745, + -0.028143626, + -0.0012573685, + 0.004024065, + -0.044356935, + -0.051310178, + -0.015289323, + -0.030234639, + 0.007162317, + 0.03509846, + 0.0075056385, + 0.016321585, + 0.002279486, + 0.008136665, + -0.013070722, + 0.047912866, + 0.0038850524, + 0.03659392, + 0.018676, + -0.027126975, + -0.008610971, + 0.003703314, + 0.021906741, + 0.017196443, + 0.0083635105, + -0.022649636, + -0.0427472, + -0.035220575, + -0.049825616, + 0.020608898, + -0.008786186, + 0.05120426, + -0.0076744757, + 0.067492664, + -0.042086206, + -0.019211868, + -0.034554653, + 0.038636506, + 0.054618992, + 0.016760439, + 0.03614989, + 0.0020738828, + 0.035541587, + -0.009483902, + 0.010591545, + -0.03670953, + -0.02300192, + 0.005059944, + 0.0033167151, + -0.008413337, + -0.06675143, + 0.021428753, + 0.057771403, + -0.036993712, + -0.038098555, + 0.03635651, + -0.05102609, + 0.023876203, + 0.004495377, + 0.009017165, + -0.004652149, + -0.018913463, + 0.0013396946, + 0.010036981, + 0.00037534232, + 0.000408126, + -0.013845552, + 0.007976019, + 0.05612355, + -0.022099286, + 0.06489436, + -0.0016591164, + 0.010839845, + -0.025517747, + -0.018433146, + 0.030932501, + -0.0146434, + 0.0077389157, + 0.0014314591, + -0.0047261086, + -0.01379333, + -0.00672064, + -0.028489156, + 0.015116883, + -0.012347785, + 0.02713658, + 0.02815122, + 0.09493939, + 0.023494598, + 0.011286576, + -0.05920509, + -0.046676517, + 0.020933578, + -0.009062939, + -0.007043584, + -0.034089938, + -0.0074305055, + 0.0435831, + -0.010437014, + 0.0077175335, + 0.013643179, + 0.00032880338, + 0.012419136, + 0.028978484, + 0.0027938702, + 0.033168048, + 0.017302865, + -0.001537013, + -0.023324333, + -0.023725992, + -0.035093762, + -0.024574025, + 0.031602282, + 0.043686762, + 0.0043900036, + -0.039813936, + -0.010593189, + -0.0121115735, + -0.034151588, + 0.009565826, + -0.038808856, + -0.012305442, + -0.00027281337, + 0.028179586, + 0.0022837583, + -0.039286092, + -0.0042402116, + 0.008249791, + 0.010657445, + -0.031379938, + -0.031366322, + -0.023170266, + 0.054060068, + 0.0048521874, + -0.01564455, + 0.01751972, + -0.0055304905, + 0.023394609, + 0.012396702, + -0.0003636368, + -0.022696953, + -0.023217125, + -0.006895921, + -0.008959929, + -0.018518882, + 0.071185775, + -0.0023356022, + -0.014435151, + 0.0037555092, + 0.025933823, + -0.00043771358, + -0.023615904, + -0.021824772, + 0.006545141, + -0.0036427176, + -0.044271503, + 0.021339668, + 0.044955056, + 0.033640694, + -0.017741505, + 0.0050410517, + 0.029788665, + -0.009972994, + -0.044430643, + -0.0060587483, + 0.03381109, + 0.032378227, + 0.029204395, + -0.01285328, + 0.011239541, + 0.011089111, + 0.032791503, + 0.01966721, + 0.01709499, + 0.04467105, + 0.006781909, + -0.0062364032, + -0.022574946, + -0.0069035026, + 0.0080041215, + -0.010154674, + -0.033628866, + 0.02108642, + 0.028107613, + 0.0055993693, + 0.017272362, + 0.0064405464, + -0.024938427, + -0.0274226, + 0.05751032, + 0.023648951, + -0.0069635683, + 0.021213165, + 0.008942416, + 0.0031823711, + 0.0020612366, + 0.0017135177, + -0.03745275, + -0.01224723, + -0.0092840865, + -0.039438073, + -0.020603599, + -0.009839146, + 0.10459197, + -0.0013756665, + -0.015315762, + -0.0004424063, + -0.035673883, + -0.22716926, + -0.011021554, + 0.009104081, + 0.033588458, + 0.007958396, + -0.00556822, + -0.00057971233, + 0.0043771993, + -0.041701302, + 0.024739638, + -0.03301154, + -0.0772926, + -0.029319951, + -0.035250075, + -0.0018914198, + -0.006261339, + -0.0055478225, + -0.006832176, + 0.023415668, + -0.039532892, + -0.020574547, + -0.012577133, + 0.05117778, + -0.0044079097, + -0.014995264, + -0.015555239, + 0.048599504, + 0.009827435, + -0.036604106, + -0.009881144, + 0.005024552, + 0.006581829, + -0.005089212, + 0.030867914, + 0.012545012, + 0.03213094, + 0.009552479, + -0.011707572, + -0.015692463, + -0.008529624, + 0.019617837, + 0.042746853, + 0.0017175317, + 0.037256774, + 0.041178584, + -0.03710732, + 0.012643423, + -0.028216833, + -0.016226137, + -0.008176746, + -0.0021341608, + -0.004784728, + 0.0022153086, + -0.027044365, + -0.0452304, + -0.00092170114, + -0.0344755, + 0.039659843, + -0.03179438, + -0.0008682657, + -0.030181633, + -0.0069265706, + 0.050441854, + 0.017631147, + -0.024619931, + 0.020767754, + 0.016655464, + -0.01860972, + 0.038021646, + -0.023390938, + 0.031038294, + 0.0038111766, + 0.0005345677, + -0.026021777, + -0.007115485, + 0.048046198, + -0.031886622, + -0.035144318, + -0.0026188153, + -0.18630543, + -0.032157507, + 0.018465653, + 0.0007996492, + 0.015417364, + -0.034394316, + -0.0075861984, + -0.0037427046, + 0.020296175, + 0.048781425, + 0.3208829, + 0.017051741, + -0.019397799, + 0.009009308, + 0.053919844, + -0.016741762, + 0.006250598, + 0.035504892, + -0.0060608485, + -0.032443427, + -0.00053168356, + 0.0076522306, + 0.025676573, + -0.0021505675, + -0.007423398, + 0.021114204, + -0.031733297, + -0.0011500829, + 0.100127555, + -0.004869494, + 0.029603617, + -0.0041950424, + 0.006506095, + 0.023717992, + -0.052787792, + -0.040853288, + -0.011165942, + 0.056000806, + -0.025902374, + 0.045923546, + -0.014623309, + 0.018445712, + 0.025848486, + 0.0029045863, + -0.053331148, + -0.018678037, + 0.011694079, + -0.013297034, + -0.0096364105, + 0.032539476, + -0.029562913, + -0.047789693, + 0.0026963938, + 0.025598904, + 0.006072212, + -0.03891818, + 0.017167594, + -0.0036721022, + -0.028613243, + -0.018510707, + -0.034060024, + -0.027170384, + -0.022034682, + 0.0024952486, + -0.0015357194, + -0.03533966, + -0.020739295, + -0.03143169, + -0.011211882, + 0.049816895, + 0.004694738, + 0.010594703, + -0.06420896, + 0.029159075, + -0.0025625115, + 0.041708685, + -0.030052194, + -0.014590074, + 0.012761965, + 0.014360401, + 0.013680634, + -0.010967563, + 0.005759606, + -0.011701792, + 0.05404799, + 0.019268606, + -0.021743063, + 0.06309377, + -0.019261587, + 0.048059467, + -0.024920318, + 0.010070914, + -0.047579683, + 0.026595078, + 0.02419197, + -0.018060323, + -0.014325064, + 0.026749337, + -0.02670927, + -0.03856533, + -0.008656995, + -0.0010423756, + -0.027002128, + 0.01249798, + 0.01479696, + 0.003004412, + 0.013617141, + -0.004383087, + -0.038900018, + -0.016454257, + -0.03198517, + -0.017905517, + -0.006894297, + -0.02221281, + 0.018678682, + -0.0025110545, + 0.00062055915, + 0.011711673, + 0.029655354, + -0.0033267613, + -0.030383127, + 0.033008993, + -0.011483914, + -0.039945323, + 0.032434426, + 0.016122635, + 0.0034836135, + 0.021274103, + 0.02966441, + 0.036415953, + -0.0307599, + -0.013897553, + -0.0090347715, + 0.008639756, + -0.027077066, + -0.031259153, + 0.00041659162, + 0.008344521, + -0.055489786, + 0.043384776, + 0.016839493, + -0.019988367, + 0.017604688, + 0.010117987, + 0.011109807, + 0.004411769, + 0.0075515187, + 0.024973173, + -0.006590006, + 0.059543025, + -0.011032581, + -0.05263824, + -0.023044582, + 0.010140496, + -0.0036169202, + -0.0006291355, + 0.04357636, + -0.016806297, + -0.021199094, + 0.010797508, + 0.0037002466, + 0.048935954, + 0.025420306, + -0.068292305, + -0.014495878, + -0.033696596, + -0.016170561, + 0.010531388, + -0.014046231, + -0.038196664, + 0.014447582, + -0.0023581628, + 0.024200277, + 0.07188329, + 0.026538994, + 0.010875151, + -0.04558475, + 0.026683357, + 0.025531683, + -0.033252317, + -0.0019728958, + -0.033328198, + -0.061308175, + 0.04225168, + 0.0017407219, + -0.018591579, + -0.043852597, + -0.0018788553, + -0.016940724, + 0.0065283165, + 0.04805281, + 0.014803647, + -0.044855554, + 0.014711851, + 0.014296282, + 0.03147689, + -0.0071145403, + -0.016696168, + 0.014320578, + -0.024143968, + -0.032781575, + 0.11330352, + 0.014250864, + 0.0010048777, + 0.010561461, + 0.012848797, + 0.009197347, + 0.027354443, + -0.009164351, + -0.017722908, + 0.001937092, + -0.007415777, + 0.003458327, + -0.040217895, + 0.005049932, + -0.013077656, + -0.013231001, + 0.015766662, + -0.009500496, + -0.016540406, + 0.0049814247, + -0.014900655, + -0.002444683, + -0.002461634, + 0.044309385, + 0.0012398985, + 0.04664923, + 0.05246022, + -0.010509221, + -0.015866991, + -0.01696081, + -0.003949499, + 0.053042635, + -0.06599373, + -0.007691855, + 0.017483734, + -0.012901056, + -0.046527628, + 0.018579615, + 0.012297996, + 0.013650844, + 0.003290136, + 0.0076800594, + -0.033333212, + 0.03522728, + -0.028282017, + 0.0096138, + 0.040518165, + 0.00096373906, + -0.031569775, + 0.024419462, + -0.01790975, + -0.0033231927, + 0.039167337, + -0.006741734, + 0.056704193, + -0.02368793, + 0.012585808, + 0.030043406, + 0.034530554, + -0.013871013, + 0.0013567432, + 0.0069410847, + -0.013012229, + -0.01794545, + 0.03237499, + -0.006095536, + -0.0015028969, + -0.022055492, + -0.0075068753, + -0.020075902, + 0.026405957, + -0.042370167, + -0.002402599, + -0.027732788, + -0.025120504, + 0.035065226, + -0.029773757, + -0.02554472, + -0.06785252, + -0.0140207475, + -0.009677745, + -0.0526383, + -0.011321278, + 0.038069192, + 0.015819497, + -0.035158947, + 0.00539325, + 0.020147596, + 0.026790883, + -0.026228523, + 0.011281811, + 0.0004322955, + 0.03357598, + -0.0312409, + 0.02051756, + 0.008903391, + -0.02679297, + 0.0059814607, + -0.039614767, + 0.02225511, + 0.0032299664, + -0.039062347, + 0.034572784, + -0.04000374, + 0.013121305, + -0.006740711, + 0.015605714, + -0.024614284, + 0.0182156, + -0.02464988, + 0.022435531, + -0.0540118, + -0.0107686315, + -0.01670897, + 0.015968842, + -0.025021473, + 0.004982866, + 0.044434078, + 0.0444515, + 0.03769919, + -0.026614936, + -0.0057781814, + 0.04286719, + 0.012986099, + -0.00009658893, + 0.057660222, + 0.002628925, + -0.0145921465, + 0.0028970172, + -0.00004345907, + 0.00540906, + 0.000009923756, + -0.01587657, + -0.013682616, + -0.0055716606, + -0.015776113, + 0.0040675635, + -0.02399378, + -0.024579993, + -0.02416092, + -0.021938631, + -0.0018802591, + -0.019586174, + 0.006322869, + -0.0037580882, + 0.011118912, + -0.00688259, + 0.046543375, + 0.0020257356, + 0.00915174, + -0.012346758, + 0.030707782, + -0.029368373, + 0.01678603, + -0.020684225, + 0.0025972757, + -0.012388426, + -0.008840013, + 0.018587714, + 0.009576551, + 0.029996252, + -0.0339124, + -0.069902465, + 0.004762207, + -0.005107531, + -0.04803004, + 0.007623275, + 0.024682391, + 0.011488783, + -0.019897707, + 0.038231988, + -0.004208973, + 0.01619736, + 0.02647576, + 0.011420114, + 0.016564004, + 0.010380385, + 0.015293031, + -0.015646528, + -0.00861725, + -0.011084581, + -0.010725576, + 0.0021767458, + -0.01672313, + 0.03839429, + -0.013438732, + -0.013672219, + 0.03522901, + 0.0024493937, + -0.009887151, + -0.03008308, + -0.026761537, + -0.020664144, + 0.022931337, + -0.008237708, + -0.0072052367, + -0.015670665, + 0.040643997, + 0.026935095, + -0.036957294, + -0.010053467, + -0.04384358, + 0.022895643, + -0.2105204, + 0.022694126, + 0.00997941, + -0.013189658, + -0.056611195, + 0.008618698, + -0.012578096, + -0.019372476, + 0.0086058965, + -0.037407827, + -0.042723864, + 0.008701772, + 0.025598338, + -0.029506119, + 0.018346047, + 0.024427475, + 0.013028314, + 0.011439589, + 0.0073115127, + 0.031192467, + -0.015018622, + -0.031565055, + 0.0016912352, + 0.025967559, + 0.0075193015, + -0.010706298, + -0.007327034, + 0.005763416, + -0.04706705, + -0.004623551, + 0.014159363, + -0.019997144, + 0.03497208, + 0.06485894, + -0.026932962, + -0.010051237, + 0.013179083, + -0.029087199, + -0.022257665, + -0.015966833, + -0.012313927, + 0.034779385, + -0.01645768, + -0.011849744, + 0.016278874, + 0.03483635, + -0.0043797335, + 0.0011064508, + -0.05597814, + 0.013468208, + -0.0057655373, + 0.04608973, + -0.031591564, + 0.00081813696, + 0.02271715, + -0.016193483, + -0.048024204, + 0.024269635, + -0.010684908, + 0.03037921, + -0.015080028, + 0.00577765, + -0.035579298, + -0.074489966, + -0.04568095, + -0.032743674, + -0.08463265, + 0.012373603, + 0.004400696, + 0.029980667, + -0.017966386, + -0.038213238, + 0.0057509523, + -0.025879731, + 0.017900858, + -0.019159615, + 0.013998882, + -0.0017255883, + -0.027130362, + -0.02789793, + -0.033315383, + 0.013056893, + -0.056488942, + 0.0058665094, + 0.060071133, + -0.009120229, + -0.003342009, + 0.0047433944, + -0.018858049, + 0.007472259, + -0.052950244, + -0.0039435257, + 0.00009774961, + 0.02106769, + -0.01170398, + 0.014035189, + -0.03197733, + -0.0038070253, + -0.014319388, + 0.0053357556, + -0.017116, + -0.0016028484, + 0.0019584813, + 0.015731098, + -0.014593648, + -0.009505084, + 0.06416616, + -0.022955375, + 0.016944945, + -0.015723076, + -0.030711906, + -0.0011726163, + -0.014695939, + 0.0066800653, + -0.046726193, + -0.0039578825, + 0.028575825, + -0.019372286, + -0.029086906, + -0.017561322, + -0.00428551, + 0.014641752, + 0.007259805, + -0.04302052, + 0.0039681853, + -0.00029460277, + 0.038807478, + 0.006130545, + -0.0048487694, + -0.018746726, + 0.039638285, + -0.03210769, + -0.012196666, + 0.029514477, + 0.030580895, + -0.011591039, + -0.013160387, + 0.066982895, + -0.019542642, + -0.053750712, + -0.023531597, + 0.0046885437, + 0.0008201537, + -0.019609794, + -0.00034986206, + -0.014274548, + -0.0038191855, + -0.013088327, + -0.042322893, + -0.029365066, + 0.051891316, + -0.0026843995, + 0.017064247, + 0.016581565, + 0.0013990946, + 0.0052255765, + -0.015575843, + 0.02221455, + -0.010577843, + 0.05188453, + 0.027795833, + -0.034560814, + -0.016681876, + -0.028297098, + 0.010590897, + 0.019287696, + 0.00036446203, + -0.01720634, + -0.037081674, + 0.0033696124, + -0.0013072222, + 0.030434122, + -0.03360222, + 0.055518307, + 0.023288932, + -0.04746223, + 0.041459426, + 0.014260388, + 0.06371192, + -0.0015586383, + 0.023453446, + -0.013434582, + 0.01792418, + 0.00756145, + 0.012596964, + 0.017840264, + 0.026810404, + 0.033809863, + 0.01566813, + 0.018697273, + 0.018435918, + -0.0015165473, + 0.0023415266, + -0.018216006, + 0.036613736, + 0.0055582654, + 0.01461495, + 0.022472767, + -0.025861593, + 0.06232265, + -0.021595163, + -0.03394643, + -0.03501842, + -0.025740786, + -0.028258167, + -0.011435451, + -0.026583094, + -0.028726535, + -0.033326574, + 0.04048446, + -0.028059425, + 0.006273973, + -0.024928257, + 0.0059108674, + -0.005911225, + 0.020267177, + 0.024992893, + 0.029467724, + 0.009110243, + 0.010474011, + 0.005127083, + 0.025516486, + 0.01224543, + -0.04925422, + -0.016988033, + -0.030158473, + -0.00012626215, + -0.0070728445, + 0.00432526, + -0.009610042, + -0.024404833, + 0.009735499, + -0.04270004, + 0.03474741, + 0.025382044, + -0.002434115, + 0.035717692, + 0.034531046, + -0.020223275, + -0.024045918, + 0.03205907, + -0.01690001, + 0.008418649, + 0.016847905 + ], + [ + -0.0777926, + -0.016130753, + -0.030252252, + 0.013064685, + 0.012630054, + -0.033431668, + -0.020723045, + -0.010652609, + 0.014717416, + 0.029943759, + -0.006884425, + -0.050565906, + -0.020473206, + 0.028232329, + -0.008742489, + -0.006928586, + -0.01848413, + -0.0062769433, + 0.0249494, + -0.026330234, + -0.044125106, + -0.027761683, + -0.047089003, + -0.018802904, + -0.0031594674, + 0.06795997, + 0.011664632, + 0.0049165445, + -0.008060829, + 0.012408625, + -0.0238953, + -0.02757731, + -0.026854679, + 0.0010435038, + -0.020860666, + 0.017221682, + 0.014673638, + 0.029738, + -0.011689957, + -0.015324875, + -0.010424466, + -0.028959932, + -0.0026987449, + -0.06396346, + -0.0044665886, + -0.051129714, + 0.008419665, + -0.016844666, + 0.03334108, + 0.008317175, + -0.015411966, + 0.007842717, + 0.061229803, + -0.05119574, + -0.013707167, + 0.012232052, + 0.012225783, + 0.045153536, + -0.0008650962, + -0.016156076, + -0.046021163, + -0.033580557, + -0.01686916, + -0.0030913, + 0.033622023, + 0.01923576, + -0.022195347, + -0.010797742, + -0.029916009, + -0.0004705949, + -0.039619286, + -0.013173107, + 0.06153379, + 0.006591758, + -0.058220953, + -0.04044149, + 0.034537926, + -0.016714765, + -0.0033219494, + 0.016891837, + 0.028455667, + -0.03353745, + -0.0025109896, + 0.0013168402, + 0.02191537, + 0.00989691, + -0.0076760734, + 0.053305842, + -0.011090245, + 0.05804909, + -0.0053123464, + -0.009720489, + 0.033220656, + -0.048008375, + -0.07702685, + 0.004185979, + 0.019034842, + 0.005118874, + 0.016340083, + 0.006550299, + 0.0013390492, + -0.0079578, + -0.017740352, + -0.029226499, + 0.03548014, + -0.030715493, + 0.035450455, + 0.002739102, + 0.008236122, + 0.0070569282, + 0.0021077737, + 0.010215997, + 0.0006403754, + 0.020998295, + -0.022850262, + -0.039860018, + -0.022919796, + 0.003358362, + -0.015362032, + -0.022922192, + 0.034358162, + -0.053381287, + 0.03479484, + -0.014682795, + 0.046078928, + -0.05120507, + -0.033427823, + 0.018008124, + -0.029796828, + 0.03957431, + -0.009962059, + 0.0014587059, + 0.035613198, + 0.023544552, + -0.0035347848, + -0.021168783, + 0.02405471, + 0.022278832, + -0.0129754245, + -0.04674764, + -0.035395015, + 0.0060356203, + 0.009195937, + -0.012415938, + 0.041733548, + -0.0042112567, + -0.0054666842, + -0.022617701, + 0.024861094, + -0.0014649403, + -0.021593269, + 0.010473995, + -0.006896913, + 0.012945232, + 0.038368367, + -0.021349257, + -0.023133717, + 0.03926201, + -0.006479839, + 0.108185984, + -0.023711825, + 0.035648413, + -0.020544553, + 0.028756576, + -0.012647061, + -0.0068361103, + -0.009214789, + -0.01819023, + -0.024630189, + -0.0158221, + -0.0076070474, + 0.016997859, + 0.028996132, + 0.0028848278, + 0.012588545, + 0.012869404, + 0.022341216, + 0.043853626, + -0.03490951, + -0.04219079, + -0.036706578, + 0.006829303, + 0.029833514, + -0.04073565, + -0.040228482, + 0.040827464, + 0.018223649, + 0.0068636555, + 0.023727717, + 0.013143164, + -0.024592938, + -0.0046767895, + 0.0073383558, + 0.065363534, + -0.054043908, + 0.010578177, + -0.0060408995, + 0.022571096, + -0.022589117, + -0.011779923, + 0.02198382, + 0.046569698, + 0.030388758, + 0.00009530159, + -0.025261797, + -0.004875701, + 0.016566563, + -0.024309523, + -0.04492145, + -0.040620018, + -0.008033184, + 0.008486595, + 0.0036873387, + -0.006312519, + -0.057184577, + -0.027954899, + -0.026917579, + -0.023880664, + -0.012670563, + -0.022808496, + 0.017791497, + 0.0046672984, + -0.021858089, + 0.017631855, + 0.023283772, + 0.025225436, + 0.03048242, + 0.008289526, + -0.027709194, + 0.022788085, + -0.037033916, + -0.019281274, + 0.042843334, + -0.020052629, + 0.062315274, + 0.017621454, + 0.0023051042, + -0.012937805, + 0.03945034, + 0.01578884, + -0.04974779, + -0.05110533, + 0.040397394, + -0.02760902, + -0.029570848, + -0.028894164, + -0.023790289, + 0.020278046, + -0.021137215, + -0.009141063, + 0.005414481, + -0.019659275, + 0.010410835, + 0.03108242, + 0.019945964, + 0.019414624, + 0.001831312, + -0.0044669346, + 0.051963087, + -0.0041192234, + 0.061908115, + 0.033783134, + -0.029776793, + 0.014804269, + 0.021261247, + -0.0008398359, + -0.062825724, + 0.008026716, + 0.0017601701, + 0.016118582, + 0.0132596325, + 0.026431667, + 0.04182074, + 0.05865137, + 0.020127004, + 0.009281466, + -0.0050892252, + -0.049665384, + 0.054135658, + 0.0067237294, + -0.018110242, + 0.034775, + 0.031828076, + 0.014316956, + -0.00709445, + 0.016639711, + 0.056842767, + 0.031007824, + 0.008321629, + -0.051327795, + -0.03239476, + 0.052417755, + 0.079580046, + -0.0044072163, + -0.025062976, + -0.034690555, + 0.0017639417, + -0.12477049, + 0.0010641285, + 0.037374467, + -0.0078232195, + -0.021236846, + -0.008217301, + -0.024288183, + 0.03848377, + -0.036963657, + -0.029888399, + -0.06894421, + -0.059106227, + -0.03341434, + -0.02493872, + 0.027394043, + 0.022943368, + -0.06030709, + -0.024837509, + -0.014572309, + -0.035931725, + 0.017025314, + -0.038330782, + 0.029428963, + -0.031423945, + 0.014545969, + -0.020500131, + 0.013090049, + -0.015320569, + -0.067829646, + -0.032127842, + -0.03852837, + 0.01132043, + 0.016142247, + 0.04502109, + 0.011946692, + 0.036663942, + 0.004616084, + -0.0040293597, + 0.008882638, + -0.0050679436, + 0.030989021, + -0.011439283, + -0.027428659, + 0.010920439, + 0.010191991, + 0.025236871, + 0.0374206, + -0.0040066508, + -0.025254825, + 0.009245347, + -0.00990044, + 0.012132254, + 0.0039520827, + 0.0044951704, + 0.00659937, + -0.0025982815, + -0.01390185, + 0.025864433, + -0.007912151, + -0.038404647, + -0.014624314, + 0.05410332, + -0.012991158, + 0.017253352, + 0.015497101, + 0.0034109775, + 0.015372937, + -0.033209722, + 0.036295604, + 0.014939996, + -0.02805046, + 0.0015722377, + 0.017167116, + -0.00017856316, + -0.045900483, + 0.0077081304, + 0.074832946, + -0.026312457, + -0.0005495474, + -0.13214304, + 0.049361903, + 0.02400141, + -0.0055426657, + 0.015336194, + -0.02816012, + 0.0046836925, + 0.0099547, + 0.03742822, + 0.011811279, + 0.19601296, + 0.033754703, + -0.010378031, + 0.001250739, + 0.008726979, + 0.018838167, + 0.043022648, + 0.0697224, + -0.010394221, + -0.030004341, + -0.04530521, + -0.046880346, + 0.018401384, + -0.01957355, + 0.041167054, + 0.058837432, + 0.015511397, + 0.0063158153, + 0.11830529, + 0.023792224, + 0.058994506, + 0.00018409223, + 0.034306847, + 0.058044955, + -0.020177843, + -0.048148416, + 0.0122626135, + 0.034675147, + -0.03752977, + 0.029120939, + -0.010040249, + 0.04624406, + -0.0073817093, + -0.011214725, + -0.04331988, + 0.0051043136, + 0.01554897, + -0.012071543, + 0.02121115, + -0.006823022, + -0.011237937, + -0.0035037443, + -0.019238824, + 0.018333014, + -0.026355784, + -0.054825306, + -0.009566956, + 0.0024437692, + -0.009667523, + -0.013058095, + -0.034837723, + -0.020705493, + -0.048979156, + 0.021577716, + -0.0041844496, + -0.0036711793, + -0.05854377, + -0.04395329, + -0.026583686, + 0.03830527, + 0.031828027, + -0.005516493, + -0.07229121, + 0.023722429, + -0.009492063, + 0.03496614, + -0.024903411, + 0.0026213636, + 0.015441066, + 0.028241813, + 0.0037205473, + 0.017424388, + -0.023927208, + 0.016457984, + 0.02520875, + 0.040443234, + 0.010766083, + 0.017123722, + -0.04108963, + -0.033335038, + 0.0030168097, + 0.0053470326, + -0.018452037, + 0.03997777, + 0.007828228, + 0.016234277, + 0.020400109, + 0.048260894, + -0.028451897, + -0.049423486, + 0.014191736, + -0.01758265, + -0.009043689, + 0.02531204, + -0.0062299054, + 0.01920355, + 0.03200232, + 0.017771985, + -0.013553965, + -0.043947473, + -0.015971966, + -0.03285715, + 0.00947157, + -0.042635962, + 0.057298675, + 0.060869828, + -0.014281702, + -0.015127428, + 0.040840045, + -0.03678748, + 0.009928272, + 0.013789576, + 0.016361559, + -0.033839006, + 0.052730188, + 0.03176229, + 0.04237319, + 0.014982663, + 0.017706575, + 0.04312265, + -0.025634803, + -0.02308356, + -0.019002479, + 0.036787376, + 0.031167256, + 0.006498465, + 0.012102121, + 0.020858966, + -0.024361406, + 0.02639614, + 0.057980295, + 0.0049070157, + -0.029224023, + 0.022465296, + 0.025334602, + 0.060530398, + 0.015762227, + 0.0035243605, + 0.034754157, + 0.027435217, + 0.060840853, + 0.01591025, + 0.009018213, + 0.00035580425, + -0.033563286, + 0.010515216, + 0.0069815437, + -0.047450293, + -0.011460179, + 0.015469139, + -0.004947935, + 0.044788565, + -0.028610881, + -0.03679307, + -0.026844844, + -0.024473835, + -0.013097169, + 0.05502105, + -0.0030086858, + 0.0054221293, + 0.00934873, + -0.05726428, + 0.028941594, + 0.021284036, + -0.018056447, + 0.076361485, + -0.028323943, + 0.010804032, + 0.03501848, + -0.041234206, + -0.010082845, + -0.023559377, + -0.047871493, + 0.043036506, + -0.0077531943, + -0.061463587, + -0.02967603, + 0.027839405, + -0.0020366006, + 0.031356577, + 0.009509519, + -0.017622339, + 0.0026792034, + -0.031537317, + 0.02021114, + 0.0575173, + -0.01732471, + 0.0070268577, + 0.0016957909, + -0.030016575, + -0.01718395, + 0.07615369, + -0.0123706665, + -0.0047786008, + -0.0060987207, + -0.0013207493, + 0.010539184, + 0.009274312, + 0.0007885846, + -0.016644027, + 0.038292266, + 0.01448823, + 0.024252761, + -0.057947274, + -0.033873387, + 0.019700998, + -0.040111557, + -0.021552257, + -0.057441693, + -0.06635977, + -0.025710544, + -0.053925313, + -0.0030804146, + -0.016332027, + 0.0361699, + 0.0059258826, + 0.042275347, + 0.024262903, + -0.011202689, + -0.03925866, + -0.026211502, + -0.018307885, + 0.04379796, + -0.076288186, + 0.032032076, + 0.007867089, + 0.0054110987, + -0.038828284, + 0.04143126, + -0.01417168, + 0.008428398, + -0.0007231599, + 0.01121787, + 0.00823, + 0.019377021, + -0.03386666, + -0.007837995, + 0.0138864545, + 0.034504473, + -0.014824415, + -0.029037168, + 0.0040237717, + 0.013956105, + -0.03657621, + -0.0029222432, + -0.0029531356, + -0.012477216, + 0.0034438937, + 0.028417397, + -0.0042160964, + -0.04023129, + -0.007177788, + -0.012188858, + -0.010640365, + -0.025378073, + 0.023323987, + 0.04131474, + -0.020508341, + -0.017246038, + -0.009083519, + 0.011987527, + 0.010934865, + -0.020992914, + -0.019448092, + 0.0035046816, + -0.0036248225, + 0.029120086, + -0.02020849, + 0.0108864745, + 0.004756365, + -0.008310412, + 0.00082247274, + -0.022503192, + -0.06487264, + 0.004785375, + -0.033616617, + -0.07876667, + 0.037286237, + 0.064570576, + -0.02501201, + 0.00021179773, + 0.019808415, + 0.01730107, + 0.0028576683, + -0.022853047, + 0.096394695, + -0.009005143, + 0.021504054, + -0.016715469, + -0.103030674, + -0.00034150484, + 0.027667, + -0.033947464, + 0.015386551, + 0.0004739991, + 0.025440171, + 0.03065169, + -0.015094892, + -0.024132984, + 0.040302265, + 0.0022036897, + 0.027904361, + -0.04102699, + 0.029432945, + -0.038208827, + -0.0062153265, + -0.011998047, + -0.013344417, + 0.033526026, + 0.021145038, + 0.0027201674, + -0.019569896, + -0.031845108, + -0.04933251, + 0.05314293, + -0.0025423346, + 0.06681798, + 0.037334353, + -0.0072135865, + 0.0014750219, + -0.008131382, + 0.0033401614, + -0.033794142, + 0.027915679, + -0.020438343, + 0.03567195, + -0.04277935, + -0.01002426, + 0.028667305, + -0.03083799, + -0.0533229, + 0.012505484, + -0.0056473175, + 0.0070565026, + 0.021628987, + -0.0054352866, + -0.039229307, + -0.0135678705, + -0.028667327, + 0.022404738, + 0.0067239925, + 0.002311097, + 0.06369203, + -0.02629535, + 0.040622883, + -0.034154095, + 0.018668704, + 0.011607976, + -0.0021320207, + -0.01759238, + 0.021003881, + 0.006119285, + -0.04615905, + -0.019840589, + -0.0129681, + 0.017631495, + -0.03321065, + 0.034223422, + 0.04391737, + -0.018810853, + -0.028161395, + 0.028423589, + -0.049154766, + 0.054977193, + 0.0000630182, + -0.0173311, + 0.0151262935, + -0.004985415, + -0.004255283, + 0.0029248288, + -0.0005668443, + -0.023171514, + -0.006701821, + -0.03586675, + -0.0022324445, + 0.01843075, + 0.0076619005, + -0.06435156, + -0.016519507, + 0.052956775, + -0.0056365584, + -0.03733352, + -0.039358534, + -0.0435778, + 0.031613506, + 0.03608112, + -0.0070490213, + 0.01944554, + 0.03693605, + -0.010692041, + -0.00599563, + 0.020131715, + -0.09125295, + -0.02595961, + -0.10351341, + 0.02379246, + -0.031193437, + -0.005686541, + -0.07574321, + -0.00818769, + 0.0012698586, + 0.031600002, + 0.029005878, + -0.034200996, + -0.07727333, + 0.03113444, + 0.002058297, + -0.009169671, + -0.02671429, + 0.0059962994, + 0.003603806, + 0.0074202335, + -0.0060870526, + -0.025415512, + -0.0050098686, + -0.018891325, + 0.015137767, + 0.0009658489, + 0.011259749, + -0.00018296832, + -0.054526947, + 0.042278267, + -0.001603754, + 0.055526804, + 0.032714244, + 0.055156004, + 0.03152486, + 0.024948584, + 0.010113138, + 0.026203973, + 0.0036432473, + -0.04414578, + 0.026116638, + -0.017378923, + 0.014874825, + 0.015792275, + -0.040725254, + -0.0027440262, + -0.016308293, + 0.019773996, + -0.032847524, + 0.009167586, + -0.08082315, + 0.008408904, + 0.020688424, + -0.019353844, + -0.028009258, + 0.026694907, + 0.0032782315, + -0.03360142, + -0.023895264, + 0.039238874, + 0.00445677, + 0.02949898, + 0.014402198, + -0.003599358, + -0.040845413, + -0.0350518, + -0.033609767, + 0.024508178, + -0.019472161, + -0.0031140537, + -0.005668693, + 0.03111455, + -0.027201336, + -0.025430666, + -0.035129692, + 0.0105588045, + 0.02917934, + -0.01097523, + -0.022186121, + 0.017050618, + 0.0037318894, + -0.03971323, + -0.0067270542, + -0.012156165, + -0.088368215, + 0.02033153, + 0.040660948, + 0.024675295, + -0.0045001414, + 0.018805915, + -0.00049041864, + 0.04464026, + -0.04455079, + -0.022046903, + 0.041305263, + 0.03293151, + -0.028352423, + -0.02059597, + -0.0019165347, + -0.006466022, + -0.009925339, + -0.008929011, + -0.076839685, + -0.00429755, + -0.014738176, + -0.024139099, + 0.03820742, + 0.003912287, + 0.030968878, + 0.004669395, + 0.025041392, + -0.013336667, + -0.056171685, + -0.023908475, + -0.0026514267, + -0.04345135, + -0.022183042, + -0.014375652, + 0.00066614884, + -0.047541592, + -0.06785615, + -0.06805146, + 0.0052735, + 0.02312607, + -0.039141715, + -0.017192993, + 0.0020956157, + 0.026551597, + -0.016486952, + -0.047485273, + 0.04242504, + -0.024608482, + -0.0063258884, + -0.025091767, + -0.0031197283, + 0.016352389, + 0.032390088, + -0.024658075, + -0.055107787, + 0.041251957, + -0.01631562, + -0.0514147, + -0.023321213, + -0.0093295155, + -0.015928332, + -0.021337397, + -0.011802844, + 0.014432149, + -0.019652303, + 0.0067840177, + -0.05091495, + -0.0055248598, + 0.0048583862, + 0.01215574, + -0.02517826, + -0.0025640433, + -0.0034858014, + 0.0013292104, + 0.034103796, + 0.020756245, + 0.0193323, + 0.040301695, + 0.00078084186, + -0.04896679, + 0.027173512, + -0.005667116, + 0.01948293, + 0.032077633, + 0.0061108964, + -0.018225638, + -0.02208757, + -0.010346406, + -0.03226915, + 0.015659394, + 0.010559285, + 0.0644997, + 0.033056695, + -0.04196774, + 0.0071754702, + -0.0038966993, + 0.020072218, + -0.03101222, + 0.03538792, + -0.008695402, + 0.0067324615, + 0.037320923, + 0.0085444255, + 0.037771806, + 0.01592961, + 0.0071725305, + 0.007101917, + 0.01527844, + 0.0028750405, + -0.016928488, + -0.013771216, + -0.023309482, + 0.030158604, + -0.011056797, + -0.000734273, + 0.039915036, + -0.007078933, + 0.041310612, + 0.03926212, + -0.021225905, + -0.03690358, + 0.00033736796, + -0.046615493, + 0.0035040032, + -0.04736803, + -0.015354346, + 0.0076449066, + 0.0646623, + -0.061303865, + 0.0072781476, + -0.022982756, + 0.04087125, + 0.020003729, + -0.0351412, + 0.0387916, + 0.044128425, + -0.0139995115, + -0.0019715482, + 0.03880407, + 0.04205457, + -0.027607651, + 0.006489501, + 0.002768678, + -0.030172173, + 0.0062397877, + -0.013040987, + -0.013252143, + -0.04237509, + 0.009563442, + 0.057689045, + -0.019734269, + 0.0113543505, + 0.02527864, + 0.0014363149, + 0.06007512, + -0.00003742828, + -0.022210408, + 0.012302623, + 0.040538836, + 0.018775605, + -0.034081753, + -0.0004998032 + ], + [ + -0.03950687, + 0.043531194, + -0.014640973, + -0.014720938, + 0.024726942, + -0.03612319, + -0.014307638, + 0.01025741, + 0.004683229, + -0.03798171, + -0.03485831, + -0.009100663, + 0.0012342628, + 0.029542759, + 0.04620511, + 0.034314673, + -0.0028652407, + -0.016328944, + 0.007256933, + -0.031210486, + 0.016605832, + -0.04016649, + -0.009275935, + 0.005787752, + -0.0058478094, + 0.029647486, + -0.03285623, + -0.036804114, + 0.0102847135, + 0.03751815, + -0.011018378, + 0.012599248, + 0.007816772, + 0.016865665, + -0.023787512, + -0.0033270551, + 0.026091896, + -0.008189237, + -0.039871283, + -0.032342162, + 0.01738121, + 0.014661393, + 0.03587866, + -0.023817893, + -0.0124929, + -0.03888124, + 0.041935273, + -0.0263411, + -0.039413948, + -0.037727747, + 0.0013563702, + -0.041180752, + 0.008936618, + -0.032369066, + 0.032389134, + 0.041318245, + -0.07018222, + -0.024456048, + -0.020840136, + 0.014526111, + 0.033790283, + 0.010664927, + -0.04341061, + -0.022219952, + -0.010782748, + 0.005418068, + -0.022497445, + 0.013246439, + 0.012736531, + -0.016248927, + 0.0008248154, + 0.009251837, + -0.06683385, + -0.0011490654, + -0.07666291, + 0.011535246, + -0.039218437, + 0.05110004, + -0.09564187, + -0.04194155, + 0.02009398, + -0.0067823115, + -0.009455593, + 0.056001958, + 0.016271655, + 0.06911042, + -0.022176139, + 0.028006595, + -0.028135093, + 0.034655333, + -0.007308119, + -0.013883404, + 0.045097373, + -0.052717112, + -0.034108102, + -0.007105286, + -0.03460486, + 0.031739317, + 0.030613171, + 0.008154331, + -0.06315366, + -0.018434746, + 0.03518978, + 0.0112801725, + 0.024134867, + -0.003977951, + 0.016525777, + 0.01839739, + -0.024246577, + -0.019862736, + 0.00031075653, + -0.013859162, + 0.013144024, + 0.024517544, + -0.038736217, + -0.02523414, + -0.010362971, + -0.027183976, + 0.021046206, + 0.014898793, + 0.003440566, + -0.016803645, + 0.0020033128, + -0.02742916, + 0.0030086606, + 0.0032655115, + -0.02966956, + 0.07162049, + 0.037842706, + -0.0046940506, + 0.00064334256, + 0.020130387, + -0.0076284185, + 0.020069573, + -0.08732818, + -0.03501705, + 0.055594362, + 0.082634926, + 0.014030701, + -0.034930494, + 0.05232482, + 0.053865146, + 0.01451553, + -0.024455464, + 0.026275275, + -0.034078658, + -0.03708409, + 0.01745932, + -0.021863382, + -0.05555264, + -0.033413053, + 0.015493354, + -0.0062087053, + -0.022264965, + -0.02127896, + -0.059868205, + -0.003401156, + 0.044210643, + 0.01787959, + 0.09501992, + 0.018742345, + -0.031885527, + -0.032269854, + 0.00025357012, + 0.042152844, + -0.03316293, + 0.02196225, + -0.0020535681, + -0.0072929896, + -0.0052945167, + -0.021792375, + 0.007816838, + -0.021683265, + -0.034982268, + -0.018460391, + -0.012000704, + 0.06461101, + 0.007490276, + 0.014343332, + -0.021530014, + -0.0075940792, + -0.02895248, + -0.014425406, + -0.040914804, + -0.03639253, + -0.08229706, + -0.004979539, + -0.017418977, + 0.016572325, + 0.037223563, + 0.00116424, + 0.019090686, + 0.010911045, + -0.010205869, + 0.048550658, + 0.035755083, + -0.018986369, + -0.001429209, + -0.027796214, + 0.025839124, + 0.022751706, + 0.054932334, + 0.020839911, + -0.017549826, + -0.044058055, + -0.042574435, + -0.05696988, + 0.024840906, + -0.017054198, + -0.054606467, + 0.0077261142, + -0.02644012, + 0.026694741, + -0.027916811, + -0.015582057, + -0.044071168, + -0.01406791, + 0.0578193, + -0.057348266, + -0.07553237, + -0.020606408, + 0.05444073, + -0.04222853, + -0.02362194, + -0.03362329, + -0.02535667, + -0.0018919092, + 0.034991592, + 0.04424803, + -0.018281374, + -0.027159164, + 0.027622392, + -0.0061814794, + 0.007855805, + -0.015342956, + 0.00082629744, + 0.008025998, + -0.0024992118, + 0.023763394, + -0.003807417, + -0.022212056, + -0.002562533, + -0.00694927, + 0.044300094, + -0.03489844, + -0.057076182, + 0.02006841, + -0.0037533976, + 0.006341216, + 0.007860421, + 0.008677003, + 0.011595276, + -0.046128266, + 0.0013428159, + 0.045055754, + 0.02239221, + 0.019835426, + -0.008117078, + 0.006887426, + -0.008277146, + 0.048709106, + -0.006205201, + 0.091035716, + 0.031401526, + 0.009654572, + -0.004819402, + -0.015429093, + 0.055476464, + -0.020625938, + 0.0040114745, + -0.0067515746, + 0.011730604, + -0.0027268373, + 0.036516953, + 0.015225207, + -0.0048017604, + -0.006393769, + -0.021304023, + 0.056664098, + 0.0026197485, + -0.0080493875, + 0.021506276, + 0.015002799, + 0.009055541, + -0.026723027, + -0.006230178, + -0.0067681912, + -0.016294137, + -0.00268294, + -0.06420861, + 0.02237568, + -0.019511264, + 0.05340581, + -0.00833766, + -0.0200639, + -0.0036080014, + -0.037964303, + -0.10868055, + 0.00020624952, + -0.06711885, + -0.006539819, + -0.0034174747, + -0.008780332, + -0.026614243, + 0.002306322, + 0.005154491, + 0.0019403903, + -0.04626778, + -0.029957443, + 0.006960841, + -0.046184577, + -0.017351516, + -0.021748034, + 0.045616314, + 0.032945316, + 0.026660904, + -0.04546675, + -0.044575658, + -0.029681908, + 0.047057368, + -0.056180052, + 0.014390304, + 0.00015773278, + 0.013120344, + 0.02658648, + 0.011372644, + 0.005863363, + 0.016664037, + 0.06376613, + 0.019382346, + 0.0802324, + -0.0016175115, + 0.021250589, + 0.00053878257, + 0.020504216, + -0.03426853, + 0.005095898, + 0.0031997939, + 0.0051565412, + -0.043827157, + 0.03209446, + 0.04712799, + -0.01782006, + 0.053983606, + -0.035910055, + -0.016622698, + -0.037207805, + 0.034288082, + -0.030350236, + -0.0011003837, + -0.02914163, + -0.03423277, + 0.009082422, + -0.001781773, + 0.032029316, + -0.0034477508, + -0.022011233, + 0.020805836, + 0.04373524, + 0.01426929, + 0.012661985, + 0.0089556845, + -0.02926414, + 0.010948363, + 0.00024500352, + 0.041760296, + -0.019010147, + -0.020999651, + 0.0043251915, + -0.018788947, + -0.03642843, + -0.029119324, + 0.05132978, + 0.008250169, + -0.023003912, + -0.026743688, + -0.08720566, + -0.059065796, + -0.000029261539, + 0.023304205, + -0.03414005, + -0.042730678, + 0.01792291, + 0.0071862466, + 0.012993562, + 0.027364269, + 0.19823818, + -0.0059806146, + -0.018968418, + 0.024375647, + 0.01791589, + 0.012285263, + -0.002662988, + -0.00074736535, + -0.014230657, + 0.03866545, + 0.044439327, + 0.007127386, + -0.007903959, + 0.0022473629, + -0.041698676, + -0.0022041672, + -0.026352853, + 0.009517888, + 0.070968114, + 0.0046247942, + 0.030372195, + 0.006002809, + 0.02578483, + 0.026675988, + 0.013856626, + 0.016871126, + -0.0014647035, + 0.052734047, + -0.009674117, + 0.017708449, + 0.0026036392, + -0.0077395584, + -0.02032638, + -0.057948228, + -0.037037287, + -0.026146235, + -0.012568703, + 0.030102657, + -0.009142211, + -0.0017773161, + 0.022137081, + -0.016034972, + -0.026643815, + 0.043836106, + -0.00026422995, + -0.012727786, + 0.06419574, + 0.006584795, + 0.018396322, + 0.017908173, + -0.050535623, + -0.02078155, + 0.010549742, + -0.014217779, + 0.024685526, + -0.023678334, + -0.020491404, + 0.006048553, + -0.023561789, + 0.018910788, + 0.08878148, + 0.007740561, + -0.059295204, + 0.03733036, + -0.016496437, + -0.018883424, + 0.018802142, + -0.007328556, + 0.016717581, + 0.04328429, + 0.05693224, + 0.017707568, + 0.046792727, + -0.014650998, + 0.062150735, + -0.012973712, + -0.018037295, + 0.016467381, + 0.0244791, + 0.0062886435, + -0.02355788, + 0.014314312, + -0.01939792, + -0.002719139, + 0.0030674103, + -0.008508544, + 0.006620206, + 0.008997434, + -0.02707569, + -0.068797246, + -0.026732523, + -0.03025209, + -0.07743795, + 0.025089052, + 0.036291923, + 0.03730403, + 0.04085414, + -0.0066735293, + -0.0055792555, + 0.031028569, + -0.0030874214, + -0.015421479, + -0.013141398, + -0.020299582, + -0.023435697, + 0.0017748995, + -0.0021218956, + 0.0012519993, + -0.001484999, + 0.008944802, + -0.027911007, + 0.012121643, + -0.020927167, + -0.0002258706, + -0.012256106, + 0.0047594383, + -0.0148541825, + -0.043066435, + 0.030682703, + 0.012971142, + -0.003666529, + 0.0108013395, + 0.025183335, + -0.028475968, + 0.021690562, + -0.0028832422, + -0.014144738, + -0.0045188447, + -0.040258396, + 0.028346978, + 0.035040945, + -0.011023868, + 0.0396404, + 0.017606566, + 0.033269204, + 0.0073707304, + -0.014222149, + 0.014156873, + 0.030270226, + 0.015319926, + 0.024624461, + -0.039615463, + 0.022913564, + 0.0031669824, + 0.002471962, + -0.039530534, + 0.04196699, + -0.017445292, + -0.030874938, + -0.0053294376, + 0.038435955, + 0.015547038, + -0.017440522, + -0.040413458, + -0.03927991, + -0.017935151, + 0.0047400724, + 0.003020446, + -0.026619047, + -0.039115086, + 0.056505933, + 0.009858637, + 0.0131826205, + 0.0284519, + 0.022894057, + 0.0022495075, + -0.021099819, + -0.0116347065, + -0.01811318, + 0.006474129, + 0.054837897, + -0.012720091, + 0.0016731841, + 0.021986602, + 0.033270527, + -0.00033776753, + -0.008643222, + 0.019710293, + -0.03264088, + 0.010028613, + 0.005217198, + -0.024148779, + -0.04014084, + -0.016549045, + 0.025656188, + 0.054638997, + 0.012246356, + -0.032795466, + -0.015674438, + -0.026266312, + -0.01646067, + 0.08210362, + 0.015601729, + -0.001658601, + 0.028211301, + -0.03853208, + -0.04279942, + 0.06524898, + -0.022642639, + -0.017951515, + -0.029549977, + -0.034688108, + 0.0061429352, + 0.010141051, + 0.00042500003, + 0.025635283, + -0.043835618, + -0.033300713, + -0.0059104236, + -0.007492045, + -0.025678229, + -0.012022994, + -0.0012621493, + -0.027511965, + 0.034545906, + 0.0027232112, + 0.0049217213, + 0.017144063, + 0.031473048, + -0.021966783, + 0.009118309, + 0.03147311, + 0.039992817, + -0.023138776, + -0.0014036369, + -0.037520673, + 0.07687327, + -0.039589405, + 0.04602321, + -0.003612067, + 0.007461662, + 0.007353311, + -0.034707982, + -0.02012695, + 0.029514175, + -0.031999454, + -0.043143217, + 0.028792208, + -0.0011134085, + -0.03371898, + 0.013939409, + -0.0019052078, + -0.004295308, + -0.007053404, + -0.0009750122, + 0.029560952, + 0.005407683, + 0.03037516, + -0.00785735, + -0.021339875, + 0.02605982, + 0.039290346, + -0.010707787, + -0.03064161, + -0.00059364137, + 0.012069025, + -0.0033701425, + -0.017966857, + -0.018942371, + 0.017725015, + -0.004986756, + 0.011548954, + 0.0028690782, + 0.038817663, + -0.021664195, + 0.013531663, + 0.0156029165, + -0.023836723, + 0.024638338, + 0.01691388, + 0.003397145, + -0.03100652, + -0.04320627, + 0.0011457364, + 0.04675279, + 0.027556699, + -0.033261135, + -0.008995849, + 0.017848724, + 0.014306368, + 0.025643473, + 0.035529383, + 0.0029430243, + 0.03153322, + -0.052132677, + -0.00248758, + -0.02161323, + -0.04709703, + -0.042139307, + -0.023811264, + 0.057088885, + -0.007749684, + 0.01378622, + 0.0038549982, + -0.042886388, + -0.02104687, + 0.00838618, + -0.0037721547, + -0.037230987, + -0.0111783985, + 0.0097314585, + 0.02131326, + -0.040853705, + 0.0057790386, + 0.021632854, + -0.022898814, + -0.012932489, + 0.004934083, + 0.010148015, + 0.014385065, + -0.003315393, + 0.017242506, + -0.007246454, + 0.009660279, + -0.019179016, + -0.010557076, + 0.016264342, + -0.029500151, + -0.012701573, + -0.012758462, + -0.029923983, + -0.02165708, + -0.052309945, + -0.033560004, + 0.0076761013, + -0.01549472, + -0.013017553, + 0.0043236334, + 0.04691618, + -0.008197069, + -0.0018741535, + -0.03084931, + -0.0054556658, + -0.016673744, + 0.039174348, + 0.04543094, + -0.023596205, + 0.001231028, + 0.035865817, + 0.0023900091, + -0.003052916, + -0.027418893, + 0.058787756, + -0.02245522, + -0.015146493, + -0.0184522, + -0.0063935583, + -0.030266108, + 0.007447998, + -0.017568134, + -0.0027355, + 0.021793561, + 0.0074607856, + -0.04652875, + -0.031837653, + -0.061417032, + -0.035766512, + 0.007060734, + 0.018031834, + 0.00045341678, + -0.0077504576, + 0.0076210555, + 0.012140915, + -0.002184006, + 0.06035298, + -0.022536587, + 0.0054653105, + -0.005007854, + -0.007992475, + -0.034879297, + 0.007152057, + -0.01913284, + -0.10116649, + 0.0022808493, + 0.008442801, + 0.036988016, + -0.016776918, + -0.07377468, + -0.01899712, + 0.07031005, + -0.034832127, + -0.03700141, + -0.01211916, + 0.0067655994, + 0.024989525, + 0.02319956, + -0.027294133, + -0.00094650587, + 0.014060532, + 0.03490878, + -0.01971701, + -0.038569264, + -0.04076684, + 0.014368246, + -0.10717569, + 0.020453772, + 0.068951204, + -0.007349555, + -0.04629167, + -0.014826132, + 0.005653124, + -0.026169593, + 0.013392487, + -0.07173489, + -0.01668766, + -0.030206582, + 0.040627755, + -0.029564377, + -0.043053318, + -0.03218649, + 0.03655294, + -0.008489806, + 0.04350242, + 0.030536214, + -0.009721366, + 0.00034617525, + -0.016816063, + 0.059502516, + -0.021497775, + -0.022350075, + -0.111227795, + 0.028571766, + -0.033449635, + 0.07857797, + -0.02256202, + 0.057096943, + 0.04939214, + 0.02692003, + 0.03341055, + 0.010906601, + 0.03452141, + -0.101317056, + -0.016416814, + 0.028970905, + -0.027896484, + 0.022908753, + -0.015605515, + 0.009761399, + 0.01375609, + 0.002159126, + -0.034757312, + 0.009949484, + -0.027204521, + 0.036121096, + -0.018719548, + 0.012673245, + -0.047391124, + -0.044187777, + 0.048009574, + -0.012766926, + 0.0116457315, + 0.014982552, + 0.019103188, + 0.022949666, + -0.039069697, + -0.04630458, + -0.028584287, + -0.040020872, + 0.015652915, + -0.013488799, + -0.049111538, + 0.026503077, + 0.03357381, + 0.03751419, + 0.02387496, + -0.0011243735, + 0.012061748, + -0.011386109, + -0.0127039915, + 0.06352219, + 0.048704118, + 0.014793782, + -0.014007755, + -0.043857392, + 0.038399946, + -0.012055639, + -0.014885214, + 0.042390678, + 0.04980625, + -0.006241386, + 0.013584097, + -0.033264153, + 0.041379217, + 0.029621469, + -0.043042388, + -0.0020997606, + -0.012933665, + 0.013286224, + -0.003282772, + 0.018547414, + -0.04327, + 0.022249907, + 0.008725586, + 0.018808063, + -0.026111113, + 0.008951248, + 0.031707752, + -0.005465531, + 0.06568567, + -0.020877102, + 0.03014087, + 0.01715928, + 0.012325161, + 0.01315365, + -0.031900663, + -0.022260122, + -0.03447207, + -0.00062158384, + -0.035049655, + 0.003485939, + 0.0046049925, + -0.05630732, + -0.011058264, + 0.007136582, + 0.016117854, + 0.0595212, + -0.03617894, + -0.027507313, + 0.002789625, + 0.01799178, + -0.0014536566, + -0.008393191, + -0.018825287, + 0.04759385, + 0.058703866, + -0.011992767, + -0.0052801687, + 0.01940323, + 0.03281311, + -0.01995503, + -0.013774053, + 0.05898928, + -0.07519091, + -0.044439603, + -0.0537019, + -0.014705708, + 0.01250235, + 0.015665539, + 0.05294212, + 0.009310713, + 0.03523993, + 0.031513117, + -0.0074070916, + 0.01826314, + 0.011835665, + -0.044761684, + -0.007041866, + 0.058125574, + 0.024500526, + -0.009871987, + -0.018237904, + 0.09113848, + -0.013172258, + -0.0041379156, + 0.016455337, + -0.022989342, + 0.016601447, + 0.014433, + 0.0053295735, + -0.024041522, + 0.01957587, + 0.018358402, + -0.017210413, + 0.043062195, + 0.0061207553, + 0.032532007, + 0.014793015, + 0.024234612, + 0.0061567454, + -0.018245239, + 0.02684064, + 0.00016797535, + 0.028608758, + 0.0046842927, + -0.010550258, + 0.02669938, + 0.030235212, + -0.0031934716, + -0.019914627, + 0.035149656, + 0.0075301835, + 0.039265014, + 0.035625007, + -0.031268924, + 0.012515024, + -0.07590218, + 0.007551814, + -0.035385616, + 0.027681876, + -0.039070603, + -0.004845278, + 0.04028714, + -0.07426009, + 0.06613951, + -0.05510454, + -0.03226479, + -0.014005947, + -0.03195615, + -0.0116673885, + 0.016437342, + -0.0076668607, + 0.028972298, + -0.025123762, + -0.017822955, + -0.011712876, + -0.012553329, + -0.037626002, + 0.00069678575, + 0.009572705, + 0.0067847413, + 0.0028660365, + -0.011129996, + 0.00090118404, + -0.0070405314, + -0.018169235, + -0.018179605, + 0.07109999, + -0.016261067, + -0.024284609, + -0.026516862, + 0.03497836, + -0.017624862, + 0.009457551, + -0.030564932, + 0.04343077, + -0.01112055, + -0.054642823, + 0.004773121, + 0.0047957487, + -0.008624442, + 0.015786134, + 0.03043451, + -0.043876424, + 0.015928723, + 0.045520768, + -0.025372392, + 0.009925128, + 0.028771762 + ] + ], + "abstain_centroids": [ + [ + -0.031148382, + 0.0053377217, + -0.051859938, + 0.006701707, + -0.013760795, + -0.046238896, + -0.0017418014, + 0.003761064, + 0.011534153, + 0.015631318, + -0.007558447, + 0.042414147, + -0.008460059, + -0.0039428133, + 0.014397731, + -0.00096799876, + 0.005969027, + -0.016228952, + -0.0066106487, + -0.01633511, + -0.017304767, + -0.015375099, + 0.00016670344, + 0.029177906, + 0.019667022, + 0.037063632, + -0.014033232, + -0.017078552, + 0.022332544, + 0.0029681043, + -0.026012668, + 0.004773388, + -0.008349955, + -0.025975648, + -0.05798201, + -0.037145726, + 0.022042587, + -0.0104322145, + -0.054713663, + -0.018552497, + 0.01407926, + 0.0066314046, + 0.027429355, + -0.026314953, + 0.021749053, + -0.05777429, + -0.011509257, + -0.020935455, + -0.00084091024, + -0.031351313, + -0.021382079, + -0.00847218, + 0.036603227, + -0.013143199, + 0.047227796, + 0.05580892, + -0.0109791085, + -0.021309411, + -0.087006725, + 0.0018242795, + -0.025122127, + -0.013786088, + -0.009168321, + 0.011006979, + 0.0012779796, + 0.09556672, + 0.00808916, + -0.008711008, + -0.008361359, + -0.034987632, + -0.03647069, + 0.021719662, + -0.032498714, + -0.017760865, + -0.06984331, + 0.011626705, + 0.020904267, + -0.008985531, + 0.0070378105, + 0.037491333, + 0.08140703, + -0.015960725, + 0.00758235, + 0.018223334, + 0.011250972, + 0.041393135, + -0.0300754, + 0.011359266, + -0.0026697272, + 0.0045864326, + -0.029182361, + -0.018836327, + 0.035417307, + -0.050186697, + -0.0169077, + 0.0030452076, + -0.031451467, + 0.021025099, + 0.03237884, + -0.017753405, + 0.024965167, + 0.006849125, + -0.0007417424, + -0.025353184, + 0.014833974, + 0.000480421, + 0.033970177, + 0.029662795, + 0.0019640021, + -0.00040594768, + 0.046204697, + 0.03329503, + 0.014799664, + 0.004400712, + -0.012687833, + -0.04885642, + -0.021582833, + -0.029335678, + 0.031864036, + -0.013479058, + 0.0061972495, + 0.01931695, + 0.039016202, + -0.0460622, + -0.011819415, + -0.03546195, + 0.030371843, + 0.045124575, + -0.00695266, + 0.026146563, + 0.0153582115, + 0.014641614, + -0.037259024, + 0.004900667, + -0.012360953, + -0.030733729, + 0.016830467, + -0.002211426, + 0.017974649, + -0.051037353, + 0.024567816, + 0.029570596, + -0.028185146, + -0.028654283, + 0.02143797, + -0.038578555, + 0.024237396, + -0.0068540815, + 0.01847857, + -0.012613435, + 0.00437136, + -0.01011591, + 0.0065389504, + -0.014050798, + 0.0061866366, + -0.01770768, + -0.019518772, + 0.053408965, + -0.013017107, + 0.016827421, + 0.022524578, + -0.0068964125, + -0.052380078, + 0.00038339154, + 0.029090617, + -0.011927654, + 0.024243768, + 0.01339041, + -0.014234689, + -0.0017058258, + -0.012792839, + -0.026187133, + 0.039857496, + -0.0025493933, + 0.030229094, + 0.026641972, + 0.07534804, + 0.022857834, + 0.010506173, + -0.058816753, + -0.051879648, + 0.016220875, + 0.017665846, + -0.054791577, + -0.039248586, + 0.014590265, + 0.021849323, + -0.018102316, + 0.015241521, + 0.02129747, + -0.022830576, + -0.01614023, + 0.027246347, + -0.00542563, + 0.046256945, + -0.007986688, + 0.026465587, + -0.0026759577, + -0.0096074, + -0.037014082, + -0.04806582, + 0.033057604, + 0.04526261, + 0.0051544863, + -0.0342738, + -0.011255881, + -0.03914743, + -0.031940874, + 0.0144135505, + -0.017719291, + 0.009187744, + 0.0078019383, + 0.02771336, + 0.0051388917, + -0.042035703, + 0.0072669606, + -0.0026950007, + 0.0030906193, + 0.004021663, + -0.02106473, + 0.0009345182, + 0.029135607, + 0.0021005718, + -0.0196823, + 0.046538655, + 0.032432303, + 0.017765502, + 0.015490211, + -0.009414902, + -0.02356354, + -0.008533621, + 0.0050160224, + -0.019913118, + -0.0035469253, + 0.03606248, + -0.022427976, + -0.04044626, + 0.015986245, + 0.009162774, + -0.00032497806, + -0.022688588, + -0.030663738, + 0.0018985462, + 0.024509203, + -0.04645868, + 0.021605283, + -0.0032775397, + 0.008400841, + 0.0053840177, + -0.0020570229, + 0.018701848, + -0.012248811, + -0.045328125, + 0.0036800986, + 0.032931987, + 0.028872136, + 0.012167973, + -0.0052398676, + 0.026528696, + 0.013797996, + 0.023905413, + 0.031488698, + 0.016135292, + 0.037903946, + 0.007559896, + 0.003637543, + -0.033904064, + -0.0027450037, + 0.0044508814, + 0.010002827, + -0.03097531, + -0.016194653, + -0.0103033185, + 0.0017212455, + -0.004217246, + -0.0003838392, + -0.028139843, + -0.037131034, + 0.059932996, + 0.016013881, + 0.0041002985, + 0.017807445, + 0.012730673, + -0.009848034, + 0.0070455708, + -0.0010963108, + -0.017336583, + 0.0052566174, + 0.006595982, + -0.045635466, + 0.01257733, + -0.015105403, + 0.08849552, + -0.0060461042, + -0.014552671, + -0.010428577, + -0.03148846, + -0.244925, + -0.022456428, + -0.01200543, + 0.018366225, + 0.012265931, + -0.024266042, + -0.033916224, + -0.0036076305, + -0.054510105, + 0.0063244584, + -0.015560582, + -0.055878177, + -0.020939475, + -0.005328696, + -0.0070734383, + 0.0048970133, + 0.0069189873, + -0.022256982, + 0.0075242044, + -0.06765787, + -0.023333378, + -0.025017215, + 0.07195793, + -0.019617401, + -0.005269894, + 0.00026103246, + 0.035949826, + 0.004107666, + -0.037599213, + -0.01236279, + -0.006176068, + 0.0069308924, + -0.0029138543, + 0.035269044, + 0.014236289, + 0.034278773, + 0.010570881, + -0.008682466, + -0.014200495, + -0.0010429567, + 0.045053545, + 0.02033515, + -0.0039721197, + 0.011877003, + 0.010973981, + -0.040029325, + -0.021862814, + -0.010063818, + -0.01741567, + -0.02046978, + 0.0024775746, + 0.00537928, + -0.0035171232, + -0.023752872, + -0.027805964, + 0.0068444465, + 0.0020820745, + 0.048062917, + -0.013278909, + -0.011143367, + -0.015697205, + -0.010483089, + 0.008951397, + 0.03141075, + -0.021271285, + 0.0060245073, + -0.005130949, + -0.013589777, + 0.028885502, + -0.019157533, + 0.033051424, + -0.0110376915, + 0.008256805, + -0.021247167, + 0.014382738, + 0.025279364, + -0.025481079, + -0.032188643, + -0.023664054, + -0.18205334, + -0.016931957, + -0.00092831947, + 0.0030238251, + 0.033743776, + -0.016215393, + -0.025330719, + 0.00043090683, + 0.052764844, + 0.045047104, + 0.34483737, + 0.021190971, + 0.029025022, + -0.029802192, + 0.05892976, + -0.027338218, + 0.017156813, + 0.01649102, + 0.0053761047, + -0.03603205, + -0.016050696, + -0.019247439, + 0.008546483, + 0.00065792014, + 0.008216663, + 0.012412621, + -0.049241338, + 0.0046835714, + 0.083209105, + 0.0044032163, + 0.015972426, + -0.035188276, + 0.0025444007, + 0.0276847, + -0.03703065, + -0.040941797, + -0.022856751, + 0.07734354, + -0.017490435, + 0.029937685, + -0.030589428, + -0.0131223295, + 0.028537748, + -0.015808787, + -0.031040976, + 0.0022384813, + 0.004584949, + -0.036500685, + -0.013998977, + 0.01908896, + -0.020744048, + -0.027405975, + 0.027832054, + 0.0009314528, + 0.021530954, + -0.015619696, + -0.006596794, + -0.008945383, + -0.02280639, + -0.00078093243, + -0.019146752, + -0.021306396, + -0.024635073, + -0.0013178922, + 0.030522594, + -0.036964312, + -0.03445982, + -0.026010966, + 0.011097982, + 0.04430233, + 0.033004954, + 0.012737286, + -0.02832643, + -0.033423617, + -0.011146303, + 0.018307667, + -0.010493131, + -0.002345288, + -0.006297813, + 0.023474589, + 0.015160519, + 0.008065754, + 0.0071064318, + -0.012013574, + 0.034734786, + 0.020278838, + 0.013851283, + 0.07314595, + 0.005167692, + 0.03851032, + -0.011801342, + -0.024405105, + -0.05664373, + -0.017895715, + 0.010395462, + 0.0045732795, + -0.017845895, + 0.052198168, + -0.007843896, + -0.03997914, + 0.011556584, + 0.019499848, + -0.03996185, + -0.007970878, + -0.00291532, + -0.05102505, + 0.023902692, + -0.024392458, + -0.019463964, + -0.021712244, + -0.028254114, + -0.022925302, + 0.008098716, + -0.009423168, + -0.0034888273, + -0.02608165, + 0.0058718375, + 0.012862887, + 0.020583529, + 0.015732484, + 0.004980414, + 0.0077576265, + -0.018226098, + -0.036115997, + 0.025706176, + 0.049908377, + 0.025328668, + 0.0090175755, + 0.020332497, + 0.040973034, + -0.016721344, + -0.01928604, + -0.012567933, + 0.0024534017, + -0.027978519, + -0.016283054, + -0.0059162932, + 0.020438466, + -0.033998214, + 0.025316799, + 0.015457932, + -0.027260445, + 0.0077517964, + 0.03414886, + 0.06149145, + -0.009142278, + 0.034868557, + 0.01562827, + -0.011573751, + 0.048809845, + -0.008835819, + -0.042271934, + -0.011501361, + -0.015698932, + 0.00765774, + 0.0040898435, + 0.017931513, + 0.01253479, + -0.026830954, + 0.009097795, + 0.008690346, + 0.044174474, + 0.0060633975, + -0.056720097, + -0.01996871, + -0.035708513, + -0.0106808655, + -0.0031720465, + -0.0111088455, + -0.029465713, + 0.012013627, + 0.014341204, + 0.052498966, + 0.13476695, + 0.046210244, + 0.052687798, + -0.00610014, + 0.045737743, + 0.043751348, + -0.032143187, + 0.0028189388, + -0.035125505, + -0.04272793, + 0.031032862, + 0.034608897, + -0.020154651, + -0.025992718, + 0.0040032133, + -0.013573706, + 0.0014319277, + 0.033563722, + -0.0124865, + -0.013652726, + -0.00095753185, + 0.018106164, + -0.019855194, + 0.00985104, + -0.017903104, + 0.013870335, + -0.02261499, + -0.025842719, + 0.123712055, + -0.016144589, + -0.0011982427, + 0.06412821, + 0.005470516, + 0.013032823, + 0.031510778, + -0.0073665287, + -0.03247822, + 0.00045605694, + -0.018492017, + 0.0098344525, + -0.025878755, + 0.011205348, + -0.012217908, + 0.004897915, + 0.015269929, + -0.02245079, + -0.0018926292, + 0.010645848, + 0.007540698, + -0.018376747, + 0.0011163224, + -0.009108631, + -0.005284527, + 0.027602877, + 0.020686561, + -0.028491242, + -0.016864398, + 0.0044273087, + 0.0062007783, + 0.047445305, + -0.06059543, + -0.023386637, + 0.031225728, + -0.01507088, + -0.030429766, + -0.0027328802, + 0.011734569, + 0.0085614575, + -0.015814755, + -0.010190624, + -0.014055305, + 0.01249472, + 0.0020188028, + -0.018316, + 0.017405652, + 0.00333235, + -0.0058355783, + 0.01903454, + -0.017335443, + 0.009071962, + 0.060741838, + -0.019548608, + 0.0028466568, + -0.0009897373, + -0.01391898, + 0.039289407, + 0.033837304, + -0.020116923, + -0.0049318965, + 0.021821506, + 0.008678599, + -0.005215469, + -0.0035473392, + 0.005850026, + -0.00011736609, + -0.038427465, + 0.004604565, + -0.017376447, + 0.03195163, + -0.037625905, + -0.00025921583, + -0.045117155, + -0.021053704, + 0.01866349, + -0.018683132, + -0.022075273, + -0.067146465, + -0.0003485933, + -0.0066958447, + -0.019604217, + -0.025007758, + 0.008463429, + 0.01903175, + -0.036018703, + 0.0002199503, + -0.014541071, + 0.03636302, + -0.024418777, + 0.036282822, + -0.015224955, + 0.03595708, + -0.040158544, + 0.03579523, + 0.02829524, + 0.011141475, + -0.01444086, + -0.03210745, + 0.012157547, + -0.005177962, + -0.056817777, + 0.01769194, + -0.03663195, + 0.038909055, + -0.009773796, + 0.0063458867, + -0.024140002, + 0.01852215, + -0.0378227, + 0.0095784925, + -0.050063416, + -0.0136081185, + -0.019632932, + 0.0028407509, + -0.0064537292, + -0.016191104, + 0.04372594, + 0.023616401, + 0.027719082, + -0.041044924, + 0.025428394, + 0.020110026, + -0.0060314494, + -0.0061647827, + 0.07147246, + -0.022163881, + -0.002495512, + 0.017374912, + 0.024532678, + 0.017868279, + -0.0123060895, + 0.009035886, + -0.023077391, + -0.00059326034, + -0.021227827, + 0.0071837874, + -0.036735952, + -0.014673909, + -0.028121512, + 0.010137441, + -0.010372104, + -0.038298663, + 0.01282512, + -0.011348841, + 0.022564255, + -0.031195194, + 0.049415413, + -0.03715284, + -0.0024479807, + 0.023421839, + 0.06413017, + -0.020511432, + 0.0113146, + -0.03853917, + 0.015962925, + -0.0053321286, + 0.01771349, + 0.016681707, + 0.020176094, + 0.045899525, + -0.03644215, + -0.05389872, + 0.03137194, + 0.008404089, + -0.038138144, + 0.014195226, + 0.04344335, + 0.0057477932, + -0.03277102, + 0.04831074, + -0.021211635, + 0.030088603, + -0.00251031, + 0.01228796, + -0.004873214, + -0.008176673, + -0.014280845, + -0.0006814086, + -0.019928552, + 0.008787194, + 0.010925872, + -0.0032950707, + -0.034151938, + 0.023929175, + -0.010394337, + -0.010170749, + 0.029074438, + 0.0061191227, + -0.023697818, + -0.021662997, + -0.03463375, + -0.0376407, + -0.008391389, + -0.01087796, + -0.008222668, + 0.0008719065, + 0.07043431, + 0.02145938, + -0.050167546, + -0.023954187, + -0.042268485, + 0.023713034, + -0.22955438, + 0.03520514, + -0.0021951396, + -0.0029458369, + -0.055697378, + 0.0052851303, + -0.008466735, + -0.00713325, + 0.0063962257, + -0.03216339, + -0.024553273, + 0.028201995, + -0.00005423209, + -0.018851798, + -0.007118189, + 0.040054683, + -0.03188417, + -0.0030668841, + -0.004163999, + 0.032002646, + 0.0047784974, + -0.02514371, + -0.005436027, + 0.014385046, + 0.018197643, + 0.0026216004, + -0.026822513, + -0.013488665, + -0.044493288, + -0.01340355, + 0.0006943536, + -0.031944275, + 0.021567853, + 0.043572214, + -0.008895598, + 0.017365277, + 0.00690336, + -0.014118812, + -0.017342428, + 0.013759935, + 0.0014441384, + 0.019967059, + -0.023214184, + -0.0035139646, + -0.0009887023, + 0.028011115, + -0.019943252, + -0.008950248, + -0.059093572, + -0.0024248771, + 0.013076103, + 0.026632652, + -0.018246919, + 0.0191107, + 0.012685868, + 0.00189008, + -0.05187209, + 0.009624937, + -0.0076693487, + 0.037904378, + 0.0033752434, + 0.012424309, + -0.03196644, + -0.03168633, + -0.04591932, + -0.02398697, + -0.08195933, + 0.013067175, + 0.013707991, + 0.02845325, + -0.022814702, + -0.036631793, + -0.0046700407, + -0.014707499, + 0.025149884, + 0.015747225, + 0.011062072, + -0.0079821665, + -0.044059474, + -0.0052218223, + -0.029661851, + -0.0010427525, + -0.052402854, + 0.010401699, + 0.012599079, + 0.018898357, + -0.024228007, + 0.012707991, + -0.01779489, + -0.0073080487, + -0.0570653, + -0.024420062, + -0.0004401083, + 0.0057198247, + -0.04850662, + 0.009310804, + -0.027014086, + -0.012875564, + -0.019028028, + -0.021750357, + -0.039091874, + 0.007929617, + -0.014110445, + -0.0059228763, + -0.027193954, + -0.008130776, + 0.05215637, + -0.009849599, + 0.026919808, + 0.021327212, + -0.013137315, + 0.0056819157, + -0.010384434, + 0.0052230777, + -0.02551496, + -0.023010256, + -0.009133841, + -0.0034335821, + -0.0066825, + -0.012148635, + -0.01445714, + 0.03445042, + -0.0021199132, + -0.04214014, + 0.01076899, + 0.010510174, + 0.035082106, + 0.008629588, + 0.0073693595, + -0.011705998, + 0.046988714, + -0.03607273, + 0.001315558, + 0.028497273, + 0.020122701, + 0.0019115491, + -0.017506052, + 0.05194846, + -0.029103, + -0.059791327, + -0.01251193, + -0.004560316, + -0.001194598, + -0.02777315, + -0.032533206, + -0.010601676, + 0.008287073, + 0.010776672, + -0.05712957, + -0.033277795, + 0.031840082, + 0.025541494, + 0.021605106, + -0.006681298, + 0.0028221495, + 0.01971035, + -0.0057943845, + 0.0026962268, + 0.0066015963, + 0.044709176, + 0.03829154, + -0.04634911, + -0.021133212, + -0.024994686, + -0.010804923, + 0.006723943, + 0.001489737, + -0.011834817, + -0.023318285, + -0.0038526892, + 0.0080185495, + 0.047896653, + -0.03021623, + 0.0534178, + 0.038851902, + -0.009745994, + 0.035945367, + 0.041955516, + 0.03538382, + -0.0023587276, + 0.03368713, + 0.00009861296, + 0.023677062, + 0.024938073, + -0.016926885, + 0.0016545699, + 0.0021329166, + 0.04145556, + -0.0030341037, + 0.026448693, + -0.013949449, + -0.00668188, + 0.0024977801, + -0.052358516, + 0.0584629, + 0.017483938, + 0.016702253, + 0.04207009, + -0.020453833, + 0.04187879, + -0.011923705, + -0.021604257, + -0.010823228, + -0.015712183, + -0.008566785, + -0.00082078524, + -0.02231372, + -0.03011328, + -0.042152412, + 0.031767588, + -0.03151925, + -0.011423307, + -0.01184754, + 0.022948263, + 0.004512842, + 0.0025419197, + 0.02291634, + 0.020818466, + 0.017587373, + 0.0005803047, + -0.016511247, + 0.009406082, + -0.015658183, + -0.035970535, + 0.01030802, + -0.027739283, + -0.022434793, + -0.003182124, + -0.011698128, + -0.014282087, + 0.010802368, + 0.011209724, + -0.037597824, + 0.06280771, + 0.030460317, + -0.014077968, + 0.052043952, + 0.03405556, + 0.0068608206, + -0.024741665, + 0.05734323, + -0.019578138, + -0.018592859, + -0.0055962713 + ] + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-6/router_report.json b/benchmarks/models/router/centroid/out-router-iter-6/router_report.json new file mode 100644 index 0000000..3cb9f76 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-6/router_report.json @@ -0,0 +1,605 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.1, + "metrics": { + "total": 141, + "accuracy": 0.53900707, + "tool_precision": 1.0, + "tool_recall": 0.225, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "tool_count": 80, + "memory_count": 46, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70918083, + "sim_memory": 0.69442564, + "sim_abstain": 0.62441397 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6285488, + "sim_memory": 0.61792433, + "sim_abstain": 0.5503659 + }, + { + "query_id": "shell_test_001", + "text": "what processes are listening on port 8003?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6625408, + "sim_memory": 0.5637741, + "sim_abstain": 0.5741711 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6475192, + "sim_memory": 0.54450005, + "sim_abstain": 0.5841964 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7063374, + "sim_memory": 0.6165706, + "sim_abstain": 0.6750375 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6848668, + "sim_memory": 0.5817895, + "sim_abstain": 0.6013617 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6619269, + "sim_memory": 0.595923, + "sim_abstain": 0.5525088 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6134916, + "sim_memory": 0.53323615, + "sim_abstain": 0.53623945 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7070505, + "sim_memory": 0.6493096, + "sim_abstain": 0.5815989 + }, + { + "query_id": "shell_test_010", + "text": "what's the uptime since klbr daemon last started", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6483902, + "sim_memory": 0.55113995, + "sim_abstain": 0.51299685 + }, + { + "query_id": "shell_test_011", + "text": "how big is agent.db right now on disk?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.618642, + "sim_memory": 0.5386792, + "sim_abstain": 0.52846694 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6492218, + "sim_memory": 0.56391144, + "sim_abstain": 0.5497022 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.65761024, + "sim_memory": 0.56834835, + "sim_abstain": 0.62134224 + }, + { + "query_id": "shell_test_014", + "text": "how many memory rows are in agent.db right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6829826, + "sim_memory": 0.58638436, + "sim_abstain": 0.5644451 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5843893, + "sim_memory": 0.547776, + "sim_abstain": 0.5217595 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6942834, + "sim_memory": 0.6637705, + "sim_abstain": 0.5931268 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66822094, + "sim_memory": 0.5693777, + "sim_abstain": 0.6224822 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66665, + "sim_memory": 0.5669212, + "sim_abstain": 0.60647064 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5745318, + "sim_memory": 0.52103066, + "sim_abstain": 0.5162513 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.64815664, + "sim_memory": 0.53975767, + "sim_abstain": 0.5891774 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5961604, + "sim_memory": 0.5583175, + "sim_abstain": 0.57129914 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6546817, + "sim_memory": 0.5749995, + "sim_abstain": 0.61477154 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.68234557, + "sim_memory": 0.60593796, + "sim_abstain": 0.60524577 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.60944146, + "sim_memory": 0.524113, + "sim_abstain": 0.54898715 + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.566589, + "sim_memory": 0.4897711, + "sim_abstain": 0.465443 + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5898618, + "sim_memory": 0.5351446, + "sim_abstain": 0.52470756 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66542095, + "sim_memory": 0.5676791, + "sim_abstain": 0.59653854 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6722767, + "sim_memory": 0.6186101, + "sim_abstain": 0.54912585 + }, + { + "query_id": "rf_test_002", + "text": "which file defines the ClientMsg type?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.53049797, + "sim_memory": 0.44120914, + "sim_abstain": 0.4389323 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6583943, + "sim_memory": 0.6532231, + "sim_abstain": 0.5341912 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.59658426, + "sim_memory": 0.5636423, + "sim_abstain": 0.5362745 + }, + { + "query_id": "rf_test_006", + "text": "config.rsのmemory_support_score_gapのデフォルト値はいくつ?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.705353, + "sim_memory": 0.6360202, + "sim_abstain": 0.55474705 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6974401, + "sim_memory": 0.65405804, + "sim_abstain": 0.58798295 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5960697, + "sim_memory": 0.55635834, + "sim_abstain": 0.5040847 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65429235, + "sim_memory": 0.63408476, + "sim_abstain": 0.554867 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5710796, + "sim_memory": 0.5308014, + "sim_abstain": 0.50004137 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6926011, + "sim_memory": 0.691817, + "sim_abstain": 0.58473253 + }, + { + "query_id": "rf_test_016", + "text": "read interrupt.rs and tell me what the Interrupt enum contains", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6431551, + "sim_memory": 0.5701346, + "sim_abstain": 0.5663854 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65750617, + "sim_memory": 0.60042876, + "sim_abstain": 0.54133034 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6355181, + "sim_memory": 0.5609783, + "sim_abstain": 0.5377031 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6916864, + "sim_memory": 0.6152147, + "sim_abstain": 0.57028234 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6794881, + "sim_memory": 0.5732701, + "sim_abstain": 0.5938102 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5705274, + "sim_memory": 0.53116524, + "sim_abstain": 0.48013124 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.55382746, + "sim_memory": 0.5629208, + "sim_abstain": 0.511596 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.654468, + "sim_memory": 0.6415955, + "sim_abstain": 0.5482988 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.56339705, + "sim_memory": 0.5195113, + "sim_abstain": 0.4231544 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6873349, + "sim_memory": 0.6034543, + "sim_abstain": 0.58547026 + }, + { + "query_id": "rf_test_031", + "text": "read the first 10 lines of agent.rs", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5996914, + "sim_memory": 0.5221693, + "sim_abstain": 0.54224586 + }, + { + "query_id": "wf_test_001", + "text": "set the compaction_keep to 15 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.69275683, + "sim_memory": 0.62913835, + "sim_abstain": 0.5867398 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65280145, + "sim_memory": 0.62437373, + "sim_abstain": 0.5488851 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.57037896, + "sim_memory": 0.5085687, + "sim_abstain": 0.52038395 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.652064, + "sim_memory": 0.57737964, + "sim_abstain": 0.5854702 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6568779, + "sim_memory": 0.6171636, + "sim_abstain": 0.5631379 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.61866033, + "sim_memory": 0.5483311, + "sim_abstain": 0.5347757 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7101654, + "sim_memory": 0.6064741, + "sim_abstain": 0.6337338 + }, + { + "query_id": "wf_test_008", + "text": "patch the memory_top_k to 5 in config.rs", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.7501923, + "sim_memory": 0.6741924, + "sim_abstain": 0.61280704 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.64629316, + "sim_memory": 0.61019343, + "sim_abstain": 0.59611 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5862066, + "sim_memory": 0.5496563, + "sim_abstain": 0.51394325 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6231395, + "sim_memory": 0.57401973, + "sim_abstain": 0.55732864 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6762118, + "sim_memory": 0.6369297, + "sim_abstain": 0.5889329 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.606107, + "sim_memory": 0.5466962, + "sim_abstain": 0.5397958 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70418376, + "sim_memory": 0.6688043, + "sim_abstain": 0.6164449 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.5306442, + "sim_memory": 0.45544416, + "sim_abstain": 0.45304802 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.50268793, + "sim_memory": 0.6389982, + "sim_abstain": 0.5120093 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.5960785, + "sim_memory": 0.5498394, + "sim_abstain": 0.5258497 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-6/router_report.md b/benchmarks/models/router/centroid/out-router-iter-6/router_report.md new file mode 100644 index 0000000..eeecf1b --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-6/router_report.md @@ -0,0 +1,87 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.100` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `80` memory: `46` abstain: `15` +- accuracy: `0.5390` +- tools precision: `1.0000` +- tools recall: `0.2250` +- memory false-tools rate: `0.0000` +- abstain false-tools rate: `0.0000` + +## Misclassified Queries + +| id | text | gold | pred | T-sim | M-sim | A-sim | +|---|---|---|---|---|---|---| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.71 | 0.69 | 0.62 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 | +| `shell_test_001` | what processes are listening on port 8003? | `Tools` | `Abstain` | 0.66 | 0.56 | 0.57 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | 0.65 | 0.54 | 0.58 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.71 | 0.62 | 0.68 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.68 | 0.58 | 0.60 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.66 | 0.60 | 0.55 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Abstain` | 0.61 | 0.53 | 0.54 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.71 | 0.65 | 0.58 | +| `shell_test_010` | what's the uptime since klbr daemon last started | `Tools` | `Memory` | 0.65 | 0.55 | 0.51 | +| `shell_test_011` | how big is agent.db right now on disk? | `Tools` | `Memory` | 0.62 | 0.54 | 0.53 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.65 | 0.56 | 0.55 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.66 | 0.57 | 0.62 | +| `shell_test_014` | how many memory rows are in agent.db right now? | `Tools` | `Memory` | 0.68 | 0.59 | 0.56 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.58 | 0.55 | 0.52 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.69 | 0.66 | 0.59 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.62 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.61 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.57 | 0.52 | 0.52 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.65 | 0.54 | 0.59 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.60 | 0.56 | 0.57 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.65 | 0.57 | 0.61 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.68 | 0.61 | 0.61 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.61 | 0.52 | 0.55 | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | 0.57 | 0.49 | 0.47 | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | 0.59 | 0.54 | 0.52 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.67 | 0.57 | 0.60 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.67 | 0.62 | 0.55 | +| `rf_test_002` | which file defines the ClientMsg type? | `Tools` | `Memory` | 0.53 | 0.44 | 0.44 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.66 | 0.65 | 0.53 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.60 | 0.56 | 0.54 | +| `rf_test_006` | config.rsのmemory_support_score_gapのデフォルト値はいくつ? | `Tools` | `Memory` | 0.71 | 0.64 | 0.55 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.70 | 0.65 | 0.59 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.60 | 0.56 | 0.50 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.65 | 0.63 | 0.55 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.57 | 0.53 | 0.50 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.69 | 0.69 | 0.58 | +| `rf_test_016` | read interrupt.rs and tell me what the Interrupt enum contains | `Tools` | `Memory` | 0.64 | 0.57 | 0.57 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.66 | 0.60 | 0.54 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.64 | 0.56 | 0.54 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.69 | 0.62 | 0.57 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.68 | 0.57 | 0.59 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.57 | 0.53 | 0.48 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.55 | 0.56 | 0.51 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.65 | 0.64 | 0.55 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.56 | 0.52 | 0.42 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.69 | 0.60 | 0.59 | +| `rf_test_031` | read the first 10 lines of agent.rs | `Tools` | `Abstain` | 0.60 | 0.52 | 0.54 | +| `wf_test_001` | set the compaction_keep to 15 in config.rs | `Tools` | `Memory` | 0.69 | 0.63 | 0.59 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.65 | 0.62 | 0.55 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Abstain` | 0.57 | 0.51 | 0.52 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Abstain` | 0.65 | 0.58 | 0.59 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.62 | 0.56 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.62 | 0.55 | 0.53 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.71 | 0.61 | 0.63 | +| `wf_test_008` | patch the memory_top_k to 5 in config.rs | `Tools` | `Memory` | 0.75 | 0.67 | 0.61 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.65 | 0.61 | 0.60 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.59 | 0.55 | 0.51 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.62 | 0.57 | 0.56 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.68 | 0.64 | 0.59 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.61 | 0.55 | 0.54 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.70 | 0.67 | 0.62 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | 0.53 | 0.46 | 0.45 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.50 | 0.64 | 0.51 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.55 | 0.53 | diff --git a/benchmarks/models/router/centroid/out-router-iter-7/router_model.json b/benchmarks/models/router/centroid/out-router-iter-7/router_model.json new file mode 100644 index 0000000..e690004 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-7/router_model.json @@ -0,0 +1,6168 @@ +{ + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "threshold": 0.05, + "memory_centroids": [ + [ + -0.02953457, + -0.023501687, + -0.05808627, + 0.0070140953, + 0.006874998, + -0.047377475, + 0.013503848, + 0.004033102, + 0.013937307, + 0.025993194, + 0.00036805047, + 0.026340032, + -0.00032751352, + 0.020008639, + 0.010437691, + -0.01599532, + 0.0378925, + -0.034139622, + 0.031576242, + -0.005721982, + -0.015650975, + -0.013299033, + 0.0015801851, + 0.05208871, + -0.0033491016, + 0.05292174, + -0.041163854, + -0.013427372, + 0.018127806, + 0.031762872, + 0.0080265, + 0.027610881, + 0.0102199325, + -0.02962109, + -0.025770932, + -0.027322631, + 0.018344872, + -0.01546087, + -0.040318888, + -0.024949342, + -0.02684663, + 0.019792331, + 0.016403709, + -0.03913061, + 0.041741397, + -0.051342268, + 0.03370111, + -0.041125838, + -0.0036108876, + 0.015794855, + -0.018666392, + -0.014134196, + 0.015846388, + -0.02152241, + 0.04721869, + 0.038556382, + -0.0057476372, + 0.01546458, + -0.04171342, + -0.007379216, + -0.0137405945, + -0.015948497, + -0.015352586, + 0.011297628, + 0.021778889, + 0.09534737, + 0.027794316, + -0.002793605, + -0.020545956, + -0.041483287, + -0.025141848, + 0.006380157, + -0.00047512646, + -0.0032311745, + -0.08699265, + 0.02291205, + 0.049483027, + -0.022220165, + 0.0006178035, + 0.015320561, + 0.047333814, + -0.006275665, + 0.034674358, + 0.014853668, + -0.0034956613, + 0.014414248, + -0.028953297, + 0.035040356, + 0.0029728715, + -0.031037042, + -0.010100751, + 0.023466574, + 0.013175919, + -0.052728992, + -0.046121895, + 0.0069394163, + -0.029717263, + -0.013612056, + 0.039096978, + 0.006514509, + 0.026238186, + 0.011335962, + 0.015135213, + -0.030236373, + 0.052462798, + -0.009011261, + 0.04137082, + 0.036443684, + -0.024784643, + -0.010885915, + 0.02339825, + 0.01153446, + 0.019820947, + 0.01667984, + -0.02987813, + -0.03124927, + -0.033248678, + -0.026805358, + 0.05532143, + 0.006061382, + 0.05987853, + 0.0053289263, + 0.06786469, + -0.03602914, + -0.015517126, + -0.04250494, + 0.020546658, + 0.012809727, + -0.00601842, + 0.008231562, + -0.004753981, + 0.0181044, + -0.02190586, + 0.0015789795, + -0.0386283, + -0.015395165, + 0.013448149, + -0.0020003526, + -0.026016362, + -0.05009787, + 0.033090923, + 0.029800063, + -0.030307502, + -0.027842177, + 0.0139083, + -0.019292444, + 0.008088462, + -0.011791759, + 0.013513014, + 0.0027809977, + -0.015044234, + -0.0142502105, + -0.011962004, + -0.013559485, + 0.025546454, + -0.0144509105, + 0.022906901, + 0.08040018, + -0.024976632, + 0.05750807, + -0.015728489, + 0.025379393, + -0.042068284, + -0.011159206, + 0.045110248, + 0.0027386474, + 0.01061638, + 0.037824012, + -0.015301461, + -0.010647753, + -0.019499803, + -0.016362367, + 0.029056897, + -0.016690247, + 0.02505767, + 0.020635538, + 0.10579964, + 0.017164556, + 0.010356794, + -0.04950358, + -0.047854308, + 0.012633971, + 0.020412799, + -0.03777352, + -0.019705521, + -0.021516891, + 0.037748206, + -0.05184238, + 0.015875751, + 0.032903414, + 0.00286073, + -0.0017315428, + 0.03207537, + 0.0024716682, + 0.036519893, + 0.019512344, + -0.005500415, + -0.028663289, + 0.006004013, + -0.028784506, + -0.009142338, + 0.037859693, + 0.063061036, + -0.007193967, + -0.059196513, + -0.00025405674, + -0.0404329, + -0.050101977, + 0.018440584, + 0.0019876033, + -0.018863054, + 0.01567874, + 0.02782502, + 0.0050265174, + -0.035830576, + 0.00025280323, + 0.024342729, + -0.0027864513, + -0.011164368, + -0.012470029, + -0.020127311, + 0.044781983, + -0.016764725, + -0.060266845, + 0.02845582, + 0.0326934, + 0.0023419242, + -0.0018993955, + -0.005613019, + -0.049475726, + -0.0010083856, + 0.024441615, + -0.022721602, + -0.024691395, + 0.034620766, + -0.00882457, + 0.0016938613, + -0.003250522, + 0.013695728, + -0.013721317, + 0.0021146091, + -0.028904684, + -0.0029565294, + -0.0042853206, + -0.05602909, + -0.0023269707, + 0.011781264, + 0.013156883, + 0.0032094966, + 0.013319434, + 0.027515475, + 0.00003426793, + -0.04807129, + -0.02646676, + 0.055103097, + 0.03457307, + 0.032444008, + -0.0027100584, + 0.024296409, + 0.0019906394, + 0.03768081, + 0.0022763326, + 0.011797177, + 0.036109027, + 0.026424909, + -0.0031692882, + -0.024819322, + -0.029096847, + 0.0043520858, + 0.0067918655, + -0.03163525, + 0.0048874402, + 0.022340473, + 0.023326822, + 0.03814576, + 0.026113065, + -0.03389849, + -0.035517395, + 0.05969392, + 0.01776524, + -0.012784303, + 0.013127809, + 0.022363255, + -0.013977365, + -0.0039680754, + -0.017222684, + -0.02858304, + -0.011452531, + -0.007811619, + -0.045651883, + 0.014498947, + -0.0013878772, + 0.100820094, + 0.005297464, + -0.028213173, + -0.03314273, + -0.04044471, + -0.22465427, + -0.02858451, + -0.003905871, + 0.00022481597, + 0.0047072093, + -0.012649562, + -0.017003896, + 0.006186442, + -0.03956627, + 0.036402363, + -0.051759057, + -0.068457276, + 0.007101277, + -0.0124830175, + -0.019360466, + 0.0033654436, + -0.011729769, + -0.009554513, + 0.018463207, + -0.03167895, + -0.006967882, + -0.019700885, + 0.023060178, + -0.020346511, + -0.0061216676, + -0.014901207, + 0.028405404, + 0.022600891, + -0.03975633, + -0.020692414, + 0.017461983, + 0.027074642, + -0.017387684, + 0.047598917, + 0.013716522, + 0.022675045, + 0.021145523, + -0.017724916, + -0.026462242, + -0.014224281, + 0.030817714, + 0.0114215035, + -0.021204576, + 0.034436375, + 0.024465613, + -0.045439, + -0.013454584, + -0.030626189, + -0.0077004014, + -0.0046026013, + -0.027327102, + 0.012236565, + 0.0023989333, + -0.0047771633, + -0.02984797, + -0.008682544, + -0.016193328, + 0.039879285, + -0.06660901, + 0.012391835, + -0.0049603703, + 0.0007947687, + 0.016125074, + 0.040164527, + -0.033208624, + 0.010781524, + -0.025258325, + -0.020662768, + 0.03224673, + -0.023760425, + 0.030194627, + -0.0016037951, + 0.0024073026, + -0.011888995, + -0.021232078, + 0.053651586, + -0.013994518, + -0.039026722, + -0.018290367, + -0.1595071, + -0.010143831, + -0.009503269, + 0.0075196954, + -0.0016893324, + -0.023991065, + -0.0004729232, + 0.005407145, + 0.045051403, + 0.03946149, + 0.31316122, + 0.019875187, + -0.017683508, + -0.011246197, + 0.041095726, + -0.01707569, + 0.02580701, + 0.020073302, + 0.0076661776, + -0.039371304, + -0.030318934, + 0.013346287, + 0.020821072, + 0.0046413513, + 0.016254276, + 0.005412236, + -0.03347545, + 0.010158307, + 0.1002082, + -0.011839762, + 0.03285164, + 0.012760006, + 0.02366014, + 0.022854084, + -0.05203756, + -0.03471872, + -0.009616535, + 0.058146782, + -0.020818148, + 0.03461769, + -0.026271882, + 0.021051634, + 0.02911293, + -0.01990685, + -0.041703526, + 0.0002907327, + 0.013849215, + 0.016160823, + 0.0024423078, + 0.03202622, + -0.0015427561, + -0.042227678, + -0.001353875, + -0.013348268, + -0.0008665041, + -0.042891808, + -0.006410304, + -0.02022797, + -0.03159226, + -0.022437904, + -0.02003756, + 0.015528935, + -0.017203221, + -0.0100704925, + -0.010856735, + -0.021080589, + -0.0504776, + 0.010053538, + -0.0030528405, + 0.027641593, + 0.027326025, + 0.0128613785, + -0.046285115, + -0.010928768, + -0.03181085, + 0.017708378, + -0.029218853, + 0.00059574813, + 0.031865608, + 0.013683232, + 0.0025548658, + -0.006452052, + 0.008344617, + -0.0018459023, + 0.04586065, + 0.007935226, + 0.00002709333, + 0.08221116, + -0.009100065, + 0.029175958, + -0.048444733, + -0.008251141, + -0.03405043, + 0.032034725, + 0.007967471, + -0.024803922, + -0.015552689, + 0.05681926, + 0.0037742213, + -0.026362995, + -0.007213026, + 0.014125172, + -0.03752151, + -0.0024824142, + 0.012550943, + 0.008332322, + 0.015813878, + -0.011721951, + -0.03736512, + -0.025358934, + -0.041780002, + -0.016802674, + 0.0075594457, + 0.012926832, + 0.0022665393, + 0.00544774, + -0.008000439, + 0.015729414, + 0.025527108, + 0.015055225, + -0.02852011, + 0.029998757, + -0.023629876, + -0.019701436, + 0.046159368, + 0.010666163, + -0.009261453, + 0.027201602, + -0.005710259, + 0.03726728, + -0.01976218, + -0.023621652, + -0.0029942847, + -0.00047694452, + -0.0040939855, + -0.0099202385, + 0.0065422435, + -0.0429584, + -0.031387664, + 0.022027241, + 0.026100025, + -0.007511666, + 0.030582216, + 0.012822923, + 0.036304314, + 0.025152108, + -0.017161628, + 0.016741537, + 0.018255189, + 0.035726435, + -0.0062032174, + -0.05698027, + -0.013355153, + 0.014536069, + -0.01766706, + -0.008239166, + 0.018501902, + -0.0010640995, + -0.018297568, + 0.019297598, + 0.01495364, + 0.036746204, + -0.02745614, + -0.055277172, + -0.029027686, + -0.032996736, + -0.010148253, + -0.0014721167, + -0.023568414, + -0.040075224, + 0.019336946, + 0.0005275909, + 0.006581408, + 0.05709383, + 0.039294206, + 0.040886384, + -0.032870255, + 0.040065862, + 0.03958012, + -0.042370867, + 0.0011705015, + -0.012909272, + -0.033453353, + 0.037587993, + 0.025471324, + -0.008906781, + -0.06545729, + 0.0017291277, + 0.008953089, + 0.016879518, + 0.026898118, + 0.027753143, + -0.03992264, + -0.004906459, + 0.0039307103, + 0.026820708, + -0.031330388, + -0.024806987, + 0.031612832, + -0.032069124, + -0.014307158, + 0.12864976, + -0.0051088952, + 0.004911811, + 0.005624591, + 0.010622269, + 0.014215928, + 0.012548023, + -0.030215545, + -0.029370805, + 0.0002464952, + 0.017944876, + 0.001236656, + -0.028763784, + 0.023505056, + 0.022390043, + -0.0015424239, + 0.027351622, + -0.04140385, + -0.030692495, + 0.0076649394, + -0.0017135493, + -0.046149958, + 0.0039872695, + 0.025925033, + -0.007340319, + 0.046820674, + 0.039672777, + -0.011907181, + -0.017452894, + -0.00021526583, + 0.0139043005, + 0.029623425, + -0.055875365, + -0.020226091, + 0.00017796963, + -0.030965574, + -0.06364141, + 0.013530132, + 0.008922023, + 0.014563754, + 0.0031031477, + -0.0030001134, + -0.030106964, + 0.03502163, + -0.0019424554, + -0.0020550822, + 0.03134754, + 0.0045823157, + -0.019614106, + -0.0048672026, + -0.022130122, + -0.009755478, + 0.03228172, + -0.019890655, + 0.008002598, + -0.00781725, + -0.011763561, + 0.01594538, + 0.03417074, + 0.011124715, + 0.00860674, + 0.0053270827, + -0.018849868, + -0.019411609, + 0.016141795, + 0.029577173, + -0.012376068, + -0.028442422, + -0.0102799805, + -0.015962493, + 0.04011505, + -0.035837717, + -0.013337596, + -0.050945375, + -0.013998839, + 0.04386832, + -0.025864763, + -0.03339192, + -0.047449946, + 0.004850084, + -0.015017307, + -0.062756956, + -0.04656785, + 0.021192448, + 0.00956155, + -0.05119442, + -0.008371866, + -0.0042842724, + 0.019156162, + -0.010939484, + 0.018840468, + -0.025657684, + 0.01106568, + -0.03244388, + 0.006420251, + 0.04826801, + -0.00057971664, + -0.006369808, + -0.031549037, + 0.049913317, + 0.007017293, + -0.030578466, + 0.02232669, + -0.04201414, + 0.026563568, + -0.015477179, + 0.026783573, + -0.022415739, + 0.016253402, + -0.025808347, + 0.052372962, + -0.05774024, + -0.011736604, + -0.010124982, + -0.0048620454, + -0.019803265, + -0.008477471, + 0.030738277, + 0.038898382, + 0.023150876, + -0.042604998, + 0.0019641898, + 0.00784871, + -0.004138505, + 0.011478454, + 0.030663332, + -0.01506021, + -0.02919572, + 0.012824615, + -0.0012030387, + 0.005259654, + -0.01432216, + -0.009896729, + -0.020819265, + -0.0027348117, + 0.00016687928, + 0.008219463, + -0.008309997, + -0.0117734615, + -0.017596774, + -0.007878184, + -0.011875525, + -0.011689367, + 0.048842385, + 0.0032761202, + -0.004180586, + -0.02784116, + 0.019389883, + 0.0059328643, + 0.017998025, + -0.0043417537, + 0.03363939, + -0.04814994, + 0.017516242, + -0.047206663, + 0.008795233, + 0.006181405, + -0.0076484066, + 0.021008575, + 0.0067271558, + 0.026135894, + -0.030862615, + -0.059556678, + -0.005249494, + -0.005896798, + -0.053860072, + 0.058404587, + 0.036468524, + 0.014157678, + -0.028333092, + 0.03760916, + 0.00519971, + 0.009662147, + -0.008499046, + 0.027133144, + 0.014945763, + -0.022028917, + 0.0066466467, + -0.012278338, + -0.025636343, + -0.0009813369, + -0.023540856, + -0.0014314126, + -0.016507363, + 0.045651384, + -0.0082562985, + -0.0041743293, + 0.05165129, + 0.025758864, + -0.029776005, + -0.012274611, + 0.01466031, + -0.038974468, + 0.0070445114, + -0.007370479, + -0.027545778, + -0.013694657, + 0.043757852, + 0.04633227, + -0.021911979, + -0.01796987, + -0.054099757, + 0.01452772, + -0.2138702, + 0.02743569, + 0.015744813, + -0.020886457, + -0.051552296, + -0.0027605426, + -0.0012711033, + -0.0047403597, + 0.009300366, + -0.029458603, + -0.02747486, + 0.025445733, + 0.0050143576, + -0.012034659, + -0.016319465, + 0.019868318, + -0.013839318, + -0.00029665322, + 0.009236322, + 0.018608531, + 0.0050179, + -0.017188085, + 0.03464569, + 0.005022059, + 0.02257357, + -0.026489973, + -0.035256382, + 0.0122413905, + -0.057871927, + -0.009198643, + 0.002898138, + 0.0062493277, + 0.032887958, + 0.047671497, + -0.026613321, + -0.005649091, + -0.0041015684, + -0.0281537, + 0.003319403, + -0.021404069, + -0.002427804, + 0.04324175, + -0.014130836, + -0.031917486, + -0.011499083, + 0.030707614, + -0.0028441278, + 0.018666685, + -0.06646727, + 0.019018704, + -0.004152146, + 0.01975428, + -0.013143967, + 0.0073776282, + 0.020081295, + -0.02697742, + -0.021839116, + 0.017184513, + 0.027456801, + 0.03954958, + 0.0016815886, + -0.016831884, + -0.027606057, + -0.050559748, + -0.051494483, + -0.012649485, + -0.07952543, + 0.004067631, + -0.0021464548, + 0.00786993, + -0.023989927, + -0.024670297, + -0.0037496786, + -0.04951884, + 0.016926184, + -0.018860383, + 0.012256827, + -0.010916814, + -0.010613337, + -0.025584185, + -0.0030827215, + 0.016111253, + -0.04742576, + -0.024564615, + 0.042982895, + 0.002380033, + -0.024394304, + 0.0062498963, + -0.009741223, + 0.00893389, + -0.06926393, + 0.017702833, + 0.003559851, + 0.023049349, + -0.012283555, + 0.03369362, + -0.012401988, + 0.0050276346, + -0.0166348, + -0.022107374, + -0.059628308, + -0.0036324025, + -0.011819672, + 0.026103102, + -0.014030879, + -0.03181233, + 0.03724609, + -0.010170614, + 0.0054339855, + -0.0066714897, + -0.03532086, + -0.015470396, + -0.019201776, + 0.010452439, + -0.035049714, + 0.008742536, + 0.017332299, + -0.0028451565, + -0.03990205, + 0.0027475304, + -0.000050377344, + 0.033374507, + -0.011835915, + -0.044426147, + -0.0012377469, + -0.0005448537, + 0.054194182, + -0.0016887394, + -0.009132979, + -0.014007341, + 0.048595123, + -0.024569178, + -0.010165418, + 0.039706923, + 0.009360816, + -0.0142467925, + -0.02289829, + 0.046355814, + -0.01972842, + -0.051768117, + 0.008552698, + 0.014160438, + 0.012196202, + -0.0075761625, + -0.019158937, + 0.0119345505, + 0.007685015, + 0.016935337, + -0.051301133, + -0.015521768, + 0.03698387, + 0.010485435, + 0.0012461421, + 0.009096625, + 0.009374426, + 0.004323882, + 0.022198806, + 0.025565172, + -0.023361158, + 0.059670344, + 0.036556274, + -0.038306747, + -0.034494575, + -0.04563513, + -0.0052286144, + 0.0026300608, + 0.022318833, + -0.0048725726, + -0.0037535578, + 0.0087553235, + -0.0037525538, + 0.052146267, + -0.02985571, + 0.07571844, + 0.014147077, + -0.050934263, + 0.041672062, + -0.0042912466, + 0.042178534, + -0.016792161, + 0.029752152, + -0.020074084, + -0.0091867, + 0.023213692, + 0.006531691, + 0.015507917, + 0.028167203, + 0.050762765, + 0.010745568, + 0.0008412688, + 0.01909061, + -0.009928744, + -0.0028815789, + -0.04545366, + 0.042247232, + -0.010490567, + 0.020409765, + 0.025339702, + -0.035115454, + 0.061531704, + -0.011911052, + -0.03500391, + -0.04524361, + -0.013206155, + -0.022409508, + -0.008728653, + -0.034059823, + -0.02689995, + -0.03300723, + 0.043033533, + -0.0290939, + 0.0121105155, + -0.02727977, + 0.0019806651, + 0.0068177576, + 0.00031189778, + 0.010153875, + 0.024259033, + 0.02276865, + -0.02847139, + -0.010577961, + 0.010186096, + 0.0016031668, + -0.04977359, + -0.034086004, + -0.009296076, + -0.015570667, + -0.007774638, + -0.006385688, + -0.011167805, + -0.021814104, + 0.03431478, + -0.037030302, + 0.032789752, + 0.015891775, + 0.00091708323, + 0.05153089, + 0.025300793, + -0.004147137, + -0.002914364, + 0.047010057, + -0.02837259, + 0.003133865, + 0.029014219 + ], + [ + -0.020254124, + -0.04116675, + -0.007372979, + -0.029964684, + 0.011190738, + -0.08159443, + 0.017661301, + -0.010246391, + -0.008209263, + 0.022077559, + -0.037495364, + 0.038561653, + 0.041310087, + -0.03827206, + -0.0030296666, + -0.04551725, + -0.0015584466, + -0.043835424, + 0.04111606, + -0.0074276403, + -0.014586625, + -0.034484424, + -0.031067863, + -0.0055164816, + 0.002870236, + 0.03311342, + 0.0054400344, + 0.020860758, + -0.012762298, + 0.020914592, + -0.037719376, + 0.019643847, + -0.026980707, + -0.01616462, + 0.0057277605, + -0.018851377, + 0.013205356, + 0.026758894, + -0.05611753, + -0.05673022, + -0.015809074, + 0.022022985, + 0.064844824, + 0.009851467, + -0.0074210535, + -0.02909688, + 0.012350218, + -0.022363547, + -0.01382957, + -0.00794799, + 0.010876325, + 0.051783428, + -0.035475973, + -0.0031464773, + 0.07489341, + 0.034771163, + -0.041550077, + 0.034477636, + -0.019864663, + 0.0066035762, + 0.025300935, + 0.041910917, + 0.018587649, + 0.020312272, + 0.017726174, + 0.073004626, + -0.005812487, + 0.016792096, + -0.025332022, + 0.0049700495, + -0.053332593, + 0.04201889, + 0.004391433, + -0.0021935226, + -0.09358781, + 0.03083754, + 0.019159859, + 0.004591123, + 0.02774286, + 0.04111074, + 0.04572332, + -0.03794063, + -0.008300797, + 0.01320226, + 0.016294988, + 0.032590497, + 0.014050795, + 0.03014458, + -0.024947153, + -0.028386062, + -0.0069098324, + 0.016842715, + -0.0002635318, + -0.052521348, + -0.056614153, + 0.0476653, + -0.002092335, + 0.0067270766, + 0.034543928, + 0.06673301, + -0.008618899, + 0.003757926, + 0.0049299044, + 0.04859857, + -0.028685993, + -0.047352344, + 0.06761084, + 0.0026073721, + -0.03719376, + -0.008147757, + 0.002335019, + -0.055548757, + -0.019844908, + 0.0003014616, + -0.04448501, + -0.053816702, + -0.06500549, + 0.033419482, + 0.023317087, + -0.012048277, + 0.045431584, + 0.0044442685, + 0.00503645, + -0.023623671, + 0.036882322, + -0.03137511, + 0.03476906, + -0.0004156654, + -0.03795088, + -0.035963524, + -0.058573436, + 0.029147573, + -0.04297972, + -0.011300198, + -0.04443065, + -0.010125407, + -0.035875123, + 0.005237153, + 0.018849581, + -0.04374896, + -0.006881187, + -0.01323515, + -0.020799844, + -0.028531378, + 0.03373202, + -0.019024812, + -0.023178076, + -0.013414461, + 0.022442773, + -0.022926392, + 0.017389378, + -0.00240939, + -0.05073249, + -0.044891775, + -0.015401027, + -0.023299117, + 0.018395206, + 0.024170302, + -0.013018237, + 0.014878734, + -0.032968245, + 0.008014404, + -0.018779429, + -0.045670286, + 0.008193353, + -0.024559846, + 0.057381175, + 0.0669771, + 0.044439804, + -0.0036185777, + 0.021347538, + 0.048340894, + 0.026817888, + 0.058665168, + 0.023300119, + 0.026641518, + 0.039867714, + 0.0049528927, + -0.0011300519, + -0.037390184, + -0.06264999, + -0.01630887, + 0.008078938, + 0.0010589276, + 0.020439938, + 0.025579099, + -0.02067246, + -0.02252045, + 0.00242239, + 0.027789189, + -0.028604046, + -0.010315909, + 0.045320645, + -0.008987895, + -0.023577353, + 0.032551922, + 0.04230767, + -0.003796409, + 0.00983521, + -0.022642719, + 0.004940319, + 0.01627678, + 0.047532525, + -0.06016555, + 0.011346706, + -0.005472442, + -0.018862773, + -0.035922226, + -0.010526478, + -0.015118645, + 0.014042482, + 0.009215666, + 0.01457269, + -0.030881861, + 0.015838949, + -0.021441704, + 0.033870716, + -0.0088826725, + -0.05239024, + -0.015585623, + -0.034875818, + -0.032844577, + 0.011226785, + -0.015373432, + 0.015121698, + -0.00072043814, + -0.0068776575, + 0.06292077, + -0.009777719, + -0.04709221, + -0.029894646, + 0.0027344727, + -0.008795243, + -0.017512025, + 0.04226805, + 0.025590824, + -0.008859926, + 0.04388061, + 0.05875191, + 0.0108080255, + -0.031908263, + 0.01467872, + -0.014533983, + 0.015212947, + 0.02041433, + 0.03890462, + -0.024720386, + 0.006734559, + -0.023964731, + 0.01345108, + 0.014990413, + 0.0030772437, + -0.058163464, + 0.047327667, + 0.00489635, + 0.005195013, + 0.0001313037, + -0.016700808, + 0.047128133, + 0.009359673, + 0.065170325, + 0.020720795, + 0.010661119, + 0.0096793715, + -0.007632, + -0.047226686, + 0.0056292997, + -0.018743971, + -0.021640094, + 0.010677068, + -0.033116397, + -0.0020473609, + -0.0034470889, + 0.048381247, + 0.04437471, + 0.024240568, + 0.005486285, + -0.05120708, + 0.0025768455, + -0.015763147, + 0.05396275, + -0.046657037, + -0.010277123, + 0.05113114, + 0.03595713, + -0.005959823, + -0.065834485, + 0.019947264, + 0.00816652, + -0.056714363, + -0.029086692, + -0.006467168, + 0.020627813, + 0.00316757, + -0.007847554, + -0.0034523793, + -0.06581499, + -0.124769256, + -0.0475397, + -0.02573181, + -0.027566018, + 0.012921132, + -0.036089245, + -0.04352001, + -0.015061321, + -0.038911663, + -0.014494528, + -0.02425402, + -0.05302295, + 0.018130956, + -0.017570471, + 0.003728436, + -0.022605387, + 0.020288603, + 0.0016745474, + 0.015943542, + 0.013685553, + -0.020636661, + 0.0014871317, + 0.036179543, + 0.0666887, + 0.018872928, + -0.011584086, + 0.0346419, + 0.0010764768, + 0.015926719, + -0.01975337, + 0.008983314, + 0.010039464, + -0.006696494, + 0.0674074, + -0.025392266, + -0.009440608, + -0.014110286, + 0.029129889, + -0.0061896862, + -0.016916197, + 0.017264564, + -0.023885023, + 0.014502518, + 0.014711948, + -0.024939105, + -0.03776957, + 0.01799964, + -0.025129559, + -0.040642563, + 0.01042123, + 0.011569937, + 0.02311178, + -0.009573609, + 0.022429014, + 0.0010554203, + -0.021534372, + 0.009123053, + 0.07520331, + -0.022060657, + 0.0064048725, + -0.026625117, + 0.016196016, + 0.004994355, + 0.05129052, + -0.07573883, + -0.019806353, + -0.0052977004, + -0.02585432, + -0.008690205, + 0.0102756955, + -0.0064524813, + 0.02608897, + 0.014926429, + -0.0153515255, + 0.014566396, + 0.027495226, + -0.05981297, + 0.013260812, + -0.017695569, + -0.103219524, + -0.060673494, + -0.015960708, + -0.01404399, + -0.000095332405, + 0.030360384, + 0.0070420117, + -0.0052885753, + 0.0328839, + -0.012404965, + 0.17907244, + 0.026853886, + -0.0055063153, + -0.019140543, + 0.022881936, + -0.0054624556, + -0.020305462, + -0.0040248292, + -0.008616788, + -0.023474207, + -0.039994232, + -0.051367916, + 0.0506136, + -0.010000171, + -0.02576667, + 0.03985849, + -0.0047355406, + 0.02381105, + 0.058388382, + 0.020415135, + -0.007172144, + 0.034434605, + -0.008059745, + 0.009382488, + -0.0031786328, + -0.04825492, + 0.0090839295, + 0.055844855, + 0.01206461, + 0.04200518, + 0.008154144, + 0.00574395, + 0.026797764, + 0.034558475, + -0.06504741, + -0.011473228, + -0.041974258, + 0.008139225, + 0.022806998, + 0.043154757, + -0.011877348, + -0.016530937, + 0.04076686, + 0.018341811, + 0.021837514, + -0.025804339, + 0.007451655, + -0.07720817, + -0.029209508, + 0.03716037, + -0.010401743, + 0.0071976758, + 0.025213012, + -0.044804268, + -0.021039924, + 0.0035612402, + -0.031583317, + -0.015959581, + -0.009585654, + -0.0027469269, + 0.020826988, + 0.0188638, + -0.005792691, + -0.023081234, + 0.0036363788, + -0.0061280904, + -0.0006764033, + 0.019075336, + 0.055916227, + 0.02952431, + 0.049550064, + 0.04268436, + -0.025420789, + 0.087470114, + 0.057473727, + 0.07013039, + 0.012614301, + 0.02380244, + -0.029213306, + 0.008438178, + -0.023142328, + -0.0111113805, + -0.020476148, + -0.009346526, + -0.01193406, + -0.006535818, + -0.025082134, + -0.008752324, + 0.0044953264, + -0.028365167, + 0.016156549, + 0.05029044, + -0.012121522, + -0.008995778, + -0.01904241, + 0.00071007677, + 0.023712004, + -0.03216477, + 0.012888373, + -0.00008797499, + -0.054210324, + -0.03657906, + 0.011120258, + -0.01213075, + -0.023088787, + 0.03602543, + -0.040457137, + -0.02300542, + -0.0150393285, + -0.01354496, + 0.0215167, + 0.05465438, + 0.028458241, + -0.0115915015, + 0.0059376894, + 0.023620417, + -0.057839744, + 0.013643552, + -0.022547174, + 0.017269548, + 0.03127004, + -0.07988349, + -0.05078919, + 0.011205213, + -0.026745658, + 0.0303635, + -0.03320978, + 0.019111408, + 0.023719456, + 0.012111504, + -0.003342989, + -0.0072348914, + 0.0030055698, + 0.0048688916, + 0.049683273, + 0.039333805, + 0.025220495, + 0.031403065, + 0.061865292, + 0.034643058, + 0.0126303295, + 0.001891789, + 0.0073859184, + -0.015541369, + -0.05020341, + -0.027491327, + 0.0065335324, + -0.00034843318, + -0.026190557, + 0.021528963, + -0.013683739, + 0.04296777, + -0.0032485006, + -0.014733479, + -0.011558493, + -0.0145478975, + 0.0024043447, + -0.029473959, + -0.0077641266, + 0.0071720267, + 0.017033016, + -0.021381047, + -0.018005215, + -0.022859164, + 0.0021507316, + 0.038735278, + -0.043056272, + 0.018944927, + 0.036730997, + -0.031711336, + -0.02730031, + 0.00054977584, + -0.01186276, + 0.051118262, + 0.014460639, + 0.028466199, + -0.021185437, + -0.06046371, + 0.02031086, + 0.031098083, + 0.02705517, + 0.00861357, + -0.023073023, + 0.040332895, + 0.0015463877, + 0.0057385997, + -0.047136843, + -0.020717619, + 0.035119373, + 0.019301428, + -0.011939856, + 0.06972406, + -0.06393006, + 0.021002274, + 0.01652762, + -0.0046303887, + -0.06746979, + -0.05053881, + 0.027266838, + -0.05120607, + 0.011592364, + -0.0007830686, + 0.026465403, + -0.061663467, + 0.076975554, + -0.025506932, + -0.021895196, + -0.019970143, + -0.007931396, + -0.030816864, + 0.01556704, + -0.012785671, + -0.010793555, + -0.0040854607, + 0.02405227, + -0.0097369645, + 0.003523927, + 0.061076246, + 0.0022672077, + -0.055630583, + 0.014753529, + 0.018235153, + -0.023194518, + -0.0654739, + 0.01683278, + -0.010466496, + -0.044046562, + -0.056332473, + 0.011695644, + 0.06960929, + 0.02427439, + -0.0040806774, + -0.03696452, + -0.029279653, + -0.014468639, + -0.0038482912, + 0.01170978, + -0.00013206947, + -0.0035896276, + -0.010911566, + 0.008625817, + -0.0061318427, + 0.011671429, + 0.049992807, + 0.002745769, + 0.0031697694, + -0.012702845, + 0.046768826, + 0.050013237, + -0.03934394, + -0.008645382, + 0.056805123, + -0.0068112477, + -0.02184254, + -0.039267957, + -0.015069269, + 0.00784059, + -0.010700507, + 0.029770492, + -0.0030366762, + 0.024371767, + 0.043727934, + -0.009476817, + -0.014411585, + 0.031865187, + 0.021500316, + -0.027218673, + -0.0048276596, + -0.009022345, + 0.008899309, + -0.023732318, + -0.04855289, + -0.032706082, + -0.08435636, + 0.0030194477, + 0.024007564, + -0.044421624, + 0.0057766563, + 0.04744126, + 0.020762306, + -0.016310865, + -0.0020700977, + -0.025132567, + 0.025563918, + -0.016010238, + -0.008089007, + 0.0058249095, + -0.038526654, + -0.01771741, + -0.025688788, + 0.015047329, + -0.0014794411, + -0.014519879, + -0.044879995, + -0.07035461, + 0.026975842, + -0.050283216, + 0.019084351, + 0.00018584942, + -0.018075237, + -0.032995265, + 0.051688373, + -0.009015666, + -0.022133581, + -0.01399461, + -0.03630475, + -0.008931077, + -0.019135477, + 0.042051293, + 0.055161092, + 0.020866865, + 0.030594286, + -0.018767308, + 0.023476098, + -0.005624278, + 0.0104467785, + -0.0336711, + -0.0070034605, + -0.023060434, + 0.007979617, + 0.016134398, + 0.055457216, + -0.036615178, + 0.0092480825, + -0.008378172, + -0.037848447, + -0.0020533525, + 0.013536509, + -0.025558382, + -0.034805316, + -0.03309949, + -0.03766695, + -0.038801335, + 0.01504471, + 0.04798496, + 0.03310903, + 0.012344884, + 0.010643564, + 0.02714804, + 0.011844159, + -0.017942522, + 0.035539888, + 0.031651203, + -0.042367086, + 0.03250244, + 0.019743834, + 0.0084450785, + 0.011004879, + 0.052851435, + -0.021892207, + 0.0071975337, + 0.013812723, + 0.0025931627, + -0.021047847, + -0.021003025, + 0.029310383, + -0.038826242, + 0.080266275, + -0.0007394495, + -0.004599728, + 0.035691377, + 0.007618633, + 0.0002773464, + 0.040907677, + -0.00071150775, + 0.02910722, + 0.026443645, + -0.007555279, + 0.039212402, + -0.054407403, + -0.03506119, + -0.07286829, + 0.022884298, + 0.02873324, + -0.014125804, + -0.013982294, + 0.004962992, + 0.030923832, + -0.01460802, + 0.006872219, + 0.0032361657, + 0.002350869, + -0.005383757, + -0.053609997, + 0.042491283, + -0.039554384, + -0.030929564, + -0.000628169, + -0.00087879057, + 0.040924307, + -0.013975817, + 0.041287072, + -0.032564737, + 0.01377594, + -0.099039905, + 0.0052515194, + 0.003380827, + -0.015652759, + -0.061392162, + -0.024736412, + -0.028811218, + 0.0068211644, + -0.00080923695, + -0.0145119205, + -0.006284979, + 0.023963012, + 0.030818338, + -0.019357484, + -0.026111864, + -0.018230068, + -0.00141907, + -0.03421442, + -0.014301486, + 0.04973083, + 0.016147241, + 0.0034317733, + 0.03079936, + -0.01633139, + 0.010416137, + 0.015177892, + -0.014367937, + 0.012394066, + -0.04084319, + 0.003077322, + -0.04215946, + 0.06567015, + 0.006222198, + 0.026923848, + 0.0003813074, + 0.021874087, + 0.029589854, + -0.021420741, + 0.032787703, + -0.03356351, + 0.02138357, + 0.027757693, + -0.014662574, + -0.013165381, + -0.030943656, + 0.023958629, + -0.016045582, + 0.029165346, + -0.03166436, + 0.0055215787, + 0.020196468, + 0.037619952, + 0.012425788, + -0.015718715, + -0.018558526, + -0.062954664, + -0.014405638, + -0.02019691, + 0.013439697, + -0.00090975413, + -0.03960515, + 0.01929136, + 0.014825985, + -0.011897969, + -0.07391725, + -0.0037334028, + -0.021778373, + -0.03345438, + -0.0048873485, + 0.03748898, + -0.006887464, + 0.033263568, + 0.026937481, + -0.026849639, + 0.015706737, + 0.004499359, + 0.026059866, + -0.030047212, + -0.035322253, + -0.013030768, + -0.09660178, + -0.020400189, + 0.014046086, + -0.040100582, + 0.023280697, + 0.0061984207, + -0.017089635, + 0.07199057, + -0.005007973, + 0.032175943, + -0.07026293, + 0.008396743, + 0.036889493, + -0.030547267, + -0.0665451, + 0.023350557, + -0.017915163, + -0.006056193, + -0.037609342, + -0.00739387, + -0.021556864, + 0.04219483, + -0.017056148, + 0.02643247, + -0.022912811, + 0.0022669423, + 0.022746345, + 0.0035540983, + 0.030907692, + 0.02280229, + -0.013390923, + 0.00013597019, + 0.023870043, + -0.022743633, + -0.0025300398, + 0.024396071, + 0.032017436, + -0.059623055, + -0.014403772, + 0.014883795, + 0.006013073, + -0.007427275, + -0.02678559, + -0.017337607, + -0.025133366, + -0.040649172, + -0.007375268, + 0.0024157013, + 0.051238976, + -0.011819185, + 0.02866894, + -0.0661211, + 0.0072578704, + 0.03772343, + 0.028456915, + -0.022677233, + -0.004822275, + 0.051148444, + -0.0075521176, + -0.0278938, + -0.0017976505, + 0.035391975, + -0.043084104, + 0.013498232, + 0.000140535, + 0.0081637185, + -0.014895366, + -0.01532493, + -0.023869285, + 0.03975661, + 0.029805588, + -0.0016607255, + -0.0066284174, + -0.047425393, + 0.06078541, + 0.009874399, + -0.01420618, + 0.012851728, + -0.056120105, + 0.056595568, + -0.0008234414, + -0.019897787, + 0.0016358016, + -0.001700002, + -0.046762414, + 0.009751421, + -0.009791319, + 0.039236393, + 0.006752303, + 0.009253158, + 0.03652968, + 0.033729088, + -0.006883442, + 0.011171952, + 0.015599616, + -0.027064255, + 0.0070749433, + -0.03184461, + 0.035316285, + -0.01576945, + 0.037113056, + 0.0086026555, + 0.05263313, + 0.065941855, + 0.025773102, + 0.041526187, + 0.014641708, + 0.032382395, + -0.012154955, + 0.0010549576, + -0.003009168, + -0.032207463, + -0.016061692, + -0.07052871, + 0.010312381, + 0.055001568, + 0.0047640097, + 0.01343715, + -0.036756456, + 0.06276159, + 0.0071619395, + -0.019194365, + -0.025343249, + -0.014627754, + 0.0021465474, + 0.01780317, + -0.015829839, + -0.058133774, + -0.030980356, + 0.0140177235, + -0.05612922, + 0.019690728, + -0.05656899, + 0.010644363, + 0.017211964, + -0.035414055, + 0.011559114, + -0.036985178, + 0.050859265, + -0.015278783, + -0.024635246, + -0.0018933638, + 0.0024701187, + -0.008400519, + -0.0036219582, + -0.002365139, + 0.005002786, + 0.004347757, + -0.0018842067, + -0.009879196, + -0.00094722357, + 0.04200999, + 0.007078176, + 0.023403713, + -0.004958603, + -0.020193184, + -0.03242245, + 0.013797558, + -0.010586125, + -0.04184174, + 0.07585164, + 0.030954061, + 0.004254381, + 0.044182718 + ] + ], + "tools_centroids": [ + [ + -0.05606899, + -0.0037082054, + -0.0406634, + 0.00789242, + -0.019758664, + -0.009364881, + 0.019557, + -0.009831778, + -0.00949407, + 0.007475803, + 0.009658697, + 0.017726261, + 0.005106813, + 0.025193945, + 0.014655598, + -0.018157613, + 0.019189123, + -0.013696539, + 0.013375534, + -0.030809665, + -0.010917298, + -0.017136682, + -0.0133627625, + 0.047115013, + 0.018693825, + 0.041793875, + -0.035732128, + -0.030892752, + 0.03770479, + 0.048461147, + 0.007146342, + -0.018489974, + 0.0021626442, + -0.055255502, + -0.033207137, + -0.03183328, + 0.023630941, + -0.008898392, + -0.06518762, + -0.009473217, + -0.02444353, + 0.009674398, + 0.007890898, + -0.042801153, + 0.021430025, + -0.038007095, + 0.02450929, + -0.040193666, + 0.0003781031, + 0.0039781453, + -0.026794637, + -0.015793024, + 0.019999465, + -0.03834564, + 0.029724747, + 0.051441368, + -0.033698015, + 0.0012862714, + -0.05337915, + -0.000660187, + -0.028856108, + -0.019998565, + -0.00937057, + 0.012524584, + 0.005108214, + 0.091537096, + 0.02503071, + -0.0009231426, + -0.0117032025, + -0.017811712, + -0.044022832, + 0.021416029, + -0.03316437, + -0.03286693, + -0.0865588, + 0.035949722, + 0.043933157, + -0.02726403, + -0.01020989, + -0.0060232165, + 0.050451577, + -0.0009674949, + 0.014217209, + 0.0017161807, + 0.020335244, + 0.052320678, + -0.043524, + 0.03319889, + -0.006663756, + -0.007431745, + -0.028143626, + -0.0012573685, + 0.004024065, + -0.044356935, + -0.051310178, + -0.015289323, + -0.030234639, + 0.007162317, + 0.03509846, + 0.0075056385, + 0.016321585, + 0.002279486, + 0.008136665, + -0.013070722, + 0.047912866, + 0.0038850524, + 0.03659392, + 0.018676, + -0.027126975, + -0.008610971, + 0.003703314, + 0.021906741, + 0.017196443, + 0.0083635105, + -0.022649636, + -0.0427472, + -0.035220575, + -0.049825616, + 0.020608898, + -0.008786186, + 0.05120426, + -0.0076744757, + 0.067492664, + -0.042086206, + -0.019211868, + -0.034554653, + 0.038636506, + 0.054618992, + 0.016760439, + 0.03614989, + 0.0020738828, + 0.035541587, + -0.009483902, + 0.010591545, + -0.03670953, + -0.02300192, + 0.005059944, + 0.0033167151, + -0.008413337, + -0.06675143, + 0.021428753, + 0.057771403, + -0.036993712, + -0.038098555, + 0.03635651, + -0.05102609, + 0.023876203, + 0.004495377, + 0.009017165, + -0.004652149, + -0.018913463, + 0.0013396946, + 0.010036981, + 0.00037534232, + 0.000408126, + -0.013845552, + 0.007976019, + 0.05612355, + -0.022099286, + 0.06489436, + -0.0016591164, + 0.010839845, + -0.025517747, + -0.018433146, + 0.030932501, + -0.0146434, + 0.0077389157, + 0.0014314591, + -0.0047261086, + -0.01379333, + -0.00672064, + -0.028489156, + 0.015116883, + -0.012347785, + 0.02713658, + 0.02815122, + 0.09493939, + 0.023494598, + 0.011286576, + -0.05920509, + -0.046676517, + 0.020933578, + -0.009062939, + -0.007043584, + -0.034089938, + -0.0074305055, + 0.0435831, + -0.010437014, + 0.0077175335, + 0.013643179, + 0.00032880338, + 0.012419136, + 0.028978484, + 0.0027938702, + 0.033168048, + 0.017302865, + -0.001537013, + -0.023324333, + -0.023725992, + -0.035093762, + -0.024574025, + 0.031602282, + 0.043686762, + 0.0043900036, + -0.039813936, + -0.010593189, + -0.0121115735, + -0.034151588, + 0.009565826, + -0.038808856, + -0.012305442, + -0.00027281337, + 0.028179586, + 0.0022837583, + -0.039286092, + -0.0042402116, + 0.008249791, + 0.010657445, + -0.031379938, + -0.031366322, + -0.023170266, + 0.054060068, + 0.0048521874, + -0.01564455, + 0.01751972, + -0.0055304905, + 0.023394609, + 0.012396702, + -0.0003636368, + -0.022696953, + -0.023217125, + -0.006895921, + -0.008959929, + -0.018518882, + 0.071185775, + -0.0023356022, + -0.014435151, + 0.0037555092, + 0.025933823, + -0.00043771358, + -0.023615904, + -0.021824772, + 0.006545141, + -0.0036427176, + -0.044271503, + 0.021339668, + 0.044955056, + 0.033640694, + -0.017741505, + 0.0050410517, + 0.029788665, + -0.009972994, + -0.044430643, + -0.0060587483, + 0.03381109, + 0.032378227, + 0.029204395, + -0.01285328, + 0.011239541, + 0.011089111, + 0.032791503, + 0.01966721, + 0.01709499, + 0.04467105, + 0.006781909, + -0.0062364032, + -0.022574946, + -0.0069035026, + 0.0080041215, + -0.010154674, + -0.033628866, + 0.02108642, + 0.028107613, + 0.0055993693, + 0.017272362, + 0.0064405464, + -0.024938427, + -0.0274226, + 0.05751032, + 0.023648951, + -0.0069635683, + 0.021213165, + 0.008942416, + 0.0031823711, + 0.0020612366, + 0.0017135177, + -0.03745275, + -0.01224723, + -0.0092840865, + -0.039438073, + -0.020603599, + -0.009839146, + 0.10459197, + -0.0013756665, + -0.015315762, + -0.0004424063, + -0.035673883, + -0.22716926, + -0.011021554, + 0.009104081, + 0.033588458, + 0.007958396, + -0.00556822, + -0.00057971233, + 0.0043771993, + -0.041701302, + 0.024739638, + -0.03301154, + -0.0772926, + -0.029319951, + -0.035250075, + -0.0018914198, + -0.006261339, + -0.0055478225, + -0.006832176, + 0.023415668, + -0.039532892, + -0.020574547, + -0.012577133, + 0.05117778, + -0.0044079097, + -0.014995264, + -0.015555239, + 0.048599504, + 0.009827435, + -0.036604106, + -0.009881144, + 0.005024552, + 0.006581829, + -0.005089212, + 0.030867914, + 0.012545012, + 0.03213094, + 0.009552479, + -0.011707572, + -0.015692463, + -0.008529624, + 0.019617837, + 0.042746853, + 0.0017175317, + 0.037256774, + 0.041178584, + -0.03710732, + 0.012643423, + -0.028216833, + -0.016226137, + -0.008176746, + -0.0021341608, + -0.004784728, + 0.0022153086, + -0.027044365, + -0.0452304, + -0.00092170114, + -0.0344755, + 0.039659843, + -0.03179438, + -0.0008682657, + -0.030181633, + -0.0069265706, + 0.050441854, + 0.017631147, + -0.024619931, + 0.020767754, + 0.016655464, + -0.01860972, + 0.038021646, + -0.023390938, + 0.031038294, + 0.0038111766, + 0.0005345677, + -0.026021777, + -0.007115485, + 0.048046198, + -0.031886622, + -0.035144318, + -0.0026188153, + -0.18630543, + -0.032157507, + 0.018465653, + 0.0007996492, + 0.015417364, + -0.034394316, + -0.0075861984, + -0.0037427046, + 0.020296175, + 0.048781425, + 0.3208829, + 0.017051741, + -0.019397799, + 0.009009308, + 0.053919844, + -0.016741762, + 0.006250598, + 0.035504892, + -0.0060608485, + -0.032443427, + -0.00053168356, + 0.0076522306, + 0.025676573, + -0.0021505675, + -0.007423398, + 0.021114204, + -0.031733297, + -0.0011500829, + 0.100127555, + -0.004869494, + 0.029603617, + -0.0041950424, + 0.006506095, + 0.023717992, + -0.052787792, + -0.040853288, + -0.011165942, + 0.056000806, + -0.025902374, + 0.045923546, + -0.014623309, + 0.018445712, + 0.025848486, + 0.0029045863, + -0.053331148, + -0.018678037, + 0.011694079, + -0.013297034, + -0.0096364105, + 0.032539476, + -0.029562913, + -0.047789693, + 0.0026963938, + 0.025598904, + 0.006072212, + -0.03891818, + 0.017167594, + -0.0036721022, + -0.028613243, + -0.018510707, + -0.034060024, + -0.027170384, + -0.022034682, + 0.0024952486, + -0.0015357194, + -0.03533966, + -0.020739295, + -0.03143169, + -0.011211882, + 0.049816895, + 0.004694738, + 0.010594703, + -0.06420896, + 0.029159075, + -0.0025625115, + 0.041708685, + -0.030052194, + -0.014590074, + 0.012761965, + 0.014360401, + 0.013680634, + -0.010967563, + 0.005759606, + -0.011701792, + 0.05404799, + 0.019268606, + -0.021743063, + 0.06309377, + -0.019261587, + 0.048059467, + -0.024920318, + 0.010070914, + -0.047579683, + 0.026595078, + 0.02419197, + -0.018060323, + -0.014325064, + 0.026749337, + -0.02670927, + -0.03856533, + -0.008656995, + -0.0010423756, + -0.027002128, + 0.01249798, + 0.01479696, + 0.003004412, + 0.013617141, + -0.004383087, + -0.038900018, + -0.016454257, + -0.03198517, + -0.017905517, + -0.006894297, + -0.02221281, + 0.018678682, + -0.0025110545, + 0.00062055915, + 0.011711673, + 0.029655354, + -0.0033267613, + -0.030383127, + 0.033008993, + -0.011483914, + -0.039945323, + 0.032434426, + 0.016122635, + 0.0034836135, + 0.021274103, + 0.02966441, + 0.036415953, + -0.0307599, + -0.013897553, + -0.0090347715, + 0.008639756, + -0.027077066, + -0.031259153, + 0.00041659162, + 0.008344521, + -0.055489786, + 0.043384776, + 0.016839493, + -0.019988367, + 0.017604688, + 0.010117987, + 0.011109807, + 0.004411769, + 0.0075515187, + 0.024973173, + -0.006590006, + 0.059543025, + -0.011032581, + -0.05263824, + -0.023044582, + 0.010140496, + -0.0036169202, + -0.0006291355, + 0.04357636, + -0.016806297, + -0.021199094, + 0.010797508, + 0.0037002466, + 0.048935954, + 0.025420306, + -0.068292305, + -0.014495878, + -0.033696596, + -0.016170561, + 0.010531388, + -0.014046231, + -0.038196664, + 0.014447582, + -0.0023581628, + 0.024200277, + 0.07188329, + 0.026538994, + 0.010875151, + -0.04558475, + 0.026683357, + 0.025531683, + -0.033252317, + -0.0019728958, + -0.033328198, + -0.061308175, + 0.04225168, + 0.0017407219, + -0.018591579, + -0.043852597, + -0.0018788553, + -0.016940724, + 0.0065283165, + 0.04805281, + 0.014803647, + -0.044855554, + 0.014711851, + 0.014296282, + 0.03147689, + -0.0071145403, + -0.016696168, + 0.014320578, + -0.024143968, + -0.032781575, + 0.11330352, + 0.014250864, + 0.0010048777, + 0.010561461, + 0.012848797, + 0.009197347, + 0.027354443, + -0.009164351, + -0.017722908, + 0.001937092, + -0.007415777, + 0.003458327, + -0.040217895, + 0.005049932, + -0.013077656, + -0.013231001, + 0.015766662, + -0.009500496, + -0.016540406, + 0.0049814247, + -0.014900655, + -0.002444683, + -0.002461634, + 0.044309385, + 0.0012398985, + 0.04664923, + 0.05246022, + -0.010509221, + -0.015866991, + -0.01696081, + -0.003949499, + 0.053042635, + -0.06599373, + -0.007691855, + 0.017483734, + -0.012901056, + -0.046527628, + 0.018579615, + 0.012297996, + 0.013650844, + 0.003290136, + 0.0076800594, + -0.033333212, + 0.03522728, + -0.028282017, + 0.0096138, + 0.040518165, + 0.00096373906, + -0.031569775, + 0.024419462, + -0.01790975, + -0.0033231927, + 0.039167337, + -0.006741734, + 0.056704193, + -0.02368793, + 0.012585808, + 0.030043406, + 0.034530554, + -0.013871013, + 0.0013567432, + 0.0069410847, + -0.013012229, + -0.01794545, + 0.03237499, + -0.006095536, + -0.0015028969, + -0.022055492, + -0.0075068753, + -0.020075902, + 0.026405957, + -0.042370167, + -0.002402599, + -0.027732788, + -0.025120504, + 0.035065226, + -0.029773757, + -0.02554472, + -0.06785252, + -0.0140207475, + -0.009677745, + -0.0526383, + -0.011321278, + 0.038069192, + 0.015819497, + -0.035158947, + 0.00539325, + 0.020147596, + 0.026790883, + -0.026228523, + 0.011281811, + 0.0004322955, + 0.03357598, + -0.0312409, + 0.02051756, + 0.008903391, + -0.02679297, + 0.0059814607, + -0.039614767, + 0.02225511, + 0.0032299664, + -0.039062347, + 0.034572784, + -0.04000374, + 0.013121305, + -0.006740711, + 0.015605714, + -0.024614284, + 0.0182156, + -0.02464988, + 0.022435531, + -0.0540118, + -0.0107686315, + -0.01670897, + 0.015968842, + -0.025021473, + 0.004982866, + 0.044434078, + 0.0444515, + 0.03769919, + -0.026614936, + -0.0057781814, + 0.04286719, + 0.012986099, + -0.00009658893, + 0.057660222, + 0.002628925, + -0.0145921465, + 0.0028970172, + -0.00004345907, + 0.00540906, + 0.000009923756, + -0.01587657, + -0.013682616, + -0.0055716606, + -0.015776113, + 0.0040675635, + -0.02399378, + -0.024579993, + -0.02416092, + -0.021938631, + -0.0018802591, + -0.019586174, + 0.006322869, + -0.0037580882, + 0.011118912, + -0.00688259, + 0.046543375, + 0.0020257356, + 0.00915174, + -0.012346758, + 0.030707782, + -0.029368373, + 0.01678603, + -0.020684225, + 0.0025972757, + -0.012388426, + -0.008840013, + 0.018587714, + 0.009576551, + 0.029996252, + -0.0339124, + -0.069902465, + 0.004762207, + -0.005107531, + -0.04803004, + 0.007623275, + 0.024682391, + 0.011488783, + -0.019897707, + 0.038231988, + -0.004208973, + 0.01619736, + 0.02647576, + 0.011420114, + 0.016564004, + 0.010380385, + 0.015293031, + -0.015646528, + -0.00861725, + -0.011084581, + -0.010725576, + 0.0021767458, + -0.01672313, + 0.03839429, + -0.013438732, + -0.013672219, + 0.03522901, + 0.0024493937, + -0.009887151, + -0.03008308, + -0.026761537, + -0.020664144, + 0.022931337, + -0.008237708, + -0.0072052367, + -0.015670665, + 0.040643997, + 0.026935095, + -0.036957294, + -0.010053467, + -0.04384358, + 0.022895643, + -0.2105204, + 0.022694126, + 0.00997941, + -0.013189658, + -0.056611195, + 0.008618698, + -0.012578096, + -0.019372476, + 0.0086058965, + -0.037407827, + -0.042723864, + 0.008701772, + 0.025598338, + -0.029506119, + 0.018346047, + 0.024427475, + 0.013028314, + 0.011439589, + 0.0073115127, + 0.031192467, + -0.015018622, + -0.031565055, + 0.0016912352, + 0.025967559, + 0.0075193015, + -0.010706298, + -0.007327034, + 0.005763416, + -0.04706705, + -0.004623551, + 0.014159363, + -0.019997144, + 0.03497208, + 0.06485894, + -0.026932962, + -0.010051237, + 0.013179083, + -0.029087199, + -0.022257665, + -0.015966833, + -0.012313927, + 0.034779385, + -0.01645768, + -0.011849744, + 0.016278874, + 0.03483635, + -0.0043797335, + 0.0011064508, + -0.05597814, + 0.013468208, + -0.0057655373, + 0.04608973, + -0.031591564, + 0.00081813696, + 0.02271715, + -0.016193483, + -0.048024204, + 0.024269635, + -0.010684908, + 0.03037921, + -0.015080028, + 0.00577765, + -0.035579298, + -0.074489966, + -0.04568095, + -0.032743674, + -0.08463265, + 0.012373603, + 0.004400696, + 0.029980667, + -0.017966386, + -0.038213238, + 0.0057509523, + -0.025879731, + 0.017900858, + -0.019159615, + 0.013998882, + -0.0017255883, + -0.027130362, + -0.02789793, + -0.033315383, + 0.013056893, + -0.056488942, + 0.0058665094, + 0.060071133, + -0.009120229, + -0.003342009, + 0.0047433944, + -0.018858049, + 0.007472259, + -0.052950244, + -0.0039435257, + 0.00009774961, + 0.02106769, + -0.01170398, + 0.014035189, + -0.03197733, + -0.0038070253, + -0.014319388, + 0.0053357556, + -0.017116, + -0.0016028484, + 0.0019584813, + 0.015731098, + -0.014593648, + -0.009505084, + 0.06416616, + -0.022955375, + 0.016944945, + -0.015723076, + -0.030711906, + -0.0011726163, + -0.014695939, + 0.0066800653, + -0.046726193, + -0.0039578825, + 0.028575825, + -0.019372286, + -0.029086906, + -0.017561322, + -0.00428551, + 0.014641752, + 0.007259805, + -0.04302052, + 0.0039681853, + -0.00029460277, + 0.038807478, + 0.006130545, + -0.0048487694, + -0.018746726, + 0.039638285, + -0.03210769, + -0.012196666, + 0.029514477, + 0.030580895, + -0.011591039, + -0.013160387, + 0.066982895, + -0.019542642, + -0.053750712, + -0.023531597, + 0.0046885437, + 0.0008201537, + -0.019609794, + -0.00034986206, + -0.014274548, + -0.0038191855, + -0.013088327, + -0.042322893, + -0.029365066, + 0.051891316, + -0.0026843995, + 0.017064247, + 0.016581565, + 0.0013990946, + 0.0052255765, + -0.015575843, + 0.02221455, + -0.010577843, + 0.05188453, + 0.027795833, + -0.034560814, + -0.016681876, + -0.028297098, + 0.010590897, + 0.019287696, + 0.00036446203, + -0.01720634, + -0.037081674, + 0.0033696124, + -0.0013072222, + 0.030434122, + -0.03360222, + 0.055518307, + 0.023288932, + -0.04746223, + 0.041459426, + 0.014260388, + 0.06371192, + -0.0015586383, + 0.023453446, + -0.013434582, + 0.01792418, + 0.00756145, + 0.012596964, + 0.017840264, + 0.026810404, + 0.033809863, + 0.01566813, + 0.018697273, + 0.018435918, + -0.0015165473, + 0.0023415266, + -0.018216006, + 0.036613736, + 0.0055582654, + 0.01461495, + 0.022472767, + -0.025861593, + 0.06232265, + -0.021595163, + -0.03394643, + -0.03501842, + -0.025740786, + -0.028258167, + -0.011435451, + -0.026583094, + -0.028726535, + -0.033326574, + 0.04048446, + -0.028059425, + 0.006273973, + -0.024928257, + 0.0059108674, + -0.005911225, + 0.020267177, + 0.024992893, + 0.029467724, + 0.009110243, + 0.010474011, + 0.005127083, + 0.025516486, + 0.01224543, + -0.04925422, + -0.016988033, + -0.030158473, + -0.00012626215, + -0.0070728445, + 0.00432526, + -0.009610042, + -0.024404833, + 0.009735499, + -0.04270004, + 0.03474741, + 0.025382044, + -0.002434115, + 0.035717692, + 0.034531046, + -0.020223275, + -0.024045918, + 0.03205907, + -0.01690001, + 0.008418649, + 0.016847905 + ], + [ + -0.0777926, + -0.016130753, + -0.030252252, + 0.013064685, + 0.012630054, + -0.033431668, + -0.020723045, + -0.010652609, + 0.014717416, + 0.029943759, + -0.006884425, + -0.050565906, + -0.020473206, + 0.028232329, + -0.008742489, + -0.006928586, + -0.01848413, + -0.0062769433, + 0.0249494, + -0.026330234, + -0.044125106, + -0.027761683, + -0.047089003, + -0.018802904, + -0.0031594674, + 0.06795997, + 0.011664632, + 0.0049165445, + -0.008060829, + 0.012408625, + -0.0238953, + -0.02757731, + -0.026854679, + 0.0010435038, + -0.020860666, + 0.017221682, + 0.014673638, + 0.029738, + -0.011689957, + -0.015324875, + -0.010424466, + -0.028959932, + -0.0026987449, + -0.06396346, + -0.0044665886, + -0.051129714, + 0.008419665, + -0.016844666, + 0.03334108, + 0.008317175, + -0.015411966, + 0.007842717, + 0.061229803, + -0.05119574, + -0.013707167, + 0.012232052, + 0.012225783, + 0.045153536, + -0.0008650962, + -0.016156076, + -0.046021163, + -0.033580557, + -0.01686916, + -0.0030913, + 0.033622023, + 0.01923576, + -0.022195347, + -0.010797742, + -0.029916009, + -0.0004705949, + -0.039619286, + -0.013173107, + 0.06153379, + 0.006591758, + -0.058220953, + -0.04044149, + 0.034537926, + -0.016714765, + -0.0033219494, + 0.016891837, + 0.028455667, + -0.03353745, + -0.0025109896, + 0.0013168402, + 0.02191537, + 0.00989691, + -0.0076760734, + 0.053305842, + -0.011090245, + 0.05804909, + -0.0053123464, + -0.009720489, + 0.033220656, + -0.048008375, + -0.07702685, + 0.004185979, + 0.019034842, + 0.005118874, + 0.016340083, + 0.006550299, + 0.0013390492, + -0.0079578, + -0.017740352, + -0.029226499, + 0.03548014, + -0.030715493, + 0.035450455, + 0.002739102, + 0.008236122, + 0.0070569282, + 0.0021077737, + 0.010215997, + 0.0006403754, + 0.020998295, + -0.022850262, + -0.039860018, + -0.022919796, + 0.003358362, + -0.015362032, + -0.022922192, + 0.034358162, + -0.053381287, + 0.03479484, + -0.014682795, + 0.046078928, + -0.05120507, + -0.033427823, + 0.018008124, + -0.029796828, + 0.03957431, + -0.009962059, + 0.0014587059, + 0.035613198, + 0.023544552, + -0.0035347848, + -0.021168783, + 0.02405471, + 0.022278832, + -0.0129754245, + -0.04674764, + -0.035395015, + 0.0060356203, + 0.009195937, + -0.012415938, + 0.041733548, + -0.0042112567, + -0.0054666842, + -0.022617701, + 0.024861094, + -0.0014649403, + -0.021593269, + 0.010473995, + -0.006896913, + 0.012945232, + 0.038368367, + -0.021349257, + -0.023133717, + 0.03926201, + -0.006479839, + 0.108185984, + -0.023711825, + 0.035648413, + -0.020544553, + 0.028756576, + -0.012647061, + -0.0068361103, + -0.009214789, + -0.01819023, + -0.024630189, + -0.0158221, + -0.0076070474, + 0.016997859, + 0.028996132, + 0.0028848278, + 0.012588545, + 0.012869404, + 0.022341216, + 0.043853626, + -0.03490951, + -0.04219079, + -0.036706578, + 0.006829303, + 0.029833514, + -0.04073565, + -0.040228482, + 0.040827464, + 0.018223649, + 0.0068636555, + 0.023727717, + 0.013143164, + -0.024592938, + -0.0046767895, + 0.0073383558, + 0.065363534, + -0.054043908, + 0.010578177, + -0.0060408995, + 0.022571096, + -0.022589117, + -0.011779923, + 0.02198382, + 0.046569698, + 0.030388758, + 0.00009530159, + -0.025261797, + -0.004875701, + 0.016566563, + -0.024309523, + -0.04492145, + -0.040620018, + -0.008033184, + 0.008486595, + 0.0036873387, + -0.006312519, + -0.057184577, + -0.027954899, + -0.026917579, + -0.023880664, + -0.012670563, + -0.022808496, + 0.017791497, + 0.0046672984, + -0.021858089, + 0.017631855, + 0.023283772, + 0.025225436, + 0.03048242, + 0.008289526, + -0.027709194, + 0.022788085, + -0.037033916, + -0.019281274, + 0.042843334, + -0.020052629, + 0.062315274, + 0.017621454, + 0.0023051042, + -0.012937805, + 0.03945034, + 0.01578884, + -0.04974779, + -0.05110533, + 0.040397394, + -0.02760902, + -0.029570848, + -0.028894164, + -0.023790289, + 0.020278046, + -0.021137215, + -0.009141063, + 0.005414481, + -0.019659275, + 0.010410835, + 0.03108242, + 0.019945964, + 0.019414624, + 0.001831312, + -0.0044669346, + 0.051963087, + -0.0041192234, + 0.061908115, + 0.033783134, + -0.029776793, + 0.014804269, + 0.021261247, + -0.0008398359, + -0.062825724, + 0.008026716, + 0.0017601701, + 0.016118582, + 0.0132596325, + 0.026431667, + 0.04182074, + 0.05865137, + 0.020127004, + 0.009281466, + -0.0050892252, + -0.049665384, + 0.054135658, + 0.0067237294, + -0.018110242, + 0.034775, + 0.031828076, + 0.014316956, + -0.00709445, + 0.016639711, + 0.056842767, + 0.031007824, + 0.008321629, + -0.051327795, + -0.03239476, + 0.052417755, + 0.079580046, + -0.0044072163, + -0.025062976, + -0.034690555, + 0.0017639417, + -0.12477049, + 0.0010641285, + 0.037374467, + -0.0078232195, + -0.021236846, + -0.008217301, + -0.024288183, + 0.03848377, + -0.036963657, + -0.029888399, + -0.06894421, + -0.059106227, + -0.03341434, + -0.02493872, + 0.027394043, + 0.022943368, + -0.06030709, + -0.024837509, + -0.014572309, + -0.035931725, + 0.017025314, + -0.038330782, + 0.029428963, + -0.031423945, + 0.014545969, + -0.020500131, + 0.013090049, + -0.015320569, + -0.067829646, + -0.032127842, + -0.03852837, + 0.01132043, + 0.016142247, + 0.04502109, + 0.011946692, + 0.036663942, + 0.004616084, + -0.0040293597, + 0.008882638, + -0.0050679436, + 0.030989021, + -0.011439283, + -0.027428659, + 0.010920439, + 0.010191991, + 0.025236871, + 0.0374206, + -0.0040066508, + -0.025254825, + 0.009245347, + -0.00990044, + 0.012132254, + 0.0039520827, + 0.0044951704, + 0.00659937, + -0.0025982815, + -0.01390185, + 0.025864433, + -0.007912151, + -0.038404647, + -0.014624314, + 0.05410332, + -0.012991158, + 0.017253352, + 0.015497101, + 0.0034109775, + 0.015372937, + -0.033209722, + 0.036295604, + 0.014939996, + -0.02805046, + 0.0015722377, + 0.017167116, + -0.00017856316, + -0.045900483, + 0.0077081304, + 0.074832946, + -0.026312457, + -0.0005495474, + -0.13214304, + 0.049361903, + 0.02400141, + -0.0055426657, + 0.015336194, + -0.02816012, + 0.0046836925, + 0.0099547, + 0.03742822, + 0.011811279, + 0.19601296, + 0.033754703, + -0.010378031, + 0.001250739, + 0.008726979, + 0.018838167, + 0.043022648, + 0.0697224, + -0.010394221, + -0.030004341, + -0.04530521, + -0.046880346, + 0.018401384, + -0.01957355, + 0.041167054, + 0.058837432, + 0.015511397, + 0.0063158153, + 0.11830529, + 0.023792224, + 0.058994506, + 0.00018409223, + 0.034306847, + 0.058044955, + -0.020177843, + -0.048148416, + 0.0122626135, + 0.034675147, + -0.03752977, + 0.029120939, + -0.010040249, + 0.04624406, + -0.0073817093, + -0.011214725, + -0.04331988, + 0.0051043136, + 0.01554897, + -0.012071543, + 0.02121115, + -0.006823022, + -0.011237937, + -0.0035037443, + -0.019238824, + 0.018333014, + -0.026355784, + -0.054825306, + -0.009566956, + 0.0024437692, + -0.009667523, + -0.013058095, + -0.034837723, + -0.020705493, + -0.048979156, + 0.021577716, + -0.0041844496, + -0.0036711793, + -0.05854377, + -0.04395329, + -0.026583686, + 0.03830527, + 0.031828027, + -0.005516493, + -0.07229121, + 0.023722429, + -0.009492063, + 0.03496614, + -0.024903411, + 0.0026213636, + 0.015441066, + 0.028241813, + 0.0037205473, + 0.017424388, + -0.023927208, + 0.016457984, + 0.02520875, + 0.040443234, + 0.010766083, + 0.017123722, + -0.04108963, + -0.033335038, + 0.0030168097, + 0.0053470326, + -0.018452037, + 0.03997777, + 0.007828228, + 0.016234277, + 0.020400109, + 0.048260894, + -0.028451897, + -0.049423486, + 0.014191736, + -0.01758265, + -0.009043689, + 0.02531204, + -0.0062299054, + 0.01920355, + 0.03200232, + 0.017771985, + -0.013553965, + -0.043947473, + -0.015971966, + -0.03285715, + 0.00947157, + -0.042635962, + 0.057298675, + 0.060869828, + -0.014281702, + -0.015127428, + 0.040840045, + -0.03678748, + 0.009928272, + 0.013789576, + 0.016361559, + -0.033839006, + 0.052730188, + 0.03176229, + 0.04237319, + 0.014982663, + 0.017706575, + 0.04312265, + -0.025634803, + -0.02308356, + -0.019002479, + 0.036787376, + 0.031167256, + 0.006498465, + 0.012102121, + 0.020858966, + -0.024361406, + 0.02639614, + 0.057980295, + 0.0049070157, + -0.029224023, + 0.022465296, + 0.025334602, + 0.060530398, + 0.015762227, + 0.0035243605, + 0.034754157, + 0.027435217, + 0.060840853, + 0.01591025, + 0.009018213, + 0.00035580425, + -0.033563286, + 0.010515216, + 0.0069815437, + -0.047450293, + -0.011460179, + 0.015469139, + -0.004947935, + 0.044788565, + -0.028610881, + -0.03679307, + -0.026844844, + -0.024473835, + -0.013097169, + 0.05502105, + -0.0030086858, + 0.0054221293, + 0.00934873, + -0.05726428, + 0.028941594, + 0.021284036, + -0.018056447, + 0.076361485, + -0.028323943, + 0.010804032, + 0.03501848, + -0.041234206, + -0.010082845, + -0.023559377, + -0.047871493, + 0.043036506, + -0.0077531943, + -0.061463587, + -0.02967603, + 0.027839405, + -0.0020366006, + 0.031356577, + 0.009509519, + -0.017622339, + 0.0026792034, + -0.031537317, + 0.02021114, + 0.0575173, + -0.01732471, + 0.0070268577, + 0.0016957909, + -0.030016575, + -0.01718395, + 0.07615369, + -0.0123706665, + -0.0047786008, + -0.0060987207, + -0.0013207493, + 0.010539184, + 0.009274312, + 0.0007885846, + -0.016644027, + 0.038292266, + 0.01448823, + 0.024252761, + -0.057947274, + -0.033873387, + 0.019700998, + -0.040111557, + -0.021552257, + -0.057441693, + -0.06635977, + -0.025710544, + -0.053925313, + -0.0030804146, + -0.016332027, + 0.0361699, + 0.0059258826, + 0.042275347, + 0.024262903, + -0.011202689, + -0.03925866, + -0.026211502, + -0.018307885, + 0.04379796, + -0.076288186, + 0.032032076, + 0.007867089, + 0.0054110987, + -0.038828284, + 0.04143126, + -0.01417168, + 0.008428398, + -0.0007231599, + 0.01121787, + 0.00823, + 0.019377021, + -0.03386666, + -0.007837995, + 0.0138864545, + 0.034504473, + -0.014824415, + -0.029037168, + 0.0040237717, + 0.013956105, + -0.03657621, + -0.0029222432, + -0.0029531356, + -0.012477216, + 0.0034438937, + 0.028417397, + -0.0042160964, + -0.04023129, + -0.007177788, + -0.012188858, + -0.010640365, + -0.025378073, + 0.023323987, + 0.04131474, + -0.020508341, + -0.017246038, + -0.009083519, + 0.011987527, + 0.010934865, + -0.020992914, + -0.019448092, + 0.0035046816, + -0.0036248225, + 0.029120086, + -0.02020849, + 0.0108864745, + 0.004756365, + -0.008310412, + 0.00082247274, + -0.022503192, + -0.06487264, + 0.004785375, + -0.033616617, + -0.07876667, + 0.037286237, + 0.064570576, + -0.02501201, + 0.00021179773, + 0.019808415, + 0.01730107, + 0.0028576683, + -0.022853047, + 0.096394695, + -0.009005143, + 0.021504054, + -0.016715469, + -0.103030674, + -0.00034150484, + 0.027667, + -0.033947464, + 0.015386551, + 0.0004739991, + 0.025440171, + 0.03065169, + -0.015094892, + -0.024132984, + 0.040302265, + 0.0022036897, + 0.027904361, + -0.04102699, + 0.029432945, + -0.038208827, + -0.0062153265, + -0.011998047, + -0.013344417, + 0.033526026, + 0.021145038, + 0.0027201674, + -0.019569896, + -0.031845108, + -0.04933251, + 0.05314293, + -0.0025423346, + 0.06681798, + 0.037334353, + -0.0072135865, + 0.0014750219, + -0.008131382, + 0.0033401614, + -0.033794142, + 0.027915679, + -0.020438343, + 0.03567195, + -0.04277935, + -0.01002426, + 0.028667305, + -0.03083799, + -0.0533229, + 0.012505484, + -0.0056473175, + 0.0070565026, + 0.021628987, + -0.0054352866, + -0.039229307, + -0.0135678705, + -0.028667327, + 0.022404738, + 0.0067239925, + 0.002311097, + 0.06369203, + -0.02629535, + 0.040622883, + -0.034154095, + 0.018668704, + 0.011607976, + -0.0021320207, + -0.01759238, + 0.021003881, + 0.006119285, + -0.04615905, + -0.019840589, + -0.0129681, + 0.017631495, + -0.03321065, + 0.034223422, + 0.04391737, + -0.018810853, + -0.028161395, + 0.028423589, + -0.049154766, + 0.054977193, + 0.0000630182, + -0.0173311, + 0.0151262935, + -0.004985415, + -0.004255283, + 0.0029248288, + -0.0005668443, + -0.023171514, + -0.006701821, + -0.03586675, + -0.0022324445, + 0.01843075, + 0.0076619005, + -0.06435156, + -0.016519507, + 0.052956775, + -0.0056365584, + -0.03733352, + -0.039358534, + -0.0435778, + 0.031613506, + 0.03608112, + -0.0070490213, + 0.01944554, + 0.03693605, + -0.010692041, + -0.00599563, + 0.020131715, + -0.09125295, + -0.02595961, + -0.10351341, + 0.02379246, + -0.031193437, + -0.005686541, + -0.07574321, + -0.00818769, + 0.0012698586, + 0.031600002, + 0.029005878, + -0.034200996, + -0.07727333, + 0.03113444, + 0.002058297, + -0.009169671, + -0.02671429, + 0.0059962994, + 0.003603806, + 0.0074202335, + -0.0060870526, + -0.025415512, + -0.0050098686, + -0.018891325, + 0.015137767, + 0.0009658489, + 0.011259749, + -0.00018296832, + -0.054526947, + 0.042278267, + -0.001603754, + 0.055526804, + 0.032714244, + 0.055156004, + 0.03152486, + 0.024948584, + 0.010113138, + 0.026203973, + 0.0036432473, + -0.04414578, + 0.026116638, + -0.017378923, + 0.014874825, + 0.015792275, + -0.040725254, + -0.0027440262, + -0.016308293, + 0.019773996, + -0.032847524, + 0.009167586, + -0.08082315, + 0.008408904, + 0.020688424, + -0.019353844, + -0.028009258, + 0.026694907, + 0.0032782315, + -0.03360142, + -0.023895264, + 0.039238874, + 0.00445677, + 0.02949898, + 0.014402198, + -0.003599358, + -0.040845413, + -0.0350518, + -0.033609767, + 0.024508178, + -0.019472161, + -0.0031140537, + -0.005668693, + 0.03111455, + -0.027201336, + -0.025430666, + -0.035129692, + 0.0105588045, + 0.02917934, + -0.01097523, + -0.022186121, + 0.017050618, + 0.0037318894, + -0.03971323, + -0.0067270542, + -0.012156165, + -0.088368215, + 0.02033153, + 0.040660948, + 0.024675295, + -0.0045001414, + 0.018805915, + -0.00049041864, + 0.04464026, + -0.04455079, + -0.022046903, + 0.041305263, + 0.03293151, + -0.028352423, + -0.02059597, + -0.0019165347, + -0.006466022, + -0.009925339, + -0.008929011, + -0.076839685, + -0.00429755, + -0.014738176, + -0.024139099, + 0.03820742, + 0.003912287, + 0.030968878, + 0.004669395, + 0.025041392, + -0.013336667, + -0.056171685, + -0.023908475, + -0.0026514267, + -0.04345135, + -0.022183042, + -0.014375652, + 0.00066614884, + -0.047541592, + -0.06785615, + -0.06805146, + 0.0052735, + 0.02312607, + -0.039141715, + -0.017192993, + 0.0020956157, + 0.026551597, + -0.016486952, + -0.047485273, + 0.04242504, + -0.024608482, + -0.0063258884, + -0.025091767, + -0.0031197283, + 0.016352389, + 0.032390088, + -0.024658075, + -0.055107787, + 0.041251957, + -0.01631562, + -0.0514147, + -0.023321213, + -0.0093295155, + -0.015928332, + -0.021337397, + -0.011802844, + 0.014432149, + -0.019652303, + 0.0067840177, + -0.05091495, + -0.0055248598, + 0.0048583862, + 0.01215574, + -0.02517826, + -0.0025640433, + -0.0034858014, + 0.0013292104, + 0.034103796, + 0.020756245, + 0.0193323, + 0.040301695, + 0.00078084186, + -0.04896679, + 0.027173512, + -0.005667116, + 0.01948293, + 0.032077633, + 0.0061108964, + -0.018225638, + -0.02208757, + -0.010346406, + -0.03226915, + 0.015659394, + 0.010559285, + 0.0644997, + 0.033056695, + -0.04196774, + 0.0071754702, + -0.0038966993, + 0.020072218, + -0.03101222, + 0.03538792, + -0.008695402, + 0.0067324615, + 0.037320923, + 0.0085444255, + 0.037771806, + 0.01592961, + 0.0071725305, + 0.007101917, + 0.01527844, + 0.0028750405, + -0.016928488, + -0.013771216, + -0.023309482, + 0.030158604, + -0.011056797, + -0.000734273, + 0.039915036, + -0.007078933, + 0.041310612, + 0.03926212, + -0.021225905, + -0.03690358, + 0.00033736796, + -0.046615493, + 0.0035040032, + -0.04736803, + -0.015354346, + 0.0076449066, + 0.0646623, + -0.061303865, + 0.0072781476, + -0.022982756, + 0.04087125, + 0.020003729, + -0.0351412, + 0.0387916, + 0.044128425, + -0.0139995115, + -0.0019715482, + 0.03880407, + 0.04205457, + -0.027607651, + 0.006489501, + 0.002768678, + -0.030172173, + 0.0062397877, + -0.013040987, + -0.013252143, + -0.04237509, + 0.009563442, + 0.057689045, + -0.019734269, + 0.0113543505, + 0.02527864, + 0.0014363149, + 0.06007512, + -0.00003742828, + -0.022210408, + 0.012302623, + 0.040538836, + 0.018775605, + -0.034081753, + -0.0004998032 + ], + [ + -0.03950687, + 0.043531194, + -0.014640973, + -0.014720938, + 0.024726942, + -0.03612319, + -0.014307638, + 0.01025741, + 0.004683229, + -0.03798171, + -0.03485831, + -0.009100663, + 0.0012342628, + 0.029542759, + 0.04620511, + 0.034314673, + -0.0028652407, + -0.016328944, + 0.007256933, + -0.031210486, + 0.016605832, + -0.04016649, + -0.009275935, + 0.005787752, + -0.0058478094, + 0.029647486, + -0.03285623, + -0.036804114, + 0.0102847135, + 0.03751815, + -0.011018378, + 0.012599248, + 0.007816772, + 0.016865665, + -0.023787512, + -0.0033270551, + 0.026091896, + -0.008189237, + -0.039871283, + -0.032342162, + 0.01738121, + 0.014661393, + 0.03587866, + -0.023817893, + -0.0124929, + -0.03888124, + 0.041935273, + -0.0263411, + -0.039413948, + -0.037727747, + 0.0013563702, + -0.041180752, + 0.008936618, + -0.032369066, + 0.032389134, + 0.041318245, + -0.07018222, + -0.024456048, + -0.020840136, + 0.014526111, + 0.033790283, + 0.010664927, + -0.04341061, + -0.022219952, + -0.010782748, + 0.005418068, + -0.022497445, + 0.013246439, + 0.012736531, + -0.016248927, + 0.0008248154, + 0.009251837, + -0.06683385, + -0.0011490654, + -0.07666291, + 0.011535246, + -0.039218437, + 0.05110004, + -0.09564187, + -0.04194155, + 0.02009398, + -0.0067823115, + -0.009455593, + 0.056001958, + 0.016271655, + 0.06911042, + -0.022176139, + 0.028006595, + -0.028135093, + 0.034655333, + -0.007308119, + -0.013883404, + 0.045097373, + -0.052717112, + -0.034108102, + -0.007105286, + -0.03460486, + 0.031739317, + 0.030613171, + 0.008154331, + -0.06315366, + -0.018434746, + 0.03518978, + 0.0112801725, + 0.024134867, + -0.003977951, + 0.016525777, + 0.01839739, + -0.024246577, + -0.019862736, + 0.00031075653, + -0.013859162, + 0.013144024, + 0.024517544, + -0.038736217, + -0.02523414, + -0.010362971, + -0.027183976, + 0.021046206, + 0.014898793, + 0.003440566, + -0.016803645, + 0.0020033128, + -0.02742916, + 0.0030086606, + 0.0032655115, + -0.02966956, + 0.07162049, + 0.037842706, + -0.0046940506, + 0.00064334256, + 0.020130387, + -0.0076284185, + 0.020069573, + -0.08732818, + -0.03501705, + 0.055594362, + 0.082634926, + 0.014030701, + -0.034930494, + 0.05232482, + 0.053865146, + 0.01451553, + -0.024455464, + 0.026275275, + -0.034078658, + -0.03708409, + 0.01745932, + -0.021863382, + -0.05555264, + -0.033413053, + 0.015493354, + -0.0062087053, + -0.022264965, + -0.02127896, + -0.059868205, + -0.003401156, + 0.044210643, + 0.01787959, + 0.09501992, + 0.018742345, + -0.031885527, + -0.032269854, + 0.00025357012, + 0.042152844, + -0.03316293, + 0.02196225, + -0.0020535681, + -0.0072929896, + -0.0052945167, + -0.021792375, + 0.007816838, + -0.021683265, + -0.034982268, + -0.018460391, + -0.012000704, + 0.06461101, + 0.007490276, + 0.014343332, + -0.021530014, + -0.0075940792, + -0.02895248, + -0.014425406, + -0.040914804, + -0.03639253, + -0.08229706, + -0.004979539, + -0.017418977, + 0.016572325, + 0.037223563, + 0.00116424, + 0.019090686, + 0.010911045, + -0.010205869, + 0.048550658, + 0.035755083, + -0.018986369, + -0.001429209, + -0.027796214, + 0.025839124, + 0.022751706, + 0.054932334, + 0.020839911, + -0.017549826, + -0.044058055, + -0.042574435, + -0.05696988, + 0.024840906, + -0.017054198, + -0.054606467, + 0.0077261142, + -0.02644012, + 0.026694741, + -0.027916811, + -0.015582057, + -0.044071168, + -0.01406791, + 0.0578193, + -0.057348266, + -0.07553237, + -0.020606408, + 0.05444073, + -0.04222853, + -0.02362194, + -0.03362329, + -0.02535667, + -0.0018919092, + 0.034991592, + 0.04424803, + -0.018281374, + -0.027159164, + 0.027622392, + -0.0061814794, + 0.007855805, + -0.015342956, + 0.00082629744, + 0.008025998, + -0.0024992118, + 0.023763394, + -0.003807417, + -0.022212056, + -0.002562533, + -0.00694927, + 0.044300094, + -0.03489844, + -0.057076182, + 0.02006841, + -0.0037533976, + 0.006341216, + 0.007860421, + 0.008677003, + 0.011595276, + -0.046128266, + 0.0013428159, + 0.045055754, + 0.02239221, + 0.019835426, + -0.008117078, + 0.006887426, + -0.008277146, + 0.048709106, + -0.006205201, + 0.091035716, + 0.031401526, + 0.009654572, + -0.004819402, + -0.015429093, + 0.055476464, + -0.020625938, + 0.0040114745, + -0.0067515746, + 0.011730604, + -0.0027268373, + 0.036516953, + 0.015225207, + -0.0048017604, + -0.006393769, + -0.021304023, + 0.056664098, + 0.0026197485, + -0.0080493875, + 0.021506276, + 0.015002799, + 0.009055541, + -0.026723027, + -0.006230178, + -0.0067681912, + -0.016294137, + -0.00268294, + -0.06420861, + 0.02237568, + -0.019511264, + 0.05340581, + -0.00833766, + -0.0200639, + -0.0036080014, + -0.037964303, + -0.10868055, + 0.00020624952, + -0.06711885, + -0.006539819, + -0.0034174747, + -0.008780332, + -0.026614243, + 0.002306322, + 0.005154491, + 0.0019403903, + -0.04626778, + -0.029957443, + 0.006960841, + -0.046184577, + -0.017351516, + -0.021748034, + 0.045616314, + 0.032945316, + 0.026660904, + -0.04546675, + -0.044575658, + -0.029681908, + 0.047057368, + -0.056180052, + 0.014390304, + 0.00015773278, + 0.013120344, + 0.02658648, + 0.011372644, + 0.005863363, + 0.016664037, + 0.06376613, + 0.019382346, + 0.0802324, + -0.0016175115, + 0.021250589, + 0.00053878257, + 0.020504216, + -0.03426853, + 0.005095898, + 0.0031997939, + 0.0051565412, + -0.043827157, + 0.03209446, + 0.04712799, + -0.01782006, + 0.053983606, + -0.035910055, + -0.016622698, + -0.037207805, + 0.034288082, + -0.030350236, + -0.0011003837, + -0.02914163, + -0.03423277, + 0.009082422, + -0.001781773, + 0.032029316, + -0.0034477508, + -0.022011233, + 0.020805836, + 0.04373524, + 0.01426929, + 0.012661985, + 0.0089556845, + -0.02926414, + 0.010948363, + 0.00024500352, + 0.041760296, + -0.019010147, + -0.020999651, + 0.0043251915, + -0.018788947, + -0.03642843, + -0.029119324, + 0.05132978, + 0.008250169, + -0.023003912, + -0.026743688, + -0.08720566, + -0.059065796, + -0.000029261539, + 0.023304205, + -0.03414005, + -0.042730678, + 0.01792291, + 0.0071862466, + 0.012993562, + 0.027364269, + 0.19823818, + -0.0059806146, + -0.018968418, + 0.024375647, + 0.01791589, + 0.012285263, + -0.002662988, + -0.00074736535, + -0.014230657, + 0.03866545, + 0.044439327, + 0.007127386, + -0.007903959, + 0.0022473629, + -0.041698676, + -0.0022041672, + -0.026352853, + 0.009517888, + 0.070968114, + 0.0046247942, + 0.030372195, + 0.006002809, + 0.02578483, + 0.026675988, + 0.013856626, + 0.016871126, + -0.0014647035, + 0.052734047, + -0.009674117, + 0.017708449, + 0.0026036392, + -0.0077395584, + -0.02032638, + -0.057948228, + -0.037037287, + -0.026146235, + -0.012568703, + 0.030102657, + -0.009142211, + -0.0017773161, + 0.022137081, + -0.016034972, + -0.026643815, + 0.043836106, + -0.00026422995, + -0.012727786, + 0.06419574, + 0.006584795, + 0.018396322, + 0.017908173, + -0.050535623, + -0.02078155, + 0.010549742, + -0.014217779, + 0.024685526, + -0.023678334, + -0.020491404, + 0.006048553, + -0.023561789, + 0.018910788, + 0.08878148, + 0.007740561, + -0.059295204, + 0.03733036, + -0.016496437, + -0.018883424, + 0.018802142, + -0.007328556, + 0.016717581, + 0.04328429, + 0.05693224, + 0.017707568, + 0.046792727, + -0.014650998, + 0.062150735, + -0.012973712, + -0.018037295, + 0.016467381, + 0.0244791, + 0.0062886435, + -0.02355788, + 0.014314312, + -0.01939792, + -0.002719139, + 0.0030674103, + -0.008508544, + 0.006620206, + 0.008997434, + -0.02707569, + -0.068797246, + -0.026732523, + -0.03025209, + -0.07743795, + 0.025089052, + 0.036291923, + 0.03730403, + 0.04085414, + -0.0066735293, + -0.0055792555, + 0.031028569, + -0.0030874214, + -0.015421479, + -0.013141398, + -0.020299582, + -0.023435697, + 0.0017748995, + -0.0021218956, + 0.0012519993, + -0.001484999, + 0.008944802, + -0.027911007, + 0.012121643, + -0.020927167, + -0.0002258706, + -0.012256106, + 0.0047594383, + -0.0148541825, + -0.043066435, + 0.030682703, + 0.012971142, + -0.003666529, + 0.0108013395, + 0.025183335, + -0.028475968, + 0.021690562, + -0.0028832422, + -0.014144738, + -0.0045188447, + -0.040258396, + 0.028346978, + 0.035040945, + -0.011023868, + 0.0396404, + 0.017606566, + 0.033269204, + 0.0073707304, + -0.014222149, + 0.014156873, + 0.030270226, + 0.015319926, + 0.024624461, + -0.039615463, + 0.022913564, + 0.0031669824, + 0.002471962, + -0.039530534, + 0.04196699, + -0.017445292, + -0.030874938, + -0.0053294376, + 0.038435955, + 0.015547038, + -0.017440522, + -0.040413458, + -0.03927991, + -0.017935151, + 0.0047400724, + 0.003020446, + -0.026619047, + -0.039115086, + 0.056505933, + 0.009858637, + 0.0131826205, + 0.0284519, + 0.022894057, + 0.0022495075, + -0.021099819, + -0.0116347065, + -0.01811318, + 0.006474129, + 0.054837897, + -0.012720091, + 0.0016731841, + 0.021986602, + 0.033270527, + -0.00033776753, + -0.008643222, + 0.019710293, + -0.03264088, + 0.010028613, + 0.005217198, + -0.024148779, + -0.04014084, + -0.016549045, + 0.025656188, + 0.054638997, + 0.012246356, + -0.032795466, + -0.015674438, + -0.026266312, + -0.01646067, + 0.08210362, + 0.015601729, + -0.001658601, + 0.028211301, + -0.03853208, + -0.04279942, + 0.06524898, + -0.022642639, + -0.017951515, + -0.029549977, + -0.034688108, + 0.0061429352, + 0.010141051, + 0.00042500003, + 0.025635283, + -0.043835618, + -0.033300713, + -0.0059104236, + -0.007492045, + -0.025678229, + -0.012022994, + -0.0012621493, + -0.027511965, + 0.034545906, + 0.0027232112, + 0.0049217213, + 0.017144063, + 0.031473048, + -0.021966783, + 0.009118309, + 0.03147311, + 0.039992817, + -0.023138776, + -0.0014036369, + -0.037520673, + 0.07687327, + -0.039589405, + 0.04602321, + -0.003612067, + 0.007461662, + 0.007353311, + -0.034707982, + -0.02012695, + 0.029514175, + -0.031999454, + -0.043143217, + 0.028792208, + -0.0011134085, + -0.03371898, + 0.013939409, + -0.0019052078, + -0.004295308, + -0.007053404, + -0.0009750122, + 0.029560952, + 0.005407683, + 0.03037516, + -0.00785735, + -0.021339875, + 0.02605982, + 0.039290346, + -0.010707787, + -0.03064161, + -0.00059364137, + 0.012069025, + -0.0033701425, + -0.017966857, + -0.018942371, + 0.017725015, + -0.004986756, + 0.011548954, + 0.0028690782, + 0.038817663, + -0.021664195, + 0.013531663, + 0.0156029165, + -0.023836723, + 0.024638338, + 0.01691388, + 0.003397145, + -0.03100652, + -0.04320627, + 0.0011457364, + 0.04675279, + 0.027556699, + -0.033261135, + -0.008995849, + 0.017848724, + 0.014306368, + 0.025643473, + 0.035529383, + 0.0029430243, + 0.03153322, + -0.052132677, + -0.00248758, + -0.02161323, + -0.04709703, + -0.042139307, + -0.023811264, + 0.057088885, + -0.007749684, + 0.01378622, + 0.0038549982, + -0.042886388, + -0.02104687, + 0.00838618, + -0.0037721547, + -0.037230987, + -0.0111783985, + 0.0097314585, + 0.02131326, + -0.040853705, + 0.0057790386, + 0.021632854, + -0.022898814, + -0.012932489, + 0.004934083, + 0.010148015, + 0.014385065, + -0.003315393, + 0.017242506, + -0.007246454, + 0.009660279, + -0.019179016, + -0.010557076, + 0.016264342, + -0.029500151, + -0.012701573, + -0.012758462, + -0.029923983, + -0.02165708, + -0.052309945, + -0.033560004, + 0.0076761013, + -0.01549472, + -0.013017553, + 0.0043236334, + 0.04691618, + -0.008197069, + -0.0018741535, + -0.03084931, + -0.0054556658, + -0.016673744, + 0.039174348, + 0.04543094, + -0.023596205, + 0.001231028, + 0.035865817, + 0.0023900091, + -0.003052916, + -0.027418893, + 0.058787756, + -0.02245522, + -0.015146493, + -0.0184522, + -0.0063935583, + -0.030266108, + 0.007447998, + -0.017568134, + -0.0027355, + 0.021793561, + 0.0074607856, + -0.04652875, + -0.031837653, + -0.061417032, + -0.035766512, + 0.007060734, + 0.018031834, + 0.00045341678, + -0.0077504576, + 0.0076210555, + 0.012140915, + -0.002184006, + 0.06035298, + -0.022536587, + 0.0054653105, + -0.005007854, + -0.007992475, + -0.034879297, + 0.007152057, + -0.01913284, + -0.10116649, + 0.0022808493, + 0.008442801, + 0.036988016, + -0.016776918, + -0.07377468, + -0.01899712, + 0.07031005, + -0.034832127, + -0.03700141, + -0.01211916, + 0.0067655994, + 0.024989525, + 0.02319956, + -0.027294133, + -0.00094650587, + 0.014060532, + 0.03490878, + -0.01971701, + -0.038569264, + -0.04076684, + 0.014368246, + -0.10717569, + 0.020453772, + 0.068951204, + -0.007349555, + -0.04629167, + -0.014826132, + 0.005653124, + -0.026169593, + 0.013392487, + -0.07173489, + -0.01668766, + -0.030206582, + 0.040627755, + -0.029564377, + -0.043053318, + -0.03218649, + 0.03655294, + -0.008489806, + 0.04350242, + 0.030536214, + -0.009721366, + 0.00034617525, + -0.016816063, + 0.059502516, + -0.021497775, + -0.022350075, + -0.111227795, + 0.028571766, + -0.033449635, + 0.07857797, + -0.02256202, + 0.057096943, + 0.04939214, + 0.02692003, + 0.03341055, + 0.010906601, + 0.03452141, + -0.101317056, + -0.016416814, + 0.028970905, + -0.027896484, + 0.022908753, + -0.015605515, + 0.009761399, + 0.01375609, + 0.002159126, + -0.034757312, + 0.009949484, + -0.027204521, + 0.036121096, + -0.018719548, + 0.012673245, + -0.047391124, + -0.044187777, + 0.048009574, + -0.012766926, + 0.0116457315, + 0.014982552, + 0.019103188, + 0.022949666, + -0.039069697, + -0.04630458, + -0.028584287, + -0.040020872, + 0.015652915, + -0.013488799, + -0.049111538, + 0.026503077, + 0.03357381, + 0.03751419, + 0.02387496, + -0.0011243735, + 0.012061748, + -0.011386109, + -0.0127039915, + 0.06352219, + 0.048704118, + 0.014793782, + -0.014007755, + -0.043857392, + 0.038399946, + -0.012055639, + -0.014885214, + 0.042390678, + 0.04980625, + -0.006241386, + 0.013584097, + -0.033264153, + 0.041379217, + 0.029621469, + -0.043042388, + -0.0020997606, + -0.012933665, + 0.013286224, + -0.003282772, + 0.018547414, + -0.04327, + 0.022249907, + 0.008725586, + 0.018808063, + -0.026111113, + 0.008951248, + 0.031707752, + -0.005465531, + 0.06568567, + -0.020877102, + 0.03014087, + 0.01715928, + 0.012325161, + 0.01315365, + -0.031900663, + -0.022260122, + -0.03447207, + -0.00062158384, + -0.035049655, + 0.003485939, + 0.0046049925, + -0.05630732, + -0.011058264, + 0.007136582, + 0.016117854, + 0.0595212, + -0.03617894, + -0.027507313, + 0.002789625, + 0.01799178, + -0.0014536566, + -0.008393191, + -0.018825287, + 0.04759385, + 0.058703866, + -0.011992767, + -0.0052801687, + 0.01940323, + 0.03281311, + -0.01995503, + -0.013774053, + 0.05898928, + -0.07519091, + -0.044439603, + -0.0537019, + -0.014705708, + 0.01250235, + 0.015665539, + 0.05294212, + 0.009310713, + 0.03523993, + 0.031513117, + -0.0074070916, + 0.01826314, + 0.011835665, + -0.044761684, + -0.007041866, + 0.058125574, + 0.024500526, + -0.009871987, + -0.018237904, + 0.09113848, + -0.013172258, + -0.0041379156, + 0.016455337, + -0.022989342, + 0.016601447, + 0.014433, + 0.0053295735, + -0.024041522, + 0.01957587, + 0.018358402, + -0.017210413, + 0.043062195, + 0.0061207553, + 0.032532007, + 0.014793015, + 0.024234612, + 0.0061567454, + -0.018245239, + 0.02684064, + 0.00016797535, + 0.028608758, + 0.0046842927, + -0.010550258, + 0.02669938, + 0.030235212, + -0.0031934716, + -0.019914627, + 0.035149656, + 0.0075301835, + 0.039265014, + 0.035625007, + -0.031268924, + 0.012515024, + -0.07590218, + 0.007551814, + -0.035385616, + 0.027681876, + -0.039070603, + -0.004845278, + 0.04028714, + -0.07426009, + 0.06613951, + -0.05510454, + -0.03226479, + -0.014005947, + -0.03195615, + -0.0116673885, + 0.016437342, + -0.0076668607, + 0.028972298, + -0.025123762, + -0.017822955, + -0.011712876, + -0.012553329, + -0.037626002, + 0.00069678575, + 0.009572705, + 0.0067847413, + 0.0028660365, + -0.011129996, + 0.00090118404, + -0.0070405314, + -0.018169235, + -0.018179605, + 0.07109999, + -0.016261067, + -0.024284609, + -0.026516862, + 0.03497836, + -0.017624862, + 0.009457551, + -0.030564932, + 0.04343077, + -0.01112055, + -0.054642823, + 0.004773121, + 0.0047957487, + -0.008624442, + 0.015786134, + 0.03043451, + -0.043876424, + 0.015928723, + 0.045520768, + -0.025372392, + 0.009925128, + 0.028771762 + ] + ], + "abstain_centroids": [ + [ + -0.031148382, + 0.0053377217, + -0.051859938, + 0.006701707, + -0.013760795, + -0.046238896, + -0.0017418014, + 0.003761064, + 0.011534153, + 0.015631318, + -0.007558447, + 0.042414147, + -0.008460059, + -0.0039428133, + 0.014397731, + -0.00096799876, + 0.005969027, + -0.016228952, + -0.0066106487, + -0.01633511, + -0.017304767, + -0.015375099, + 0.00016670344, + 0.029177906, + 0.019667022, + 0.037063632, + -0.014033232, + -0.017078552, + 0.022332544, + 0.0029681043, + -0.026012668, + 0.004773388, + -0.008349955, + -0.025975648, + -0.05798201, + -0.037145726, + 0.022042587, + -0.0104322145, + -0.054713663, + -0.018552497, + 0.01407926, + 0.0066314046, + 0.027429355, + -0.026314953, + 0.021749053, + -0.05777429, + -0.011509257, + -0.020935455, + -0.00084091024, + -0.031351313, + -0.021382079, + -0.00847218, + 0.036603227, + -0.013143199, + 0.047227796, + 0.05580892, + -0.0109791085, + -0.021309411, + -0.087006725, + 0.0018242795, + -0.025122127, + -0.013786088, + -0.009168321, + 0.011006979, + 0.0012779796, + 0.09556672, + 0.00808916, + -0.008711008, + -0.008361359, + -0.034987632, + -0.03647069, + 0.021719662, + -0.032498714, + -0.017760865, + -0.06984331, + 0.011626705, + 0.020904267, + -0.008985531, + 0.0070378105, + 0.037491333, + 0.08140703, + -0.015960725, + 0.00758235, + 0.018223334, + 0.011250972, + 0.041393135, + -0.0300754, + 0.011359266, + -0.0026697272, + 0.0045864326, + -0.029182361, + -0.018836327, + 0.035417307, + -0.050186697, + -0.0169077, + 0.0030452076, + -0.031451467, + 0.021025099, + 0.03237884, + -0.017753405, + 0.024965167, + 0.006849125, + -0.0007417424, + -0.025353184, + 0.014833974, + 0.000480421, + 0.033970177, + 0.029662795, + 0.0019640021, + -0.00040594768, + 0.046204697, + 0.03329503, + 0.014799664, + 0.004400712, + -0.012687833, + -0.04885642, + -0.021582833, + -0.029335678, + 0.031864036, + -0.013479058, + 0.0061972495, + 0.01931695, + 0.039016202, + -0.0460622, + -0.011819415, + -0.03546195, + 0.030371843, + 0.045124575, + -0.00695266, + 0.026146563, + 0.0153582115, + 0.014641614, + -0.037259024, + 0.004900667, + -0.012360953, + -0.030733729, + 0.016830467, + -0.002211426, + 0.017974649, + -0.051037353, + 0.024567816, + 0.029570596, + -0.028185146, + -0.028654283, + 0.02143797, + -0.038578555, + 0.024237396, + -0.0068540815, + 0.01847857, + -0.012613435, + 0.00437136, + -0.01011591, + 0.0065389504, + -0.014050798, + 0.0061866366, + -0.01770768, + -0.019518772, + 0.053408965, + -0.013017107, + 0.016827421, + 0.022524578, + -0.0068964125, + -0.052380078, + 0.00038339154, + 0.029090617, + -0.011927654, + 0.024243768, + 0.01339041, + -0.014234689, + -0.0017058258, + -0.012792839, + -0.026187133, + 0.039857496, + -0.0025493933, + 0.030229094, + 0.026641972, + 0.07534804, + 0.022857834, + 0.010506173, + -0.058816753, + -0.051879648, + 0.016220875, + 0.017665846, + -0.054791577, + -0.039248586, + 0.014590265, + 0.021849323, + -0.018102316, + 0.015241521, + 0.02129747, + -0.022830576, + -0.01614023, + 0.027246347, + -0.00542563, + 0.046256945, + -0.007986688, + 0.026465587, + -0.0026759577, + -0.0096074, + -0.037014082, + -0.04806582, + 0.033057604, + 0.04526261, + 0.0051544863, + -0.0342738, + -0.011255881, + -0.03914743, + -0.031940874, + 0.0144135505, + -0.017719291, + 0.009187744, + 0.0078019383, + 0.02771336, + 0.0051388917, + -0.042035703, + 0.0072669606, + -0.0026950007, + 0.0030906193, + 0.004021663, + -0.02106473, + 0.0009345182, + 0.029135607, + 0.0021005718, + -0.0196823, + 0.046538655, + 0.032432303, + 0.017765502, + 0.015490211, + -0.009414902, + -0.02356354, + -0.008533621, + 0.0050160224, + -0.019913118, + -0.0035469253, + 0.03606248, + -0.022427976, + -0.04044626, + 0.015986245, + 0.009162774, + -0.00032497806, + -0.022688588, + -0.030663738, + 0.0018985462, + 0.024509203, + -0.04645868, + 0.021605283, + -0.0032775397, + 0.008400841, + 0.0053840177, + -0.0020570229, + 0.018701848, + -0.012248811, + -0.045328125, + 0.0036800986, + 0.032931987, + 0.028872136, + 0.012167973, + -0.0052398676, + 0.026528696, + 0.013797996, + 0.023905413, + 0.031488698, + 0.016135292, + 0.037903946, + 0.007559896, + 0.003637543, + -0.033904064, + -0.0027450037, + 0.0044508814, + 0.010002827, + -0.03097531, + -0.016194653, + -0.0103033185, + 0.0017212455, + -0.004217246, + -0.0003838392, + -0.028139843, + -0.037131034, + 0.059932996, + 0.016013881, + 0.0041002985, + 0.017807445, + 0.012730673, + -0.009848034, + 0.0070455708, + -0.0010963108, + -0.017336583, + 0.0052566174, + 0.006595982, + -0.045635466, + 0.01257733, + -0.015105403, + 0.08849552, + -0.0060461042, + -0.014552671, + -0.010428577, + -0.03148846, + -0.244925, + -0.022456428, + -0.01200543, + 0.018366225, + 0.012265931, + -0.024266042, + -0.033916224, + -0.0036076305, + -0.054510105, + 0.0063244584, + -0.015560582, + -0.055878177, + -0.020939475, + -0.005328696, + -0.0070734383, + 0.0048970133, + 0.0069189873, + -0.022256982, + 0.0075242044, + -0.06765787, + -0.023333378, + -0.025017215, + 0.07195793, + -0.019617401, + -0.005269894, + 0.00026103246, + 0.035949826, + 0.004107666, + -0.037599213, + -0.01236279, + -0.006176068, + 0.0069308924, + -0.0029138543, + 0.035269044, + 0.014236289, + 0.034278773, + 0.010570881, + -0.008682466, + -0.014200495, + -0.0010429567, + 0.045053545, + 0.02033515, + -0.0039721197, + 0.011877003, + 0.010973981, + -0.040029325, + -0.021862814, + -0.010063818, + -0.01741567, + -0.02046978, + 0.0024775746, + 0.00537928, + -0.0035171232, + -0.023752872, + -0.027805964, + 0.0068444465, + 0.0020820745, + 0.048062917, + -0.013278909, + -0.011143367, + -0.015697205, + -0.010483089, + 0.008951397, + 0.03141075, + -0.021271285, + 0.0060245073, + -0.005130949, + -0.013589777, + 0.028885502, + -0.019157533, + 0.033051424, + -0.0110376915, + 0.008256805, + -0.021247167, + 0.014382738, + 0.025279364, + -0.025481079, + -0.032188643, + -0.023664054, + -0.18205334, + -0.016931957, + -0.00092831947, + 0.0030238251, + 0.033743776, + -0.016215393, + -0.025330719, + 0.00043090683, + 0.052764844, + 0.045047104, + 0.34483737, + 0.021190971, + 0.029025022, + -0.029802192, + 0.05892976, + -0.027338218, + 0.017156813, + 0.01649102, + 0.0053761047, + -0.03603205, + -0.016050696, + -0.019247439, + 0.008546483, + 0.00065792014, + 0.008216663, + 0.012412621, + -0.049241338, + 0.0046835714, + 0.083209105, + 0.0044032163, + 0.015972426, + -0.035188276, + 0.0025444007, + 0.0276847, + -0.03703065, + -0.040941797, + -0.022856751, + 0.07734354, + -0.017490435, + 0.029937685, + -0.030589428, + -0.0131223295, + 0.028537748, + -0.015808787, + -0.031040976, + 0.0022384813, + 0.004584949, + -0.036500685, + -0.013998977, + 0.01908896, + -0.020744048, + -0.027405975, + 0.027832054, + 0.0009314528, + 0.021530954, + -0.015619696, + -0.006596794, + -0.008945383, + -0.02280639, + -0.00078093243, + -0.019146752, + -0.021306396, + -0.024635073, + -0.0013178922, + 0.030522594, + -0.036964312, + -0.03445982, + -0.026010966, + 0.011097982, + 0.04430233, + 0.033004954, + 0.012737286, + -0.02832643, + -0.033423617, + -0.011146303, + 0.018307667, + -0.010493131, + -0.002345288, + -0.006297813, + 0.023474589, + 0.015160519, + 0.008065754, + 0.0071064318, + -0.012013574, + 0.034734786, + 0.020278838, + 0.013851283, + 0.07314595, + 0.005167692, + 0.03851032, + -0.011801342, + -0.024405105, + -0.05664373, + -0.017895715, + 0.010395462, + 0.0045732795, + -0.017845895, + 0.052198168, + -0.007843896, + -0.03997914, + 0.011556584, + 0.019499848, + -0.03996185, + -0.007970878, + -0.00291532, + -0.05102505, + 0.023902692, + -0.024392458, + -0.019463964, + -0.021712244, + -0.028254114, + -0.022925302, + 0.008098716, + -0.009423168, + -0.0034888273, + -0.02608165, + 0.0058718375, + 0.012862887, + 0.020583529, + 0.015732484, + 0.004980414, + 0.0077576265, + -0.018226098, + -0.036115997, + 0.025706176, + 0.049908377, + 0.025328668, + 0.0090175755, + 0.020332497, + 0.040973034, + -0.016721344, + -0.01928604, + -0.012567933, + 0.0024534017, + -0.027978519, + -0.016283054, + -0.0059162932, + 0.020438466, + -0.033998214, + 0.025316799, + 0.015457932, + -0.027260445, + 0.0077517964, + 0.03414886, + 0.06149145, + -0.009142278, + 0.034868557, + 0.01562827, + -0.011573751, + 0.048809845, + -0.008835819, + -0.042271934, + -0.011501361, + -0.015698932, + 0.00765774, + 0.0040898435, + 0.017931513, + 0.01253479, + -0.026830954, + 0.009097795, + 0.008690346, + 0.044174474, + 0.0060633975, + -0.056720097, + -0.01996871, + -0.035708513, + -0.0106808655, + -0.0031720465, + -0.0111088455, + -0.029465713, + 0.012013627, + 0.014341204, + 0.052498966, + 0.13476695, + 0.046210244, + 0.052687798, + -0.00610014, + 0.045737743, + 0.043751348, + -0.032143187, + 0.0028189388, + -0.035125505, + -0.04272793, + 0.031032862, + 0.034608897, + -0.020154651, + -0.025992718, + 0.0040032133, + -0.013573706, + 0.0014319277, + 0.033563722, + -0.0124865, + -0.013652726, + -0.00095753185, + 0.018106164, + -0.019855194, + 0.00985104, + -0.017903104, + 0.013870335, + -0.02261499, + -0.025842719, + 0.123712055, + -0.016144589, + -0.0011982427, + 0.06412821, + 0.005470516, + 0.013032823, + 0.031510778, + -0.0073665287, + -0.03247822, + 0.00045605694, + -0.018492017, + 0.0098344525, + -0.025878755, + 0.011205348, + -0.012217908, + 0.004897915, + 0.015269929, + -0.02245079, + -0.0018926292, + 0.010645848, + 0.007540698, + -0.018376747, + 0.0011163224, + -0.009108631, + -0.005284527, + 0.027602877, + 0.020686561, + -0.028491242, + -0.016864398, + 0.0044273087, + 0.0062007783, + 0.047445305, + -0.06059543, + -0.023386637, + 0.031225728, + -0.01507088, + -0.030429766, + -0.0027328802, + 0.011734569, + 0.0085614575, + -0.015814755, + -0.010190624, + -0.014055305, + 0.01249472, + 0.0020188028, + -0.018316, + 0.017405652, + 0.00333235, + -0.0058355783, + 0.01903454, + -0.017335443, + 0.009071962, + 0.060741838, + -0.019548608, + 0.0028466568, + -0.0009897373, + -0.01391898, + 0.039289407, + 0.033837304, + -0.020116923, + -0.0049318965, + 0.021821506, + 0.008678599, + -0.005215469, + -0.0035473392, + 0.005850026, + -0.00011736609, + -0.038427465, + 0.004604565, + -0.017376447, + 0.03195163, + -0.037625905, + -0.00025921583, + -0.045117155, + -0.021053704, + 0.01866349, + -0.018683132, + -0.022075273, + -0.067146465, + -0.0003485933, + -0.0066958447, + -0.019604217, + -0.025007758, + 0.008463429, + 0.01903175, + -0.036018703, + 0.0002199503, + -0.014541071, + 0.03636302, + -0.024418777, + 0.036282822, + -0.015224955, + 0.03595708, + -0.040158544, + 0.03579523, + 0.02829524, + 0.011141475, + -0.01444086, + -0.03210745, + 0.012157547, + -0.005177962, + -0.056817777, + 0.01769194, + -0.03663195, + 0.038909055, + -0.009773796, + 0.0063458867, + -0.024140002, + 0.01852215, + -0.0378227, + 0.0095784925, + -0.050063416, + -0.0136081185, + -0.019632932, + 0.0028407509, + -0.0064537292, + -0.016191104, + 0.04372594, + 0.023616401, + 0.027719082, + -0.041044924, + 0.025428394, + 0.020110026, + -0.0060314494, + -0.0061647827, + 0.07147246, + -0.022163881, + -0.002495512, + 0.017374912, + 0.024532678, + 0.017868279, + -0.0123060895, + 0.009035886, + -0.023077391, + -0.00059326034, + -0.021227827, + 0.0071837874, + -0.036735952, + -0.014673909, + -0.028121512, + 0.010137441, + -0.010372104, + -0.038298663, + 0.01282512, + -0.011348841, + 0.022564255, + -0.031195194, + 0.049415413, + -0.03715284, + -0.0024479807, + 0.023421839, + 0.06413017, + -0.020511432, + 0.0113146, + -0.03853917, + 0.015962925, + -0.0053321286, + 0.01771349, + 0.016681707, + 0.020176094, + 0.045899525, + -0.03644215, + -0.05389872, + 0.03137194, + 0.008404089, + -0.038138144, + 0.014195226, + 0.04344335, + 0.0057477932, + -0.03277102, + 0.04831074, + -0.021211635, + 0.030088603, + -0.00251031, + 0.01228796, + -0.004873214, + -0.008176673, + -0.014280845, + -0.0006814086, + -0.019928552, + 0.008787194, + 0.010925872, + -0.0032950707, + -0.034151938, + 0.023929175, + -0.010394337, + -0.010170749, + 0.029074438, + 0.0061191227, + -0.023697818, + -0.021662997, + -0.03463375, + -0.0376407, + -0.008391389, + -0.01087796, + -0.008222668, + 0.0008719065, + 0.07043431, + 0.02145938, + -0.050167546, + -0.023954187, + -0.042268485, + 0.023713034, + -0.22955438, + 0.03520514, + -0.0021951396, + -0.0029458369, + -0.055697378, + 0.0052851303, + -0.008466735, + -0.00713325, + 0.0063962257, + -0.03216339, + -0.024553273, + 0.028201995, + -0.00005423209, + -0.018851798, + -0.007118189, + 0.040054683, + -0.03188417, + -0.0030668841, + -0.004163999, + 0.032002646, + 0.0047784974, + -0.02514371, + -0.005436027, + 0.014385046, + 0.018197643, + 0.0026216004, + -0.026822513, + -0.013488665, + -0.044493288, + -0.01340355, + 0.0006943536, + -0.031944275, + 0.021567853, + 0.043572214, + -0.008895598, + 0.017365277, + 0.00690336, + -0.014118812, + -0.017342428, + 0.013759935, + 0.0014441384, + 0.019967059, + -0.023214184, + -0.0035139646, + -0.0009887023, + 0.028011115, + -0.019943252, + -0.008950248, + -0.059093572, + -0.0024248771, + 0.013076103, + 0.026632652, + -0.018246919, + 0.0191107, + 0.012685868, + 0.00189008, + -0.05187209, + 0.009624937, + -0.0076693487, + 0.037904378, + 0.0033752434, + 0.012424309, + -0.03196644, + -0.03168633, + -0.04591932, + -0.02398697, + -0.08195933, + 0.013067175, + 0.013707991, + 0.02845325, + -0.022814702, + -0.036631793, + -0.0046700407, + -0.014707499, + 0.025149884, + 0.015747225, + 0.011062072, + -0.0079821665, + -0.044059474, + -0.0052218223, + -0.029661851, + -0.0010427525, + -0.052402854, + 0.010401699, + 0.012599079, + 0.018898357, + -0.024228007, + 0.012707991, + -0.01779489, + -0.0073080487, + -0.0570653, + -0.024420062, + -0.0004401083, + 0.0057198247, + -0.04850662, + 0.009310804, + -0.027014086, + -0.012875564, + -0.019028028, + -0.021750357, + -0.039091874, + 0.007929617, + -0.014110445, + -0.0059228763, + -0.027193954, + -0.008130776, + 0.05215637, + -0.009849599, + 0.026919808, + 0.021327212, + -0.013137315, + 0.0056819157, + -0.010384434, + 0.0052230777, + -0.02551496, + -0.023010256, + -0.009133841, + -0.0034335821, + -0.0066825, + -0.012148635, + -0.01445714, + 0.03445042, + -0.0021199132, + -0.04214014, + 0.01076899, + 0.010510174, + 0.035082106, + 0.008629588, + 0.0073693595, + -0.011705998, + 0.046988714, + -0.03607273, + 0.001315558, + 0.028497273, + 0.020122701, + 0.0019115491, + -0.017506052, + 0.05194846, + -0.029103, + -0.059791327, + -0.01251193, + -0.004560316, + -0.001194598, + -0.02777315, + -0.032533206, + -0.010601676, + 0.008287073, + 0.010776672, + -0.05712957, + -0.033277795, + 0.031840082, + 0.025541494, + 0.021605106, + -0.006681298, + 0.0028221495, + 0.01971035, + -0.0057943845, + 0.0026962268, + 0.0066015963, + 0.044709176, + 0.03829154, + -0.04634911, + -0.021133212, + -0.024994686, + -0.010804923, + 0.006723943, + 0.001489737, + -0.011834817, + -0.023318285, + -0.0038526892, + 0.0080185495, + 0.047896653, + -0.03021623, + 0.0534178, + 0.038851902, + -0.009745994, + 0.035945367, + 0.041955516, + 0.03538382, + -0.0023587276, + 0.03368713, + 0.00009861296, + 0.023677062, + 0.024938073, + -0.016926885, + 0.0016545699, + 0.0021329166, + 0.04145556, + -0.0030341037, + 0.026448693, + -0.013949449, + -0.00668188, + 0.0024977801, + -0.052358516, + 0.0584629, + 0.017483938, + 0.016702253, + 0.04207009, + -0.020453833, + 0.04187879, + -0.011923705, + -0.021604257, + -0.010823228, + -0.015712183, + -0.008566785, + -0.00082078524, + -0.02231372, + -0.03011328, + -0.042152412, + 0.031767588, + -0.03151925, + -0.011423307, + -0.01184754, + 0.022948263, + 0.004512842, + 0.0025419197, + 0.02291634, + 0.020818466, + 0.017587373, + 0.0005803047, + -0.016511247, + 0.009406082, + -0.015658183, + -0.035970535, + 0.01030802, + -0.027739283, + -0.022434793, + -0.003182124, + -0.011698128, + -0.014282087, + 0.010802368, + 0.011209724, + -0.037597824, + 0.06280771, + 0.030460317, + -0.014077968, + 0.052043952, + 0.03405556, + 0.0068608206, + -0.024741665, + 0.05734323, + -0.019578138, + -0.018592859, + -0.0055962713 + ] + ] +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-7/router_report.json b/benchmarks/models/router/centroid/out-router-iter-7/router_report.json new file mode 100644 index 0000000..4dca779 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-7/router_report.json @@ -0,0 +1,317 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "threshold": 0.05, + "metrics": { + "total": 141, + "accuracy": 0.7659575, + "tool_precision": 0.94545454, + "tool_recall": 0.65, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.2, + "tool_count": 80, + "memory_count": 46, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70918083, + "sim_memory": 0.69442564, + "sim_abstain": 0.62441397 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6285488, + "sim_memory": 0.61792433, + "sim_abstain": 0.5503659 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.7063374, + "sim_memory": 0.6165706, + "sim_abstain": 0.6750375 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.65761024, + "sim_memory": 0.56834835, + "sim_abstain": 0.62134224 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5843893, + "sim_memory": 0.547776, + "sim_abstain": 0.5217595 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6942834, + "sim_memory": 0.6637705, + "sim_abstain": 0.5931268 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.66822094, + "sim_memory": 0.5693777, + "sim_abstain": 0.6224822 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.5961604, + "sim_memory": 0.5583175, + "sim_abstain": 0.57129914 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.6546817, + "sim_memory": 0.5749995, + "sim_abstain": 0.61477154 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6583943, + "sim_memory": 0.6532231, + "sim_abstain": 0.5341912 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.59658426, + "sim_memory": 0.5636423, + "sim_abstain": 0.5362745 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6974401, + "sim_memory": 0.65405804, + "sim_abstain": 0.58798295 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5960697, + "sim_memory": 0.55635834, + "sim_abstain": 0.5040847 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65429235, + "sim_memory": 0.63408476, + "sim_abstain": 0.554867 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5710796, + "sim_memory": 0.5308014, + "sim_abstain": 0.50004137 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6926011, + "sim_memory": 0.691817, + "sim_abstain": 0.58473253 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5705274, + "sim_memory": 0.53116524, + "sim_abstain": 0.48013124 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.55382746, + "sim_memory": 0.5629208, + "sim_abstain": 0.511596 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.654468, + "sim_memory": 0.6415955, + "sim_abstain": 0.5482988 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.56339705, + "sim_memory": 0.5195113, + "sim_abstain": 0.4231544 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.65280145, + "sim_memory": 0.62437373, + "sim_abstain": 0.5488851 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "abstain", + "sim_tools": 0.57037896, + "sim_memory": 0.5085687, + "sim_abstain": 0.52038395 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6568779, + "sim_memory": 0.6171636, + "sim_abstain": 0.5631379 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.64629316, + "sim_memory": 0.61019343, + "sim_abstain": 0.59611 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.5862066, + "sim_memory": 0.5496563, + "sim_abstain": 0.51394325 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6231395, + "sim_memory": 0.57401973, + "sim_abstain": 0.55732864 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.6762118, + "sim_memory": 0.6369297, + "sim_abstain": 0.5889329 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "sim_tools": 0.70418376, + "sim_memory": 0.6688043, + "sim_abstain": 0.6164449 + }, + { + "query_id": "abs_test_006", + "text": "what's the github token for the ci runner?", + "gold": "abstain", + "pred": "tools", + "sim_tools": 0.6197224, + "sim_memory": 0.5374777, + "sim_abstain": 0.568083 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "sim_tools": 0.5306442, + "sim_memory": 0.45544416, + "sim_abstain": 0.45304802 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.50268793, + "sim_memory": 0.6389982, + "sim_abstain": 0.5120093 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "sim_tools": 0.5960785, + "sim_memory": 0.5498394, + "sim_abstain": 0.5258497 + }, + { + "query_id": "abs_test_015", + "text": "can you log into the remote postgres and show me the schema?", + "gold": "abstain", + "pred": "tools", + "sim_tools": 0.6982557, + "sim_memory": 0.59161407, + "sim_abstain": 0.6236215 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/centroid/out-router-iter-7/router_report.md b/benchmarks/models/router/centroid/out-router-iter-7/router_report.md new file mode 100644 index 0000000..93cd5d1 --- /dev/null +++ b/benchmarks/models/router/centroid/out-router-iter-7/router_report.md @@ -0,0 +1,55 @@ +# Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` +threshold: `0.050` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `80` memory: `46` abstain: `15` +- accuracy: `0.7660` +- tools precision: `0.9455` +- tools recall: `0.6500` +- memory false-tools rate: `0.0000` +- abstain false-tools rate: `0.2000` + +## Misclassified Queries + +| id | text | gold | pred | T-sim | M-sim | A-sim | +|---|---|---|---|---|---|---| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.71 | 0.69 | 0.62 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.71 | 0.62 | 0.68 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.66 | 0.57 | 0.62 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.58 | 0.55 | 0.52 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.69 | 0.66 | 0.59 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.67 | 0.57 | 0.62 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.60 | 0.56 | 0.57 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.65 | 0.57 | 0.61 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.66 | 0.65 | 0.53 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.60 | 0.56 | 0.54 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.70 | 0.65 | 0.59 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.60 | 0.56 | 0.50 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.65 | 0.63 | 0.55 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.57 | 0.53 | 0.50 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.69 | 0.69 | 0.58 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.57 | 0.53 | 0.48 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.55 | 0.56 | 0.51 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.65 | 0.64 | 0.55 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.56 | 0.52 | 0.42 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.65 | 0.62 | 0.55 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Abstain` | 0.57 | 0.51 | 0.52 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.62 | 0.56 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.65 | 0.61 | 0.60 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.59 | 0.55 | 0.51 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.62 | 0.57 | 0.56 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.68 | 0.64 | 0.59 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.70 | 0.67 | 0.62 | +| `abs_test_006` | what's the github token for the ci runner? | `Abstain` | `Tools` | 0.62 | 0.54 | 0.57 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.53 | 0.46 | 0.45 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.50 | 0.64 | 0.51 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.55 | 0.53 | +| `abs_test_015` | can you log into the remote postgres and show me the schema? | `Abstain` | `Tools` | 0.70 | 0.59 | 0.62 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-1/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-1/router_model.json new file mode 100644 index 0000000..ba7b593 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-1/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.23592426, + 0.07949028, + 0.12828025, + 0.020902963, + -0.14630364, + 0.2927499, + 0.06966109, + -0.08523073, + -0.16648786, + -0.113038614, + 0.075005926, + -0.13153438, + 0.04556273, + 0.1155168, + 0.030217323, + -0.050269037, + -0.10385784, + 0.12613581, + -0.046347305, + -0.1806265, + -0.0074652135, + -0.024326801, + -0.133352, + 0.0062017944, + 0.13812687, + -0.046048444, + -0.035603765, + -0.14249863, + 0.13594781, + 0.20004377, + 0.067069486, + -0.32760555, + -0.02300786, + -0.2257146, + 0.009997282, + -0.01574743, + 0.022478333, + 0.025442492, + -0.15943898, + 0.11212006, + -0.06273372, + -0.07784379, + -0.11847325, + -0.10418217, + -0.11747773, + 0.10935527, + 0.04392976, + -0.050310243, + 0.021614216, + 0.016519511, + -0.06274579, + -0.035048507, + 0.024466934, + -0.1573378, + -0.16322778, + 0.08302193, + -0.21318288, + -0.026558505, + -0.0031204561, + 0.021045184, + -0.10830819, + -0.053354923, + 0.007069701, + 0.003236166, + -0.054645468, + -0.066746205, + 0.0042206966, + 0.011725452, + 0.024868056, + 0.1641807, + -0.12174166, + 0.08036685, + -0.18606532, + -0.18648595, + -0.055612143, + 0.1113537, + 0.020987004, + -0.055373207, + -0.10797911, + -0.26858276, + -0.055051066, + 0.042448446, + -0.10084089, + -0.11356134, + 0.13724102, + 0.25910544, + -0.13518274, + 0.04425172, + -0.050655782, + 0.13767378, + -0.0948142, + -0.10503194, + -0.095695175, + 0.051466744, + -0.11693792, + -0.18811321, + -0.021318266, + 0.084969796, + -0.000039007555, + 0.038942024, + -0.07426399, + -0.048653837, + -0.017212326, + 0.12696455, + 0.07994347, + 0.07255431, + -0.023122568, + -0.10119431, + -0.06391866, + -0.009974052, + -0.22291102, + 0.04796371, + 0.0050148666, + -0.032373007, + 0.0017591142, + -0.06311281, + -0.04166572, + -0.16159615, + -0.22823016, + -0.050940126, + 0.06667, + -0.17553227, + 0.07846491, + -0.02724876, + -0.018249461, + 0.04215583, + 0.090505816, + 0.2814388, + 0.18407907, + 0.20622618, + 0.024842309, + 0.15443164, + 0.15413754, + 0.06652858, + -0.047738686, + -0.033041824, + -0.041732833, + 0.049973257, + 0.03409717, + -0.17384486, + -0.053682525, + 0.248795, + -0.054270744, + -0.07033317, + 0.1876099, + -0.21418522, + 0.06829151, + 0.11003408, + -0.03164994, + -0.022757377, + -0.08888097, + 0.13937046, + 0.15536013, + 0.11985994, + -0.12203828, + 0.012693621, + -0.029066604, + -0.10698433, + 0.0034464756, + 0.20514335, + 0.019293183, + -0.022996629, + 0.15026903, + -0.07309379, + -0.077310584, + -0.11743237, + -0.06254132, + -0.2564027, + 0.05387124, + -0.07310932, + 0.08163549, + -0.076425664, + -0.13267939, + -0.0035350712, + 0.0057923766, + 0.04745503, + 0.010319806, + 0.051259685, + 0.0014403726, + -0.070171736, + 0.016842423, + 0.08128754, + -0.20343794, + 0.27244997, + -0.09540211, + 0.028516164, + 0.10335394, + 0.25332108, + -0.04973971, + -0.12993562, + 0.00938431, + 0.14829825, + -0.02070785, + 0.038483165, + -0.04951867, + 0.049828425, + -0.07171106, + -0.033850256, + -0.1940812, + -0.009546966, + -0.0112374, + -0.025064934, + -0.13010886, + 0.060540106, + 0.06995255, + -0.07389344, + 0.19672123, + 0.07199163, + -0.10079826, + -0.2906196, + -0.02971355, + -0.11642807, + 0.010727101, + -0.01761672, + -0.024298944, + -0.09158621, + -0.061693575, + 0.10451686, + -0.19965115, + -0.16541333, + -0.07537575, + 0.1343618, + 0.123472676, + 0.26711482, + -0.123669244, + -0.27683747, + 0.15709896, + 0.063465126, + 0.0615493, + 0.15080597, + -0.17174396, + -0.18264005, + 0.1107746, + 0.0067112264, + 0.31034142, + 0.07599152, + -0.0509026, + 0.009105364, + 0.11131923, + 0.060135517, + -0.17910625, + 0.035435993, + 0.086380795, + -0.06450933, + 0.055679657, + 0.09171713, + 0.2905432, + 0.18213777, + -0.16435088, + -0.033460796, + 0.032459106, + -0.06308164, + 0.021363614, + 0.10111545, + -0.10463408, + -0.009399844, + 0.023662485, + -0.075450815, + -0.08625308, + 0.03425609, + 0.0023995063, + 0.06754814, + 0.028587475, + 0.07141926, + -0.10899464, + -0.047308013, + 0.026210437, + 0.13202062, + 0.019719098, + -0.12279105, + -0.007736201, + 0.18840674, + 0.13595977, + -0.08788261, + -0.0676793, + -0.081188634, + 0.05922845, + 0.054195378, + 0.019523846, + 0.03297895, + -0.029749334, + 0.062380422, + -0.063839525, + 0.11746603, + 0.011709679, + 0.10049474, + -0.05289085, + -0.03948768, + -0.054421388, + 0.01328617, + -0.2833275, + -0.022533081, + 0.10512336, + -0.029588947, + 0.052098297, + 0.17958073, + 0.03313654, + -0.019853612, + 0.13525973, + 0.1127346, + 0.2221017, + 0.0035275596, + 0.06705968, + 0.14495128, + 0.019332787, + 0.01311561, + -0.030842064, + 0.037300523, + -0.12344496, + -0.24704461, + -0.20622453, + 0.11744558, + -0.084686, + 0.013970482, + 0.035322703, + 0.05975246, + -0.0038897332, + -0.07505102, + 0.051193774, + 0.10900311, + 0.083961606, + -0.0867449, + -0.032312248, + 0.14108476, + -0.06840601, + 0.0072576064, + 0.058360774, + -0.08088482, + -0.11373591, + 0.07676662, + -0.09543414, + 0.011987333, + 0.08126872, + -0.06535875, + 0.03102977, + 0.047054287, + 0.02478329, + -0.09908036, + 0.21217299, + 0.1383378, + 0.082068495, + 0.17698748, + 0.07193557, + 0.2344127, + -0.03208311, + -0.04145339, + 0.023168517, + 0.1236235, + -0.12498678, + 0.016663183, + -0.12038983, + -0.14532413, + 0.038567517, + -0.18034275, + -0.015200795, + 0.16699447, + -0.074562944, + -0.16833735, + -0.015414858, + 0.29435298, + -0.16669613, + 0.06419434, + 0.07672183, + 0.2750522, + -0.0076842927, + 0.06513065, + 0.0045497296, + -0.02302735, + 0.05604058, + -0.015285121, + -0.090965, + 0.013308471, + 0.02668532, + -0.09131679, + 0.007411643, + 0.11996509, + -0.20453662, + -0.109074995, + 0.21115391, + -0.048814315, + 0.094790995, + -0.12637646, + 0.00034142684, + -0.058226816, + -0.21736974, + 0.05987529, + 0.0670604, + -0.008285919, + -0.100198664, + 0.21597408, + 0.038501117, + 0.04248908, + -0.12107101, + 0.14713411, + -0.10302279, + 0.04554846, + 0.17807189, + 0.00014714275, + 0.053678766, + -0.05470793, + -0.1468645, + 0.13801947, + 0.06114068, + -0.06495822, + 0.08971697, + 0.012763572, + 0.050100397, + -0.047830675, + -0.069687314, + 0.03206434, + -0.026934445, + -0.039816517, + 0.026246255, + -0.053072933, + -0.069831535, + 0.09188684, + 0.09086396, + 0.05913434, + -0.0142745795, + 0.15986186, + -0.114475, + -0.15942223, + -0.01427824, + -0.10161266, + -0.068465814, + 0.03319923, + -0.16473934, + -0.078048386, + -0.06126856, + 0.27868116, + -0.0040932256, + -0.0390591, + 0.18008506, + 0.11380149, + 0.008093586, + -0.025110045, + -0.13809349, + -0.26707006, + -0.03321406, + 0.10180727, + -0.012883773, + -0.06458295, + 0.17459899, + -0.24968287, + -0.10938166, + 0.14730081, + -0.16311902, + -0.02548285, + -0.19513816, + 0.37816283, + 0.17008424, + 0.18801682, + -0.068766676, + -0.10833149, + -0.06584093, + 0.005282863, + 0.05705433, + -0.06359804, + -0.011743776, + -0.05460606, + 0.10435172, + 0.069224454, + -0.2127962, + -0.1429357, + -0.08955556, + 0.105138846, + 0.108842775, + 0.18029512, + -0.0558439, + 0.095631614, + 0.12960093, + 0.0037637828, + 0.03426417, + -0.20179549, + -0.21381183, + -0.11177129, + -0.05186325, + -0.15315644, + 0.083750695, + 0.14294466, + 0.06936303, + 0.097568415, + -0.027851103, + 0.10691856, + -0.04703076, + 0.056984775, + 0.056401536, + -0.0071451035, + -0.11501074, + -0.24542804, + 0.17336452, + 0.028761476, + 0.052476797, + -0.038022522, + 0.044432264, + -0.16253045, + -0.09647879, + 0.07095663, + 0.08605165, + -0.11119516, + -0.058758993, + -0.027047373, + 0.05835111, + -0.004415728, + 0.24493673, + 0.012733293, + -0.12034067, + 0.076820046, + -0.007837615, + 0.061275806, + -0.100934975, + -0.15823844, + -0.009439666, + 0.28618845, + -0.21604861, + 0.18269648, + -0.043275967, + -0.05298801, + -0.06984093, + -0.062074646, + -0.2558363, + -0.082004085, + 0.08930997, + 0.05140848, + -0.11594375, + 0.16008247, + -0.01647563, + 0.005979772, + -0.055181064, + 0.03094402, + 0.057006262, + 0.037506513, + 0.22195199, + -0.15976258, + -0.003430983, + -0.03736374, + -0.052652735, + 0.09027966, + 0.32088894, + -0.117658705, + 0.07841281, + -0.007944, + -0.037882786, + 0.13079576, + 0.030691518, + -0.012422775, + -0.005092901, + -0.06255673, + 0.032830406, + -0.03967016, + -0.112258725, + -0.22631066, + -0.16136739, + -0.15279353, + -0.14530544, + 0.027644537, + -0.013307016, + -0.12973621, + -0.22201909, + 0.047920972, + -0.21610388, + -0.05441284, + 0.0869944, + -0.0120201865, + -0.16603912, + -0.027903512, + 0.14737375, + -0.045106143, + -0.10809847, + 0.12977672, + 0.06122886, + 0.1714344, + 0.10619558, + 0.05405039, + -0.095592745, + 0.022597203, + -0.10200969, + -0.06988007, + 0.17373843, + -0.041029654, + -0.096191384, + 0.036363162, + -0.040415987, + 0.113239214, + 0.11083686, + 0.11057566, + 0.021456886, + -0.1187632, + -0.005398725, + -0.09654723, + -0.13158664, + -0.16231221, + -0.13153927, + -0.07664805, + 0.18590742, + 0.022832684, + -0.03676107, + -0.16832295, + 0.27945006, + -0.03835614, + 0.25090364, + 0.08288016, + 0.056393255, + 0.15483865, + 0.036503028, + 0.012308527, + -0.16759615, + -0.14400983, + 0.15403487, + -0.07984099, + 0.103513256, + 0.05800183, + 0.13537936, + 0.06597946, + 0.09718602, + -0.0009663588, + 0.009373034, + 0.049974643, + 0.10922762, + -0.0666244, + 0.054338492, + -0.2523631, + 0.12637684, + 0.12406432, + -0.0133141745, + -0.14440738, + 0.18514237, + 0.03871962, + -0.00856082, + -0.03739144, + 0.103693984, + 0.38904634, + -0.12921004, + 0.20685571, + 0.0717717, + -0.017677164, + -0.13859999, + -0.047213227, + -0.035712585, + -0.018850666, + -0.032312583, + 0.1963019, + -0.22302505, + 0.059585784, + 0.07076368, + -0.0037386497, + -0.021374866, + -0.09076489, + -0.05121408, + 0.06617326, + 0.16769287, + -0.0759916, + 0.00011955293, + -0.071625695, + 0.05203951, + -0.11306046, + -0.13112223, + 0.024701066, + -0.045229748, + 0.21586764, + 0.1884639, + 0.014671893, + 0.07167805, + 0.086855225, + 0.21806249, + 0.0010663769, + -0.09161673, + -0.07338588, + 0.18400705, + 0.11373605, + -0.007007729, + 0.08980646, + -0.27008447, + -0.22743385, + 0.108092554, + -0.10263347, + -0.12530118, + 0.011223255, + -0.02534834, + 0.11997918, + 0.00043334352, + -0.12639672, + 0.07851988, + -0.06799492, + -0.021804351, + 0.022667186, + 0.039111875, + -0.14440686, + -0.00038854606, + 0.04258943, + -0.03997176, + 0.16821238, + -0.068418495, + 0.11187042, + 0.07045411, + 0.054948222, + 0.09370843, + 0.11527065, + -0.12156669, + 0.23117429, + 0.14954045, + -0.050761856, + 0.18862389, + 0.17474213, + 0.04331295, + -0.08087375, + -0.057447176, + -0.03303781, + 0.10136958, + -0.09814324, + 0.07164336, + 0.01354199, + -0.090047404, + -0.019989517, + -0.024555784, + -0.0873952, + -0.021014484, + -0.1468437, + 0.06360805, + -0.009553678, + -0.2520642, + -0.0009906373, + 0.043536548, + 0.15728356, + 0.1157493, + 0.07684819, + -0.0066023595, + -0.15029529, + -0.04891407, + 0.07041565, + 0.019646177, + 0.15459754, + -0.055917192, + -0.13702592, + -0.08028114, + -0.005777796, + 0.005830356, + -0.022148266, + -0.03957986, + -0.1078105, + -0.00035169302, + -0.041368164, + 0.024077183, + -0.29544196, + -0.08077371, + -0.013104905, + 0.041448615, + -0.010947219, + -0.034304015, + 0.016416628, + 0.27215916, + -0.104368046, + 0.07733927, + 0.22092457, + 0.1287746, + -0.030926289, + 0.14851579, + -0.096314825, + 0.00039009872, + 0.010219875, + 0.056555286, + 0.00847581, + -0.04259313, + -0.0931727, + -0.0684084, + -0.115641244, + 0.14368759, + -0.16151942, + -0.24075364, + 0.13465847, + 0.16644302, + 0.005077254, + 0.13074507, + -0.023705306, + -0.061494812, + -0.08048761, + -0.060518295, + 0.06539182, + 0.029792016, + 0.03118727, + 0.006684678, + -0.06381983, + -0.0007743956, + 0.015529249, + -0.06939081, + 0.07917838, + -0.0718619, + -0.11937301, + 0.010090283, + -0.08640019, + -0.18818526, + -0.11591951, + 0.18826221, + -0.1300556, + 0.24455251, + -0.010264277, + 0.24497488, + 0.09312586, + 0.022135185, + 0.051192943, + -0.16545711, + -0.11049563, + -0.1811448, + 0.14125228, + -0.13107261, + 0.028541027, + 0.15770547, + 0.026281191, + 0.050461553, + 0.061548013, + 0.12506539, + -0.102001585, + 0.061886996, + 0.15549985, + -0.026153754, + -0.084029496, + 0.10270605, + -0.06070375, + -0.16488475, + -0.027996136, + -0.074855156, + -0.006563445, + 0.006692481, + 0.11110623, + 0.21137123, + 0.06501346, + 0.007817524, + -0.08232954, + 0.0263958, + 0.011379713, + -0.04867242, + 0.18348722, + -0.13841671, + -0.06814913, + 0.04329171, + 0.03316773, + -0.14990935, + 0.10580919, + -0.22892036, + -0.04298803, + -0.12009234, + 0.092555754, + -0.07022957, + -0.25958672, + 0.04043409, + -0.116474554, + -0.07289303, + 0.056018278, + 0.009886767, + 0.1267809, + 0.04049609, + -0.08856585, + 0.04917458, + 0.10745919, + 0.0031032898, + -0.0776394, + 0.021187214, + 0.07920715, + -0.053854663, + -0.09190022, + -0.15109119, + 0.017716136, + -0.10367998, + 0.18685561, + 0.22195326, + -0.12602778, + 0.1713986, + -0.012507223, + -0.05165178, + 0.046761766, + 0.09106876, + -0.074691705, + 0.017120015, + 0.037856784, + 0.11998435, + -0.11249925, + -0.111552395, + -0.022593446, + 0.023815157, + 0.19325908, + 0.2730344, + -0.007951937, + 0.12512314, + -0.024566812, + 0.047931004, + 0.12275496, + 0.16077788, + -0.09789672, + 0.03344916, + -0.16171698, + -0.016622083, + 0.038192052, + 0.016207283, + -0.0202107, + -0.13984105, + -0.03407714, + 0.12702416, + -0.14115557, + -0.019655803, + -0.16221006, + -0.01791215, + -0.13691811, + 0.12804243, + -0.011204896, + 0.01841803, + 0.0050965617, + -0.08466767, + 0.016319202, + 0.007212669, + -0.04629895, + -0.06361673, + -0.027919175, + -0.041548878, + -0.06237486, + 0.1745566, + -0.043050725, + 0.038423937, + 0.18671963, + 0.018245509, + -0.029951867, + -0.22200696, + -0.057974942, + -0.057651896, + -0.055361606, + 0.18368134, + -0.15395811, + -0.080219485, + -0.21031596, + 0.09167013, + -0.06837816, + 0.13859786, + -0.13372809, + 0.07269533, + 0.11164036, + -0.072583176, + -0.021096015, + -0.23613524, + 0.05250386, + 0.044930402, + -0.01711093, + -0.07612404, + 0.03333045, + 0.11559862, + 0.10009551, + 0.15378518, + 0.11826541, + -0.1216371, + -0.087640956, + -0.23119548, + -0.023505269, + -0.039743233, + -0.15849829, + -0.022412464, + -0.098236606, + 0.018338203, + -0.056111015, + 0.01976757, + 0.047145586, + 0.1796975, + 0.07133825, + -0.05501143, + 0.0137379235, + 0.15215507, + -0.13460286, + 0.09070147, + 0.069535986, + 0.061831266, + -0.087164275, + 0.080333784, + 0.08281889, + 0.08870611, + 0.042467095, + 0.03364361, + 0.23295124, + -0.071372524, + 0.023481961, + -0.041838918, + -0.04865416, + 0.016903803, + 0.057054404, + -0.046940424, + -0.029821644, + 0.0018828119, + -0.11095171, + -0.07097106, + -0.055843648, + 0.00956623, + 0.018680362, + 0.0153555, + 0.017266344, + 0.00007468156, + 0.01172443, + -0.027056027, + 0.0065923487, + -0.06067196, + 0.13302639, + 0.10897519, + 0.08939442, + -0.12072664, + 0.22262308, + 0.16682461, + 0.15896632, + 0.11093958, + -0.03812856, + 0.041086815, + -0.16033763, + 0.13180858, + -0.01072775, + 0.104514994, + -0.0033954063, + -0.0977714, + -0.14432882, + -0.059497964, + -0.051815588, + 0.06572544, + 0.021578567, + -0.09776904, + 0.06565402, + -0.16587934, + -0.116609134, + -0.14652133, + 0.06313389, + 0.08367709, + -0.026228959 + ], + [ + 0.15846656, + -0.14894673, + -0.13188975, + -0.015371586, + 0.16705966, + -0.231484, + 0.0024328774, + 0.061065182, + 0.12463436, + 0.11706655, + -0.026946174, + 0.05513764, + -0.0051243478, + -0.008197161, + -0.032258887, + -0.016351407, + 0.18757029, + -0.15723023, + 0.16273841, + 0.15276295, + 0.014594901, + 0.009599693, + 0.09730296, + 0.077078134, + -0.17151691, + 0.10118174, + -0.06627298, + 0.11285701, + -0.103246175, + -0.04298179, + 0.057788603, + 0.3094538, + 0.073913105, + 0.14185263, + 0.08249015, + 0.034514878, + -0.015602923, + -0.029827323, + 0.13692331, + -0.109577194, + -0.09081349, + 0.10803755, + 0.06682779, + 0.029051548, + 0.1537387, + -0.075789124, + 0.1146328, + -0.038770474, + -0.024302071, + 0.13382743, + 0.049123622, + 0.008879571, + -0.07886671, + 0.074562654, + 0.14038633, + -0.10135582, + 0.15901671, + 0.13758342, + 0.13179119, + -0.044150967, + 0.10968406, + 0.028949987, + -0.021487163, + 0.0023873842, + 0.102438875, + 0.09836604, + 0.06578889, + 0.013700457, + -0.061243135, + -0.1439024, + 0.10743138, + -0.09807392, + 0.23552205, + 0.16922787, + -0.04566979, + -0.031479042, + 0.08064524, + -0.0140977055, + 0.052643966, + 0.14185508, + -0.04078149, + -0.0034885583, + 0.16458786, + 0.0752994, + -0.13292527, + -0.25739685, + 0.09674331, + 0.05962877, + 0.04741129, + -0.21546471, + 0.11757735, + 0.20755276, + 0.0030378932, + -0.06636101, + -0.028773654, + 0.153386, + 0.01590099, + -0.16970557, + 0.033176962, + 0.057676144, + 0.064098015, + 0.049269557, + 0.06532966, + -0.11312617, + 0.06794883, + -0.08994207, + 0.059467416, + 0.09691956, + -0.048683733, + -0.027496036, + 0.10316365, + -0.10428701, + 0.013541405, + 0.063007005, + -0.06387908, + 0.08398428, + -0.022002866, + 0.1102383, + 0.24936323, + 0.08888533, + 0.13566265, + 0.08911813, + 0.05534134, + 0.03426087, + -0.003912369, + -0.06930861, + -0.07926059, + -0.2927976, + -0.13223682, + -0.2020717, + -0.085529335, + -0.08397994, + -0.07512756, + -0.056016676, + -0.067682855, + 0.06265628, + 0.022324614, + -0.02904347, + -0.16194701, + 0.11525205, + 0.06782437, + -0.16947006, + 0.02137571, + 0.035497002, + -0.1502951, + 0.2020105, + -0.09598874, + -0.09472741, + 0.009307279, + 0.058853816, + 0.0008540991, + -0.11725075, + -0.18493211, + -0.088182926, + 0.14129642, + -0.006272462, + 0.15148452, + 0.17936167, + -0.051006615, + -0.005678193, + -0.12936504, + 0.118420176, + -0.091464624, + 0.010349933, + 0.114841565, + 0.12727495, + 0.01069694, + 0.27586663, + -0.03692025, + 0.026927534, + -0.08407196, + 0.079921246, + 0.07302816, + -0.03934301, + -0.008794134, + -0.039965097, + 0.11838427, + -0.048990693, + 0.0011952232, + 0.059630655, + -0.02073507, + -0.06985642, + 0.15482605, + -0.1547007, + 0.1256582, + -0.1299996, + -0.01868761, + -0.29615396, + 0.040472984, + 0.13947299, + 0.071826965, + -0.064604826, + 0.045524985, + -0.0013657572, + 0.00922413, + 0.054285023, + -0.037322026, + -0.055743903, + 0.1780619, + 0.019885443, + 0.12101634, + 0.048781477, + 0.17347799, + -0.0873945, + -0.13729812, + 0.082692884, + -0.15274675, + -0.12076281, + 0.093869224, + 0.25696653, + -0.060197536, + 0.11230131, + 0.0016297848, + 0.0036486778, + 0.025571777, + 0.047004957, + 0.1287568, + -0.09368466, + 0.08403961, + 0.14358337, + -0.018372541, + -0.040335115, + -0.14648063, + -0.32904556, + 0.037121747, + 0.19827436, + -0.16016959, + -0.08684652, + -0.03874674, + -0.20288484, + 0.14046177, + 0.19002816, + -0.090223774, + -0.08097232, + -0.21057487, + -0.010953297, + 0.17159285, + -0.060857728, + -0.057242908, + -0.07777571, + 0.1968416, + -0.027824707, + -0.08062887, + -0.042103395, + -0.080151476, + -0.12742521, + -0.15232758, + -0.11119279, + 0.10303499, + 0.074524805, + 0.018627008, + 0.083202824, + -0.038138278, + -0.16862473, + 0.15880917, + 0.03960908, + 0.053431354, + 0.056068275, + 0.06306108, + -0.057394594, + 0.060965676, + -0.12809429, + -0.025608107, + -0.04439726, + 0.14331713, + 0.0075398274, + 0.0030347155, + -0.18464282, + -0.009869805, + 0.073502645, + -0.009123688, + -0.069161676, + 0.005321287, + 0.13778484, + 0.18855691, + 0.14130345, + -0.06832302, + -0.047158428, + -0.0030905677, + -0.011824866, + -0.024031462, + -0.0598138, + 0.07862906, + -0.09125566, + -0.040174685, + -0.1162901, + -0.019563397, + -0.022721067, + -0.002791389, + -0.028123276, + 0.20853533, + 0.055247508, + -0.009078611, + 0.05638552, + -0.0861065, + -0.19664645, + -0.068449125, + -0.004284521, + -0.12818453, + -0.054515485, + -0.21327844, + -0.022304073, + -0.014190555, + -0.059023093, + 0.014396681, + 0.015331843, + 0.121404916, + -0.15422125, + 0.023859633, + 0.262496, + 0.12140163, + -0.12423673, + 0.053574026, + -0.07480659, + 0.009966959, + 0.0005245576, + 0.10235568, + 0.10102288, + -0.023910532, + -0.20969674, + -0.05751784, + 0.06074621, + -0.030705856, + -0.108660564, + 0.10981022, + -0.022167629, + -0.074983954, + 0.14298001, + 0.15487315, + -0.105472706, + 0.12458858, + -0.011921662, + -0.0843357, + 0.080602966, + -0.05240729, + -0.078765005, + -0.06236435, + 0.033246577, + -0.16713138, + -0.16168198, + 0.02604236, + -0.08032973, + -0.09142687, + -0.13910621, + -0.050133, + 0.05249491, + 0.029800884, + -0.18045552, + 0.11533404, + 0.0036546546, + 0.13957272, + 0.09065534, + -0.07983853, + 0.065196075, + 0.00483309, + -0.30357614, + 0.1300592, + 0.14113303, + 0.045899827, + -0.17832467, + 0.16170332, + -0.101678856, + -0.03701216, + -0.25509036, + -0.025909375, + -0.024960186, + -0.027482023, + 0.018042693, + -0.009376487, + -0.011157068, + 0.08628339, + -0.123575084, + 0.08299967, + 0.09093684, + -0.035794947, + -0.0690082, + 0.15115522, + 0.07879298, + -0.17823504, + 0.052165683, + -0.18495956, + 0.06324202, + 0.078495614, + 0.057631716, + 0.14990783, + -0.04525075, + -0.029693574, + 0.0128731625, + -0.08203428, + -0.10327234, + -0.06030997, + -0.0077230334, + 0.115389444, + -0.08682941, + 0.079778865, + -0.060847305, + -0.17431593, + 0.10438322, + 0.012507549, + 0.052554544, + 0.12766378, + -0.115853414, + -0.008095368, + 0.06318096, + 0.022107618, + -0.058577165, + 0.025625072, + 0.18896843, + 0.11701097, + -0.03345069, + -0.046744175, + 0.02987199, + 0.018711489, + 0.004343268, + 0.034436073, + -0.033576973, + -0.056621, + 0.07035622, + 0.019453403, + -0.12953557, + 0.026528751, + 0.10356629, + 0.0403922, + 0.23169217, + 0.10239691, + 0.028832285, + 0.1697025, + -0.003531267, + -0.033742674, + -0.23828392, + -0.0671558, + -0.07060434, + -0.12074501, + -0.12699483, + -0.04256371, + -0.04909332, + 0.09225179, + 0.309755, + 0.04118996, + -0.10893188, + -0.124176696, + 0.08212391, + -0.18666762, + 0.28523374, + 0.030693185, + -0.14264916, + 0.10614599, + 0.02713271, + 0.06335521, + -0.19940498, + -0.18611884, + -0.1278629, + -0.015504656, + 0.08729648, + 0.17531921, + -0.02825543, + -0.06725592, + -0.0005201547, + 0.011050643, + 0.077033885, + -0.020947594, + -0.0773544, + 0.10796558, + 0.15403922, + 0.0063380264, + -0.08978346, + -0.204632, + -0.08047593, + 0.09311416, + 0.09013907, + -0.09525472, + -0.10154838, + -0.024830816, + 0.17287362, + 0.18489592, + 0.11841823, + -0.019611297, + 0.10096258, + -0.06806371, + -0.08465169, + -0.0013794691, + 0.111672156, + 0.005498556, + -0.047415145, + -0.03359179, + -0.059774265, + -0.098724715, + 0.013236173, + 0.08048248, + 0.24152869, + -0.10831418, + 0.07953544, + -0.09127184, + 0.044242486, + -0.009510906, + 0.12005419, + -0.040822737, + 0.03278505, + -0.07833108, + 0.11430967, + 0.11994177, + -0.098867826, + -0.15677744, + 0.069522895, + -0.25672966, + -0.011805372, + 0.07292897, + -0.082478516, + 0.02561567, + -0.047698513, + 0.13341406, + 0.12818106, + 0.044617876, + -0.40938342, + 0.15651107, + -0.13003437, + 0.07099373, + 0.09285106, + 0.12612762, + -0.017611964, + 0.12529397, + 0.17717384, + -0.2243484, + -0.019332271, + 0.1802624, + -0.13767661, + 0.01961787, + -0.065574445, + 0.024493206, + 0.07592765, + -0.12865902, + -0.06986012, + -0.15317339, + 0.06604938, + 0.014160292, + 0.06368592, + 0.05778993, + -0.06879842, + -0.3333686, + 0.07081244, + -0.08865506, + 0.0010657859, + 0.020205896, + -0.094921045, + -0.061092082, + -0.034534458, + 0.033048082, + -0.0042976537, + -0.15719782, + -0.1924763, + 0.06796789, + 0.1328857, + 0.018388335, + 0.11101402, + 0.109840795, + -0.06254849, + 0.004293073, + 0.15740435, + 0.17644696, + 0.0058195884, + 0.13029478, + 0.071573466, + -0.20421873, + -0.0050412193, + 0.18835844, + 0.07271274, + -0.112578, + 0.16781381, + -0.014131367, + -0.10212361, + -0.08584278, + 0.031238567, + -0.21272327, + -0.0694137, + 0.1308, + -0.053156145, + 0.096017756, + 0.109670356, + -0.09303574, + 0.0563351, + -0.1033751, + -0.010472144, + 0.032520525, + -0.1469482, + -0.15429942, + -0.09321035, + -0.016487926, + 0.20083293, + -0.01745232, + 0.042721696, + 0.14180313, + 0.21574003, + 0.073189884, + 0.09661394, + -0.19908611, + -0.11430697, + 0.017988099, + 0.09320381, + -0.29125306, + 0.033173177, + -0.06468759, + -0.07561015, + 0.03300521, + -0.033276286, + 0.024739988, + -0.026963402, + 0.10801799, + 0.13346595, + -0.14806463, + 0.047130696, + -0.06427405, + -0.13276349, + -0.1571873, + -0.1720413, + -0.019900773, + 0.004509936, + 0.017361667, + 0.021702256, + -0.06365898, + -0.012174513, + 0.043270823, + 0.16917661, + -0.040842917, + -0.036232848, + 0.012658868, + 0.054902602, + -0.20585628, + -0.054428756, + -0.045206092, + -0.04531314, + -0.0789444, + -0.25269538, + 0.061517183, + -0.1425821, + -0.11450696, + 0.028789515, + 0.19709642, + 0.08766006, + -0.021093508, + -0.07813938, + -0.032772005, + -0.07292086, + 0.24164371, + -0.08440194, + -0.027143363, + -0.048491545, + 0.015090795, + 0.1070741, + 0.030939583, + -0.09336118, + -0.14727467, + 0.07165781, + 0.08620502, + 0.02424046, + -0.08498235, + 0.12279331, + 0.10719259, + -0.049552344, + -0.116854385, + -0.23771396, + -0.0920385, + -0.032208342, + -0.11301597, + -0.08382864, + -0.11445808, + -0.04390611, + 0.1032361, + 0.0013361671, + -0.16949497, + -0.14225605, + 0.021628316, + -0.15692602, + 0.26118505, + 0.11900645, + -0.053895447, + 0.06433893, + 0.21862704, + 0.029760117, + 0.09066999, + -0.06886031, + -0.03807151, + 0.05981284, + -0.08681997, + 0.1286995, + 0.01431086, + -0.022292813, + -0.0005013467, + 0.25885803, + -0.04367855, + -0.035725366, + 0.05130664, + -0.15069792, + 0.00079145544, + -0.056092262, + -0.072985694, + 0.030023174, + -0.06899001, + -0.0975411, + 0.008925394, + -0.19356744, + -0.096645854, + 0.08922534, + -0.26041025, + -0.11038632, + -0.12035151, + 0.047138903, + -0.04014047, + -0.005079972, + -0.08453478, + 0.0108782975, + -0.055437442, + -0.024233581, + 0.13133909, + 0.017852798, + 0.09230822, + 0.05934381, + 0.032781035, + 0.044325385, + -0.056726612, + 0.09054025, + 0.30902827, + 0.042634513, + -0.11351706, + -0.10184977, + -0.15892436, + 0.07752234, + 0.063255265, + 0.03392058, + -0.05138785, + -0.1493424, + 0.01198075, + -0.14310691, + 0.021757988, + 0.13613899, + -0.01396091, + 0.01895428, + -0.041307848, + -0.032796502, + 0.039798666, + 0.03955335, + -0.11712413, + -0.007823799, + -0.08758357, + 0.36238423, + 0.045195565, + 0.042123098, + -0.0118451305, + -0.011359909, + 0.10570221, + -0.06508535, + -0.21233514, + 0.12879884, + 0.00801969, + -0.20314407, + -0.02493685, + -0.026470685, + -0.13859881, + 0.027010472, + -0.10404782, + -0.00038773075, + 0.0040764627, + 0.07198745, + 0.03613616, + 0.082790814, + 0.12304106, + 0.15090622, + -0.12975182, + 0.1422943, + 0.32201743, + -0.11158703, + -0.06097697, + 0.007564108, + -0.1664216, + -0.035444904, + -0.028393103, + 0.14315064, + 0.11850086, + -0.024217011, + -0.07844546, + -0.040807687, + -0.028631993, + 0.039424993, + 0.059636302, + -0.073344104, + 0.04405877, + -0.08296971, + 0.06776528, + 0.095045164, + 0.0041737542, + 0.063781425, + 0.12505543, + 0.08125786, + -0.11541801, + 0.106116205, + -0.21122476, + -0.051326115, + -0.11984397, + -0.05864495, + 0.025218667, + -0.06094618, + 0.12133363, + 0.09911744, + 0.26462877, + -0.12545985, + 0.11903367, + -0.105698176, + -0.14354075, + 0.0644074, + -0.09552613, + -0.033709824, + -0.09168388, + 0.18870622, + 0.0010486201, + -0.0814779, + -0.044366695, + -0.007999213, + -0.099657305, + -0.012411787, + 0.19009852, + -0.09644196, + 0.03648562, + 0.08932092, + 0.011254463, + -0.18018445, + -0.18829237, + -0.031496696, + 0.04360538, + 0.15007122, + -0.06060969, + 0.06423649, + -0.018384177, + -0.1379392, + 0.10824323, + 0.011449396, + -0.005312135, + -0.12633747, + 0.18985581, + -0.050494656, + 0.27826023, + 0.04518124, + 0.0762414, + -0.15507232, + 0.05498624, + 0.11095998, + -0.07603957, + 0.11104304, + 0.038726956, + -0.06857704, + -0.050024584, + -0.14961046, + -0.042043075, + 0.09258056, + -0.027747488, + -0.19722545, + -0.025414722, + -0.05644636, + -0.0046709403, + -0.07148298, + 0.13278374, + 0.00003596043, + 0.17292266, + 0.0392831, + 0.076638624, + -0.25348988, + -0.048064888, + 0.040796805, + -0.1317034, + -0.010435862, + 0.057497643, + 0.02138421, + -0.12928724, + 0.1843052, + -0.0032561622, + 0.02560748, + 0.0040482255, + 0.16645108, + 0.1124439, + 0.07103201, + -0.017360581, + -0.14021014, + -0.2761286, + -0.027650936, + -0.087520726, + 0.1284345, + -0.0020231095, + -0.16560674, + -0.13692503, + 0.06287243, + -0.07822542, + 0.035219744, + -0.072541155, + -0.09010822, + -0.04351065, + 0.027134832, + 0.06755826, + 0.12301915, + 0.0049245586, + 0.09186711, + -0.09780905, + 0.16602924, + 0.060643964, + 0.09915961, + -0.12730426, + -0.012174859, + -0.05269128, + -0.041920573, + 0.13201751, + -0.038913008, + -0.051302686, + 0.021553233, + 0.06974902, + 0.037005715, + -0.0075630434, + 0.095330544, + -0.1538154, + -0.020467883, + -0.046345387, + -0.13503817, + 0.0069053033, + 0.03509617, + 0.22232045, + 0.10840425, + 0.077698134, + 0.09895408, + -0.0935128, + 0.17952754, + 0.055660255, + 0.17371093, + -0.07007724, + 0.09766003, + -0.07297232, + 0.048130978, + -0.11572883, + -0.032660715, + 0.08643092, + -0.033975136, + 0.25975278, + 0.035782024, + -0.13298692, + 0.07752003, + 0.057088308, + -0.011036775, + -0.1296041, + -0.14754817, + -0.09915362, + -0.09061552, + 0.15483025, + 0.086246595, + 0.21902007, + 0.06317361, + -0.0017731533, + 0.14258327, + 0.009094075, + 0.16113617, + -0.080414616, + -0.10949779, + 0.01014872, + -0.17496696, + -0.084411696, + -0.10173003, + 0.04112154, + -0.07685115, + -0.20684911, + 0.10517152, + 0.01482347, + -0.0013761881, + 0.03887268, + 0.09994203, + -0.014113231, + -0.13717343, + 0.03788843, + -0.042142607, + -0.04741655, + -0.16435277, + 0.015987214, + -0.093827575, + 0.04590369, + -0.0027739417, + -0.06706411, + 0.044482827, + 0.022706142, + -0.028011689, + -0.12265332, + 0.083518095, + -0.0049355505, + 0.01921485, + -0.050097317, + -0.02169908, + 0.0050034286, + 0.034458358, + -0.009163651, + 0.065465175, + -0.03930148, + -0.061662227, + 0.041233227, + -0.099589385, + -0.11675253, + -0.053536303, + 0.11452381, + -0.24844523, + -0.1071091, + -0.11363072, + -0.02298112, + -0.025474198, + -0.17504856, + 0.16830038, + -0.07467595, + -0.008802671, + -0.06480381, + 0.011079027, + -0.03456703, + 0.18757644, + 0.031754997, + -0.038898297, + -0.09223578, + 0.024588306, + 0.07565296, + -0.0666662, + 0.083090164, + 0.14365932, + 0.09664895, + -0.07777618, + 0.0065826504, + 0.13500947 + ], + [ + 0.07745762, + 0.06945646, + 0.0036094207, + -0.0055313776, + -0.020755991, + -0.0612659, + -0.072094016, + 0.024165524, + 0.041853517, + -0.0040279394, + -0.048059803, + 0.076396756, + -0.040438388, + -0.10731963, + 0.002041552, + 0.06662044, + -0.083712466, + 0.03109451, + -0.11639113, + 0.027863607, + -0.0071296943, + 0.014727122, + 0.036048975, + -0.08327996, + 0.03339005, + -0.055133313, + 0.101876795, + 0.029641671, + -0.03270176, + -0.15706189, + -0.12485806, + 0.01815152, + -0.050905254, + 0.08386188, + -0.09248747, + -0.01876746, + -0.006875415, + 0.004384827, + 0.022515684, + -0.002542826, + 0.15354718, + -0.030193746, + 0.05164544, + 0.075130545, + -0.03626094, + -0.033566207, + -0.15856253, + 0.0890807, + 0.0026878468, + -0.15034702, + 0.01362214, + 0.026168939, + 0.054399718, + 0.08277522, + 0.02284142, + 0.018333904, + 0.054166045, + -0.11102487, + -0.12867075, + 0.023105752, + -0.0013759588, + 0.024404915, + 0.01441747, + -0.0056235488, + -0.047793347, + -0.03161983, + -0.07000958, + -0.025425902, + 0.036375098, + -0.020278316, + 0.014310252, + 0.01770713, + -0.049456652, + 0.017257914, + 0.101281986, + -0.07987475, + -0.10163229, + 0.06947092, + 0.055335134, + 0.12672764, + 0.09583257, + -0.038959898, + -0.063746974, + 0.03826194, + -0.004315776, + -0.0017085372, + 0.03843952, + -0.103880465, + 0.00324448, + 0.07779084, + -0.02276321, + -0.102520965, + 0.09265732, + 0.014894296, + 0.14571168, + 0.03472724, + 0.005417286, + 0.08473566, + -0.033137944, + -0.09661821, + 0.010165947, + -0.0006157164, + -0.048117362, + -0.013838345, + -0.14789234, + 0.01738775, + -0.036344845, + 0.0042748144, + 0.112602405, + 0.037470084, + 0.119747445, + 0.05632335, + -0.018556254, + -0.030634018, + 0.062119953, + -0.020871485, + 0.06366857, + 0.051357854, + -0.021132894, + -0.037945222, + -0.20233276, + 0.08641417, + -0.1338063, + -0.00701211, + 0.02216182, + 0.0271528, + -0.01124527, + 0.011358738, + -0.05184234, + -0.0041546402, + 0.060687024, + -0.07045171, + -0.079010054, + -0.010511909, + 0.115421556, + -0.02961444, + 0.019408198, + -0.020929804, + 0.12784983, + 0.058592763, + -0.01414183, + -0.07932492, + 0.032895025, + 0.034836117, + -0.037314806, + 0.012174778, + 0.027697226, + -0.015306631, + 0.022342695, + -0.036096424, + 0.088026896, + -0.02211972, + 0.029572042, + -0.031676978, + -0.019258121, + -0.006421155, + -0.122418076, + -0.0723774, + 0.047560133, + -0.19946514, + 0.11007185, + -0.09542355, + -0.058804367, + 0.06274386, + -0.037530933, + -0.009842537, + 0.051844392, + -0.019464014, + -0.016950997, + 0.04618182, + 0.0024364518, + -0.0034955442, + 0.059651207, + 0.0428781, + 0.0030017633, + -0.0074899355, + -0.12870409, + -0.0022689754, + -0.0026355973, + 0.010541111, + 0.0038926431, + -0.01143113, + 0.048611894, + -0.117749445, + -0.030256076, + 0.10148345, + -0.08466635, + 0.042832922, + 0.009266726, + -0.009537506, + -0.081211306, + -0.08369359, + -0.024817122, + -0.03711743, + 0.040294554, + -0.10411341, + 0.10903301, + 0.08959415, + 0.016019406, + -0.010338482, + -0.10977886, + -0.023716526, + -0.04336907, + 0.026854422, + 0.067345515, + -0.008799443, + -0.043974437, + 0.04877117, + 0.006929042, + 0.033653125, + 0.08991103, + 0.0041267015, + -0.012356885, + 0.0139680505, + -0.0012728166, + 0.04458126, + -0.06706323, + -0.010832285, + 0.11561156, + 0.021829976, + 0.09374831, + -0.09402673, + 0.023007879, + 0.061930645, + 0.086547494, + 0.078563154, + 0.0030706376, + 0.023381444, + -0.022802526, + 0.05207883, + 0.031282157, + -0.007388119, + -0.020550888, + 0.07426106, + -0.099766485, + -0.06503823, + -0.120690316, + 0.05175232, + -0.05407624, + 0.017640108, + -0.017735312, + -0.0076112943, + -0.005751937, + 0.10661272, + 0.024471821, + 0.03570812, + -0.13821566, + -0.07094514, + 0.06131591, + -0.041064017, + -0.05108608, + -0.020121247, + 0.016774653, + 0.067509264, + -0.05417521, + -0.030209228, + -0.07709386, + 0.019382522, + 0.023191983, + 0.023138497, + -0.0633652, + 0.06054607, + -0.0029793808, + -0.027022064, + -0.034322478, + 0.039768178, + -0.029245159, + 0.05262219, + -0.009849279, + 0.049288478, + 0.01685988, + -0.11924519, + -0.14128108, + -0.049902253, + -0.12087765, + -0.060114726, + 0.009094624, + -0.0070369425, + -0.016433297, + -0.021154065, + 0.053780776, + -0.0025666319, + -0.014789462, + -0.026210396, + 0.028465025, + 0.01579537, + 0.07245419, + 0.062208753, + 0.05721281, + 0.014837121, + 0.074792154, + -0.03271447, + -0.09604473, + -0.026796577, + 0.03400821, + 0.017065603, + 0.035312586, + 0.02413816, + -0.00707516, + -0.058219112, + -0.00882335, + 0.01877651, + -0.05286912, + -0.085928194, + -0.033729453, + -0.028447466, + -0.090562835, + 0.116920725, + 0.099585325, + -0.015451496, + 0.08482293, + 0.006791124, + 0.031112006, + 0.060836084, + -0.04528965, + -0.060277015, + -0.09846596, + -0.025971867, + -0.027283201, + 0.10069358, + -0.026443765, + 0.02599866, + 0.063018076, + -0.03242416, + -0.0414042, + 0.014910042, + 0.016623162, + -0.06209513, + -0.041137267, + 0.028706074, + -0.029154455, + -0.00006567799, + 0.0030669977, + -0.015244235, + 0.021377513, + 0.03171063, + 0.037581064, + 0.06583381, + -0.045041483, + 0.0233442, + -0.108110875, + -0.096657746, + 0.019491378, + -0.095306545, + 0.08221612, + -0.011041563, + -0.05296942, + 0.056832075, + 0.009652748, + -0.020317838, + -0.01918292, + 0.054668766, + 0.04127102, + 0.1151467, + 0.010367705, + 0.13658163, + -0.055496324, + 0.0272043, + -0.030484973, + -0.116028234, + 0.00499293, + 0.037484445, + -0.03970968, + -0.01996194, + 0.033593677, + -0.040170476, + 0.022932295, + 0.004984673, + -0.04666409, + 0.026442192, + 0.00468162, + 0.1102666, + -0.10968494, + 0.00037995595, + 0.028383309, + -0.050956897, + 0.053381402, + 0.03028198, + -0.032919087, + -0.0033513985, + 0.09016863, + 0.063134365, + -0.078837015, + 0.0005950799, + 0.067461945, + -0.014624532, + -0.037366915, + -0.004587243, + 0.18223287, + -0.11270174, + 0.02180889, + -0.034766078, + 0.0056815967, + -0.06030472, + 0.023243932, + 0.015298831, + -0.0037559464, + -0.104530334, + -0.0661863, + 0.0021533796, + 0.01920063, + -0.022166068, + -0.053045332, + 0.001777255, + -0.11182458, + 0.04581356, + -0.07572549, + -0.14113775, + -0.047323607, + 0.0013863543, + 0.07367861, + 0.009944513, + -0.044957746, + 0.04872967, + 0.035395484, + -0.058309883, + -0.034243003, + -0.12949067, + -0.0051788217, + -0.030326268, + 0.087946266, + 0.055856027, + -0.026113957, + -0.13007952, + -0.0339311, + -0.06203155, + -0.004963134, + 0.0815797, + 0.09501125, + -0.040397406, + 0.07124904, + 0.10966349, + -0.05934003, + 0.013193269, + 0.034470137, + 0.07420334, + 0.04584168, + -0.04268478, + -0.007975875, + 0.007124554, + 0.13706046, + -0.017540988, + 0.012068503, + -0.03555097, + 0.07868844, + -0.0046516946, + 0.05697312, + -0.0016498672, + 0.13178287, + -0.17875795, + 0.01603461, + -0.060153756, + 0.08427135, + 0.021034995, + -0.10947829, + 0.02297257, + 0.010201626, + 0.06411817, + 0.00069312344, + -0.022427784, + -0.08340414, + 0.008129976, + 0.10483062, + -0.0111034345, + 0.08321758, + -0.015355383, + 0.0957891, + -0.09981907, + -0.03727025, + -0.18577074, + -0.03434626, + 0.09778465, + -0.009433364, + 0.028921766, + 0.028915899, + -0.0066469046, + 0.071474515, + 0.05219391, + -0.015687006, + -0.058292978, + -0.06798357, + -0.20924047, + 0.022352543, + -0.059503425, + 0.08062257, + 0.002789503, + 0.042323086, + -0.006091077, + 0.034528166, + 0.003899257, + -0.06505021, + -0.10829687, + 0.038795065, + -0.0062199766, + -0.034921397, + 0.042476345, + 0.1373016, + -0.103741765, + -0.0077205664, + -0.0031144773, + -0.061182786, + 0.12591518, + 0.09842636, + -0.06510717, + 0.011792917, + -0.000927913, + 0.04741174, + 0.0056584463, + -0.01777804, + -0.013577285, + -0.032479167, + 0.03005739, + -0.035178192, + 0.12319496, + 0.05953756, + -0.052662153, + -0.027717786, + -0.03986303, + -0.056286726, + 0.079686634, + 0.1305422, + -0.095169686, + 0.13503848, + -0.032076195, + -0.06431863, + -0.022405755, + -0.003142242, + 0.059594706, + 0.03068783, + -0.10687165, + 0.07165271, + 0.03235361, + -0.06877855, + 0.09371309, + -0.010729307, + -0.026322195, + -0.0051372023, + -0.021481242, + 0.012479718, + 0.046846237, + 0.010242278, + 0.0068782144, + 0.017676892, + -0.03587468, + 0.03040056, + 0.04695722, + -0.027955173, + 0.06685435, + 0.12436746, + 0.23214655, + 0.044290863, + 0.093425006, + 0.14297906, + 0.041779425, + 0.035464626, + 0.034903977, + 0.009013933, + -0.027668172, + 0.045572154, + -0.05374057, + 0.08580905, + -0.01716065, + 0.1172244, + 0.017061407, + -0.02231919, + -0.04480924, + -0.034795754, + -0.1227076, + 0.12222982, + -0.027653115, + 0.024613868, + -0.20267302, + 0.106527664, + 0.015363324, + -0.035207216, + 0.030558974, + 0.0059918873, + -0.03979042, + -0.080702506, + -0.015305442, + 0.1995665, + -0.02589101, + 0.0078954445, + 0.03370895, + 0.04346243, + -0.01736537, + -0.004968941, + -0.08206979, + 0.022851046, + 0.053825565, + -0.010216455, + -0.05342773, + 0.05834939, + -0.01996591, + 0.013178858, + 0.09147428, + 0.018772963, + 0.07511911, + 0.01180301, + 0.005182977, + -0.18621622, + -0.0072700414, + -0.089398436, + -0.12156247, + -0.061243024, + 0.014654888, + 0.059578173, + 0.010543929, + -0.005970224, + 0.03271037, + -0.03923921, + 0.074761786, + 0.021807993, + 0.10606188, + -0.07728519, + -0.0035435762, + -0.02673473, + -0.07167695, + -0.04556865, + 0.078798905, + -0.09760936, + 0.08318658, + -0.08553383, + -0.08783146, + 0.0006552894, + 0.08950479, + 0.020713896, + 0.015709128, + 0.053766903, + 0.08270455, + -0.024749562, + -0.13635102, + 0.06769282, + -0.06427349, + 0.042735264, + -0.011112339, + -0.05849649, + -0.04044677, + 0.056806073, + 0.09699002, + 0.065084584, + -0.12338098, + -0.018618584, + 0.024816224, + -0.043620303, + 0.05223019, + 0.006284062, + -0.01630919, + 0.020274458, + 0.027187906, + -0.020418243, + 0.0043337992, + -0.08632459, + 0.04738525, + 0.032942872, + -0.009732838, + 0.023929657, + 0.02485127, + 0.162084, + 0.021846166, + -0.096425354, + 0.017536443, + 0.041337963, + -0.0030266028, + -0.10360438, + 0.042839766, + -0.011619359, + 0.07204971, + -0.014512203, + 0.028519962, + -0.014620583, + 0.06711956, + 0.0088992305, + 0.10842747, + -0.05419707, + 0.03829462, + -0.09332558, + -0.040983323, + -0.0653216, + -0.051118877, + 0.037638128, + 0.066583864, + 0.008300097, + -0.0607046, + 0.0074935053, + -0.00037436237, + -0.038610496, + -0.114451, + 0.044067096, + -0.0068640895, + -0.0113348495, + -0.017514303, + 0.06762703, + -0.055778116, + 0.0025316353, + -0.084971406, + -0.024718441, + -0.017729495, + 0.112641305, + -0.037606817, + -0.05289465, + -0.03846344, + 0.07178632, + -0.06435578, + 0.07703845, + 0.03373481, + 0.09758765, + 0.038117815, + -0.016834803, + 0.08726492, + -0.016205939, + 0.010691579, + -0.041291684, + 0.0021367143, + -0.06775239, + 0.028051322, + -0.011766549, + 0.10251835, + -0.006881461, + -0.080986604, + -0.056964066, + -0.04164388, + 0.06998048, + -0.05543376, + 0.04317498, + -0.15437047, + -0.056652922, + 0.11637467, + 0.10030192, + 0.07892662, + -0.031626917, + -0.011490581, + 0.03415925, + 0.0008869665, + 0.09424204, + -0.013176491, + 0.035477515, + 0.054944754, + -0.0002188088, + 0.06825712, + 0.11747579, + 0.049191963, + 0.06350638, + -0.06694228, + 0.035578184, + -0.029018175, + -0.029603489, + 0.022307126, + -0.07139821, + 0.048668716, + -0.05982387, + -0.024430871, + -0.08535895, + -0.01778066, + -0.103837736, + 0.057396974, + -0.0099169, + 0.069304325, + 0.10365774, + -0.009832133, + -0.06063173, + -0.08046321, + 0.0064569516, + 0.010381851, + -0.054632653, + -0.035264913, + -0.013935891, + 0.01922498, + -0.08126367, + -0.023071425, + -0.105466105, + -0.012641356, + 0.03567662, + 0.05915022, + 0.08988791, + -0.06266305, + -0.057982516, + -0.04117482, + 0.048653405, + 0.009620415, + 0.021947324, + 0.024394862, + -0.05886191, + 0.057814877, + 0.025332076, + 0.003791334, + 0.0040966417, + 0.024327794, + -0.014264028, + 0.022618687, + 0.06312975, + 0.03466166, + -0.0728442, + 0.023939375, + -0.03332764, + 0.061590437, + -0.12513098, + -0.03448089, + -0.047353834, + 0.009753265, + 0.04412345, + 0.011378179, + -0.08348392, + -0.01579243, + 0.012038838, + 0.0771572, + -0.01416469, + -0.09068855, + 0.04506465, + -0.027838208, + -0.03338156, + -0.08670475, + -0.062935546, + -0.07402203, + 0.07052046, + 0.09202871, + -0.0030487257, + 0.0731155, + -0.025213849, + 0.12443811, + 0.038369555, + -0.08275741, + -0.017946951, + 0.06907822, + -0.023078818, + -0.033516727, + -0.051422942, + -0.067741655, + 0.034213886, + -0.075616196, + 0.06705659, + -0.04554798, + 0.030173553, + 0.056699738, + -0.03797959, + 0.09316969, + -0.039946508, + -0.055314485, + -0.049339972, + -0.0021931415, + 0.043850936, + 0.0625165, + 0.015243344, + 0.1486268, + 0.03560544, + 0.0054315627, + 0.034166023, + 0.01255885, + 0.040137827, + 0.022829562, + 0.0015469731, + -0.0040146876, + -0.021427106, + 0.08976617, + 0.022311423, + 0.13408577, + -0.016516278, + -0.0077241906, + -0.07892908, + 0.0918642, + -0.021831458, + -0.056999244, + 0.027041351, + 0.06663442, + -0.1738884, + 0.085231, + -0.039695214, + 0.02294308, + -0.005845859, + -0.06814599, + 0.038218528, + -0.109613486, + -0.013863854, + -0.06346433, + -0.12403258, + -0.053951886, + -0.00089150923, + -0.048438534, + -0.0064545753, + -0.053049006, + 0.003094217, + 0.03560286, + -0.0376025, + -0.10386766, + -0.045907885, + 0.04285174, + -0.02385285, + 0.035024274, + 0.04477627, + 0.12649728, + 0.08916322, + 0.05191616, + 0.027303383, + -0.006924128, + 0.07228284, + -0.088942, + -0.1319488, + 0.049288463, + 0.1174649, + -0.0038192864, + -0.042731818, + 0.03775852, + -0.00073822733, + 0.023379738, + 0.034273278, + 0.03682401, + -0.04734978, + 0.022593817, + 0.04409001, + 0.02474576, + -0.0061323484, + -0.009086511, + 0.049111933, + -0.032955654, + -0.020741219, + 0.063518584, + 0.007921433, + -0.05168147, + -0.025150804, + -0.005144285, + -0.00031368487, + -0.05042929, + -0.020046197, + -0.04359245, + -0.09016853, + -0.025569437, + 0.0245593, + 0.03660497, + -0.021592885, + -0.029281888, + -0.065625526, + 0.08559711, + 0.043033604, + -0.07897961, + -0.013847679, + 0.055071138, + -0.023617532, + -0.08828593, + 0.08805651, + -0.06040911, + 0.01903576, + -0.022293685, + 0.014005562, + 0.047452588, + -0.054631684, + -0.027649907, + -0.033193264, + 0.0013944489, + 0.012175467, + -0.03966833, + 0.04151638, + 0.01591493, + 0.013318377, + -0.06289957, + 0.06207641, + 0.1656088, + -0.029916283, + 0.12782148, + -0.09528566, + 0.030391762, + 0.013889852, + 0.063113265, + 0.054693975, + 0.029431297, + -0.1055249, + -0.068159774, + -0.10070397, + -0.012777722, + -0.06622054, + 0.05435452, + -0.12659454, + -0.0003244871, + 0.013772954, + -0.06859853, + 0.055385318, + 0.07034561, + -0.0040647984, + 0.05142809, + 0.05016032, + -0.10153724, + 0.0242343, + 0.05783332, + 0.12077048, + 0.027433638, + 0.075906634, + 0.03662882, + 0.040531076, + 0.0030187052, + -0.02035892, + -0.051724717, + 0.009088971, + -0.07718964, + 0.066357516, + 0.05506988, + 0.019438691, + -0.033437017, + 0.0077773957, + -0.035858143, + 0.00620277, + 0.025822306, + -0.05971552, + -0.04533555, + -0.08795847, + 0.06360275, + 0.1339617, + -0.007962772, + -0.057132564, + 0.019530416, + -0.039711196, + -0.0076836203, + 0.13233833, + -0.043247588, + 0.027742952, + 0.090713866, + 0.026510363, + -0.046166863, + 0.022116156, + 0.0010122113, + 0.08278919, + -0.027050173, + 0.049872346, + 0.014642328, + -0.09025972, + -0.10878053 + ] + ], + "bias": [ + 0.409906, + 0.11315371, + -0.5230597 + ], + "decision": { + "tools_prob_threshold": 0.7, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json new file mode 100644 index 0000000..92898bf --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json @@ -0,0 +1,439 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.7, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 141, + "accuracy": 0.6879433, + "confusion": [ + [ + 51, + 29, + 0 + ], + [ + 0, + 46, + 0 + ], + [ + 3, + 12, + 0 + ] + ], + "tool_precision": 0.9444444, + "tool_recall": 0.6375, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.2, + "memory_false_abstain_rate": 0.0, + "tool_count": 80, + "memory_count": 46, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6104362, + "p_memory": 0.30589545, + "p_abstain": 0.0836684 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.59472257, + "p_memory": 0.3161544, + "p_abstain": 0.08912299 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6297685, + "p_memory": 0.26889145, + "p_abstain": 0.10134005 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6624538, + "p_memory": 0.26015484, + "p_abstain": 0.07739138 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6639559, + "p_memory": 0.23188965, + "p_abstain": 0.1041544 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.60731936, + "p_memory": 0.27004346, + "p_abstain": 0.12263718 + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6722137, + "p_memory": 0.23017572, + "p_abstain": 0.09761066 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.60558313, + "p_memory": 0.32460892, + "p_abstain": 0.06980796 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6429548, + "p_memory": 0.25670224, + "p_abstain": 0.10034299 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.68191427, + "p_memory": 0.24376579, + "p_abstain": 0.074319914 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6615207, + "p_memory": 0.253586, + "p_abstain": 0.08489324 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6061652, + "p_memory": 0.31448147, + "p_abstain": 0.07935343 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6499309, + "p_memory": 0.2537558, + "p_abstain": 0.096313275 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.59418523, + "p_memory": 0.33327773, + "p_abstain": 0.07253697 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5657255, + "p_memory": 0.34358782, + "p_abstain": 0.0906867 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6546773, + "p_memory": 0.25830474, + "p_abstain": 0.08701797 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47816014, + "p_memory": 0.41151565, + "p_abstain": 0.110324204 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.576397, + "p_memory": 0.3468361, + "p_abstain": 0.07676686 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6846916, + "p_memory": 0.24798164, + "p_abstain": 0.06732679 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.628443, + "p_memory": 0.29244968, + "p_abstain": 0.07910734 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.691014, + "p_memory": 0.20424092, + "p_abstain": 0.10474504 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.66433394, + "p_memory": 0.25342876, + "p_abstain": 0.082237266 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.63012874, + "p_memory": 0.26535815, + "p_abstain": 0.10451312 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6360849, + "p_memory": 0.2686135, + "p_abstain": 0.09530164 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.65475535, + "p_memory": 0.24794452, + "p_abstain": 0.09730011 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.65992075, + "p_memory": 0.25475225, + "p_abstain": 0.08532704 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6974221, + "p_memory": 0.21716036, + "p_abstain": 0.08541755 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.69476, + "p_memory": 0.2100455, + "p_abstain": 0.09519449 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.66223246, + "p_memory": 0.2545751, + "p_abstain": 0.08319243 + }, + { + "query_id": "abs_test_001", + "text": "can you help me?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.55861247, + "p_memory": 0.2375059, + "p_abstain": 0.20388162 + }, + { + "query_id": "abs_test_002", + "text": "what's happening on hackernews right now?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.69352347, + "p_memory": 0.19897409, + "p_abstain": 0.10750248 + }, + { + "query_id": "abs_test_003", + "text": "what will my project look like in 6 months?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.47491404, + "p_memory": 0.35513982, + "p_abstain": 0.16994612 + }, + { + "query_id": "abs_test_004", + "text": "what's the best programming language?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46780536, + "p_memory": 0.31017643, + "p_abstain": 0.22201824 + }, + { + "query_id": "abs_test_005", + "text": "help", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.59031075, + "p_memory": 0.22899751, + "p_abstain": 0.18069172 + }, + { + "query_id": "abs_test_006", + "text": "what's the github token for the ci runner?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.7357465, + "p_memory": 0.16489287, + "p_abstain": 0.09936059 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5205462, + "p_memory": 0.31795332, + "p_abstain": 0.16150045 + }, + { + "query_id": "abs_test_008", + "text": "something is wrong", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.51049954, + "p_memory": 0.28777307, + "p_abstain": 0.20172736 + }, + { + "query_id": "abs_test_009", + "text": "どのモデルが今一番スコアが高いですか — ベンチマークサイトで", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.61282194, + "p_memory": 0.25715584, + "p_abstain": 0.13002223 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.66652524, + "p_memory": 0.22985739, + "p_abstain": 0.10361738 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49091458, + "p_memory": 0.3661909, + "p_abstain": 0.1428945 + }, + { + "query_id": "abs_test_012", + "text": "y", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.47903177, + "p_memory": 0.2892732, + "p_abstain": 0.23169498 + }, + { + "query_id": "abs_test_013", + "text": "is the llama.cpp repo trending on github today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.73314124, + "p_memory": 0.1612506, + "p_abstain": 0.105608135 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.60404634, + "p_memory": 0.2927036, + "p_abstain": 0.10325001 + }, + { + "query_id": "abs_test_015", + "text": "can you log into the remote postgres and show me the schema?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.77282834, + "p_memory": 0.14105494, + "p_abstain": 0.086116806 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.md new file mode 100644 index 0000000..c99cb5c --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-1/router_report.md @@ -0,0 +1,81 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.700` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `80` memory: `46` abstain: `15` +- accuracy: `0.6879` +- tools precision: `0.9444` +- tools recall: `0.6375` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.2000` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 51 | 29 | 0 | +| memory | 0 | 46 | 0 | +| abstain | 3 | 12 | 0 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.61 | 0.31 | 0.08 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.59 | 0.32 | 0.09 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.63 | 0.27 | 0.10 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.66 | 0.26 | 0.08 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.66 | 0.23 | 0.10 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.61 | 0.27 | 0.12 | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | 0.67 | 0.23 | 0.10 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.61 | 0.32 | 0.07 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.64 | 0.26 | 0.10 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.68 | 0.24 | 0.07 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.61 | 0.31 | 0.08 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.65 | 0.25 | 0.10 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.59 | 0.33 | 0.07 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | 0.57 | 0.34 | 0.09 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.65 | 0.26 | 0.09 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.48 | 0.41 | 0.11 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.58 | 0.35 | 0.08 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.68 | 0.25 | 0.07 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.63 | 0.29 | 0.08 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.69 | 0.20 | 0.10 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.63 | 0.27 | 0.10 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.64 | 0.27 | 0.10 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.65 | 0.25 | 0.10 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.66 | 0.25 | 0.09 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.70 | 0.22 | 0.09 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.69 | 0.21 | 0.10 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `abs_test_001` | can you help me? | `Abstain` | `Memory` | 0.56 | 0.24 | 0.20 | +| `abs_test_002` | what's happening on hackernews right now? | `Abstain` | `Memory` | 0.69 | 0.20 | 0.11 | +| `abs_test_003` | what will my project look like in 6 months? | `Abstain` | `Memory` | 0.47 | 0.36 | 0.17 | +| `abs_test_004` | what's the best programming language? | `Abstain` | `Memory` | 0.47 | 0.31 | 0.22 | +| `abs_test_005` | help | `Abstain` | `Memory` | 0.59 | 0.23 | 0.18 | +| `abs_test_006` | what's the github token for the ci runner? | `Abstain` | `Tools` | 0.74 | 0.16 | 0.10 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.52 | 0.32 | 0.16 | +| `abs_test_008` | something is wrong | `Abstain` | `Memory` | 0.51 | 0.29 | 0.20 | +| `abs_test_009` | どのモデルが今一番スコアが高いですか — ベンチマークサイトで | `Abstain` | `Memory` | 0.61 | 0.26 | 0.13 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | 0.67 | 0.23 | 0.10 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.49 | 0.37 | 0.14 | +| `abs_test_012` | y | `Abstain` | `Memory` | 0.48 | 0.29 | 0.23 | +| `abs_test_013` | is the llama.cpp repo trending on github today? | `Abstain` | `Tools` | 0.73 | 0.16 | 0.11 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.29 | 0.10 | +| `abs_test_015` | can you log into the remote postgres and show me the schema? | `Abstain` | `Tools` | 0.77 | 0.14 | 0.09 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-10/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-10/router_model.json new file mode 100644 index 0000000..c7f1171 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-10/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.1537661, + -0.035649825, + -0.08693668, + -0.039169595, + -0.07801271, + 0.022404537, + 0.13668533, + -0.04904432, + -0.084624946, + -0.12592527, + 0.015163703, + -0.18208449, + 0.0055660303, + 0.07221892, + 0.06448449, + -0.1870893, + -0.075359516, + -0.03543284, + -0.008072806, + 0.035137396, + 0.1856608, + -0.047060464, + -0.026945204, + 0.038430814, + -0.027119577, + -0.07783137, + -0.028168011, + -0.04998577, + 0.13851744, + 0.22334306, + -0.11444658, + -0.10880823, + -0.013960583, + -0.002960124, + 0.1886642, + 0.008615434, + 0.06759129, + 0.025011372, + -0.119343616, + 0.033081945, + -0.08350601, + -0.0484712, + -0.074073195, + 0.045808908, + -0.01146472, + 0.10831494, + -0.0040111416, + -0.09824766, + -0.02530775, + -0.06901607, + -0.050421126, + -0.126361, + -0.088383175, + -0.08048365, + -0.001986429, + 0.18562548, + -0.10405873, + 0.02529612, + 0.108271234, + 0.24374901, + -0.011125183, + 0.0132962605, + -0.07309199, + 0.029683625, + -0.08636377, + -0.03715244, + 0.083628885, + 0.005959518, + 0.07466751, + 0.11360453, + -0.041455425, + 0.09902782, + -0.02341183, + 0.05831913, + -0.042435482, + 0.08605521, + 0.15117645, + -0.019223474, + -0.038496863, + -0.12737247, + 0.058691185, + -0.018187318, + -0.10621924, + -0.027555455, + 0.05125111, + 0.16719356, + -0.17868775, + -0.17773148, + -0.08739252, + 0.10273777, + -0.034631487, + -0.0539006, + -0.096600875, + 0.08291881, + -0.15143853, + -0.09585049, + 0.027222762, + 0.007994819, + 0.060303994, + -0.018851208, + -0.016228046, + 0.08862469, + 0.19132261, + -0.019297566, + -0.053104565, + 0.14122458, + -0.10131717, + -0.09569557, + -0.068272986, + 0.000808413, + -0.15115495, + 0.047998995, + 0.039926194, + 0.037234508, + 0.032307126, + -0.075684965, + -0.073772445, + -0.10967773, + -0.09304202, + 0.040335774, + -0.016109457, + -0.17040172, + 0.13639012, + 0.034102615, + -0.09065539, + 0.1110392, + 0.17052025, + 0.16978532, + 0.1330525, + 0.0534311, + 0.10709471, + 0.12975323, + 0.0031307633, + 0.11619781, + -0.20192865, + -0.04178235, + 0.07954132, + 0.047113087, + -0.03501471, + -0.1652226, + -0.09810865, + 0.086311735, + 0.020574726, + 0.026401358, + 0.15123715, + -0.11407882, + -0.0010843852, + -0.032467104, + 0.04454825, + 0.08361806, + -0.06101439, + 0.060647525, + 0.18783487, + -0.12666312, + 0.084901564, + 0.10627836, + -0.047494754, + -0.0030847418, + 0.13515899, + 0.16071515, + 0.009656678, + -0.032863, + 0.27564478, + -0.07924913, + -0.046650242, + 0.075491354, + -0.06973335, + 0.06965276, + -0.001548173, + -0.142573, + 0.00032827802, + 0.014488301, + -0.1260245, + -0.08115438, + -0.044923816, + -0.03615714, + 0.0039115055, + -0.06106897, + -0.041726876, + 0.046486966, + 0.068862185, + -0.0059390427, + -0.20666197, + 0.11289427, + -0.011771635, + -0.06389011, + 0.028167449, + 0.2907871, + 0.01976517, + -0.08579038, + 0.08004743, + 0.16933607, + 0.05288272, + 0.038429778, + 0.03427874, + -0.044819575, + 0.04110185, + -0.13571876, + -0.19524126, + 0.06988574, + 0.073779844, + -0.046239115, + -0.040948626, + -0.084544, + -0.020007487, + -0.092407554, + -0.17292632, + 0.23797427, + -0.1945358, + -0.49743015, + -0.07454593, + -0.1673993, + -0.10651807, + -0.04177226, + -0.05472722, + -0.15188706, + -0.022152808, + -0.13939627, + -0.019187424, + -0.15158167, + -0.19280887, + 0.10705503, + 0.062492274, + 0.16923375, + -0.26784095, + -0.15662816, + 0.025555544, + -0.17363223, + 0.08134375, + 0.06117359, + -0.09012711, + -0.06951844, + 0.061694615, + -0.003797633, + 0.21708345, + 0.12185522, + 0.19993393, + 0.08121296, + 0.012878898, + -0.054898884, + -0.1080671, + 0.17580487, + -0.06257184, + -0.13819437, + 0.07154935, + 0.09852028, + 0.16923477, + -0.005906965, + -0.0075131687, + 0.119097345, + -0.0682252, + -0.030815927, + -0.19546556, + 0.037838846, + 0.00044135028, + 0.11877078, + 0.10327466, + -0.06929159, + 0.020809863, + 0.13948017, + -0.18564449, + 0.21409702, + 0.14669497, + 0.045995284, + 0.010650101, + -0.07551283, + -0.0006766264, + 0.07865089, + -0.11002661, + -0.024456186, + -0.08400181, + -0.13899222, + 0.09306608, + 0.10094795, + 0.07228482, + 0.019025564, + 0.09661008, + 0.13605566, + 0.028701328, + 0.020753454, + 0.0604083, + 0.05287096, + -0.07340115, + 0.009560703, + -0.025192428, + 0.07975566, + -0.28597078, + 0.021488205, + -0.0889583, + 0.12257723, + -0.13694072, + 0.12471303, + 0.061077848, + 0.004631609, + -0.10417203, + 0.22062522, + -0.013706877, + -0.049198896, + 0.00029548293, + 0.075458646, + 0.1765529, + 0.11541535, + 0.008701007, + 0.24391726, + 0.028608661, + 0.06389106, + 0.08642011, + -0.09138413, + -0.10007181, + -0.0695514, + -0.00900465, + -0.03647545, + 0.019589651, + 0.10178171, + 0.17542481, + 0.11783792, + 0.049075227, + -0.10664901, + 0.03252592, + 0.050435152, + -0.008125645, + -0.10447342, + 0.009139559, + 0.1275379, + -0.021951595, + -0.0136193605, + 0.031333257, + -0.15114506, + -0.1765369, + 0.06522931, + 0.06019238, + -0.035919093, + -0.15519899, + -0.01969548, + -0.03430187, + 0.10910889, + -0.0056543187, + -0.10220757, + 0.26145938, + 0.15574658, + 0.049058672, + -0.0064308834, + 0.042335905, + 0.005810607, + 0.038708996, + -0.0057784626, + 0.049044974, + 0.017823728, + 0.087167464, + 0.074433744, + 0.0077878316, + -0.05596981, + 0.0045128716, + -0.049364593, + 0.0015845216, + 0.14348508, + -0.13488854, + -0.1561442, + 0.054857284, + 0.018423049, + -0.034320723, + 0.0040649255, + -0.04828976, + 0.28164595, + 0.047446065, + -0.11510787, + 0.11150978, + -0.04681086, + 0.071485296, + -0.045360688, + -0.009777533, + 0.038308833, + -0.052017767, + 0.03695938, + 0.057656713, + -0.060405314, + -0.29814774, + -0.071221784, + 0.1714612, + -0.07547322, + 0.005367531, + -0.13997321, + -0.16026817, + 0.04131712, + -0.17275667, + -0.029355615, + 0.08490756, + -0.115701295, + 0.0016283661, + 0.20621264, + 0.16213463, + -0.07390191, + -0.19121905, + -0.13373351, + 0.054568663, + 0.11002628, + 0.025603758, + -0.06534632, + 0.057530094, + 0.018174551, + -0.14876643, + 0.035836782, + -0.01137345, + -0.075621784, + 0.07126445, + -0.04957483, + -0.09046384, + -0.067094974, + -0.042324323, + -0.057362486, + 0.1400903, + -0.03241678, + 0.10672305, + -0.19880442, + 0.09854313, + 0.0760169, + -0.091766715, + -0.12132613, + 0.027035655, + -0.3583752, + -0.13381411, + -0.30593142, + -0.0072146636, + 0.12702839, + -0.1360928, + -0.011511107, + -0.045212187, + -0.032329787, + 0.013346648, + -0.016038861, + 0.07367348, + -0.19516303, + 0.09781957, + -0.023986258, + 0.14528868, + 0.08888904, + -0.0009244085, + -0.05984577, + 0.018362086, + -0.0895728, + -0.023944547, + -0.0010550144, + 0.117598146, + -0.02050585, + -0.12902373, + 0.077714995, + -0.012136869, + -0.028286034, + 0.01971836, + 0.23491275, + 0.00014187285, + -0.025502866, + -0.160924, + -0.10722113, + -0.028722614, + -0.11093872, + 0.12761506, + 0.032540575, + -0.028944362, + 0.026266946, + 0.15018588, + 0.065873004, + -0.2041029, + -0.026748825, + -0.15338984, + 0.019006187, + -0.09869299, + -0.10463372, + 0.056366608, + 0.019882223, + 0.024008995, + -0.06550811, + 0.040581826, + -0.09272168, + -0.021980435, + 0.009398623, + -0.055842966, + -0.067572854, + 0.2544045, + 0.2720369, + 0.056282323, + 0.14016761, + 0.1229671, + 0.085792094, + 0.082071744, + 0.015617369, + 0.0025832395, + -0.11643445, + -0.08638553, + -0.052909072, + -0.16713358, + 0.003093768, + 0.07404296, + 0.031308185, + -0.117112026, + -0.08167997, + -0.1499687, + 0.043127168, + 0.14035194, + -0.07871639, + -0.037212748, + -0.0598638, + -0.14055473, + -0.036376033, + 0.11357092, + -0.050674643, + 0.022856299, + 0.057805985, + 0.089862354, + -0.044336755, + -0.08881311, + -0.10180266, + 0.09055733, + -0.04494839, + 0.09060898, + -0.13318132, + 0.032166876, + 0.00562406, + -0.072895855, + 0.008453028, + -0.10890039, + -0.14855209, + -0.009387831, + 0.07252449, + 0.1000593, + -0.1428223, + -0.08014342, + -0.040895056, + -0.039497916, + 0.36770174, + -0.0024050162, + -0.007834549, + 0.1724918, + 0.0037634445, + -0.18319333, + 0.07710567, + -0.19161667, + -0.014408101, + 0.4058824, + -0.06781774, + -0.004210171, + -0.020524722, + 0.03903467, + -0.065584004, + 0.12053713, + 0.07669467, + 0.019268565, + -0.1424542, + 0.07729574, + -0.1408175, + -0.027871037, + -0.07438413, + -0.24066758, + 0.0007071577, + -0.11960982, + 0.079575315, + 0.026175413, + 0.06513677, + -0.07926052, + 0.053233802, + -0.23602884, + 0.009922975, + -0.024497086, + 0.07963983, + -0.03895233, + 0.012983701, + 0.02903441, + 0.09618848, + -0.086873636, + 0.15702735, + 0.051104963, + 0.23331824, + 0.05189599, + -0.012449083, + 0.18062961, + 0.06019235, + 0.056962494, + 0.03108089, + -0.021042427, + 0.0064071524, + -0.088212445, + 0.08572126, + 0.0037502868, + 0.07312031, + 0.12206003, + 0.15648857, + 0.19765435, + -0.08641273, + -0.02523061, + -0.11257553, + -0.07819527, + -0.028061705, + -0.06571461, + -0.015628025, + 0.062462434, + 0.11131009, + -0.0013901697, + 0.058562018, + 0.06027097, + 0.023649702, + 0.15201351, + 0.08222728, + 0.06710522, + 0.06597227, + 0.15405323, + -0.106588826, + -0.048434135, + -0.08093238, + 0.13272074, + -0.104526795, + 0.066636786, + -0.14262883, + 0.11796827, + 0.01689741, + 0.10963646, + -0.043205373, + 0.030428115, + 0.06697563, + 0.18351023, + -0.12014605, + 0.007847034, + -0.21682698, + -0.053551763, + -0.030757116, + -0.058873527, + -0.039070714, + -0.00346026, + 0.09289441, + -0.14382905, + 0.1066636, + 0.06449059, + 0.21688591, + -0.36460355, + 0.15066512, + -0.064523116, + -0.018895637, + -0.14244261, + 0.018194063, + -0.08151443, + 0.008980479, + -0.08436932, + 0.123211086, + -0.16318616, + 0.11563229, + -0.06480912, + -0.005352673, + 0.068340674, + -0.07209151, + 0.0028943075, + 0.0013212605, + 0.11254483, + 0.098231606, + -0.082876496, + -0.018098809, + -0.019039044, + -0.10377387, + -0.055182043, + -0.045605227, + -0.06043637, + 0.19120796, + 0.10237104, + 0.07438044, + 0.13820165, + 0.17585173, + 0.27409434, + -0.07378421, + 0.025601309, + -0.12231862, + 0.12022717, + -0.101080395, + 0.032167993, + -0.08084088, + -0.17208248, + -0.1681819, + -0.03261707, + 0.02148205, + -0.075628266, + 0.024750838, + 0.03915467, + -0.13104486, + 0.034098193, + -0.034001082, + 0.08776243, + -0.049719255, + -0.06473143, + 0.10588895, + 0.08131149, + -0.052911863, + -0.054036725, + -0.035009447, + 0.08870277, + 0.07426314, + 0.040366404, + 0.077888794, + 0.021348773, + 0.226514, + 0.15032294, + 0.16204213, + 0.000056462246, + 0.09213504, + 0.19880146, + 0.08437888, + 0.021916471, + 0.08471564, + 0.03781062, + -0.066586, + -0.046965122, + -0.10133214, + 0.08557293, + -0.14368777, + -0.06882834, + -0.12171229, + 0.0055409498, + -0.09227392, + -0.10453962, + -0.07994333, + 0.097816125, + 0.0027960786, + -0.11214976, + 0.05112316, + -0.040688016, + 0.068019, + -0.25952643, + 0.16106968, + 0.17121106, + 0.25099227, + 0.076975055, + -0.2097073, + -0.023342844, + 0.015645755, + 0.12014807, + 0.07335009, + -0.05761358, + -0.08484111, + -0.000030331716, + 0.15092224, + 0.06777547, + -0.08367785, + 0.048469383, + -0.11079949, + 0.003651169, + -0.07147883, + -0.014249957, + -0.24292563, + -0.15647426, + -0.17459014, + 0.010819486, + -0.105703846, + -0.030978357, + 0.06532192, + 0.08745108, + -0.053935952, + 0.0688913, + 0.12390802, + 0.21340448, + -0.014140771, + 0.27248612, + -0.2464558, + 0.05955756, + 0.104456566, + 0.009404783, + -0.08957156, + -0.13419013, + -0.026675185, + -0.1552804, + -0.008024584, + 0.005712407, + -0.124870405, + -0.2592146, + 0.19014545, + 0.0327858, + -0.1229455, + 0.010203828, + 0.037715424, + -0.0053597577, + 0.05868922, + 0.010626316, + 0.019355819, + 0.08497353, + 0.119429246, + 0.004582282, + -0.06729474, + 0.07995998, + -0.04374404, + -0.015687706, + -0.053967472, + -0.08461543, + -0.0808064, + -0.07503357, + -0.03532297, + 0.0011162171, + -0.11756368, + 0.11231443, + 0.0066749663, + 0.087183446, + 0.033798646, + 0.17816873, + 0.16899641, + 0.032384265, + 0.037937198, + -0.1306509, + -0.08966298, + -0.11874975, + 0.04818552, + -0.08112427, + -0.08979903, + 0.016811714, + -0.12227783, + 0.006724136, + 0.07726029, + 0.072018504, + 0.047206745, + -0.020475753, + 0.21668829, + 0.06144242, + -0.010373841, + 0.08095411, + -0.00984799, + -0.18237619, + 0.08362384, + -0.06917313, + 0.003942984, + 0.08102712, + -0.08224079, + 0.026713282, + -0.021954091, + -0.21172519, + 0.061612993, + 0.11611484, + 0.15850388, + -0.07526881, + -0.011174921, + 0.12953082, + -0.04849906, + 0.08364964, + 0.045158945, + 0.10626501, + 0.119382195, + -0.20448637, + -0.033443533, + -0.044285737, + -0.078928396, + -0.027338818, + -0.12594555, + 0.047167778, + -0.03249266, + -0.050701614, + 0.083851635, + 0.06785768, + -0.038283225, + -0.02997082, + -0.10361641, + 0.040058605, + 0.013622663, + -0.029170217, + -0.039008163, + 0.08126564, + 0.023967572, + -0.040728178, + 0.020921214, + -0.2012302, + 0.029383505, + 0.039341018, + 0.07352988, + 0.149539, + -0.18081951, + 0.042585555, + -0.07393927, + -0.0520452, + -0.055225402, + -0.02532332, + 0.07143403, + 0.14533009, + 0.020230262, + 0.06286708, + -0.016043179, + -0.0058425125, + -0.1894284, + 0.081516296, + 0.01061884, + 0.08930471, + -0.070286885, + 0.12981166, + 0.01348248, + 0.043093804, + 0.11957937, + 0.022259962, + -0.032027576, + 0.025988672, + -0.16140743, + 0.07117117, + -0.027395064, + -0.059419967, + -0.0067737456, + -0.09964316, + 0.049351126, + 0.21752295, + 0.046031915, + -0.024564896, + -0.011924736, + -0.059805218, + 0.02614514, + -0.015359368, + -0.02796093, + -0.012214168, + 0.0056245904, + -0.032312065, + 0.109036505, + 0.07137153, + 0.108339995, + 0.052375022, + -0.017451666, + -0.14530696, + -0.11981482, + 0.079305254, + -0.0025598651, + -0.05310435, + 0.11399684, + 0.03127478, + 0.0064025996, + -0.03253582, + 0.089398734, + -0.18456937, + -0.0016192505, + 0.30008864, + -0.18448582, + -0.11759908, + -0.15413254, + 0.018025307, + 0.025112813, + -0.0009730928, + -0.2881156, + 0.12879921, + 0.13925475, + -0.09416232, + 0.0006413473, + -0.13858742, + 0.06543319, + -0.08643667, + -0.07211853, + 0.052677996, + 0.1292809, + -0.041060735, + -0.010105186, + 0.017167171, + 0.10278289, + -0.02560506, + 0.01444451, + -0.24326459, + 0.087474845, + -0.029556967, + -0.018887402, + 0.0519785, + 0.07257978, + -0.06718943, + -0.06401816, + 0.009868641, + 0.29014003, + 0.014569991, + -0.13224676, + -0.021046856, + 0.008031937, + 0.029677179, + -0.12090361, + -0.030684829, + -0.12032807, + 0.14864889, + 0.0730822, + 0.04875755, + -0.053761955, + -0.013974497, + -0.03588524, + 0.14112757, + 0.20620118, + 0.0026121754, + 0.057062473, + 0.006303307, + -0.100188285, + 0.030305644, + 0.030009354, + 0.08998002, + -0.0975479, + 0.17385758, + -0.14035688, + -0.12630624, + 0.20372146, + -0.031224119, + -0.015600041, + -0.000572721, + -0.12464091, + 0.028028218, + -0.08736926, + -0.064607315, + 0.0074855303, + -0.003159628, + 0.06802779, + -0.1313798, + 0.06092782, + -0.1787316, + 0.09524019, + 0.2142574, + 0.07154194, + -0.0078775445, + -0.08554093, + 0.17139089, + -0.00896565, + 0.20121369, + -0.08606445, + 0.10155439, + 0.120717004, + -0.05526951, + -0.25100285, + -0.110797964, + -0.06828172, + -0.07158169, + 0.12045773, + -0.19262262, + -0.03129522, + -0.17944223, + 0.02532566, + 0.09656396, + -0.091062434, + -0.0006519022, + 0.11245893 + ], + [ + 0.08650064, + -0.114138156, + 0.11560147, + 0.05716735, + 0.07363487, + 0.055551223, + 0.042977456, + 0.13866937, + 0.07197206, + 0.057236373, + -0.04786727, + 0.078654975, + 0.020689813, + -0.026271248, + -0.02856729, + 0.048773885, + 0.13842425, + -0.06279018, + 0.12849218, + 0.16308266, + -0.09831704, + -0.01208826, + 0.031379998, + 0.074512295, + -0.14470626, + 0.022406168, + -0.0015860604, + 0.08554748, + -0.13143538, + -0.0886885, + -0.05255464, + 0.30100065, + 0.030713795, + 0.030090688, + -0.0038725059, + -0.036318548, + -0.028928364, + 0.000039066, + 0.098410994, + -0.28426936, + -0.05369149, + 0.11395493, + -0.0016404965, + 0.02959418, + 0.1296119, + -0.08816001, + 0.12523289, + -0.046427406, + -0.024127232, + 0.10684162, + -0.050801836, + -0.036207974, + -0.09595485, + 0.078260094, + 0.06545781, + -0.14052771, + 0.096398816, + -0.020581724, + 0.1009272, + 0.016509471, + 0.032654297, + -0.10522052, + 0.07001931, + -0.052200574, + 0.11428169, + -0.16297819, + -0.022918481, + 0.008518012, + -0.052359655, + -0.15543026, + -0.030120252, + -0.21617754, + 0.057570234, + 0.012278141, + 0.0100514935, + -0.008567871, + 0.03985517, + -0.0071163354, + 0.14116311, + -0.043021444, + 0.046327017, + 0.06556431, + 0.19307518, + 0.048381023, + -0.054358784, + -0.14242582, + 0.052703325, + -0.016104024, + 0.11762951, + -0.27093124, + 0.071561694, + 0.14363705, + -0.10874558, + -0.042457294, + -0.045092538, + 0.073277965, + 0.081697024, + -0.005809052, + -0.10392095, + 0.055779688, + 0.040752683, + 0.023003666, + -0.07039203, + -0.098758176, + 0.11601843, + -0.09822991, + -0.014778384, + 0.019168867, + -0.08343514, + 0.013522634, + 0.018998768, + -0.08619179, + 0.05457085, + 0.008922722, + -0.15973733, + 0.021987697, + 0.04509556, + 0.12129829, + 0.20903002, + 0.11825309, + -0.009349105, + 0.0066330372, + 0.07822109, + 0.09448989, + 0.04427069, + -0.05089293, + -0.0634784, + -0.29010263, + -0.1817537, + -0.10901791, + -0.07024844, + -0.09644515, + 0.04460636, + 0.025619725, + 0.054481138, + -0.017032826, + -0.0024525498, + 0.0051498828, + -0.16207719, + 0.13890503, + 0.06168882, + -0.14186028, + -0.017634777, + 0.035919037, + -0.13157438, + 0.2913547, + -0.048506882, + 0.061833624, + -0.049157888, + -0.029682238, + -0.018465182, + -0.14546509, + -0.08058027, + 0.049560603, + 0.13289733, + -0.124585554, + 0.106291436, + 0.18051346, + -0.032141753, + 0.0047566677, + -0.096933044, + 0.21418223, + -0.12096488, + -0.014764312, + 0.047446866, + 0.03750072, + 0.020264208, + 0.18554664, + -0.1021362, + 0.013693483, + -0.05330727, + 0.021761887, + 0.099549085, + 0.0121648405, + 0.018702097, + -0.09758032, + 0.15901814, + -0.065417156, + 0.057077993, + 0.00047911698, + -0.04200102, + -0.075910345, + 0.16953775, + -0.023566708, + 0.058020957, + -0.1151307, + 0.065792955, + -0.29199713, + -0.019268267, + 0.024025127, + 0.052528866, + -0.10454245, + 0.028018963, + 0.00668617, + -0.0009638491, + 0.15003029, + -0.0796975, + 0.08568932, + 0.14248814, + -0.047471944, + 0.154569, + 0.105354466, + -0.09005179, + 0.0016345638, + -0.19058277, + 0.1462792, + -0.088306405, + -0.19980139, + 0.07397957, + 0.2989383, + -0.031951997, + 0.10095756, + 0.11718432, + 0.0045800325, + 0.09298005, + -0.074017435, + -0.043252986, + 0.075847, + 0.009191615, + 0.09819291, + 0.010901586, + -0.12441394, + -0.15651459, + -0.36471352, + 0.107327834, + 0.2367279, + -0.007892925, + -0.13905, + -0.114933364, + -0.05437099, + 0.030067027, + 0.099514276, + -0.037281614, + -0.037492216, + -0.1981845, + -0.1384136, + 0.08877338, + -0.13132341, + 0.00768169, + -0.047017664, + 0.03620687, + -0.099904016, + 0.019379376, + 0.0981703, + 0.011796661, + -0.09512934, + -0.18267427, + -0.13936168, + 0.06484512, + -0.030278102, + 0.090696916, + 0.12653728, + 0.037380584, + -0.15862349, + 0.14395134, + -0.021009054, + 0.054963164, + 0.046817254, + 0.06660228, + -0.035252098, + 0.090558656, + -0.09402542, + -0.056013796, + -0.061302137, + 0.031734105, + 0.015802667, + -0.16472177, + -0.101517014, + 0.048385564, + 0.16827406, + -0.0048398217, + 0.064329885, + 0.060761847, + 0.08469305, + 0.10910126, + -0.043873362, + -0.089271344, + -0.055935968, + 0.034083303, + -0.013393669, + 0.03643514, + -0.120853394, + 0.0704519, + 0.0065425616, + -0.04119151, + -0.09728708, + 0.05142146, + -0.102816895, + -0.016405446, + 0.10037102, + 0.114227235, + 0.020892173, + 0.0033101249, + 0.09104857, + -0.12247543, + -0.2576225, + -0.1403052, + 0.11442854, + -0.099400446, + 0.00058794674, + -0.19586149, + -0.113684855, + 0.030646956, + -0.11775976, + -0.06694089, + 0.086007595, + 0.09736035, + -0.13316418, + 0.023876686, + 0.06294222, + 0.060473986, + -0.0043298365, + -0.03389869, + -0.0055077346, + 0.03175626, + 0.03684258, + 0.09002136, + 0.12259016, + 0.017571814, + -0.27465117, + -0.033611458, + 0.08492853, + -0.064178616, + -0.09749589, + 0.08281812, + 0.074508965, + -0.08160397, + 0.1836253, + 0.12195985, + -0.047170453, + 0.11666565, + 0.00809368, + -0.1302758, + 0.21874562, + 0.08840849, + -0.09259481, + -0.015606064, + -0.030824287, + -0.2201201, + -0.05294996, + -0.025827821, + 0.028233208, + -0.032888405, + -0.011495582, + -0.11593533, + 0.0070121544, + 0.0074381963, + -0.0030233257, + 0.06750082, + -0.015932027, + 0.06266208, + 0.090228274, + -0.007783951, + 0.09262657, + -0.04413849, + -0.258896, + -0.08814282, + 0.19863154, + 0.21488272, + -0.047835387, + 0.12803216, + -0.017444558, + 0.075741306, + -0.19025156, + 0.027734958, + 0.03320167, + -0.123028815, + -0.028187469, + 0.013636808, + 0.06271145, + 0.18612179, + -0.29361016, + 0.10948807, + 0.027112115, + -0.031513385, + 0.028932175, + 0.25757912, + 0.047562208, + -0.1354802, + 0.08021137, + -0.27474785, + 0.032081064, + 0.03791089, + 0.025726352, + 0.069445595, + 0.0021079297, + -0.02449264, + 0.011238894, + -0.02319162, + -0.026542418, + -0.18316846, + 0.10666311, + 0.19345199, + -0.06720772, + 0.00815098, + -0.049136266, + -0.14031912, + 0.15578413, + 0.008095201, + 0.01659755, + 0.18123667, + -0.17131653, + 0.063718066, + 0.07473296, + -0.027271748, + -0.058085024, + -0.100015745, + -0.001066607, + 0.025115151, + 0.086855374, + 0.03166735, + 0.024315033, + 0.007213102, + 0.054682832, + 0.04690719, + -0.17995097, + -0.004744909, + -0.05241575, + 0.12825188, + 0.056677263, + -0.08422418, + 0.08251111, + 0.009535376, + 0.19880669, + 0.2963894, + -0.07653299, + 0.1401372, + 0.019019231, + -0.09808532, + 0.026957024, + -0.17146498, + 0.026395286, + -0.096108206, + -0.11671051, + -0.0031023317, + 0.030857945, + 0.00259537, + 0.17649667, + -0.009633149, + 0.09164389, + -0.11770125, + 0.035321847, + -0.25525537, + 0.2554045, + -0.042877495, + -0.19594881, + 0.1396774, + 0.012059957, + -0.00449406, + -0.19431978, + -0.088604175, + 0.015468876, + -0.13193911, + 0.10843306, + 0.23043013, + 0.06712104, + -0.07877263, + 0.03013501, + 0.22939627, + 0.047045074, + -0.083643205, + -0.066756636, + 0.15640521, + 0.14078821, + -0.011661555, + -0.07418068, + -0.0354833, + 0.025781255, + 0.06630863, + 0.09891263, + -0.034469504, + 0.028824922, + -0.056631226, + 0.08970828, + 0.09523603, + 0.050132398, + -0.1690745, + 0.046013247, + 0.013893757, + -0.12068828, + -0.048534885, + 0.07168909, + -0.037411164, + -0.008103455, + 0.006886762, + -0.19633815, + -0.17443606, + 0.045308072, + 0.09340842, + 0.1490397, + -0.030604543, + 0.02490058, + -0.051153444, + -0.04850469, + -0.051614046, + 0.07135396, + 0.04567157, + 0.0073676747, + -0.15031311, + 0.16379681, + 0.0757311, + -0.1616459, + -0.1270838, + 0.023255069, + -0.1826222, + 0.15810522, + -0.044493705, + -0.22102857, + 0.045563675, + -0.008063402, + 0.05133512, + 0.12316835, + 0.10462279, + -0.2890721, + -0.017548595, + -0.11253995, + 0.10683944, + 0.032772627, + 0.011404846, + -0.031123182, + -0.008722479, + 0.1220427, + -0.23846045, + -0.05996672, + -0.05013295, + -0.029948775, + 0.060149055, + -0.13687938, + 0.054654896, + -0.0076787327, + -0.08335814, + -0.11796455, + -0.101908155, + -0.050144844, + 0.21571128, + -0.06343264, + 0.107729964, + -0.013642982, + -0.40179682, + -0.013399297, + 0.02881357, + -0.018139042, + 0.013398127, + -0.031053284, + -0.06259348, + -0.10992421, + 0.03368614, + 0.1299659, + -0.14629827, + -0.17959803, + -0.02231733, + 0.14280543, + 0.06939315, + 0.08924989, + 0.06705628, + 0.071522295, + -0.029402977, + 0.059425004, + 0.11423234, + 0.09920461, + 0.1667185, + 0.077411644, + -0.061828777, + -0.07963011, + 0.10044846, + -0.029872157, + -0.1763701, + 0.110166654, + 0.17331573, + -0.14678414, + -0.13495205, + -0.015179527, + -0.105202034, + 0.03376242, + 0.045832768, + 0.050930824, + 0.113197684, + 0.022861745, + -0.081894, + 0.0609409, + 0.07157385, + -0.030135512, + -0.13592605, + -0.17098251, + -0.037453867, + 0.0445514, + 0.005111366, + 0.22748575, + -0.031975, + 0.17256372, + 0.15765712, + 0.2115168, + -0.01236721, + -0.010223999, + -0.2236333, + -0.2040943, + 0.008426578, + 0.037784062, + -0.21303326, + 0.0014444012, + 0.10017615, + -0.03439675, + -0.03444695, + 0.07566503, + 0.11491664, + 0.14344919, + 0.03547391, + 0.1023221, + -0.26562563, + 0.03324227, + -0.12612627, + -0.025097914, + -0.16231543, + -0.07071355, + -0.031565484, + 0.05040618, + 0.019868962, + 0.12765361, + -0.11720381, + 0.042901862, + 0.04383294, + 0.048617877, + 0.046922315, + -0.03069157, + 0.014754203, + -0.046216488, + -0.09432203, + -0.040246278, + -0.026649779, + -0.072701216, + -0.15752329, + -0.36157733, + 0.040658437, + -0.093095146, + -0.08237973, + 0.13206859, + 0.21616566, + 0.13615578, + -0.052190773, + -0.031580392, + 0.100822076, + -0.012105965, + 0.28700614, + -0.0798311, + -0.030761806, + 0.031985268, + 0.06104218, + 0.017198546, + -0.008219104, + -0.006477159, + -0.030536303, + -0.07336686, + 0.0806302, + -0.015533479, + -0.0037378035, + 0.057539888, + -0.015810573, + 0.0075652716, + -0.058744423, + -0.26509422, + -0.07290028, + -0.042361405, + -0.10962765, + -0.2524032, + -0.13754942, + -0.03327559, + 0.028698724, + -0.032883186, + -0.119565666, + -0.1226996, + 0.05682807, + -0.048052542, + 0.37282622, + 0.028903697, + 0.035287928, + 0.02152677, + 0.23741001, + 0.06756465, + 0.122916296, + -0.07250478, + -0.04318548, + 0.1850775, + -0.01884087, + 0.08287051, + 0.020896921, + 0.010066692, + -0.03891263, + 0.18874517, + 0.010514654, + -0.011000999, + -0.08137152, + -0.035857525, + -0.12775403, + 0.02271913, + -0.09061829, + -0.007581237, + -0.10853437, + 0.03644659, + -0.04371966, + -0.20594738, + -0.10544621, + -0.008287322, + -0.15238318, + -0.08166857, + -0.085347965, + 0.1146354, + 0.079319455, + 0.011942096, + -0.07547264, + 0.037925895, + 0.04891033, + 0.09160054, + 0.03913848, + 0.10733035, + 0.12855045, + -0.008485813, + -0.09220337, + 0.13284315, + -0.068138555, + -0.06429005, + 0.30747658, + 0.017451087, + -0.1090665, + 0.033579294, + -0.22076453, + 0.12192298, + 0.14958231, + -0.13429965, + -0.009634048, + -0.14999968, + -0.00200675, + -0.12121864, + -0.013977665, + 0.048683085, + -0.024490103, + 0.0498644, + -0.011909416, + 0.0015371294, + 0.037581872, + 0.07797013, + -0.06255501, + -0.17288956, + -0.07691344, + 0.37672377, + -0.026362475, + 0.06730378, + 0.0104083195, + -0.024101874, + 0.14750732, + -0.02757014, + -0.1706281, + -0.0031450412, + -0.0055671176, + -0.1802068, + 0.12573895, + -0.032311182, + -0.2111977, + 0.064462945, + -0.057173625, + 0.035591144, + 0.1605281, + 0.07954016, + -0.026228633, + 0.18851891, + 0.13628644, + 0.033962764, + -0.008227327, + 0.103921, + 0.386116, + -0.13597281, + -0.12757441, + 0.043079887, + -0.09839026, + -0.009575337, + -0.005930024, + 0.11509768, + 0.113127336, + -0.063924, + -0.16976811, + -0.026487622, + 0.16157718, + 0.039270367, + 0.013407921, + 0.05706089, + -0.024494743, + -0.032472625, + 0.20470479, + 0.062012397, + 0.013509699, + 0.10476464, + 0.02340273, + 0.22813578, + -0.17417786, + 0.05276678, + -0.2060439, + -0.003265173, + -0.16086264, + -0.23557913, + 0.0522266, + -0.173802, + 0.0074773612, + -0.028467707, + 0.20351472, + -0.057790864, + 0.08757821, + -0.06914482, + -0.14491978, + 0.09239386, + 0.020376774, + 0.1097607, + -0.088697076, + 0.16945688, + 0.01119273, + -0.1528324, + -0.173479, + -0.026488239, + -0.029663468, + -0.037529018, + 0.08184811, + -0.116736144, + 0.020757439, + 0.037805155, + 0.081116766, + -0.08899582, + -0.23994243, + 0.00085073564, + 0.015741583, + 0.14916396, + -0.049735896, + 0.06661528, + -0.05483578, + 0.05611641, + 0.07030592, + -0.14783034, + 0.07137445, + -0.106677264, + 0.060104955, + -0.10194723, + 0.14962104, + 0.028547723, + 0.078555, + -0.09768789, + 0.11221817, + 0.0854506, + -0.15803969, + 0.009801034, + 0.078489214, + -0.0693264, + -0.1491822, + -0.09020748, + -0.03234147, + 0.044914614, + 0.02052608, + -0.1493524, + 0.17103215, + -0.063729316, + -0.10966809, + -0.029281124, + 0.09548159, + -0.048648804, + 0.23593551, + -0.012712764, + 0.07028848, + -0.061728034, + 0.05049216, + -0.021263314, + -0.06874739, + 0.07710166, + 0.08121541, + 0.0734596, + -0.023434158, + 0.16007483, + -0.10867535, + -0.11891655, + 0.22188166, + 0.06489395, + 0.010524853, + 0.19451497, + -0.053407338, + -0.08928299, + -0.14070098, + 0.20396641, + -0.062143497, + 0.125486, + 0.12106229, + -0.1373563, + -0.12767154, + 0.05849606, + -0.1128916, + 0.040035207, + -0.036183015, + -0.13515732, + 0.056579936, + -0.15353481, + 0.17633565, + 0.17068711, + -0.031218339, + -0.06275934, + 0.045422144, + 0.020727132, + -0.015841577, + 0.028296215, + -0.1515143, + 0.05359048, + -0.13335627, + -0.102366865, + 0.09058614, + -0.16648483, + 0.021478737, + -0.02842722, + 0.01834001, + 0.029639145, + -0.023922393, + 0.026784806, + -0.1001365, + -0.042642847, + -0.03436985, + -0.097866334, + 0.019808806, + -0.042747162, + 0.14128022, + -0.033543553, + 0.08283731, + 0.07131058, + -0.15540025, + 0.28758714, + 0.0058351206, + 0.15492027, + 0.013075218, + 0.08945752, + 0.07358258, + 0.054398827, + -0.02437751, + -0.07341751, + 0.10022484, + -0.050757684, + 0.1440264, + 0.013197424, + -0.109852426, + 0.22719486, + 0.080075815, + 0.00024181453, + -0.00097611186, + -0.04847994, + -0.07244215, + 0.006288356, + 0.030367048, + 0.12748545, + 0.1944054, + -0.060363494, + -0.0036989984, + 0.06789716, + 0.03632896, + 0.026532803, + -0.06583803, + -0.13583247, + 0.035222605, + -0.08362427, + -0.18340993, + -0.19657335, + -0.0031432216, + 0.023255847, + -0.18112065, + 0.12979667, + 0.04521475, + -0.037557703, + 0.0030770968, + 0.2226345, + 0.14577474, + 0.0025226166, + -0.023376113, + -0.054195154, + -0.05671215, + -0.11162617, + -0.06985026, + -0.028117025, + 0.0053079883, + 0.061277986, + -0.14143746, + 0.23066823, + 0.19300221, + 0.041472983, + -0.18196483, + 0.05748335, + 0.073920846, + -0.11640508, + -0.11107834, + -0.18051128, + 0.009662717, + 0.13686214, + -0.022840913, + 0.08510747, + 0.012901924, + -0.10250301, + 0.0596237, + -0.18144847, + 0.105939075, + 0.07167126, + 0.14054418, + -0.18191113, + -0.1436565, + -0.0597559, + 0.030895803, + -0.030637655, + -0.13535596, + 0.08015554, + -0.07608762, + -0.040301584, + -0.19534457, + -0.2292024, + -0.04393903, + 0.038195375, + 0.099135764, + -0.021666853, + -0.029471708, + -0.009995175, + 0.25933316, + 0.032328885, + 0.09000577, + 0.03263837, + 0.079661906, + -0.0012039868, + 0.11770895, + -0.0007853804 + ], + [ + 0.067265354, + 0.14978805, + -0.02866475, + -0.017997744, + 0.0043778527, + -0.077955745, + -0.17966285, + -0.08962502, + 0.012652886, + 0.068688974, + 0.032703526, + 0.103429645, + -0.026255865, + -0.04594767, + -0.03591724, + 0.13831529, + -0.063064694, + 0.098223, + -0.120419286, + -0.19822003, + -0.087343685, + 0.05914871, + -0.004434789, + -0.1129431, + 0.17182596, + 0.055425245, + 0.029754065, + -0.03556171, + -0.0070821294, + -0.13465452, + 0.16700113, + -0.19219238, + -0.01675321, + -0.027130553, + -0.18479164, + 0.027703132, + -0.038662884, + -0.025050428, + 0.020932555, + 0.25118732, + 0.13719758, + -0.065483645, + 0.07571365, + -0.07540309, + -0.11814715, + -0.020154916, + -0.1212217, + 0.14467515, + 0.049435012, + -0.03782557, + 0.10122299, + 0.16256903, + 0.18433803, + 0.002223497, + -0.06347142, + -0.045097824, + 0.0076598977, + -0.0047144056, + -0.20919853, + -0.26025832, + -0.021529123, + 0.09192429, + 0.0030727165, + 0.022516953, + -0.02791795, + 0.20013073, + -0.060710344, + -0.014477534, + -0.022307828, + 0.04182581, + 0.07157571, + 0.11714966, + -0.034158412, + -0.07059728, + 0.03238398, + -0.077487394, + -0.19103165, + 0.026339812, + -0.10266619, + 0.17039405, + -0.10501824, + -0.047377028, + -0.086855955, + -0.020825548, + 0.0031076584, + -0.024767814, + 0.12598439, + 0.19383541, + -0.030236963, + 0.16819346, + -0.036930185, + -0.08973632, + 0.20534639, + -0.04046148, + 0.19653097, + 0.022572542, + -0.10891978, + -0.0021857636, + 0.043617047, + -0.03692846, + -0.02452463, + -0.1116284, + -0.120930575, + 0.11805574, + -0.062913835, + -0.04299464, + 0.11609556, + 0.07652675, + 0.15170819, + -0.0143310465, + 0.13215616, + 0.038192805, + -0.09449705, + -0.04615726, + 0.12743029, + 0.053697243, + 0.028676936, + -0.011620548, + -0.115987964, + -0.15858902, + 0.025458544, + 0.16376868, + -0.21461114, + -0.12859254, + 0.046384696, + -0.060146254, + -0.10704182, + 0.1203173, + 0.04870109, + 0.055586778, + -0.036846314, + -0.033308074, + -0.04773714, + -0.1418176, + 0.14744744, + 0.058815204, + -0.077088796, + -0.052262966, + 0.19709195, + 0.026317546, + 0.036419798, + 0.05554856, + -0.0029399341, + -0.062320363, + -0.01966279, + -0.17727584, + 0.04959125, + -0.029366525, + 0.004609626, + -0.053935826, + 0.07947955, + 0.08481755, + -0.10725464, + 0.07710253, + -0.2177989, + 0.018307095, + -0.058796752, + -0.17742877, + -0.103017226, + -0.16547178, + 0.08727635, + -0.18131922, + -0.15467994, + 0.094013415, + -0.0007966098, + -0.11299204, + 0.049469113, + -0.2551994, + 0.10368437, + 0.12887946, + 0.052979, + -0.036250193, + 0.026475381, + 0.068989575, + 0.026221694, + 0.13373744, + -0.16292971, + 0.1264861, + -0.015351114, + -0.046966083, + -0.026861196, + 0.08184943, + 0.03712424, + -0.0893275, + -0.04624934, + 0.17902076, + -0.09396041, + 0.0012100477, + -0.0004969027, + 0.061765272, + -0.13257642, + -0.06479358, + -0.08090168, + -0.045115944, + -0.033314876, + -0.10521074, + 0.038595706, + 0.05002954, + 0.05275309, + -0.022413827, + -0.228349, + -0.05911533, + 0.13100033, + 0.08290936, + 0.21059032, + -0.053871688, + 0.2612328, + -0.03817294, + 0.12055628, + 0.19849202, + 0.1064979, + 0.06644159, + -0.010666335, + 0.037192244, + -0.03825279, + 0.22590448, + 0.06540578, + 0.0635492, + 0.009995832, + 0.053388823, + 0.1819073, + 0.017358864, + 0.0940223, + 0.19547985, + 0.16051306, + -0.08009981, + -0.01766264, + 0.31268245, + 0.03358963, + -0.006802613, + 0.060060095, + -0.02999579, + -0.02441298, + 0.041289836, + -0.018898971, + 0.016558355, + -0.2887073, + 0.05011042, + -0.020560574, + 0.10191658, + 0.07186017, + -0.07590083, + 0.043192476, + 0.04002405, + -0.08334605, + -0.0033909143, + 0.013439505, + 0.1452686, + -0.05733201, + -0.088819325, + -0.022471705, + -0.095721394, + 0.15808508, + 0.120784655, + -0.14439268, + -0.097761706, + -0.15823784, + 0.02247433, + -0.08741217, + -0.10422804, + 0.095085815, + -0.12007163, + -0.090681195, + 0.015306839, + -0.042384245, + 0.059710126, + 0.16539843, + 0.022866111, + 0.061641064, + -0.14381783, + 0.08884166, + 0.07466225, + -0.15382792, + -0.185641, + -0.1813861, + 0.02484775, + -0.0073386994, + -0.0801197, + -0.06278462, + -0.0073598013, + -0.09684346, + 0.06798241, + 0.0029492439, + -0.016103258, + 0.06638396, + 0.01753141, + 0.23454934, + 0.08132862, + 0.10536375, + -0.2229483, + 0.022713436, + -0.14560527, + -0.064387955, + -0.095680125, + 0.22664733, + 0.036997385, + 0.15401214, + -0.0652297, + 0.09910494, + -0.07604658, + 0.019308623, + -0.0017304748, + -0.039347965, + -0.12615763, + 0.038332216, + -0.1498985, + -0.18378036, + 0.22454844, + 0.076195106, + 0.006609157, + -0.051469374, + 0.04080529, + 0.014309058, + -0.096273944, + -0.207181, + -0.15468046, + -0.13909651, + -0.015941054, + -0.050097745, + 0.22421612, + 0.041737087, + 0.01954481, + 0.05503906, + -0.030042086, + -0.060866546, + -0.060889628, + 0.050270688, + -0.032480314, + 0.05457713, + -0.018058803, + -0.17685808, + 0.027825413, + 0.2854748, + -0.1990501, + -0.05410666, + -0.016514089, + 0.021260383, + 0.13303187, + -0.04133936, + -0.102796614, + -0.023230858, + -0.021802323, + -0.009447499, + 0.0056849695, + 0.077226385, + -0.001233707, + -0.056483172, + -0.0148004005, + -0.15466832, + -0.058501728, + -0.07044992, + -0.034258507, + 0.0032710754, + -0.043261908, + 0.042553976, + 0.11541086, + 0.22303134, + -0.042487398, + -0.2697398, + 0.029412314, + -0.093711354, + 0.013379649, + -0.02745153, + -0.09139424, + -0.07518103, + 0.081906185, + 0.01151903, + 0.07499832, + -0.08512211, + -0.017350763, + -0.17634419, + 0.25530154, + -0.05747039, + -0.06407152, + -0.026143344, + 0.031473134, + 0.040568653, + 0.02365954, + -0.03598118, + -0.004738165, + 0.26938042, + 0.107892066, + 0.12235724, + -0.0670435, + 0.103311166, + 0.027247678, + -0.060414925, + 0.10446229, + 0.021563247, + -0.17967018, + 0.021033838, + -0.032761253, + -0.002232972, + 0.20094131, + -0.06271966, + -0.06088999, + 0.114715405, + -0.09043788, + -0.065625295, + -0.034772128, + -0.03247024, + 0.13547981, + -0.05234461, + 0.0008887822, + -0.04399269, + 0.1076599, + 0.19047959, + 0.06816161, + 0.017209189, + -0.02949286, + -0.17175756, + 0.008101737, + -0.11393619, + 0.14412154, + -0.14545026, + 0.10393415, + 0.09651161, + 0.17374185, + -0.15528761, + 0.30169812, + 0.21803834, + 0.22342007, + -0.0023207148, + -0.32583496, + -0.1602966, + 0.08804405, + -0.09492502, + 0.013310529, + 0.08473863, + -0.010918157, + 0.09779154, + 0.1687677, + -0.0017114308, + 0.14069672, + -0.14218633, + -0.119746976, + -0.0016709599, + -0.116650805, + -0.0087289335, + -0.0020710814, + 0.14164573, + -0.03426684, + 0.13765714, + -0.23489869, + 0.17190114, + 0.11823379, + -0.1275406, + 0.016226083, + -0.015224294, + -0.040592875, + 0.088462226, + 0.0100339865, + 0.29286325, + -0.0012119255, + -0.20170744, + 0.043817673, + -0.048842527, + -0.06267558, + -0.20045178, + -0.07331201, + -0.0665427, + 0.0008836231, + 0.047697656, + -0.114039324, + 0.16505136, + 0.055174492, + 0.1341762, + 0.07885245, + -0.12267522, + -0.118794784, + 0.010460504, + 0.036683165, + 0.016049402, + 0.0030134192, + -0.07325558, + -0.059531037, + 0.22491747, + 0.021559626, + -0.2682984, + -0.1513486, + -0.007747446, + -0.21185677, + -0.085555956, + -0.07768858, + -0.0889585, + 0.18072057, + 0.17185259, + 0.071126275, + -0.0070228805, + -0.09613079, + 0.19773807, + -0.027994337, + -0.022889512, + 0.017196476, + 0.16872603, + 0.010325995, + 0.10429709, + -0.050494857, + 0.009961071, + -0.08508044, + -0.03851841, + 0.22150967, + 0.26763844, + 0.013120958, + 0.06905126, + -0.1074305, + 0.021637388, + 0.16322272, + -0.13542607, + 0.05240015, + 0.03747799, + -0.02136567, + -0.19518022, + 0.33402026, + -0.07306034, + 0.24572115, + -0.13900623, + -0.038396675, + 0.06149101, + 0.022670144, + 0.11762282, + 0.02650935, + 0.24784814, + -0.012557801, + -0.049926393, + 0.17277104, + 0.019994384, + 0.17777458, + -0.015156963, + -0.36002278, + 0.08576319, + 0.12579906, + -0.070583634, + 0.046381455, + -0.032518055, + -0.013673033, + 0.08388661, + 0.02805109, + -0.0040858416, + 0.08121704, + -0.024603426, + 0.038663764, + -0.052432787, + 0.09663728, + -0.05794367, + 0.03322958, + -0.05295467, + 0.012488321, + 0.06900259, + 0.32041568, + 0.050188363, + -0.06842132, + 0.17127438, + -0.08995696, + 0.052553497, + -0.15109767, + 0.0032275803, + -0.124561824, + -0.0349717, + -0.15243848, + 0.06931027, + -0.08733464, + 0.08632587, + -0.000009701724, + -0.061496064, + 0.016888466, + 0.14733578, + -0.20635521, + -0.086442135, + -0.010243254, + 0.08384721, + -0.21813874, + 0.053306054, + -0.021313356, + -0.22646224, + -0.111123234, + -0.17016025, + -0.053942624, + 0.10293637, + -0.067348056, + 0.016638568, + -0.055585764, + 0.1321758, + 0.09786219, + -0.0846062, + -0.20103993, + -0.20276587, + -0.14107303, + 0.0572056, + -0.05998819, + -0.079461865, + -0.18345493, + 0.07808181, + 0.025852023, + 0.16117097, + 0.09278437, + -0.0070364075, + -0.09634602, + 0.15276241, + -0.025094103, + -0.25218967, + -0.047830544, + -0.03265829, + -0.14163731, + -0.2689699, + -0.03686031, + 0.012960216, + -0.02138974, + 0.13290478, + 0.071284525, + 0.059489492, + 0.1677266, + 0.04434716, + 0.05381613, + -0.07807097, + -0.007200855, + -0.05029709, + -0.19462912, + -0.06630631, + 0.077244155, + -0.051679987, + 0.16820908, + 0.006629443, + 0.06144869, + 0.044119336, + 0.08528723, + 0.09778228, + -0.05264809, + 0.17047869, + -0.033962384, + 0.09303276, + 0.14469127, + 0.32394484, + -0.05757008, + 0.14690277, + -0.113172874, + -0.073722996, + -0.1543499, + 0.13370518, + 0.022599915, + -0.016452743, + -0.1111051, + -0.123819724, + -0.0358012, + 0.095570944, + -0.026632585, + -0.12938291, + 0.054892946, + 0.0053247954, + 0.0051558986, + -0.08200846, + -0.024864692, + 0.0022463088, + 0.033632293, + 0.022776838, + 0.046233963, + 0.07099262, + 0.03803995, + 0.11918089, + 0.073886305, + -0.02947079, + -0.032019015, + -0.028573902, + 0.07655141, + -0.13654494, + 0.10705977, + -0.054300036, + 0.1552018, + -0.000661521, + 0.22377999, + -0.08899608, + 0.12889342, + -0.20074363, + 0.13927813, + -0.0026708662, + -0.04300883, + -0.16178162, + -0.0923155, + -0.16207108, + 0.20354961, + 0.009087291, + -0.15107653, + -0.06892152, + -0.033151213, + 0.043834474, + -0.115955636, + -0.042398877, + -0.13583317, + 0.0435221, + 0.046010397, + -0.007331226, + -0.03840562, + 0.08738765, + -0.10060804, + 0.06926944, + -0.21893275, + -0.04178857, + -0.19848873, + 0.043663215, + 0.11381242, + -0.09335524, + -0.076091655, + 0.13046667, + -0.0030471336, + 0.04753741, + -0.04804935, + -0.032354355, + 0.08939008, + -0.010100283, + 0.10576193, + 0.01991799, + 0.030111695, + -0.04467943, + -0.015056431, + -0.024010837, + 0.08842912, + -0.005612796, + -0.13563918, + 0.18028839, + 0.013166858, + -0.2667886, + -0.085470125, + 0.3685931, + -0.1946491, + 0.049553536, + -0.37291518, + -0.22655754, + 0.34400705, + 0.032976877, + 0.13435382, + -0.11814129, + 0.047868554, + 0.07159131, + 0.036158036, + 0.024520447, + -0.20078675, + -0.05586612, + 0.08214067, + -0.08605121, + 0.032829337, + 0.058903836, + 0.24436851, + 0.09116331, + -0.13379814, + 0.1828366, + 0.1072864, + -0.021227801, + 0.12980577, + -0.11652907, + -0.03775177, + 0.083177045, + 0.057081014, + -0.06332418, + 0.056298833, + -0.3391434, + 0.04645195, + -0.06128842, + 0.18199284, + -0.002383945, + -0.14004768, + -0.16993281, + 0.010031427, + 0.16041878, + -0.16184394, + 0.018993879, + -0.025938176, + 0.002514922, + 0.020949349, + -0.1269015, + -0.054172676, + 0.094788544, + 0.07986568, + 0.088186376, + -0.028140098, + 0.011289786, + -0.17378683, + -0.12375356, + 0.04456818, + 0.08479448, + -0.09294157, + -0.16615918, + 0.028024394, + -0.093367904, + -0.013316832, + 0.04018244, + 0.08644012, + -0.12008931, + 0.018793972, + 0.061523847, + -0.0694417, + -0.024518935, + -0.11057199, + 0.061863527, + -0.05944173, + 0.11886046, + -0.030533476, + -0.017306136, + 0.0665828, + -0.08461085, + 0.13586487, + 0.12317356, + 0.118130654, + -0.084764935, + 0.009605317, + -0.0064539798, + 0.15894376, + 0.12810811, + 0.029883908, + -0.027100915, + -0.18702109, + 0.016678592, + -0.21666358, + 0.009283024, + -0.06385588, + 0.11203676, + 0.036862098, + -0.051290695, + 0.04737701, + 0.100528136, + 0.033112366, + 0.048415672, + -0.04174812, + -0.16214389, + 0.17123649, + 0.21322905, + 0.021103352, + 0.19598372, + -0.21077679, + -0.06637894, + -0.22511913, + 0.13010465, + -0.044941496, + -0.19983673, + 0.1963295, + -0.15502413, + 0.0615183, + -0.16637003, + -0.01743503, + 0.054865364, + 0.004895811, + -0.034269266, + 0.1766163, + -0.084879376, + 0.04049494, + 0.11087186, + 0.02269163, + -0.027787592, + -0.0145252375, + 0.08132456, + 0.12849073, + 0.06231228, + 0.058701742, + -0.06058468, + 0.13572963, + -0.14186189, + 0.1027375, + 0.028402401, + 0.0053135604, + -0.054753385, + 0.027727582, + -0.034705237, + -0.016670734, + -0.10962946, + -0.011801861, + -0.20003115, + 0.2020829, + 0.026161859, + -0.0031623817, + -0.029170213, + -0.018234199, + 0.04875749, + -0.23150893, + -0.036654674, + 0.09868626, + -0.28474876, + -0.048850782, + -0.0046823416, + -0.0050867144, + -0.028108975, + 0.078664176, + 0.05139621, + -0.13367955, + -0.06766825, + -0.13896853, + -0.16415608, + 0.017776988, + 0.10541162, + -0.0264685, + 0.08690293, + 0.12137225, + -0.034988154, + 0.16255242, + 0.002840024, + 0.16030852, + -0.076692425, + -0.22003824, + -0.18630469, + 0.016727459, + -0.020857252, + -0.008802409, + 0.07564682, + -0.05444138, + 0.1668737, + -0.025629545, + 0.14557064, + 0.09674229, + -0.058274068, + 0.05744842, + -0.0928503, + -0.0799128, + -0.07071504, + -0.0121874735, + 0.1692294, + 0.093030006, + 0.020831311, + 0.04520271, + 0.087474234, + -0.016130574, + -0.05108359, + 0.036344565, + -0.108744465, + -0.055855155, + 0.1017321, + -0.069691285, + -0.14468838, + -0.10310156, + 0.11176399, + -0.00078769, + -0.03110051, + -0.1145704, + -0.07260943, + 0.23371693, + -0.104421645, + -0.06583719, + -0.006062506, + 0.05011631, + -0.0054390067, + -0.078630604, + 0.19628902, + -0.1550763, + -0.13275377, + -0.12952268, + 0.042036884, + 0.05858515, + 0.055275004, + -0.10907125, + -0.00476199, + -0.14193, + 0.04885924, + -0.02711134, + 0.03325598, + -0.04900974, + -0.08830747, + -0.099112585, + 0.13302748, + 0.19985054, + -0.045091256, + -0.20651561, + 0.16883984, + 0.3288204, + 0.024190074, + -0.031287767, + 0.15144359, + -0.008892988, + -0.01452989, + 0.15788583, + -0.151726, + -0.2957167, + -0.1945324, + 0.05123937, + 0.037350606, + 0.09008041, + -0.08441538, + -0.09457509, + 0.067238115, + -0.02894548, + -0.011611294, + 0.03891026, + 0.111131825, + -0.26067775, + -0.28298238, + 0.056074947, + 0.008107262, + 0.0828736, + 0.052385423, + -0.08731629, + 0.14230244, + 0.19611123, + -0.0090899905, + -0.012221198, + -0.00518729, + 0.0022617683, + 0.051705398, + 0.095017456, + -0.056464072, + 0.11342066, + 0.0254407, + -0.13259909, + 0.038187362, + 0.08667088, + -0.07060096, + -0.011785999, + -0.023018276, + 0.11617855, + -0.036035065, + -0.07118992, + -0.12512614, + 0.12636593, + 0.09379038, + 0.10848534, + 0.099208534, + 0.21280734, + 0.011662269, + 0.08994858, + 0.101053335, + -0.11046256, + -0.066710554, + -0.001033645, + 0.08943645, + -0.057964046, + -0.17622581, + 0.09226646, + -0.11705704, + -0.11167349 + ] + ], + "bias": [ + 0.23896483, + -0.3402354, + 0.101270564 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.json new file mode 100644 index 0000000..5c115ef --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.json @@ -0,0 +1,1717 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.71159875, + "confusion": [ + [ + 69, + 75, + 5 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 11, + 58 + ] + ], + "tool_precision": 0.98571426, + "tool_recall": 0.46308726, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5340551, + "p_memory": 0.21771747, + "p_abstain": 0.24822739 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5626822, + "p_memory": 0.1836759, + "p_abstain": 0.2536419 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6036589, + "p_memory": 0.11206787, + "p_abstain": 0.28427315 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40643284, + "p_memory": 0.19013943, + "p_abstain": 0.40342778 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51935685, + "p_memory": 0.09203593, + "p_abstain": 0.3886073 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3248475, + "p_memory": 0.08496061, + "p_abstain": 0.5901919 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5663625, + "p_memory": 0.18072063, + "p_abstain": 0.2529169 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53524184, + "p_memory": 0.17327352, + "p_abstain": 0.2914846 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55211174, + "p_memory": 0.14331223, + "p_abstain": 0.30457598 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48472118, + "p_memory": 0.20033272, + "p_abstain": 0.31494612 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48654568, + "p_memory": 0.20197286, + "p_abstain": 0.31148145 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3951337, + "p_memory": 0.11869106, + "p_abstain": 0.48617515 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5669123, + "p_memory": 0.11284301, + "p_abstain": 0.32024476 + }, + { + "query_id": "shell_test_023", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6097893, + "p_memory": 0.08166945, + "p_abstain": 0.30854127 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "memory", + "p_tools": 0.532595, + "p_memory": 0.09771287, + "p_abstain": 0.36969218 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5313089, + "p_memory": 0.07768788, + "p_abstain": 0.39100322 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5733041, + "p_memory": 0.16126691, + "p_abstain": 0.26542896 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5433677, + "p_memory": 0.18911023, + "p_abstain": 0.2675221 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47030315, + "p_memory": 0.17896877, + "p_abstain": 0.3507281 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5621234, + "p_memory": 0.14502367, + "p_abstain": 0.29285294 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57606184, + "p_memory": 0.1570898, + "p_abstain": 0.2668484 + }, + { + "query_id": "rf_test_020", + "text": "read the existing router_report.md from the last benchmark run", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5553279, + "p_memory": 0.09895797, + "p_abstain": 0.3457141 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5518937, + "p_memory": 0.17088455, + "p_abstain": 0.2772218 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53405726, + "p_memory": 0.19761977, + "p_abstain": 0.268323 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4904085, + "p_memory": 0.08687752, + "p_abstain": 0.42271397 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5497213, + "p_memory": 0.09366132, + "p_abstain": 0.3566174 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41655815, + "p_memory": 0.17062807, + "p_abstain": 0.41281372 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5488636, + "p_memory": 0.2709307, + "p_abstain": 0.1802057 + }, + { + "query_id": "rf_test_028", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54564625, + "p_memory": 0.10946755, + "p_abstain": 0.3448862 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5033046, + "p_memory": 0.11161256, + "p_abstain": 0.3850828 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46809536, + "p_memory": 0.20201841, + "p_abstain": 0.32988623 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41650668, + "p_memory": 0.15908073, + "p_abstain": 0.4244126 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55804354, + "p_memory": 0.1696324, + "p_abstain": 0.27232406 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4662468, + "p_memory": 0.18212193, + "p_abstain": 0.35163125 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56596255, + "p_memory": 0.1841741, + "p_abstain": 0.24986333 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3543384, + "p_memory": 0.07887757, + "p_abstain": 0.566784 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50478405, + "p_memory": 0.16049631, + "p_abstain": 0.33471963 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42139858, + "p_memory": 0.14304228, + "p_abstain": 0.43555915 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49103788, + "p_memory": 0.14342807, + "p_abstain": 0.36553407 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42337945, + "p_memory": 0.1434221, + "p_abstain": 0.4331984 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5013689, + "p_memory": 0.1826258, + "p_abstain": 0.31600538 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.456451, + "p_memory": 0.15955734, + "p_abstain": 0.3839917 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52091247, + "p_memory": 0.18335688, + "p_abstain": 0.29573065 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46343142, + "p_memory": 0.2125959, + "p_abstain": 0.32397267 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5892539, + "p_memory": 0.1738571, + "p_abstain": 0.23688899 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44753295, + "p_memory": 0.27299076, + "p_abstain": 0.27947626 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5022449, + "p_memory": 0.23418446, + "p_abstain": 0.2635706 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.7777778, + "confusion": [ + [ + 38, + 31, + 1 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 8, + 47 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54285717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 6, + 4, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 7, + 33 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.6, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 5, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.6166667, + "confusion": [ + [ + 27, + 22, + 1 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.md new file mode 100644 index 0000000..37de7de --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-10/router_report.md @@ -0,0 +1,146 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.340` +- abstain_prob_threshold: `0.450` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.7116` +- tools precision: `0.9857` +- tools recall: `0.4631` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 69 | 75 | 5 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 11 | 58 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.53 | 0.22 | 0.25 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.56 | 0.18 | 0.25 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Memory` | 0.58 | 0.09 | 0.33 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.57 | 0.17 | 0.26 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Memory` | 0.58 | 0.10 | 0.32 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.58 | 0.18 | 0.24 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_017` | append a line "router ok" to docs/router_holdout_log.txt | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_tools_recall_tools_018` | create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it | `Tools` | `Memory` | 0.58 | 0.16 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Memory` | 0.56 | 0.12 | 0.31 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.56 | 0.11 | 0.33 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.56 | 0.16 | 0.28 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Memory` | 0.51 | 0.08 | 0.41 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Memory` | 0.56 | 0.09 | 0.34 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Memory` | 0.48 | 0.09 | 0.43 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Memory` | 0.50 | 0.13 | 0.36 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.45 | 0.17 | 0.39 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.52 | 0.20 | 0.28 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Memory` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.34 | 0.25 | 0.40 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.35 | 0.32 | 0.33 | +| `holdout_abstain_toolish_abstain_024` | fix this pls | `Abstain` | `Memory` | 0.46 | 0.11 | 0.43 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Memory` | 0.54 | 0.12 | 0.34 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.42 | 0.21 | 0.37 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.40 | 0.15 | 0.45 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Memory` | 0.56 | 0.08 | 0.35 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.47 | 0.21 | 0.32 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Memory` | 0.56 | 0.09 | 0.35 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Memory` | 0.62 | 0.08 | 0.30 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.53 | 0.18 | 0.29 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.60 | 0.11 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.41 | 0.19 | 0.40 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Memory` | 0.52 | 0.09 | 0.39 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.32 | 0.08 | 0.59 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.54 | 0.17 | 0.29 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.55 | 0.14 | 0.30 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.20 | 0.31 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.49 | 0.20 | 0.31 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.40 | 0.12 | 0.49 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Memory` | 0.57 | 0.11 | 0.32 | +| `shell_test_023` | 現在のプロセス一覧でklbrが含まれるものを見せて | `Tools` | `Memory` | 0.61 | 0.08 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Memory` | 0.53 | 0.10 | 0.37 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Memory` | 0.53 | 0.08 | 0.39 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.57 | 0.16 | 0.27 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.54 | 0.19 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.56 | 0.15 | 0.29 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.58 | 0.16 | 0.27 | +| `rf_test_020` | read the existing router_report.md from the last benchmark run | `Tools` | `Memory` | 0.56 | 0.10 | 0.35 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.55 | 0.17 | 0.28 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.53 | 0.20 | 0.27 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Memory` | 0.49 | 0.09 | 0.42 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Memory` | 0.55 | 0.09 | 0.36 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.42 | 0.17 | 0.41 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.55 | 0.27 | 0.18 | +| `rf_test_028` | benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて | `Tools` | `Memory` | 0.55 | 0.11 | 0.34 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.50 | 0.11 | 0.39 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.47 | 0.20 | 0.33 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.42 | 0.16 | 0.42 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.56 | 0.17 | 0.27 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.35 | 0.08 | 0.57 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.50 | 0.16 | 0.33 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.42 | 0.14 | 0.44 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.49 | 0.14 | 0.37 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.42 | 0.14 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.50 | 0.18 | 0.32 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.46 | 0.16 | 0.38 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.52 | 0.18 | 0.30 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.46 | 0.21 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.59 | 0.17 | 0.24 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.45 | 0.27 | 0.28 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.50 | 0.23 | 0.26 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.5429` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7778` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.6000 | 0.0000 | 0.0000 | 0.8167 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.5400 | 0.0000 | 0.0000 | 0.6167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-11/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-11/router_model.json new file mode 100644 index 0000000..c7f1171 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-11/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.1537661, + -0.035649825, + -0.08693668, + -0.039169595, + -0.07801271, + 0.022404537, + 0.13668533, + -0.04904432, + -0.084624946, + -0.12592527, + 0.015163703, + -0.18208449, + 0.0055660303, + 0.07221892, + 0.06448449, + -0.1870893, + -0.075359516, + -0.03543284, + -0.008072806, + 0.035137396, + 0.1856608, + -0.047060464, + -0.026945204, + 0.038430814, + -0.027119577, + -0.07783137, + -0.028168011, + -0.04998577, + 0.13851744, + 0.22334306, + -0.11444658, + -0.10880823, + -0.013960583, + -0.002960124, + 0.1886642, + 0.008615434, + 0.06759129, + 0.025011372, + -0.119343616, + 0.033081945, + -0.08350601, + -0.0484712, + -0.074073195, + 0.045808908, + -0.01146472, + 0.10831494, + -0.0040111416, + -0.09824766, + -0.02530775, + -0.06901607, + -0.050421126, + -0.126361, + -0.088383175, + -0.08048365, + -0.001986429, + 0.18562548, + -0.10405873, + 0.02529612, + 0.108271234, + 0.24374901, + -0.011125183, + 0.0132962605, + -0.07309199, + 0.029683625, + -0.08636377, + -0.03715244, + 0.083628885, + 0.005959518, + 0.07466751, + 0.11360453, + -0.041455425, + 0.09902782, + -0.02341183, + 0.05831913, + -0.042435482, + 0.08605521, + 0.15117645, + -0.019223474, + -0.038496863, + -0.12737247, + 0.058691185, + -0.018187318, + -0.10621924, + -0.027555455, + 0.05125111, + 0.16719356, + -0.17868775, + -0.17773148, + -0.08739252, + 0.10273777, + -0.034631487, + -0.0539006, + -0.096600875, + 0.08291881, + -0.15143853, + -0.09585049, + 0.027222762, + 0.007994819, + 0.060303994, + -0.018851208, + -0.016228046, + 0.08862469, + 0.19132261, + -0.019297566, + -0.053104565, + 0.14122458, + -0.10131717, + -0.09569557, + -0.068272986, + 0.000808413, + -0.15115495, + 0.047998995, + 0.039926194, + 0.037234508, + 0.032307126, + -0.075684965, + -0.073772445, + -0.10967773, + -0.09304202, + 0.040335774, + -0.016109457, + -0.17040172, + 0.13639012, + 0.034102615, + -0.09065539, + 0.1110392, + 0.17052025, + 0.16978532, + 0.1330525, + 0.0534311, + 0.10709471, + 0.12975323, + 0.0031307633, + 0.11619781, + -0.20192865, + -0.04178235, + 0.07954132, + 0.047113087, + -0.03501471, + -0.1652226, + -0.09810865, + 0.086311735, + 0.020574726, + 0.026401358, + 0.15123715, + -0.11407882, + -0.0010843852, + -0.032467104, + 0.04454825, + 0.08361806, + -0.06101439, + 0.060647525, + 0.18783487, + -0.12666312, + 0.084901564, + 0.10627836, + -0.047494754, + -0.0030847418, + 0.13515899, + 0.16071515, + 0.009656678, + -0.032863, + 0.27564478, + -0.07924913, + -0.046650242, + 0.075491354, + -0.06973335, + 0.06965276, + -0.001548173, + -0.142573, + 0.00032827802, + 0.014488301, + -0.1260245, + -0.08115438, + -0.044923816, + -0.03615714, + 0.0039115055, + -0.06106897, + -0.041726876, + 0.046486966, + 0.068862185, + -0.0059390427, + -0.20666197, + 0.11289427, + -0.011771635, + -0.06389011, + 0.028167449, + 0.2907871, + 0.01976517, + -0.08579038, + 0.08004743, + 0.16933607, + 0.05288272, + 0.038429778, + 0.03427874, + -0.044819575, + 0.04110185, + -0.13571876, + -0.19524126, + 0.06988574, + 0.073779844, + -0.046239115, + -0.040948626, + -0.084544, + -0.020007487, + -0.092407554, + -0.17292632, + 0.23797427, + -0.1945358, + -0.49743015, + -0.07454593, + -0.1673993, + -0.10651807, + -0.04177226, + -0.05472722, + -0.15188706, + -0.022152808, + -0.13939627, + -0.019187424, + -0.15158167, + -0.19280887, + 0.10705503, + 0.062492274, + 0.16923375, + -0.26784095, + -0.15662816, + 0.025555544, + -0.17363223, + 0.08134375, + 0.06117359, + -0.09012711, + -0.06951844, + 0.061694615, + -0.003797633, + 0.21708345, + 0.12185522, + 0.19993393, + 0.08121296, + 0.012878898, + -0.054898884, + -0.1080671, + 0.17580487, + -0.06257184, + -0.13819437, + 0.07154935, + 0.09852028, + 0.16923477, + -0.005906965, + -0.0075131687, + 0.119097345, + -0.0682252, + -0.030815927, + -0.19546556, + 0.037838846, + 0.00044135028, + 0.11877078, + 0.10327466, + -0.06929159, + 0.020809863, + 0.13948017, + -0.18564449, + 0.21409702, + 0.14669497, + 0.045995284, + 0.010650101, + -0.07551283, + -0.0006766264, + 0.07865089, + -0.11002661, + -0.024456186, + -0.08400181, + -0.13899222, + 0.09306608, + 0.10094795, + 0.07228482, + 0.019025564, + 0.09661008, + 0.13605566, + 0.028701328, + 0.020753454, + 0.0604083, + 0.05287096, + -0.07340115, + 0.009560703, + -0.025192428, + 0.07975566, + -0.28597078, + 0.021488205, + -0.0889583, + 0.12257723, + -0.13694072, + 0.12471303, + 0.061077848, + 0.004631609, + -0.10417203, + 0.22062522, + -0.013706877, + -0.049198896, + 0.00029548293, + 0.075458646, + 0.1765529, + 0.11541535, + 0.008701007, + 0.24391726, + 0.028608661, + 0.06389106, + 0.08642011, + -0.09138413, + -0.10007181, + -0.0695514, + -0.00900465, + -0.03647545, + 0.019589651, + 0.10178171, + 0.17542481, + 0.11783792, + 0.049075227, + -0.10664901, + 0.03252592, + 0.050435152, + -0.008125645, + -0.10447342, + 0.009139559, + 0.1275379, + -0.021951595, + -0.0136193605, + 0.031333257, + -0.15114506, + -0.1765369, + 0.06522931, + 0.06019238, + -0.035919093, + -0.15519899, + -0.01969548, + -0.03430187, + 0.10910889, + -0.0056543187, + -0.10220757, + 0.26145938, + 0.15574658, + 0.049058672, + -0.0064308834, + 0.042335905, + 0.005810607, + 0.038708996, + -0.0057784626, + 0.049044974, + 0.017823728, + 0.087167464, + 0.074433744, + 0.0077878316, + -0.05596981, + 0.0045128716, + -0.049364593, + 0.0015845216, + 0.14348508, + -0.13488854, + -0.1561442, + 0.054857284, + 0.018423049, + -0.034320723, + 0.0040649255, + -0.04828976, + 0.28164595, + 0.047446065, + -0.11510787, + 0.11150978, + -0.04681086, + 0.071485296, + -0.045360688, + -0.009777533, + 0.038308833, + -0.052017767, + 0.03695938, + 0.057656713, + -0.060405314, + -0.29814774, + -0.071221784, + 0.1714612, + -0.07547322, + 0.005367531, + -0.13997321, + -0.16026817, + 0.04131712, + -0.17275667, + -0.029355615, + 0.08490756, + -0.115701295, + 0.0016283661, + 0.20621264, + 0.16213463, + -0.07390191, + -0.19121905, + -0.13373351, + 0.054568663, + 0.11002628, + 0.025603758, + -0.06534632, + 0.057530094, + 0.018174551, + -0.14876643, + 0.035836782, + -0.01137345, + -0.075621784, + 0.07126445, + -0.04957483, + -0.09046384, + -0.067094974, + -0.042324323, + -0.057362486, + 0.1400903, + -0.03241678, + 0.10672305, + -0.19880442, + 0.09854313, + 0.0760169, + -0.091766715, + -0.12132613, + 0.027035655, + -0.3583752, + -0.13381411, + -0.30593142, + -0.0072146636, + 0.12702839, + -0.1360928, + -0.011511107, + -0.045212187, + -0.032329787, + 0.013346648, + -0.016038861, + 0.07367348, + -0.19516303, + 0.09781957, + -0.023986258, + 0.14528868, + 0.08888904, + -0.0009244085, + -0.05984577, + 0.018362086, + -0.0895728, + -0.023944547, + -0.0010550144, + 0.117598146, + -0.02050585, + -0.12902373, + 0.077714995, + -0.012136869, + -0.028286034, + 0.01971836, + 0.23491275, + 0.00014187285, + -0.025502866, + -0.160924, + -0.10722113, + -0.028722614, + -0.11093872, + 0.12761506, + 0.032540575, + -0.028944362, + 0.026266946, + 0.15018588, + 0.065873004, + -0.2041029, + -0.026748825, + -0.15338984, + 0.019006187, + -0.09869299, + -0.10463372, + 0.056366608, + 0.019882223, + 0.024008995, + -0.06550811, + 0.040581826, + -0.09272168, + -0.021980435, + 0.009398623, + -0.055842966, + -0.067572854, + 0.2544045, + 0.2720369, + 0.056282323, + 0.14016761, + 0.1229671, + 0.085792094, + 0.082071744, + 0.015617369, + 0.0025832395, + -0.11643445, + -0.08638553, + -0.052909072, + -0.16713358, + 0.003093768, + 0.07404296, + 0.031308185, + -0.117112026, + -0.08167997, + -0.1499687, + 0.043127168, + 0.14035194, + -0.07871639, + -0.037212748, + -0.0598638, + -0.14055473, + -0.036376033, + 0.11357092, + -0.050674643, + 0.022856299, + 0.057805985, + 0.089862354, + -0.044336755, + -0.08881311, + -0.10180266, + 0.09055733, + -0.04494839, + 0.09060898, + -0.13318132, + 0.032166876, + 0.00562406, + -0.072895855, + 0.008453028, + -0.10890039, + -0.14855209, + -0.009387831, + 0.07252449, + 0.1000593, + -0.1428223, + -0.08014342, + -0.040895056, + -0.039497916, + 0.36770174, + -0.0024050162, + -0.007834549, + 0.1724918, + 0.0037634445, + -0.18319333, + 0.07710567, + -0.19161667, + -0.014408101, + 0.4058824, + -0.06781774, + -0.004210171, + -0.020524722, + 0.03903467, + -0.065584004, + 0.12053713, + 0.07669467, + 0.019268565, + -0.1424542, + 0.07729574, + -0.1408175, + -0.027871037, + -0.07438413, + -0.24066758, + 0.0007071577, + -0.11960982, + 0.079575315, + 0.026175413, + 0.06513677, + -0.07926052, + 0.053233802, + -0.23602884, + 0.009922975, + -0.024497086, + 0.07963983, + -0.03895233, + 0.012983701, + 0.02903441, + 0.09618848, + -0.086873636, + 0.15702735, + 0.051104963, + 0.23331824, + 0.05189599, + -0.012449083, + 0.18062961, + 0.06019235, + 0.056962494, + 0.03108089, + -0.021042427, + 0.0064071524, + -0.088212445, + 0.08572126, + 0.0037502868, + 0.07312031, + 0.12206003, + 0.15648857, + 0.19765435, + -0.08641273, + -0.02523061, + -0.11257553, + -0.07819527, + -0.028061705, + -0.06571461, + -0.015628025, + 0.062462434, + 0.11131009, + -0.0013901697, + 0.058562018, + 0.06027097, + 0.023649702, + 0.15201351, + 0.08222728, + 0.06710522, + 0.06597227, + 0.15405323, + -0.106588826, + -0.048434135, + -0.08093238, + 0.13272074, + -0.104526795, + 0.066636786, + -0.14262883, + 0.11796827, + 0.01689741, + 0.10963646, + -0.043205373, + 0.030428115, + 0.06697563, + 0.18351023, + -0.12014605, + 0.007847034, + -0.21682698, + -0.053551763, + -0.030757116, + -0.058873527, + -0.039070714, + -0.00346026, + 0.09289441, + -0.14382905, + 0.1066636, + 0.06449059, + 0.21688591, + -0.36460355, + 0.15066512, + -0.064523116, + -0.018895637, + -0.14244261, + 0.018194063, + -0.08151443, + 0.008980479, + -0.08436932, + 0.123211086, + -0.16318616, + 0.11563229, + -0.06480912, + -0.005352673, + 0.068340674, + -0.07209151, + 0.0028943075, + 0.0013212605, + 0.11254483, + 0.098231606, + -0.082876496, + -0.018098809, + -0.019039044, + -0.10377387, + -0.055182043, + -0.045605227, + -0.06043637, + 0.19120796, + 0.10237104, + 0.07438044, + 0.13820165, + 0.17585173, + 0.27409434, + -0.07378421, + 0.025601309, + -0.12231862, + 0.12022717, + -0.101080395, + 0.032167993, + -0.08084088, + -0.17208248, + -0.1681819, + -0.03261707, + 0.02148205, + -0.075628266, + 0.024750838, + 0.03915467, + -0.13104486, + 0.034098193, + -0.034001082, + 0.08776243, + -0.049719255, + -0.06473143, + 0.10588895, + 0.08131149, + -0.052911863, + -0.054036725, + -0.035009447, + 0.08870277, + 0.07426314, + 0.040366404, + 0.077888794, + 0.021348773, + 0.226514, + 0.15032294, + 0.16204213, + 0.000056462246, + 0.09213504, + 0.19880146, + 0.08437888, + 0.021916471, + 0.08471564, + 0.03781062, + -0.066586, + -0.046965122, + -0.10133214, + 0.08557293, + -0.14368777, + -0.06882834, + -0.12171229, + 0.0055409498, + -0.09227392, + -0.10453962, + -0.07994333, + 0.097816125, + 0.0027960786, + -0.11214976, + 0.05112316, + -0.040688016, + 0.068019, + -0.25952643, + 0.16106968, + 0.17121106, + 0.25099227, + 0.076975055, + -0.2097073, + -0.023342844, + 0.015645755, + 0.12014807, + 0.07335009, + -0.05761358, + -0.08484111, + -0.000030331716, + 0.15092224, + 0.06777547, + -0.08367785, + 0.048469383, + -0.11079949, + 0.003651169, + -0.07147883, + -0.014249957, + -0.24292563, + -0.15647426, + -0.17459014, + 0.010819486, + -0.105703846, + -0.030978357, + 0.06532192, + 0.08745108, + -0.053935952, + 0.0688913, + 0.12390802, + 0.21340448, + -0.014140771, + 0.27248612, + -0.2464558, + 0.05955756, + 0.104456566, + 0.009404783, + -0.08957156, + -0.13419013, + -0.026675185, + -0.1552804, + -0.008024584, + 0.005712407, + -0.124870405, + -0.2592146, + 0.19014545, + 0.0327858, + -0.1229455, + 0.010203828, + 0.037715424, + -0.0053597577, + 0.05868922, + 0.010626316, + 0.019355819, + 0.08497353, + 0.119429246, + 0.004582282, + -0.06729474, + 0.07995998, + -0.04374404, + -0.015687706, + -0.053967472, + -0.08461543, + -0.0808064, + -0.07503357, + -0.03532297, + 0.0011162171, + -0.11756368, + 0.11231443, + 0.0066749663, + 0.087183446, + 0.033798646, + 0.17816873, + 0.16899641, + 0.032384265, + 0.037937198, + -0.1306509, + -0.08966298, + -0.11874975, + 0.04818552, + -0.08112427, + -0.08979903, + 0.016811714, + -0.12227783, + 0.006724136, + 0.07726029, + 0.072018504, + 0.047206745, + -0.020475753, + 0.21668829, + 0.06144242, + -0.010373841, + 0.08095411, + -0.00984799, + -0.18237619, + 0.08362384, + -0.06917313, + 0.003942984, + 0.08102712, + -0.08224079, + 0.026713282, + -0.021954091, + -0.21172519, + 0.061612993, + 0.11611484, + 0.15850388, + -0.07526881, + -0.011174921, + 0.12953082, + -0.04849906, + 0.08364964, + 0.045158945, + 0.10626501, + 0.119382195, + -0.20448637, + -0.033443533, + -0.044285737, + -0.078928396, + -0.027338818, + -0.12594555, + 0.047167778, + -0.03249266, + -0.050701614, + 0.083851635, + 0.06785768, + -0.038283225, + -0.02997082, + -0.10361641, + 0.040058605, + 0.013622663, + -0.029170217, + -0.039008163, + 0.08126564, + 0.023967572, + -0.040728178, + 0.020921214, + -0.2012302, + 0.029383505, + 0.039341018, + 0.07352988, + 0.149539, + -0.18081951, + 0.042585555, + -0.07393927, + -0.0520452, + -0.055225402, + -0.02532332, + 0.07143403, + 0.14533009, + 0.020230262, + 0.06286708, + -0.016043179, + -0.0058425125, + -0.1894284, + 0.081516296, + 0.01061884, + 0.08930471, + -0.070286885, + 0.12981166, + 0.01348248, + 0.043093804, + 0.11957937, + 0.022259962, + -0.032027576, + 0.025988672, + -0.16140743, + 0.07117117, + -0.027395064, + -0.059419967, + -0.0067737456, + -0.09964316, + 0.049351126, + 0.21752295, + 0.046031915, + -0.024564896, + -0.011924736, + -0.059805218, + 0.02614514, + -0.015359368, + -0.02796093, + -0.012214168, + 0.0056245904, + -0.032312065, + 0.109036505, + 0.07137153, + 0.108339995, + 0.052375022, + -0.017451666, + -0.14530696, + -0.11981482, + 0.079305254, + -0.0025598651, + -0.05310435, + 0.11399684, + 0.03127478, + 0.0064025996, + -0.03253582, + 0.089398734, + -0.18456937, + -0.0016192505, + 0.30008864, + -0.18448582, + -0.11759908, + -0.15413254, + 0.018025307, + 0.025112813, + -0.0009730928, + -0.2881156, + 0.12879921, + 0.13925475, + -0.09416232, + 0.0006413473, + -0.13858742, + 0.06543319, + -0.08643667, + -0.07211853, + 0.052677996, + 0.1292809, + -0.041060735, + -0.010105186, + 0.017167171, + 0.10278289, + -0.02560506, + 0.01444451, + -0.24326459, + 0.087474845, + -0.029556967, + -0.018887402, + 0.0519785, + 0.07257978, + -0.06718943, + -0.06401816, + 0.009868641, + 0.29014003, + 0.014569991, + -0.13224676, + -0.021046856, + 0.008031937, + 0.029677179, + -0.12090361, + -0.030684829, + -0.12032807, + 0.14864889, + 0.0730822, + 0.04875755, + -0.053761955, + -0.013974497, + -0.03588524, + 0.14112757, + 0.20620118, + 0.0026121754, + 0.057062473, + 0.006303307, + -0.100188285, + 0.030305644, + 0.030009354, + 0.08998002, + -0.0975479, + 0.17385758, + -0.14035688, + -0.12630624, + 0.20372146, + -0.031224119, + -0.015600041, + -0.000572721, + -0.12464091, + 0.028028218, + -0.08736926, + -0.064607315, + 0.0074855303, + -0.003159628, + 0.06802779, + -0.1313798, + 0.06092782, + -0.1787316, + 0.09524019, + 0.2142574, + 0.07154194, + -0.0078775445, + -0.08554093, + 0.17139089, + -0.00896565, + 0.20121369, + -0.08606445, + 0.10155439, + 0.120717004, + -0.05526951, + -0.25100285, + -0.110797964, + -0.06828172, + -0.07158169, + 0.12045773, + -0.19262262, + -0.03129522, + -0.17944223, + 0.02532566, + 0.09656396, + -0.091062434, + -0.0006519022, + 0.11245893 + ], + [ + 0.08650064, + -0.114138156, + 0.11560147, + 0.05716735, + 0.07363487, + 0.055551223, + 0.042977456, + 0.13866937, + 0.07197206, + 0.057236373, + -0.04786727, + 0.078654975, + 0.020689813, + -0.026271248, + -0.02856729, + 0.048773885, + 0.13842425, + -0.06279018, + 0.12849218, + 0.16308266, + -0.09831704, + -0.01208826, + 0.031379998, + 0.074512295, + -0.14470626, + 0.022406168, + -0.0015860604, + 0.08554748, + -0.13143538, + -0.0886885, + -0.05255464, + 0.30100065, + 0.030713795, + 0.030090688, + -0.0038725059, + -0.036318548, + -0.028928364, + 0.000039066, + 0.098410994, + -0.28426936, + -0.05369149, + 0.11395493, + -0.0016404965, + 0.02959418, + 0.1296119, + -0.08816001, + 0.12523289, + -0.046427406, + -0.024127232, + 0.10684162, + -0.050801836, + -0.036207974, + -0.09595485, + 0.078260094, + 0.06545781, + -0.14052771, + 0.096398816, + -0.020581724, + 0.1009272, + 0.016509471, + 0.032654297, + -0.10522052, + 0.07001931, + -0.052200574, + 0.11428169, + -0.16297819, + -0.022918481, + 0.008518012, + -0.052359655, + -0.15543026, + -0.030120252, + -0.21617754, + 0.057570234, + 0.012278141, + 0.0100514935, + -0.008567871, + 0.03985517, + -0.0071163354, + 0.14116311, + -0.043021444, + 0.046327017, + 0.06556431, + 0.19307518, + 0.048381023, + -0.054358784, + -0.14242582, + 0.052703325, + -0.016104024, + 0.11762951, + -0.27093124, + 0.071561694, + 0.14363705, + -0.10874558, + -0.042457294, + -0.045092538, + 0.073277965, + 0.081697024, + -0.005809052, + -0.10392095, + 0.055779688, + 0.040752683, + 0.023003666, + -0.07039203, + -0.098758176, + 0.11601843, + -0.09822991, + -0.014778384, + 0.019168867, + -0.08343514, + 0.013522634, + 0.018998768, + -0.08619179, + 0.05457085, + 0.008922722, + -0.15973733, + 0.021987697, + 0.04509556, + 0.12129829, + 0.20903002, + 0.11825309, + -0.009349105, + 0.0066330372, + 0.07822109, + 0.09448989, + 0.04427069, + -0.05089293, + -0.0634784, + -0.29010263, + -0.1817537, + -0.10901791, + -0.07024844, + -0.09644515, + 0.04460636, + 0.025619725, + 0.054481138, + -0.017032826, + -0.0024525498, + 0.0051498828, + -0.16207719, + 0.13890503, + 0.06168882, + -0.14186028, + -0.017634777, + 0.035919037, + -0.13157438, + 0.2913547, + -0.048506882, + 0.061833624, + -0.049157888, + -0.029682238, + -0.018465182, + -0.14546509, + -0.08058027, + 0.049560603, + 0.13289733, + -0.124585554, + 0.106291436, + 0.18051346, + -0.032141753, + 0.0047566677, + -0.096933044, + 0.21418223, + -0.12096488, + -0.014764312, + 0.047446866, + 0.03750072, + 0.020264208, + 0.18554664, + -0.1021362, + 0.013693483, + -0.05330727, + 0.021761887, + 0.099549085, + 0.0121648405, + 0.018702097, + -0.09758032, + 0.15901814, + -0.065417156, + 0.057077993, + 0.00047911698, + -0.04200102, + -0.075910345, + 0.16953775, + -0.023566708, + 0.058020957, + -0.1151307, + 0.065792955, + -0.29199713, + -0.019268267, + 0.024025127, + 0.052528866, + -0.10454245, + 0.028018963, + 0.00668617, + -0.0009638491, + 0.15003029, + -0.0796975, + 0.08568932, + 0.14248814, + -0.047471944, + 0.154569, + 0.105354466, + -0.09005179, + 0.0016345638, + -0.19058277, + 0.1462792, + -0.088306405, + -0.19980139, + 0.07397957, + 0.2989383, + -0.031951997, + 0.10095756, + 0.11718432, + 0.0045800325, + 0.09298005, + -0.074017435, + -0.043252986, + 0.075847, + 0.009191615, + 0.09819291, + 0.010901586, + -0.12441394, + -0.15651459, + -0.36471352, + 0.107327834, + 0.2367279, + -0.007892925, + -0.13905, + -0.114933364, + -0.05437099, + 0.030067027, + 0.099514276, + -0.037281614, + -0.037492216, + -0.1981845, + -0.1384136, + 0.08877338, + -0.13132341, + 0.00768169, + -0.047017664, + 0.03620687, + -0.099904016, + 0.019379376, + 0.0981703, + 0.011796661, + -0.09512934, + -0.18267427, + -0.13936168, + 0.06484512, + -0.030278102, + 0.090696916, + 0.12653728, + 0.037380584, + -0.15862349, + 0.14395134, + -0.021009054, + 0.054963164, + 0.046817254, + 0.06660228, + -0.035252098, + 0.090558656, + -0.09402542, + -0.056013796, + -0.061302137, + 0.031734105, + 0.015802667, + -0.16472177, + -0.101517014, + 0.048385564, + 0.16827406, + -0.0048398217, + 0.064329885, + 0.060761847, + 0.08469305, + 0.10910126, + -0.043873362, + -0.089271344, + -0.055935968, + 0.034083303, + -0.013393669, + 0.03643514, + -0.120853394, + 0.0704519, + 0.0065425616, + -0.04119151, + -0.09728708, + 0.05142146, + -0.102816895, + -0.016405446, + 0.10037102, + 0.114227235, + 0.020892173, + 0.0033101249, + 0.09104857, + -0.12247543, + -0.2576225, + -0.1403052, + 0.11442854, + -0.099400446, + 0.00058794674, + -0.19586149, + -0.113684855, + 0.030646956, + -0.11775976, + -0.06694089, + 0.086007595, + 0.09736035, + -0.13316418, + 0.023876686, + 0.06294222, + 0.060473986, + -0.0043298365, + -0.03389869, + -0.0055077346, + 0.03175626, + 0.03684258, + 0.09002136, + 0.12259016, + 0.017571814, + -0.27465117, + -0.033611458, + 0.08492853, + -0.064178616, + -0.09749589, + 0.08281812, + 0.074508965, + -0.08160397, + 0.1836253, + 0.12195985, + -0.047170453, + 0.11666565, + 0.00809368, + -0.1302758, + 0.21874562, + 0.08840849, + -0.09259481, + -0.015606064, + -0.030824287, + -0.2201201, + -0.05294996, + -0.025827821, + 0.028233208, + -0.032888405, + -0.011495582, + -0.11593533, + 0.0070121544, + 0.0074381963, + -0.0030233257, + 0.06750082, + -0.015932027, + 0.06266208, + 0.090228274, + -0.007783951, + 0.09262657, + -0.04413849, + -0.258896, + -0.08814282, + 0.19863154, + 0.21488272, + -0.047835387, + 0.12803216, + -0.017444558, + 0.075741306, + -0.19025156, + 0.027734958, + 0.03320167, + -0.123028815, + -0.028187469, + 0.013636808, + 0.06271145, + 0.18612179, + -0.29361016, + 0.10948807, + 0.027112115, + -0.031513385, + 0.028932175, + 0.25757912, + 0.047562208, + -0.1354802, + 0.08021137, + -0.27474785, + 0.032081064, + 0.03791089, + 0.025726352, + 0.069445595, + 0.0021079297, + -0.02449264, + 0.011238894, + -0.02319162, + -0.026542418, + -0.18316846, + 0.10666311, + 0.19345199, + -0.06720772, + 0.00815098, + -0.049136266, + -0.14031912, + 0.15578413, + 0.008095201, + 0.01659755, + 0.18123667, + -0.17131653, + 0.063718066, + 0.07473296, + -0.027271748, + -0.058085024, + -0.100015745, + -0.001066607, + 0.025115151, + 0.086855374, + 0.03166735, + 0.024315033, + 0.007213102, + 0.054682832, + 0.04690719, + -0.17995097, + -0.004744909, + -0.05241575, + 0.12825188, + 0.056677263, + -0.08422418, + 0.08251111, + 0.009535376, + 0.19880669, + 0.2963894, + -0.07653299, + 0.1401372, + 0.019019231, + -0.09808532, + 0.026957024, + -0.17146498, + 0.026395286, + -0.096108206, + -0.11671051, + -0.0031023317, + 0.030857945, + 0.00259537, + 0.17649667, + -0.009633149, + 0.09164389, + -0.11770125, + 0.035321847, + -0.25525537, + 0.2554045, + -0.042877495, + -0.19594881, + 0.1396774, + 0.012059957, + -0.00449406, + -0.19431978, + -0.088604175, + 0.015468876, + -0.13193911, + 0.10843306, + 0.23043013, + 0.06712104, + -0.07877263, + 0.03013501, + 0.22939627, + 0.047045074, + -0.083643205, + -0.066756636, + 0.15640521, + 0.14078821, + -0.011661555, + -0.07418068, + -0.0354833, + 0.025781255, + 0.06630863, + 0.09891263, + -0.034469504, + 0.028824922, + -0.056631226, + 0.08970828, + 0.09523603, + 0.050132398, + -0.1690745, + 0.046013247, + 0.013893757, + -0.12068828, + -0.048534885, + 0.07168909, + -0.037411164, + -0.008103455, + 0.006886762, + -0.19633815, + -0.17443606, + 0.045308072, + 0.09340842, + 0.1490397, + -0.030604543, + 0.02490058, + -0.051153444, + -0.04850469, + -0.051614046, + 0.07135396, + 0.04567157, + 0.0073676747, + -0.15031311, + 0.16379681, + 0.0757311, + -0.1616459, + -0.1270838, + 0.023255069, + -0.1826222, + 0.15810522, + -0.044493705, + -0.22102857, + 0.045563675, + -0.008063402, + 0.05133512, + 0.12316835, + 0.10462279, + -0.2890721, + -0.017548595, + -0.11253995, + 0.10683944, + 0.032772627, + 0.011404846, + -0.031123182, + -0.008722479, + 0.1220427, + -0.23846045, + -0.05996672, + -0.05013295, + -0.029948775, + 0.060149055, + -0.13687938, + 0.054654896, + -0.0076787327, + -0.08335814, + -0.11796455, + -0.101908155, + -0.050144844, + 0.21571128, + -0.06343264, + 0.107729964, + -0.013642982, + -0.40179682, + -0.013399297, + 0.02881357, + -0.018139042, + 0.013398127, + -0.031053284, + -0.06259348, + -0.10992421, + 0.03368614, + 0.1299659, + -0.14629827, + -0.17959803, + -0.02231733, + 0.14280543, + 0.06939315, + 0.08924989, + 0.06705628, + 0.071522295, + -0.029402977, + 0.059425004, + 0.11423234, + 0.09920461, + 0.1667185, + 0.077411644, + -0.061828777, + -0.07963011, + 0.10044846, + -0.029872157, + -0.1763701, + 0.110166654, + 0.17331573, + -0.14678414, + -0.13495205, + -0.015179527, + -0.105202034, + 0.03376242, + 0.045832768, + 0.050930824, + 0.113197684, + 0.022861745, + -0.081894, + 0.0609409, + 0.07157385, + -0.030135512, + -0.13592605, + -0.17098251, + -0.037453867, + 0.0445514, + 0.005111366, + 0.22748575, + -0.031975, + 0.17256372, + 0.15765712, + 0.2115168, + -0.01236721, + -0.010223999, + -0.2236333, + -0.2040943, + 0.008426578, + 0.037784062, + -0.21303326, + 0.0014444012, + 0.10017615, + -0.03439675, + -0.03444695, + 0.07566503, + 0.11491664, + 0.14344919, + 0.03547391, + 0.1023221, + -0.26562563, + 0.03324227, + -0.12612627, + -0.025097914, + -0.16231543, + -0.07071355, + -0.031565484, + 0.05040618, + 0.019868962, + 0.12765361, + -0.11720381, + 0.042901862, + 0.04383294, + 0.048617877, + 0.046922315, + -0.03069157, + 0.014754203, + -0.046216488, + -0.09432203, + -0.040246278, + -0.026649779, + -0.072701216, + -0.15752329, + -0.36157733, + 0.040658437, + -0.093095146, + -0.08237973, + 0.13206859, + 0.21616566, + 0.13615578, + -0.052190773, + -0.031580392, + 0.100822076, + -0.012105965, + 0.28700614, + -0.0798311, + -0.030761806, + 0.031985268, + 0.06104218, + 0.017198546, + -0.008219104, + -0.006477159, + -0.030536303, + -0.07336686, + 0.0806302, + -0.015533479, + -0.0037378035, + 0.057539888, + -0.015810573, + 0.0075652716, + -0.058744423, + -0.26509422, + -0.07290028, + -0.042361405, + -0.10962765, + -0.2524032, + -0.13754942, + -0.03327559, + 0.028698724, + -0.032883186, + -0.119565666, + -0.1226996, + 0.05682807, + -0.048052542, + 0.37282622, + 0.028903697, + 0.035287928, + 0.02152677, + 0.23741001, + 0.06756465, + 0.122916296, + -0.07250478, + -0.04318548, + 0.1850775, + -0.01884087, + 0.08287051, + 0.020896921, + 0.010066692, + -0.03891263, + 0.18874517, + 0.010514654, + -0.011000999, + -0.08137152, + -0.035857525, + -0.12775403, + 0.02271913, + -0.09061829, + -0.007581237, + -0.10853437, + 0.03644659, + -0.04371966, + -0.20594738, + -0.10544621, + -0.008287322, + -0.15238318, + -0.08166857, + -0.085347965, + 0.1146354, + 0.079319455, + 0.011942096, + -0.07547264, + 0.037925895, + 0.04891033, + 0.09160054, + 0.03913848, + 0.10733035, + 0.12855045, + -0.008485813, + -0.09220337, + 0.13284315, + -0.068138555, + -0.06429005, + 0.30747658, + 0.017451087, + -0.1090665, + 0.033579294, + -0.22076453, + 0.12192298, + 0.14958231, + -0.13429965, + -0.009634048, + -0.14999968, + -0.00200675, + -0.12121864, + -0.013977665, + 0.048683085, + -0.024490103, + 0.0498644, + -0.011909416, + 0.0015371294, + 0.037581872, + 0.07797013, + -0.06255501, + -0.17288956, + -0.07691344, + 0.37672377, + -0.026362475, + 0.06730378, + 0.0104083195, + -0.024101874, + 0.14750732, + -0.02757014, + -0.1706281, + -0.0031450412, + -0.0055671176, + -0.1802068, + 0.12573895, + -0.032311182, + -0.2111977, + 0.064462945, + -0.057173625, + 0.035591144, + 0.1605281, + 0.07954016, + -0.026228633, + 0.18851891, + 0.13628644, + 0.033962764, + -0.008227327, + 0.103921, + 0.386116, + -0.13597281, + -0.12757441, + 0.043079887, + -0.09839026, + -0.009575337, + -0.005930024, + 0.11509768, + 0.113127336, + -0.063924, + -0.16976811, + -0.026487622, + 0.16157718, + 0.039270367, + 0.013407921, + 0.05706089, + -0.024494743, + -0.032472625, + 0.20470479, + 0.062012397, + 0.013509699, + 0.10476464, + 0.02340273, + 0.22813578, + -0.17417786, + 0.05276678, + -0.2060439, + -0.003265173, + -0.16086264, + -0.23557913, + 0.0522266, + -0.173802, + 0.0074773612, + -0.028467707, + 0.20351472, + -0.057790864, + 0.08757821, + -0.06914482, + -0.14491978, + 0.09239386, + 0.020376774, + 0.1097607, + -0.088697076, + 0.16945688, + 0.01119273, + -0.1528324, + -0.173479, + -0.026488239, + -0.029663468, + -0.037529018, + 0.08184811, + -0.116736144, + 0.020757439, + 0.037805155, + 0.081116766, + -0.08899582, + -0.23994243, + 0.00085073564, + 0.015741583, + 0.14916396, + -0.049735896, + 0.06661528, + -0.05483578, + 0.05611641, + 0.07030592, + -0.14783034, + 0.07137445, + -0.106677264, + 0.060104955, + -0.10194723, + 0.14962104, + 0.028547723, + 0.078555, + -0.09768789, + 0.11221817, + 0.0854506, + -0.15803969, + 0.009801034, + 0.078489214, + -0.0693264, + -0.1491822, + -0.09020748, + -0.03234147, + 0.044914614, + 0.02052608, + -0.1493524, + 0.17103215, + -0.063729316, + -0.10966809, + -0.029281124, + 0.09548159, + -0.048648804, + 0.23593551, + -0.012712764, + 0.07028848, + -0.061728034, + 0.05049216, + -0.021263314, + -0.06874739, + 0.07710166, + 0.08121541, + 0.0734596, + -0.023434158, + 0.16007483, + -0.10867535, + -0.11891655, + 0.22188166, + 0.06489395, + 0.010524853, + 0.19451497, + -0.053407338, + -0.08928299, + -0.14070098, + 0.20396641, + -0.062143497, + 0.125486, + 0.12106229, + -0.1373563, + -0.12767154, + 0.05849606, + -0.1128916, + 0.040035207, + -0.036183015, + -0.13515732, + 0.056579936, + -0.15353481, + 0.17633565, + 0.17068711, + -0.031218339, + -0.06275934, + 0.045422144, + 0.020727132, + -0.015841577, + 0.028296215, + -0.1515143, + 0.05359048, + -0.13335627, + -0.102366865, + 0.09058614, + -0.16648483, + 0.021478737, + -0.02842722, + 0.01834001, + 0.029639145, + -0.023922393, + 0.026784806, + -0.1001365, + -0.042642847, + -0.03436985, + -0.097866334, + 0.019808806, + -0.042747162, + 0.14128022, + -0.033543553, + 0.08283731, + 0.07131058, + -0.15540025, + 0.28758714, + 0.0058351206, + 0.15492027, + 0.013075218, + 0.08945752, + 0.07358258, + 0.054398827, + -0.02437751, + -0.07341751, + 0.10022484, + -0.050757684, + 0.1440264, + 0.013197424, + -0.109852426, + 0.22719486, + 0.080075815, + 0.00024181453, + -0.00097611186, + -0.04847994, + -0.07244215, + 0.006288356, + 0.030367048, + 0.12748545, + 0.1944054, + -0.060363494, + -0.0036989984, + 0.06789716, + 0.03632896, + 0.026532803, + -0.06583803, + -0.13583247, + 0.035222605, + -0.08362427, + -0.18340993, + -0.19657335, + -0.0031432216, + 0.023255847, + -0.18112065, + 0.12979667, + 0.04521475, + -0.037557703, + 0.0030770968, + 0.2226345, + 0.14577474, + 0.0025226166, + -0.023376113, + -0.054195154, + -0.05671215, + -0.11162617, + -0.06985026, + -0.028117025, + 0.0053079883, + 0.061277986, + -0.14143746, + 0.23066823, + 0.19300221, + 0.041472983, + -0.18196483, + 0.05748335, + 0.073920846, + -0.11640508, + -0.11107834, + -0.18051128, + 0.009662717, + 0.13686214, + -0.022840913, + 0.08510747, + 0.012901924, + -0.10250301, + 0.0596237, + -0.18144847, + 0.105939075, + 0.07167126, + 0.14054418, + -0.18191113, + -0.1436565, + -0.0597559, + 0.030895803, + -0.030637655, + -0.13535596, + 0.08015554, + -0.07608762, + -0.040301584, + -0.19534457, + -0.2292024, + -0.04393903, + 0.038195375, + 0.099135764, + -0.021666853, + -0.029471708, + -0.009995175, + 0.25933316, + 0.032328885, + 0.09000577, + 0.03263837, + 0.079661906, + -0.0012039868, + 0.11770895, + -0.0007853804 + ], + [ + 0.067265354, + 0.14978805, + -0.02866475, + -0.017997744, + 0.0043778527, + -0.077955745, + -0.17966285, + -0.08962502, + 0.012652886, + 0.068688974, + 0.032703526, + 0.103429645, + -0.026255865, + -0.04594767, + -0.03591724, + 0.13831529, + -0.063064694, + 0.098223, + -0.120419286, + -0.19822003, + -0.087343685, + 0.05914871, + -0.004434789, + -0.1129431, + 0.17182596, + 0.055425245, + 0.029754065, + -0.03556171, + -0.0070821294, + -0.13465452, + 0.16700113, + -0.19219238, + -0.01675321, + -0.027130553, + -0.18479164, + 0.027703132, + -0.038662884, + -0.025050428, + 0.020932555, + 0.25118732, + 0.13719758, + -0.065483645, + 0.07571365, + -0.07540309, + -0.11814715, + -0.020154916, + -0.1212217, + 0.14467515, + 0.049435012, + -0.03782557, + 0.10122299, + 0.16256903, + 0.18433803, + 0.002223497, + -0.06347142, + -0.045097824, + 0.0076598977, + -0.0047144056, + -0.20919853, + -0.26025832, + -0.021529123, + 0.09192429, + 0.0030727165, + 0.022516953, + -0.02791795, + 0.20013073, + -0.060710344, + -0.014477534, + -0.022307828, + 0.04182581, + 0.07157571, + 0.11714966, + -0.034158412, + -0.07059728, + 0.03238398, + -0.077487394, + -0.19103165, + 0.026339812, + -0.10266619, + 0.17039405, + -0.10501824, + -0.047377028, + -0.086855955, + -0.020825548, + 0.0031076584, + -0.024767814, + 0.12598439, + 0.19383541, + -0.030236963, + 0.16819346, + -0.036930185, + -0.08973632, + 0.20534639, + -0.04046148, + 0.19653097, + 0.022572542, + -0.10891978, + -0.0021857636, + 0.043617047, + -0.03692846, + -0.02452463, + -0.1116284, + -0.120930575, + 0.11805574, + -0.062913835, + -0.04299464, + 0.11609556, + 0.07652675, + 0.15170819, + -0.0143310465, + 0.13215616, + 0.038192805, + -0.09449705, + -0.04615726, + 0.12743029, + 0.053697243, + 0.028676936, + -0.011620548, + -0.115987964, + -0.15858902, + 0.025458544, + 0.16376868, + -0.21461114, + -0.12859254, + 0.046384696, + -0.060146254, + -0.10704182, + 0.1203173, + 0.04870109, + 0.055586778, + -0.036846314, + -0.033308074, + -0.04773714, + -0.1418176, + 0.14744744, + 0.058815204, + -0.077088796, + -0.052262966, + 0.19709195, + 0.026317546, + 0.036419798, + 0.05554856, + -0.0029399341, + -0.062320363, + -0.01966279, + -0.17727584, + 0.04959125, + -0.029366525, + 0.004609626, + -0.053935826, + 0.07947955, + 0.08481755, + -0.10725464, + 0.07710253, + -0.2177989, + 0.018307095, + -0.058796752, + -0.17742877, + -0.103017226, + -0.16547178, + 0.08727635, + -0.18131922, + -0.15467994, + 0.094013415, + -0.0007966098, + -0.11299204, + 0.049469113, + -0.2551994, + 0.10368437, + 0.12887946, + 0.052979, + -0.036250193, + 0.026475381, + 0.068989575, + 0.026221694, + 0.13373744, + -0.16292971, + 0.1264861, + -0.015351114, + -0.046966083, + -0.026861196, + 0.08184943, + 0.03712424, + -0.0893275, + -0.04624934, + 0.17902076, + -0.09396041, + 0.0012100477, + -0.0004969027, + 0.061765272, + -0.13257642, + -0.06479358, + -0.08090168, + -0.045115944, + -0.033314876, + -0.10521074, + 0.038595706, + 0.05002954, + 0.05275309, + -0.022413827, + -0.228349, + -0.05911533, + 0.13100033, + 0.08290936, + 0.21059032, + -0.053871688, + 0.2612328, + -0.03817294, + 0.12055628, + 0.19849202, + 0.1064979, + 0.06644159, + -0.010666335, + 0.037192244, + -0.03825279, + 0.22590448, + 0.06540578, + 0.0635492, + 0.009995832, + 0.053388823, + 0.1819073, + 0.017358864, + 0.0940223, + 0.19547985, + 0.16051306, + -0.08009981, + -0.01766264, + 0.31268245, + 0.03358963, + -0.006802613, + 0.060060095, + -0.02999579, + -0.02441298, + 0.041289836, + -0.018898971, + 0.016558355, + -0.2887073, + 0.05011042, + -0.020560574, + 0.10191658, + 0.07186017, + -0.07590083, + 0.043192476, + 0.04002405, + -0.08334605, + -0.0033909143, + 0.013439505, + 0.1452686, + -0.05733201, + -0.088819325, + -0.022471705, + -0.095721394, + 0.15808508, + 0.120784655, + -0.14439268, + -0.097761706, + -0.15823784, + 0.02247433, + -0.08741217, + -0.10422804, + 0.095085815, + -0.12007163, + -0.090681195, + 0.015306839, + -0.042384245, + 0.059710126, + 0.16539843, + 0.022866111, + 0.061641064, + -0.14381783, + 0.08884166, + 0.07466225, + -0.15382792, + -0.185641, + -0.1813861, + 0.02484775, + -0.0073386994, + -0.0801197, + -0.06278462, + -0.0073598013, + -0.09684346, + 0.06798241, + 0.0029492439, + -0.016103258, + 0.06638396, + 0.01753141, + 0.23454934, + 0.08132862, + 0.10536375, + -0.2229483, + 0.022713436, + -0.14560527, + -0.064387955, + -0.095680125, + 0.22664733, + 0.036997385, + 0.15401214, + -0.0652297, + 0.09910494, + -0.07604658, + 0.019308623, + -0.0017304748, + -0.039347965, + -0.12615763, + 0.038332216, + -0.1498985, + -0.18378036, + 0.22454844, + 0.076195106, + 0.006609157, + -0.051469374, + 0.04080529, + 0.014309058, + -0.096273944, + -0.207181, + -0.15468046, + -0.13909651, + -0.015941054, + -0.050097745, + 0.22421612, + 0.041737087, + 0.01954481, + 0.05503906, + -0.030042086, + -0.060866546, + -0.060889628, + 0.050270688, + -0.032480314, + 0.05457713, + -0.018058803, + -0.17685808, + 0.027825413, + 0.2854748, + -0.1990501, + -0.05410666, + -0.016514089, + 0.021260383, + 0.13303187, + -0.04133936, + -0.102796614, + -0.023230858, + -0.021802323, + -0.009447499, + 0.0056849695, + 0.077226385, + -0.001233707, + -0.056483172, + -0.0148004005, + -0.15466832, + -0.058501728, + -0.07044992, + -0.034258507, + 0.0032710754, + -0.043261908, + 0.042553976, + 0.11541086, + 0.22303134, + -0.042487398, + -0.2697398, + 0.029412314, + -0.093711354, + 0.013379649, + -0.02745153, + -0.09139424, + -0.07518103, + 0.081906185, + 0.01151903, + 0.07499832, + -0.08512211, + -0.017350763, + -0.17634419, + 0.25530154, + -0.05747039, + -0.06407152, + -0.026143344, + 0.031473134, + 0.040568653, + 0.02365954, + -0.03598118, + -0.004738165, + 0.26938042, + 0.107892066, + 0.12235724, + -0.0670435, + 0.103311166, + 0.027247678, + -0.060414925, + 0.10446229, + 0.021563247, + -0.17967018, + 0.021033838, + -0.032761253, + -0.002232972, + 0.20094131, + -0.06271966, + -0.06088999, + 0.114715405, + -0.09043788, + -0.065625295, + -0.034772128, + -0.03247024, + 0.13547981, + -0.05234461, + 0.0008887822, + -0.04399269, + 0.1076599, + 0.19047959, + 0.06816161, + 0.017209189, + -0.02949286, + -0.17175756, + 0.008101737, + -0.11393619, + 0.14412154, + -0.14545026, + 0.10393415, + 0.09651161, + 0.17374185, + -0.15528761, + 0.30169812, + 0.21803834, + 0.22342007, + -0.0023207148, + -0.32583496, + -0.1602966, + 0.08804405, + -0.09492502, + 0.013310529, + 0.08473863, + -0.010918157, + 0.09779154, + 0.1687677, + -0.0017114308, + 0.14069672, + -0.14218633, + -0.119746976, + -0.0016709599, + -0.116650805, + -0.0087289335, + -0.0020710814, + 0.14164573, + -0.03426684, + 0.13765714, + -0.23489869, + 0.17190114, + 0.11823379, + -0.1275406, + 0.016226083, + -0.015224294, + -0.040592875, + 0.088462226, + 0.0100339865, + 0.29286325, + -0.0012119255, + -0.20170744, + 0.043817673, + -0.048842527, + -0.06267558, + -0.20045178, + -0.07331201, + -0.0665427, + 0.0008836231, + 0.047697656, + -0.114039324, + 0.16505136, + 0.055174492, + 0.1341762, + 0.07885245, + -0.12267522, + -0.118794784, + 0.010460504, + 0.036683165, + 0.016049402, + 0.0030134192, + -0.07325558, + -0.059531037, + 0.22491747, + 0.021559626, + -0.2682984, + -0.1513486, + -0.007747446, + -0.21185677, + -0.085555956, + -0.07768858, + -0.0889585, + 0.18072057, + 0.17185259, + 0.071126275, + -0.0070228805, + -0.09613079, + 0.19773807, + -0.027994337, + -0.022889512, + 0.017196476, + 0.16872603, + 0.010325995, + 0.10429709, + -0.050494857, + 0.009961071, + -0.08508044, + -0.03851841, + 0.22150967, + 0.26763844, + 0.013120958, + 0.06905126, + -0.1074305, + 0.021637388, + 0.16322272, + -0.13542607, + 0.05240015, + 0.03747799, + -0.02136567, + -0.19518022, + 0.33402026, + -0.07306034, + 0.24572115, + -0.13900623, + -0.038396675, + 0.06149101, + 0.022670144, + 0.11762282, + 0.02650935, + 0.24784814, + -0.012557801, + -0.049926393, + 0.17277104, + 0.019994384, + 0.17777458, + -0.015156963, + -0.36002278, + 0.08576319, + 0.12579906, + -0.070583634, + 0.046381455, + -0.032518055, + -0.013673033, + 0.08388661, + 0.02805109, + -0.0040858416, + 0.08121704, + -0.024603426, + 0.038663764, + -0.052432787, + 0.09663728, + -0.05794367, + 0.03322958, + -0.05295467, + 0.012488321, + 0.06900259, + 0.32041568, + 0.050188363, + -0.06842132, + 0.17127438, + -0.08995696, + 0.052553497, + -0.15109767, + 0.0032275803, + -0.124561824, + -0.0349717, + -0.15243848, + 0.06931027, + -0.08733464, + 0.08632587, + -0.000009701724, + -0.061496064, + 0.016888466, + 0.14733578, + -0.20635521, + -0.086442135, + -0.010243254, + 0.08384721, + -0.21813874, + 0.053306054, + -0.021313356, + -0.22646224, + -0.111123234, + -0.17016025, + -0.053942624, + 0.10293637, + -0.067348056, + 0.016638568, + -0.055585764, + 0.1321758, + 0.09786219, + -0.0846062, + -0.20103993, + -0.20276587, + -0.14107303, + 0.0572056, + -0.05998819, + -0.079461865, + -0.18345493, + 0.07808181, + 0.025852023, + 0.16117097, + 0.09278437, + -0.0070364075, + -0.09634602, + 0.15276241, + -0.025094103, + -0.25218967, + -0.047830544, + -0.03265829, + -0.14163731, + -0.2689699, + -0.03686031, + 0.012960216, + -0.02138974, + 0.13290478, + 0.071284525, + 0.059489492, + 0.1677266, + 0.04434716, + 0.05381613, + -0.07807097, + -0.007200855, + -0.05029709, + -0.19462912, + -0.06630631, + 0.077244155, + -0.051679987, + 0.16820908, + 0.006629443, + 0.06144869, + 0.044119336, + 0.08528723, + 0.09778228, + -0.05264809, + 0.17047869, + -0.033962384, + 0.09303276, + 0.14469127, + 0.32394484, + -0.05757008, + 0.14690277, + -0.113172874, + -0.073722996, + -0.1543499, + 0.13370518, + 0.022599915, + -0.016452743, + -0.1111051, + -0.123819724, + -0.0358012, + 0.095570944, + -0.026632585, + -0.12938291, + 0.054892946, + 0.0053247954, + 0.0051558986, + -0.08200846, + -0.024864692, + 0.0022463088, + 0.033632293, + 0.022776838, + 0.046233963, + 0.07099262, + 0.03803995, + 0.11918089, + 0.073886305, + -0.02947079, + -0.032019015, + -0.028573902, + 0.07655141, + -0.13654494, + 0.10705977, + -0.054300036, + 0.1552018, + -0.000661521, + 0.22377999, + -0.08899608, + 0.12889342, + -0.20074363, + 0.13927813, + -0.0026708662, + -0.04300883, + -0.16178162, + -0.0923155, + -0.16207108, + 0.20354961, + 0.009087291, + -0.15107653, + -0.06892152, + -0.033151213, + 0.043834474, + -0.115955636, + -0.042398877, + -0.13583317, + 0.0435221, + 0.046010397, + -0.007331226, + -0.03840562, + 0.08738765, + -0.10060804, + 0.06926944, + -0.21893275, + -0.04178857, + -0.19848873, + 0.043663215, + 0.11381242, + -0.09335524, + -0.076091655, + 0.13046667, + -0.0030471336, + 0.04753741, + -0.04804935, + -0.032354355, + 0.08939008, + -0.010100283, + 0.10576193, + 0.01991799, + 0.030111695, + -0.04467943, + -0.015056431, + -0.024010837, + 0.08842912, + -0.005612796, + -0.13563918, + 0.18028839, + 0.013166858, + -0.2667886, + -0.085470125, + 0.3685931, + -0.1946491, + 0.049553536, + -0.37291518, + -0.22655754, + 0.34400705, + 0.032976877, + 0.13435382, + -0.11814129, + 0.047868554, + 0.07159131, + 0.036158036, + 0.024520447, + -0.20078675, + -0.05586612, + 0.08214067, + -0.08605121, + 0.032829337, + 0.058903836, + 0.24436851, + 0.09116331, + -0.13379814, + 0.1828366, + 0.1072864, + -0.021227801, + 0.12980577, + -0.11652907, + -0.03775177, + 0.083177045, + 0.057081014, + -0.06332418, + 0.056298833, + -0.3391434, + 0.04645195, + -0.06128842, + 0.18199284, + -0.002383945, + -0.14004768, + -0.16993281, + 0.010031427, + 0.16041878, + -0.16184394, + 0.018993879, + -0.025938176, + 0.002514922, + 0.020949349, + -0.1269015, + -0.054172676, + 0.094788544, + 0.07986568, + 0.088186376, + -0.028140098, + 0.011289786, + -0.17378683, + -0.12375356, + 0.04456818, + 0.08479448, + -0.09294157, + -0.16615918, + 0.028024394, + -0.093367904, + -0.013316832, + 0.04018244, + 0.08644012, + -0.12008931, + 0.018793972, + 0.061523847, + -0.0694417, + -0.024518935, + -0.11057199, + 0.061863527, + -0.05944173, + 0.11886046, + -0.030533476, + -0.017306136, + 0.0665828, + -0.08461085, + 0.13586487, + 0.12317356, + 0.118130654, + -0.084764935, + 0.009605317, + -0.0064539798, + 0.15894376, + 0.12810811, + 0.029883908, + -0.027100915, + -0.18702109, + 0.016678592, + -0.21666358, + 0.009283024, + -0.06385588, + 0.11203676, + 0.036862098, + -0.051290695, + 0.04737701, + 0.100528136, + 0.033112366, + 0.048415672, + -0.04174812, + -0.16214389, + 0.17123649, + 0.21322905, + 0.021103352, + 0.19598372, + -0.21077679, + -0.06637894, + -0.22511913, + 0.13010465, + -0.044941496, + -0.19983673, + 0.1963295, + -0.15502413, + 0.0615183, + -0.16637003, + -0.01743503, + 0.054865364, + 0.004895811, + -0.034269266, + 0.1766163, + -0.084879376, + 0.04049494, + 0.11087186, + 0.02269163, + -0.027787592, + -0.0145252375, + 0.08132456, + 0.12849073, + 0.06231228, + 0.058701742, + -0.06058468, + 0.13572963, + -0.14186189, + 0.1027375, + 0.028402401, + 0.0053135604, + -0.054753385, + 0.027727582, + -0.034705237, + -0.016670734, + -0.10962946, + -0.011801861, + -0.20003115, + 0.2020829, + 0.026161859, + -0.0031623817, + -0.029170213, + -0.018234199, + 0.04875749, + -0.23150893, + -0.036654674, + 0.09868626, + -0.28474876, + -0.048850782, + -0.0046823416, + -0.0050867144, + -0.028108975, + 0.078664176, + 0.05139621, + -0.13367955, + -0.06766825, + -0.13896853, + -0.16415608, + 0.017776988, + 0.10541162, + -0.0264685, + 0.08690293, + 0.12137225, + -0.034988154, + 0.16255242, + 0.002840024, + 0.16030852, + -0.076692425, + -0.22003824, + -0.18630469, + 0.016727459, + -0.020857252, + -0.008802409, + 0.07564682, + -0.05444138, + 0.1668737, + -0.025629545, + 0.14557064, + 0.09674229, + -0.058274068, + 0.05744842, + -0.0928503, + -0.0799128, + -0.07071504, + -0.0121874735, + 0.1692294, + 0.093030006, + 0.020831311, + 0.04520271, + 0.087474234, + -0.016130574, + -0.05108359, + 0.036344565, + -0.108744465, + -0.055855155, + 0.1017321, + -0.069691285, + -0.14468838, + -0.10310156, + 0.11176399, + -0.00078769, + -0.03110051, + -0.1145704, + -0.07260943, + 0.23371693, + -0.104421645, + -0.06583719, + -0.006062506, + 0.05011631, + -0.0054390067, + -0.078630604, + 0.19628902, + -0.1550763, + -0.13275377, + -0.12952268, + 0.042036884, + 0.05858515, + 0.055275004, + -0.10907125, + -0.00476199, + -0.14193, + 0.04885924, + -0.02711134, + 0.03325598, + -0.04900974, + -0.08830747, + -0.099112585, + 0.13302748, + 0.19985054, + -0.045091256, + -0.20651561, + 0.16883984, + 0.3288204, + 0.024190074, + -0.031287767, + 0.15144359, + -0.008892988, + -0.01452989, + 0.15788583, + -0.151726, + -0.2957167, + -0.1945324, + 0.05123937, + 0.037350606, + 0.09008041, + -0.08441538, + -0.09457509, + 0.067238115, + -0.02894548, + -0.011611294, + 0.03891026, + 0.111131825, + -0.26067775, + -0.28298238, + 0.056074947, + 0.008107262, + 0.0828736, + 0.052385423, + -0.08731629, + 0.14230244, + 0.19611123, + -0.0090899905, + -0.012221198, + -0.00518729, + 0.0022617683, + 0.051705398, + 0.095017456, + -0.056464072, + 0.11342066, + 0.0254407, + -0.13259909, + 0.038187362, + 0.08667088, + -0.07060096, + -0.011785999, + -0.023018276, + 0.11617855, + -0.036035065, + -0.07118992, + -0.12512614, + 0.12636593, + 0.09379038, + 0.10848534, + 0.099208534, + 0.21280734, + 0.011662269, + 0.08994858, + 0.101053335, + -0.11046256, + -0.066710554, + -0.001033645, + 0.08943645, + -0.057964046, + -0.17622581, + 0.09226646, + -0.11705704, + -0.11167349 + ] + ], + "bias": [ + 0.23896483, + -0.3402354, + 0.101270564 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.json new file mode 100644 index 0000000..5c115ef --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.json @@ -0,0 +1,1717 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.71159875, + "confusion": [ + [ + 69, + 75, + 5 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 11, + 58 + ] + ], + "tool_precision": 0.98571426, + "tool_recall": 0.46308726, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5340551, + "p_memory": 0.21771747, + "p_abstain": 0.24822739 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5626822, + "p_memory": 0.1836759, + "p_abstain": 0.2536419 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6036589, + "p_memory": 0.11206787, + "p_abstain": 0.28427315 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40643284, + "p_memory": 0.19013943, + "p_abstain": 0.40342778 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51935685, + "p_memory": 0.09203593, + "p_abstain": 0.3886073 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3248475, + "p_memory": 0.08496061, + "p_abstain": 0.5901919 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5663625, + "p_memory": 0.18072063, + "p_abstain": 0.2529169 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53524184, + "p_memory": 0.17327352, + "p_abstain": 0.2914846 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55211174, + "p_memory": 0.14331223, + "p_abstain": 0.30457598 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48472118, + "p_memory": 0.20033272, + "p_abstain": 0.31494612 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48654568, + "p_memory": 0.20197286, + "p_abstain": 0.31148145 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3951337, + "p_memory": 0.11869106, + "p_abstain": 0.48617515 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5669123, + "p_memory": 0.11284301, + "p_abstain": 0.32024476 + }, + { + "query_id": "shell_test_023", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6097893, + "p_memory": 0.08166945, + "p_abstain": 0.30854127 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "memory", + "p_tools": 0.532595, + "p_memory": 0.09771287, + "p_abstain": 0.36969218 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5313089, + "p_memory": 0.07768788, + "p_abstain": 0.39100322 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5733041, + "p_memory": 0.16126691, + "p_abstain": 0.26542896 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5433677, + "p_memory": 0.18911023, + "p_abstain": 0.2675221 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47030315, + "p_memory": 0.17896877, + "p_abstain": 0.3507281 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5621234, + "p_memory": 0.14502367, + "p_abstain": 0.29285294 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57606184, + "p_memory": 0.1570898, + "p_abstain": 0.2668484 + }, + { + "query_id": "rf_test_020", + "text": "read the existing router_report.md from the last benchmark run", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5553279, + "p_memory": 0.09895797, + "p_abstain": 0.3457141 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5518937, + "p_memory": 0.17088455, + "p_abstain": 0.2772218 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53405726, + "p_memory": 0.19761977, + "p_abstain": 0.268323 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4904085, + "p_memory": 0.08687752, + "p_abstain": 0.42271397 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5497213, + "p_memory": 0.09366132, + "p_abstain": 0.3566174 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41655815, + "p_memory": 0.17062807, + "p_abstain": 0.41281372 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5488636, + "p_memory": 0.2709307, + "p_abstain": 0.1802057 + }, + { + "query_id": "rf_test_028", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54564625, + "p_memory": 0.10946755, + "p_abstain": 0.3448862 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5033046, + "p_memory": 0.11161256, + "p_abstain": 0.3850828 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46809536, + "p_memory": 0.20201841, + "p_abstain": 0.32988623 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41650668, + "p_memory": 0.15908073, + "p_abstain": 0.4244126 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55804354, + "p_memory": 0.1696324, + "p_abstain": 0.27232406 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4662468, + "p_memory": 0.18212193, + "p_abstain": 0.35163125 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56596255, + "p_memory": 0.1841741, + "p_abstain": 0.24986333 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3543384, + "p_memory": 0.07887757, + "p_abstain": 0.566784 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50478405, + "p_memory": 0.16049631, + "p_abstain": 0.33471963 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42139858, + "p_memory": 0.14304228, + "p_abstain": 0.43555915 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49103788, + "p_memory": 0.14342807, + "p_abstain": 0.36553407 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42337945, + "p_memory": 0.1434221, + "p_abstain": 0.4331984 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5013689, + "p_memory": 0.1826258, + "p_abstain": 0.31600538 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.456451, + "p_memory": 0.15955734, + "p_abstain": 0.3839917 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52091247, + "p_memory": 0.18335688, + "p_abstain": 0.29573065 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46343142, + "p_memory": 0.2125959, + "p_abstain": 0.32397267 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5892539, + "p_memory": 0.1738571, + "p_abstain": 0.23688899 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44753295, + "p_memory": 0.27299076, + "p_abstain": 0.27947626 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5022449, + "p_memory": 0.23418446, + "p_abstain": 0.2635706 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.7777778, + "confusion": [ + [ + 38, + 31, + 1 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 8, + 47 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54285717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 6, + 4, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 7, + 33 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.6, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 5, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.6166667, + "confusion": [ + [ + 27, + 22, + 1 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.md new file mode 100644 index 0000000..37de7de --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-11/router_report.md @@ -0,0 +1,146 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.340` +- abstain_prob_threshold: `0.450` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.7116` +- tools precision: `0.9857` +- tools recall: `0.4631` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 69 | 75 | 5 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 11 | 58 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.53 | 0.22 | 0.25 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.56 | 0.18 | 0.25 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Memory` | 0.58 | 0.09 | 0.33 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.57 | 0.17 | 0.26 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Memory` | 0.58 | 0.10 | 0.32 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.58 | 0.18 | 0.24 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_017` | append a line "router ok" to docs/router_holdout_log.txt | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_tools_recall_tools_018` | create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it | `Tools` | `Memory` | 0.58 | 0.16 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Memory` | 0.56 | 0.12 | 0.31 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.56 | 0.11 | 0.33 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.56 | 0.16 | 0.28 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Memory` | 0.51 | 0.08 | 0.41 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Memory` | 0.56 | 0.09 | 0.34 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Memory` | 0.48 | 0.09 | 0.43 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Memory` | 0.50 | 0.13 | 0.36 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.45 | 0.17 | 0.39 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.52 | 0.20 | 0.28 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Memory` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.34 | 0.25 | 0.40 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.35 | 0.32 | 0.33 | +| `holdout_abstain_toolish_abstain_024` | fix this pls | `Abstain` | `Memory` | 0.46 | 0.11 | 0.43 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Memory` | 0.54 | 0.12 | 0.34 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.42 | 0.21 | 0.37 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.40 | 0.15 | 0.45 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Memory` | 0.56 | 0.08 | 0.35 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.47 | 0.21 | 0.32 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Memory` | 0.56 | 0.09 | 0.35 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Memory` | 0.62 | 0.08 | 0.30 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.53 | 0.18 | 0.29 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.60 | 0.11 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.41 | 0.19 | 0.40 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Memory` | 0.52 | 0.09 | 0.39 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.32 | 0.08 | 0.59 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.54 | 0.17 | 0.29 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.55 | 0.14 | 0.30 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.20 | 0.31 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.49 | 0.20 | 0.31 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.40 | 0.12 | 0.49 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Memory` | 0.57 | 0.11 | 0.32 | +| `shell_test_023` | 現在のプロセス一覧でklbrが含まれるものを見せて | `Tools` | `Memory` | 0.61 | 0.08 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Memory` | 0.53 | 0.10 | 0.37 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Memory` | 0.53 | 0.08 | 0.39 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.57 | 0.16 | 0.27 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.54 | 0.19 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.56 | 0.15 | 0.29 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.58 | 0.16 | 0.27 | +| `rf_test_020` | read the existing router_report.md from the last benchmark run | `Tools` | `Memory` | 0.56 | 0.10 | 0.35 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.55 | 0.17 | 0.28 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.53 | 0.20 | 0.27 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Memory` | 0.49 | 0.09 | 0.42 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Memory` | 0.55 | 0.09 | 0.36 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.42 | 0.17 | 0.41 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.55 | 0.27 | 0.18 | +| `rf_test_028` | benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて | `Tools` | `Memory` | 0.55 | 0.11 | 0.34 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.50 | 0.11 | 0.39 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.47 | 0.20 | 0.33 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.42 | 0.16 | 0.42 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.56 | 0.17 | 0.27 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.35 | 0.08 | 0.57 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.50 | 0.16 | 0.33 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.42 | 0.14 | 0.44 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.49 | 0.14 | 0.37 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.42 | 0.14 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.50 | 0.18 | 0.32 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.46 | 0.16 | 0.38 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.52 | 0.18 | 0.30 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.46 | 0.21 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.59 | 0.17 | 0.24 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.45 | 0.27 | 0.28 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.50 | 0.23 | 0.26 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.5429` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7778` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.6000 | 0.0000 | 0.0000 | 0.8167 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.5400 | 0.0000 | 0.0000 | 0.6167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-12/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-12/router_model.json new file mode 100644 index 0000000..c7f1171 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-12/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.1537661, + -0.035649825, + -0.08693668, + -0.039169595, + -0.07801271, + 0.022404537, + 0.13668533, + -0.04904432, + -0.084624946, + -0.12592527, + 0.015163703, + -0.18208449, + 0.0055660303, + 0.07221892, + 0.06448449, + -0.1870893, + -0.075359516, + -0.03543284, + -0.008072806, + 0.035137396, + 0.1856608, + -0.047060464, + -0.026945204, + 0.038430814, + -0.027119577, + -0.07783137, + -0.028168011, + -0.04998577, + 0.13851744, + 0.22334306, + -0.11444658, + -0.10880823, + -0.013960583, + -0.002960124, + 0.1886642, + 0.008615434, + 0.06759129, + 0.025011372, + -0.119343616, + 0.033081945, + -0.08350601, + -0.0484712, + -0.074073195, + 0.045808908, + -0.01146472, + 0.10831494, + -0.0040111416, + -0.09824766, + -0.02530775, + -0.06901607, + -0.050421126, + -0.126361, + -0.088383175, + -0.08048365, + -0.001986429, + 0.18562548, + -0.10405873, + 0.02529612, + 0.108271234, + 0.24374901, + -0.011125183, + 0.0132962605, + -0.07309199, + 0.029683625, + -0.08636377, + -0.03715244, + 0.083628885, + 0.005959518, + 0.07466751, + 0.11360453, + -0.041455425, + 0.09902782, + -0.02341183, + 0.05831913, + -0.042435482, + 0.08605521, + 0.15117645, + -0.019223474, + -0.038496863, + -0.12737247, + 0.058691185, + -0.018187318, + -0.10621924, + -0.027555455, + 0.05125111, + 0.16719356, + -0.17868775, + -0.17773148, + -0.08739252, + 0.10273777, + -0.034631487, + -0.0539006, + -0.096600875, + 0.08291881, + -0.15143853, + -0.09585049, + 0.027222762, + 0.007994819, + 0.060303994, + -0.018851208, + -0.016228046, + 0.08862469, + 0.19132261, + -0.019297566, + -0.053104565, + 0.14122458, + -0.10131717, + -0.09569557, + -0.068272986, + 0.000808413, + -0.15115495, + 0.047998995, + 0.039926194, + 0.037234508, + 0.032307126, + -0.075684965, + -0.073772445, + -0.10967773, + -0.09304202, + 0.040335774, + -0.016109457, + -0.17040172, + 0.13639012, + 0.034102615, + -0.09065539, + 0.1110392, + 0.17052025, + 0.16978532, + 0.1330525, + 0.0534311, + 0.10709471, + 0.12975323, + 0.0031307633, + 0.11619781, + -0.20192865, + -0.04178235, + 0.07954132, + 0.047113087, + -0.03501471, + -0.1652226, + -0.09810865, + 0.086311735, + 0.020574726, + 0.026401358, + 0.15123715, + -0.11407882, + -0.0010843852, + -0.032467104, + 0.04454825, + 0.08361806, + -0.06101439, + 0.060647525, + 0.18783487, + -0.12666312, + 0.084901564, + 0.10627836, + -0.047494754, + -0.0030847418, + 0.13515899, + 0.16071515, + 0.009656678, + -0.032863, + 0.27564478, + -0.07924913, + -0.046650242, + 0.075491354, + -0.06973335, + 0.06965276, + -0.001548173, + -0.142573, + 0.00032827802, + 0.014488301, + -0.1260245, + -0.08115438, + -0.044923816, + -0.03615714, + 0.0039115055, + -0.06106897, + -0.041726876, + 0.046486966, + 0.068862185, + -0.0059390427, + -0.20666197, + 0.11289427, + -0.011771635, + -0.06389011, + 0.028167449, + 0.2907871, + 0.01976517, + -0.08579038, + 0.08004743, + 0.16933607, + 0.05288272, + 0.038429778, + 0.03427874, + -0.044819575, + 0.04110185, + -0.13571876, + -0.19524126, + 0.06988574, + 0.073779844, + -0.046239115, + -0.040948626, + -0.084544, + -0.020007487, + -0.092407554, + -0.17292632, + 0.23797427, + -0.1945358, + -0.49743015, + -0.07454593, + -0.1673993, + -0.10651807, + -0.04177226, + -0.05472722, + -0.15188706, + -0.022152808, + -0.13939627, + -0.019187424, + -0.15158167, + -0.19280887, + 0.10705503, + 0.062492274, + 0.16923375, + -0.26784095, + -0.15662816, + 0.025555544, + -0.17363223, + 0.08134375, + 0.06117359, + -0.09012711, + -0.06951844, + 0.061694615, + -0.003797633, + 0.21708345, + 0.12185522, + 0.19993393, + 0.08121296, + 0.012878898, + -0.054898884, + -0.1080671, + 0.17580487, + -0.06257184, + -0.13819437, + 0.07154935, + 0.09852028, + 0.16923477, + -0.005906965, + -0.0075131687, + 0.119097345, + -0.0682252, + -0.030815927, + -0.19546556, + 0.037838846, + 0.00044135028, + 0.11877078, + 0.10327466, + -0.06929159, + 0.020809863, + 0.13948017, + -0.18564449, + 0.21409702, + 0.14669497, + 0.045995284, + 0.010650101, + -0.07551283, + -0.0006766264, + 0.07865089, + -0.11002661, + -0.024456186, + -0.08400181, + -0.13899222, + 0.09306608, + 0.10094795, + 0.07228482, + 0.019025564, + 0.09661008, + 0.13605566, + 0.028701328, + 0.020753454, + 0.0604083, + 0.05287096, + -0.07340115, + 0.009560703, + -0.025192428, + 0.07975566, + -0.28597078, + 0.021488205, + -0.0889583, + 0.12257723, + -0.13694072, + 0.12471303, + 0.061077848, + 0.004631609, + -0.10417203, + 0.22062522, + -0.013706877, + -0.049198896, + 0.00029548293, + 0.075458646, + 0.1765529, + 0.11541535, + 0.008701007, + 0.24391726, + 0.028608661, + 0.06389106, + 0.08642011, + -0.09138413, + -0.10007181, + -0.0695514, + -0.00900465, + -0.03647545, + 0.019589651, + 0.10178171, + 0.17542481, + 0.11783792, + 0.049075227, + -0.10664901, + 0.03252592, + 0.050435152, + -0.008125645, + -0.10447342, + 0.009139559, + 0.1275379, + -0.021951595, + -0.0136193605, + 0.031333257, + -0.15114506, + -0.1765369, + 0.06522931, + 0.06019238, + -0.035919093, + -0.15519899, + -0.01969548, + -0.03430187, + 0.10910889, + -0.0056543187, + -0.10220757, + 0.26145938, + 0.15574658, + 0.049058672, + -0.0064308834, + 0.042335905, + 0.005810607, + 0.038708996, + -0.0057784626, + 0.049044974, + 0.017823728, + 0.087167464, + 0.074433744, + 0.0077878316, + -0.05596981, + 0.0045128716, + -0.049364593, + 0.0015845216, + 0.14348508, + -0.13488854, + -0.1561442, + 0.054857284, + 0.018423049, + -0.034320723, + 0.0040649255, + -0.04828976, + 0.28164595, + 0.047446065, + -0.11510787, + 0.11150978, + -0.04681086, + 0.071485296, + -0.045360688, + -0.009777533, + 0.038308833, + -0.052017767, + 0.03695938, + 0.057656713, + -0.060405314, + -0.29814774, + -0.071221784, + 0.1714612, + -0.07547322, + 0.005367531, + -0.13997321, + -0.16026817, + 0.04131712, + -0.17275667, + -0.029355615, + 0.08490756, + -0.115701295, + 0.0016283661, + 0.20621264, + 0.16213463, + -0.07390191, + -0.19121905, + -0.13373351, + 0.054568663, + 0.11002628, + 0.025603758, + -0.06534632, + 0.057530094, + 0.018174551, + -0.14876643, + 0.035836782, + -0.01137345, + -0.075621784, + 0.07126445, + -0.04957483, + -0.09046384, + -0.067094974, + -0.042324323, + -0.057362486, + 0.1400903, + -0.03241678, + 0.10672305, + -0.19880442, + 0.09854313, + 0.0760169, + -0.091766715, + -0.12132613, + 0.027035655, + -0.3583752, + -0.13381411, + -0.30593142, + -0.0072146636, + 0.12702839, + -0.1360928, + -0.011511107, + -0.045212187, + -0.032329787, + 0.013346648, + -0.016038861, + 0.07367348, + -0.19516303, + 0.09781957, + -0.023986258, + 0.14528868, + 0.08888904, + -0.0009244085, + -0.05984577, + 0.018362086, + -0.0895728, + -0.023944547, + -0.0010550144, + 0.117598146, + -0.02050585, + -0.12902373, + 0.077714995, + -0.012136869, + -0.028286034, + 0.01971836, + 0.23491275, + 0.00014187285, + -0.025502866, + -0.160924, + -0.10722113, + -0.028722614, + -0.11093872, + 0.12761506, + 0.032540575, + -0.028944362, + 0.026266946, + 0.15018588, + 0.065873004, + -0.2041029, + -0.026748825, + -0.15338984, + 0.019006187, + -0.09869299, + -0.10463372, + 0.056366608, + 0.019882223, + 0.024008995, + -0.06550811, + 0.040581826, + -0.09272168, + -0.021980435, + 0.009398623, + -0.055842966, + -0.067572854, + 0.2544045, + 0.2720369, + 0.056282323, + 0.14016761, + 0.1229671, + 0.085792094, + 0.082071744, + 0.015617369, + 0.0025832395, + -0.11643445, + -0.08638553, + -0.052909072, + -0.16713358, + 0.003093768, + 0.07404296, + 0.031308185, + -0.117112026, + -0.08167997, + -0.1499687, + 0.043127168, + 0.14035194, + -0.07871639, + -0.037212748, + -0.0598638, + -0.14055473, + -0.036376033, + 0.11357092, + -0.050674643, + 0.022856299, + 0.057805985, + 0.089862354, + -0.044336755, + -0.08881311, + -0.10180266, + 0.09055733, + -0.04494839, + 0.09060898, + -0.13318132, + 0.032166876, + 0.00562406, + -0.072895855, + 0.008453028, + -0.10890039, + -0.14855209, + -0.009387831, + 0.07252449, + 0.1000593, + -0.1428223, + -0.08014342, + -0.040895056, + -0.039497916, + 0.36770174, + -0.0024050162, + -0.007834549, + 0.1724918, + 0.0037634445, + -0.18319333, + 0.07710567, + -0.19161667, + -0.014408101, + 0.4058824, + -0.06781774, + -0.004210171, + -0.020524722, + 0.03903467, + -0.065584004, + 0.12053713, + 0.07669467, + 0.019268565, + -0.1424542, + 0.07729574, + -0.1408175, + -0.027871037, + -0.07438413, + -0.24066758, + 0.0007071577, + -0.11960982, + 0.079575315, + 0.026175413, + 0.06513677, + -0.07926052, + 0.053233802, + -0.23602884, + 0.009922975, + -0.024497086, + 0.07963983, + -0.03895233, + 0.012983701, + 0.02903441, + 0.09618848, + -0.086873636, + 0.15702735, + 0.051104963, + 0.23331824, + 0.05189599, + -0.012449083, + 0.18062961, + 0.06019235, + 0.056962494, + 0.03108089, + -0.021042427, + 0.0064071524, + -0.088212445, + 0.08572126, + 0.0037502868, + 0.07312031, + 0.12206003, + 0.15648857, + 0.19765435, + -0.08641273, + -0.02523061, + -0.11257553, + -0.07819527, + -0.028061705, + -0.06571461, + -0.015628025, + 0.062462434, + 0.11131009, + -0.0013901697, + 0.058562018, + 0.06027097, + 0.023649702, + 0.15201351, + 0.08222728, + 0.06710522, + 0.06597227, + 0.15405323, + -0.106588826, + -0.048434135, + -0.08093238, + 0.13272074, + -0.104526795, + 0.066636786, + -0.14262883, + 0.11796827, + 0.01689741, + 0.10963646, + -0.043205373, + 0.030428115, + 0.06697563, + 0.18351023, + -0.12014605, + 0.007847034, + -0.21682698, + -0.053551763, + -0.030757116, + -0.058873527, + -0.039070714, + -0.00346026, + 0.09289441, + -0.14382905, + 0.1066636, + 0.06449059, + 0.21688591, + -0.36460355, + 0.15066512, + -0.064523116, + -0.018895637, + -0.14244261, + 0.018194063, + -0.08151443, + 0.008980479, + -0.08436932, + 0.123211086, + -0.16318616, + 0.11563229, + -0.06480912, + -0.005352673, + 0.068340674, + -0.07209151, + 0.0028943075, + 0.0013212605, + 0.11254483, + 0.098231606, + -0.082876496, + -0.018098809, + -0.019039044, + -0.10377387, + -0.055182043, + -0.045605227, + -0.06043637, + 0.19120796, + 0.10237104, + 0.07438044, + 0.13820165, + 0.17585173, + 0.27409434, + -0.07378421, + 0.025601309, + -0.12231862, + 0.12022717, + -0.101080395, + 0.032167993, + -0.08084088, + -0.17208248, + -0.1681819, + -0.03261707, + 0.02148205, + -0.075628266, + 0.024750838, + 0.03915467, + -0.13104486, + 0.034098193, + -0.034001082, + 0.08776243, + -0.049719255, + -0.06473143, + 0.10588895, + 0.08131149, + -0.052911863, + -0.054036725, + -0.035009447, + 0.08870277, + 0.07426314, + 0.040366404, + 0.077888794, + 0.021348773, + 0.226514, + 0.15032294, + 0.16204213, + 0.000056462246, + 0.09213504, + 0.19880146, + 0.08437888, + 0.021916471, + 0.08471564, + 0.03781062, + -0.066586, + -0.046965122, + -0.10133214, + 0.08557293, + -0.14368777, + -0.06882834, + -0.12171229, + 0.0055409498, + -0.09227392, + -0.10453962, + -0.07994333, + 0.097816125, + 0.0027960786, + -0.11214976, + 0.05112316, + -0.040688016, + 0.068019, + -0.25952643, + 0.16106968, + 0.17121106, + 0.25099227, + 0.076975055, + -0.2097073, + -0.023342844, + 0.015645755, + 0.12014807, + 0.07335009, + -0.05761358, + -0.08484111, + -0.000030331716, + 0.15092224, + 0.06777547, + -0.08367785, + 0.048469383, + -0.11079949, + 0.003651169, + -0.07147883, + -0.014249957, + -0.24292563, + -0.15647426, + -0.17459014, + 0.010819486, + -0.105703846, + -0.030978357, + 0.06532192, + 0.08745108, + -0.053935952, + 0.0688913, + 0.12390802, + 0.21340448, + -0.014140771, + 0.27248612, + -0.2464558, + 0.05955756, + 0.104456566, + 0.009404783, + -0.08957156, + -0.13419013, + -0.026675185, + -0.1552804, + -0.008024584, + 0.005712407, + -0.124870405, + -0.2592146, + 0.19014545, + 0.0327858, + -0.1229455, + 0.010203828, + 0.037715424, + -0.0053597577, + 0.05868922, + 0.010626316, + 0.019355819, + 0.08497353, + 0.119429246, + 0.004582282, + -0.06729474, + 0.07995998, + -0.04374404, + -0.015687706, + -0.053967472, + -0.08461543, + -0.0808064, + -0.07503357, + -0.03532297, + 0.0011162171, + -0.11756368, + 0.11231443, + 0.0066749663, + 0.087183446, + 0.033798646, + 0.17816873, + 0.16899641, + 0.032384265, + 0.037937198, + -0.1306509, + -0.08966298, + -0.11874975, + 0.04818552, + -0.08112427, + -0.08979903, + 0.016811714, + -0.12227783, + 0.006724136, + 0.07726029, + 0.072018504, + 0.047206745, + -0.020475753, + 0.21668829, + 0.06144242, + -0.010373841, + 0.08095411, + -0.00984799, + -0.18237619, + 0.08362384, + -0.06917313, + 0.003942984, + 0.08102712, + -0.08224079, + 0.026713282, + -0.021954091, + -0.21172519, + 0.061612993, + 0.11611484, + 0.15850388, + -0.07526881, + -0.011174921, + 0.12953082, + -0.04849906, + 0.08364964, + 0.045158945, + 0.10626501, + 0.119382195, + -0.20448637, + -0.033443533, + -0.044285737, + -0.078928396, + -0.027338818, + -0.12594555, + 0.047167778, + -0.03249266, + -0.050701614, + 0.083851635, + 0.06785768, + -0.038283225, + -0.02997082, + -0.10361641, + 0.040058605, + 0.013622663, + -0.029170217, + -0.039008163, + 0.08126564, + 0.023967572, + -0.040728178, + 0.020921214, + -0.2012302, + 0.029383505, + 0.039341018, + 0.07352988, + 0.149539, + -0.18081951, + 0.042585555, + -0.07393927, + -0.0520452, + -0.055225402, + -0.02532332, + 0.07143403, + 0.14533009, + 0.020230262, + 0.06286708, + -0.016043179, + -0.0058425125, + -0.1894284, + 0.081516296, + 0.01061884, + 0.08930471, + -0.070286885, + 0.12981166, + 0.01348248, + 0.043093804, + 0.11957937, + 0.022259962, + -0.032027576, + 0.025988672, + -0.16140743, + 0.07117117, + -0.027395064, + -0.059419967, + -0.0067737456, + -0.09964316, + 0.049351126, + 0.21752295, + 0.046031915, + -0.024564896, + -0.011924736, + -0.059805218, + 0.02614514, + -0.015359368, + -0.02796093, + -0.012214168, + 0.0056245904, + -0.032312065, + 0.109036505, + 0.07137153, + 0.108339995, + 0.052375022, + -0.017451666, + -0.14530696, + -0.11981482, + 0.079305254, + -0.0025598651, + -0.05310435, + 0.11399684, + 0.03127478, + 0.0064025996, + -0.03253582, + 0.089398734, + -0.18456937, + -0.0016192505, + 0.30008864, + -0.18448582, + -0.11759908, + -0.15413254, + 0.018025307, + 0.025112813, + -0.0009730928, + -0.2881156, + 0.12879921, + 0.13925475, + -0.09416232, + 0.0006413473, + -0.13858742, + 0.06543319, + -0.08643667, + -0.07211853, + 0.052677996, + 0.1292809, + -0.041060735, + -0.010105186, + 0.017167171, + 0.10278289, + -0.02560506, + 0.01444451, + -0.24326459, + 0.087474845, + -0.029556967, + -0.018887402, + 0.0519785, + 0.07257978, + -0.06718943, + -0.06401816, + 0.009868641, + 0.29014003, + 0.014569991, + -0.13224676, + -0.021046856, + 0.008031937, + 0.029677179, + -0.12090361, + -0.030684829, + -0.12032807, + 0.14864889, + 0.0730822, + 0.04875755, + -0.053761955, + -0.013974497, + -0.03588524, + 0.14112757, + 0.20620118, + 0.0026121754, + 0.057062473, + 0.006303307, + -0.100188285, + 0.030305644, + 0.030009354, + 0.08998002, + -0.0975479, + 0.17385758, + -0.14035688, + -0.12630624, + 0.20372146, + -0.031224119, + -0.015600041, + -0.000572721, + -0.12464091, + 0.028028218, + -0.08736926, + -0.064607315, + 0.0074855303, + -0.003159628, + 0.06802779, + -0.1313798, + 0.06092782, + -0.1787316, + 0.09524019, + 0.2142574, + 0.07154194, + -0.0078775445, + -0.08554093, + 0.17139089, + -0.00896565, + 0.20121369, + -0.08606445, + 0.10155439, + 0.120717004, + -0.05526951, + -0.25100285, + -0.110797964, + -0.06828172, + -0.07158169, + 0.12045773, + -0.19262262, + -0.03129522, + -0.17944223, + 0.02532566, + 0.09656396, + -0.091062434, + -0.0006519022, + 0.11245893 + ], + [ + 0.08650064, + -0.114138156, + 0.11560147, + 0.05716735, + 0.07363487, + 0.055551223, + 0.042977456, + 0.13866937, + 0.07197206, + 0.057236373, + -0.04786727, + 0.078654975, + 0.020689813, + -0.026271248, + -0.02856729, + 0.048773885, + 0.13842425, + -0.06279018, + 0.12849218, + 0.16308266, + -0.09831704, + -0.01208826, + 0.031379998, + 0.074512295, + -0.14470626, + 0.022406168, + -0.0015860604, + 0.08554748, + -0.13143538, + -0.0886885, + -0.05255464, + 0.30100065, + 0.030713795, + 0.030090688, + -0.0038725059, + -0.036318548, + -0.028928364, + 0.000039066, + 0.098410994, + -0.28426936, + -0.05369149, + 0.11395493, + -0.0016404965, + 0.02959418, + 0.1296119, + -0.08816001, + 0.12523289, + -0.046427406, + -0.024127232, + 0.10684162, + -0.050801836, + -0.036207974, + -0.09595485, + 0.078260094, + 0.06545781, + -0.14052771, + 0.096398816, + -0.020581724, + 0.1009272, + 0.016509471, + 0.032654297, + -0.10522052, + 0.07001931, + -0.052200574, + 0.11428169, + -0.16297819, + -0.022918481, + 0.008518012, + -0.052359655, + -0.15543026, + -0.030120252, + -0.21617754, + 0.057570234, + 0.012278141, + 0.0100514935, + -0.008567871, + 0.03985517, + -0.0071163354, + 0.14116311, + -0.043021444, + 0.046327017, + 0.06556431, + 0.19307518, + 0.048381023, + -0.054358784, + -0.14242582, + 0.052703325, + -0.016104024, + 0.11762951, + -0.27093124, + 0.071561694, + 0.14363705, + -0.10874558, + -0.042457294, + -0.045092538, + 0.073277965, + 0.081697024, + -0.005809052, + -0.10392095, + 0.055779688, + 0.040752683, + 0.023003666, + -0.07039203, + -0.098758176, + 0.11601843, + -0.09822991, + -0.014778384, + 0.019168867, + -0.08343514, + 0.013522634, + 0.018998768, + -0.08619179, + 0.05457085, + 0.008922722, + -0.15973733, + 0.021987697, + 0.04509556, + 0.12129829, + 0.20903002, + 0.11825309, + -0.009349105, + 0.0066330372, + 0.07822109, + 0.09448989, + 0.04427069, + -0.05089293, + -0.0634784, + -0.29010263, + -0.1817537, + -0.10901791, + -0.07024844, + -0.09644515, + 0.04460636, + 0.025619725, + 0.054481138, + -0.017032826, + -0.0024525498, + 0.0051498828, + -0.16207719, + 0.13890503, + 0.06168882, + -0.14186028, + -0.017634777, + 0.035919037, + -0.13157438, + 0.2913547, + -0.048506882, + 0.061833624, + -0.049157888, + -0.029682238, + -0.018465182, + -0.14546509, + -0.08058027, + 0.049560603, + 0.13289733, + -0.124585554, + 0.106291436, + 0.18051346, + -0.032141753, + 0.0047566677, + -0.096933044, + 0.21418223, + -0.12096488, + -0.014764312, + 0.047446866, + 0.03750072, + 0.020264208, + 0.18554664, + -0.1021362, + 0.013693483, + -0.05330727, + 0.021761887, + 0.099549085, + 0.0121648405, + 0.018702097, + -0.09758032, + 0.15901814, + -0.065417156, + 0.057077993, + 0.00047911698, + -0.04200102, + -0.075910345, + 0.16953775, + -0.023566708, + 0.058020957, + -0.1151307, + 0.065792955, + -0.29199713, + -0.019268267, + 0.024025127, + 0.052528866, + -0.10454245, + 0.028018963, + 0.00668617, + -0.0009638491, + 0.15003029, + -0.0796975, + 0.08568932, + 0.14248814, + -0.047471944, + 0.154569, + 0.105354466, + -0.09005179, + 0.0016345638, + -0.19058277, + 0.1462792, + -0.088306405, + -0.19980139, + 0.07397957, + 0.2989383, + -0.031951997, + 0.10095756, + 0.11718432, + 0.0045800325, + 0.09298005, + -0.074017435, + -0.043252986, + 0.075847, + 0.009191615, + 0.09819291, + 0.010901586, + -0.12441394, + -0.15651459, + -0.36471352, + 0.107327834, + 0.2367279, + -0.007892925, + -0.13905, + -0.114933364, + -0.05437099, + 0.030067027, + 0.099514276, + -0.037281614, + -0.037492216, + -0.1981845, + -0.1384136, + 0.08877338, + -0.13132341, + 0.00768169, + -0.047017664, + 0.03620687, + -0.099904016, + 0.019379376, + 0.0981703, + 0.011796661, + -0.09512934, + -0.18267427, + -0.13936168, + 0.06484512, + -0.030278102, + 0.090696916, + 0.12653728, + 0.037380584, + -0.15862349, + 0.14395134, + -0.021009054, + 0.054963164, + 0.046817254, + 0.06660228, + -0.035252098, + 0.090558656, + -0.09402542, + -0.056013796, + -0.061302137, + 0.031734105, + 0.015802667, + -0.16472177, + -0.101517014, + 0.048385564, + 0.16827406, + -0.0048398217, + 0.064329885, + 0.060761847, + 0.08469305, + 0.10910126, + -0.043873362, + -0.089271344, + -0.055935968, + 0.034083303, + -0.013393669, + 0.03643514, + -0.120853394, + 0.0704519, + 0.0065425616, + -0.04119151, + -0.09728708, + 0.05142146, + -0.102816895, + -0.016405446, + 0.10037102, + 0.114227235, + 0.020892173, + 0.0033101249, + 0.09104857, + -0.12247543, + -0.2576225, + -0.1403052, + 0.11442854, + -0.099400446, + 0.00058794674, + -0.19586149, + -0.113684855, + 0.030646956, + -0.11775976, + -0.06694089, + 0.086007595, + 0.09736035, + -0.13316418, + 0.023876686, + 0.06294222, + 0.060473986, + -0.0043298365, + -0.03389869, + -0.0055077346, + 0.03175626, + 0.03684258, + 0.09002136, + 0.12259016, + 0.017571814, + -0.27465117, + -0.033611458, + 0.08492853, + -0.064178616, + -0.09749589, + 0.08281812, + 0.074508965, + -0.08160397, + 0.1836253, + 0.12195985, + -0.047170453, + 0.11666565, + 0.00809368, + -0.1302758, + 0.21874562, + 0.08840849, + -0.09259481, + -0.015606064, + -0.030824287, + -0.2201201, + -0.05294996, + -0.025827821, + 0.028233208, + -0.032888405, + -0.011495582, + -0.11593533, + 0.0070121544, + 0.0074381963, + -0.0030233257, + 0.06750082, + -0.015932027, + 0.06266208, + 0.090228274, + -0.007783951, + 0.09262657, + -0.04413849, + -0.258896, + -0.08814282, + 0.19863154, + 0.21488272, + -0.047835387, + 0.12803216, + -0.017444558, + 0.075741306, + -0.19025156, + 0.027734958, + 0.03320167, + -0.123028815, + -0.028187469, + 0.013636808, + 0.06271145, + 0.18612179, + -0.29361016, + 0.10948807, + 0.027112115, + -0.031513385, + 0.028932175, + 0.25757912, + 0.047562208, + -0.1354802, + 0.08021137, + -0.27474785, + 0.032081064, + 0.03791089, + 0.025726352, + 0.069445595, + 0.0021079297, + -0.02449264, + 0.011238894, + -0.02319162, + -0.026542418, + -0.18316846, + 0.10666311, + 0.19345199, + -0.06720772, + 0.00815098, + -0.049136266, + -0.14031912, + 0.15578413, + 0.008095201, + 0.01659755, + 0.18123667, + -0.17131653, + 0.063718066, + 0.07473296, + -0.027271748, + -0.058085024, + -0.100015745, + -0.001066607, + 0.025115151, + 0.086855374, + 0.03166735, + 0.024315033, + 0.007213102, + 0.054682832, + 0.04690719, + -0.17995097, + -0.004744909, + -0.05241575, + 0.12825188, + 0.056677263, + -0.08422418, + 0.08251111, + 0.009535376, + 0.19880669, + 0.2963894, + -0.07653299, + 0.1401372, + 0.019019231, + -0.09808532, + 0.026957024, + -0.17146498, + 0.026395286, + -0.096108206, + -0.11671051, + -0.0031023317, + 0.030857945, + 0.00259537, + 0.17649667, + -0.009633149, + 0.09164389, + -0.11770125, + 0.035321847, + -0.25525537, + 0.2554045, + -0.042877495, + -0.19594881, + 0.1396774, + 0.012059957, + -0.00449406, + -0.19431978, + -0.088604175, + 0.015468876, + -0.13193911, + 0.10843306, + 0.23043013, + 0.06712104, + -0.07877263, + 0.03013501, + 0.22939627, + 0.047045074, + -0.083643205, + -0.066756636, + 0.15640521, + 0.14078821, + -0.011661555, + -0.07418068, + -0.0354833, + 0.025781255, + 0.06630863, + 0.09891263, + -0.034469504, + 0.028824922, + -0.056631226, + 0.08970828, + 0.09523603, + 0.050132398, + -0.1690745, + 0.046013247, + 0.013893757, + -0.12068828, + -0.048534885, + 0.07168909, + -0.037411164, + -0.008103455, + 0.006886762, + -0.19633815, + -0.17443606, + 0.045308072, + 0.09340842, + 0.1490397, + -0.030604543, + 0.02490058, + -0.051153444, + -0.04850469, + -0.051614046, + 0.07135396, + 0.04567157, + 0.0073676747, + -0.15031311, + 0.16379681, + 0.0757311, + -0.1616459, + -0.1270838, + 0.023255069, + -0.1826222, + 0.15810522, + -0.044493705, + -0.22102857, + 0.045563675, + -0.008063402, + 0.05133512, + 0.12316835, + 0.10462279, + -0.2890721, + -0.017548595, + -0.11253995, + 0.10683944, + 0.032772627, + 0.011404846, + -0.031123182, + -0.008722479, + 0.1220427, + -0.23846045, + -0.05996672, + -0.05013295, + -0.029948775, + 0.060149055, + -0.13687938, + 0.054654896, + -0.0076787327, + -0.08335814, + -0.11796455, + -0.101908155, + -0.050144844, + 0.21571128, + -0.06343264, + 0.107729964, + -0.013642982, + -0.40179682, + -0.013399297, + 0.02881357, + -0.018139042, + 0.013398127, + -0.031053284, + -0.06259348, + -0.10992421, + 0.03368614, + 0.1299659, + -0.14629827, + -0.17959803, + -0.02231733, + 0.14280543, + 0.06939315, + 0.08924989, + 0.06705628, + 0.071522295, + -0.029402977, + 0.059425004, + 0.11423234, + 0.09920461, + 0.1667185, + 0.077411644, + -0.061828777, + -0.07963011, + 0.10044846, + -0.029872157, + -0.1763701, + 0.110166654, + 0.17331573, + -0.14678414, + -0.13495205, + -0.015179527, + -0.105202034, + 0.03376242, + 0.045832768, + 0.050930824, + 0.113197684, + 0.022861745, + -0.081894, + 0.0609409, + 0.07157385, + -0.030135512, + -0.13592605, + -0.17098251, + -0.037453867, + 0.0445514, + 0.005111366, + 0.22748575, + -0.031975, + 0.17256372, + 0.15765712, + 0.2115168, + -0.01236721, + -0.010223999, + -0.2236333, + -0.2040943, + 0.008426578, + 0.037784062, + -0.21303326, + 0.0014444012, + 0.10017615, + -0.03439675, + -0.03444695, + 0.07566503, + 0.11491664, + 0.14344919, + 0.03547391, + 0.1023221, + -0.26562563, + 0.03324227, + -0.12612627, + -0.025097914, + -0.16231543, + -0.07071355, + -0.031565484, + 0.05040618, + 0.019868962, + 0.12765361, + -0.11720381, + 0.042901862, + 0.04383294, + 0.048617877, + 0.046922315, + -0.03069157, + 0.014754203, + -0.046216488, + -0.09432203, + -0.040246278, + -0.026649779, + -0.072701216, + -0.15752329, + -0.36157733, + 0.040658437, + -0.093095146, + -0.08237973, + 0.13206859, + 0.21616566, + 0.13615578, + -0.052190773, + -0.031580392, + 0.100822076, + -0.012105965, + 0.28700614, + -0.0798311, + -0.030761806, + 0.031985268, + 0.06104218, + 0.017198546, + -0.008219104, + -0.006477159, + -0.030536303, + -0.07336686, + 0.0806302, + -0.015533479, + -0.0037378035, + 0.057539888, + -0.015810573, + 0.0075652716, + -0.058744423, + -0.26509422, + -0.07290028, + -0.042361405, + -0.10962765, + -0.2524032, + -0.13754942, + -0.03327559, + 0.028698724, + -0.032883186, + -0.119565666, + -0.1226996, + 0.05682807, + -0.048052542, + 0.37282622, + 0.028903697, + 0.035287928, + 0.02152677, + 0.23741001, + 0.06756465, + 0.122916296, + -0.07250478, + -0.04318548, + 0.1850775, + -0.01884087, + 0.08287051, + 0.020896921, + 0.010066692, + -0.03891263, + 0.18874517, + 0.010514654, + -0.011000999, + -0.08137152, + -0.035857525, + -0.12775403, + 0.02271913, + -0.09061829, + -0.007581237, + -0.10853437, + 0.03644659, + -0.04371966, + -0.20594738, + -0.10544621, + -0.008287322, + -0.15238318, + -0.08166857, + -0.085347965, + 0.1146354, + 0.079319455, + 0.011942096, + -0.07547264, + 0.037925895, + 0.04891033, + 0.09160054, + 0.03913848, + 0.10733035, + 0.12855045, + -0.008485813, + -0.09220337, + 0.13284315, + -0.068138555, + -0.06429005, + 0.30747658, + 0.017451087, + -0.1090665, + 0.033579294, + -0.22076453, + 0.12192298, + 0.14958231, + -0.13429965, + -0.009634048, + -0.14999968, + -0.00200675, + -0.12121864, + -0.013977665, + 0.048683085, + -0.024490103, + 0.0498644, + -0.011909416, + 0.0015371294, + 0.037581872, + 0.07797013, + -0.06255501, + -0.17288956, + -0.07691344, + 0.37672377, + -0.026362475, + 0.06730378, + 0.0104083195, + -0.024101874, + 0.14750732, + -0.02757014, + -0.1706281, + -0.0031450412, + -0.0055671176, + -0.1802068, + 0.12573895, + -0.032311182, + -0.2111977, + 0.064462945, + -0.057173625, + 0.035591144, + 0.1605281, + 0.07954016, + -0.026228633, + 0.18851891, + 0.13628644, + 0.033962764, + -0.008227327, + 0.103921, + 0.386116, + -0.13597281, + -0.12757441, + 0.043079887, + -0.09839026, + -0.009575337, + -0.005930024, + 0.11509768, + 0.113127336, + -0.063924, + -0.16976811, + -0.026487622, + 0.16157718, + 0.039270367, + 0.013407921, + 0.05706089, + -0.024494743, + -0.032472625, + 0.20470479, + 0.062012397, + 0.013509699, + 0.10476464, + 0.02340273, + 0.22813578, + -0.17417786, + 0.05276678, + -0.2060439, + -0.003265173, + -0.16086264, + -0.23557913, + 0.0522266, + -0.173802, + 0.0074773612, + -0.028467707, + 0.20351472, + -0.057790864, + 0.08757821, + -0.06914482, + -0.14491978, + 0.09239386, + 0.020376774, + 0.1097607, + -0.088697076, + 0.16945688, + 0.01119273, + -0.1528324, + -0.173479, + -0.026488239, + -0.029663468, + -0.037529018, + 0.08184811, + -0.116736144, + 0.020757439, + 0.037805155, + 0.081116766, + -0.08899582, + -0.23994243, + 0.00085073564, + 0.015741583, + 0.14916396, + -0.049735896, + 0.06661528, + -0.05483578, + 0.05611641, + 0.07030592, + -0.14783034, + 0.07137445, + -0.106677264, + 0.060104955, + -0.10194723, + 0.14962104, + 0.028547723, + 0.078555, + -0.09768789, + 0.11221817, + 0.0854506, + -0.15803969, + 0.009801034, + 0.078489214, + -0.0693264, + -0.1491822, + -0.09020748, + -0.03234147, + 0.044914614, + 0.02052608, + -0.1493524, + 0.17103215, + -0.063729316, + -0.10966809, + -0.029281124, + 0.09548159, + -0.048648804, + 0.23593551, + -0.012712764, + 0.07028848, + -0.061728034, + 0.05049216, + -0.021263314, + -0.06874739, + 0.07710166, + 0.08121541, + 0.0734596, + -0.023434158, + 0.16007483, + -0.10867535, + -0.11891655, + 0.22188166, + 0.06489395, + 0.010524853, + 0.19451497, + -0.053407338, + -0.08928299, + -0.14070098, + 0.20396641, + -0.062143497, + 0.125486, + 0.12106229, + -0.1373563, + -0.12767154, + 0.05849606, + -0.1128916, + 0.040035207, + -0.036183015, + -0.13515732, + 0.056579936, + -0.15353481, + 0.17633565, + 0.17068711, + -0.031218339, + -0.06275934, + 0.045422144, + 0.020727132, + -0.015841577, + 0.028296215, + -0.1515143, + 0.05359048, + -0.13335627, + -0.102366865, + 0.09058614, + -0.16648483, + 0.021478737, + -0.02842722, + 0.01834001, + 0.029639145, + -0.023922393, + 0.026784806, + -0.1001365, + -0.042642847, + -0.03436985, + -0.097866334, + 0.019808806, + -0.042747162, + 0.14128022, + -0.033543553, + 0.08283731, + 0.07131058, + -0.15540025, + 0.28758714, + 0.0058351206, + 0.15492027, + 0.013075218, + 0.08945752, + 0.07358258, + 0.054398827, + -0.02437751, + -0.07341751, + 0.10022484, + -0.050757684, + 0.1440264, + 0.013197424, + -0.109852426, + 0.22719486, + 0.080075815, + 0.00024181453, + -0.00097611186, + -0.04847994, + -0.07244215, + 0.006288356, + 0.030367048, + 0.12748545, + 0.1944054, + -0.060363494, + -0.0036989984, + 0.06789716, + 0.03632896, + 0.026532803, + -0.06583803, + -0.13583247, + 0.035222605, + -0.08362427, + -0.18340993, + -0.19657335, + -0.0031432216, + 0.023255847, + -0.18112065, + 0.12979667, + 0.04521475, + -0.037557703, + 0.0030770968, + 0.2226345, + 0.14577474, + 0.0025226166, + -0.023376113, + -0.054195154, + -0.05671215, + -0.11162617, + -0.06985026, + -0.028117025, + 0.0053079883, + 0.061277986, + -0.14143746, + 0.23066823, + 0.19300221, + 0.041472983, + -0.18196483, + 0.05748335, + 0.073920846, + -0.11640508, + -0.11107834, + -0.18051128, + 0.009662717, + 0.13686214, + -0.022840913, + 0.08510747, + 0.012901924, + -0.10250301, + 0.0596237, + -0.18144847, + 0.105939075, + 0.07167126, + 0.14054418, + -0.18191113, + -0.1436565, + -0.0597559, + 0.030895803, + -0.030637655, + -0.13535596, + 0.08015554, + -0.07608762, + -0.040301584, + -0.19534457, + -0.2292024, + -0.04393903, + 0.038195375, + 0.099135764, + -0.021666853, + -0.029471708, + -0.009995175, + 0.25933316, + 0.032328885, + 0.09000577, + 0.03263837, + 0.079661906, + -0.0012039868, + 0.11770895, + -0.0007853804 + ], + [ + 0.067265354, + 0.14978805, + -0.02866475, + -0.017997744, + 0.0043778527, + -0.077955745, + -0.17966285, + -0.08962502, + 0.012652886, + 0.068688974, + 0.032703526, + 0.103429645, + -0.026255865, + -0.04594767, + -0.03591724, + 0.13831529, + -0.063064694, + 0.098223, + -0.120419286, + -0.19822003, + -0.087343685, + 0.05914871, + -0.004434789, + -0.1129431, + 0.17182596, + 0.055425245, + 0.029754065, + -0.03556171, + -0.0070821294, + -0.13465452, + 0.16700113, + -0.19219238, + -0.01675321, + -0.027130553, + -0.18479164, + 0.027703132, + -0.038662884, + -0.025050428, + 0.020932555, + 0.25118732, + 0.13719758, + -0.065483645, + 0.07571365, + -0.07540309, + -0.11814715, + -0.020154916, + -0.1212217, + 0.14467515, + 0.049435012, + -0.03782557, + 0.10122299, + 0.16256903, + 0.18433803, + 0.002223497, + -0.06347142, + -0.045097824, + 0.0076598977, + -0.0047144056, + -0.20919853, + -0.26025832, + -0.021529123, + 0.09192429, + 0.0030727165, + 0.022516953, + -0.02791795, + 0.20013073, + -0.060710344, + -0.014477534, + -0.022307828, + 0.04182581, + 0.07157571, + 0.11714966, + -0.034158412, + -0.07059728, + 0.03238398, + -0.077487394, + -0.19103165, + 0.026339812, + -0.10266619, + 0.17039405, + -0.10501824, + -0.047377028, + -0.086855955, + -0.020825548, + 0.0031076584, + -0.024767814, + 0.12598439, + 0.19383541, + -0.030236963, + 0.16819346, + -0.036930185, + -0.08973632, + 0.20534639, + -0.04046148, + 0.19653097, + 0.022572542, + -0.10891978, + -0.0021857636, + 0.043617047, + -0.03692846, + -0.02452463, + -0.1116284, + -0.120930575, + 0.11805574, + -0.062913835, + -0.04299464, + 0.11609556, + 0.07652675, + 0.15170819, + -0.0143310465, + 0.13215616, + 0.038192805, + -0.09449705, + -0.04615726, + 0.12743029, + 0.053697243, + 0.028676936, + -0.011620548, + -0.115987964, + -0.15858902, + 0.025458544, + 0.16376868, + -0.21461114, + -0.12859254, + 0.046384696, + -0.060146254, + -0.10704182, + 0.1203173, + 0.04870109, + 0.055586778, + -0.036846314, + -0.033308074, + -0.04773714, + -0.1418176, + 0.14744744, + 0.058815204, + -0.077088796, + -0.052262966, + 0.19709195, + 0.026317546, + 0.036419798, + 0.05554856, + -0.0029399341, + -0.062320363, + -0.01966279, + -0.17727584, + 0.04959125, + -0.029366525, + 0.004609626, + -0.053935826, + 0.07947955, + 0.08481755, + -0.10725464, + 0.07710253, + -0.2177989, + 0.018307095, + -0.058796752, + -0.17742877, + -0.103017226, + -0.16547178, + 0.08727635, + -0.18131922, + -0.15467994, + 0.094013415, + -0.0007966098, + -0.11299204, + 0.049469113, + -0.2551994, + 0.10368437, + 0.12887946, + 0.052979, + -0.036250193, + 0.026475381, + 0.068989575, + 0.026221694, + 0.13373744, + -0.16292971, + 0.1264861, + -0.015351114, + -0.046966083, + -0.026861196, + 0.08184943, + 0.03712424, + -0.0893275, + -0.04624934, + 0.17902076, + -0.09396041, + 0.0012100477, + -0.0004969027, + 0.061765272, + -0.13257642, + -0.06479358, + -0.08090168, + -0.045115944, + -0.033314876, + -0.10521074, + 0.038595706, + 0.05002954, + 0.05275309, + -0.022413827, + -0.228349, + -0.05911533, + 0.13100033, + 0.08290936, + 0.21059032, + -0.053871688, + 0.2612328, + -0.03817294, + 0.12055628, + 0.19849202, + 0.1064979, + 0.06644159, + -0.010666335, + 0.037192244, + -0.03825279, + 0.22590448, + 0.06540578, + 0.0635492, + 0.009995832, + 0.053388823, + 0.1819073, + 0.017358864, + 0.0940223, + 0.19547985, + 0.16051306, + -0.08009981, + -0.01766264, + 0.31268245, + 0.03358963, + -0.006802613, + 0.060060095, + -0.02999579, + -0.02441298, + 0.041289836, + -0.018898971, + 0.016558355, + -0.2887073, + 0.05011042, + -0.020560574, + 0.10191658, + 0.07186017, + -0.07590083, + 0.043192476, + 0.04002405, + -0.08334605, + -0.0033909143, + 0.013439505, + 0.1452686, + -0.05733201, + -0.088819325, + -0.022471705, + -0.095721394, + 0.15808508, + 0.120784655, + -0.14439268, + -0.097761706, + -0.15823784, + 0.02247433, + -0.08741217, + -0.10422804, + 0.095085815, + -0.12007163, + -0.090681195, + 0.015306839, + -0.042384245, + 0.059710126, + 0.16539843, + 0.022866111, + 0.061641064, + -0.14381783, + 0.08884166, + 0.07466225, + -0.15382792, + -0.185641, + -0.1813861, + 0.02484775, + -0.0073386994, + -0.0801197, + -0.06278462, + -0.0073598013, + -0.09684346, + 0.06798241, + 0.0029492439, + -0.016103258, + 0.06638396, + 0.01753141, + 0.23454934, + 0.08132862, + 0.10536375, + -0.2229483, + 0.022713436, + -0.14560527, + -0.064387955, + -0.095680125, + 0.22664733, + 0.036997385, + 0.15401214, + -0.0652297, + 0.09910494, + -0.07604658, + 0.019308623, + -0.0017304748, + -0.039347965, + -0.12615763, + 0.038332216, + -0.1498985, + -0.18378036, + 0.22454844, + 0.076195106, + 0.006609157, + -0.051469374, + 0.04080529, + 0.014309058, + -0.096273944, + -0.207181, + -0.15468046, + -0.13909651, + -0.015941054, + -0.050097745, + 0.22421612, + 0.041737087, + 0.01954481, + 0.05503906, + -0.030042086, + -0.060866546, + -0.060889628, + 0.050270688, + -0.032480314, + 0.05457713, + -0.018058803, + -0.17685808, + 0.027825413, + 0.2854748, + -0.1990501, + -0.05410666, + -0.016514089, + 0.021260383, + 0.13303187, + -0.04133936, + -0.102796614, + -0.023230858, + -0.021802323, + -0.009447499, + 0.0056849695, + 0.077226385, + -0.001233707, + -0.056483172, + -0.0148004005, + -0.15466832, + -0.058501728, + -0.07044992, + -0.034258507, + 0.0032710754, + -0.043261908, + 0.042553976, + 0.11541086, + 0.22303134, + -0.042487398, + -0.2697398, + 0.029412314, + -0.093711354, + 0.013379649, + -0.02745153, + -0.09139424, + -0.07518103, + 0.081906185, + 0.01151903, + 0.07499832, + -0.08512211, + -0.017350763, + -0.17634419, + 0.25530154, + -0.05747039, + -0.06407152, + -0.026143344, + 0.031473134, + 0.040568653, + 0.02365954, + -0.03598118, + -0.004738165, + 0.26938042, + 0.107892066, + 0.12235724, + -0.0670435, + 0.103311166, + 0.027247678, + -0.060414925, + 0.10446229, + 0.021563247, + -0.17967018, + 0.021033838, + -0.032761253, + -0.002232972, + 0.20094131, + -0.06271966, + -0.06088999, + 0.114715405, + -0.09043788, + -0.065625295, + -0.034772128, + -0.03247024, + 0.13547981, + -0.05234461, + 0.0008887822, + -0.04399269, + 0.1076599, + 0.19047959, + 0.06816161, + 0.017209189, + -0.02949286, + -0.17175756, + 0.008101737, + -0.11393619, + 0.14412154, + -0.14545026, + 0.10393415, + 0.09651161, + 0.17374185, + -0.15528761, + 0.30169812, + 0.21803834, + 0.22342007, + -0.0023207148, + -0.32583496, + -0.1602966, + 0.08804405, + -0.09492502, + 0.013310529, + 0.08473863, + -0.010918157, + 0.09779154, + 0.1687677, + -0.0017114308, + 0.14069672, + -0.14218633, + -0.119746976, + -0.0016709599, + -0.116650805, + -0.0087289335, + -0.0020710814, + 0.14164573, + -0.03426684, + 0.13765714, + -0.23489869, + 0.17190114, + 0.11823379, + -0.1275406, + 0.016226083, + -0.015224294, + -0.040592875, + 0.088462226, + 0.0100339865, + 0.29286325, + -0.0012119255, + -0.20170744, + 0.043817673, + -0.048842527, + -0.06267558, + -0.20045178, + -0.07331201, + -0.0665427, + 0.0008836231, + 0.047697656, + -0.114039324, + 0.16505136, + 0.055174492, + 0.1341762, + 0.07885245, + -0.12267522, + -0.118794784, + 0.010460504, + 0.036683165, + 0.016049402, + 0.0030134192, + -0.07325558, + -0.059531037, + 0.22491747, + 0.021559626, + -0.2682984, + -0.1513486, + -0.007747446, + -0.21185677, + -0.085555956, + -0.07768858, + -0.0889585, + 0.18072057, + 0.17185259, + 0.071126275, + -0.0070228805, + -0.09613079, + 0.19773807, + -0.027994337, + -0.022889512, + 0.017196476, + 0.16872603, + 0.010325995, + 0.10429709, + -0.050494857, + 0.009961071, + -0.08508044, + -0.03851841, + 0.22150967, + 0.26763844, + 0.013120958, + 0.06905126, + -0.1074305, + 0.021637388, + 0.16322272, + -0.13542607, + 0.05240015, + 0.03747799, + -0.02136567, + -0.19518022, + 0.33402026, + -0.07306034, + 0.24572115, + -0.13900623, + -0.038396675, + 0.06149101, + 0.022670144, + 0.11762282, + 0.02650935, + 0.24784814, + -0.012557801, + -0.049926393, + 0.17277104, + 0.019994384, + 0.17777458, + -0.015156963, + -0.36002278, + 0.08576319, + 0.12579906, + -0.070583634, + 0.046381455, + -0.032518055, + -0.013673033, + 0.08388661, + 0.02805109, + -0.0040858416, + 0.08121704, + -0.024603426, + 0.038663764, + -0.052432787, + 0.09663728, + -0.05794367, + 0.03322958, + -0.05295467, + 0.012488321, + 0.06900259, + 0.32041568, + 0.050188363, + -0.06842132, + 0.17127438, + -0.08995696, + 0.052553497, + -0.15109767, + 0.0032275803, + -0.124561824, + -0.0349717, + -0.15243848, + 0.06931027, + -0.08733464, + 0.08632587, + -0.000009701724, + -0.061496064, + 0.016888466, + 0.14733578, + -0.20635521, + -0.086442135, + -0.010243254, + 0.08384721, + -0.21813874, + 0.053306054, + -0.021313356, + -0.22646224, + -0.111123234, + -0.17016025, + -0.053942624, + 0.10293637, + -0.067348056, + 0.016638568, + -0.055585764, + 0.1321758, + 0.09786219, + -0.0846062, + -0.20103993, + -0.20276587, + -0.14107303, + 0.0572056, + -0.05998819, + -0.079461865, + -0.18345493, + 0.07808181, + 0.025852023, + 0.16117097, + 0.09278437, + -0.0070364075, + -0.09634602, + 0.15276241, + -0.025094103, + -0.25218967, + -0.047830544, + -0.03265829, + -0.14163731, + -0.2689699, + -0.03686031, + 0.012960216, + -0.02138974, + 0.13290478, + 0.071284525, + 0.059489492, + 0.1677266, + 0.04434716, + 0.05381613, + -0.07807097, + -0.007200855, + -0.05029709, + -0.19462912, + -0.06630631, + 0.077244155, + -0.051679987, + 0.16820908, + 0.006629443, + 0.06144869, + 0.044119336, + 0.08528723, + 0.09778228, + -0.05264809, + 0.17047869, + -0.033962384, + 0.09303276, + 0.14469127, + 0.32394484, + -0.05757008, + 0.14690277, + -0.113172874, + -0.073722996, + -0.1543499, + 0.13370518, + 0.022599915, + -0.016452743, + -0.1111051, + -0.123819724, + -0.0358012, + 0.095570944, + -0.026632585, + -0.12938291, + 0.054892946, + 0.0053247954, + 0.0051558986, + -0.08200846, + -0.024864692, + 0.0022463088, + 0.033632293, + 0.022776838, + 0.046233963, + 0.07099262, + 0.03803995, + 0.11918089, + 0.073886305, + -0.02947079, + -0.032019015, + -0.028573902, + 0.07655141, + -0.13654494, + 0.10705977, + -0.054300036, + 0.1552018, + -0.000661521, + 0.22377999, + -0.08899608, + 0.12889342, + -0.20074363, + 0.13927813, + -0.0026708662, + -0.04300883, + -0.16178162, + -0.0923155, + -0.16207108, + 0.20354961, + 0.009087291, + -0.15107653, + -0.06892152, + -0.033151213, + 0.043834474, + -0.115955636, + -0.042398877, + -0.13583317, + 0.0435221, + 0.046010397, + -0.007331226, + -0.03840562, + 0.08738765, + -0.10060804, + 0.06926944, + -0.21893275, + -0.04178857, + -0.19848873, + 0.043663215, + 0.11381242, + -0.09335524, + -0.076091655, + 0.13046667, + -0.0030471336, + 0.04753741, + -0.04804935, + -0.032354355, + 0.08939008, + -0.010100283, + 0.10576193, + 0.01991799, + 0.030111695, + -0.04467943, + -0.015056431, + -0.024010837, + 0.08842912, + -0.005612796, + -0.13563918, + 0.18028839, + 0.013166858, + -0.2667886, + -0.085470125, + 0.3685931, + -0.1946491, + 0.049553536, + -0.37291518, + -0.22655754, + 0.34400705, + 0.032976877, + 0.13435382, + -0.11814129, + 0.047868554, + 0.07159131, + 0.036158036, + 0.024520447, + -0.20078675, + -0.05586612, + 0.08214067, + -0.08605121, + 0.032829337, + 0.058903836, + 0.24436851, + 0.09116331, + -0.13379814, + 0.1828366, + 0.1072864, + -0.021227801, + 0.12980577, + -0.11652907, + -0.03775177, + 0.083177045, + 0.057081014, + -0.06332418, + 0.056298833, + -0.3391434, + 0.04645195, + -0.06128842, + 0.18199284, + -0.002383945, + -0.14004768, + -0.16993281, + 0.010031427, + 0.16041878, + -0.16184394, + 0.018993879, + -0.025938176, + 0.002514922, + 0.020949349, + -0.1269015, + -0.054172676, + 0.094788544, + 0.07986568, + 0.088186376, + -0.028140098, + 0.011289786, + -0.17378683, + -0.12375356, + 0.04456818, + 0.08479448, + -0.09294157, + -0.16615918, + 0.028024394, + -0.093367904, + -0.013316832, + 0.04018244, + 0.08644012, + -0.12008931, + 0.018793972, + 0.061523847, + -0.0694417, + -0.024518935, + -0.11057199, + 0.061863527, + -0.05944173, + 0.11886046, + -0.030533476, + -0.017306136, + 0.0665828, + -0.08461085, + 0.13586487, + 0.12317356, + 0.118130654, + -0.084764935, + 0.009605317, + -0.0064539798, + 0.15894376, + 0.12810811, + 0.029883908, + -0.027100915, + -0.18702109, + 0.016678592, + -0.21666358, + 0.009283024, + -0.06385588, + 0.11203676, + 0.036862098, + -0.051290695, + 0.04737701, + 0.100528136, + 0.033112366, + 0.048415672, + -0.04174812, + -0.16214389, + 0.17123649, + 0.21322905, + 0.021103352, + 0.19598372, + -0.21077679, + -0.06637894, + -0.22511913, + 0.13010465, + -0.044941496, + -0.19983673, + 0.1963295, + -0.15502413, + 0.0615183, + -0.16637003, + -0.01743503, + 0.054865364, + 0.004895811, + -0.034269266, + 0.1766163, + -0.084879376, + 0.04049494, + 0.11087186, + 0.02269163, + -0.027787592, + -0.0145252375, + 0.08132456, + 0.12849073, + 0.06231228, + 0.058701742, + -0.06058468, + 0.13572963, + -0.14186189, + 0.1027375, + 0.028402401, + 0.0053135604, + -0.054753385, + 0.027727582, + -0.034705237, + -0.016670734, + -0.10962946, + -0.011801861, + -0.20003115, + 0.2020829, + 0.026161859, + -0.0031623817, + -0.029170213, + -0.018234199, + 0.04875749, + -0.23150893, + -0.036654674, + 0.09868626, + -0.28474876, + -0.048850782, + -0.0046823416, + -0.0050867144, + -0.028108975, + 0.078664176, + 0.05139621, + -0.13367955, + -0.06766825, + -0.13896853, + -0.16415608, + 0.017776988, + 0.10541162, + -0.0264685, + 0.08690293, + 0.12137225, + -0.034988154, + 0.16255242, + 0.002840024, + 0.16030852, + -0.076692425, + -0.22003824, + -0.18630469, + 0.016727459, + -0.020857252, + -0.008802409, + 0.07564682, + -0.05444138, + 0.1668737, + -0.025629545, + 0.14557064, + 0.09674229, + -0.058274068, + 0.05744842, + -0.0928503, + -0.0799128, + -0.07071504, + -0.0121874735, + 0.1692294, + 0.093030006, + 0.020831311, + 0.04520271, + 0.087474234, + -0.016130574, + -0.05108359, + 0.036344565, + -0.108744465, + -0.055855155, + 0.1017321, + -0.069691285, + -0.14468838, + -0.10310156, + 0.11176399, + -0.00078769, + -0.03110051, + -0.1145704, + -0.07260943, + 0.23371693, + -0.104421645, + -0.06583719, + -0.006062506, + 0.05011631, + -0.0054390067, + -0.078630604, + 0.19628902, + -0.1550763, + -0.13275377, + -0.12952268, + 0.042036884, + 0.05858515, + 0.055275004, + -0.10907125, + -0.00476199, + -0.14193, + 0.04885924, + -0.02711134, + 0.03325598, + -0.04900974, + -0.08830747, + -0.099112585, + 0.13302748, + 0.19985054, + -0.045091256, + -0.20651561, + 0.16883984, + 0.3288204, + 0.024190074, + -0.031287767, + 0.15144359, + -0.008892988, + -0.01452989, + 0.15788583, + -0.151726, + -0.2957167, + -0.1945324, + 0.05123937, + 0.037350606, + 0.09008041, + -0.08441538, + -0.09457509, + 0.067238115, + -0.02894548, + -0.011611294, + 0.03891026, + 0.111131825, + -0.26067775, + -0.28298238, + 0.056074947, + 0.008107262, + 0.0828736, + 0.052385423, + -0.08731629, + 0.14230244, + 0.19611123, + -0.0090899905, + -0.012221198, + -0.00518729, + 0.0022617683, + 0.051705398, + 0.095017456, + -0.056464072, + 0.11342066, + 0.0254407, + -0.13259909, + 0.038187362, + 0.08667088, + -0.07060096, + -0.011785999, + -0.023018276, + 0.11617855, + -0.036035065, + -0.07118992, + -0.12512614, + 0.12636593, + 0.09379038, + 0.10848534, + 0.099208534, + 0.21280734, + 0.011662269, + 0.08994858, + 0.101053335, + -0.11046256, + -0.066710554, + -0.001033645, + 0.08943645, + -0.057964046, + -0.17622581, + 0.09226646, + -0.11705704, + -0.11167349 + ] + ], + "bias": [ + 0.23896483, + -0.3402354, + 0.101270564 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.json new file mode 100644 index 0000000..5c115ef --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.json @@ -0,0 +1,1717 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.71159875, + "confusion": [ + [ + 69, + 75, + 5 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 11, + 58 + ] + ], + "tool_precision": 0.98571426, + "tool_recall": 0.46308726, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5340551, + "p_memory": 0.21771747, + "p_abstain": 0.24822739 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5626822, + "p_memory": 0.1836759, + "p_abstain": 0.2536419 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6036589, + "p_memory": 0.11206787, + "p_abstain": 0.28427315 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40643284, + "p_memory": 0.19013943, + "p_abstain": 0.40342778 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51935685, + "p_memory": 0.09203593, + "p_abstain": 0.3886073 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3248475, + "p_memory": 0.08496061, + "p_abstain": 0.5901919 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5663625, + "p_memory": 0.18072063, + "p_abstain": 0.2529169 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53524184, + "p_memory": 0.17327352, + "p_abstain": 0.2914846 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55211174, + "p_memory": 0.14331223, + "p_abstain": 0.30457598 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48472118, + "p_memory": 0.20033272, + "p_abstain": 0.31494612 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48654568, + "p_memory": 0.20197286, + "p_abstain": 0.31148145 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3951337, + "p_memory": 0.11869106, + "p_abstain": 0.48617515 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5669123, + "p_memory": 0.11284301, + "p_abstain": 0.32024476 + }, + { + "query_id": "shell_test_023", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6097893, + "p_memory": 0.08166945, + "p_abstain": 0.30854127 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "memory", + "p_tools": 0.532595, + "p_memory": 0.09771287, + "p_abstain": 0.36969218 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5313089, + "p_memory": 0.07768788, + "p_abstain": 0.39100322 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5733041, + "p_memory": 0.16126691, + "p_abstain": 0.26542896 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5433677, + "p_memory": 0.18911023, + "p_abstain": 0.2675221 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47030315, + "p_memory": 0.17896877, + "p_abstain": 0.3507281 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5621234, + "p_memory": 0.14502367, + "p_abstain": 0.29285294 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57606184, + "p_memory": 0.1570898, + "p_abstain": 0.2668484 + }, + { + "query_id": "rf_test_020", + "text": "read the existing router_report.md from the last benchmark run", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5553279, + "p_memory": 0.09895797, + "p_abstain": 0.3457141 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5518937, + "p_memory": 0.17088455, + "p_abstain": 0.2772218 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53405726, + "p_memory": 0.19761977, + "p_abstain": 0.268323 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4904085, + "p_memory": 0.08687752, + "p_abstain": 0.42271397 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5497213, + "p_memory": 0.09366132, + "p_abstain": 0.3566174 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41655815, + "p_memory": 0.17062807, + "p_abstain": 0.41281372 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5488636, + "p_memory": 0.2709307, + "p_abstain": 0.1802057 + }, + { + "query_id": "rf_test_028", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54564625, + "p_memory": 0.10946755, + "p_abstain": 0.3448862 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5033046, + "p_memory": 0.11161256, + "p_abstain": 0.3850828 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46809536, + "p_memory": 0.20201841, + "p_abstain": 0.32988623 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41650668, + "p_memory": 0.15908073, + "p_abstain": 0.4244126 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55804354, + "p_memory": 0.1696324, + "p_abstain": 0.27232406 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4662468, + "p_memory": 0.18212193, + "p_abstain": 0.35163125 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56596255, + "p_memory": 0.1841741, + "p_abstain": 0.24986333 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3543384, + "p_memory": 0.07887757, + "p_abstain": 0.566784 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50478405, + "p_memory": 0.16049631, + "p_abstain": 0.33471963 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42139858, + "p_memory": 0.14304228, + "p_abstain": 0.43555915 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49103788, + "p_memory": 0.14342807, + "p_abstain": 0.36553407 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42337945, + "p_memory": 0.1434221, + "p_abstain": 0.4331984 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5013689, + "p_memory": 0.1826258, + "p_abstain": 0.31600538 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.456451, + "p_memory": 0.15955734, + "p_abstain": 0.3839917 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52091247, + "p_memory": 0.18335688, + "p_abstain": 0.29573065 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46343142, + "p_memory": 0.2125959, + "p_abstain": 0.32397267 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5892539, + "p_memory": 0.1738571, + "p_abstain": 0.23688899 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44753295, + "p_memory": 0.27299076, + "p_abstain": 0.27947626 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5022449, + "p_memory": 0.23418446, + "p_abstain": 0.2635706 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.7777778, + "confusion": [ + [ + 38, + 31, + 1 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 8, + 47 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54285717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 6, + 4, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 7, + 33 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.6, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 5, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.6166667, + "confusion": [ + [ + 27, + 22, + 1 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.md new file mode 100644 index 0000000..37de7de --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-12/router_report.md @@ -0,0 +1,146 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.340` +- abstain_prob_threshold: `0.450` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.7116` +- tools precision: `0.9857` +- tools recall: `0.4631` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 69 | 75 | 5 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 11 | 58 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.53 | 0.22 | 0.25 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.56 | 0.18 | 0.25 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Memory` | 0.58 | 0.09 | 0.33 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.57 | 0.17 | 0.26 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Memory` | 0.58 | 0.10 | 0.32 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.58 | 0.18 | 0.24 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_017` | append a line "router ok" to docs/router_holdout_log.txt | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_tools_recall_tools_018` | create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it | `Tools` | `Memory` | 0.58 | 0.16 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Memory` | 0.56 | 0.12 | 0.31 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.56 | 0.11 | 0.33 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.56 | 0.16 | 0.28 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Memory` | 0.51 | 0.08 | 0.41 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Memory` | 0.56 | 0.09 | 0.34 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Memory` | 0.48 | 0.09 | 0.43 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Memory` | 0.50 | 0.13 | 0.36 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.45 | 0.17 | 0.39 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.52 | 0.20 | 0.28 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Memory` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.34 | 0.25 | 0.40 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.35 | 0.32 | 0.33 | +| `holdout_abstain_toolish_abstain_024` | fix this pls | `Abstain` | `Memory` | 0.46 | 0.11 | 0.43 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Memory` | 0.54 | 0.12 | 0.34 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.42 | 0.21 | 0.37 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.40 | 0.15 | 0.45 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Memory` | 0.56 | 0.08 | 0.35 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.47 | 0.21 | 0.32 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Memory` | 0.56 | 0.09 | 0.35 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Memory` | 0.62 | 0.08 | 0.30 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.53 | 0.18 | 0.29 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.60 | 0.11 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.41 | 0.19 | 0.40 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Memory` | 0.52 | 0.09 | 0.39 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.32 | 0.08 | 0.59 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.54 | 0.17 | 0.29 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.55 | 0.14 | 0.30 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.20 | 0.31 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.49 | 0.20 | 0.31 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.40 | 0.12 | 0.49 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Memory` | 0.57 | 0.11 | 0.32 | +| `shell_test_023` | 現在のプロセス一覧でklbrが含まれるものを見せて | `Tools` | `Memory` | 0.61 | 0.08 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Memory` | 0.53 | 0.10 | 0.37 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Memory` | 0.53 | 0.08 | 0.39 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.57 | 0.16 | 0.27 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.54 | 0.19 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.56 | 0.15 | 0.29 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.58 | 0.16 | 0.27 | +| `rf_test_020` | read the existing router_report.md from the last benchmark run | `Tools` | `Memory` | 0.56 | 0.10 | 0.35 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.55 | 0.17 | 0.28 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.53 | 0.20 | 0.27 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Memory` | 0.49 | 0.09 | 0.42 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Memory` | 0.55 | 0.09 | 0.36 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.42 | 0.17 | 0.41 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.55 | 0.27 | 0.18 | +| `rf_test_028` | benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて | `Tools` | `Memory` | 0.55 | 0.11 | 0.34 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.50 | 0.11 | 0.39 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.47 | 0.20 | 0.33 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.42 | 0.16 | 0.42 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.56 | 0.17 | 0.27 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.35 | 0.08 | 0.57 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.50 | 0.16 | 0.33 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.42 | 0.14 | 0.44 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.49 | 0.14 | 0.37 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.42 | 0.14 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.50 | 0.18 | 0.32 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.46 | 0.16 | 0.38 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.52 | 0.18 | 0.30 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.46 | 0.21 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.59 | 0.17 | 0.24 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.45 | 0.27 | 0.28 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.50 | 0.23 | 0.26 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.5429` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7778` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.6000 | 0.0000 | 0.0000 | 0.8167 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.5400 | 0.0000 | 0.0000 | 0.6167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-13/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-13/router_model.json new file mode 100644 index 0000000..c7f1171 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-13/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.1537661, + -0.035649825, + -0.08693668, + -0.039169595, + -0.07801271, + 0.022404537, + 0.13668533, + -0.04904432, + -0.084624946, + -0.12592527, + 0.015163703, + -0.18208449, + 0.0055660303, + 0.07221892, + 0.06448449, + -0.1870893, + -0.075359516, + -0.03543284, + -0.008072806, + 0.035137396, + 0.1856608, + -0.047060464, + -0.026945204, + 0.038430814, + -0.027119577, + -0.07783137, + -0.028168011, + -0.04998577, + 0.13851744, + 0.22334306, + -0.11444658, + -0.10880823, + -0.013960583, + -0.002960124, + 0.1886642, + 0.008615434, + 0.06759129, + 0.025011372, + -0.119343616, + 0.033081945, + -0.08350601, + -0.0484712, + -0.074073195, + 0.045808908, + -0.01146472, + 0.10831494, + -0.0040111416, + -0.09824766, + -0.02530775, + -0.06901607, + -0.050421126, + -0.126361, + -0.088383175, + -0.08048365, + -0.001986429, + 0.18562548, + -0.10405873, + 0.02529612, + 0.108271234, + 0.24374901, + -0.011125183, + 0.0132962605, + -0.07309199, + 0.029683625, + -0.08636377, + -0.03715244, + 0.083628885, + 0.005959518, + 0.07466751, + 0.11360453, + -0.041455425, + 0.09902782, + -0.02341183, + 0.05831913, + -0.042435482, + 0.08605521, + 0.15117645, + -0.019223474, + -0.038496863, + -0.12737247, + 0.058691185, + -0.018187318, + -0.10621924, + -0.027555455, + 0.05125111, + 0.16719356, + -0.17868775, + -0.17773148, + -0.08739252, + 0.10273777, + -0.034631487, + -0.0539006, + -0.096600875, + 0.08291881, + -0.15143853, + -0.09585049, + 0.027222762, + 0.007994819, + 0.060303994, + -0.018851208, + -0.016228046, + 0.08862469, + 0.19132261, + -0.019297566, + -0.053104565, + 0.14122458, + -0.10131717, + -0.09569557, + -0.068272986, + 0.000808413, + -0.15115495, + 0.047998995, + 0.039926194, + 0.037234508, + 0.032307126, + -0.075684965, + -0.073772445, + -0.10967773, + -0.09304202, + 0.040335774, + -0.016109457, + -0.17040172, + 0.13639012, + 0.034102615, + -0.09065539, + 0.1110392, + 0.17052025, + 0.16978532, + 0.1330525, + 0.0534311, + 0.10709471, + 0.12975323, + 0.0031307633, + 0.11619781, + -0.20192865, + -0.04178235, + 0.07954132, + 0.047113087, + -0.03501471, + -0.1652226, + -0.09810865, + 0.086311735, + 0.020574726, + 0.026401358, + 0.15123715, + -0.11407882, + -0.0010843852, + -0.032467104, + 0.04454825, + 0.08361806, + -0.06101439, + 0.060647525, + 0.18783487, + -0.12666312, + 0.084901564, + 0.10627836, + -0.047494754, + -0.0030847418, + 0.13515899, + 0.16071515, + 0.009656678, + -0.032863, + 0.27564478, + -0.07924913, + -0.046650242, + 0.075491354, + -0.06973335, + 0.06965276, + -0.001548173, + -0.142573, + 0.00032827802, + 0.014488301, + -0.1260245, + -0.08115438, + -0.044923816, + -0.03615714, + 0.0039115055, + -0.06106897, + -0.041726876, + 0.046486966, + 0.068862185, + -0.0059390427, + -0.20666197, + 0.11289427, + -0.011771635, + -0.06389011, + 0.028167449, + 0.2907871, + 0.01976517, + -0.08579038, + 0.08004743, + 0.16933607, + 0.05288272, + 0.038429778, + 0.03427874, + -0.044819575, + 0.04110185, + -0.13571876, + -0.19524126, + 0.06988574, + 0.073779844, + -0.046239115, + -0.040948626, + -0.084544, + -0.020007487, + -0.092407554, + -0.17292632, + 0.23797427, + -0.1945358, + -0.49743015, + -0.07454593, + -0.1673993, + -0.10651807, + -0.04177226, + -0.05472722, + -0.15188706, + -0.022152808, + -0.13939627, + -0.019187424, + -0.15158167, + -0.19280887, + 0.10705503, + 0.062492274, + 0.16923375, + -0.26784095, + -0.15662816, + 0.025555544, + -0.17363223, + 0.08134375, + 0.06117359, + -0.09012711, + -0.06951844, + 0.061694615, + -0.003797633, + 0.21708345, + 0.12185522, + 0.19993393, + 0.08121296, + 0.012878898, + -0.054898884, + -0.1080671, + 0.17580487, + -0.06257184, + -0.13819437, + 0.07154935, + 0.09852028, + 0.16923477, + -0.005906965, + -0.0075131687, + 0.119097345, + -0.0682252, + -0.030815927, + -0.19546556, + 0.037838846, + 0.00044135028, + 0.11877078, + 0.10327466, + -0.06929159, + 0.020809863, + 0.13948017, + -0.18564449, + 0.21409702, + 0.14669497, + 0.045995284, + 0.010650101, + -0.07551283, + -0.0006766264, + 0.07865089, + -0.11002661, + -0.024456186, + -0.08400181, + -0.13899222, + 0.09306608, + 0.10094795, + 0.07228482, + 0.019025564, + 0.09661008, + 0.13605566, + 0.028701328, + 0.020753454, + 0.0604083, + 0.05287096, + -0.07340115, + 0.009560703, + -0.025192428, + 0.07975566, + -0.28597078, + 0.021488205, + -0.0889583, + 0.12257723, + -0.13694072, + 0.12471303, + 0.061077848, + 0.004631609, + -0.10417203, + 0.22062522, + -0.013706877, + -0.049198896, + 0.00029548293, + 0.075458646, + 0.1765529, + 0.11541535, + 0.008701007, + 0.24391726, + 0.028608661, + 0.06389106, + 0.08642011, + -0.09138413, + -0.10007181, + -0.0695514, + -0.00900465, + -0.03647545, + 0.019589651, + 0.10178171, + 0.17542481, + 0.11783792, + 0.049075227, + -0.10664901, + 0.03252592, + 0.050435152, + -0.008125645, + -0.10447342, + 0.009139559, + 0.1275379, + -0.021951595, + -0.0136193605, + 0.031333257, + -0.15114506, + -0.1765369, + 0.06522931, + 0.06019238, + -0.035919093, + -0.15519899, + -0.01969548, + -0.03430187, + 0.10910889, + -0.0056543187, + -0.10220757, + 0.26145938, + 0.15574658, + 0.049058672, + -0.0064308834, + 0.042335905, + 0.005810607, + 0.038708996, + -0.0057784626, + 0.049044974, + 0.017823728, + 0.087167464, + 0.074433744, + 0.0077878316, + -0.05596981, + 0.0045128716, + -0.049364593, + 0.0015845216, + 0.14348508, + -0.13488854, + -0.1561442, + 0.054857284, + 0.018423049, + -0.034320723, + 0.0040649255, + -0.04828976, + 0.28164595, + 0.047446065, + -0.11510787, + 0.11150978, + -0.04681086, + 0.071485296, + -0.045360688, + -0.009777533, + 0.038308833, + -0.052017767, + 0.03695938, + 0.057656713, + -0.060405314, + -0.29814774, + -0.071221784, + 0.1714612, + -0.07547322, + 0.005367531, + -0.13997321, + -0.16026817, + 0.04131712, + -0.17275667, + -0.029355615, + 0.08490756, + -0.115701295, + 0.0016283661, + 0.20621264, + 0.16213463, + -0.07390191, + -0.19121905, + -0.13373351, + 0.054568663, + 0.11002628, + 0.025603758, + -0.06534632, + 0.057530094, + 0.018174551, + -0.14876643, + 0.035836782, + -0.01137345, + -0.075621784, + 0.07126445, + -0.04957483, + -0.09046384, + -0.067094974, + -0.042324323, + -0.057362486, + 0.1400903, + -0.03241678, + 0.10672305, + -0.19880442, + 0.09854313, + 0.0760169, + -0.091766715, + -0.12132613, + 0.027035655, + -0.3583752, + -0.13381411, + -0.30593142, + -0.0072146636, + 0.12702839, + -0.1360928, + -0.011511107, + -0.045212187, + -0.032329787, + 0.013346648, + -0.016038861, + 0.07367348, + -0.19516303, + 0.09781957, + -0.023986258, + 0.14528868, + 0.08888904, + -0.0009244085, + -0.05984577, + 0.018362086, + -0.0895728, + -0.023944547, + -0.0010550144, + 0.117598146, + -0.02050585, + -0.12902373, + 0.077714995, + -0.012136869, + -0.028286034, + 0.01971836, + 0.23491275, + 0.00014187285, + -0.025502866, + -0.160924, + -0.10722113, + -0.028722614, + -0.11093872, + 0.12761506, + 0.032540575, + -0.028944362, + 0.026266946, + 0.15018588, + 0.065873004, + -0.2041029, + -0.026748825, + -0.15338984, + 0.019006187, + -0.09869299, + -0.10463372, + 0.056366608, + 0.019882223, + 0.024008995, + -0.06550811, + 0.040581826, + -0.09272168, + -0.021980435, + 0.009398623, + -0.055842966, + -0.067572854, + 0.2544045, + 0.2720369, + 0.056282323, + 0.14016761, + 0.1229671, + 0.085792094, + 0.082071744, + 0.015617369, + 0.0025832395, + -0.11643445, + -0.08638553, + -0.052909072, + -0.16713358, + 0.003093768, + 0.07404296, + 0.031308185, + -0.117112026, + -0.08167997, + -0.1499687, + 0.043127168, + 0.14035194, + -0.07871639, + -0.037212748, + -0.0598638, + -0.14055473, + -0.036376033, + 0.11357092, + -0.050674643, + 0.022856299, + 0.057805985, + 0.089862354, + -0.044336755, + -0.08881311, + -0.10180266, + 0.09055733, + -0.04494839, + 0.09060898, + -0.13318132, + 0.032166876, + 0.00562406, + -0.072895855, + 0.008453028, + -0.10890039, + -0.14855209, + -0.009387831, + 0.07252449, + 0.1000593, + -0.1428223, + -0.08014342, + -0.040895056, + -0.039497916, + 0.36770174, + -0.0024050162, + -0.007834549, + 0.1724918, + 0.0037634445, + -0.18319333, + 0.07710567, + -0.19161667, + -0.014408101, + 0.4058824, + -0.06781774, + -0.004210171, + -0.020524722, + 0.03903467, + -0.065584004, + 0.12053713, + 0.07669467, + 0.019268565, + -0.1424542, + 0.07729574, + -0.1408175, + -0.027871037, + -0.07438413, + -0.24066758, + 0.0007071577, + -0.11960982, + 0.079575315, + 0.026175413, + 0.06513677, + -0.07926052, + 0.053233802, + -0.23602884, + 0.009922975, + -0.024497086, + 0.07963983, + -0.03895233, + 0.012983701, + 0.02903441, + 0.09618848, + -0.086873636, + 0.15702735, + 0.051104963, + 0.23331824, + 0.05189599, + -0.012449083, + 0.18062961, + 0.06019235, + 0.056962494, + 0.03108089, + -0.021042427, + 0.0064071524, + -0.088212445, + 0.08572126, + 0.0037502868, + 0.07312031, + 0.12206003, + 0.15648857, + 0.19765435, + -0.08641273, + -0.02523061, + -0.11257553, + -0.07819527, + -0.028061705, + -0.06571461, + -0.015628025, + 0.062462434, + 0.11131009, + -0.0013901697, + 0.058562018, + 0.06027097, + 0.023649702, + 0.15201351, + 0.08222728, + 0.06710522, + 0.06597227, + 0.15405323, + -0.106588826, + -0.048434135, + -0.08093238, + 0.13272074, + -0.104526795, + 0.066636786, + -0.14262883, + 0.11796827, + 0.01689741, + 0.10963646, + -0.043205373, + 0.030428115, + 0.06697563, + 0.18351023, + -0.12014605, + 0.007847034, + -0.21682698, + -0.053551763, + -0.030757116, + -0.058873527, + -0.039070714, + -0.00346026, + 0.09289441, + -0.14382905, + 0.1066636, + 0.06449059, + 0.21688591, + -0.36460355, + 0.15066512, + -0.064523116, + -0.018895637, + -0.14244261, + 0.018194063, + -0.08151443, + 0.008980479, + -0.08436932, + 0.123211086, + -0.16318616, + 0.11563229, + -0.06480912, + -0.005352673, + 0.068340674, + -0.07209151, + 0.0028943075, + 0.0013212605, + 0.11254483, + 0.098231606, + -0.082876496, + -0.018098809, + -0.019039044, + -0.10377387, + -0.055182043, + -0.045605227, + -0.06043637, + 0.19120796, + 0.10237104, + 0.07438044, + 0.13820165, + 0.17585173, + 0.27409434, + -0.07378421, + 0.025601309, + -0.12231862, + 0.12022717, + -0.101080395, + 0.032167993, + -0.08084088, + -0.17208248, + -0.1681819, + -0.03261707, + 0.02148205, + -0.075628266, + 0.024750838, + 0.03915467, + -0.13104486, + 0.034098193, + -0.034001082, + 0.08776243, + -0.049719255, + -0.06473143, + 0.10588895, + 0.08131149, + -0.052911863, + -0.054036725, + -0.035009447, + 0.08870277, + 0.07426314, + 0.040366404, + 0.077888794, + 0.021348773, + 0.226514, + 0.15032294, + 0.16204213, + 0.000056462246, + 0.09213504, + 0.19880146, + 0.08437888, + 0.021916471, + 0.08471564, + 0.03781062, + -0.066586, + -0.046965122, + -0.10133214, + 0.08557293, + -0.14368777, + -0.06882834, + -0.12171229, + 0.0055409498, + -0.09227392, + -0.10453962, + -0.07994333, + 0.097816125, + 0.0027960786, + -0.11214976, + 0.05112316, + -0.040688016, + 0.068019, + -0.25952643, + 0.16106968, + 0.17121106, + 0.25099227, + 0.076975055, + -0.2097073, + -0.023342844, + 0.015645755, + 0.12014807, + 0.07335009, + -0.05761358, + -0.08484111, + -0.000030331716, + 0.15092224, + 0.06777547, + -0.08367785, + 0.048469383, + -0.11079949, + 0.003651169, + -0.07147883, + -0.014249957, + -0.24292563, + -0.15647426, + -0.17459014, + 0.010819486, + -0.105703846, + -0.030978357, + 0.06532192, + 0.08745108, + -0.053935952, + 0.0688913, + 0.12390802, + 0.21340448, + -0.014140771, + 0.27248612, + -0.2464558, + 0.05955756, + 0.104456566, + 0.009404783, + -0.08957156, + -0.13419013, + -0.026675185, + -0.1552804, + -0.008024584, + 0.005712407, + -0.124870405, + -0.2592146, + 0.19014545, + 0.0327858, + -0.1229455, + 0.010203828, + 0.037715424, + -0.0053597577, + 0.05868922, + 0.010626316, + 0.019355819, + 0.08497353, + 0.119429246, + 0.004582282, + -0.06729474, + 0.07995998, + -0.04374404, + -0.015687706, + -0.053967472, + -0.08461543, + -0.0808064, + -0.07503357, + -0.03532297, + 0.0011162171, + -0.11756368, + 0.11231443, + 0.0066749663, + 0.087183446, + 0.033798646, + 0.17816873, + 0.16899641, + 0.032384265, + 0.037937198, + -0.1306509, + -0.08966298, + -0.11874975, + 0.04818552, + -0.08112427, + -0.08979903, + 0.016811714, + -0.12227783, + 0.006724136, + 0.07726029, + 0.072018504, + 0.047206745, + -0.020475753, + 0.21668829, + 0.06144242, + -0.010373841, + 0.08095411, + -0.00984799, + -0.18237619, + 0.08362384, + -0.06917313, + 0.003942984, + 0.08102712, + -0.08224079, + 0.026713282, + -0.021954091, + -0.21172519, + 0.061612993, + 0.11611484, + 0.15850388, + -0.07526881, + -0.011174921, + 0.12953082, + -0.04849906, + 0.08364964, + 0.045158945, + 0.10626501, + 0.119382195, + -0.20448637, + -0.033443533, + -0.044285737, + -0.078928396, + -0.027338818, + -0.12594555, + 0.047167778, + -0.03249266, + -0.050701614, + 0.083851635, + 0.06785768, + -0.038283225, + -0.02997082, + -0.10361641, + 0.040058605, + 0.013622663, + -0.029170217, + -0.039008163, + 0.08126564, + 0.023967572, + -0.040728178, + 0.020921214, + -0.2012302, + 0.029383505, + 0.039341018, + 0.07352988, + 0.149539, + -0.18081951, + 0.042585555, + -0.07393927, + -0.0520452, + -0.055225402, + -0.02532332, + 0.07143403, + 0.14533009, + 0.020230262, + 0.06286708, + -0.016043179, + -0.0058425125, + -0.1894284, + 0.081516296, + 0.01061884, + 0.08930471, + -0.070286885, + 0.12981166, + 0.01348248, + 0.043093804, + 0.11957937, + 0.022259962, + -0.032027576, + 0.025988672, + -0.16140743, + 0.07117117, + -0.027395064, + -0.059419967, + -0.0067737456, + -0.09964316, + 0.049351126, + 0.21752295, + 0.046031915, + -0.024564896, + -0.011924736, + -0.059805218, + 0.02614514, + -0.015359368, + -0.02796093, + -0.012214168, + 0.0056245904, + -0.032312065, + 0.109036505, + 0.07137153, + 0.108339995, + 0.052375022, + -0.017451666, + -0.14530696, + -0.11981482, + 0.079305254, + -0.0025598651, + -0.05310435, + 0.11399684, + 0.03127478, + 0.0064025996, + -0.03253582, + 0.089398734, + -0.18456937, + -0.0016192505, + 0.30008864, + -0.18448582, + -0.11759908, + -0.15413254, + 0.018025307, + 0.025112813, + -0.0009730928, + -0.2881156, + 0.12879921, + 0.13925475, + -0.09416232, + 0.0006413473, + -0.13858742, + 0.06543319, + -0.08643667, + -0.07211853, + 0.052677996, + 0.1292809, + -0.041060735, + -0.010105186, + 0.017167171, + 0.10278289, + -0.02560506, + 0.01444451, + -0.24326459, + 0.087474845, + -0.029556967, + -0.018887402, + 0.0519785, + 0.07257978, + -0.06718943, + -0.06401816, + 0.009868641, + 0.29014003, + 0.014569991, + -0.13224676, + -0.021046856, + 0.008031937, + 0.029677179, + -0.12090361, + -0.030684829, + -0.12032807, + 0.14864889, + 0.0730822, + 0.04875755, + -0.053761955, + -0.013974497, + -0.03588524, + 0.14112757, + 0.20620118, + 0.0026121754, + 0.057062473, + 0.006303307, + -0.100188285, + 0.030305644, + 0.030009354, + 0.08998002, + -0.0975479, + 0.17385758, + -0.14035688, + -0.12630624, + 0.20372146, + -0.031224119, + -0.015600041, + -0.000572721, + -0.12464091, + 0.028028218, + -0.08736926, + -0.064607315, + 0.0074855303, + -0.003159628, + 0.06802779, + -0.1313798, + 0.06092782, + -0.1787316, + 0.09524019, + 0.2142574, + 0.07154194, + -0.0078775445, + -0.08554093, + 0.17139089, + -0.00896565, + 0.20121369, + -0.08606445, + 0.10155439, + 0.120717004, + -0.05526951, + -0.25100285, + -0.110797964, + -0.06828172, + -0.07158169, + 0.12045773, + -0.19262262, + -0.03129522, + -0.17944223, + 0.02532566, + 0.09656396, + -0.091062434, + -0.0006519022, + 0.11245893 + ], + [ + 0.08650064, + -0.114138156, + 0.11560147, + 0.05716735, + 0.07363487, + 0.055551223, + 0.042977456, + 0.13866937, + 0.07197206, + 0.057236373, + -0.04786727, + 0.078654975, + 0.020689813, + -0.026271248, + -0.02856729, + 0.048773885, + 0.13842425, + -0.06279018, + 0.12849218, + 0.16308266, + -0.09831704, + -0.01208826, + 0.031379998, + 0.074512295, + -0.14470626, + 0.022406168, + -0.0015860604, + 0.08554748, + -0.13143538, + -0.0886885, + -0.05255464, + 0.30100065, + 0.030713795, + 0.030090688, + -0.0038725059, + -0.036318548, + -0.028928364, + 0.000039066, + 0.098410994, + -0.28426936, + -0.05369149, + 0.11395493, + -0.0016404965, + 0.02959418, + 0.1296119, + -0.08816001, + 0.12523289, + -0.046427406, + -0.024127232, + 0.10684162, + -0.050801836, + -0.036207974, + -0.09595485, + 0.078260094, + 0.06545781, + -0.14052771, + 0.096398816, + -0.020581724, + 0.1009272, + 0.016509471, + 0.032654297, + -0.10522052, + 0.07001931, + -0.052200574, + 0.11428169, + -0.16297819, + -0.022918481, + 0.008518012, + -0.052359655, + -0.15543026, + -0.030120252, + -0.21617754, + 0.057570234, + 0.012278141, + 0.0100514935, + -0.008567871, + 0.03985517, + -0.0071163354, + 0.14116311, + -0.043021444, + 0.046327017, + 0.06556431, + 0.19307518, + 0.048381023, + -0.054358784, + -0.14242582, + 0.052703325, + -0.016104024, + 0.11762951, + -0.27093124, + 0.071561694, + 0.14363705, + -0.10874558, + -0.042457294, + -0.045092538, + 0.073277965, + 0.081697024, + -0.005809052, + -0.10392095, + 0.055779688, + 0.040752683, + 0.023003666, + -0.07039203, + -0.098758176, + 0.11601843, + -0.09822991, + -0.014778384, + 0.019168867, + -0.08343514, + 0.013522634, + 0.018998768, + -0.08619179, + 0.05457085, + 0.008922722, + -0.15973733, + 0.021987697, + 0.04509556, + 0.12129829, + 0.20903002, + 0.11825309, + -0.009349105, + 0.0066330372, + 0.07822109, + 0.09448989, + 0.04427069, + -0.05089293, + -0.0634784, + -0.29010263, + -0.1817537, + -0.10901791, + -0.07024844, + -0.09644515, + 0.04460636, + 0.025619725, + 0.054481138, + -0.017032826, + -0.0024525498, + 0.0051498828, + -0.16207719, + 0.13890503, + 0.06168882, + -0.14186028, + -0.017634777, + 0.035919037, + -0.13157438, + 0.2913547, + -0.048506882, + 0.061833624, + -0.049157888, + -0.029682238, + -0.018465182, + -0.14546509, + -0.08058027, + 0.049560603, + 0.13289733, + -0.124585554, + 0.106291436, + 0.18051346, + -0.032141753, + 0.0047566677, + -0.096933044, + 0.21418223, + -0.12096488, + -0.014764312, + 0.047446866, + 0.03750072, + 0.020264208, + 0.18554664, + -0.1021362, + 0.013693483, + -0.05330727, + 0.021761887, + 0.099549085, + 0.0121648405, + 0.018702097, + -0.09758032, + 0.15901814, + -0.065417156, + 0.057077993, + 0.00047911698, + -0.04200102, + -0.075910345, + 0.16953775, + -0.023566708, + 0.058020957, + -0.1151307, + 0.065792955, + -0.29199713, + -0.019268267, + 0.024025127, + 0.052528866, + -0.10454245, + 0.028018963, + 0.00668617, + -0.0009638491, + 0.15003029, + -0.0796975, + 0.08568932, + 0.14248814, + -0.047471944, + 0.154569, + 0.105354466, + -0.09005179, + 0.0016345638, + -0.19058277, + 0.1462792, + -0.088306405, + -0.19980139, + 0.07397957, + 0.2989383, + -0.031951997, + 0.10095756, + 0.11718432, + 0.0045800325, + 0.09298005, + -0.074017435, + -0.043252986, + 0.075847, + 0.009191615, + 0.09819291, + 0.010901586, + -0.12441394, + -0.15651459, + -0.36471352, + 0.107327834, + 0.2367279, + -0.007892925, + -0.13905, + -0.114933364, + -0.05437099, + 0.030067027, + 0.099514276, + -0.037281614, + -0.037492216, + -0.1981845, + -0.1384136, + 0.08877338, + -0.13132341, + 0.00768169, + -0.047017664, + 0.03620687, + -0.099904016, + 0.019379376, + 0.0981703, + 0.011796661, + -0.09512934, + -0.18267427, + -0.13936168, + 0.06484512, + -0.030278102, + 0.090696916, + 0.12653728, + 0.037380584, + -0.15862349, + 0.14395134, + -0.021009054, + 0.054963164, + 0.046817254, + 0.06660228, + -0.035252098, + 0.090558656, + -0.09402542, + -0.056013796, + -0.061302137, + 0.031734105, + 0.015802667, + -0.16472177, + -0.101517014, + 0.048385564, + 0.16827406, + -0.0048398217, + 0.064329885, + 0.060761847, + 0.08469305, + 0.10910126, + -0.043873362, + -0.089271344, + -0.055935968, + 0.034083303, + -0.013393669, + 0.03643514, + -0.120853394, + 0.0704519, + 0.0065425616, + -0.04119151, + -0.09728708, + 0.05142146, + -0.102816895, + -0.016405446, + 0.10037102, + 0.114227235, + 0.020892173, + 0.0033101249, + 0.09104857, + -0.12247543, + -0.2576225, + -0.1403052, + 0.11442854, + -0.099400446, + 0.00058794674, + -0.19586149, + -0.113684855, + 0.030646956, + -0.11775976, + -0.06694089, + 0.086007595, + 0.09736035, + -0.13316418, + 0.023876686, + 0.06294222, + 0.060473986, + -0.0043298365, + -0.03389869, + -0.0055077346, + 0.03175626, + 0.03684258, + 0.09002136, + 0.12259016, + 0.017571814, + -0.27465117, + -0.033611458, + 0.08492853, + -0.064178616, + -0.09749589, + 0.08281812, + 0.074508965, + -0.08160397, + 0.1836253, + 0.12195985, + -0.047170453, + 0.11666565, + 0.00809368, + -0.1302758, + 0.21874562, + 0.08840849, + -0.09259481, + -0.015606064, + -0.030824287, + -0.2201201, + -0.05294996, + -0.025827821, + 0.028233208, + -0.032888405, + -0.011495582, + -0.11593533, + 0.0070121544, + 0.0074381963, + -0.0030233257, + 0.06750082, + -0.015932027, + 0.06266208, + 0.090228274, + -0.007783951, + 0.09262657, + -0.04413849, + -0.258896, + -0.08814282, + 0.19863154, + 0.21488272, + -0.047835387, + 0.12803216, + -0.017444558, + 0.075741306, + -0.19025156, + 0.027734958, + 0.03320167, + -0.123028815, + -0.028187469, + 0.013636808, + 0.06271145, + 0.18612179, + -0.29361016, + 0.10948807, + 0.027112115, + -0.031513385, + 0.028932175, + 0.25757912, + 0.047562208, + -0.1354802, + 0.08021137, + -0.27474785, + 0.032081064, + 0.03791089, + 0.025726352, + 0.069445595, + 0.0021079297, + -0.02449264, + 0.011238894, + -0.02319162, + -0.026542418, + -0.18316846, + 0.10666311, + 0.19345199, + -0.06720772, + 0.00815098, + -0.049136266, + -0.14031912, + 0.15578413, + 0.008095201, + 0.01659755, + 0.18123667, + -0.17131653, + 0.063718066, + 0.07473296, + -0.027271748, + -0.058085024, + -0.100015745, + -0.001066607, + 0.025115151, + 0.086855374, + 0.03166735, + 0.024315033, + 0.007213102, + 0.054682832, + 0.04690719, + -0.17995097, + -0.004744909, + -0.05241575, + 0.12825188, + 0.056677263, + -0.08422418, + 0.08251111, + 0.009535376, + 0.19880669, + 0.2963894, + -0.07653299, + 0.1401372, + 0.019019231, + -0.09808532, + 0.026957024, + -0.17146498, + 0.026395286, + -0.096108206, + -0.11671051, + -0.0031023317, + 0.030857945, + 0.00259537, + 0.17649667, + -0.009633149, + 0.09164389, + -0.11770125, + 0.035321847, + -0.25525537, + 0.2554045, + -0.042877495, + -0.19594881, + 0.1396774, + 0.012059957, + -0.00449406, + -0.19431978, + -0.088604175, + 0.015468876, + -0.13193911, + 0.10843306, + 0.23043013, + 0.06712104, + -0.07877263, + 0.03013501, + 0.22939627, + 0.047045074, + -0.083643205, + -0.066756636, + 0.15640521, + 0.14078821, + -0.011661555, + -0.07418068, + -0.0354833, + 0.025781255, + 0.06630863, + 0.09891263, + -0.034469504, + 0.028824922, + -0.056631226, + 0.08970828, + 0.09523603, + 0.050132398, + -0.1690745, + 0.046013247, + 0.013893757, + -0.12068828, + -0.048534885, + 0.07168909, + -0.037411164, + -0.008103455, + 0.006886762, + -0.19633815, + -0.17443606, + 0.045308072, + 0.09340842, + 0.1490397, + -0.030604543, + 0.02490058, + -0.051153444, + -0.04850469, + -0.051614046, + 0.07135396, + 0.04567157, + 0.0073676747, + -0.15031311, + 0.16379681, + 0.0757311, + -0.1616459, + -0.1270838, + 0.023255069, + -0.1826222, + 0.15810522, + -0.044493705, + -0.22102857, + 0.045563675, + -0.008063402, + 0.05133512, + 0.12316835, + 0.10462279, + -0.2890721, + -0.017548595, + -0.11253995, + 0.10683944, + 0.032772627, + 0.011404846, + -0.031123182, + -0.008722479, + 0.1220427, + -0.23846045, + -0.05996672, + -0.05013295, + -0.029948775, + 0.060149055, + -0.13687938, + 0.054654896, + -0.0076787327, + -0.08335814, + -0.11796455, + -0.101908155, + -0.050144844, + 0.21571128, + -0.06343264, + 0.107729964, + -0.013642982, + -0.40179682, + -0.013399297, + 0.02881357, + -0.018139042, + 0.013398127, + -0.031053284, + -0.06259348, + -0.10992421, + 0.03368614, + 0.1299659, + -0.14629827, + -0.17959803, + -0.02231733, + 0.14280543, + 0.06939315, + 0.08924989, + 0.06705628, + 0.071522295, + -0.029402977, + 0.059425004, + 0.11423234, + 0.09920461, + 0.1667185, + 0.077411644, + -0.061828777, + -0.07963011, + 0.10044846, + -0.029872157, + -0.1763701, + 0.110166654, + 0.17331573, + -0.14678414, + -0.13495205, + -0.015179527, + -0.105202034, + 0.03376242, + 0.045832768, + 0.050930824, + 0.113197684, + 0.022861745, + -0.081894, + 0.0609409, + 0.07157385, + -0.030135512, + -0.13592605, + -0.17098251, + -0.037453867, + 0.0445514, + 0.005111366, + 0.22748575, + -0.031975, + 0.17256372, + 0.15765712, + 0.2115168, + -0.01236721, + -0.010223999, + -0.2236333, + -0.2040943, + 0.008426578, + 0.037784062, + -0.21303326, + 0.0014444012, + 0.10017615, + -0.03439675, + -0.03444695, + 0.07566503, + 0.11491664, + 0.14344919, + 0.03547391, + 0.1023221, + -0.26562563, + 0.03324227, + -0.12612627, + -0.025097914, + -0.16231543, + -0.07071355, + -0.031565484, + 0.05040618, + 0.019868962, + 0.12765361, + -0.11720381, + 0.042901862, + 0.04383294, + 0.048617877, + 0.046922315, + -0.03069157, + 0.014754203, + -0.046216488, + -0.09432203, + -0.040246278, + -0.026649779, + -0.072701216, + -0.15752329, + -0.36157733, + 0.040658437, + -0.093095146, + -0.08237973, + 0.13206859, + 0.21616566, + 0.13615578, + -0.052190773, + -0.031580392, + 0.100822076, + -0.012105965, + 0.28700614, + -0.0798311, + -0.030761806, + 0.031985268, + 0.06104218, + 0.017198546, + -0.008219104, + -0.006477159, + -0.030536303, + -0.07336686, + 0.0806302, + -0.015533479, + -0.0037378035, + 0.057539888, + -0.015810573, + 0.0075652716, + -0.058744423, + -0.26509422, + -0.07290028, + -0.042361405, + -0.10962765, + -0.2524032, + -0.13754942, + -0.03327559, + 0.028698724, + -0.032883186, + -0.119565666, + -0.1226996, + 0.05682807, + -0.048052542, + 0.37282622, + 0.028903697, + 0.035287928, + 0.02152677, + 0.23741001, + 0.06756465, + 0.122916296, + -0.07250478, + -0.04318548, + 0.1850775, + -0.01884087, + 0.08287051, + 0.020896921, + 0.010066692, + -0.03891263, + 0.18874517, + 0.010514654, + -0.011000999, + -0.08137152, + -0.035857525, + -0.12775403, + 0.02271913, + -0.09061829, + -0.007581237, + -0.10853437, + 0.03644659, + -0.04371966, + -0.20594738, + -0.10544621, + -0.008287322, + -0.15238318, + -0.08166857, + -0.085347965, + 0.1146354, + 0.079319455, + 0.011942096, + -0.07547264, + 0.037925895, + 0.04891033, + 0.09160054, + 0.03913848, + 0.10733035, + 0.12855045, + -0.008485813, + -0.09220337, + 0.13284315, + -0.068138555, + -0.06429005, + 0.30747658, + 0.017451087, + -0.1090665, + 0.033579294, + -0.22076453, + 0.12192298, + 0.14958231, + -0.13429965, + -0.009634048, + -0.14999968, + -0.00200675, + -0.12121864, + -0.013977665, + 0.048683085, + -0.024490103, + 0.0498644, + -0.011909416, + 0.0015371294, + 0.037581872, + 0.07797013, + -0.06255501, + -0.17288956, + -0.07691344, + 0.37672377, + -0.026362475, + 0.06730378, + 0.0104083195, + -0.024101874, + 0.14750732, + -0.02757014, + -0.1706281, + -0.0031450412, + -0.0055671176, + -0.1802068, + 0.12573895, + -0.032311182, + -0.2111977, + 0.064462945, + -0.057173625, + 0.035591144, + 0.1605281, + 0.07954016, + -0.026228633, + 0.18851891, + 0.13628644, + 0.033962764, + -0.008227327, + 0.103921, + 0.386116, + -0.13597281, + -0.12757441, + 0.043079887, + -0.09839026, + -0.009575337, + -0.005930024, + 0.11509768, + 0.113127336, + -0.063924, + -0.16976811, + -0.026487622, + 0.16157718, + 0.039270367, + 0.013407921, + 0.05706089, + -0.024494743, + -0.032472625, + 0.20470479, + 0.062012397, + 0.013509699, + 0.10476464, + 0.02340273, + 0.22813578, + -0.17417786, + 0.05276678, + -0.2060439, + -0.003265173, + -0.16086264, + -0.23557913, + 0.0522266, + -0.173802, + 0.0074773612, + -0.028467707, + 0.20351472, + -0.057790864, + 0.08757821, + -0.06914482, + -0.14491978, + 0.09239386, + 0.020376774, + 0.1097607, + -0.088697076, + 0.16945688, + 0.01119273, + -0.1528324, + -0.173479, + -0.026488239, + -0.029663468, + -0.037529018, + 0.08184811, + -0.116736144, + 0.020757439, + 0.037805155, + 0.081116766, + -0.08899582, + -0.23994243, + 0.00085073564, + 0.015741583, + 0.14916396, + -0.049735896, + 0.06661528, + -0.05483578, + 0.05611641, + 0.07030592, + -0.14783034, + 0.07137445, + -0.106677264, + 0.060104955, + -0.10194723, + 0.14962104, + 0.028547723, + 0.078555, + -0.09768789, + 0.11221817, + 0.0854506, + -0.15803969, + 0.009801034, + 0.078489214, + -0.0693264, + -0.1491822, + -0.09020748, + -0.03234147, + 0.044914614, + 0.02052608, + -0.1493524, + 0.17103215, + -0.063729316, + -0.10966809, + -0.029281124, + 0.09548159, + -0.048648804, + 0.23593551, + -0.012712764, + 0.07028848, + -0.061728034, + 0.05049216, + -0.021263314, + -0.06874739, + 0.07710166, + 0.08121541, + 0.0734596, + -0.023434158, + 0.16007483, + -0.10867535, + -0.11891655, + 0.22188166, + 0.06489395, + 0.010524853, + 0.19451497, + -0.053407338, + -0.08928299, + -0.14070098, + 0.20396641, + -0.062143497, + 0.125486, + 0.12106229, + -0.1373563, + -0.12767154, + 0.05849606, + -0.1128916, + 0.040035207, + -0.036183015, + -0.13515732, + 0.056579936, + -0.15353481, + 0.17633565, + 0.17068711, + -0.031218339, + -0.06275934, + 0.045422144, + 0.020727132, + -0.015841577, + 0.028296215, + -0.1515143, + 0.05359048, + -0.13335627, + -0.102366865, + 0.09058614, + -0.16648483, + 0.021478737, + -0.02842722, + 0.01834001, + 0.029639145, + -0.023922393, + 0.026784806, + -0.1001365, + -0.042642847, + -0.03436985, + -0.097866334, + 0.019808806, + -0.042747162, + 0.14128022, + -0.033543553, + 0.08283731, + 0.07131058, + -0.15540025, + 0.28758714, + 0.0058351206, + 0.15492027, + 0.013075218, + 0.08945752, + 0.07358258, + 0.054398827, + -0.02437751, + -0.07341751, + 0.10022484, + -0.050757684, + 0.1440264, + 0.013197424, + -0.109852426, + 0.22719486, + 0.080075815, + 0.00024181453, + -0.00097611186, + -0.04847994, + -0.07244215, + 0.006288356, + 0.030367048, + 0.12748545, + 0.1944054, + -0.060363494, + -0.0036989984, + 0.06789716, + 0.03632896, + 0.026532803, + -0.06583803, + -0.13583247, + 0.035222605, + -0.08362427, + -0.18340993, + -0.19657335, + -0.0031432216, + 0.023255847, + -0.18112065, + 0.12979667, + 0.04521475, + -0.037557703, + 0.0030770968, + 0.2226345, + 0.14577474, + 0.0025226166, + -0.023376113, + -0.054195154, + -0.05671215, + -0.11162617, + -0.06985026, + -0.028117025, + 0.0053079883, + 0.061277986, + -0.14143746, + 0.23066823, + 0.19300221, + 0.041472983, + -0.18196483, + 0.05748335, + 0.073920846, + -0.11640508, + -0.11107834, + -0.18051128, + 0.009662717, + 0.13686214, + -0.022840913, + 0.08510747, + 0.012901924, + -0.10250301, + 0.0596237, + -0.18144847, + 0.105939075, + 0.07167126, + 0.14054418, + -0.18191113, + -0.1436565, + -0.0597559, + 0.030895803, + -0.030637655, + -0.13535596, + 0.08015554, + -0.07608762, + -0.040301584, + -0.19534457, + -0.2292024, + -0.04393903, + 0.038195375, + 0.099135764, + -0.021666853, + -0.029471708, + -0.009995175, + 0.25933316, + 0.032328885, + 0.09000577, + 0.03263837, + 0.079661906, + -0.0012039868, + 0.11770895, + -0.0007853804 + ], + [ + 0.067265354, + 0.14978805, + -0.02866475, + -0.017997744, + 0.0043778527, + -0.077955745, + -0.17966285, + -0.08962502, + 0.012652886, + 0.068688974, + 0.032703526, + 0.103429645, + -0.026255865, + -0.04594767, + -0.03591724, + 0.13831529, + -0.063064694, + 0.098223, + -0.120419286, + -0.19822003, + -0.087343685, + 0.05914871, + -0.004434789, + -0.1129431, + 0.17182596, + 0.055425245, + 0.029754065, + -0.03556171, + -0.0070821294, + -0.13465452, + 0.16700113, + -0.19219238, + -0.01675321, + -0.027130553, + -0.18479164, + 0.027703132, + -0.038662884, + -0.025050428, + 0.020932555, + 0.25118732, + 0.13719758, + -0.065483645, + 0.07571365, + -0.07540309, + -0.11814715, + -0.020154916, + -0.1212217, + 0.14467515, + 0.049435012, + -0.03782557, + 0.10122299, + 0.16256903, + 0.18433803, + 0.002223497, + -0.06347142, + -0.045097824, + 0.0076598977, + -0.0047144056, + -0.20919853, + -0.26025832, + -0.021529123, + 0.09192429, + 0.0030727165, + 0.022516953, + -0.02791795, + 0.20013073, + -0.060710344, + -0.014477534, + -0.022307828, + 0.04182581, + 0.07157571, + 0.11714966, + -0.034158412, + -0.07059728, + 0.03238398, + -0.077487394, + -0.19103165, + 0.026339812, + -0.10266619, + 0.17039405, + -0.10501824, + -0.047377028, + -0.086855955, + -0.020825548, + 0.0031076584, + -0.024767814, + 0.12598439, + 0.19383541, + -0.030236963, + 0.16819346, + -0.036930185, + -0.08973632, + 0.20534639, + -0.04046148, + 0.19653097, + 0.022572542, + -0.10891978, + -0.0021857636, + 0.043617047, + -0.03692846, + -0.02452463, + -0.1116284, + -0.120930575, + 0.11805574, + -0.062913835, + -0.04299464, + 0.11609556, + 0.07652675, + 0.15170819, + -0.0143310465, + 0.13215616, + 0.038192805, + -0.09449705, + -0.04615726, + 0.12743029, + 0.053697243, + 0.028676936, + -0.011620548, + -0.115987964, + -0.15858902, + 0.025458544, + 0.16376868, + -0.21461114, + -0.12859254, + 0.046384696, + -0.060146254, + -0.10704182, + 0.1203173, + 0.04870109, + 0.055586778, + -0.036846314, + -0.033308074, + -0.04773714, + -0.1418176, + 0.14744744, + 0.058815204, + -0.077088796, + -0.052262966, + 0.19709195, + 0.026317546, + 0.036419798, + 0.05554856, + -0.0029399341, + -0.062320363, + -0.01966279, + -0.17727584, + 0.04959125, + -0.029366525, + 0.004609626, + -0.053935826, + 0.07947955, + 0.08481755, + -0.10725464, + 0.07710253, + -0.2177989, + 0.018307095, + -0.058796752, + -0.17742877, + -0.103017226, + -0.16547178, + 0.08727635, + -0.18131922, + -0.15467994, + 0.094013415, + -0.0007966098, + -0.11299204, + 0.049469113, + -0.2551994, + 0.10368437, + 0.12887946, + 0.052979, + -0.036250193, + 0.026475381, + 0.068989575, + 0.026221694, + 0.13373744, + -0.16292971, + 0.1264861, + -0.015351114, + -0.046966083, + -0.026861196, + 0.08184943, + 0.03712424, + -0.0893275, + -0.04624934, + 0.17902076, + -0.09396041, + 0.0012100477, + -0.0004969027, + 0.061765272, + -0.13257642, + -0.06479358, + -0.08090168, + -0.045115944, + -0.033314876, + -0.10521074, + 0.038595706, + 0.05002954, + 0.05275309, + -0.022413827, + -0.228349, + -0.05911533, + 0.13100033, + 0.08290936, + 0.21059032, + -0.053871688, + 0.2612328, + -0.03817294, + 0.12055628, + 0.19849202, + 0.1064979, + 0.06644159, + -0.010666335, + 0.037192244, + -0.03825279, + 0.22590448, + 0.06540578, + 0.0635492, + 0.009995832, + 0.053388823, + 0.1819073, + 0.017358864, + 0.0940223, + 0.19547985, + 0.16051306, + -0.08009981, + -0.01766264, + 0.31268245, + 0.03358963, + -0.006802613, + 0.060060095, + -0.02999579, + -0.02441298, + 0.041289836, + -0.018898971, + 0.016558355, + -0.2887073, + 0.05011042, + -0.020560574, + 0.10191658, + 0.07186017, + -0.07590083, + 0.043192476, + 0.04002405, + -0.08334605, + -0.0033909143, + 0.013439505, + 0.1452686, + -0.05733201, + -0.088819325, + -0.022471705, + -0.095721394, + 0.15808508, + 0.120784655, + -0.14439268, + -0.097761706, + -0.15823784, + 0.02247433, + -0.08741217, + -0.10422804, + 0.095085815, + -0.12007163, + -0.090681195, + 0.015306839, + -0.042384245, + 0.059710126, + 0.16539843, + 0.022866111, + 0.061641064, + -0.14381783, + 0.08884166, + 0.07466225, + -0.15382792, + -0.185641, + -0.1813861, + 0.02484775, + -0.0073386994, + -0.0801197, + -0.06278462, + -0.0073598013, + -0.09684346, + 0.06798241, + 0.0029492439, + -0.016103258, + 0.06638396, + 0.01753141, + 0.23454934, + 0.08132862, + 0.10536375, + -0.2229483, + 0.022713436, + -0.14560527, + -0.064387955, + -0.095680125, + 0.22664733, + 0.036997385, + 0.15401214, + -0.0652297, + 0.09910494, + -0.07604658, + 0.019308623, + -0.0017304748, + -0.039347965, + -0.12615763, + 0.038332216, + -0.1498985, + -0.18378036, + 0.22454844, + 0.076195106, + 0.006609157, + -0.051469374, + 0.04080529, + 0.014309058, + -0.096273944, + -0.207181, + -0.15468046, + -0.13909651, + -0.015941054, + -0.050097745, + 0.22421612, + 0.041737087, + 0.01954481, + 0.05503906, + -0.030042086, + -0.060866546, + -0.060889628, + 0.050270688, + -0.032480314, + 0.05457713, + -0.018058803, + -0.17685808, + 0.027825413, + 0.2854748, + -0.1990501, + -0.05410666, + -0.016514089, + 0.021260383, + 0.13303187, + -0.04133936, + -0.102796614, + -0.023230858, + -0.021802323, + -0.009447499, + 0.0056849695, + 0.077226385, + -0.001233707, + -0.056483172, + -0.0148004005, + -0.15466832, + -0.058501728, + -0.07044992, + -0.034258507, + 0.0032710754, + -0.043261908, + 0.042553976, + 0.11541086, + 0.22303134, + -0.042487398, + -0.2697398, + 0.029412314, + -0.093711354, + 0.013379649, + -0.02745153, + -0.09139424, + -0.07518103, + 0.081906185, + 0.01151903, + 0.07499832, + -0.08512211, + -0.017350763, + -0.17634419, + 0.25530154, + -0.05747039, + -0.06407152, + -0.026143344, + 0.031473134, + 0.040568653, + 0.02365954, + -0.03598118, + -0.004738165, + 0.26938042, + 0.107892066, + 0.12235724, + -0.0670435, + 0.103311166, + 0.027247678, + -0.060414925, + 0.10446229, + 0.021563247, + -0.17967018, + 0.021033838, + -0.032761253, + -0.002232972, + 0.20094131, + -0.06271966, + -0.06088999, + 0.114715405, + -0.09043788, + -0.065625295, + -0.034772128, + -0.03247024, + 0.13547981, + -0.05234461, + 0.0008887822, + -0.04399269, + 0.1076599, + 0.19047959, + 0.06816161, + 0.017209189, + -0.02949286, + -0.17175756, + 0.008101737, + -0.11393619, + 0.14412154, + -0.14545026, + 0.10393415, + 0.09651161, + 0.17374185, + -0.15528761, + 0.30169812, + 0.21803834, + 0.22342007, + -0.0023207148, + -0.32583496, + -0.1602966, + 0.08804405, + -0.09492502, + 0.013310529, + 0.08473863, + -0.010918157, + 0.09779154, + 0.1687677, + -0.0017114308, + 0.14069672, + -0.14218633, + -0.119746976, + -0.0016709599, + -0.116650805, + -0.0087289335, + -0.0020710814, + 0.14164573, + -0.03426684, + 0.13765714, + -0.23489869, + 0.17190114, + 0.11823379, + -0.1275406, + 0.016226083, + -0.015224294, + -0.040592875, + 0.088462226, + 0.0100339865, + 0.29286325, + -0.0012119255, + -0.20170744, + 0.043817673, + -0.048842527, + -0.06267558, + -0.20045178, + -0.07331201, + -0.0665427, + 0.0008836231, + 0.047697656, + -0.114039324, + 0.16505136, + 0.055174492, + 0.1341762, + 0.07885245, + -0.12267522, + -0.118794784, + 0.010460504, + 0.036683165, + 0.016049402, + 0.0030134192, + -0.07325558, + -0.059531037, + 0.22491747, + 0.021559626, + -0.2682984, + -0.1513486, + -0.007747446, + -0.21185677, + -0.085555956, + -0.07768858, + -0.0889585, + 0.18072057, + 0.17185259, + 0.071126275, + -0.0070228805, + -0.09613079, + 0.19773807, + -0.027994337, + -0.022889512, + 0.017196476, + 0.16872603, + 0.010325995, + 0.10429709, + -0.050494857, + 0.009961071, + -0.08508044, + -0.03851841, + 0.22150967, + 0.26763844, + 0.013120958, + 0.06905126, + -0.1074305, + 0.021637388, + 0.16322272, + -0.13542607, + 0.05240015, + 0.03747799, + -0.02136567, + -0.19518022, + 0.33402026, + -0.07306034, + 0.24572115, + -0.13900623, + -0.038396675, + 0.06149101, + 0.022670144, + 0.11762282, + 0.02650935, + 0.24784814, + -0.012557801, + -0.049926393, + 0.17277104, + 0.019994384, + 0.17777458, + -0.015156963, + -0.36002278, + 0.08576319, + 0.12579906, + -0.070583634, + 0.046381455, + -0.032518055, + -0.013673033, + 0.08388661, + 0.02805109, + -0.0040858416, + 0.08121704, + -0.024603426, + 0.038663764, + -0.052432787, + 0.09663728, + -0.05794367, + 0.03322958, + -0.05295467, + 0.012488321, + 0.06900259, + 0.32041568, + 0.050188363, + -0.06842132, + 0.17127438, + -0.08995696, + 0.052553497, + -0.15109767, + 0.0032275803, + -0.124561824, + -0.0349717, + -0.15243848, + 0.06931027, + -0.08733464, + 0.08632587, + -0.000009701724, + -0.061496064, + 0.016888466, + 0.14733578, + -0.20635521, + -0.086442135, + -0.010243254, + 0.08384721, + -0.21813874, + 0.053306054, + -0.021313356, + -0.22646224, + -0.111123234, + -0.17016025, + -0.053942624, + 0.10293637, + -0.067348056, + 0.016638568, + -0.055585764, + 0.1321758, + 0.09786219, + -0.0846062, + -0.20103993, + -0.20276587, + -0.14107303, + 0.0572056, + -0.05998819, + -0.079461865, + -0.18345493, + 0.07808181, + 0.025852023, + 0.16117097, + 0.09278437, + -0.0070364075, + -0.09634602, + 0.15276241, + -0.025094103, + -0.25218967, + -0.047830544, + -0.03265829, + -0.14163731, + -0.2689699, + -0.03686031, + 0.012960216, + -0.02138974, + 0.13290478, + 0.071284525, + 0.059489492, + 0.1677266, + 0.04434716, + 0.05381613, + -0.07807097, + -0.007200855, + -0.05029709, + -0.19462912, + -0.06630631, + 0.077244155, + -0.051679987, + 0.16820908, + 0.006629443, + 0.06144869, + 0.044119336, + 0.08528723, + 0.09778228, + -0.05264809, + 0.17047869, + -0.033962384, + 0.09303276, + 0.14469127, + 0.32394484, + -0.05757008, + 0.14690277, + -0.113172874, + -0.073722996, + -0.1543499, + 0.13370518, + 0.022599915, + -0.016452743, + -0.1111051, + -0.123819724, + -0.0358012, + 0.095570944, + -0.026632585, + -0.12938291, + 0.054892946, + 0.0053247954, + 0.0051558986, + -0.08200846, + -0.024864692, + 0.0022463088, + 0.033632293, + 0.022776838, + 0.046233963, + 0.07099262, + 0.03803995, + 0.11918089, + 0.073886305, + -0.02947079, + -0.032019015, + -0.028573902, + 0.07655141, + -0.13654494, + 0.10705977, + -0.054300036, + 0.1552018, + -0.000661521, + 0.22377999, + -0.08899608, + 0.12889342, + -0.20074363, + 0.13927813, + -0.0026708662, + -0.04300883, + -0.16178162, + -0.0923155, + -0.16207108, + 0.20354961, + 0.009087291, + -0.15107653, + -0.06892152, + -0.033151213, + 0.043834474, + -0.115955636, + -0.042398877, + -0.13583317, + 0.0435221, + 0.046010397, + -0.007331226, + -0.03840562, + 0.08738765, + -0.10060804, + 0.06926944, + -0.21893275, + -0.04178857, + -0.19848873, + 0.043663215, + 0.11381242, + -0.09335524, + -0.076091655, + 0.13046667, + -0.0030471336, + 0.04753741, + -0.04804935, + -0.032354355, + 0.08939008, + -0.010100283, + 0.10576193, + 0.01991799, + 0.030111695, + -0.04467943, + -0.015056431, + -0.024010837, + 0.08842912, + -0.005612796, + -0.13563918, + 0.18028839, + 0.013166858, + -0.2667886, + -0.085470125, + 0.3685931, + -0.1946491, + 0.049553536, + -0.37291518, + -0.22655754, + 0.34400705, + 0.032976877, + 0.13435382, + -0.11814129, + 0.047868554, + 0.07159131, + 0.036158036, + 0.024520447, + -0.20078675, + -0.05586612, + 0.08214067, + -0.08605121, + 0.032829337, + 0.058903836, + 0.24436851, + 0.09116331, + -0.13379814, + 0.1828366, + 0.1072864, + -0.021227801, + 0.12980577, + -0.11652907, + -0.03775177, + 0.083177045, + 0.057081014, + -0.06332418, + 0.056298833, + -0.3391434, + 0.04645195, + -0.06128842, + 0.18199284, + -0.002383945, + -0.14004768, + -0.16993281, + 0.010031427, + 0.16041878, + -0.16184394, + 0.018993879, + -0.025938176, + 0.002514922, + 0.020949349, + -0.1269015, + -0.054172676, + 0.094788544, + 0.07986568, + 0.088186376, + -0.028140098, + 0.011289786, + -0.17378683, + -0.12375356, + 0.04456818, + 0.08479448, + -0.09294157, + -0.16615918, + 0.028024394, + -0.093367904, + -0.013316832, + 0.04018244, + 0.08644012, + -0.12008931, + 0.018793972, + 0.061523847, + -0.0694417, + -0.024518935, + -0.11057199, + 0.061863527, + -0.05944173, + 0.11886046, + -0.030533476, + -0.017306136, + 0.0665828, + -0.08461085, + 0.13586487, + 0.12317356, + 0.118130654, + -0.084764935, + 0.009605317, + -0.0064539798, + 0.15894376, + 0.12810811, + 0.029883908, + -0.027100915, + -0.18702109, + 0.016678592, + -0.21666358, + 0.009283024, + -0.06385588, + 0.11203676, + 0.036862098, + -0.051290695, + 0.04737701, + 0.100528136, + 0.033112366, + 0.048415672, + -0.04174812, + -0.16214389, + 0.17123649, + 0.21322905, + 0.021103352, + 0.19598372, + -0.21077679, + -0.06637894, + -0.22511913, + 0.13010465, + -0.044941496, + -0.19983673, + 0.1963295, + -0.15502413, + 0.0615183, + -0.16637003, + -0.01743503, + 0.054865364, + 0.004895811, + -0.034269266, + 0.1766163, + -0.084879376, + 0.04049494, + 0.11087186, + 0.02269163, + -0.027787592, + -0.0145252375, + 0.08132456, + 0.12849073, + 0.06231228, + 0.058701742, + -0.06058468, + 0.13572963, + -0.14186189, + 0.1027375, + 0.028402401, + 0.0053135604, + -0.054753385, + 0.027727582, + -0.034705237, + -0.016670734, + -0.10962946, + -0.011801861, + -0.20003115, + 0.2020829, + 0.026161859, + -0.0031623817, + -0.029170213, + -0.018234199, + 0.04875749, + -0.23150893, + -0.036654674, + 0.09868626, + -0.28474876, + -0.048850782, + -0.0046823416, + -0.0050867144, + -0.028108975, + 0.078664176, + 0.05139621, + -0.13367955, + -0.06766825, + -0.13896853, + -0.16415608, + 0.017776988, + 0.10541162, + -0.0264685, + 0.08690293, + 0.12137225, + -0.034988154, + 0.16255242, + 0.002840024, + 0.16030852, + -0.076692425, + -0.22003824, + -0.18630469, + 0.016727459, + -0.020857252, + -0.008802409, + 0.07564682, + -0.05444138, + 0.1668737, + -0.025629545, + 0.14557064, + 0.09674229, + -0.058274068, + 0.05744842, + -0.0928503, + -0.0799128, + -0.07071504, + -0.0121874735, + 0.1692294, + 0.093030006, + 0.020831311, + 0.04520271, + 0.087474234, + -0.016130574, + -0.05108359, + 0.036344565, + -0.108744465, + -0.055855155, + 0.1017321, + -0.069691285, + -0.14468838, + -0.10310156, + 0.11176399, + -0.00078769, + -0.03110051, + -0.1145704, + -0.07260943, + 0.23371693, + -0.104421645, + -0.06583719, + -0.006062506, + 0.05011631, + -0.0054390067, + -0.078630604, + 0.19628902, + -0.1550763, + -0.13275377, + -0.12952268, + 0.042036884, + 0.05858515, + 0.055275004, + -0.10907125, + -0.00476199, + -0.14193, + 0.04885924, + -0.02711134, + 0.03325598, + -0.04900974, + -0.08830747, + -0.099112585, + 0.13302748, + 0.19985054, + -0.045091256, + -0.20651561, + 0.16883984, + 0.3288204, + 0.024190074, + -0.031287767, + 0.15144359, + -0.008892988, + -0.01452989, + 0.15788583, + -0.151726, + -0.2957167, + -0.1945324, + 0.05123937, + 0.037350606, + 0.09008041, + -0.08441538, + -0.09457509, + 0.067238115, + -0.02894548, + -0.011611294, + 0.03891026, + 0.111131825, + -0.26067775, + -0.28298238, + 0.056074947, + 0.008107262, + 0.0828736, + 0.052385423, + -0.08731629, + 0.14230244, + 0.19611123, + -0.0090899905, + -0.012221198, + -0.00518729, + 0.0022617683, + 0.051705398, + 0.095017456, + -0.056464072, + 0.11342066, + 0.0254407, + -0.13259909, + 0.038187362, + 0.08667088, + -0.07060096, + -0.011785999, + -0.023018276, + 0.11617855, + -0.036035065, + -0.07118992, + -0.12512614, + 0.12636593, + 0.09379038, + 0.10848534, + 0.099208534, + 0.21280734, + 0.011662269, + 0.08994858, + 0.101053335, + -0.11046256, + -0.066710554, + -0.001033645, + 0.08943645, + -0.057964046, + -0.17622581, + 0.09226646, + -0.11705704, + -0.11167349 + ] + ], + "bias": [ + 0.23896483, + -0.3402354, + 0.101270564 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.json new file mode 100644 index 0000000..5c115ef --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.json @@ -0,0 +1,1717 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.71159875, + "confusion": [ + [ + 69, + 75, + 5 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 11, + 58 + ] + ], + "tool_precision": 0.98571426, + "tool_recall": 0.46308726, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5340551, + "p_memory": 0.21771747, + "p_abstain": 0.24822739 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5626822, + "p_memory": 0.1836759, + "p_abstain": 0.2536419 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6036589, + "p_memory": 0.11206787, + "p_abstain": 0.28427315 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40643284, + "p_memory": 0.19013943, + "p_abstain": 0.40342778 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51935685, + "p_memory": 0.09203593, + "p_abstain": 0.3886073 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3248475, + "p_memory": 0.08496061, + "p_abstain": 0.5901919 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5663625, + "p_memory": 0.18072063, + "p_abstain": 0.2529169 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53524184, + "p_memory": 0.17327352, + "p_abstain": 0.2914846 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55211174, + "p_memory": 0.14331223, + "p_abstain": 0.30457598 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48472118, + "p_memory": 0.20033272, + "p_abstain": 0.31494612 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48654568, + "p_memory": 0.20197286, + "p_abstain": 0.31148145 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3951337, + "p_memory": 0.11869106, + "p_abstain": 0.48617515 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5669123, + "p_memory": 0.11284301, + "p_abstain": 0.32024476 + }, + { + "query_id": "shell_test_023", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6097893, + "p_memory": 0.08166945, + "p_abstain": 0.30854127 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "memory", + "p_tools": 0.532595, + "p_memory": 0.09771287, + "p_abstain": 0.36969218 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5313089, + "p_memory": 0.07768788, + "p_abstain": 0.39100322 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5733041, + "p_memory": 0.16126691, + "p_abstain": 0.26542896 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5433677, + "p_memory": 0.18911023, + "p_abstain": 0.2675221 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47030315, + "p_memory": 0.17896877, + "p_abstain": 0.3507281 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5621234, + "p_memory": 0.14502367, + "p_abstain": 0.29285294 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57606184, + "p_memory": 0.1570898, + "p_abstain": 0.2668484 + }, + { + "query_id": "rf_test_020", + "text": "read the existing router_report.md from the last benchmark run", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5553279, + "p_memory": 0.09895797, + "p_abstain": 0.3457141 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5518937, + "p_memory": 0.17088455, + "p_abstain": 0.2772218 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53405726, + "p_memory": 0.19761977, + "p_abstain": 0.268323 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4904085, + "p_memory": 0.08687752, + "p_abstain": 0.42271397 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5497213, + "p_memory": 0.09366132, + "p_abstain": 0.3566174 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41655815, + "p_memory": 0.17062807, + "p_abstain": 0.41281372 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5488636, + "p_memory": 0.2709307, + "p_abstain": 0.1802057 + }, + { + "query_id": "rf_test_028", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54564625, + "p_memory": 0.10946755, + "p_abstain": 0.3448862 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5033046, + "p_memory": 0.11161256, + "p_abstain": 0.3850828 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46809536, + "p_memory": 0.20201841, + "p_abstain": 0.32988623 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41650668, + "p_memory": 0.15908073, + "p_abstain": 0.4244126 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55804354, + "p_memory": 0.1696324, + "p_abstain": 0.27232406 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4662468, + "p_memory": 0.18212193, + "p_abstain": 0.35163125 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56596255, + "p_memory": 0.1841741, + "p_abstain": 0.24986333 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3543384, + "p_memory": 0.07887757, + "p_abstain": 0.566784 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50478405, + "p_memory": 0.16049631, + "p_abstain": 0.33471963 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42139858, + "p_memory": 0.14304228, + "p_abstain": 0.43555915 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49103788, + "p_memory": 0.14342807, + "p_abstain": 0.36553407 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42337945, + "p_memory": 0.1434221, + "p_abstain": 0.4331984 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5013689, + "p_memory": 0.1826258, + "p_abstain": 0.31600538 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.456451, + "p_memory": 0.15955734, + "p_abstain": 0.3839917 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52091247, + "p_memory": 0.18335688, + "p_abstain": 0.29573065 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46343142, + "p_memory": 0.2125959, + "p_abstain": 0.32397267 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5892539, + "p_memory": 0.1738571, + "p_abstain": 0.23688899 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44753295, + "p_memory": 0.27299076, + "p_abstain": 0.27947626 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5022449, + "p_memory": 0.23418446, + "p_abstain": 0.2635706 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.7777778, + "confusion": [ + [ + 38, + 31, + 1 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 8, + 47 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54285717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 6, + 4, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 7, + 33 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.6, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 5, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.6166667, + "confusion": [ + [ + 27, + 22, + 1 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.md new file mode 100644 index 0000000..37de7de --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-13/router_report.md @@ -0,0 +1,146 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.340` +- abstain_prob_threshold: `0.450` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.7116` +- tools precision: `0.9857` +- tools recall: `0.4631` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 69 | 75 | 5 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 11 | 58 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.53 | 0.22 | 0.25 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.56 | 0.18 | 0.25 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Memory` | 0.58 | 0.09 | 0.33 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.57 | 0.17 | 0.26 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Memory` | 0.58 | 0.10 | 0.32 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.58 | 0.18 | 0.24 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_017` | append a line "router ok" to docs/router_holdout_log.txt | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_tools_recall_tools_018` | create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it | `Tools` | `Memory` | 0.58 | 0.16 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Memory` | 0.56 | 0.12 | 0.31 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.56 | 0.11 | 0.33 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.56 | 0.16 | 0.28 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Memory` | 0.51 | 0.08 | 0.41 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Memory` | 0.56 | 0.09 | 0.34 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Memory` | 0.48 | 0.09 | 0.43 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Memory` | 0.50 | 0.13 | 0.36 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.45 | 0.17 | 0.39 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.52 | 0.20 | 0.28 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Memory` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.34 | 0.25 | 0.40 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.35 | 0.32 | 0.33 | +| `holdout_abstain_toolish_abstain_024` | fix this pls | `Abstain` | `Memory` | 0.46 | 0.11 | 0.43 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Memory` | 0.54 | 0.12 | 0.34 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.42 | 0.21 | 0.37 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.40 | 0.15 | 0.45 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Memory` | 0.56 | 0.08 | 0.35 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.47 | 0.21 | 0.32 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Memory` | 0.56 | 0.09 | 0.35 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Memory` | 0.62 | 0.08 | 0.30 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.53 | 0.18 | 0.29 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.60 | 0.11 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.41 | 0.19 | 0.40 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Memory` | 0.52 | 0.09 | 0.39 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.32 | 0.08 | 0.59 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.54 | 0.17 | 0.29 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.55 | 0.14 | 0.30 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.20 | 0.31 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.49 | 0.20 | 0.31 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.40 | 0.12 | 0.49 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Memory` | 0.57 | 0.11 | 0.32 | +| `shell_test_023` | 現在のプロセス一覧でklbrが含まれるものを見せて | `Tools` | `Memory` | 0.61 | 0.08 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Memory` | 0.53 | 0.10 | 0.37 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Memory` | 0.53 | 0.08 | 0.39 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.57 | 0.16 | 0.27 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.54 | 0.19 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.56 | 0.15 | 0.29 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.58 | 0.16 | 0.27 | +| `rf_test_020` | read the existing router_report.md from the last benchmark run | `Tools` | `Memory` | 0.56 | 0.10 | 0.35 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.55 | 0.17 | 0.28 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.53 | 0.20 | 0.27 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Memory` | 0.49 | 0.09 | 0.42 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Memory` | 0.55 | 0.09 | 0.36 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.42 | 0.17 | 0.41 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.55 | 0.27 | 0.18 | +| `rf_test_028` | benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて | `Tools` | `Memory` | 0.55 | 0.11 | 0.34 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.50 | 0.11 | 0.39 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.47 | 0.20 | 0.33 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.42 | 0.16 | 0.42 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.56 | 0.17 | 0.27 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.35 | 0.08 | 0.57 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.50 | 0.16 | 0.33 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.42 | 0.14 | 0.44 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.49 | 0.14 | 0.37 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.42 | 0.14 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.50 | 0.18 | 0.32 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.46 | 0.16 | 0.38 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.52 | 0.18 | 0.30 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.46 | 0.21 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.59 | 0.17 | 0.24 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.45 | 0.27 | 0.28 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.50 | 0.23 | 0.26 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.5429` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7778` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.6000 | 0.0000 | 0.0000 | 0.8167 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.5400 | 0.0000 | 0.0000 | 0.6167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-14/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-14/router_model.json new file mode 100644 index 0000000..c7f1171 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-14/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.1537661, + -0.035649825, + -0.08693668, + -0.039169595, + -0.07801271, + 0.022404537, + 0.13668533, + -0.04904432, + -0.084624946, + -0.12592527, + 0.015163703, + -0.18208449, + 0.0055660303, + 0.07221892, + 0.06448449, + -0.1870893, + -0.075359516, + -0.03543284, + -0.008072806, + 0.035137396, + 0.1856608, + -0.047060464, + -0.026945204, + 0.038430814, + -0.027119577, + -0.07783137, + -0.028168011, + -0.04998577, + 0.13851744, + 0.22334306, + -0.11444658, + -0.10880823, + -0.013960583, + -0.002960124, + 0.1886642, + 0.008615434, + 0.06759129, + 0.025011372, + -0.119343616, + 0.033081945, + -0.08350601, + -0.0484712, + -0.074073195, + 0.045808908, + -0.01146472, + 0.10831494, + -0.0040111416, + -0.09824766, + -0.02530775, + -0.06901607, + -0.050421126, + -0.126361, + -0.088383175, + -0.08048365, + -0.001986429, + 0.18562548, + -0.10405873, + 0.02529612, + 0.108271234, + 0.24374901, + -0.011125183, + 0.0132962605, + -0.07309199, + 0.029683625, + -0.08636377, + -0.03715244, + 0.083628885, + 0.005959518, + 0.07466751, + 0.11360453, + -0.041455425, + 0.09902782, + -0.02341183, + 0.05831913, + -0.042435482, + 0.08605521, + 0.15117645, + -0.019223474, + -0.038496863, + -0.12737247, + 0.058691185, + -0.018187318, + -0.10621924, + -0.027555455, + 0.05125111, + 0.16719356, + -0.17868775, + -0.17773148, + -0.08739252, + 0.10273777, + -0.034631487, + -0.0539006, + -0.096600875, + 0.08291881, + -0.15143853, + -0.09585049, + 0.027222762, + 0.007994819, + 0.060303994, + -0.018851208, + -0.016228046, + 0.08862469, + 0.19132261, + -0.019297566, + -0.053104565, + 0.14122458, + -0.10131717, + -0.09569557, + -0.068272986, + 0.000808413, + -0.15115495, + 0.047998995, + 0.039926194, + 0.037234508, + 0.032307126, + -0.075684965, + -0.073772445, + -0.10967773, + -0.09304202, + 0.040335774, + -0.016109457, + -0.17040172, + 0.13639012, + 0.034102615, + -0.09065539, + 0.1110392, + 0.17052025, + 0.16978532, + 0.1330525, + 0.0534311, + 0.10709471, + 0.12975323, + 0.0031307633, + 0.11619781, + -0.20192865, + -0.04178235, + 0.07954132, + 0.047113087, + -0.03501471, + -0.1652226, + -0.09810865, + 0.086311735, + 0.020574726, + 0.026401358, + 0.15123715, + -0.11407882, + -0.0010843852, + -0.032467104, + 0.04454825, + 0.08361806, + -0.06101439, + 0.060647525, + 0.18783487, + -0.12666312, + 0.084901564, + 0.10627836, + -0.047494754, + -0.0030847418, + 0.13515899, + 0.16071515, + 0.009656678, + -0.032863, + 0.27564478, + -0.07924913, + -0.046650242, + 0.075491354, + -0.06973335, + 0.06965276, + -0.001548173, + -0.142573, + 0.00032827802, + 0.014488301, + -0.1260245, + -0.08115438, + -0.044923816, + -0.03615714, + 0.0039115055, + -0.06106897, + -0.041726876, + 0.046486966, + 0.068862185, + -0.0059390427, + -0.20666197, + 0.11289427, + -0.011771635, + -0.06389011, + 0.028167449, + 0.2907871, + 0.01976517, + -0.08579038, + 0.08004743, + 0.16933607, + 0.05288272, + 0.038429778, + 0.03427874, + -0.044819575, + 0.04110185, + -0.13571876, + -0.19524126, + 0.06988574, + 0.073779844, + -0.046239115, + -0.040948626, + -0.084544, + -0.020007487, + -0.092407554, + -0.17292632, + 0.23797427, + -0.1945358, + -0.49743015, + -0.07454593, + -0.1673993, + -0.10651807, + -0.04177226, + -0.05472722, + -0.15188706, + -0.022152808, + -0.13939627, + -0.019187424, + -0.15158167, + -0.19280887, + 0.10705503, + 0.062492274, + 0.16923375, + -0.26784095, + -0.15662816, + 0.025555544, + -0.17363223, + 0.08134375, + 0.06117359, + -0.09012711, + -0.06951844, + 0.061694615, + -0.003797633, + 0.21708345, + 0.12185522, + 0.19993393, + 0.08121296, + 0.012878898, + -0.054898884, + -0.1080671, + 0.17580487, + -0.06257184, + -0.13819437, + 0.07154935, + 0.09852028, + 0.16923477, + -0.005906965, + -0.0075131687, + 0.119097345, + -0.0682252, + -0.030815927, + -0.19546556, + 0.037838846, + 0.00044135028, + 0.11877078, + 0.10327466, + -0.06929159, + 0.020809863, + 0.13948017, + -0.18564449, + 0.21409702, + 0.14669497, + 0.045995284, + 0.010650101, + -0.07551283, + -0.0006766264, + 0.07865089, + -0.11002661, + -0.024456186, + -0.08400181, + -0.13899222, + 0.09306608, + 0.10094795, + 0.07228482, + 0.019025564, + 0.09661008, + 0.13605566, + 0.028701328, + 0.020753454, + 0.0604083, + 0.05287096, + -0.07340115, + 0.009560703, + -0.025192428, + 0.07975566, + -0.28597078, + 0.021488205, + -0.0889583, + 0.12257723, + -0.13694072, + 0.12471303, + 0.061077848, + 0.004631609, + -0.10417203, + 0.22062522, + -0.013706877, + -0.049198896, + 0.00029548293, + 0.075458646, + 0.1765529, + 0.11541535, + 0.008701007, + 0.24391726, + 0.028608661, + 0.06389106, + 0.08642011, + -0.09138413, + -0.10007181, + -0.0695514, + -0.00900465, + -0.03647545, + 0.019589651, + 0.10178171, + 0.17542481, + 0.11783792, + 0.049075227, + -0.10664901, + 0.03252592, + 0.050435152, + -0.008125645, + -0.10447342, + 0.009139559, + 0.1275379, + -0.021951595, + -0.0136193605, + 0.031333257, + -0.15114506, + -0.1765369, + 0.06522931, + 0.06019238, + -0.035919093, + -0.15519899, + -0.01969548, + -0.03430187, + 0.10910889, + -0.0056543187, + -0.10220757, + 0.26145938, + 0.15574658, + 0.049058672, + -0.0064308834, + 0.042335905, + 0.005810607, + 0.038708996, + -0.0057784626, + 0.049044974, + 0.017823728, + 0.087167464, + 0.074433744, + 0.0077878316, + -0.05596981, + 0.0045128716, + -0.049364593, + 0.0015845216, + 0.14348508, + -0.13488854, + -0.1561442, + 0.054857284, + 0.018423049, + -0.034320723, + 0.0040649255, + -0.04828976, + 0.28164595, + 0.047446065, + -0.11510787, + 0.11150978, + -0.04681086, + 0.071485296, + -0.045360688, + -0.009777533, + 0.038308833, + -0.052017767, + 0.03695938, + 0.057656713, + -0.060405314, + -0.29814774, + -0.071221784, + 0.1714612, + -0.07547322, + 0.005367531, + -0.13997321, + -0.16026817, + 0.04131712, + -0.17275667, + -0.029355615, + 0.08490756, + -0.115701295, + 0.0016283661, + 0.20621264, + 0.16213463, + -0.07390191, + -0.19121905, + -0.13373351, + 0.054568663, + 0.11002628, + 0.025603758, + -0.06534632, + 0.057530094, + 0.018174551, + -0.14876643, + 0.035836782, + -0.01137345, + -0.075621784, + 0.07126445, + -0.04957483, + -0.09046384, + -0.067094974, + -0.042324323, + -0.057362486, + 0.1400903, + -0.03241678, + 0.10672305, + -0.19880442, + 0.09854313, + 0.0760169, + -0.091766715, + -0.12132613, + 0.027035655, + -0.3583752, + -0.13381411, + -0.30593142, + -0.0072146636, + 0.12702839, + -0.1360928, + -0.011511107, + -0.045212187, + -0.032329787, + 0.013346648, + -0.016038861, + 0.07367348, + -0.19516303, + 0.09781957, + -0.023986258, + 0.14528868, + 0.08888904, + -0.0009244085, + -0.05984577, + 0.018362086, + -0.0895728, + -0.023944547, + -0.0010550144, + 0.117598146, + -0.02050585, + -0.12902373, + 0.077714995, + -0.012136869, + -0.028286034, + 0.01971836, + 0.23491275, + 0.00014187285, + -0.025502866, + -0.160924, + -0.10722113, + -0.028722614, + -0.11093872, + 0.12761506, + 0.032540575, + -0.028944362, + 0.026266946, + 0.15018588, + 0.065873004, + -0.2041029, + -0.026748825, + -0.15338984, + 0.019006187, + -0.09869299, + -0.10463372, + 0.056366608, + 0.019882223, + 0.024008995, + -0.06550811, + 0.040581826, + -0.09272168, + -0.021980435, + 0.009398623, + -0.055842966, + -0.067572854, + 0.2544045, + 0.2720369, + 0.056282323, + 0.14016761, + 0.1229671, + 0.085792094, + 0.082071744, + 0.015617369, + 0.0025832395, + -0.11643445, + -0.08638553, + -0.052909072, + -0.16713358, + 0.003093768, + 0.07404296, + 0.031308185, + -0.117112026, + -0.08167997, + -0.1499687, + 0.043127168, + 0.14035194, + -0.07871639, + -0.037212748, + -0.0598638, + -0.14055473, + -0.036376033, + 0.11357092, + -0.050674643, + 0.022856299, + 0.057805985, + 0.089862354, + -0.044336755, + -0.08881311, + -0.10180266, + 0.09055733, + -0.04494839, + 0.09060898, + -0.13318132, + 0.032166876, + 0.00562406, + -0.072895855, + 0.008453028, + -0.10890039, + -0.14855209, + -0.009387831, + 0.07252449, + 0.1000593, + -0.1428223, + -0.08014342, + -0.040895056, + -0.039497916, + 0.36770174, + -0.0024050162, + -0.007834549, + 0.1724918, + 0.0037634445, + -0.18319333, + 0.07710567, + -0.19161667, + -0.014408101, + 0.4058824, + -0.06781774, + -0.004210171, + -0.020524722, + 0.03903467, + -0.065584004, + 0.12053713, + 0.07669467, + 0.019268565, + -0.1424542, + 0.07729574, + -0.1408175, + -0.027871037, + -0.07438413, + -0.24066758, + 0.0007071577, + -0.11960982, + 0.079575315, + 0.026175413, + 0.06513677, + -0.07926052, + 0.053233802, + -0.23602884, + 0.009922975, + -0.024497086, + 0.07963983, + -0.03895233, + 0.012983701, + 0.02903441, + 0.09618848, + -0.086873636, + 0.15702735, + 0.051104963, + 0.23331824, + 0.05189599, + -0.012449083, + 0.18062961, + 0.06019235, + 0.056962494, + 0.03108089, + -0.021042427, + 0.0064071524, + -0.088212445, + 0.08572126, + 0.0037502868, + 0.07312031, + 0.12206003, + 0.15648857, + 0.19765435, + -0.08641273, + -0.02523061, + -0.11257553, + -0.07819527, + -0.028061705, + -0.06571461, + -0.015628025, + 0.062462434, + 0.11131009, + -0.0013901697, + 0.058562018, + 0.06027097, + 0.023649702, + 0.15201351, + 0.08222728, + 0.06710522, + 0.06597227, + 0.15405323, + -0.106588826, + -0.048434135, + -0.08093238, + 0.13272074, + -0.104526795, + 0.066636786, + -0.14262883, + 0.11796827, + 0.01689741, + 0.10963646, + -0.043205373, + 0.030428115, + 0.06697563, + 0.18351023, + -0.12014605, + 0.007847034, + -0.21682698, + -0.053551763, + -0.030757116, + -0.058873527, + -0.039070714, + -0.00346026, + 0.09289441, + -0.14382905, + 0.1066636, + 0.06449059, + 0.21688591, + -0.36460355, + 0.15066512, + -0.064523116, + -0.018895637, + -0.14244261, + 0.018194063, + -0.08151443, + 0.008980479, + -0.08436932, + 0.123211086, + -0.16318616, + 0.11563229, + -0.06480912, + -0.005352673, + 0.068340674, + -0.07209151, + 0.0028943075, + 0.0013212605, + 0.11254483, + 0.098231606, + -0.082876496, + -0.018098809, + -0.019039044, + -0.10377387, + -0.055182043, + -0.045605227, + -0.06043637, + 0.19120796, + 0.10237104, + 0.07438044, + 0.13820165, + 0.17585173, + 0.27409434, + -0.07378421, + 0.025601309, + -0.12231862, + 0.12022717, + -0.101080395, + 0.032167993, + -0.08084088, + -0.17208248, + -0.1681819, + -0.03261707, + 0.02148205, + -0.075628266, + 0.024750838, + 0.03915467, + -0.13104486, + 0.034098193, + -0.034001082, + 0.08776243, + -0.049719255, + -0.06473143, + 0.10588895, + 0.08131149, + -0.052911863, + -0.054036725, + -0.035009447, + 0.08870277, + 0.07426314, + 0.040366404, + 0.077888794, + 0.021348773, + 0.226514, + 0.15032294, + 0.16204213, + 0.000056462246, + 0.09213504, + 0.19880146, + 0.08437888, + 0.021916471, + 0.08471564, + 0.03781062, + -0.066586, + -0.046965122, + -0.10133214, + 0.08557293, + -0.14368777, + -0.06882834, + -0.12171229, + 0.0055409498, + -0.09227392, + -0.10453962, + -0.07994333, + 0.097816125, + 0.0027960786, + -0.11214976, + 0.05112316, + -0.040688016, + 0.068019, + -0.25952643, + 0.16106968, + 0.17121106, + 0.25099227, + 0.076975055, + -0.2097073, + -0.023342844, + 0.015645755, + 0.12014807, + 0.07335009, + -0.05761358, + -0.08484111, + -0.000030331716, + 0.15092224, + 0.06777547, + -0.08367785, + 0.048469383, + -0.11079949, + 0.003651169, + -0.07147883, + -0.014249957, + -0.24292563, + -0.15647426, + -0.17459014, + 0.010819486, + -0.105703846, + -0.030978357, + 0.06532192, + 0.08745108, + -0.053935952, + 0.0688913, + 0.12390802, + 0.21340448, + -0.014140771, + 0.27248612, + -0.2464558, + 0.05955756, + 0.104456566, + 0.009404783, + -0.08957156, + -0.13419013, + -0.026675185, + -0.1552804, + -0.008024584, + 0.005712407, + -0.124870405, + -0.2592146, + 0.19014545, + 0.0327858, + -0.1229455, + 0.010203828, + 0.037715424, + -0.0053597577, + 0.05868922, + 0.010626316, + 0.019355819, + 0.08497353, + 0.119429246, + 0.004582282, + -0.06729474, + 0.07995998, + -0.04374404, + -0.015687706, + -0.053967472, + -0.08461543, + -0.0808064, + -0.07503357, + -0.03532297, + 0.0011162171, + -0.11756368, + 0.11231443, + 0.0066749663, + 0.087183446, + 0.033798646, + 0.17816873, + 0.16899641, + 0.032384265, + 0.037937198, + -0.1306509, + -0.08966298, + -0.11874975, + 0.04818552, + -0.08112427, + -0.08979903, + 0.016811714, + -0.12227783, + 0.006724136, + 0.07726029, + 0.072018504, + 0.047206745, + -0.020475753, + 0.21668829, + 0.06144242, + -0.010373841, + 0.08095411, + -0.00984799, + -0.18237619, + 0.08362384, + -0.06917313, + 0.003942984, + 0.08102712, + -0.08224079, + 0.026713282, + -0.021954091, + -0.21172519, + 0.061612993, + 0.11611484, + 0.15850388, + -0.07526881, + -0.011174921, + 0.12953082, + -0.04849906, + 0.08364964, + 0.045158945, + 0.10626501, + 0.119382195, + -0.20448637, + -0.033443533, + -0.044285737, + -0.078928396, + -0.027338818, + -0.12594555, + 0.047167778, + -0.03249266, + -0.050701614, + 0.083851635, + 0.06785768, + -0.038283225, + -0.02997082, + -0.10361641, + 0.040058605, + 0.013622663, + -0.029170217, + -0.039008163, + 0.08126564, + 0.023967572, + -0.040728178, + 0.020921214, + -0.2012302, + 0.029383505, + 0.039341018, + 0.07352988, + 0.149539, + -0.18081951, + 0.042585555, + -0.07393927, + -0.0520452, + -0.055225402, + -0.02532332, + 0.07143403, + 0.14533009, + 0.020230262, + 0.06286708, + -0.016043179, + -0.0058425125, + -0.1894284, + 0.081516296, + 0.01061884, + 0.08930471, + -0.070286885, + 0.12981166, + 0.01348248, + 0.043093804, + 0.11957937, + 0.022259962, + -0.032027576, + 0.025988672, + -0.16140743, + 0.07117117, + -0.027395064, + -0.059419967, + -0.0067737456, + -0.09964316, + 0.049351126, + 0.21752295, + 0.046031915, + -0.024564896, + -0.011924736, + -0.059805218, + 0.02614514, + -0.015359368, + -0.02796093, + -0.012214168, + 0.0056245904, + -0.032312065, + 0.109036505, + 0.07137153, + 0.108339995, + 0.052375022, + -0.017451666, + -0.14530696, + -0.11981482, + 0.079305254, + -0.0025598651, + -0.05310435, + 0.11399684, + 0.03127478, + 0.0064025996, + -0.03253582, + 0.089398734, + -0.18456937, + -0.0016192505, + 0.30008864, + -0.18448582, + -0.11759908, + -0.15413254, + 0.018025307, + 0.025112813, + -0.0009730928, + -0.2881156, + 0.12879921, + 0.13925475, + -0.09416232, + 0.0006413473, + -0.13858742, + 0.06543319, + -0.08643667, + -0.07211853, + 0.052677996, + 0.1292809, + -0.041060735, + -0.010105186, + 0.017167171, + 0.10278289, + -0.02560506, + 0.01444451, + -0.24326459, + 0.087474845, + -0.029556967, + -0.018887402, + 0.0519785, + 0.07257978, + -0.06718943, + -0.06401816, + 0.009868641, + 0.29014003, + 0.014569991, + -0.13224676, + -0.021046856, + 0.008031937, + 0.029677179, + -0.12090361, + -0.030684829, + -0.12032807, + 0.14864889, + 0.0730822, + 0.04875755, + -0.053761955, + -0.013974497, + -0.03588524, + 0.14112757, + 0.20620118, + 0.0026121754, + 0.057062473, + 0.006303307, + -0.100188285, + 0.030305644, + 0.030009354, + 0.08998002, + -0.0975479, + 0.17385758, + -0.14035688, + -0.12630624, + 0.20372146, + -0.031224119, + -0.015600041, + -0.000572721, + -0.12464091, + 0.028028218, + -0.08736926, + -0.064607315, + 0.0074855303, + -0.003159628, + 0.06802779, + -0.1313798, + 0.06092782, + -0.1787316, + 0.09524019, + 0.2142574, + 0.07154194, + -0.0078775445, + -0.08554093, + 0.17139089, + -0.00896565, + 0.20121369, + -0.08606445, + 0.10155439, + 0.120717004, + -0.05526951, + -0.25100285, + -0.110797964, + -0.06828172, + -0.07158169, + 0.12045773, + -0.19262262, + -0.03129522, + -0.17944223, + 0.02532566, + 0.09656396, + -0.091062434, + -0.0006519022, + 0.11245893 + ], + [ + 0.08650064, + -0.114138156, + 0.11560147, + 0.05716735, + 0.07363487, + 0.055551223, + 0.042977456, + 0.13866937, + 0.07197206, + 0.057236373, + -0.04786727, + 0.078654975, + 0.020689813, + -0.026271248, + -0.02856729, + 0.048773885, + 0.13842425, + -0.06279018, + 0.12849218, + 0.16308266, + -0.09831704, + -0.01208826, + 0.031379998, + 0.074512295, + -0.14470626, + 0.022406168, + -0.0015860604, + 0.08554748, + -0.13143538, + -0.0886885, + -0.05255464, + 0.30100065, + 0.030713795, + 0.030090688, + -0.0038725059, + -0.036318548, + -0.028928364, + 0.000039066, + 0.098410994, + -0.28426936, + -0.05369149, + 0.11395493, + -0.0016404965, + 0.02959418, + 0.1296119, + -0.08816001, + 0.12523289, + -0.046427406, + -0.024127232, + 0.10684162, + -0.050801836, + -0.036207974, + -0.09595485, + 0.078260094, + 0.06545781, + -0.14052771, + 0.096398816, + -0.020581724, + 0.1009272, + 0.016509471, + 0.032654297, + -0.10522052, + 0.07001931, + -0.052200574, + 0.11428169, + -0.16297819, + -0.022918481, + 0.008518012, + -0.052359655, + -0.15543026, + -0.030120252, + -0.21617754, + 0.057570234, + 0.012278141, + 0.0100514935, + -0.008567871, + 0.03985517, + -0.0071163354, + 0.14116311, + -0.043021444, + 0.046327017, + 0.06556431, + 0.19307518, + 0.048381023, + -0.054358784, + -0.14242582, + 0.052703325, + -0.016104024, + 0.11762951, + -0.27093124, + 0.071561694, + 0.14363705, + -0.10874558, + -0.042457294, + -0.045092538, + 0.073277965, + 0.081697024, + -0.005809052, + -0.10392095, + 0.055779688, + 0.040752683, + 0.023003666, + -0.07039203, + -0.098758176, + 0.11601843, + -0.09822991, + -0.014778384, + 0.019168867, + -0.08343514, + 0.013522634, + 0.018998768, + -0.08619179, + 0.05457085, + 0.008922722, + -0.15973733, + 0.021987697, + 0.04509556, + 0.12129829, + 0.20903002, + 0.11825309, + -0.009349105, + 0.0066330372, + 0.07822109, + 0.09448989, + 0.04427069, + -0.05089293, + -0.0634784, + -0.29010263, + -0.1817537, + -0.10901791, + -0.07024844, + -0.09644515, + 0.04460636, + 0.025619725, + 0.054481138, + -0.017032826, + -0.0024525498, + 0.0051498828, + -0.16207719, + 0.13890503, + 0.06168882, + -0.14186028, + -0.017634777, + 0.035919037, + -0.13157438, + 0.2913547, + -0.048506882, + 0.061833624, + -0.049157888, + -0.029682238, + -0.018465182, + -0.14546509, + -0.08058027, + 0.049560603, + 0.13289733, + -0.124585554, + 0.106291436, + 0.18051346, + -0.032141753, + 0.0047566677, + -0.096933044, + 0.21418223, + -0.12096488, + -0.014764312, + 0.047446866, + 0.03750072, + 0.020264208, + 0.18554664, + -0.1021362, + 0.013693483, + -0.05330727, + 0.021761887, + 0.099549085, + 0.0121648405, + 0.018702097, + -0.09758032, + 0.15901814, + -0.065417156, + 0.057077993, + 0.00047911698, + -0.04200102, + -0.075910345, + 0.16953775, + -0.023566708, + 0.058020957, + -0.1151307, + 0.065792955, + -0.29199713, + -0.019268267, + 0.024025127, + 0.052528866, + -0.10454245, + 0.028018963, + 0.00668617, + -0.0009638491, + 0.15003029, + -0.0796975, + 0.08568932, + 0.14248814, + -0.047471944, + 0.154569, + 0.105354466, + -0.09005179, + 0.0016345638, + -0.19058277, + 0.1462792, + -0.088306405, + -0.19980139, + 0.07397957, + 0.2989383, + -0.031951997, + 0.10095756, + 0.11718432, + 0.0045800325, + 0.09298005, + -0.074017435, + -0.043252986, + 0.075847, + 0.009191615, + 0.09819291, + 0.010901586, + -0.12441394, + -0.15651459, + -0.36471352, + 0.107327834, + 0.2367279, + -0.007892925, + -0.13905, + -0.114933364, + -0.05437099, + 0.030067027, + 0.099514276, + -0.037281614, + -0.037492216, + -0.1981845, + -0.1384136, + 0.08877338, + -0.13132341, + 0.00768169, + -0.047017664, + 0.03620687, + -0.099904016, + 0.019379376, + 0.0981703, + 0.011796661, + -0.09512934, + -0.18267427, + -0.13936168, + 0.06484512, + -0.030278102, + 0.090696916, + 0.12653728, + 0.037380584, + -0.15862349, + 0.14395134, + -0.021009054, + 0.054963164, + 0.046817254, + 0.06660228, + -0.035252098, + 0.090558656, + -0.09402542, + -0.056013796, + -0.061302137, + 0.031734105, + 0.015802667, + -0.16472177, + -0.101517014, + 0.048385564, + 0.16827406, + -0.0048398217, + 0.064329885, + 0.060761847, + 0.08469305, + 0.10910126, + -0.043873362, + -0.089271344, + -0.055935968, + 0.034083303, + -0.013393669, + 0.03643514, + -0.120853394, + 0.0704519, + 0.0065425616, + -0.04119151, + -0.09728708, + 0.05142146, + -0.102816895, + -0.016405446, + 0.10037102, + 0.114227235, + 0.020892173, + 0.0033101249, + 0.09104857, + -0.12247543, + -0.2576225, + -0.1403052, + 0.11442854, + -0.099400446, + 0.00058794674, + -0.19586149, + -0.113684855, + 0.030646956, + -0.11775976, + -0.06694089, + 0.086007595, + 0.09736035, + -0.13316418, + 0.023876686, + 0.06294222, + 0.060473986, + -0.0043298365, + -0.03389869, + -0.0055077346, + 0.03175626, + 0.03684258, + 0.09002136, + 0.12259016, + 0.017571814, + -0.27465117, + -0.033611458, + 0.08492853, + -0.064178616, + -0.09749589, + 0.08281812, + 0.074508965, + -0.08160397, + 0.1836253, + 0.12195985, + -0.047170453, + 0.11666565, + 0.00809368, + -0.1302758, + 0.21874562, + 0.08840849, + -0.09259481, + -0.015606064, + -0.030824287, + -0.2201201, + -0.05294996, + -0.025827821, + 0.028233208, + -0.032888405, + -0.011495582, + -0.11593533, + 0.0070121544, + 0.0074381963, + -0.0030233257, + 0.06750082, + -0.015932027, + 0.06266208, + 0.090228274, + -0.007783951, + 0.09262657, + -0.04413849, + -0.258896, + -0.08814282, + 0.19863154, + 0.21488272, + -0.047835387, + 0.12803216, + -0.017444558, + 0.075741306, + -0.19025156, + 0.027734958, + 0.03320167, + -0.123028815, + -0.028187469, + 0.013636808, + 0.06271145, + 0.18612179, + -0.29361016, + 0.10948807, + 0.027112115, + -0.031513385, + 0.028932175, + 0.25757912, + 0.047562208, + -0.1354802, + 0.08021137, + -0.27474785, + 0.032081064, + 0.03791089, + 0.025726352, + 0.069445595, + 0.0021079297, + -0.02449264, + 0.011238894, + -0.02319162, + -0.026542418, + -0.18316846, + 0.10666311, + 0.19345199, + -0.06720772, + 0.00815098, + -0.049136266, + -0.14031912, + 0.15578413, + 0.008095201, + 0.01659755, + 0.18123667, + -0.17131653, + 0.063718066, + 0.07473296, + -0.027271748, + -0.058085024, + -0.100015745, + -0.001066607, + 0.025115151, + 0.086855374, + 0.03166735, + 0.024315033, + 0.007213102, + 0.054682832, + 0.04690719, + -0.17995097, + -0.004744909, + -0.05241575, + 0.12825188, + 0.056677263, + -0.08422418, + 0.08251111, + 0.009535376, + 0.19880669, + 0.2963894, + -0.07653299, + 0.1401372, + 0.019019231, + -0.09808532, + 0.026957024, + -0.17146498, + 0.026395286, + -0.096108206, + -0.11671051, + -0.0031023317, + 0.030857945, + 0.00259537, + 0.17649667, + -0.009633149, + 0.09164389, + -0.11770125, + 0.035321847, + -0.25525537, + 0.2554045, + -0.042877495, + -0.19594881, + 0.1396774, + 0.012059957, + -0.00449406, + -0.19431978, + -0.088604175, + 0.015468876, + -0.13193911, + 0.10843306, + 0.23043013, + 0.06712104, + -0.07877263, + 0.03013501, + 0.22939627, + 0.047045074, + -0.083643205, + -0.066756636, + 0.15640521, + 0.14078821, + -0.011661555, + -0.07418068, + -0.0354833, + 0.025781255, + 0.06630863, + 0.09891263, + -0.034469504, + 0.028824922, + -0.056631226, + 0.08970828, + 0.09523603, + 0.050132398, + -0.1690745, + 0.046013247, + 0.013893757, + -0.12068828, + -0.048534885, + 0.07168909, + -0.037411164, + -0.008103455, + 0.006886762, + -0.19633815, + -0.17443606, + 0.045308072, + 0.09340842, + 0.1490397, + -0.030604543, + 0.02490058, + -0.051153444, + -0.04850469, + -0.051614046, + 0.07135396, + 0.04567157, + 0.0073676747, + -0.15031311, + 0.16379681, + 0.0757311, + -0.1616459, + -0.1270838, + 0.023255069, + -0.1826222, + 0.15810522, + -0.044493705, + -0.22102857, + 0.045563675, + -0.008063402, + 0.05133512, + 0.12316835, + 0.10462279, + -0.2890721, + -0.017548595, + -0.11253995, + 0.10683944, + 0.032772627, + 0.011404846, + -0.031123182, + -0.008722479, + 0.1220427, + -0.23846045, + -0.05996672, + -0.05013295, + -0.029948775, + 0.060149055, + -0.13687938, + 0.054654896, + -0.0076787327, + -0.08335814, + -0.11796455, + -0.101908155, + -0.050144844, + 0.21571128, + -0.06343264, + 0.107729964, + -0.013642982, + -0.40179682, + -0.013399297, + 0.02881357, + -0.018139042, + 0.013398127, + -0.031053284, + -0.06259348, + -0.10992421, + 0.03368614, + 0.1299659, + -0.14629827, + -0.17959803, + -0.02231733, + 0.14280543, + 0.06939315, + 0.08924989, + 0.06705628, + 0.071522295, + -0.029402977, + 0.059425004, + 0.11423234, + 0.09920461, + 0.1667185, + 0.077411644, + -0.061828777, + -0.07963011, + 0.10044846, + -0.029872157, + -0.1763701, + 0.110166654, + 0.17331573, + -0.14678414, + -0.13495205, + -0.015179527, + -0.105202034, + 0.03376242, + 0.045832768, + 0.050930824, + 0.113197684, + 0.022861745, + -0.081894, + 0.0609409, + 0.07157385, + -0.030135512, + -0.13592605, + -0.17098251, + -0.037453867, + 0.0445514, + 0.005111366, + 0.22748575, + -0.031975, + 0.17256372, + 0.15765712, + 0.2115168, + -0.01236721, + -0.010223999, + -0.2236333, + -0.2040943, + 0.008426578, + 0.037784062, + -0.21303326, + 0.0014444012, + 0.10017615, + -0.03439675, + -0.03444695, + 0.07566503, + 0.11491664, + 0.14344919, + 0.03547391, + 0.1023221, + -0.26562563, + 0.03324227, + -0.12612627, + -0.025097914, + -0.16231543, + -0.07071355, + -0.031565484, + 0.05040618, + 0.019868962, + 0.12765361, + -0.11720381, + 0.042901862, + 0.04383294, + 0.048617877, + 0.046922315, + -0.03069157, + 0.014754203, + -0.046216488, + -0.09432203, + -0.040246278, + -0.026649779, + -0.072701216, + -0.15752329, + -0.36157733, + 0.040658437, + -0.093095146, + -0.08237973, + 0.13206859, + 0.21616566, + 0.13615578, + -0.052190773, + -0.031580392, + 0.100822076, + -0.012105965, + 0.28700614, + -0.0798311, + -0.030761806, + 0.031985268, + 0.06104218, + 0.017198546, + -0.008219104, + -0.006477159, + -0.030536303, + -0.07336686, + 0.0806302, + -0.015533479, + -0.0037378035, + 0.057539888, + -0.015810573, + 0.0075652716, + -0.058744423, + -0.26509422, + -0.07290028, + -0.042361405, + -0.10962765, + -0.2524032, + -0.13754942, + -0.03327559, + 0.028698724, + -0.032883186, + -0.119565666, + -0.1226996, + 0.05682807, + -0.048052542, + 0.37282622, + 0.028903697, + 0.035287928, + 0.02152677, + 0.23741001, + 0.06756465, + 0.122916296, + -0.07250478, + -0.04318548, + 0.1850775, + -0.01884087, + 0.08287051, + 0.020896921, + 0.010066692, + -0.03891263, + 0.18874517, + 0.010514654, + -0.011000999, + -0.08137152, + -0.035857525, + -0.12775403, + 0.02271913, + -0.09061829, + -0.007581237, + -0.10853437, + 0.03644659, + -0.04371966, + -0.20594738, + -0.10544621, + -0.008287322, + -0.15238318, + -0.08166857, + -0.085347965, + 0.1146354, + 0.079319455, + 0.011942096, + -0.07547264, + 0.037925895, + 0.04891033, + 0.09160054, + 0.03913848, + 0.10733035, + 0.12855045, + -0.008485813, + -0.09220337, + 0.13284315, + -0.068138555, + -0.06429005, + 0.30747658, + 0.017451087, + -0.1090665, + 0.033579294, + -0.22076453, + 0.12192298, + 0.14958231, + -0.13429965, + -0.009634048, + -0.14999968, + -0.00200675, + -0.12121864, + -0.013977665, + 0.048683085, + -0.024490103, + 0.0498644, + -0.011909416, + 0.0015371294, + 0.037581872, + 0.07797013, + -0.06255501, + -0.17288956, + -0.07691344, + 0.37672377, + -0.026362475, + 0.06730378, + 0.0104083195, + -0.024101874, + 0.14750732, + -0.02757014, + -0.1706281, + -0.0031450412, + -0.0055671176, + -0.1802068, + 0.12573895, + -0.032311182, + -0.2111977, + 0.064462945, + -0.057173625, + 0.035591144, + 0.1605281, + 0.07954016, + -0.026228633, + 0.18851891, + 0.13628644, + 0.033962764, + -0.008227327, + 0.103921, + 0.386116, + -0.13597281, + -0.12757441, + 0.043079887, + -0.09839026, + -0.009575337, + -0.005930024, + 0.11509768, + 0.113127336, + -0.063924, + -0.16976811, + -0.026487622, + 0.16157718, + 0.039270367, + 0.013407921, + 0.05706089, + -0.024494743, + -0.032472625, + 0.20470479, + 0.062012397, + 0.013509699, + 0.10476464, + 0.02340273, + 0.22813578, + -0.17417786, + 0.05276678, + -0.2060439, + -0.003265173, + -0.16086264, + -0.23557913, + 0.0522266, + -0.173802, + 0.0074773612, + -0.028467707, + 0.20351472, + -0.057790864, + 0.08757821, + -0.06914482, + -0.14491978, + 0.09239386, + 0.020376774, + 0.1097607, + -0.088697076, + 0.16945688, + 0.01119273, + -0.1528324, + -0.173479, + -0.026488239, + -0.029663468, + -0.037529018, + 0.08184811, + -0.116736144, + 0.020757439, + 0.037805155, + 0.081116766, + -0.08899582, + -0.23994243, + 0.00085073564, + 0.015741583, + 0.14916396, + -0.049735896, + 0.06661528, + -0.05483578, + 0.05611641, + 0.07030592, + -0.14783034, + 0.07137445, + -0.106677264, + 0.060104955, + -0.10194723, + 0.14962104, + 0.028547723, + 0.078555, + -0.09768789, + 0.11221817, + 0.0854506, + -0.15803969, + 0.009801034, + 0.078489214, + -0.0693264, + -0.1491822, + -0.09020748, + -0.03234147, + 0.044914614, + 0.02052608, + -0.1493524, + 0.17103215, + -0.063729316, + -0.10966809, + -0.029281124, + 0.09548159, + -0.048648804, + 0.23593551, + -0.012712764, + 0.07028848, + -0.061728034, + 0.05049216, + -0.021263314, + -0.06874739, + 0.07710166, + 0.08121541, + 0.0734596, + -0.023434158, + 0.16007483, + -0.10867535, + -0.11891655, + 0.22188166, + 0.06489395, + 0.010524853, + 0.19451497, + -0.053407338, + -0.08928299, + -0.14070098, + 0.20396641, + -0.062143497, + 0.125486, + 0.12106229, + -0.1373563, + -0.12767154, + 0.05849606, + -0.1128916, + 0.040035207, + -0.036183015, + -0.13515732, + 0.056579936, + -0.15353481, + 0.17633565, + 0.17068711, + -0.031218339, + -0.06275934, + 0.045422144, + 0.020727132, + -0.015841577, + 0.028296215, + -0.1515143, + 0.05359048, + -0.13335627, + -0.102366865, + 0.09058614, + -0.16648483, + 0.021478737, + -0.02842722, + 0.01834001, + 0.029639145, + -0.023922393, + 0.026784806, + -0.1001365, + -0.042642847, + -0.03436985, + -0.097866334, + 0.019808806, + -0.042747162, + 0.14128022, + -0.033543553, + 0.08283731, + 0.07131058, + -0.15540025, + 0.28758714, + 0.0058351206, + 0.15492027, + 0.013075218, + 0.08945752, + 0.07358258, + 0.054398827, + -0.02437751, + -0.07341751, + 0.10022484, + -0.050757684, + 0.1440264, + 0.013197424, + -0.109852426, + 0.22719486, + 0.080075815, + 0.00024181453, + -0.00097611186, + -0.04847994, + -0.07244215, + 0.006288356, + 0.030367048, + 0.12748545, + 0.1944054, + -0.060363494, + -0.0036989984, + 0.06789716, + 0.03632896, + 0.026532803, + -0.06583803, + -0.13583247, + 0.035222605, + -0.08362427, + -0.18340993, + -0.19657335, + -0.0031432216, + 0.023255847, + -0.18112065, + 0.12979667, + 0.04521475, + -0.037557703, + 0.0030770968, + 0.2226345, + 0.14577474, + 0.0025226166, + -0.023376113, + -0.054195154, + -0.05671215, + -0.11162617, + -0.06985026, + -0.028117025, + 0.0053079883, + 0.061277986, + -0.14143746, + 0.23066823, + 0.19300221, + 0.041472983, + -0.18196483, + 0.05748335, + 0.073920846, + -0.11640508, + -0.11107834, + -0.18051128, + 0.009662717, + 0.13686214, + -0.022840913, + 0.08510747, + 0.012901924, + -0.10250301, + 0.0596237, + -0.18144847, + 0.105939075, + 0.07167126, + 0.14054418, + -0.18191113, + -0.1436565, + -0.0597559, + 0.030895803, + -0.030637655, + -0.13535596, + 0.08015554, + -0.07608762, + -0.040301584, + -0.19534457, + -0.2292024, + -0.04393903, + 0.038195375, + 0.099135764, + -0.021666853, + -0.029471708, + -0.009995175, + 0.25933316, + 0.032328885, + 0.09000577, + 0.03263837, + 0.079661906, + -0.0012039868, + 0.11770895, + -0.0007853804 + ], + [ + 0.067265354, + 0.14978805, + -0.02866475, + -0.017997744, + 0.0043778527, + -0.077955745, + -0.17966285, + -0.08962502, + 0.012652886, + 0.068688974, + 0.032703526, + 0.103429645, + -0.026255865, + -0.04594767, + -0.03591724, + 0.13831529, + -0.063064694, + 0.098223, + -0.120419286, + -0.19822003, + -0.087343685, + 0.05914871, + -0.004434789, + -0.1129431, + 0.17182596, + 0.055425245, + 0.029754065, + -0.03556171, + -0.0070821294, + -0.13465452, + 0.16700113, + -0.19219238, + -0.01675321, + -0.027130553, + -0.18479164, + 0.027703132, + -0.038662884, + -0.025050428, + 0.020932555, + 0.25118732, + 0.13719758, + -0.065483645, + 0.07571365, + -0.07540309, + -0.11814715, + -0.020154916, + -0.1212217, + 0.14467515, + 0.049435012, + -0.03782557, + 0.10122299, + 0.16256903, + 0.18433803, + 0.002223497, + -0.06347142, + -0.045097824, + 0.0076598977, + -0.0047144056, + -0.20919853, + -0.26025832, + -0.021529123, + 0.09192429, + 0.0030727165, + 0.022516953, + -0.02791795, + 0.20013073, + -0.060710344, + -0.014477534, + -0.022307828, + 0.04182581, + 0.07157571, + 0.11714966, + -0.034158412, + -0.07059728, + 0.03238398, + -0.077487394, + -0.19103165, + 0.026339812, + -0.10266619, + 0.17039405, + -0.10501824, + -0.047377028, + -0.086855955, + -0.020825548, + 0.0031076584, + -0.024767814, + 0.12598439, + 0.19383541, + -0.030236963, + 0.16819346, + -0.036930185, + -0.08973632, + 0.20534639, + -0.04046148, + 0.19653097, + 0.022572542, + -0.10891978, + -0.0021857636, + 0.043617047, + -0.03692846, + -0.02452463, + -0.1116284, + -0.120930575, + 0.11805574, + -0.062913835, + -0.04299464, + 0.11609556, + 0.07652675, + 0.15170819, + -0.0143310465, + 0.13215616, + 0.038192805, + -0.09449705, + -0.04615726, + 0.12743029, + 0.053697243, + 0.028676936, + -0.011620548, + -0.115987964, + -0.15858902, + 0.025458544, + 0.16376868, + -0.21461114, + -0.12859254, + 0.046384696, + -0.060146254, + -0.10704182, + 0.1203173, + 0.04870109, + 0.055586778, + -0.036846314, + -0.033308074, + -0.04773714, + -0.1418176, + 0.14744744, + 0.058815204, + -0.077088796, + -0.052262966, + 0.19709195, + 0.026317546, + 0.036419798, + 0.05554856, + -0.0029399341, + -0.062320363, + -0.01966279, + -0.17727584, + 0.04959125, + -0.029366525, + 0.004609626, + -0.053935826, + 0.07947955, + 0.08481755, + -0.10725464, + 0.07710253, + -0.2177989, + 0.018307095, + -0.058796752, + -0.17742877, + -0.103017226, + -0.16547178, + 0.08727635, + -0.18131922, + -0.15467994, + 0.094013415, + -0.0007966098, + -0.11299204, + 0.049469113, + -0.2551994, + 0.10368437, + 0.12887946, + 0.052979, + -0.036250193, + 0.026475381, + 0.068989575, + 0.026221694, + 0.13373744, + -0.16292971, + 0.1264861, + -0.015351114, + -0.046966083, + -0.026861196, + 0.08184943, + 0.03712424, + -0.0893275, + -0.04624934, + 0.17902076, + -0.09396041, + 0.0012100477, + -0.0004969027, + 0.061765272, + -0.13257642, + -0.06479358, + -0.08090168, + -0.045115944, + -0.033314876, + -0.10521074, + 0.038595706, + 0.05002954, + 0.05275309, + -0.022413827, + -0.228349, + -0.05911533, + 0.13100033, + 0.08290936, + 0.21059032, + -0.053871688, + 0.2612328, + -0.03817294, + 0.12055628, + 0.19849202, + 0.1064979, + 0.06644159, + -0.010666335, + 0.037192244, + -0.03825279, + 0.22590448, + 0.06540578, + 0.0635492, + 0.009995832, + 0.053388823, + 0.1819073, + 0.017358864, + 0.0940223, + 0.19547985, + 0.16051306, + -0.08009981, + -0.01766264, + 0.31268245, + 0.03358963, + -0.006802613, + 0.060060095, + -0.02999579, + -0.02441298, + 0.041289836, + -0.018898971, + 0.016558355, + -0.2887073, + 0.05011042, + -0.020560574, + 0.10191658, + 0.07186017, + -0.07590083, + 0.043192476, + 0.04002405, + -0.08334605, + -0.0033909143, + 0.013439505, + 0.1452686, + -0.05733201, + -0.088819325, + -0.022471705, + -0.095721394, + 0.15808508, + 0.120784655, + -0.14439268, + -0.097761706, + -0.15823784, + 0.02247433, + -0.08741217, + -0.10422804, + 0.095085815, + -0.12007163, + -0.090681195, + 0.015306839, + -0.042384245, + 0.059710126, + 0.16539843, + 0.022866111, + 0.061641064, + -0.14381783, + 0.08884166, + 0.07466225, + -0.15382792, + -0.185641, + -0.1813861, + 0.02484775, + -0.0073386994, + -0.0801197, + -0.06278462, + -0.0073598013, + -0.09684346, + 0.06798241, + 0.0029492439, + -0.016103258, + 0.06638396, + 0.01753141, + 0.23454934, + 0.08132862, + 0.10536375, + -0.2229483, + 0.022713436, + -0.14560527, + -0.064387955, + -0.095680125, + 0.22664733, + 0.036997385, + 0.15401214, + -0.0652297, + 0.09910494, + -0.07604658, + 0.019308623, + -0.0017304748, + -0.039347965, + -0.12615763, + 0.038332216, + -0.1498985, + -0.18378036, + 0.22454844, + 0.076195106, + 0.006609157, + -0.051469374, + 0.04080529, + 0.014309058, + -0.096273944, + -0.207181, + -0.15468046, + -0.13909651, + -0.015941054, + -0.050097745, + 0.22421612, + 0.041737087, + 0.01954481, + 0.05503906, + -0.030042086, + -0.060866546, + -0.060889628, + 0.050270688, + -0.032480314, + 0.05457713, + -0.018058803, + -0.17685808, + 0.027825413, + 0.2854748, + -0.1990501, + -0.05410666, + -0.016514089, + 0.021260383, + 0.13303187, + -0.04133936, + -0.102796614, + -0.023230858, + -0.021802323, + -0.009447499, + 0.0056849695, + 0.077226385, + -0.001233707, + -0.056483172, + -0.0148004005, + -0.15466832, + -0.058501728, + -0.07044992, + -0.034258507, + 0.0032710754, + -0.043261908, + 0.042553976, + 0.11541086, + 0.22303134, + -0.042487398, + -0.2697398, + 0.029412314, + -0.093711354, + 0.013379649, + -0.02745153, + -0.09139424, + -0.07518103, + 0.081906185, + 0.01151903, + 0.07499832, + -0.08512211, + -0.017350763, + -0.17634419, + 0.25530154, + -0.05747039, + -0.06407152, + -0.026143344, + 0.031473134, + 0.040568653, + 0.02365954, + -0.03598118, + -0.004738165, + 0.26938042, + 0.107892066, + 0.12235724, + -0.0670435, + 0.103311166, + 0.027247678, + -0.060414925, + 0.10446229, + 0.021563247, + -0.17967018, + 0.021033838, + -0.032761253, + -0.002232972, + 0.20094131, + -0.06271966, + -0.06088999, + 0.114715405, + -0.09043788, + -0.065625295, + -0.034772128, + -0.03247024, + 0.13547981, + -0.05234461, + 0.0008887822, + -0.04399269, + 0.1076599, + 0.19047959, + 0.06816161, + 0.017209189, + -0.02949286, + -0.17175756, + 0.008101737, + -0.11393619, + 0.14412154, + -0.14545026, + 0.10393415, + 0.09651161, + 0.17374185, + -0.15528761, + 0.30169812, + 0.21803834, + 0.22342007, + -0.0023207148, + -0.32583496, + -0.1602966, + 0.08804405, + -0.09492502, + 0.013310529, + 0.08473863, + -0.010918157, + 0.09779154, + 0.1687677, + -0.0017114308, + 0.14069672, + -0.14218633, + -0.119746976, + -0.0016709599, + -0.116650805, + -0.0087289335, + -0.0020710814, + 0.14164573, + -0.03426684, + 0.13765714, + -0.23489869, + 0.17190114, + 0.11823379, + -0.1275406, + 0.016226083, + -0.015224294, + -0.040592875, + 0.088462226, + 0.0100339865, + 0.29286325, + -0.0012119255, + -0.20170744, + 0.043817673, + -0.048842527, + -0.06267558, + -0.20045178, + -0.07331201, + -0.0665427, + 0.0008836231, + 0.047697656, + -0.114039324, + 0.16505136, + 0.055174492, + 0.1341762, + 0.07885245, + -0.12267522, + -0.118794784, + 0.010460504, + 0.036683165, + 0.016049402, + 0.0030134192, + -0.07325558, + -0.059531037, + 0.22491747, + 0.021559626, + -0.2682984, + -0.1513486, + -0.007747446, + -0.21185677, + -0.085555956, + -0.07768858, + -0.0889585, + 0.18072057, + 0.17185259, + 0.071126275, + -0.0070228805, + -0.09613079, + 0.19773807, + -0.027994337, + -0.022889512, + 0.017196476, + 0.16872603, + 0.010325995, + 0.10429709, + -0.050494857, + 0.009961071, + -0.08508044, + -0.03851841, + 0.22150967, + 0.26763844, + 0.013120958, + 0.06905126, + -0.1074305, + 0.021637388, + 0.16322272, + -0.13542607, + 0.05240015, + 0.03747799, + -0.02136567, + -0.19518022, + 0.33402026, + -0.07306034, + 0.24572115, + -0.13900623, + -0.038396675, + 0.06149101, + 0.022670144, + 0.11762282, + 0.02650935, + 0.24784814, + -0.012557801, + -0.049926393, + 0.17277104, + 0.019994384, + 0.17777458, + -0.015156963, + -0.36002278, + 0.08576319, + 0.12579906, + -0.070583634, + 0.046381455, + -0.032518055, + -0.013673033, + 0.08388661, + 0.02805109, + -0.0040858416, + 0.08121704, + -0.024603426, + 0.038663764, + -0.052432787, + 0.09663728, + -0.05794367, + 0.03322958, + -0.05295467, + 0.012488321, + 0.06900259, + 0.32041568, + 0.050188363, + -0.06842132, + 0.17127438, + -0.08995696, + 0.052553497, + -0.15109767, + 0.0032275803, + -0.124561824, + -0.0349717, + -0.15243848, + 0.06931027, + -0.08733464, + 0.08632587, + -0.000009701724, + -0.061496064, + 0.016888466, + 0.14733578, + -0.20635521, + -0.086442135, + -0.010243254, + 0.08384721, + -0.21813874, + 0.053306054, + -0.021313356, + -0.22646224, + -0.111123234, + -0.17016025, + -0.053942624, + 0.10293637, + -0.067348056, + 0.016638568, + -0.055585764, + 0.1321758, + 0.09786219, + -0.0846062, + -0.20103993, + -0.20276587, + -0.14107303, + 0.0572056, + -0.05998819, + -0.079461865, + -0.18345493, + 0.07808181, + 0.025852023, + 0.16117097, + 0.09278437, + -0.0070364075, + -0.09634602, + 0.15276241, + -0.025094103, + -0.25218967, + -0.047830544, + -0.03265829, + -0.14163731, + -0.2689699, + -0.03686031, + 0.012960216, + -0.02138974, + 0.13290478, + 0.071284525, + 0.059489492, + 0.1677266, + 0.04434716, + 0.05381613, + -0.07807097, + -0.007200855, + -0.05029709, + -0.19462912, + -0.06630631, + 0.077244155, + -0.051679987, + 0.16820908, + 0.006629443, + 0.06144869, + 0.044119336, + 0.08528723, + 0.09778228, + -0.05264809, + 0.17047869, + -0.033962384, + 0.09303276, + 0.14469127, + 0.32394484, + -0.05757008, + 0.14690277, + -0.113172874, + -0.073722996, + -0.1543499, + 0.13370518, + 0.022599915, + -0.016452743, + -0.1111051, + -0.123819724, + -0.0358012, + 0.095570944, + -0.026632585, + -0.12938291, + 0.054892946, + 0.0053247954, + 0.0051558986, + -0.08200846, + -0.024864692, + 0.0022463088, + 0.033632293, + 0.022776838, + 0.046233963, + 0.07099262, + 0.03803995, + 0.11918089, + 0.073886305, + -0.02947079, + -0.032019015, + -0.028573902, + 0.07655141, + -0.13654494, + 0.10705977, + -0.054300036, + 0.1552018, + -0.000661521, + 0.22377999, + -0.08899608, + 0.12889342, + -0.20074363, + 0.13927813, + -0.0026708662, + -0.04300883, + -0.16178162, + -0.0923155, + -0.16207108, + 0.20354961, + 0.009087291, + -0.15107653, + -0.06892152, + -0.033151213, + 0.043834474, + -0.115955636, + -0.042398877, + -0.13583317, + 0.0435221, + 0.046010397, + -0.007331226, + -0.03840562, + 0.08738765, + -0.10060804, + 0.06926944, + -0.21893275, + -0.04178857, + -0.19848873, + 0.043663215, + 0.11381242, + -0.09335524, + -0.076091655, + 0.13046667, + -0.0030471336, + 0.04753741, + -0.04804935, + -0.032354355, + 0.08939008, + -0.010100283, + 0.10576193, + 0.01991799, + 0.030111695, + -0.04467943, + -0.015056431, + -0.024010837, + 0.08842912, + -0.005612796, + -0.13563918, + 0.18028839, + 0.013166858, + -0.2667886, + -0.085470125, + 0.3685931, + -0.1946491, + 0.049553536, + -0.37291518, + -0.22655754, + 0.34400705, + 0.032976877, + 0.13435382, + -0.11814129, + 0.047868554, + 0.07159131, + 0.036158036, + 0.024520447, + -0.20078675, + -0.05586612, + 0.08214067, + -0.08605121, + 0.032829337, + 0.058903836, + 0.24436851, + 0.09116331, + -0.13379814, + 0.1828366, + 0.1072864, + -0.021227801, + 0.12980577, + -0.11652907, + -0.03775177, + 0.083177045, + 0.057081014, + -0.06332418, + 0.056298833, + -0.3391434, + 0.04645195, + -0.06128842, + 0.18199284, + -0.002383945, + -0.14004768, + -0.16993281, + 0.010031427, + 0.16041878, + -0.16184394, + 0.018993879, + -0.025938176, + 0.002514922, + 0.020949349, + -0.1269015, + -0.054172676, + 0.094788544, + 0.07986568, + 0.088186376, + -0.028140098, + 0.011289786, + -0.17378683, + -0.12375356, + 0.04456818, + 0.08479448, + -0.09294157, + -0.16615918, + 0.028024394, + -0.093367904, + -0.013316832, + 0.04018244, + 0.08644012, + -0.12008931, + 0.018793972, + 0.061523847, + -0.0694417, + -0.024518935, + -0.11057199, + 0.061863527, + -0.05944173, + 0.11886046, + -0.030533476, + -0.017306136, + 0.0665828, + -0.08461085, + 0.13586487, + 0.12317356, + 0.118130654, + -0.084764935, + 0.009605317, + -0.0064539798, + 0.15894376, + 0.12810811, + 0.029883908, + -0.027100915, + -0.18702109, + 0.016678592, + -0.21666358, + 0.009283024, + -0.06385588, + 0.11203676, + 0.036862098, + -0.051290695, + 0.04737701, + 0.100528136, + 0.033112366, + 0.048415672, + -0.04174812, + -0.16214389, + 0.17123649, + 0.21322905, + 0.021103352, + 0.19598372, + -0.21077679, + -0.06637894, + -0.22511913, + 0.13010465, + -0.044941496, + -0.19983673, + 0.1963295, + -0.15502413, + 0.0615183, + -0.16637003, + -0.01743503, + 0.054865364, + 0.004895811, + -0.034269266, + 0.1766163, + -0.084879376, + 0.04049494, + 0.11087186, + 0.02269163, + -0.027787592, + -0.0145252375, + 0.08132456, + 0.12849073, + 0.06231228, + 0.058701742, + -0.06058468, + 0.13572963, + -0.14186189, + 0.1027375, + 0.028402401, + 0.0053135604, + -0.054753385, + 0.027727582, + -0.034705237, + -0.016670734, + -0.10962946, + -0.011801861, + -0.20003115, + 0.2020829, + 0.026161859, + -0.0031623817, + -0.029170213, + -0.018234199, + 0.04875749, + -0.23150893, + -0.036654674, + 0.09868626, + -0.28474876, + -0.048850782, + -0.0046823416, + -0.0050867144, + -0.028108975, + 0.078664176, + 0.05139621, + -0.13367955, + -0.06766825, + -0.13896853, + -0.16415608, + 0.017776988, + 0.10541162, + -0.0264685, + 0.08690293, + 0.12137225, + -0.034988154, + 0.16255242, + 0.002840024, + 0.16030852, + -0.076692425, + -0.22003824, + -0.18630469, + 0.016727459, + -0.020857252, + -0.008802409, + 0.07564682, + -0.05444138, + 0.1668737, + -0.025629545, + 0.14557064, + 0.09674229, + -0.058274068, + 0.05744842, + -0.0928503, + -0.0799128, + -0.07071504, + -0.0121874735, + 0.1692294, + 0.093030006, + 0.020831311, + 0.04520271, + 0.087474234, + -0.016130574, + -0.05108359, + 0.036344565, + -0.108744465, + -0.055855155, + 0.1017321, + -0.069691285, + -0.14468838, + -0.10310156, + 0.11176399, + -0.00078769, + -0.03110051, + -0.1145704, + -0.07260943, + 0.23371693, + -0.104421645, + -0.06583719, + -0.006062506, + 0.05011631, + -0.0054390067, + -0.078630604, + 0.19628902, + -0.1550763, + -0.13275377, + -0.12952268, + 0.042036884, + 0.05858515, + 0.055275004, + -0.10907125, + -0.00476199, + -0.14193, + 0.04885924, + -0.02711134, + 0.03325598, + -0.04900974, + -0.08830747, + -0.099112585, + 0.13302748, + 0.19985054, + -0.045091256, + -0.20651561, + 0.16883984, + 0.3288204, + 0.024190074, + -0.031287767, + 0.15144359, + -0.008892988, + -0.01452989, + 0.15788583, + -0.151726, + -0.2957167, + -0.1945324, + 0.05123937, + 0.037350606, + 0.09008041, + -0.08441538, + -0.09457509, + 0.067238115, + -0.02894548, + -0.011611294, + 0.03891026, + 0.111131825, + -0.26067775, + -0.28298238, + 0.056074947, + 0.008107262, + 0.0828736, + 0.052385423, + -0.08731629, + 0.14230244, + 0.19611123, + -0.0090899905, + -0.012221198, + -0.00518729, + 0.0022617683, + 0.051705398, + 0.095017456, + -0.056464072, + 0.11342066, + 0.0254407, + -0.13259909, + 0.038187362, + 0.08667088, + -0.07060096, + -0.011785999, + -0.023018276, + 0.11617855, + -0.036035065, + -0.07118992, + -0.12512614, + 0.12636593, + 0.09379038, + 0.10848534, + 0.099208534, + 0.21280734, + 0.011662269, + 0.08994858, + 0.101053335, + -0.11046256, + -0.066710554, + -0.001033645, + 0.08943645, + -0.057964046, + -0.17622581, + 0.09226646, + -0.11705704, + -0.11167349 + ] + ], + "bias": [ + 0.23896483, + -0.3402354, + 0.101270564 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.json new file mode 100644 index 0000000..5c115ef --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.json @@ -0,0 +1,1717 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.34, + "abstain_prob_threshold": 0.45000002, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.71159875, + "confusion": [ + [ + 69, + 75, + 5 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 11, + 58 + ] + ], + "tool_precision": 0.98571426, + "tool_recall": 0.46308726, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5340551, + "p_memory": 0.21771747, + "p_abstain": 0.24822739 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5626822, + "p_memory": 0.1836759, + "p_abstain": 0.2536419 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6036589, + "p_memory": 0.11206787, + "p_abstain": 0.28427315 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40643284, + "p_memory": 0.19013943, + "p_abstain": 0.40342778 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51935685, + "p_memory": 0.09203593, + "p_abstain": 0.3886073 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3248475, + "p_memory": 0.08496061, + "p_abstain": 0.5901919 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5663625, + "p_memory": 0.18072063, + "p_abstain": 0.2529169 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53524184, + "p_memory": 0.17327352, + "p_abstain": 0.2914846 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55211174, + "p_memory": 0.14331223, + "p_abstain": 0.30457598 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48472118, + "p_memory": 0.20033272, + "p_abstain": 0.31494612 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48654568, + "p_memory": 0.20197286, + "p_abstain": 0.31148145 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3951337, + "p_memory": 0.11869106, + "p_abstain": 0.48617515 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5669123, + "p_memory": 0.11284301, + "p_abstain": 0.32024476 + }, + { + "query_id": "shell_test_023", + "text": "現在のプロセス一覧でklbrが含まれるものを見せて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6097893, + "p_memory": 0.08166945, + "p_abstain": 0.30854127 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "memory", + "p_tools": 0.532595, + "p_memory": 0.09771287, + "p_abstain": 0.36969218 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5313089, + "p_memory": 0.07768788, + "p_abstain": 0.39100322 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5733041, + "p_memory": 0.16126691, + "p_abstain": 0.26542896 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5433677, + "p_memory": 0.18911023, + "p_abstain": 0.2675221 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47030315, + "p_memory": 0.17896877, + "p_abstain": 0.3507281 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5621234, + "p_memory": 0.14502367, + "p_abstain": 0.29285294 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57606184, + "p_memory": 0.1570898, + "p_abstain": 0.2668484 + }, + { + "query_id": "rf_test_020", + "text": "read the existing router_report.md from the last benchmark run", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5553279, + "p_memory": 0.09895797, + "p_abstain": 0.3457141 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5518937, + "p_memory": 0.17088455, + "p_abstain": 0.2772218 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53405726, + "p_memory": 0.19761977, + "p_abstain": 0.268323 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4904085, + "p_memory": 0.08687752, + "p_abstain": 0.42271397 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5497213, + "p_memory": 0.09366132, + "p_abstain": 0.3566174 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41655815, + "p_memory": 0.17062807, + "p_abstain": 0.41281372 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5488636, + "p_memory": 0.2709307, + "p_abstain": 0.1802057 + }, + { + "query_id": "rf_test_028", + "text": "benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54564625, + "p_memory": 0.10946755, + "p_abstain": 0.3448862 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5033046, + "p_memory": 0.11161256, + "p_abstain": 0.3850828 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46809536, + "p_memory": 0.20201841, + "p_abstain": 0.32988623 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41650668, + "p_memory": 0.15908073, + "p_abstain": 0.4244126 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55804354, + "p_memory": 0.1696324, + "p_abstain": 0.27232406 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4662468, + "p_memory": 0.18212193, + "p_abstain": 0.35163125 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56596255, + "p_memory": 0.1841741, + "p_abstain": 0.24986333 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3543384, + "p_memory": 0.07887757, + "p_abstain": 0.566784 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50478405, + "p_memory": 0.16049631, + "p_abstain": 0.33471963 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42139858, + "p_memory": 0.14304228, + "p_abstain": 0.43555915 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49103788, + "p_memory": 0.14342807, + "p_abstain": 0.36553407 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42337945, + "p_memory": 0.1434221, + "p_abstain": 0.4331984 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5013689, + "p_memory": 0.1826258, + "p_abstain": 0.31600538 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.456451, + "p_memory": 0.15955734, + "p_abstain": 0.3839917 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52091247, + "p_memory": 0.18335688, + "p_abstain": 0.29573065 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46343142, + "p_memory": 0.2125959, + "p_abstain": 0.32397267 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5892539, + "p_memory": 0.1738571, + "p_abstain": 0.23688899 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44753295, + "p_memory": 0.27299076, + "p_abstain": 0.27947626 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5022449, + "p_memory": 0.23418446, + "p_abstain": 0.2635706 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.7777778, + "confusion": [ + [ + 38, + 31, + 1 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 8, + 47 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54285717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 6, + 4, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 7, + 33 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.6, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34340474, + "p_memory": 0.25163478, + "p_abstain": 0.40496042 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3519324, + "p_memory": 0.31789282, + "p_abstain": 0.33017477 + }, + { + "query_id": "holdout_abstain_toolish_abstain_024", + "text": "fix this pls", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46415684, + "p_memory": 0.11063545, + "p_abstain": 0.4252077 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5380341, + "p_memory": 0.11918207, + "p_abstain": 0.3427838 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42133805, + "p_memory": 0.21229622, + "p_abstain": 0.3663658 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40329978, + "p_memory": 0.15085411, + "p_abstain": 0.4458461 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56161386, + "p_memory": 0.08422203, + "p_abstain": 0.35416412 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4686281, + "p_memory": 0.20746468, + "p_abstain": 0.32390714 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 5, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56028265, + "p_memory": 0.09171384, + "p_abstain": 0.34800354 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6163145, + "p_memory": 0.083830416, + "p_abstain": 0.29985517 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53365076, + "p_memory": 0.1765899, + "p_abstain": 0.2897593 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5660403, + "p_memory": 0.12957314, + "p_abstain": 0.30438662 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.588117, + "p_memory": 0.12349621, + "p_abstain": 0.2883868 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49780443, + "p_memory": 0.1976606, + "p_abstain": 0.30453497 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.6166667, + "confusion": [ + [ + 27, + 22, + 1 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.54, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5814607, + "p_memory": 0.090691835, + "p_abstain": 0.3278474 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43580824, + "p_memory": 0.1266781, + "p_abstain": 0.43751368 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5679764, + "p_memory": 0.17214762, + "p_abstain": 0.25987604 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48265192, + "p_memory": 0.11242944, + "p_abstain": 0.40491858 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5766949, + "p_memory": 0.1020035, + "p_abstain": 0.32130158 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.57704926, + "p_memory": 0.17888491, + "p_abstain": 0.24406579 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53964037, + "p_memory": 0.16215126, + "p_abstain": 0.29820833 + }, + { + "query_id": "holdout_tools_recall_tools_017", + "text": "append a line \"router ok\" to docs/router_holdout_log.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5688708, + "p_memory": 0.13480045, + "p_abstain": 0.29632878 + }, + { + "query_id": "holdout_tools_recall_tools_018", + "text": "create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5791878, + "p_memory": 0.15656106, + "p_abstain": 0.26425117 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5642287, + "p_memory": 0.12317955, + "p_abstain": 0.3125918 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30877766, + "p_memory": 0.09008908, + "p_abstain": 0.6011332 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.55833334, + "p_memory": 0.11444883, + "p_abstain": 0.32721782 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.56498855, + "p_memory": 0.1571439, + "p_abstain": 0.27786747 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5090132, + "p_memory": 0.08257267, + "p_abstain": 0.4084142 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5643572, + "p_memory": 0.09330997, + "p_abstain": 0.34233275 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "memory", + "p_tools": 0.479116, + "p_memory": 0.09175641, + "p_abstain": 0.42912757 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49324355, + "p_memory": 0.07097059, + "p_abstain": 0.43578583 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.50321186, + "p_memory": 0.13390279, + "p_abstain": 0.3628854 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4472453, + "p_memory": 0.1665224, + "p_abstain": 0.38623226 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42895338, + "p_memory": 0.13105285, + "p_abstain": 0.43999377 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.53821284, + "p_memory": 0.15968174, + "p_abstain": 0.30210543 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5234248, + "p_memory": 0.19924532, + "p_abstain": 0.27732992 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5949896, + "p_memory": 0.12630968, + "p_abstain": 0.2787007 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.md new file mode 100644 index 0000000..37de7de --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-14/router_report.md @@ -0,0 +1,146 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.340` +- abstain_prob_threshold: `0.450` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.7116` +- tools precision: `0.9857` +- tools recall: `0.4631` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 69 | 75 | 5 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 11 | 58 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.53 | 0.22 | 0.25 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.56 | 0.18 | 0.25 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Memory` | 0.58 | 0.09 | 0.33 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.57 | 0.17 | 0.26 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Memory` | 0.58 | 0.10 | 0.32 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.58 | 0.18 | 0.24 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_017` | append a line "router ok" to docs/router_holdout_log.txt | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_tools_recall_tools_018` | create a file benchmarks/router_holdout_tmp.txt and put the number 123 in it | `Tools` | `Memory` | 0.58 | 0.16 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Memory` | 0.56 | 0.12 | 0.31 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.56 | 0.11 | 0.33 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.56 | 0.16 | 0.28 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Memory` | 0.51 | 0.08 | 0.41 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Memory` | 0.56 | 0.09 | 0.34 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Memory` | 0.48 | 0.09 | 0.43 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Memory` | 0.50 | 0.13 | 0.36 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.45 | 0.17 | 0.39 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.54 | 0.16 | 0.30 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.52 | 0.20 | 0.28 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Memory` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.34 | 0.25 | 0.40 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.35 | 0.32 | 0.33 | +| `holdout_abstain_toolish_abstain_024` | fix this pls | `Abstain` | `Memory` | 0.46 | 0.11 | 0.43 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Memory` | 0.54 | 0.12 | 0.34 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.42 | 0.21 | 0.37 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.40 | 0.15 | 0.45 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Memory` | 0.56 | 0.08 | 0.35 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.47 | 0.21 | 0.32 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Memory` | 0.56 | 0.09 | 0.35 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Memory` | 0.62 | 0.08 | 0.30 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.53 | 0.18 | 0.29 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.57 | 0.13 | 0.30 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Memory` | 0.59 | 0.12 | 0.29 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.50 | 0.20 | 0.30 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.60 | 0.11 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.41 | 0.19 | 0.40 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Memory` | 0.52 | 0.09 | 0.39 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.32 | 0.08 | 0.59 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.54 | 0.17 | 0.29 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.55 | 0.14 | 0.30 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Memory` | 0.44 | 0.13 | 0.44 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.20 | 0.31 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.49 | 0.20 | 0.31 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.40 | 0.12 | 0.49 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Memory` | 0.57 | 0.11 | 0.32 | +| `shell_test_023` | 現在のプロセス一覧でklbrが含まれるものを見せて | `Tools` | `Memory` | 0.61 | 0.08 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.31 | 0.09 | 0.60 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.48 | 0.11 | 0.40 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Memory` | 0.53 | 0.10 | 0.37 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Memory` | 0.43 | 0.13 | 0.44 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Memory` | 0.49 | 0.07 | 0.44 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Memory` | 0.53 | 0.08 | 0.39 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.57 | 0.16 | 0.27 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.54 | 0.19 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.56 | 0.15 | 0.29 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.58 | 0.16 | 0.27 | +| `rf_test_020` | read the existing router_report.md from the last benchmark run | `Tools` | `Memory` | 0.56 | 0.10 | 0.35 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.55 | 0.17 | 0.28 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.53 | 0.20 | 0.27 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Memory` | 0.49 | 0.09 | 0.42 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Memory` | 0.55 | 0.09 | 0.36 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.42 | 0.17 | 0.41 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.55 | 0.27 | 0.18 | +| `rf_test_028` | benchmarks/router_dataset_log.mdを読んで最新のエントリを教えて | `Tools` | `Memory` | 0.55 | 0.11 | 0.34 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.50 | 0.11 | 0.39 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.47 | 0.20 | 0.33 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.42 | 0.16 | 0.42 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.56 | 0.17 | 0.27 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.47 | 0.18 | 0.35 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.57 | 0.18 | 0.25 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.35 | 0.08 | 0.57 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.50 | 0.16 | 0.33 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.42 | 0.14 | 0.44 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.49 | 0.14 | 0.37 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.42 | 0.14 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.50 | 0.18 | 0.32 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.46 | 0.16 | 0.38 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.52 | 0.18 | 0.30 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.46 | 0.21 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.59 | 0.17 | 0.24 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.45 | 0.27 | 0.28 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.50 | 0.23 | 0.26 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.5429` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7778` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.6000 | 0.0000 | 0.0000 | 0.8167 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.5400 | 0.0000 | 0.0000 | 0.6167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-2/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-2/router_model.json new file mode 100644 index 0000000..ba7b593 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-2/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "dev", + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.23592426, + 0.07949028, + 0.12828025, + 0.020902963, + -0.14630364, + 0.2927499, + 0.06966109, + -0.08523073, + -0.16648786, + -0.113038614, + 0.075005926, + -0.13153438, + 0.04556273, + 0.1155168, + 0.030217323, + -0.050269037, + -0.10385784, + 0.12613581, + -0.046347305, + -0.1806265, + -0.0074652135, + -0.024326801, + -0.133352, + 0.0062017944, + 0.13812687, + -0.046048444, + -0.035603765, + -0.14249863, + 0.13594781, + 0.20004377, + 0.067069486, + -0.32760555, + -0.02300786, + -0.2257146, + 0.009997282, + -0.01574743, + 0.022478333, + 0.025442492, + -0.15943898, + 0.11212006, + -0.06273372, + -0.07784379, + -0.11847325, + -0.10418217, + -0.11747773, + 0.10935527, + 0.04392976, + -0.050310243, + 0.021614216, + 0.016519511, + -0.06274579, + -0.035048507, + 0.024466934, + -0.1573378, + -0.16322778, + 0.08302193, + -0.21318288, + -0.026558505, + -0.0031204561, + 0.021045184, + -0.10830819, + -0.053354923, + 0.007069701, + 0.003236166, + -0.054645468, + -0.066746205, + 0.0042206966, + 0.011725452, + 0.024868056, + 0.1641807, + -0.12174166, + 0.08036685, + -0.18606532, + -0.18648595, + -0.055612143, + 0.1113537, + 0.020987004, + -0.055373207, + -0.10797911, + -0.26858276, + -0.055051066, + 0.042448446, + -0.10084089, + -0.11356134, + 0.13724102, + 0.25910544, + -0.13518274, + 0.04425172, + -0.050655782, + 0.13767378, + -0.0948142, + -0.10503194, + -0.095695175, + 0.051466744, + -0.11693792, + -0.18811321, + -0.021318266, + 0.084969796, + -0.000039007555, + 0.038942024, + -0.07426399, + -0.048653837, + -0.017212326, + 0.12696455, + 0.07994347, + 0.07255431, + -0.023122568, + -0.10119431, + -0.06391866, + -0.009974052, + -0.22291102, + 0.04796371, + 0.0050148666, + -0.032373007, + 0.0017591142, + -0.06311281, + -0.04166572, + -0.16159615, + -0.22823016, + -0.050940126, + 0.06667, + -0.17553227, + 0.07846491, + -0.02724876, + -0.018249461, + 0.04215583, + 0.090505816, + 0.2814388, + 0.18407907, + 0.20622618, + 0.024842309, + 0.15443164, + 0.15413754, + 0.06652858, + -0.047738686, + -0.033041824, + -0.041732833, + 0.049973257, + 0.03409717, + -0.17384486, + -0.053682525, + 0.248795, + -0.054270744, + -0.07033317, + 0.1876099, + -0.21418522, + 0.06829151, + 0.11003408, + -0.03164994, + -0.022757377, + -0.08888097, + 0.13937046, + 0.15536013, + 0.11985994, + -0.12203828, + 0.012693621, + -0.029066604, + -0.10698433, + 0.0034464756, + 0.20514335, + 0.019293183, + -0.022996629, + 0.15026903, + -0.07309379, + -0.077310584, + -0.11743237, + -0.06254132, + -0.2564027, + 0.05387124, + -0.07310932, + 0.08163549, + -0.076425664, + -0.13267939, + -0.0035350712, + 0.0057923766, + 0.04745503, + 0.010319806, + 0.051259685, + 0.0014403726, + -0.070171736, + 0.016842423, + 0.08128754, + -0.20343794, + 0.27244997, + -0.09540211, + 0.028516164, + 0.10335394, + 0.25332108, + -0.04973971, + -0.12993562, + 0.00938431, + 0.14829825, + -0.02070785, + 0.038483165, + -0.04951867, + 0.049828425, + -0.07171106, + -0.033850256, + -0.1940812, + -0.009546966, + -0.0112374, + -0.025064934, + -0.13010886, + 0.060540106, + 0.06995255, + -0.07389344, + 0.19672123, + 0.07199163, + -0.10079826, + -0.2906196, + -0.02971355, + -0.11642807, + 0.010727101, + -0.01761672, + -0.024298944, + -0.09158621, + -0.061693575, + 0.10451686, + -0.19965115, + -0.16541333, + -0.07537575, + 0.1343618, + 0.123472676, + 0.26711482, + -0.123669244, + -0.27683747, + 0.15709896, + 0.063465126, + 0.0615493, + 0.15080597, + -0.17174396, + -0.18264005, + 0.1107746, + 0.0067112264, + 0.31034142, + 0.07599152, + -0.0509026, + 0.009105364, + 0.11131923, + 0.060135517, + -0.17910625, + 0.035435993, + 0.086380795, + -0.06450933, + 0.055679657, + 0.09171713, + 0.2905432, + 0.18213777, + -0.16435088, + -0.033460796, + 0.032459106, + -0.06308164, + 0.021363614, + 0.10111545, + -0.10463408, + -0.009399844, + 0.023662485, + -0.075450815, + -0.08625308, + 0.03425609, + 0.0023995063, + 0.06754814, + 0.028587475, + 0.07141926, + -0.10899464, + -0.047308013, + 0.026210437, + 0.13202062, + 0.019719098, + -0.12279105, + -0.007736201, + 0.18840674, + 0.13595977, + -0.08788261, + -0.0676793, + -0.081188634, + 0.05922845, + 0.054195378, + 0.019523846, + 0.03297895, + -0.029749334, + 0.062380422, + -0.063839525, + 0.11746603, + 0.011709679, + 0.10049474, + -0.05289085, + -0.03948768, + -0.054421388, + 0.01328617, + -0.2833275, + -0.022533081, + 0.10512336, + -0.029588947, + 0.052098297, + 0.17958073, + 0.03313654, + -0.019853612, + 0.13525973, + 0.1127346, + 0.2221017, + 0.0035275596, + 0.06705968, + 0.14495128, + 0.019332787, + 0.01311561, + -0.030842064, + 0.037300523, + -0.12344496, + -0.24704461, + -0.20622453, + 0.11744558, + -0.084686, + 0.013970482, + 0.035322703, + 0.05975246, + -0.0038897332, + -0.07505102, + 0.051193774, + 0.10900311, + 0.083961606, + -0.0867449, + -0.032312248, + 0.14108476, + -0.06840601, + 0.0072576064, + 0.058360774, + -0.08088482, + -0.11373591, + 0.07676662, + -0.09543414, + 0.011987333, + 0.08126872, + -0.06535875, + 0.03102977, + 0.047054287, + 0.02478329, + -0.09908036, + 0.21217299, + 0.1383378, + 0.082068495, + 0.17698748, + 0.07193557, + 0.2344127, + -0.03208311, + -0.04145339, + 0.023168517, + 0.1236235, + -0.12498678, + 0.016663183, + -0.12038983, + -0.14532413, + 0.038567517, + -0.18034275, + -0.015200795, + 0.16699447, + -0.074562944, + -0.16833735, + -0.015414858, + 0.29435298, + -0.16669613, + 0.06419434, + 0.07672183, + 0.2750522, + -0.0076842927, + 0.06513065, + 0.0045497296, + -0.02302735, + 0.05604058, + -0.015285121, + -0.090965, + 0.013308471, + 0.02668532, + -0.09131679, + 0.007411643, + 0.11996509, + -0.20453662, + -0.109074995, + 0.21115391, + -0.048814315, + 0.094790995, + -0.12637646, + 0.00034142684, + -0.058226816, + -0.21736974, + 0.05987529, + 0.0670604, + -0.008285919, + -0.100198664, + 0.21597408, + 0.038501117, + 0.04248908, + -0.12107101, + 0.14713411, + -0.10302279, + 0.04554846, + 0.17807189, + 0.00014714275, + 0.053678766, + -0.05470793, + -0.1468645, + 0.13801947, + 0.06114068, + -0.06495822, + 0.08971697, + 0.012763572, + 0.050100397, + -0.047830675, + -0.069687314, + 0.03206434, + -0.026934445, + -0.039816517, + 0.026246255, + -0.053072933, + -0.069831535, + 0.09188684, + 0.09086396, + 0.05913434, + -0.0142745795, + 0.15986186, + -0.114475, + -0.15942223, + -0.01427824, + -0.10161266, + -0.068465814, + 0.03319923, + -0.16473934, + -0.078048386, + -0.06126856, + 0.27868116, + -0.0040932256, + -0.0390591, + 0.18008506, + 0.11380149, + 0.008093586, + -0.025110045, + -0.13809349, + -0.26707006, + -0.03321406, + 0.10180727, + -0.012883773, + -0.06458295, + 0.17459899, + -0.24968287, + -0.10938166, + 0.14730081, + -0.16311902, + -0.02548285, + -0.19513816, + 0.37816283, + 0.17008424, + 0.18801682, + -0.068766676, + -0.10833149, + -0.06584093, + 0.005282863, + 0.05705433, + -0.06359804, + -0.011743776, + -0.05460606, + 0.10435172, + 0.069224454, + -0.2127962, + -0.1429357, + -0.08955556, + 0.105138846, + 0.108842775, + 0.18029512, + -0.0558439, + 0.095631614, + 0.12960093, + 0.0037637828, + 0.03426417, + -0.20179549, + -0.21381183, + -0.11177129, + -0.05186325, + -0.15315644, + 0.083750695, + 0.14294466, + 0.06936303, + 0.097568415, + -0.027851103, + 0.10691856, + -0.04703076, + 0.056984775, + 0.056401536, + -0.0071451035, + -0.11501074, + -0.24542804, + 0.17336452, + 0.028761476, + 0.052476797, + -0.038022522, + 0.044432264, + -0.16253045, + -0.09647879, + 0.07095663, + 0.08605165, + -0.11119516, + -0.058758993, + -0.027047373, + 0.05835111, + -0.004415728, + 0.24493673, + 0.012733293, + -0.12034067, + 0.076820046, + -0.007837615, + 0.061275806, + -0.100934975, + -0.15823844, + -0.009439666, + 0.28618845, + -0.21604861, + 0.18269648, + -0.043275967, + -0.05298801, + -0.06984093, + -0.062074646, + -0.2558363, + -0.082004085, + 0.08930997, + 0.05140848, + -0.11594375, + 0.16008247, + -0.01647563, + 0.005979772, + -0.055181064, + 0.03094402, + 0.057006262, + 0.037506513, + 0.22195199, + -0.15976258, + -0.003430983, + -0.03736374, + -0.052652735, + 0.09027966, + 0.32088894, + -0.117658705, + 0.07841281, + -0.007944, + -0.037882786, + 0.13079576, + 0.030691518, + -0.012422775, + -0.005092901, + -0.06255673, + 0.032830406, + -0.03967016, + -0.112258725, + -0.22631066, + -0.16136739, + -0.15279353, + -0.14530544, + 0.027644537, + -0.013307016, + -0.12973621, + -0.22201909, + 0.047920972, + -0.21610388, + -0.05441284, + 0.0869944, + -0.0120201865, + -0.16603912, + -0.027903512, + 0.14737375, + -0.045106143, + -0.10809847, + 0.12977672, + 0.06122886, + 0.1714344, + 0.10619558, + 0.05405039, + -0.095592745, + 0.022597203, + -0.10200969, + -0.06988007, + 0.17373843, + -0.041029654, + -0.096191384, + 0.036363162, + -0.040415987, + 0.113239214, + 0.11083686, + 0.11057566, + 0.021456886, + -0.1187632, + -0.005398725, + -0.09654723, + -0.13158664, + -0.16231221, + -0.13153927, + -0.07664805, + 0.18590742, + 0.022832684, + -0.03676107, + -0.16832295, + 0.27945006, + -0.03835614, + 0.25090364, + 0.08288016, + 0.056393255, + 0.15483865, + 0.036503028, + 0.012308527, + -0.16759615, + -0.14400983, + 0.15403487, + -0.07984099, + 0.103513256, + 0.05800183, + 0.13537936, + 0.06597946, + 0.09718602, + -0.0009663588, + 0.009373034, + 0.049974643, + 0.10922762, + -0.0666244, + 0.054338492, + -0.2523631, + 0.12637684, + 0.12406432, + -0.0133141745, + -0.14440738, + 0.18514237, + 0.03871962, + -0.00856082, + -0.03739144, + 0.103693984, + 0.38904634, + -0.12921004, + 0.20685571, + 0.0717717, + -0.017677164, + -0.13859999, + -0.047213227, + -0.035712585, + -0.018850666, + -0.032312583, + 0.1963019, + -0.22302505, + 0.059585784, + 0.07076368, + -0.0037386497, + -0.021374866, + -0.09076489, + -0.05121408, + 0.06617326, + 0.16769287, + -0.0759916, + 0.00011955293, + -0.071625695, + 0.05203951, + -0.11306046, + -0.13112223, + 0.024701066, + -0.045229748, + 0.21586764, + 0.1884639, + 0.014671893, + 0.07167805, + 0.086855225, + 0.21806249, + 0.0010663769, + -0.09161673, + -0.07338588, + 0.18400705, + 0.11373605, + -0.007007729, + 0.08980646, + -0.27008447, + -0.22743385, + 0.108092554, + -0.10263347, + -0.12530118, + 0.011223255, + -0.02534834, + 0.11997918, + 0.00043334352, + -0.12639672, + 0.07851988, + -0.06799492, + -0.021804351, + 0.022667186, + 0.039111875, + -0.14440686, + -0.00038854606, + 0.04258943, + -0.03997176, + 0.16821238, + -0.068418495, + 0.11187042, + 0.07045411, + 0.054948222, + 0.09370843, + 0.11527065, + -0.12156669, + 0.23117429, + 0.14954045, + -0.050761856, + 0.18862389, + 0.17474213, + 0.04331295, + -0.08087375, + -0.057447176, + -0.03303781, + 0.10136958, + -0.09814324, + 0.07164336, + 0.01354199, + -0.090047404, + -0.019989517, + -0.024555784, + -0.0873952, + -0.021014484, + -0.1468437, + 0.06360805, + -0.009553678, + -0.2520642, + -0.0009906373, + 0.043536548, + 0.15728356, + 0.1157493, + 0.07684819, + -0.0066023595, + -0.15029529, + -0.04891407, + 0.07041565, + 0.019646177, + 0.15459754, + -0.055917192, + -0.13702592, + -0.08028114, + -0.005777796, + 0.005830356, + -0.022148266, + -0.03957986, + -0.1078105, + -0.00035169302, + -0.041368164, + 0.024077183, + -0.29544196, + -0.08077371, + -0.013104905, + 0.041448615, + -0.010947219, + -0.034304015, + 0.016416628, + 0.27215916, + -0.104368046, + 0.07733927, + 0.22092457, + 0.1287746, + -0.030926289, + 0.14851579, + -0.096314825, + 0.00039009872, + 0.010219875, + 0.056555286, + 0.00847581, + -0.04259313, + -0.0931727, + -0.0684084, + -0.115641244, + 0.14368759, + -0.16151942, + -0.24075364, + 0.13465847, + 0.16644302, + 0.005077254, + 0.13074507, + -0.023705306, + -0.061494812, + -0.08048761, + -0.060518295, + 0.06539182, + 0.029792016, + 0.03118727, + 0.006684678, + -0.06381983, + -0.0007743956, + 0.015529249, + -0.06939081, + 0.07917838, + -0.0718619, + -0.11937301, + 0.010090283, + -0.08640019, + -0.18818526, + -0.11591951, + 0.18826221, + -0.1300556, + 0.24455251, + -0.010264277, + 0.24497488, + 0.09312586, + 0.022135185, + 0.051192943, + -0.16545711, + -0.11049563, + -0.1811448, + 0.14125228, + -0.13107261, + 0.028541027, + 0.15770547, + 0.026281191, + 0.050461553, + 0.061548013, + 0.12506539, + -0.102001585, + 0.061886996, + 0.15549985, + -0.026153754, + -0.084029496, + 0.10270605, + -0.06070375, + -0.16488475, + -0.027996136, + -0.074855156, + -0.006563445, + 0.006692481, + 0.11110623, + 0.21137123, + 0.06501346, + 0.007817524, + -0.08232954, + 0.0263958, + 0.011379713, + -0.04867242, + 0.18348722, + -0.13841671, + -0.06814913, + 0.04329171, + 0.03316773, + -0.14990935, + 0.10580919, + -0.22892036, + -0.04298803, + -0.12009234, + 0.092555754, + -0.07022957, + -0.25958672, + 0.04043409, + -0.116474554, + -0.07289303, + 0.056018278, + 0.009886767, + 0.1267809, + 0.04049609, + -0.08856585, + 0.04917458, + 0.10745919, + 0.0031032898, + -0.0776394, + 0.021187214, + 0.07920715, + -0.053854663, + -0.09190022, + -0.15109119, + 0.017716136, + -0.10367998, + 0.18685561, + 0.22195326, + -0.12602778, + 0.1713986, + -0.012507223, + -0.05165178, + 0.046761766, + 0.09106876, + -0.074691705, + 0.017120015, + 0.037856784, + 0.11998435, + -0.11249925, + -0.111552395, + -0.022593446, + 0.023815157, + 0.19325908, + 0.2730344, + -0.007951937, + 0.12512314, + -0.024566812, + 0.047931004, + 0.12275496, + 0.16077788, + -0.09789672, + 0.03344916, + -0.16171698, + -0.016622083, + 0.038192052, + 0.016207283, + -0.0202107, + -0.13984105, + -0.03407714, + 0.12702416, + -0.14115557, + -0.019655803, + -0.16221006, + -0.01791215, + -0.13691811, + 0.12804243, + -0.011204896, + 0.01841803, + 0.0050965617, + -0.08466767, + 0.016319202, + 0.007212669, + -0.04629895, + -0.06361673, + -0.027919175, + -0.041548878, + -0.06237486, + 0.1745566, + -0.043050725, + 0.038423937, + 0.18671963, + 0.018245509, + -0.029951867, + -0.22200696, + -0.057974942, + -0.057651896, + -0.055361606, + 0.18368134, + -0.15395811, + -0.080219485, + -0.21031596, + 0.09167013, + -0.06837816, + 0.13859786, + -0.13372809, + 0.07269533, + 0.11164036, + -0.072583176, + -0.021096015, + -0.23613524, + 0.05250386, + 0.044930402, + -0.01711093, + -0.07612404, + 0.03333045, + 0.11559862, + 0.10009551, + 0.15378518, + 0.11826541, + -0.1216371, + -0.087640956, + -0.23119548, + -0.023505269, + -0.039743233, + -0.15849829, + -0.022412464, + -0.098236606, + 0.018338203, + -0.056111015, + 0.01976757, + 0.047145586, + 0.1796975, + 0.07133825, + -0.05501143, + 0.0137379235, + 0.15215507, + -0.13460286, + 0.09070147, + 0.069535986, + 0.061831266, + -0.087164275, + 0.080333784, + 0.08281889, + 0.08870611, + 0.042467095, + 0.03364361, + 0.23295124, + -0.071372524, + 0.023481961, + -0.041838918, + -0.04865416, + 0.016903803, + 0.057054404, + -0.046940424, + -0.029821644, + 0.0018828119, + -0.11095171, + -0.07097106, + -0.055843648, + 0.00956623, + 0.018680362, + 0.0153555, + 0.017266344, + 0.00007468156, + 0.01172443, + -0.027056027, + 0.0065923487, + -0.06067196, + 0.13302639, + 0.10897519, + 0.08939442, + -0.12072664, + 0.22262308, + 0.16682461, + 0.15896632, + 0.11093958, + -0.03812856, + 0.041086815, + -0.16033763, + 0.13180858, + -0.01072775, + 0.104514994, + -0.0033954063, + -0.0977714, + -0.14432882, + -0.059497964, + -0.051815588, + 0.06572544, + 0.021578567, + -0.09776904, + 0.06565402, + -0.16587934, + -0.116609134, + -0.14652133, + 0.06313389, + 0.08367709, + -0.026228959 + ], + [ + 0.15846656, + -0.14894673, + -0.13188975, + -0.015371586, + 0.16705966, + -0.231484, + 0.0024328774, + 0.061065182, + 0.12463436, + 0.11706655, + -0.026946174, + 0.05513764, + -0.0051243478, + -0.008197161, + -0.032258887, + -0.016351407, + 0.18757029, + -0.15723023, + 0.16273841, + 0.15276295, + 0.014594901, + 0.009599693, + 0.09730296, + 0.077078134, + -0.17151691, + 0.10118174, + -0.06627298, + 0.11285701, + -0.103246175, + -0.04298179, + 0.057788603, + 0.3094538, + 0.073913105, + 0.14185263, + 0.08249015, + 0.034514878, + -0.015602923, + -0.029827323, + 0.13692331, + -0.109577194, + -0.09081349, + 0.10803755, + 0.06682779, + 0.029051548, + 0.1537387, + -0.075789124, + 0.1146328, + -0.038770474, + -0.024302071, + 0.13382743, + 0.049123622, + 0.008879571, + -0.07886671, + 0.074562654, + 0.14038633, + -0.10135582, + 0.15901671, + 0.13758342, + 0.13179119, + -0.044150967, + 0.10968406, + 0.028949987, + -0.021487163, + 0.0023873842, + 0.102438875, + 0.09836604, + 0.06578889, + 0.013700457, + -0.061243135, + -0.1439024, + 0.10743138, + -0.09807392, + 0.23552205, + 0.16922787, + -0.04566979, + -0.031479042, + 0.08064524, + -0.0140977055, + 0.052643966, + 0.14185508, + -0.04078149, + -0.0034885583, + 0.16458786, + 0.0752994, + -0.13292527, + -0.25739685, + 0.09674331, + 0.05962877, + 0.04741129, + -0.21546471, + 0.11757735, + 0.20755276, + 0.0030378932, + -0.06636101, + -0.028773654, + 0.153386, + 0.01590099, + -0.16970557, + 0.033176962, + 0.057676144, + 0.064098015, + 0.049269557, + 0.06532966, + -0.11312617, + 0.06794883, + -0.08994207, + 0.059467416, + 0.09691956, + -0.048683733, + -0.027496036, + 0.10316365, + -0.10428701, + 0.013541405, + 0.063007005, + -0.06387908, + 0.08398428, + -0.022002866, + 0.1102383, + 0.24936323, + 0.08888533, + 0.13566265, + 0.08911813, + 0.05534134, + 0.03426087, + -0.003912369, + -0.06930861, + -0.07926059, + -0.2927976, + -0.13223682, + -0.2020717, + -0.085529335, + -0.08397994, + -0.07512756, + -0.056016676, + -0.067682855, + 0.06265628, + 0.022324614, + -0.02904347, + -0.16194701, + 0.11525205, + 0.06782437, + -0.16947006, + 0.02137571, + 0.035497002, + -0.1502951, + 0.2020105, + -0.09598874, + -0.09472741, + 0.009307279, + 0.058853816, + 0.0008540991, + -0.11725075, + -0.18493211, + -0.088182926, + 0.14129642, + -0.006272462, + 0.15148452, + 0.17936167, + -0.051006615, + -0.005678193, + -0.12936504, + 0.118420176, + -0.091464624, + 0.010349933, + 0.114841565, + 0.12727495, + 0.01069694, + 0.27586663, + -0.03692025, + 0.026927534, + -0.08407196, + 0.079921246, + 0.07302816, + -0.03934301, + -0.008794134, + -0.039965097, + 0.11838427, + -0.048990693, + 0.0011952232, + 0.059630655, + -0.02073507, + -0.06985642, + 0.15482605, + -0.1547007, + 0.1256582, + -0.1299996, + -0.01868761, + -0.29615396, + 0.040472984, + 0.13947299, + 0.071826965, + -0.064604826, + 0.045524985, + -0.0013657572, + 0.00922413, + 0.054285023, + -0.037322026, + -0.055743903, + 0.1780619, + 0.019885443, + 0.12101634, + 0.048781477, + 0.17347799, + -0.0873945, + -0.13729812, + 0.082692884, + -0.15274675, + -0.12076281, + 0.093869224, + 0.25696653, + -0.060197536, + 0.11230131, + 0.0016297848, + 0.0036486778, + 0.025571777, + 0.047004957, + 0.1287568, + -0.09368466, + 0.08403961, + 0.14358337, + -0.018372541, + -0.040335115, + -0.14648063, + -0.32904556, + 0.037121747, + 0.19827436, + -0.16016959, + -0.08684652, + -0.03874674, + -0.20288484, + 0.14046177, + 0.19002816, + -0.090223774, + -0.08097232, + -0.21057487, + -0.010953297, + 0.17159285, + -0.060857728, + -0.057242908, + -0.07777571, + 0.1968416, + -0.027824707, + -0.08062887, + -0.042103395, + -0.080151476, + -0.12742521, + -0.15232758, + -0.11119279, + 0.10303499, + 0.074524805, + 0.018627008, + 0.083202824, + -0.038138278, + -0.16862473, + 0.15880917, + 0.03960908, + 0.053431354, + 0.056068275, + 0.06306108, + -0.057394594, + 0.060965676, + -0.12809429, + -0.025608107, + -0.04439726, + 0.14331713, + 0.0075398274, + 0.0030347155, + -0.18464282, + -0.009869805, + 0.073502645, + -0.009123688, + -0.069161676, + 0.005321287, + 0.13778484, + 0.18855691, + 0.14130345, + -0.06832302, + -0.047158428, + -0.0030905677, + -0.011824866, + -0.024031462, + -0.0598138, + 0.07862906, + -0.09125566, + -0.040174685, + -0.1162901, + -0.019563397, + -0.022721067, + -0.002791389, + -0.028123276, + 0.20853533, + 0.055247508, + -0.009078611, + 0.05638552, + -0.0861065, + -0.19664645, + -0.068449125, + -0.004284521, + -0.12818453, + -0.054515485, + -0.21327844, + -0.022304073, + -0.014190555, + -0.059023093, + 0.014396681, + 0.015331843, + 0.121404916, + -0.15422125, + 0.023859633, + 0.262496, + 0.12140163, + -0.12423673, + 0.053574026, + -0.07480659, + 0.009966959, + 0.0005245576, + 0.10235568, + 0.10102288, + -0.023910532, + -0.20969674, + -0.05751784, + 0.06074621, + -0.030705856, + -0.108660564, + 0.10981022, + -0.022167629, + -0.074983954, + 0.14298001, + 0.15487315, + -0.105472706, + 0.12458858, + -0.011921662, + -0.0843357, + 0.080602966, + -0.05240729, + -0.078765005, + -0.06236435, + 0.033246577, + -0.16713138, + -0.16168198, + 0.02604236, + -0.08032973, + -0.09142687, + -0.13910621, + -0.050133, + 0.05249491, + 0.029800884, + -0.18045552, + 0.11533404, + 0.0036546546, + 0.13957272, + 0.09065534, + -0.07983853, + 0.065196075, + 0.00483309, + -0.30357614, + 0.1300592, + 0.14113303, + 0.045899827, + -0.17832467, + 0.16170332, + -0.101678856, + -0.03701216, + -0.25509036, + -0.025909375, + -0.024960186, + -0.027482023, + 0.018042693, + -0.009376487, + -0.011157068, + 0.08628339, + -0.123575084, + 0.08299967, + 0.09093684, + -0.035794947, + -0.0690082, + 0.15115522, + 0.07879298, + -0.17823504, + 0.052165683, + -0.18495956, + 0.06324202, + 0.078495614, + 0.057631716, + 0.14990783, + -0.04525075, + -0.029693574, + 0.0128731625, + -0.08203428, + -0.10327234, + -0.06030997, + -0.0077230334, + 0.115389444, + -0.08682941, + 0.079778865, + -0.060847305, + -0.17431593, + 0.10438322, + 0.012507549, + 0.052554544, + 0.12766378, + -0.115853414, + -0.008095368, + 0.06318096, + 0.022107618, + -0.058577165, + 0.025625072, + 0.18896843, + 0.11701097, + -0.03345069, + -0.046744175, + 0.02987199, + 0.018711489, + 0.004343268, + 0.034436073, + -0.033576973, + -0.056621, + 0.07035622, + 0.019453403, + -0.12953557, + 0.026528751, + 0.10356629, + 0.0403922, + 0.23169217, + 0.10239691, + 0.028832285, + 0.1697025, + -0.003531267, + -0.033742674, + -0.23828392, + -0.0671558, + -0.07060434, + -0.12074501, + -0.12699483, + -0.04256371, + -0.04909332, + 0.09225179, + 0.309755, + 0.04118996, + -0.10893188, + -0.124176696, + 0.08212391, + -0.18666762, + 0.28523374, + 0.030693185, + -0.14264916, + 0.10614599, + 0.02713271, + 0.06335521, + -0.19940498, + -0.18611884, + -0.1278629, + -0.015504656, + 0.08729648, + 0.17531921, + -0.02825543, + -0.06725592, + -0.0005201547, + 0.011050643, + 0.077033885, + -0.020947594, + -0.0773544, + 0.10796558, + 0.15403922, + 0.0063380264, + -0.08978346, + -0.204632, + -0.08047593, + 0.09311416, + 0.09013907, + -0.09525472, + -0.10154838, + -0.024830816, + 0.17287362, + 0.18489592, + 0.11841823, + -0.019611297, + 0.10096258, + -0.06806371, + -0.08465169, + -0.0013794691, + 0.111672156, + 0.005498556, + -0.047415145, + -0.03359179, + -0.059774265, + -0.098724715, + 0.013236173, + 0.08048248, + 0.24152869, + -0.10831418, + 0.07953544, + -0.09127184, + 0.044242486, + -0.009510906, + 0.12005419, + -0.040822737, + 0.03278505, + -0.07833108, + 0.11430967, + 0.11994177, + -0.098867826, + -0.15677744, + 0.069522895, + -0.25672966, + -0.011805372, + 0.07292897, + -0.082478516, + 0.02561567, + -0.047698513, + 0.13341406, + 0.12818106, + 0.044617876, + -0.40938342, + 0.15651107, + -0.13003437, + 0.07099373, + 0.09285106, + 0.12612762, + -0.017611964, + 0.12529397, + 0.17717384, + -0.2243484, + -0.019332271, + 0.1802624, + -0.13767661, + 0.01961787, + -0.065574445, + 0.024493206, + 0.07592765, + -0.12865902, + -0.06986012, + -0.15317339, + 0.06604938, + 0.014160292, + 0.06368592, + 0.05778993, + -0.06879842, + -0.3333686, + 0.07081244, + -0.08865506, + 0.0010657859, + 0.020205896, + -0.094921045, + -0.061092082, + -0.034534458, + 0.033048082, + -0.0042976537, + -0.15719782, + -0.1924763, + 0.06796789, + 0.1328857, + 0.018388335, + 0.11101402, + 0.109840795, + -0.06254849, + 0.004293073, + 0.15740435, + 0.17644696, + 0.0058195884, + 0.13029478, + 0.071573466, + -0.20421873, + -0.0050412193, + 0.18835844, + 0.07271274, + -0.112578, + 0.16781381, + -0.014131367, + -0.10212361, + -0.08584278, + 0.031238567, + -0.21272327, + -0.0694137, + 0.1308, + -0.053156145, + 0.096017756, + 0.109670356, + -0.09303574, + 0.0563351, + -0.1033751, + -0.010472144, + 0.032520525, + -0.1469482, + -0.15429942, + -0.09321035, + -0.016487926, + 0.20083293, + -0.01745232, + 0.042721696, + 0.14180313, + 0.21574003, + 0.073189884, + 0.09661394, + -0.19908611, + -0.11430697, + 0.017988099, + 0.09320381, + -0.29125306, + 0.033173177, + -0.06468759, + -0.07561015, + 0.03300521, + -0.033276286, + 0.024739988, + -0.026963402, + 0.10801799, + 0.13346595, + -0.14806463, + 0.047130696, + -0.06427405, + -0.13276349, + -0.1571873, + -0.1720413, + -0.019900773, + 0.004509936, + 0.017361667, + 0.021702256, + -0.06365898, + -0.012174513, + 0.043270823, + 0.16917661, + -0.040842917, + -0.036232848, + 0.012658868, + 0.054902602, + -0.20585628, + -0.054428756, + -0.045206092, + -0.04531314, + -0.0789444, + -0.25269538, + 0.061517183, + -0.1425821, + -0.11450696, + 0.028789515, + 0.19709642, + 0.08766006, + -0.021093508, + -0.07813938, + -0.032772005, + -0.07292086, + 0.24164371, + -0.08440194, + -0.027143363, + -0.048491545, + 0.015090795, + 0.1070741, + 0.030939583, + -0.09336118, + -0.14727467, + 0.07165781, + 0.08620502, + 0.02424046, + -0.08498235, + 0.12279331, + 0.10719259, + -0.049552344, + -0.116854385, + -0.23771396, + -0.0920385, + -0.032208342, + -0.11301597, + -0.08382864, + -0.11445808, + -0.04390611, + 0.1032361, + 0.0013361671, + -0.16949497, + -0.14225605, + 0.021628316, + -0.15692602, + 0.26118505, + 0.11900645, + -0.053895447, + 0.06433893, + 0.21862704, + 0.029760117, + 0.09066999, + -0.06886031, + -0.03807151, + 0.05981284, + -0.08681997, + 0.1286995, + 0.01431086, + -0.022292813, + -0.0005013467, + 0.25885803, + -0.04367855, + -0.035725366, + 0.05130664, + -0.15069792, + 0.00079145544, + -0.056092262, + -0.072985694, + 0.030023174, + -0.06899001, + -0.0975411, + 0.008925394, + -0.19356744, + -0.096645854, + 0.08922534, + -0.26041025, + -0.11038632, + -0.12035151, + 0.047138903, + -0.04014047, + -0.005079972, + -0.08453478, + 0.0108782975, + -0.055437442, + -0.024233581, + 0.13133909, + 0.017852798, + 0.09230822, + 0.05934381, + 0.032781035, + 0.044325385, + -0.056726612, + 0.09054025, + 0.30902827, + 0.042634513, + -0.11351706, + -0.10184977, + -0.15892436, + 0.07752234, + 0.063255265, + 0.03392058, + -0.05138785, + -0.1493424, + 0.01198075, + -0.14310691, + 0.021757988, + 0.13613899, + -0.01396091, + 0.01895428, + -0.041307848, + -0.032796502, + 0.039798666, + 0.03955335, + -0.11712413, + -0.007823799, + -0.08758357, + 0.36238423, + 0.045195565, + 0.042123098, + -0.0118451305, + -0.011359909, + 0.10570221, + -0.06508535, + -0.21233514, + 0.12879884, + 0.00801969, + -0.20314407, + -0.02493685, + -0.026470685, + -0.13859881, + 0.027010472, + -0.10404782, + -0.00038773075, + 0.0040764627, + 0.07198745, + 0.03613616, + 0.082790814, + 0.12304106, + 0.15090622, + -0.12975182, + 0.1422943, + 0.32201743, + -0.11158703, + -0.06097697, + 0.007564108, + -0.1664216, + -0.035444904, + -0.028393103, + 0.14315064, + 0.11850086, + -0.024217011, + -0.07844546, + -0.040807687, + -0.028631993, + 0.039424993, + 0.059636302, + -0.073344104, + 0.04405877, + -0.08296971, + 0.06776528, + 0.095045164, + 0.0041737542, + 0.063781425, + 0.12505543, + 0.08125786, + -0.11541801, + 0.106116205, + -0.21122476, + -0.051326115, + -0.11984397, + -0.05864495, + 0.025218667, + -0.06094618, + 0.12133363, + 0.09911744, + 0.26462877, + -0.12545985, + 0.11903367, + -0.105698176, + -0.14354075, + 0.0644074, + -0.09552613, + -0.033709824, + -0.09168388, + 0.18870622, + 0.0010486201, + -0.0814779, + -0.044366695, + -0.007999213, + -0.099657305, + -0.012411787, + 0.19009852, + -0.09644196, + 0.03648562, + 0.08932092, + 0.011254463, + -0.18018445, + -0.18829237, + -0.031496696, + 0.04360538, + 0.15007122, + -0.06060969, + 0.06423649, + -0.018384177, + -0.1379392, + 0.10824323, + 0.011449396, + -0.005312135, + -0.12633747, + 0.18985581, + -0.050494656, + 0.27826023, + 0.04518124, + 0.0762414, + -0.15507232, + 0.05498624, + 0.11095998, + -0.07603957, + 0.11104304, + 0.038726956, + -0.06857704, + -0.050024584, + -0.14961046, + -0.042043075, + 0.09258056, + -0.027747488, + -0.19722545, + -0.025414722, + -0.05644636, + -0.0046709403, + -0.07148298, + 0.13278374, + 0.00003596043, + 0.17292266, + 0.0392831, + 0.076638624, + -0.25348988, + -0.048064888, + 0.040796805, + -0.1317034, + -0.010435862, + 0.057497643, + 0.02138421, + -0.12928724, + 0.1843052, + -0.0032561622, + 0.02560748, + 0.0040482255, + 0.16645108, + 0.1124439, + 0.07103201, + -0.017360581, + -0.14021014, + -0.2761286, + -0.027650936, + -0.087520726, + 0.1284345, + -0.0020231095, + -0.16560674, + -0.13692503, + 0.06287243, + -0.07822542, + 0.035219744, + -0.072541155, + -0.09010822, + -0.04351065, + 0.027134832, + 0.06755826, + 0.12301915, + 0.0049245586, + 0.09186711, + -0.09780905, + 0.16602924, + 0.060643964, + 0.09915961, + -0.12730426, + -0.012174859, + -0.05269128, + -0.041920573, + 0.13201751, + -0.038913008, + -0.051302686, + 0.021553233, + 0.06974902, + 0.037005715, + -0.0075630434, + 0.095330544, + -0.1538154, + -0.020467883, + -0.046345387, + -0.13503817, + 0.0069053033, + 0.03509617, + 0.22232045, + 0.10840425, + 0.077698134, + 0.09895408, + -0.0935128, + 0.17952754, + 0.055660255, + 0.17371093, + -0.07007724, + 0.09766003, + -0.07297232, + 0.048130978, + -0.11572883, + -0.032660715, + 0.08643092, + -0.033975136, + 0.25975278, + 0.035782024, + -0.13298692, + 0.07752003, + 0.057088308, + -0.011036775, + -0.1296041, + -0.14754817, + -0.09915362, + -0.09061552, + 0.15483025, + 0.086246595, + 0.21902007, + 0.06317361, + -0.0017731533, + 0.14258327, + 0.009094075, + 0.16113617, + -0.080414616, + -0.10949779, + 0.01014872, + -0.17496696, + -0.084411696, + -0.10173003, + 0.04112154, + -0.07685115, + -0.20684911, + 0.10517152, + 0.01482347, + -0.0013761881, + 0.03887268, + 0.09994203, + -0.014113231, + -0.13717343, + 0.03788843, + -0.042142607, + -0.04741655, + -0.16435277, + 0.015987214, + -0.093827575, + 0.04590369, + -0.0027739417, + -0.06706411, + 0.044482827, + 0.022706142, + -0.028011689, + -0.12265332, + 0.083518095, + -0.0049355505, + 0.01921485, + -0.050097317, + -0.02169908, + 0.0050034286, + 0.034458358, + -0.009163651, + 0.065465175, + -0.03930148, + -0.061662227, + 0.041233227, + -0.099589385, + -0.11675253, + -0.053536303, + 0.11452381, + -0.24844523, + -0.1071091, + -0.11363072, + -0.02298112, + -0.025474198, + -0.17504856, + 0.16830038, + -0.07467595, + -0.008802671, + -0.06480381, + 0.011079027, + -0.03456703, + 0.18757644, + 0.031754997, + -0.038898297, + -0.09223578, + 0.024588306, + 0.07565296, + -0.0666662, + 0.083090164, + 0.14365932, + 0.09664895, + -0.07777618, + 0.0065826504, + 0.13500947 + ], + [ + 0.07745762, + 0.06945646, + 0.0036094207, + -0.0055313776, + -0.020755991, + -0.0612659, + -0.072094016, + 0.024165524, + 0.041853517, + -0.0040279394, + -0.048059803, + 0.076396756, + -0.040438388, + -0.10731963, + 0.002041552, + 0.06662044, + -0.083712466, + 0.03109451, + -0.11639113, + 0.027863607, + -0.0071296943, + 0.014727122, + 0.036048975, + -0.08327996, + 0.03339005, + -0.055133313, + 0.101876795, + 0.029641671, + -0.03270176, + -0.15706189, + -0.12485806, + 0.01815152, + -0.050905254, + 0.08386188, + -0.09248747, + -0.01876746, + -0.006875415, + 0.004384827, + 0.022515684, + -0.002542826, + 0.15354718, + -0.030193746, + 0.05164544, + 0.075130545, + -0.03626094, + -0.033566207, + -0.15856253, + 0.0890807, + 0.0026878468, + -0.15034702, + 0.01362214, + 0.026168939, + 0.054399718, + 0.08277522, + 0.02284142, + 0.018333904, + 0.054166045, + -0.11102487, + -0.12867075, + 0.023105752, + -0.0013759588, + 0.024404915, + 0.01441747, + -0.0056235488, + -0.047793347, + -0.03161983, + -0.07000958, + -0.025425902, + 0.036375098, + -0.020278316, + 0.014310252, + 0.01770713, + -0.049456652, + 0.017257914, + 0.101281986, + -0.07987475, + -0.10163229, + 0.06947092, + 0.055335134, + 0.12672764, + 0.09583257, + -0.038959898, + -0.063746974, + 0.03826194, + -0.004315776, + -0.0017085372, + 0.03843952, + -0.103880465, + 0.00324448, + 0.07779084, + -0.02276321, + -0.102520965, + 0.09265732, + 0.014894296, + 0.14571168, + 0.03472724, + 0.005417286, + 0.08473566, + -0.033137944, + -0.09661821, + 0.010165947, + -0.0006157164, + -0.048117362, + -0.013838345, + -0.14789234, + 0.01738775, + -0.036344845, + 0.0042748144, + 0.112602405, + 0.037470084, + 0.119747445, + 0.05632335, + -0.018556254, + -0.030634018, + 0.062119953, + -0.020871485, + 0.06366857, + 0.051357854, + -0.021132894, + -0.037945222, + -0.20233276, + 0.08641417, + -0.1338063, + -0.00701211, + 0.02216182, + 0.0271528, + -0.01124527, + 0.011358738, + -0.05184234, + -0.0041546402, + 0.060687024, + -0.07045171, + -0.079010054, + -0.010511909, + 0.115421556, + -0.02961444, + 0.019408198, + -0.020929804, + 0.12784983, + 0.058592763, + -0.01414183, + -0.07932492, + 0.032895025, + 0.034836117, + -0.037314806, + 0.012174778, + 0.027697226, + -0.015306631, + 0.022342695, + -0.036096424, + 0.088026896, + -0.02211972, + 0.029572042, + -0.031676978, + -0.019258121, + -0.006421155, + -0.122418076, + -0.0723774, + 0.047560133, + -0.19946514, + 0.11007185, + -0.09542355, + -0.058804367, + 0.06274386, + -0.037530933, + -0.009842537, + 0.051844392, + -0.019464014, + -0.016950997, + 0.04618182, + 0.0024364518, + -0.0034955442, + 0.059651207, + 0.0428781, + 0.0030017633, + -0.0074899355, + -0.12870409, + -0.0022689754, + -0.0026355973, + 0.010541111, + 0.0038926431, + -0.01143113, + 0.048611894, + -0.117749445, + -0.030256076, + 0.10148345, + -0.08466635, + 0.042832922, + 0.009266726, + -0.009537506, + -0.081211306, + -0.08369359, + -0.024817122, + -0.03711743, + 0.040294554, + -0.10411341, + 0.10903301, + 0.08959415, + 0.016019406, + -0.010338482, + -0.10977886, + -0.023716526, + -0.04336907, + 0.026854422, + 0.067345515, + -0.008799443, + -0.043974437, + 0.04877117, + 0.006929042, + 0.033653125, + 0.08991103, + 0.0041267015, + -0.012356885, + 0.0139680505, + -0.0012728166, + 0.04458126, + -0.06706323, + -0.010832285, + 0.11561156, + 0.021829976, + 0.09374831, + -0.09402673, + 0.023007879, + 0.061930645, + 0.086547494, + 0.078563154, + 0.0030706376, + 0.023381444, + -0.022802526, + 0.05207883, + 0.031282157, + -0.007388119, + -0.020550888, + 0.07426106, + -0.099766485, + -0.06503823, + -0.120690316, + 0.05175232, + -0.05407624, + 0.017640108, + -0.017735312, + -0.0076112943, + -0.005751937, + 0.10661272, + 0.024471821, + 0.03570812, + -0.13821566, + -0.07094514, + 0.06131591, + -0.041064017, + -0.05108608, + -0.020121247, + 0.016774653, + 0.067509264, + -0.05417521, + -0.030209228, + -0.07709386, + 0.019382522, + 0.023191983, + 0.023138497, + -0.0633652, + 0.06054607, + -0.0029793808, + -0.027022064, + -0.034322478, + 0.039768178, + -0.029245159, + 0.05262219, + -0.009849279, + 0.049288478, + 0.01685988, + -0.11924519, + -0.14128108, + -0.049902253, + -0.12087765, + -0.060114726, + 0.009094624, + -0.0070369425, + -0.016433297, + -0.021154065, + 0.053780776, + -0.0025666319, + -0.014789462, + -0.026210396, + 0.028465025, + 0.01579537, + 0.07245419, + 0.062208753, + 0.05721281, + 0.014837121, + 0.074792154, + -0.03271447, + -0.09604473, + -0.026796577, + 0.03400821, + 0.017065603, + 0.035312586, + 0.02413816, + -0.00707516, + -0.058219112, + -0.00882335, + 0.01877651, + -0.05286912, + -0.085928194, + -0.033729453, + -0.028447466, + -0.090562835, + 0.116920725, + 0.099585325, + -0.015451496, + 0.08482293, + 0.006791124, + 0.031112006, + 0.060836084, + -0.04528965, + -0.060277015, + -0.09846596, + -0.025971867, + -0.027283201, + 0.10069358, + -0.026443765, + 0.02599866, + 0.063018076, + -0.03242416, + -0.0414042, + 0.014910042, + 0.016623162, + -0.06209513, + -0.041137267, + 0.028706074, + -0.029154455, + -0.00006567799, + 0.0030669977, + -0.015244235, + 0.021377513, + 0.03171063, + 0.037581064, + 0.06583381, + -0.045041483, + 0.0233442, + -0.108110875, + -0.096657746, + 0.019491378, + -0.095306545, + 0.08221612, + -0.011041563, + -0.05296942, + 0.056832075, + 0.009652748, + -0.020317838, + -0.01918292, + 0.054668766, + 0.04127102, + 0.1151467, + 0.010367705, + 0.13658163, + -0.055496324, + 0.0272043, + -0.030484973, + -0.116028234, + 0.00499293, + 0.037484445, + -0.03970968, + -0.01996194, + 0.033593677, + -0.040170476, + 0.022932295, + 0.004984673, + -0.04666409, + 0.026442192, + 0.00468162, + 0.1102666, + -0.10968494, + 0.00037995595, + 0.028383309, + -0.050956897, + 0.053381402, + 0.03028198, + -0.032919087, + -0.0033513985, + 0.09016863, + 0.063134365, + -0.078837015, + 0.0005950799, + 0.067461945, + -0.014624532, + -0.037366915, + -0.004587243, + 0.18223287, + -0.11270174, + 0.02180889, + -0.034766078, + 0.0056815967, + -0.06030472, + 0.023243932, + 0.015298831, + -0.0037559464, + -0.104530334, + -0.0661863, + 0.0021533796, + 0.01920063, + -0.022166068, + -0.053045332, + 0.001777255, + -0.11182458, + 0.04581356, + -0.07572549, + -0.14113775, + -0.047323607, + 0.0013863543, + 0.07367861, + 0.009944513, + -0.044957746, + 0.04872967, + 0.035395484, + -0.058309883, + -0.034243003, + -0.12949067, + -0.0051788217, + -0.030326268, + 0.087946266, + 0.055856027, + -0.026113957, + -0.13007952, + -0.0339311, + -0.06203155, + -0.004963134, + 0.0815797, + 0.09501125, + -0.040397406, + 0.07124904, + 0.10966349, + -0.05934003, + 0.013193269, + 0.034470137, + 0.07420334, + 0.04584168, + -0.04268478, + -0.007975875, + 0.007124554, + 0.13706046, + -0.017540988, + 0.012068503, + -0.03555097, + 0.07868844, + -0.0046516946, + 0.05697312, + -0.0016498672, + 0.13178287, + -0.17875795, + 0.01603461, + -0.060153756, + 0.08427135, + 0.021034995, + -0.10947829, + 0.02297257, + 0.010201626, + 0.06411817, + 0.00069312344, + -0.022427784, + -0.08340414, + 0.008129976, + 0.10483062, + -0.0111034345, + 0.08321758, + -0.015355383, + 0.0957891, + -0.09981907, + -0.03727025, + -0.18577074, + -0.03434626, + 0.09778465, + -0.009433364, + 0.028921766, + 0.028915899, + -0.0066469046, + 0.071474515, + 0.05219391, + -0.015687006, + -0.058292978, + -0.06798357, + -0.20924047, + 0.022352543, + -0.059503425, + 0.08062257, + 0.002789503, + 0.042323086, + -0.006091077, + 0.034528166, + 0.003899257, + -0.06505021, + -0.10829687, + 0.038795065, + -0.0062199766, + -0.034921397, + 0.042476345, + 0.1373016, + -0.103741765, + -0.0077205664, + -0.0031144773, + -0.061182786, + 0.12591518, + 0.09842636, + -0.06510717, + 0.011792917, + -0.000927913, + 0.04741174, + 0.0056584463, + -0.01777804, + -0.013577285, + -0.032479167, + 0.03005739, + -0.035178192, + 0.12319496, + 0.05953756, + -0.052662153, + -0.027717786, + -0.03986303, + -0.056286726, + 0.079686634, + 0.1305422, + -0.095169686, + 0.13503848, + -0.032076195, + -0.06431863, + -0.022405755, + -0.003142242, + 0.059594706, + 0.03068783, + -0.10687165, + 0.07165271, + 0.03235361, + -0.06877855, + 0.09371309, + -0.010729307, + -0.026322195, + -0.0051372023, + -0.021481242, + 0.012479718, + 0.046846237, + 0.010242278, + 0.0068782144, + 0.017676892, + -0.03587468, + 0.03040056, + 0.04695722, + -0.027955173, + 0.06685435, + 0.12436746, + 0.23214655, + 0.044290863, + 0.093425006, + 0.14297906, + 0.041779425, + 0.035464626, + 0.034903977, + 0.009013933, + -0.027668172, + 0.045572154, + -0.05374057, + 0.08580905, + -0.01716065, + 0.1172244, + 0.017061407, + -0.02231919, + -0.04480924, + -0.034795754, + -0.1227076, + 0.12222982, + -0.027653115, + 0.024613868, + -0.20267302, + 0.106527664, + 0.015363324, + -0.035207216, + 0.030558974, + 0.0059918873, + -0.03979042, + -0.080702506, + -0.015305442, + 0.1995665, + -0.02589101, + 0.0078954445, + 0.03370895, + 0.04346243, + -0.01736537, + -0.004968941, + -0.08206979, + 0.022851046, + 0.053825565, + -0.010216455, + -0.05342773, + 0.05834939, + -0.01996591, + 0.013178858, + 0.09147428, + 0.018772963, + 0.07511911, + 0.01180301, + 0.005182977, + -0.18621622, + -0.0072700414, + -0.089398436, + -0.12156247, + -0.061243024, + 0.014654888, + 0.059578173, + 0.010543929, + -0.005970224, + 0.03271037, + -0.03923921, + 0.074761786, + 0.021807993, + 0.10606188, + -0.07728519, + -0.0035435762, + -0.02673473, + -0.07167695, + -0.04556865, + 0.078798905, + -0.09760936, + 0.08318658, + -0.08553383, + -0.08783146, + 0.0006552894, + 0.08950479, + 0.020713896, + 0.015709128, + 0.053766903, + 0.08270455, + -0.024749562, + -0.13635102, + 0.06769282, + -0.06427349, + 0.042735264, + -0.011112339, + -0.05849649, + -0.04044677, + 0.056806073, + 0.09699002, + 0.065084584, + -0.12338098, + -0.018618584, + 0.024816224, + -0.043620303, + 0.05223019, + 0.006284062, + -0.01630919, + 0.020274458, + 0.027187906, + -0.020418243, + 0.0043337992, + -0.08632459, + 0.04738525, + 0.032942872, + -0.009732838, + 0.023929657, + 0.02485127, + 0.162084, + 0.021846166, + -0.096425354, + 0.017536443, + 0.041337963, + -0.0030266028, + -0.10360438, + 0.042839766, + -0.011619359, + 0.07204971, + -0.014512203, + 0.028519962, + -0.014620583, + 0.06711956, + 0.0088992305, + 0.10842747, + -0.05419707, + 0.03829462, + -0.09332558, + -0.040983323, + -0.0653216, + -0.051118877, + 0.037638128, + 0.066583864, + 0.008300097, + -0.0607046, + 0.0074935053, + -0.00037436237, + -0.038610496, + -0.114451, + 0.044067096, + -0.0068640895, + -0.0113348495, + -0.017514303, + 0.06762703, + -0.055778116, + 0.0025316353, + -0.084971406, + -0.024718441, + -0.017729495, + 0.112641305, + -0.037606817, + -0.05289465, + -0.03846344, + 0.07178632, + -0.06435578, + 0.07703845, + 0.03373481, + 0.09758765, + 0.038117815, + -0.016834803, + 0.08726492, + -0.016205939, + 0.010691579, + -0.041291684, + 0.0021367143, + -0.06775239, + 0.028051322, + -0.011766549, + 0.10251835, + -0.006881461, + -0.080986604, + -0.056964066, + -0.04164388, + 0.06998048, + -0.05543376, + 0.04317498, + -0.15437047, + -0.056652922, + 0.11637467, + 0.10030192, + 0.07892662, + -0.031626917, + -0.011490581, + 0.03415925, + 0.0008869665, + 0.09424204, + -0.013176491, + 0.035477515, + 0.054944754, + -0.0002188088, + 0.06825712, + 0.11747579, + 0.049191963, + 0.06350638, + -0.06694228, + 0.035578184, + -0.029018175, + -0.029603489, + 0.022307126, + -0.07139821, + 0.048668716, + -0.05982387, + -0.024430871, + -0.08535895, + -0.01778066, + -0.103837736, + 0.057396974, + -0.0099169, + 0.069304325, + 0.10365774, + -0.009832133, + -0.06063173, + -0.08046321, + 0.0064569516, + 0.010381851, + -0.054632653, + -0.035264913, + -0.013935891, + 0.01922498, + -0.08126367, + -0.023071425, + -0.105466105, + -0.012641356, + 0.03567662, + 0.05915022, + 0.08988791, + -0.06266305, + -0.057982516, + -0.04117482, + 0.048653405, + 0.009620415, + 0.021947324, + 0.024394862, + -0.05886191, + 0.057814877, + 0.025332076, + 0.003791334, + 0.0040966417, + 0.024327794, + -0.014264028, + 0.022618687, + 0.06312975, + 0.03466166, + -0.0728442, + 0.023939375, + -0.03332764, + 0.061590437, + -0.12513098, + -0.03448089, + -0.047353834, + 0.009753265, + 0.04412345, + 0.011378179, + -0.08348392, + -0.01579243, + 0.012038838, + 0.0771572, + -0.01416469, + -0.09068855, + 0.04506465, + -0.027838208, + -0.03338156, + -0.08670475, + -0.062935546, + -0.07402203, + 0.07052046, + 0.09202871, + -0.0030487257, + 0.0731155, + -0.025213849, + 0.12443811, + 0.038369555, + -0.08275741, + -0.017946951, + 0.06907822, + -0.023078818, + -0.033516727, + -0.051422942, + -0.067741655, + 0.034213886, + -0.075616196, + 0.06705659, + -0.04554798, + 0.030173553, + 0.056699738, + -0.03797959, + 0.09316969, + -0.039946508, + -0.055314485, + -0.049339972, + -0.0021931415, + 0.043850936, + 0.0625165, + 0.015243344, + 0.1486268, + 0.03560544, + 0.0054315627, + 0.034166023, + 0.01255885, + 0.040137827, + 0.022829562, + 0.0015469731, + -0.0040146876, + -0.021427106, + 0.08976617, + 0.022311423, + 0.13408577, + -0.016516278, + -0.0077241906, + -0.07892908, + 0.0918642, + -0.021831458, + -0.056999244, + 0.027041351, + 0.06663442, + -0.1738884, + 0.085231, + -0.039695214, + 0.02294308, + -0.005845859, + -0.06814599, + 0.038218528, + -0.109613486, + -0.013863854, + -0.06346433, + -0.12403258, + -0.053951886, + -0.00089150923, + -0.048438534, + -0.0064545753, + -0.053049006, + 0.003094217, + 0.03560286, + -0.0376025, + -0.10386766, + -0.045907885, + 0.04285174, + -0.02385285, + 0.035024274, + 0.04477627, + 0.12649728, + 0.08916322, + 0.05191616, + 0.027303383, + -0.006924128, + 0.07228284, + -0.088942, + -0.1319488, + 0.049288463, + 0.1174649, + -0.0038192864, + -0.042731818, + 0.03775852, + -0.00073822733, + 0.023379738, + 0.034273278, + 0.03682401, + -0.04734978, + 0.022593817, + 0.04409001, + 0.02474576, + -0.0061323484, + -0.009086511, + 0.049111933, + -0.032955654, + -0.020741219, + 0.063518584, + 0.007921433, + -0.05168147, + -0.025150804, + -0.005144285, + -0.00031368487, + -0.05042929, + -0.020046197, + -0.04359245, + -0.09016853, + -0.025569437, + 0.0245593, + 0.03660497, + -0.021592885, + -0.029281888, + -0.065625526, + 0.08559711, + 0.043033604, + -0.07897961, + -0.013847679, + 0.055071138, + -0.023617532, + -0.08828593, + 0.08805651, + -0.06040911, + 0.01903576, + -0.022293685, + 0.014005562, + 0.047452588, + -0.054631684, + -0.027649907, + -0.033193264, + 0.0013944489, + 0.012175467, + -0.03966833, + 0.04151638, + 0.01591493, + 0.013318377, + -0.06289957, + 0.06207641, + 0.1656088, + -0.029916283, + 0.12782148, + -0.09528566, + 0.030391762, + 0.013889852, + 0.063113265, + 0.054693975, + 0.029431297, + -0.1055249, + -0.068159774, + -0.10070397, + -0.012777722, + -0.06622054, + 0.05435452, + -0.12659454, + -0.0003244871, + 0.013772954, + -0.06859853, + 0.055385318, + 0.07034561, + -0.0040647984, + 0.05142809, + 0.05016032, + -0.10153724, + 0.0242343, + 0.05783332, + 0.12077048, + 0.027433638, + 0.075906634, + 0.03662882, + 0.040531076, + 0.0030187052, + -0.02035892, + -0.051724717, + 0.009088971, + -0.07718964, + 0.066357516, + 0.05506988, + 0.019438691, + -0.033437017, + 0.0077773957, + -0.035858143, + 0.00620277, + 0.025822306, + -0.05971552, + -0.04533555, + -0.08795847, + 0.06360275, + 0.1339617, + -0.007962772, + -0.057132564, + 0.019530416, + -0.039711196, + -0.0076836203, + 0.13233833, + -0.043247588, + 0.027742952, + 0.090713866, + 0.026510363, + -0.046166863, + 0.022116156, + 0.0010122113, + 0.08278919, + -0.027050173, + 0.049872346, + 0.014642328, + -0.09025972, + -0.10878053 + ] + ], + "bias": [ + 0.409906, + 0.11315371, + -0.5230597 + ], + "decision": { + "tools_prob_threshold": 0.7, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.json new file mode 100644 index 0000000..92898bf --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.json @@ -0,0 +1,439 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "dev", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.7, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 141, + "accuracy": 0.6879433, + "confusion": [ + [ + 51, + 29, + 0 + ], + [ + 0, + 46, + 0 + ], + [ + 3, + 12, + 0 + ] + ], + "tool_precision": 0.9444444, + "tool_recall": 0.6375, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.2, + "memory_false_abstain_rate": 0.0, + "tool_count": 80, + "memory_count": 46, + "abstain_count": 15, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6104362, + "p_memory": 0.30589545, + "p_abstain": 0.0836684 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.59472257, + "p_memory": 0.3161544, + "p_abstain": 0.08912299 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6297685, + "p_memory": 0.26889145, + "p_abstain": 0.10134005 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6624538, + "p_memory": 0.26015484, + "p_abstain": 0.07739138 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6639559, + "p_memory": 0.23188965, + "p_abstain": 0.1041544 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.60731936, + "p_memory": 0.27004346, + "p_abstain": 0.12263718 + }, + { + "query_id": "shell_test_031", + "text": "is the disk partition containing agent.db almost full?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6722137, + "p_memory": 0.23017572, + "p_abstain": 0.09761066 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.60558313, + "p_memory": 0.32460892, + "p_abstain": 0.06980796 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6429548, + "p_memory": 0.25670224, + "p_abstain": 0.10034299 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.68191427, + "p_memory": 0.24376579, + "p_abstain": 0.074319914 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6615207, + "p_memory": 0.253586, + "p_abstain": 0.08489324 + }, + { + "query_id": "rf_test_013", + "text": "what is the embed_dim value in the support calibrated config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6061652, + "p_memory": 0.31448147, + "p_abstain": 0.07935343 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6499309, + "p_memory": 0.2537558, + "p_abstain": 0.096313275 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.59418523, + "p_memory": 0.33327773, + "p_abstain": 0.07253697 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5657255, + "p_memory": 0.34358782, + "p_abstain": 0.0906867 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6546773, + "p_memory": 0.25830474, + "p_abstain": 0.08701797 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47816014, + "p_memory": 0.41151565, + "p_abstain": 0.110324204 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.576397, + "p_memory": 0.3468361, + "p_abstain": 0.07676686 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6846916, + "p_memory": 0.24798164, + "p_abstain": 0.06732679 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.628443, + "p_memory": 0.29244968, + "p_abstain": 0.07910734 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.691014, + "p_memory": 0.20424092, + "p_abstain": 0.10474504 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.66433394, + "p_memory": 0.25342876, + "p_abstain": 0.082237266 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.63012874, + "p_memory": 0.26535815, + "p_abstain": 0.10451312 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6360849, + "p_memory": 0.2686135, + "p_abstain": 0.09530164 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.65475535, + "p_memory": 0.24794452, + "p_abstain": 0.09730011 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.65992075, + "p_memory": 0.25475225, + "p_abstain": 0.08532704 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.6974221, + "p_memory": 0.21716036, + "p_abstain": 0.08541755 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.69476, + "p_memory": 0.2100455, + "p_abstain": 0.09519449 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.66223246, + "p_memory": 0.2545751, + "p_abstain": 0.08319243 + }, + { + "query_id": "abs_test_001", + "text": "can you help me?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.55861247, + "p_memory": 0.2375059, + "p_abstain": 0.20388162 + }, + { + "query_id": "abs_test_002", + "text": "what's happening on hackernews right now?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.69352347, + "p_memory": 0.19897409, + "p_abstain": 0.10750248 + }, + { + "query_id": "abs_test_003", + "text": "what will my project look like in 6 months?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.47491404, + "p_memory": 0.35513982, + "p_abstain": 0.16994612 + }, + { + "query_id": "abs_test_004", + "text": "what's the best programming language?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.46780536, + "p_memory": 0.31017643, + "p_abstain": 0.22201824 + }, + { + "query_id": "abs_test_005", + "text": "help", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.59031075, + "p_memory": 0.22899751, + "p_abstain": 0.18069172 + }, + { + "query_id": "abs_test_006", + "text": "what's the github token for the ci runner?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.7357465, + "p_memory": 0.16489287, + "p_abstain": 0.09936059 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.5205462, + "p_memory": 0.31795332, + "p_abstain": 0.16150045 + }, + { + "query_id": "abs_test_008", + "text": "something is wrong", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.51049954, + "p_memory": 0.28777307, + "p_abstain": 0.20172736 + }, + { + "query_id": "abs_test_009", + "text": "どのモデルが今一番スコアが高いですか — ベンチマークサイトで", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.61282194, + "p_memory": 0.25715584, + "p_abstain": 0.13002223 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.66652524, + "p_memory": 0.22985739, + "p_abstain": 0.10361738 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.49091458, + "p_memory": 0.3661909, + "p_abstain": 0.1428945 + }, + { + "query_id": "abs_test_012", + "text": "y", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.47903177, + "p_memory": 0.2892732, + "p_abstain": 0.23169498 + }, + { + "query_id": "abs_test_013", + "text": "is the llama.cpp repo trending on github today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.73314124, + "p_memory": 0.1612506, + "p_abstain": 0.105608135 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.60404634, + "p_memory": 0.2927036, + "p_abstain": 0.10325001 + }, + { + "query_id": "abs_test_015", + "text": "can you log into the remote postgres and show me the schema?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.77282834, + "p_memory": 0.14105494, + "p_abstain": 0.086116806 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.md new file mode 100644 index 0000000..c99cb5c --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-2/router_report.md @@ -0,0 +1,81 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-read-file-v2+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-memory-lane-v2+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Dev` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.700` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `141` +- tools: `80` memory: `46` abstain: `15` +- accuracy: `0.6879` +- tools precision: `0.9444` +- tools recall: `0.6375` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.2000` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 51 | 29 | 0 | +| memory | 0 | 46 | 0 | +| abstain | 3 | 12 | 0 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.61 | 0.31 | 0.08 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.59 | 0.32 | 0.09 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.63 | 0.27 | 0.10 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.66 | 0.26 | 0.08 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.66 | 0.23 | 0.10 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Memory` | 0.61 | 0.27 | 0.12 | +| `shell_test_031` | is the disk partition containing agent.db almost full? | `Tools` | `Memory` | 0.67 | 0.23 | 0.10 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.61 | 0.32 | 0.07 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.64 | 0.26 | 0.10 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.68 | 0.24 | 0.07 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `rf_test_013` | what is the embed_dim value in the support calibrated config? | `Tools` | `Memory` | 0.61 | 0.31 | 0.08 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.65 | 0.25 | 0.10 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.59 | 0.33 | 0.07 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | 0.57 | 0.34 | 0.09 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.65 | 0.26 | 0.09 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.48 | 0.41 | 0.11 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.58 | 0.35 | 0.08 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.68 | 0.25 | 0.07 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.63 | 0.29 | 0.08 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.69 | 0.20 | 0.10 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.63 | 0.27 | 0.10 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.64 | 0.27 | 0.10 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.65 | 0.25 | 0.10 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.66 | 0.25 | 0.09 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.70 | 0.22 | 0.09 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.69 | 0.21 | 0.10 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.66 | 0.25 | 0.08 | +| `abs_test_001` | can you help me? | `Abstain` | `Memory` | 0.56 | 0.24 | 0.20 | +| `abs_test_002` | what's happening on hackernews right now? | `Abstain` | `Memory` | 0.69 | 0.20 | 0.11 | +| `abs_test_003` | what will my project look like in 6 months? | `Abstain` | `Memory` | 0.47 | 0.36 | 0.17 | +| `abs_test_004` | what's the best programming language? | `Abstain` | `Memory` | 0.47 | 0.31 | 0.22 | +| `abs_test_005` | help | `Abstain` | `Memory` | 0.59 | 0.23 | 0.18 | +| `abs_test_006` | what's the github token for the ci runner? | `Abstain` | `Tools` | 0.74 | 0.16 | 0.10 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.52 | 0.32 | 0.16 | +| `abs_test_008` | something is wrong | `Abstain` | `Memory` | 0.51 | 0.29 | 0.20 | +| `abs_test_009` | どのモデルが今一番スコアが高いですか — ベンチマークサイトで | `Abstain` | `Memory` | 0.61 | 0.26 | 0.13 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Memory` | 0.67 | 0.23 | 0.10 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.49 | 0.37 | 0.14 | +| `abs_test_012` | y | `Abstain` | `Memory` | 0.48 | 0.29 | 0.23 | +| `abs_test_013` | is the llama.cpp repo trending on github today? | `Abstain` | `Tools` | 0.73 | 0.16 | 0.11 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.60 | 0.29 | 0.10 | +| `abs_test_015` | can you log into the remote postgres and show me the schema? | `Abstain` | `Tools` | 0.77 | 0.14 | 0.09 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-3/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-3/router_model.json new file mode 100644 index 0000000..1fec4a5 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-3/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.18633848, + -0.021990247, + -0.09598119, + -0.12475248, + -0.0055247764, + 0.06369566, + -0.008723138, + -0.19863072, + -0.06277868, + -0.099104874, + 0.085971214, + -0.114912264, + 0.011969069, + -0.011524559, + 0.19476768, + -0.11838336, + -0.03345108, + -0.07276929, + -0.027851067, + -0.06088229, + 0.09280635, + 0.068293154, + -0.068615414, + 0.104422644, + 0.28788283, + -0.102640204, + 0.1655996, + -0.0056493874, + 0.2648823, + 0.18839236, + 0.11906588, + -0.08070814, + -0.11075673, + -0.048513148, + 0.1482414, + 0.043183617, + 0.07661222, + -0.11964615, + -0.12162281, + 0.11422304, + -0.05089721, + -0.2086355, + -0.17418046, + -0.117384106, + 0.10907311, + 0.0814679, + -0.02460272, + -0.08618438, + -0.111958876, + 0.019398285, + 0.06338135, + 0.06671663, + -0.018391166, + -0.13034785, + -0.044448815, + 0.2374514, + 0.00870324, + 0.085715555, + 0.034128625, + 0.21414743, + -0.07857612, + 0.08836732, + -0.19596298, + 0.07679612, + 0.020521877, + 0.05828172, + 0.14005251, + -0.12677951, + 0.0846198, + 0.15728568, + -0.03908312, + 0.17048323, + 0.084221445, + -0.013166092, + -0.14352055, + 0.119596496, + 0.014491269, + 0.02896192, + -0.034508154, + -0.113518804, + -0.07383001, + -0.025141595, + -0.110220656, + 0.13085045, + 0.07905992, + 0.1644883, + -0.1553705, + -0.024855055, + -0.07725549, + 0.045213725, + -0.062506184, + 0.0158601, + -0.059938118, + 0.030852143, + -0.037213616, + -0.11498651, + -0.21289913, + -0.16038676, + 0.08329657, + -0.034475576, + 0.01727555, + -0.08468778, + 0.2650441, + 0.097090185, + -0.15225862, + 0.06705284, + 0.05224962, + -0.065318406, + -0.040154513, + 0.009635824, + -0.29516324, + 0.10499831, + 0.07110597, + 0.03876518, + 0.078121774, + 0.08497595, + -0.07566165, + -0.081970595, + -0.16383497, + 0.068018995, + 0.19277292, + -0.025526213, + 0.037149314, + -0.1211539, + -0.16565678, + 0.0900694, + 0.16598481, + 0.13508134, + 0.12723272, + 0.064582154, + 0.14409442, + 0.051390126, + 0.09308874, + -0.0053218165, + -0.14926353, + 0.10023623, + 0.052404538, + 0.118025355, + 0.064098574, + -0.035400648, + -0.1167498, + 0.11188998, + 0.12400007, + 0.060254753, + -0.0014532692, + -0.32071766, + -0.0011266157, + -0.054838896, + 0.04992481, + -0.005961276, + -0.08831957, + 0.08546277, + 0.103863135, + -0.21817385, + 0.04985002, + -0.07579833, + -0.02296212, + -0.16555277, + 0.09859051, + 0.13479902, + 0.055023395, + -0.02733191, + -0.041447084, + -0.04039115, + 0.07048925, + 0.086557135, + -0.039628442, + 0.11102648, + 0.021148635, + -0.16282822, + 0.14222291, + -0.07848284, + -0.22176567, + -0.044396807, + -0.08138735, + 0.030739442, + -0.11531769, + -0.09774903, + -0.082107425, + 0.046270154, + 0.067457, + 0.2850858, + -0.33103317, + 0.07849784, + -0.023498492, + -0.102756575, + -0.17244335, + 0.3278435, + 0.06095586, + -0.09701801, + 0.10624031, + 0.19086596, + -0.1159302, + 0.051547915, + 0.019621905, + -0.05067868, + 0.081711896, + -0.03655443, + -0.032190118, + 0.25896105, + 0.04414717, + -0.1454319, + 0.1655049, + -0.03875817, + 0.07561866, + -0.19182944, + -0.0029918475, + 0.16781612, + -0.11417916, + -0.23184198, + -0.15177523, + -0.19467123, + -0.23595092, + -0.08488743, + -0.04964707, + -0.10689684, + 0.075020626, + 0.045934547, + 0.023282593, + -0.17801835, + -0.21553747, + 0.14769518, + 0.115120836, + 0.14128429, + -0.18796399, + -0.13099259, + -0.30767933, + -0.035177387, + 0.15527754, + 0.15546124, + -0.019328687, + -0.04424327, + 0.06824624, + -0.07744808, + 0.23629627, + 0.16880615, + -0.01786926, + 0.097564556, + -0.07779348, + 0.039635386, + -0.19111083, + 0.1443258, + -0.058470096, + -0.2673507, + 0.06965227, + 0.148964, + 0.22784793, + -0.0035397268, + 0.018858295, + 0.1077301, + -0.09406661, + -0.18655677, + -0.08204047, + -0.036849435, + 0.045027044, + -0.0013961178, + 0.07994172, + -0.11097627, + -0.10617173, + 0.051952574, + -0.16665302, + 0.14288223, + 0.08364722, + 0.04902033, + 0.056778733, + -0.11843705, + 0.08718353, + -0.03218126, + 0.02587419, + -0.06395128, + -0.009066266, + -0.047516793, + 0.23327143, + -0.04678851, + 0.04037096, + -0.013538329, + 0.08178902, + 0.06668387, + -0.16897848, + -0.015004748, + -0.01572572, + 0.14406358, + 0.071664244, + 0.042464685, + 0.08947636, + 0.092256874, + -0.25032964, + 0.07105301, + -0.108864136, + 0.08742515, + 0.04301848, + 0.017901175, + 0.10187706, + -0.016301403, + 0.061324365, + 0.2776516, + -0.060695387, + -0.033483397, + 0.073651, + 0.0057025487, + 0.3352942, + 0.10394482, + 0.16374, + 0.09518288, + 0.08158211, + -0.09344818, + 0.021589026, + 0.15027705, + -0.07937315, + -0.019393088, + -0.05662475, + -0.05123898, + -0.050836004, + 0.1343026, + 0.10637051, + -0.0577773, + 0.037862033, + -0.247351, + 0.106672175, + -0.0030619581, + -0.00014678766, + -0.08657385, + 0.113408186, + 0.23169582, + 0.019879716, + -0.034681514, + -0.0020551863, + -0.16076934, + -0.14288679, + 0.010082046, + -0.06617522, + -0.10194228, + -0.1415618, + 0.101942606, + -0.08793393, + 0.10894591, + 0.16146699, + -0.12228665, + 0.14080705, + 0.24692212, + 0.14946236, + 0.092784494, + 0.15893684, + -0.008467448, + 0.027090807, + 0.12665439, + -0.13623457, + -0.090713635, + 0.12896347, + 0.21177316, + 0.028683344, + -0.15380841, + -0.0006147844, + -0.1393724, + 0.036622833, + 0.045594625, + -0.023290219, + -0.17021397, + -0.08726953, + -0.015805764, + 0.026924638, + 0.07847709, + -0.04327303, + 0.3590176, + 0.060146403, + -0.15717702, + 0.1325123, + -0.0035076342, + 0.19028895, + -0.079874925, + -0.13333586, + 0.19525443, + -0.06502711, + -0.03750315, + -0.041022416, + -0.043243382, + -0.35507184, + -0.21726933, + 0.16190492, + -0.158557, + 0.13389374, + -0.18338445, + -0.0932396, + 0.12955591, + -0.053707905, + 0.0006459714, + -0.0032740647, + -0.28709257, + -0.056750856, + 0.23210952, + 0.30267164, + -0.08682682, + -0.17241643, + -0.03326459, + 0.054191496, + 0.097175844, + 0.16940773, + -0.12762201, + 0.009070134, + 0.11668817, + -0.21143344, + 0.05499155, + -0.15847756, + -0.14995135, + -0.0049438695, + 0.045848697, + 0.13417763, + -0.02462848, + 0.05411647, + -0.15435664, + 0.04004719, + -0.17902878, + 0.16244677, + -0.17061608, + -0.12348929, + 0.09180731, + 0.11858081, + 0.04388564, + -0.050241712, + -0.3794224, + -0.023072392, + -0.15187621, + -0.06022391, + 0.0389537, + -0.11311294, + -0.04483756, + -0.03191812, + -0.09286949, + 0.11451646, + -0.18656348, + 0.03569287, + -0.15736677, + 0.05963041, + -0.025123972, + 0.04769077, + 0.057496574, + 0.11523707, + -0.08233095, + -0.032269277, + -0.08317559, + 0.04284503, + 0.1099772, + 0.29609403, + -0.059371803, + -0.1258368, + 0.16488141, + 0.029701294, + -0.0871117, + 0.15366647, + 0.2648822, + -0.025175504, + 0.070027724, + -0.16725753, + -0.02301268, + 0.008337837, + -0.19743346, + -0.035971083, + -0.0139340125, + -0.0352934, + 0.2689722, + 0.012080048, + -0.11762006, + -0.21998496, + -0.09839513, + -0.06674763, + 0.04683752, + -0.11146877, + -0.04599476, + 0.14974731, + 0.05422002, + 0.012376041, + -0.14231902, + 0.030036619, + -0.12907441, + -0.22337681, + 0.035535805, + 0.06476209, + -0.18803062, + 0.04536368, + 0.24490592, + 0.18265249, + 0.22729625, + 0.10897362, + 0.11467191, + 0.05619355, + 0.09987299, + 0.06857587, + 0.12826896, + -0.16517407, + -0.075386114, + -0.25283834, + 0.042725638, + 0.109347545, + 0.014253335, + -0.23104662, + -0.13149749, + -0.25347164, + -0.0050595454, + 0.20712197, + -0.12284579, + -0.103947915, + 0.15963243, + 0.079586536, + 0.046046272, + 0.12228099, + -0.15825438, + -0.014796799, + 0.07996475, + 0.05532982, + -0.08836719, + -0.002184571, + -0.04528829, + -0.057088953, + 0.10949727, + 0.08618899, + 0.040242728, + -0.041859094, + -0.08223082, + -0.104844384, + -0.101053454, + -0.031028843, + -0.05574357, + 0.13322599, + 0.088604, + 0.107746825, + -0.14654899, + 0.07539338, + 0.013862365, + -0.09069706, + 0.1796719, + -0.016659677, + -0.04956159, + 0.27165785, + -0.11160872, + -0.2989888, + 0.09245948, + -0.098325185, + -0.24016048, + 0.52027804, + -0.064231955, + -0.005032319, + -0.035204574, + -0.054530594, + 0.0322089, + 0.18308656, + 0.08390766, + 0.043010466, + -0.15488197, + 0.08767366, + 0.004138853, + -0.045949485, + -0.1160313, + -0.07836478, + -0.0026241415, + -0.19997384, + -0.03938704, + -0.084648326, + -0.06306966, + -0.0952212, + 0.049785048, + -0.20149958, + -0.015529836, + 0.04424018, + 0.21384019, + -0.059389725, + 0.17041788, + 0.20170075, + 0.08455694, + -0.22512946, + 0.14936431, + 0.06936717, + 0.18910527, + 0.004311936, + -0.11028445, + 0.24718592, + -0.016151149, + 0.052220903, + -0.028108425, + 0.10092346, + -0.08499343, + -0.094193466, + 0.18906602, + -0.061153974, + 0.3099695, + 0.062192697, + 0.23875803, + 0.21906589, + 0.10406714, + 0.12670133, + 0.030697094, + -0.01277904, + -0.1554076, + -0.033657804, + -0.020216823, + 0.1222573, + 0.13814458, + 0.13875861, + -0.06178161, + 0.08828033, + -0.03528705, + 0.113277875, + 0.07364842, + -0.032665182, + 0.121137686, + 0.050435707, + -0.08710971, + -0.04304125, + -0.12949078, + 0.28858903, + 0.026131703, + 0.06545478, + -0.22433525, + 0.03651575, + -0.0020827467, + 0.10167747, + 0.052595302, + 0.03274926, + 0.0072248536, + 0.15257488, + -0.19761868, + -0.099838115, + -0.21286489, + -0.050637424, + 0.004828881, + -0.03982375, + -0.0035783183, + -0.03838706, + 0.13523293, + 0.061031763, + -0.0036069269, + 0.24715978, + 0.30907214, + -0.1487525, + 0.02132856, + 0.016107956, + 0.047291737, + -0.19320416, + 0.05628742, + 0.014700451, + -0.011124475, + -0.09308549, + 0.11523371, + -0.2443315, + 0.2313448, + -0.25128835, + 0.04598335, + 0.040417917, + -0.09665347, + 0.04653424, + -0.07673263, + 0.12003437, + -0.017339345, + -0.103531435, + 0.022593237, + -0.09782897, + 0.061050087, + -0.13742255, + -0.047381733, + -0.119925104, + 0.2250597, + 0.10950321, + -0.0006340932, + 0.15462278, + 0.16510288, + 0.26863286, + -0.105883054, + -0.047063254, + 0.028948596, + 0.17239384, + 0.056860864, + 0.017096903, + -0.04228211, + -0.29253602, + -0.16056961, + 0.036930505, + 0.024778143, + -0.17934024, + -0.057446398, + -0.053578038, + 0.049655244, + 0.18780835, + -0.14032418, + 0.05718126, + -0.050660625, + -0.07751246, + 0.1379321, + 0.031953067, + -0.11338463, + -0.06786446, + -0.033356477, + 0.20826721, + 0.09957642, + 0.17668569, + 0.037627928, + -0.08140153, + 0.19429025, + 0.17275688, + -0.016087506, + 0.010439479, + 0.26191953, + 0.16416778, + 0.046684586, + 0.014290053, + 0.27286097, + 0.054265194, + -0.11745537, + 0.014185265, + -0.05908016, + 0.020058561, + -0.08973265, + -0.19595727, + -0.08870827, + 0.07308811, + -0.17287685, + -0.1589863, + -0.09653189, + 0.0048071207, + -0.13724232, + -0.08037632, + -0.047368392, + -0.16322568, + -0.033112556, + -0.13488308, + 0.15020229, + 0.1550236, + 0.10108509, + 0.040011637, + -0.20094907, + -0.090874165, + -0.0939116, + 0.080748454, + 0.115656935, + -0.03451729, + -0.13090292, + -0.13617615, + 0.0730974, + 0.122894086, + -0.038970716, + 0.07054392, + -0.06039169, + 0.02978956, + 0.1455779, + -0.077636614, + -0.24305566, + -0.10163222, + -0.10691119, + 0.037590254, + -0.034741804, + -0.06074967, + 0.09297616, + 0.14142899, + -0.11809402, + -0.15412627, + 0.065455005, + 0.09679129, + 0.019191982, + 0.20682472, + -0.20068286, + 0.0781541, + 0.23031051, + -0.18984596, + -0.13075766, + -0.01675578, + -0.0005947731, + -0.07893368, + -0.090689294, + 0.016939832, + -0.22322744, + -0.20472436, + 0.22921742, + -0.09677777, + -0.08641651, + 0.10703374, + 0.02100416, + -0.056980975, + 0.028346613, + -0.14496057, + 0.010396989, + 0.07659849, + 0.07763536, + -0.019762618, + -0.009585714, + 0.14585073, + -0.21997699, + 0.058942523, + -0.05442393, + -0.05415349, + -0.2622914, + 0.0067495466, + 0.036460627, + -0.15851912, + -0.14385591, + 0.15922575, + 0.039899103, + 0.078442976, + 0.08507526, + 0.18382412, + 0.22244017, + -0.03963535, + 0.06646716, + -0.032181084, + 0.005002472, + -0.11370807, + 0.16499059, + -0.09635935, + -0.19694255, + 0.021960784, + -0.27661255, + -0.05737264, + 0.1593896, + 0.09378417, + 0.047629766, + 0.01862993, + 0.14267302, + 0.062311556, + 0.057066027, + -0.0026629304, + -0.0600522, + -0.07380065, + 0.039031584, + -0.16306129, + 0.029224617, + 0.10198643, + -0.050818797, + 0.08438627, + 0.03716007, + -0.0443205, + -0.09043356, + 0.06818091, + 0.056742877, + -0.09950669, + 0.07628545, + -0.106138945, + 0.051655952, + -0.11948614, + 0.10950823, + -0.042758968, + 0.14772765, + -0.07351131, + -0.039129984, + -0.13518056, + -0.010262115, + -0.0797974, + -0.22309522, + -0.041619502, + 0.017691836, + -0.26448753, + 0.048773687, + -0.0430489, + 0.116527185, + -0.092469476, + -0.102719575, + 0.19830416, + -0.0104989875, + -0.108391374, + -0.09932554, + 0.07718134, + 0.02247461, + -0.051388126, + 0.090667345, + -0.28471762, + 0.044905305, + 0.17018011, + 0.0883001, + 0.22816132, + -0.12958865, + 0.16366321, + -0.07649908, + -0.04187491, + 0.031642366, + 0.02797218, + -0.015719814, + 0.17807175, + 0.25526682, + -0.023230469, + -0.051920697, + 0.066116184, + -0.2540213, + 0.033616856, + 0.06835817, + 0.10677274, + -0.14640501, + 0.053824387, + -0.08425958, + 0.06938902, + 0.18427739, + 0.050563883, + -0.12735263, + 0.070671864, + -0.26665702, + 0.11795721, + -0.02624768, + -0.09283501, + 0.024224505, + -0.20036814, + 0.007591098, + 0.13092197, + -0.038981788, + 0.009218034, + 0.036291324, + 0.03673069, + -0.14297684, + 0.06404598, + -0.04193293, + 0.03271318, + -0.015589479, + -0.011117657, + 0.27253702, + 0.1462861, + 0.003484278, + 0.0281606, + 0.015206004, + -0.015842091, + -0.16562876, + 0.10613876, + 0.07942983, + -0.027591115, + 0.01765117, + -0.15868017, + -0.06422185, + -0.16085465, + 0.055914916, + -0.17279945, + -0.08020601, + 0.43271828, + -0.2222178, + -0.35625243, + -0.17985259, + 0.1014227, + -0.054314714, + -0.049593076, + -0.30154017, + 0.09971668, + 0.09626297, + -0.05056628, + 0.057815276, + -0.2127709, + 0.07285609, + -0.08156592, + -0.18285358, + 0.0055787605, + 0.050411053, + -0.04962786, + 0.034054108, + 0.025982443, + 0.042589404, + -0.024576876, + -0.090059645, + -0.30343422, + 0.035683475, + 0.056787435, + -0.18377149, + -0.07379626, + -0.06505526, + 0.044090565, + 0.119362965, + -0.08053844, + 0.21147929, + -0.03585316, + -0.044990774, + -0.04109666, + 0.08407934, + 0.23574571, + -0.2137305, + -0.10426127, + -0.01860227, + 0.12616, + -0.0708649, + 0.022942567, + 0.011244215, + 0.031471517, + 0.0930586, + 0.03214131, + 0.32868665, + 0.03207292, + 0.08603695, + 0.057994302, + -0.13777441, + 0.09182316, + 0.014774016, + 0.021573544, + 0.0076492485, + 0.17600806, + -0.1502997, + -0.24296235, + 0.20674296, + 0.04510429, + 0.22502612, + 0.031383082, + -0.061237697, + 0.050558675, + -0.06667626, + 0.021187605, + -0.09565701, + -0.03821771, + 0.16841638, + -0.100745626, + -0.05147383, + -0.14504951, + 0.019862775, + 0.12780803, + 0.004019054, + 0.03778424, + 0.014132394, + 0.12381858, + -0.04886466, + 0.20842083, + -0.052135132, + 0.031716947, + 0.11628498, + -0.11497015, + -0.17629582, + -0.070079945, + -0.11253226, + -0.07588835, + 0.038207293, + -0.27067733, + -0.13882011, + -0.107785426, + 0.018694866, + 0.08630753, + -0.09765054, + -0.05338821, + -0.14067116 + ], + [ + 0.08710954, + -0.17130332, + 0.18518627, + 0.034484398, + -0.015136474, + -0.11925981, + 0.13832627, + 0.34316385, + 0.119799584, + 0.098551616, + -0.038712684, + 0.07383681, + 0.06289237, + -0.03994811, + -0.16898105, + 0.10380005, + 0.054381933, + -0.01971661, + 0.13839778, + 0.16946474, + -0.13109885, + -0.06390203, + 0.061225582, + 0.030190721, + -0.23004332, + 0.067906775, + -0.18641347, + 0.058921356, + -0.14940989, + -0.14585148, + -0.05028953, + 0.23240112, + 0.053929616, + 0.03085122, + 0.035810877, + -0.06495588, + 0.04786252, + 0.04149577, + 0.0872867, + -0.3250427, + -0.04962545, + 0.11174253, + 0.002483554, + 0.038309716, + 0.034978088, + -0.04172436, + 0.1876067, + -0.07495243, + -0.027371367, + 0.14645548, + -0.033528112, + -0.06964625, + -0.055938557, + 0.17258711, + 0.043243863, + -0.13666941, + -0.050095692, + -0.17020623, + 0.23184434, + 0.053598404, + 0.011893593, + -0.13508283, + 0.10040416, + -0.019216675, + 0.2241509, + -0.23771235, + -0.060662683, + 0.09703199, + -0.112910956, + -0.18161695, + -0.06245686, + -0.33224294, + -0.041275404, + 0.034993168, + 0.075077124, + 0.069832034, + 0.12258471, + 0.037918355, + -0.035581246, + -0.03809896, + 0.106061384, + 0.037776574, + 0.2677748, + 0.017203985, + -0.19279915, + -0.060687363, + 0.1294681, + 0.0033134802, + 0.20178464, + -0.40706864, + 0.034403134, + 0.14862113, + -0.1569005, + -0.06778971, + -0.08919004, + 0.06502744, + 0.20957167, + -0.07431113, + -0.009827447, + 0.14637196, + -0.043831594, + 0.20640296, + -0.12957036, + -0.10145491, + 0.21949573, + -0.1981791, + -0.114104114, + 0.057041492, + -0.14220239, + -0.06584, + 0.08439022, + -0.13847621, + -0.0043671746, + -0.1280747, + -0.2209941, + -0.06541769, + 0.098690294, + 0.12322519, + 0.3025779, + 0.110712335, + -0.13243516, + 0.07160524, + 0.17707202, + 0.11139699, + 0.1694134, + -0.078767486, + -0.13430579, + -0.24052751, + -0.0995972, + -0.22442292, + -0.14696601, + 0.011035744, + -0.005659949, + -0.02199578, + 0.058117013, + -0.022728508, + -0.05242452, + -0.06867182, + -0.23656839, + 0.037021764, + 0.14837083, + -0.30180052, + -0.021341268, + -0.00328619, + -0.14210492, + 0.34642032, + -0.080925204, + 0.025348207, + 0.040403865, + -0.08036848, + -0.026028905, + -0.13547285, + -0.13658845, + 0.08929891, + 0.17939393, + -0.16283756, + 0.20934752, + 0.2804383, + -0.042361386, + 0.0037535643, + -0.08561987, + 0.2610427, + -0.16887006, + -0.0804164, + -0.016786562, + -0.0086533185, + 0.10278585, + 0.21424574, + -0.06778377, + -0.07107834, + -0.023322714, + 0.038586013, + 0.15721613, + 0.010754087, + 0.02862506, + -0.043462764, + 0.2998912, + -0.058243856, + 0.11280774, + 0.02401249, + -0.10704567, + -0.0711847, + 0.22480705, + 0.009597078, + 0.10986745, + -0.18179896, + 0.1457943, + -0.42035487, + -0.06941299, + 0.033767574, + -0.015188636, + -0.14821082, + 0.048311736, + -0.009381438, + 0.058467768, + 0.22949363, + -0.17324351, + 0.09156386, + 0.09434916, + -0.10727511, + 0.17464036, + 0.1562835, + -0.12607798, + 0.0289634, + -0.14662074, + 0.16115622, + 0.006686656, + -0.31932178, + 0.08756386, + 0.30337483, + 0.009069734, + 0.16290994, + 0.2164364, + -0.099484086, + 0.13662401, + -0.13326345, + 0.081423216, + 0.16968766, + 0.0050415867, + 0.15819815, + -0.022486081, + -0.17130947, + -0.18132602, + -0.34165516, + 0.15751578, + 0.09290999, + 0.11265368, + -0.04221395, + -0.10326879, + -0.21709505, + 0.075194344, + 0.16578491, + -0.0228574, + 0.0064968187, + -0.24290843, + -0.17301916, + 0.0010360496, + -0.08306041, + 0.13103732, + -0.059812766, + 0.05592717, + -0.14495893, + -0.1183925, + 0.17915757, + -0.027477909, + -0.1789974, + -0.18129501, + -0.14337839, + 0.01016965, + -0.13785207, + -0.00971535, + 0.11592945, + 0.11714097, + -0.22951867, + 0.16646625, + -0.0031756428, + 0.06898733, + 0.015582822, + 0.11184965, + -0.11426219, + 0.15530753, + -0.12028912, + -0.08832732, + -0.0884876, + 0.01425523, + 0.010757544, + -0.041930813, + -0.056576736, + 0.030607315, + 0.13479832, + -0.07811483, + 0.058088824, + 0.00008467862, + 0.024450693, + 0.17910321, + 0.047229327, + -0.18732254, + -0.045666203, + 0.12912987, + -0.09150285, + 0.03890589, + -0.23575307, + 0.029016903, + -0.040441103, + -0.107373886, + -0.10694674, + 0.036835346, + -0.20357336, + -0.057427242, + 0.037563592, + -0.07291938, + -0.05247231, + 0.016445952, + 0.101862565, + -0.2436902, + -0.24760891, + -0.15459998, + 0.019506909, + -0.21173312, + -0.087886855, + -0.37426487, + -0.07298867, + -0.059791256, + -0.15710446, + -0.08189467, + 0.026206193, + 0.22261252, + -0.24138017, + 0.00890099, + 0.075444795, + 0.030475946, + 0.024316594, + 0.018440438, + -0.024992568, + -0.026872009, + 0.108047344, + 0.11162974, + 0.18837136, + -0.0804132, + -0.32997715, + -0.021764146, + 0.08130553, + -0.1064667, + -0.08348454, + -0.058926456, + 0.123180114, + -0.049904585, + 0.21330495, + 0.08938906, + -0.054655578, + 0.21257395, + 0.05705831, + -0.040726066, + 0.14231788, + 0.023860166, + 0.021737259, + -0.08625983, + -0.046179447, + -0.22169155, + -0.096731246, + 0.07239878, + -0.002620855, + -0.13451672, + -0.0035110759, + -0.100999646, + 0.045851093, + 0.115883544, + -0.040573798, + -0.0038128404, + -0.120611295, + 0.12036302, + 0.14173028, + -0.020041298, + 0.10767743, + 0.05669824, + -0.21955228, + -0.027810711, + 0.2384975, + 0.20769784, + 0.058436036, + 0.13167174, + -0.03997825, + 0.06490375, + -0.39682728, + -0.009131579, + 0.0038182035, + -0.062273886, + 0.06298824, + -0.055486213, + 0.07763187, + 0.18734314, + -0.31007335, + 0.10445774, + 0.0037420318, + 0.011656824, + 0.06310175, + 0.28920949, + 0.046282753, + -0.17289883, + 0.08784476, + -0.28481385, + 0.110921256, + 0.17077385, + -0.11537249, + 0.0025526553, + 0.06728107, + 0.03187741, + 0.24450828, + 0.013957283, + -0.18511797, + -0.28993872, + 0.18045323, + 0.19787051, + -0.08506221, + -0.071446285, + -0.116508596, + -0.19188306, + 0.17366874, + 0.13747092, + -0.030285724, + 0.24382776, + -0.16537562, + 0.17491636, + 0.11145762, + 0.0145614585, + -0.078121655, + -0.15256482, + 0.023452919, + -0.03198221, + 0.15302968, + -0.007858731, + 0.081439614, + -0.03475655, + 0.048131734, + 0.04107332, + -0.18774486, + -0.06102128, + -0.080510825, + 0.14107823, + 0.20489639, + -0.07205382, + 0.09343803, + -0.0017656386, + 0.15036091, + 0.37663803, + -0.0154301, + 0.10145759, + 0.04513303, + -0.15015142, + 0.23778017, + -0.17112422, + -0.011182279, + -0.1409076, + -0.106038846, + -0.020715188, + -0.013257168, + -0.01857237, + 0.10767726, + 0.018119281, + 0.017791973, + -0.185015, + -0.025156049, + -0.31205723, + 0.3476952, + 0.056714073, + -0.13445053, + 0.049857687, + 0.077027015, + -0.086866625, + -0.23081313, + -0.07399999, + -0.07555021, + -0.0629564, + 0.076873794, + 0.19624448, + 0.16596378, + -0.038202465, + 0.1258459, + 0.24286169, + 0.011493793, + 0.007160016, + -0.029425204, + 0.16966641, + 0.12608604, + 0.023404934, + -0.025459873, + -0.011254301, + 0.13206586, + -0.019274868, + 0.09385937, + -0.010656316, + -0.065400034, + -0.14098474, + 0.03497467, + 0.1471968, + -0.08455206, + -0.14741223, + 0.070851095, + 0.021002669, + -0.19233584, + -0.016548818, + 0.035071168, + 0.015510567, + -0.07111342, + -0.0008209783, + -0.17425494, + -0.22124435, + 0.073263645, + 0.14981963, + 0.20124836, + 0.053815912, + -0.013117615, + -0.06897596, + -0.025157409, + 0.008126072, + 0.06966679, + 0.19344465, + 0.091212906, + -0.14537519, + 0.2256169, + 0.05493476, + -0.16982616, + -0.2294222, + -0.020211222, + -0.30311102, + 0.20485239, + -0.0036436268, + -0.38836393, + 0.113643125, + -0.00074193213, + 0.014368115, + 0.16776425, + 0.16274397, + -0.24023412, + -0.0065946416, + -0.08178381, + -0.062067654, + 0.050150722, + 0.08572824, + -0.049099766, + 0.038218405, + 0.13753733, + -0.27165624, + -0.06660632, + -0.0031024336, + 0.00077022106, + 0.109839894, + -0.13052694, + 0.12741303, + -0.02386286, + 0.027174694, + -0.056120556, + -0.22025867, + -0.055497155, + 0.26162866, + 0.05429381, + 0.022575926, + 0.14617692, + -0.48583928, + -0.026855517, + -0.035127908, + -0.0994388, + 0.06336697, + -0.0013796594, + -0.18111397, + -0.11681097, + 0.05168338, + 0.09935661, + -0.13242848, + -0.25558785, + 0.02271768, + 0.15886833, + 0.1357752, + 0.020422582, + 0.05690636, + 0.10379993, + 0.015206449, + 0.042851932, + 0.14122868, + 0.03441443, + 0.17019148, + 0.09627194, + -0.11068253, + -0.17213245, + 0.10069426, + -0.16084917, + -0.20603392, + 0.1537688, + 0.18053167, + 0.012052561, + -0.14493415, + 0.009886556, + -0.100698516, + 0.13298142, + -0.0011233576, + 0.09706364, + 0.11539116, + 0.08948318, + -0.16341476, + 0.076917924, + 0.08169475, + -0.095494755, + -0.15150273, + -0.17631614, + -0.04698625, + -0.10497289, + -0.1551321, + 0.19398408, + -0.04757996, + 0.14968508, + 0.14501125, + 0.3260735, + -0.09190358, + -0.09226908, + -0.29598865, + -0.24046373, + -0.014669314, + -0.064815916, + -0.13214196, + 0.092982225, + -0.06129808, + -0.020856455, + 0.07223847, + 0.15493158, + 0.045643926, + 0.16655073, + 0.07079807, + 0.04770307, + -0.27394435, + -0.017774954, + -0.26278344, + 0.060371034, + -0.1546492, + -0.05711425, + -0.14442088, + 0.10931744, + 0.013118476, + 0.20232604, + -0.21293196, + 0.028680857, + 0.065254785, + 0.10172164, + 0.19185011, + -0.033591248, + -0.009484618, + -0.12940541, + -0.05190162, + -0.0575119, + -0.11933636, + 0.08953299, + -0.30165416, + -0.37290353, + -0.023061203, + -0.08670423, + -0.028334, + 0.06946971, + 0.21275698, + 0.17013383, + -0.08542582, + 0.045555767, + -0.015354397, + -0.1544775, + 0.347188, + -0.1362879, + 0.031593885, + -0.063385, + 0.07919194, + 0.03388464, + -0.07173834, + 0.038540523, + -0.07981687, + -0.09306369, + 0.15257466, + 0.11125348, + 0.03917901, + 0.02519694, + -0.0038859278, + -0.033267844, + -0.24917944, + -0.23705544, + -0.10430534, + -0.05478128, + -0.061792918, + -0.38342047, + -0.08392769, + 0.0018478394, + 0.06320872, + -0.00330005, + -0.09136997, + -0.14779916, + 0.018767918, + -0.17030604, + 0.48196173, + 0.07686573, + 0.013626745, + -0.021895628, + 0.28598508, + 0.13888015, + 0.11149308, + -0.13487104, + -0.23114635, + 0.2573732, + -0.059814, + 0.14756495, + 0.064012565, + -0.08390431, + -0.08256339, + 0.17762811, + 0.065454476, + -0.017735574, + -0.18665063, + -0.0832109, + -0.13875094, + -0.120969504, + 0.034954272, + -0.022674376, + -0.15303893, + -0.0941168, + -0.07137932, + -0.1681534, + -0.17633656, + -0.059532795, + -0.113158494, + 0.006298596, + -0.09926128, + 0.0132873, + 0.13816498, + 0.023504218, + -0.09607285, + -0.013130624, + 0.15760157, + 0.14307156, + -0.058194783, + 0.13315538, + 0.17142235, + 0.014921406, + -0.09991027, + 0.08165643, + -0.03796723, + 0.0028694954, + 0.37064257, + 0.13076137, + -0.09495863, + 0.048868448, + -0.30687767, + 0.058660306, + 0.02056709, + -0.021638418, + -0.02370854, + -0.16527307, + -0.0034045777, + -0.11812108, + -0.052942898, + 0.01853251, + 0.04409421, + 0.08230125, + -0.072519235, + -0.08816045, + -0.01623577, + 0.017485151, + -0.023712784, + -0.17598683, + -0.036672812, + 0.52452826, + -0.030806748, + 0.18347956, + -0.040979996, + -0.03811666, + 0.25729805, + -0.048534125, + -0.2373455, + 0.14856055, + 0.084020026, + -0.048066158, + 0.18652984, + -0.056927446, + -0.29441887, + 0.08345022, + -0.0005996005, + 0.031345766, + 0.27583638, + 0.13197714, + -0.04237792, + 0.24010932, + 0.12292464, + -0.0576011, + 0.006550551, + 0.104579575, + 0.40900844, + -0.18993206, + -0.0060352357, + 0.021329733, + -0.122469254, + 0.05516141, + -0.10217497, + 0.17811313, + 0.22834168, + -0.0727938, + -0.06289365, + -0.0612119, + 0.0885646, + 0.08026745, + -0.0053653144, + 0.18809816, + 0.0068571805, + 0.004589373, + 0.26332837, + 0.19965443, + 0.023766935, + 0.075609, + 0.022701783, + 0.24352224, + -0.12722453, + -0.0043050097, + -0.22488694, + -0.04397989, + -0.12078576, + -0.30110332, + 0.07128283, + -0.16407505, + 0.04377039, + -0.016103752, + 0.2800282, + -0.11904939, + 0.20880961, + -0.014415037, + -0.24259716, + 0.2187837, + 0.018473664, + 0.05934518, + -0.12305252, + 0.23674639, + -0.026103668, + -0.1798324, + -0.21792579, + -0.0977618, + -0.050768767, + -0.016194953, + 0.0853139, + -0.049470134, + 0.045779597, + 0.090835765, + 0.049913734, + -0.11232327, + -0.21321629, + 0.0716266, + -0.003955598, + 0.22519064, + -0.009941131, + 0.19688989, + 0.008872793, + -0.02055702, + 0.09081416, + -0.188783, + 0.11419467, + -0.16248907, + -0.044919066, + -0.07653471, + 0.12145175, + 0.1500241, + 0.13012478, + -0.057526883, + 0.21240865, + 0.105857186, + -0.076245934, + -0.02195677, + 0.06403661, + -0.23577379, + -0.12655307, + -0.11207305, + 0.045876216, + 0.06294753, + -0.16967238, + -0.16886263, + 0.20803007, + -0.071986, + -0.14971863, + -0.039650787, + 0.18356499, + -0.12795182, + 0.2708319, + -0.036815282, + 0.049104527, + -0.122806, + 0.01948398, + -0.07615092, + -0.20949979, + 0.20418935, + 0.07240039, + 0.11423065, + -0.027510138, + 0.17900626, + -0.118062556, + -0.19533488, + 0.3006342, + 0.13511322, + 0.053908553, + 0.20639807, + -0.10189352, + -0.05439522, + -0.17225397, + 0.1793962, + -0.13950582, + 0.17308019, + 0.10287988, + -0.13179243, + -0.11794216, + 0.06293473, + -0.085337915, + 0.11593874, + -0.014016872, + -0.15914717, + 0.08646496, + -0.22406147, + 0.2338852, + 0.17168793, + -0.066047914, + -0.006837371, + 0.0618302, + -0.053112675, + -0.12019638, + 0.03951125, + -0.1434713, + 0.05452687, + -0.16260725, + -0.115297765, + 0.124994054, + -0.24352738, + 0.0137317935, + -0.019007314, + 0.0733735, + 0.0527939, + 0.014726399, + 0.07390042, + 0.006591537, + -0.11127264, + -0.02154871, + -0.030611224, + 0.068587966, + -0.05893293, + 0.23582877, + -0.0012231775, + 0.09105914, + 0.10211014, + -0.2383829, + 0.2864423, + 0.14851955, + 0.19865727, + -0.059792183, + 0.06848801, + 0.13465886, + 0.1286977, + 0.0067498614, + -0.04016777, + 0.20123987, + 0.061112855, + 0.18267728, + -0.071847156, + -0.11303111, + 0.3798, + 0.11107034, + 0.036331534, + -0.009910368, + -0.06711268, + -0.030826552, + 0.00086975336, + 0.011719601, + 0.13172266, + 0.3593029, + 0.036299367, + -0.106546186, + 0.10521198, + 0.07050957, + -0.0057226345, + -0.10432761, + -0.12922813, + 0.059869226, + -0.19951992, + -0.25756156, + -0.220191, + 0.022794224, + 0.028878445, + -0.11174812, + 0.19299944, + 0.13134119, + -0.017544553, + -0.028695328, + 0.25374243, + 0.15130068, + -0.10371478, + -0.06168496, + -0.07604035, + 0.028159369, + -0.20027314, + -0.1225441, + -0.15416595, + -0.060561605, + 0.0145160835, + -0.17459767, + 0.32457855, + 0.15947826, + 0.04655447, + -0.25514564, + 0.08199647, + 0.11665995, + -0.21419841, + -0.14407481, + -0.28173584, + -0.05530346, + 0.11907159, + -0.03749379, + 0.13516708, + -0.122134365, + -0.07007785, + 0.03407505, + -0.18017942, + -0.009898455, + 0.11660031, + 0.2261957, + -0.13030855, + -0.08201036, + -0.037830215, + 0.023683347, + -0.052337818, + -0.08290516, + 0.049878456, + -0.105940744, + -0.03673183, + -0.04835046, + -0.14271665, + -0.03498139, + -0.025459593, + 0.22258753, + 0.07466624, + 0.028863654, + -0.032301944, + 0.22064988, + 0.15382591, + 0.08489334, + 0.06317825, + 0.06779215, + 0.037101578, + 0.10427194, + 0.0017950593 + ], + [ + 0.09922909, + 0.19329362, + -0.089205086, + 0.0902681, + 0.020661248, + 0.055564135, + -0.12960309, + -0.14453314, + -0.057020947, + 0.00055327086, + -0.04725846, + 0.041075412, + -0.074861415, + 0.05147266, + -0.025786527, + 0.014583264, + -0.020930862, + 0.092485875, + -0.110546716, + -0.10858244, + 0.03829249, + -0.0043910975, + 0.007389824, + -0.13461338, + -0.05783951, + 0.034733385, + 0.020813826, + -0.053271953, + -0.115472406, + -0.042540982, + -0.068776384, + -0.15169284, + 0.056827094, + 0.017661918, + -0.1840523, + 0.021772237, + -0.12447468, + 0.07815046, + 0.03433609, + 0.21081972, + 0.10052267, + 0.09689305, + 0.171697, + 0.07907441, + -0.14405127, + -0.039743524, + -0.16300403, + 0.16113685, + 0.13933033, + -0.16585381, + -0.029853264, + 0.0029296472, + 0.07432966, + -0.042239234, + 0.0012049297, + -0.1007821, + 0.04139244, + 0.08449067, + -0.26597312, + -0.267746, + 0.06668256, + 0.046715535, + 0.09555873, + -0.05757943, + -0.24467292, + 0.1794307, + -0.07938974, + 0.029747458, + 0.028291203, + 0.024331363, + 0.10153997, + 0.16175944, + -0.042946056, + -0.021827083, + 0.06844339, + -0.18942854, + -0.13707602, + -0.06688024, + 0.07008942, + 0.1516177, + -0.032231364, + -0.012634971, + -0.15755424, + -0.1480545, + 0.11373915, + -0.10380098, + 0.025902417, + 0.02154159, + -0.12452926, + 0.36185467, + 0.028103083, + -0.16448122, + 0.21683855, + 0.036937624, + 0.12640364, + 0.049958993, + 0.0033274589, + 0.23469786, + -0.073469125, + -0.111896284, + 0.026556036, + -0.121715285, + -0.13547362, + 0.0043646568, + -0.06723719, + 0.13112622, + 0.061854582, + 0.00827685, + 0.18235683, + 0.056204185, + 0.21077286, + 0.03347793, + -0.06673879, + 0.08930961, + 0.14287215, + -0.019558305, + -0.023028698, + -0.04125462, + -0.13874297, + -0.17873141, + -0.06033782, + -0.046079062, + -0.21422128, + 0.009756932, + -0.0037566265, + -0.011301864, + -0.031679, + 0.105446085, + -0.027635487, + 0.15984066, + 0.0028714947, + -0.062425885, + -0.08742873, + 0.027317606, + 0.091146655, + -0.07750772, + 0.00001997856, + -0.04935356, + 0.1724698, + -0.0016211232, + -0.031620953, + 0.18991055, + -0.102658816, + -0.0569686, + 0.14355817, + -0.02570266, + 0.08205178, + 0.029490678, + -0.09032867, + 0.08632978, + 0.11434846, + 0.050010186, + 0.032725327, + 0.12887496, + -0.22924386, + 0.23863578, + -0.18638536, + -0.11488565, + -0.056229144, + -0.13855264, + 0.030596541, + -0.23371081, + 0.21031708, + 0.12080758, + -0.053702675, + -0.077903844, + -0.06315739, + -0.3252722, + 0.046635125, + 0.23390658, + -0.11890019, + 0.0398968, + 0.06454974, + 0.03364276, + 0.052762296, + 0.012723345, + -0.18457334, + 0.1559928, + -0.030700224, + -0.07028264, + 0.03958868, + -0.21390094, + 0.10622579, + -0.08809487, + -0.08636894, + 0.2845554, + 0.026649011, + 0.09251134, + 0.00845717, + 0.063250445, + -0.091051705, + -0.042655118, + 0.067618415, + -0.042166475, + -0.07808972, + -0.17881471, + 0.091531634, + -0.05500939, + -0.06215904, + -0.15168588, + -0.21878763, + -0.010851546, + -0.03942697, + 0.009794777, + 0.071002096, + 0.030673252, + -0.0036948097, + 0.15150587, + 0.026615286, + -0.071532965, + 0.14270544, + 0.031761274, + 0.019514496, + 0.18437156, + -0.0869769, + 0.24016027, + -0.15644383, + -0.21562234, + -0.028324183, + 0.01982025, + 0.23802343, + 0.023614297, + 0.06620521, + 0.20037079, + 0.030448142, + 0.038082566, + 0.19502579, + 0.07739141, + -0.05200885, + 0.06163384, + -0.055865686, + -0.12154154, + -0.045388855, + 0.07095126, + 0.0066120788, + 0.0042130393, + 0.016833203, + -0.01450416, + -0.05324384, + 0.020177407, + 0.1351836, + 0.0006332288, + 0.17686254, + 0.08819308, + -0.042174343, + 0.03003325, + -0.046552762, + 0.14691816, + -0.029027943, + 0.030122029, + 0.10378191, + 0.07062717, + -0.035100482, + 0.2663682, + -0.21149327, + 0.0045717624, + -0.1489291, + 0.09539345, + -0.0056779007, + 0.06230959, + 0.011345457, + -0.022593074, + 0.004680072, + 0.03946722, + -0.07103394, + 0.1076795, + -0.04525275, + 0.08875799, + -0.056481507, + -0.07084696, + 0.08718119, + -0.010572039, + -0.23335618, + 0.022337815, + -0.21947427, + -0.033690974, + 0.10553345, + -0.021017643, + 0.039848566, + 0.10650752, + -0.023180155, + 0.09168934, + -0.100681156, + -0.002023562, + 0.017897502, + 0.014689954, + 0.21349426, + 0.13252033, + 0.16629137, + -0.12498875, + 0.029900944, + 0.034571122, + -0.1183231, + -0.08556117, + 0.18236583, + -0.030042706, + 0.21529557, + 0.013976496, + 0.13808198, + 0.08218435, + 0.03897052, + -0.0309562, + -0.103948675, + 0.06192161, + 0.0003125374, + 0.06724196, + -0.2442015, + 0.09110316, + 0.070472196, + -0.05605174, + 0.026148817, + 0.026922356, + 0.03239555, + -0.10931005, + -0.079498544, + -0.05027006, + -0.14949176, + 0.05897984, + -0.026259027, + 0.33303934, + 0.021910945, + 0.0052683633, + -0.006941438, + -0.1482114, + 0.03904678, + -0.088498645, + 0.051959783, + -0.052535605, + 0.053497687, + 0.044573534, + -0.14639854, + 0.04488398, + 0.18228793, + -0.24426037, + 0.064073786, + -0.13068326, + -0.07520704, + 0.168466, + 0.080884494, + -0.15019079, + -0.22186117, + -0.09016358, + -0.024420097, + 0.011978521, + 0.07390882, + -0.17250529, + 0.020350983, + 0.1312874, + -0.12515071, + -0.091161884, + -0.14904639, + 0.012078115, + 0.020656077, + 0.03169497, + -0.09332102, + 0.17395759, + 0.051100943, + -0.0682836, + -0.12042831, + -0.042630278, + -0.15859638, + -0.038498797, + -0.02163074, + 0.037809588, + -0.051014822, + 0.15335885, + -0.07023838, + -0.059480634, + -0.13480279, + 0.0022430932, + -0.05400732, + 0.11481892, + -0.03943057, + 0.03376113, + 0.029365588, + -0.01985839, + 0.06586234, + 0.17098652, + 0.010993923, + 0.07071217, + 0.15091984, + 0.07246314, + -0.07753422, + -0.014183399, + 0.05115523, + -0.06792704, + -0.028603341, + 0.042584494, + 0.0427936, + -0.046991456, + -0.012732995, + -0.093626425, + -0.025454156, + 0.118326806, + 0.01725479, + 0.019332752, + 0.022475427, + -0.046046827, + -0.14654095, + -0.08640246, + -0.03239433, + 0.11038407, + -0.016438928, + 0.038493652, + -0.00961758, + 0.03227294, + 0.018387197, + 0.0011755466, + -0.022134272, + 0.0013270652, + -0.032188423, + 0.0975892, + -0.1276902, + 0.12248453, + 0.08241598, + 0.09593749, + -0.05755952, + 0.03662516, + -0.09083649, + 0.17452618, + 0.09512618, + 0.058438238, + 0.06198956, + -0.18931474, + -0.26352504, + 0.060267664, + -0.06953952, + 0.047736462, + 0.035634864, + -0.05121662, + 0.13543147, + 0.16854906, + 0.08127721, + 0.1311628, + -0.026975583, + -0.044239435, + -0.09666469, + -0.025346389, + 0.014149978, + 0.06538364, + 0.14216997, + -0.08482117, + 0.01596345, + -0.28832352, + 0.069122784, + -0.030430982, + -0.07955898, + 0.010084634, + -0.06679984, + -0.034068897, + 0.09917545, + 0.005522545, + 0.23021385, + -0.053861145, + -0.20458226, + 0.031469673, + 0.07417353, + -0.111911826, + -0.20756844, + -0.28046608, + -0.019240063, + 0.14704527, + 0.050318588, + -0.027690839, + 0.0433427, + -0.021377675, + 0.122723065, + -0.08607106, + -0.13047236, + -0.1480793, + -0.0017197174, + 0.207719, + 0.11094812, + 0.094099835, + 0.07618005, + 0.049016297, + 0.08265005, + 0.11717951, + -0.06636636, + -0.052570045, + -0.16610357, + -0.2623675, + -0.12448414, + -0.04355849, + -0.055372536, + 0.074382015, + 0.15266834, + -0.20153268, + 0.0153543055, + -0.1258622, + 0.19902235, + -0.029608008, + -0.040371638, + 0.010904064, + 0.2229205, + 0.061830718, + 0.06002709, + -0.08615332, + -0.06174688, + -0.102771245, + 0.049013123, + 0.010193833, + 0.14983569, + -0.025835007, + 0.18083006, + -0.04659801, + 0.018440437, + 0.30839917, + -0.16897294, + 0.08910914, + -0.012183556, + -0.122475974, + -0.10565502, + 0.13073695, + -0.07959442, + 0.041541137, + 0.10392677, + 0.032080103, + 0.019116145, + 0.1501532, + -0.007189581, + -0.081793725, + 0.13843022, + -0.021997655, + -0.10464448, + 0.14577876, + -0.18523328, + 0.11666456, + -0.03671598, + -0.15580918, + -0.0105150165, + 0.10568207, + -0.05139898, + 0.167106, + 0.037360042, + -0.1467535, + 0.07574925, + 0.09398352, + -0.03443882, + 0.09108746, + 0.04016023, + 0.13464329, + -0.008836393, + -0.030829234, + -0.0019725922, + 0.03290327, + -0.09469384, + 0.055525407, + 0.04475484, + 0.25144902, + 0.023231843, + -0.042837147, + -0.057410453, + -0.017798437, + 0.14306754, + -0.06441287, + 0.069441855, + 0.020217743, + -0.046007477, + -0.08419948, + 0.031308196, + -0.08074212, + 0.06644234, + -0.041707832, + -0.04130455, + -0.009568846, + 0.004333094, + -0.23832574, + 0.0445979, + -0.1614168, + 0.07556689, + -0.1989918, + 0.09638659, + -0.022696951, + -0.24606256, + -0.08091246, + -0.16761199, + -0.06137476, + 0.06249124, + 0.008075503, + 0.012498684, + -0.093571134, + 0.21265674, + -0.13365346, + -0.015206457, + -0.13378516, + -0.0639338, + -0.2980513, + -0.07912139, + -0.18038224, + -0.13223216, + -0.17066601, + 0.12556143, + 0.11248599, + 0.17373152, + 0.10231915, + -0.12408944, + 0.12659758, + 0.043861717, + -0.057695147, + -0.051979873, + -0.05279198, + -0.039573293, + -0.27606925, + -0.096079595, + -0.07944101, + -0.027756765, + 0.08178766, + -0.014644812, + -0.008356735, + 0.19732872, + 0.16396426, + 0.118133426, + 0.059197, + 0.04274339, + -0.16191266, + -0.04586772, + -0.2095507, + 0.060357083, + 0.1689379, + 0.034583356, + 0.11114319, + -0.14121269, + 0.02876236, + 0.04930838, + 0.13298379, + 0.090288736, + -0.0777211, + 0.058304645, + -0.085926, + 0.054494467, + 0.0638312, + 0.17181379, + 0.065375715, + 0.012226054, + -0.116761416, + -0.01955284, + -0.22642134, + 0.07072532, + -0.034431268, + 0.10843991, + 0.039243765, + -0.10285648, + -0.095056795, + 0.21969451, + 0.017401649, + -0.11960973, + 0.06276881, + 0.025204107, + 0.038192134, + -0.04021747, + 0.11040308, + -0.049043182, + -0.13384683, + 0.05864996, + -0.08624704, + 0.14130844, + 0.08064963, + 0.36910453, + 0.011995751, + -0.00519787, + 0.055415392, + -0.09282995, + 0.21831799, + -0.18470502, + 0.10403515, + -0.016145466, + -0.02564856, + -0.081023864, + 0.090938374, + -0.035864808, + 0.21258816, + -0.18942566, + 0.083703935, + -0.050557304, + -0.002882522, + -0.10664499, + -0.081433766, + -0.05791506, + 0.085215755, + 0.043338016, + -0.11704907, + 0.002632758, + -0.09690419, + 0.013499867, + -0.054027792, + 0.05061038, + -0.06424348, + 0.0024100016, + 0.051092062, + -0.02161671, + -0.016365506, + -0.037934802, + 0.08334156, + 0.046447176, + -0.17161596, + -0.019718044, + 0.11020432, + 0.06093988, + -0.093766004, + 0.012168797, + 0.012848215, + 0.09886849, + -0.2791596, + 0.044996105, + 0.10416806, + -0.15235019, + 0.03557597, + 0.07601429, + 0.102863245, + 0.038355656, + -0.05436331, + -0.01489334, + 0.03972165, + -0.0124360835, + 0.081610486, + 0.09510311, + 0.055585824, + 0.1183436, + 0.04449894, + -0.20741712, + -0.09764877, + 0.22984175, + -0.19907053, + 0.15185396, + -0.15974532, + -0.060578734, + 0.22258739, + 0.11458275, + 0.2591845, + -0.0773439, + 0.0024641708, + 0.087460205, + 0.11237043, + 0.09208186, + -0.15539859, + -0.050374884, + 0.12713112, + -0.054308232, + 0.04290657, + -0.0060767806, + 0.030408895, + 0.1143094, + -0.28147283, + 0.13243896, + -0.076568425, + 0.0033897345, + 0.072858445, + -0.19654825, + -0.044442017, + 0.09591671, + -0.030466663, + 0.07010624, + -0.017388823, + -0.28332117, + 0.03773549, + 0.08759413, + 0.11723249, + -0.07755449, + -0.26165622, + -0.08599037, + -0.0012194687, + 0.05913376, + -0.23951457, + -0.043991003, + 0.1482903, + -0.02349037, + 0.11864776, + -0.20428394, + -0.0392854, + 0.10281299, + 0.06508676, + 0.01543562, + -0.07616559, + 0.15915586, + -0.20645969, + -0.08338114, + 0.06239686, + -0.01370483, + -0.016423445, + -0.06880195, + -0.07068177, + -0.1404855, + 0.03187874, + -0.065799706, + 0.049834546, + -0.20917483, + 0.06263697, + -0.030516472, + -0.11206959, + 0.1358173, + -0.09966633, + -0.03200116, + -0.03559406, + 0.14644402, + -0.041095372, + -0.063038506, + 0.07866325, + -0.031647455, + 0.097607866, + -0.011589293, + 0.011101283, + -0.1663201, + -0.045941252, + -0.11245013, + 0.21135762, + 0.22063637, + 0.057828937, + 0.03889893, + -0.21873465, + 0.029268285, + -0.28437626, + 0.007473741, + 0.037159376, + 0.1556142, + 0.0406958, + 0.053431734, + 0.07624715, + -0.011513231, + 0.010438517, + 0.117281705, + -0.120060414, + -0.15190017, + 0.16314197, + 0.12882996, + -0.10878672, + 0.0482761, + -0.13475692, + -0.05823976, + -0.25363272, + 0.09063389, + -0.055728428, + 0.015324757, + 0.1371272, + 0.005291453, + 0.052980922, + 0.08767807, + -0.07119296, + -0.047940455, + -0.11089401, + 0.0050558276, + 0.06778891, + -0.13261117, + 0.117238015, + 0.117865466, + 0.004264933, + 0.2004508, + 0.18700008, + 0.16960195, + -0.004454132, + 0.046593163, + 0.03977202, + -0.028631704, + 0.17936163, + -0.09963885, + 0.17131142, + 0.07253733, + 0.01717618, + -0.13217688, + 0.037284497, + 0.013885482, + -0.008089996, + -0.21928462, + 0.034505904, + -0.24764518, + 0.20573963, + 0.045836683, + -0.12769024, + -0.030525493, + -0.14587297, + -0.00046203224, + -0.16328663, + -0.06000922, + -0.059931837, + -0.27740368, + -0.08319252, + -0.12002472, + 0.047623277, + 0.06827662, + -0.013962914, + 0.06548123, + -0.032991156, + 0.08568136, + -0.08882054, + -0.17226891, + -0.052484937, + 0.067378275, + 0.06441795, + 0.01466604, + 0.15071812, + -0.10394031, + 0.18539482, + 0.0063700625, + 0.19983687, + -0.03351717, + -0.17927904, + -0.0648741, + 0.045819163, + -0.07104821, + 0.016821377, + 0.083465725, + 0.1034657, + 0.07942522, + -0.012593937, + 0.12989402, + 0.13088727, + -0.113876365, + -0.029009558, + -0.1600178, + 0.015523028, + -0.101534136, + -0.06799992, + 0.0011157023, + 0.09172839, + -0.11273034, + 0.031842764, + 0.04913983, + 0.012960052, + 0.09009213, + 0.12315478, + -0.07497427, + -0.05469171, + 0.08174025, + -0.021904139, + -0.19433546, + -0.0642246, + 0.20773315, + -0.018804684, + -0.04163051, + -0.014173299, + -0.08506588, + 0.17284273, + -0.10646656, + -0.056095254, + -0.15067372, + -0.118928105, + 0.030093474, + -0.0010089284, + 0.19459692, + -0.1969463, + -0.11664909, + -0.08674267, + 0.059538268, + 0.033058587, + 0.0048441044, + -0.04345918, + 0.012857262, + -0.041662965, + -0.055868775, + -0.07198284, + 0.04975875, + 0.07855949, + 0.003286673, + 0.07077788, + 0.06023691, + 0.0098650325, + 0.020669239, + -0.011959251, + 0.29341477, + 0.26518163, + 0.018302424, + -0.11295785, + -0.123997554, + 0.020731002, + -0.027079932, + 0.036146816, + -0.09746471, + -0.18287756, + -0.17424327, + 0.09247057, + 0.030213475, + -0.017018242, + -0.060300678, + -0.1284134, + 0.090471216, + 0.06812903, + 0.002567259, + 0.12325836, + 0.08277443, + -0.33935252, + -0.18105176, + -0.05420371, + 0.0791375, + 0.06830327, + 0.1263024, + 0.0074555012, + 0.09897055, + 0.05670974, + 0.023920402, + -0.057833895, + -0.013064893, + -0.06849094, + 0.1009468, + 0.16573484, + 0.0041426737, + 0.011762998, + 0.110644124, + -0.06512651, + -0.08114622, + 0.11044569, + -0.04579765, + 0.033811156, + -0.06146758, + 0.038205404, + -0.040913455, + -0.0010137758, + -0.10248012, + 0.08886698, + 0.016633533, + 0.02643175, + 0.14995141, + 0.20175526, + -0.15250771, + 0.037865993, + 0.047024727, + -0.0059053525, + 0.050027758, + -0.015005728, + 0.02289207, + -0.08187314, + -0.15409966, + 0.060548972, + -0.050883785, + 0.13887611 + ] + ], + "bias": [ + 0.15158902, + 0.03748021, + -0.18906918 + ], + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.json new file mode 100644 index 0000000..0c56d62 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.json @@ -0,0 +1,434 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 157, + "accuracy": 0.77707005, + "confusion": [ + [ + 58, + 23, + 4 + ], + [ + 0, + 51, + 0 + ], + [ + 2, + 6, + 13 + ] + ], + "tool_precision": 0.96666664, + "tool_recall": 0.68235296, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0952381, + "memory_false_abstain_rate": 0.0, + "tool_count": 85, + "memory_count": 51, + "abstain_count": 21, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.27311695, + "p_memory": 0.5416255, + "p_abstain": 0.18525758 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.421286, + "p_memory": 0.37514666, + "p_abstain": 0.20356733 + }, + { + "query_id": "holdout_tools_006", + "text": "which crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38631028, + "p_memory": 0.43678588, + "p_abstain": 0.17690389 + }, + { + "query_id": "holdout_abs_001", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.58328974, + "p_memory": 0.2393619, + "p_abstain": 0.17734835 + }, + { + "query_id": "holdout_abs_003", + "text": "fix this pls", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5047037, + "p_memory": 0.19701391, + "p_abstain": 0.29828238 + }, + { + "query_id": "holdout_abs_004", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.23068972, + "p_memory": 0.55771327, + "p_abstain": 0.21159701 + }, + { + "query_id": "holdout_abs_006", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44023973, + "p_memory": 0.2884513, + "p_abstain": 0.27130887 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.32584342, + "p_memory": 0.4522681, + "p_abstain": 0.2218885 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47907954, + "p_memory": 0.32599244, + "p_abstain": 0.19492812 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42581502, + "p_memory": 0.2707809, + "p_abstain": 0.3034041 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48411453, + "p_memory": 0.33207738, + "p_abstain": 0.18380813 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41490114, + "p_memory": 0.39232263, + "p_abstain": 0.19277626 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.498076, + "p_memory": 0.2635602, + "p_abstain": 0.23836374 + }, + { + "query_id": "shell_test_019", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.48745245, + "p_memory": 0.17580202, + "p_abstain": 0.33674553 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44542202, + "p_memory": 0.17025512, + "p_abstain": 0.38432285 + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49682242, + "p_memory": 0.21907358, + "p_abstain": 0.28410402 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49529323, + "p_memory": 0.33562484, + "p_abstain": 0.16908191 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47192854, + "p_memory": 0.29691777, + "p_abstain": 0.2311537 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.35084134, + "p_memory": 0.5222145, + "p_abstain": 0.12694421 + }, + { + "query_id": "rf_test_010", + "text": "what's in the .gitignore for klbr?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.43999156, + "p_memory": 0.21865375, + "p_abstain": 0.3413547 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4496186, + "p_memory": 0.29366884, + "p_abstain": 0.25671256 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46442676, + "p_memory": 0.39125812, + "p_abstain": 0.14431503 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3966753, + "p_memory": 0.3473245, + "p_abstain": 0.2560003 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4669734, + "p_memory": 0.30586943, + "p_abstain": 0.22715719 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4187118, + "p_memory": 0.4667664, + "p_abstain": 0.114521824 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4648565, + "p_memory": 0.34376398, + "p_abstain": 0.19137956 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4916422, + "p_memory": 0.2993488, + "p_abstain": 0.20900898 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4452266, + "p_memory": 0.38029882, + "p_abstain": 0.17447457 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48173755, + "p_memory": 0.22332487, + "p_abstain": 0.2949376 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49144813, + "p_memory": 0.2733501, + "p_abstain": 0.2352018 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44083598, + "p_memory": 0.38097113, + "p_abstain": 0.17819287 + }, + { + "query_id": "abs_test_004", + "text": "what's the best programming language?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36859047, + "p_memory": 0.31743553, + "p_abstain": 0.313974 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.37783423, + "p_memory": 0.4140439, + "p_abstain": 0.20812187 + }, + { + "query_id": "abs_test_009", + "text": "どのモデルが今一番スコアが高いですか — ベンチマークサイトで", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36733645, + "p_memory": 0.37445688, + "p_abstain": 0.25820664 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38499674, + "p_memory": 0.4603757, + "p_abstain": 0.15462756 + } + ] + }, + "holdout_metrics": { + "total": 18, + "accuracy": 0.7222222, + "confusion": [ + [ + 5, + 1, + 0 + ], + [ + 0, + 6, + 0 + ], + [ + 2, + 2, + 2 + ] + ], + "tool_precision": 0.71428573, + "tool_recall": 0.8333333, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.33333334, + "memory_false_abstain_rate": 0.0, + "tool_count": 6, + "memory_count": 6, + "abstain_count": 6, + "misclassified": [ + { + "query_id": "holdout_tools_006", + "text": "which crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38631028, + "p_memory": 0.43678588, + "p_abstain": 0.17690389 + }, + { + "query_id": "holdout_abs_001", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.58328974, + "p_memory": 0.2393619, + "p_abstain": 0.17734835 + }, + { + "query_id": "holdout_abs_003", + "text": "fix this pls", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5047037, + "p_memory": 0.19701391, + "p_abstain": 0.29828238 + }, + { + "query_id": "holdout_abs_004", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.23068972, + "p_memory": 0.55771327, + "p_abstain": 0.21159701 + }, + { + "query_id": "holdout_abs_006", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44023973, + "p_memory": 0.2884513, + "p_abstain": 0.27130887 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.md new file mode 100644 index 0000000..032b8b1 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-3/router_report.md @@ -0,0 +1,81 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.500` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `157` +- tools: `85` memory: `51` abstain: `21` +- accuracy: `0.7771` +- tools precision: `0.9667` +- tools recall: `0.6824` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0952` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 58 | 23 | 4 | +| memory | 0 | 51 | 0 | +| abstain | 2 | 6 | 13 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.27 | 0.54 | 0.19 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.42 | 0.38 | 0.20 | +| `holdout_tools_006` | which crate defines the LlmClient type? | `Tools` | `Memory` | 0.39 | 0.44 | 0.18 | +| `holdout_abs_001` | what's the latest NVIDIA driver version today? | `Abstain` | `Tools` | 0.58 | 0.24 | 0.18 | +| `holdout_abs_003` | fix this pls | `Abstain` | `Tools` | 0.50 | 0.20 | 0.30 | +| `holdout_abs_004` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.23 | 0.56 | 0.21 | +| `holdout_abs_006` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.44 | 0.29 | 0.27 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.33 | 0.45 | 0.22 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.48 | 0.33 | 0.19 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Abstain` | 0.43 | 0.27 | 0.30 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.33 | 0.18 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.41 | 0.39 | 0.19 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Memory` | 0.50 | 0.26 | 0.24 | +| `shell_test_019` | is the klbr tui currently connected to the daemon socket? | `Tools` | `Abstain` | 0.49 | 0.18 | 0.34 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.45 | 0.17 | 0.38 | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | 0.50 | 0.22 | 0.28 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.50 | 0.34 | 0.17 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.47 | 0.30 | 0.23 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.35 | 0.52 | 0.13 | +| `rf_test_010` | what's in the .gitignore for klbr? | `Tools` | `Abstain` | 0.44 | 0.22 | 0.34 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.45 | 0.29 | 0.26 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.46 | 0.39 | 0.14 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.40 | 0.35 | 0.26 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.47 | 0.31 | 0.23 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.42 | 0.47 | 0.11 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.46 | 0.34 | 0.19 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.49 | 0.30 | 0.21 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.45 | 0.38 | 0.17 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.48 | 0.22 | 0.29 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.49 | 0.27 | 0.24 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.44 | 0.38 | 0.18 | +| `abs_test_004` | what's the best programming language? | `Abstain` | `Memory` | 0.37 | 0.32 | 0.31 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.38 | 0.41 | 0.21 | +| `abs_test_009` | どのモデルが今一番スコアが高いですか — ベンチマークサイトで | `Abstain` | `Memory` | 0.37 | 0.37 | 0.26 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.38 | 0.46 | 0.15 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `18` +- tools precision: `0.7143` +- tools recall: `0.8333` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.3333` +- accuracy: `0.7222` diff --git a/benchmarks/models/router/linear/out-router-linear-iter-4/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-4/router_model.json new file mode 100644 index 0000000..1fec4a5 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-4/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.18633848, + -0.021990247, + -0.09598119, + -0.12475248, + -0.0055247764, + 0.06369566, + -0.008723138, + -0.19863072, + -0.06277868, + -0.099104874, + 0.085971214, + -0.114912264, + 0.011969069, + -0.011524559, + 0.19476768, + -0.11838336, + -0.03345108, + -0.07276929, + -0.027851067, + -0.06088229, + 0.09280635, + 0.068293154, + -0.068615414, + 0.104422644, + 0.28788283, + -0.102640204, + 0.1655996, + -0.0056493874, + 0.2648823, + 0.18839236, + 0.11906588, + -0.08070814, + -0.11075673, + -0.048513148, + 0.1482414, + 0.043183617, + 0.07661222, + -0.11964615, + -0.12162281, + 0.11422304, + -0.05089721, + -0.2086355, + -0.17418046, + -0.117384106, + 0.10907311, + 0.0814679, + -0.02460272, + -0.08618438, + -0.111958876, + 0.019398285, + 0.06338135, + 0.06671663, + -0.018391166, + -0.13034785, + -0.044448815, + 0.2374514, + 0.00870324, + 0.085715555, + 0.034128625, + 0.21414743, + -0.07857612, + 0.08836732, + -0.19596298, + 0.07679612, + 0.020521877, + 0.05828172, + 0.14005251, + -0.12677951, + 0.0846198, + 0.15728568, + -0.03908312, + 0.17048323, + 0.084221445, + -0.013166092, + -0.14352055, + 0.119596496, + 0.014491269, + 0.02896192, + -0.034508154, + -0.113518804, + -0.07383001, + -0.025141595, + -0.110220656, + 0.13085045, + 0.07905992, + 0.1644883, + -0.1553705, + -0.024855055, + -0.07725549, + 0.045213725, + -0.062506184, + 0.0158601, + -0.059938118, + 0.030852143, + -0.037213616, + -0.11498651, + -0.21289913, + -0.16038676, + 0.08329657, + -0.034475576, + 0.01727555, + -0.08468778, + 0.2650441, + 0.097090185, + -0.15225862, + 0.06705284, + 0.05224962, + -0.065318406, + -0.040154513, + 0.009635824, + -0.29516324, + 0.10499831, + 0.07110597, + 0.03876518, + 0.078121774, + 0.08497595, + -0.07566165, + -0.081970595, + -0.16383497, + 0.068018995, + 0.19277292, + -0.025526213, + 0.037149314, + -0.1211539, + -0.16565678, + 0.0900694, + 0.16598481, + 0.13508134, + 0.12723272, + 0.064582154, + 0.14409442, + 0.051390126, + 0.09308874, + -0.0053218165, + -0.14926353, + 0.10023623, + 0.052404538, + 0.118025355, + 0.064098574, + -0.035400648, + -0.1167498, + 0.11188998, + 0.12400007, + 0.060254753, + -0.0014532692, + -0.32071766, + -0.0011266157, + -0.054838896, + 0.04992481, + -0.005961276, + -0.08831957, + 0.08546277, + 0.103863135, + -0.21817385, + 0.04985002, + -0.07579833, + -0.02296212, + -0.16555277, + 0.09859051, + 0.13479902, + 0.055023395, + -0.02733191, + -0.041447084, + -0.04039115, + 0.07048925, + 0.086557135, + -0.039628442, + 0.11102648, + 0.021148635, + -0.16282822, + 0.14222291, + -0.07848284, + -0.22176567, + -0.044396807, + -0.08138735, + 0.030739442, + -0.11531769, + -0.09774903, + -0.082107425, + 0.046270154, + 0.067457, + 0.2850858, + -0.33103317, + 0.07849784, + -0.023498492, + -0.102756575, + -0.17244335, + 0.3278435, + 0.06095586, + -0.09701801, + 0.10624031, + 0.19086596, + -0.1159302, + 0.051547915, + 0.019621905, + -0.05067868, + 0.081711896, + -0.03655443, + -0.032190118, + 0.25896105, + 0.04414717, + -0.1454319, + 0.1655049, + -0.03875817, + 0.07561866, + -0.19182944, + -0.0029918475, + 0.16781612, + -0.11417916, + -0.23184198, + -0.15177523, + -0.19467123, + -0.23595092, + -0.08488743, + -0.04964707, + -0.10689684, + 0.075020626, + 0.045934547, + 0.023282593, + -0.17801835, + -0.21553747, + 0.14769518, + 0.115120836, + 0.14128429, + -0.18796399, + -0.13099259, + -0.30767933, + -0.035177387, + 0.15527754, + 0.15546124, + -0.019328687, + -0.04424327, + 0.06824624, + -0.07744808, + 0.23629627, + 0.16880615, + -0.01786926, + 0.097564556, + -0.07779348, + 0.039635386, + -0.19111083, + 0.1443258, + -0.058470096, + -0.2673507, + 0.06965227, + 0.148964, + 0.22784793, + -0.0035397268, + 0.018858295, + 0.1077301, + -0.09406661, + -0.18655677, + -0.08204047, + -0.036849435, + 0.045027044, + -0.0013961178, + 0.07994172, + -0.11097627, + -0.10617173, + 0.051952574, + -0.16665302, + 0.14288223, + 0.08364722, + 0.04902033, + 0.056778733, + -0.11843705, + 0.08718353, + -0.03218126, + 0.02587419, + -0.06395128, + -0.009066266, + -0.047516793, + 0.23327143, + -0.04678851, + 0.04037096, + -0.013538329, + 0.08178902, + 0.06668387, + -0.16897848, + -0.015004748, + -0.01572572, + 0.14406358, + 0.071664244, + 0.042464685, + 0.08947636, + 0.092256874, + -0.25032964, + 0.07105301, + -0.108864136, + 0.08742515, + 0.04301848, + 0.017901175, + 0.10187706, + -0.016301403, + 0.061324365, + 0.2776516, + -0.060695387, + -0.033483397, + 0.073651, + 0.0057025487, + 0.3352942, + 0.10394482, + 0.16374, + 0.09518288, + 0.08158211, + -0.09344818, + 0.021589026, + 0.15027705, + -0.07937315, + -0.019393088, + -0.05662475, + -0.05123898, + -0.050836004, + 0.1343026, + 0.10637051, + -0.0577773, + 0.037862033, + -0.247351, + 0.106672175, + -0.0030619581, + -0.00014678766, + -0.08657385, + 0.113408186, + 0.23169582, + 0.019879716, + -0.034681514, + -0.0020551863, + -0.16076934, + -0.14288679, + 0.010082046, + -0.06617522, + -0.10194228, + -0.1415618, + 0.101942606, + -0.08793393, + 0.10894591, + 0.16146699, + -0.12228665, + 0.14080705, + 0.24692212, + 0.14946236, + 0.092784494, + 0.15893684, + -0.008467448, + 0.027090807, + 0.12665439, + -0.13623457, + -0.090713635, + 0.12896347, + 0.21177316, + 0.028683344, + -0.15380841, + -0.0006147844, + -0.1393724, + 0.036622833, + 0.045594625, + -0.023290219, + -0.17021397, + -0.08726953, + -0.015805764, + 0.026924638, + 0.07847709, + -0.04327303, + 0.3590176, + 0.060146403, + -0.15717702, + 0.1325123, + -0.0035076342, + 0.19028895, + -0.079874925, + -0.13333586, + 0.19525443, + -0.06502711, + -0.03750315, + -0.041022416, + -0.043243382, + -0.35507184, + -0.21726933, + 0.16190492, + -0.158557, + 0.13389374, + -0.18338445, + -0.0932396, + 0.12955591, + -0.053707905, + 0.0006459714, + -0.0032740647, + -0.28709257, + -0.056750856, + 0.23210952, + 0.30267164, + -0.08682682, + -0.17241643, + -0.03326459, + 0.054191496, + 0.097175844, + 0.16940773, + -0.12762201, + 0.009070134, + 0.11668817, + -0.21143344, + 0.05499155, + -0.15847756, + -0.14995135, + -0.0049438695, + 0.045848697, + 0.13417763, + -0.02462848, + 0.05411647, + -0.15435664, + 0.04004719, + -0.17902878, + 0.16244677, + -0.17061608, + -0.12348929, + 0.09180731, + 0.11858081, + 0.04388564, + -0.050241712, + -0.3794224, + -0.023072392, + -0.15187621, + -0.06022391, + 0.0389537, + -0.11311294, + -0.04483756, + -0.03191812, + -0.09286949, + 0.11451646, + -0.18656348, + 0.03569287, + -0.15736677, + 0.05963041, + -0.025123972, + 0.04769077, + 0.057496574, + 0.11523707, + -0.08233095, + -0.032269277, + -0.08317559, + 0.04284503, + 0.1099772, + 0.29609403, + -0.059371803, + -0.1258368, + 0.16488141, + 0.029701294, + -0.0871117, + 0.15366647, + 0.2648822, + -0.025175504, + 0.070027724, + -0.16725753, + -0.02301268, + 0.008337837, + -0.19743346, + -0.035971083, + -0.0139340125, + -0.0352934, + 0.2689722, + 0.012080048, + -0.11762006, + -0.21998496, + -0.09839513, + -0.06674763, + 0.04683752, + -0.11146877, + -0.04599476, + 0.14974731, + 0.05422002, + 0.012376041, + -0.14231902, + 0.030036619, + -0.12907441, + -0.22337681, + 0.035535805, + 0.06476209, + -0.18803062, + 0.04536368, + 0.24490592, + 0.18265249, + 0.22729625, + 0.10897362, + 0.11467191, + 0.05619355, + 0.09987299, + 0.06857587, + 0.12826896, + -0.16517407, + -0.075386114, + -0.25283834, + 0.042725638, + 0.109347545, + 0.014253335, + -0.23104662, + -0.13149749, + -0.25347164, + -0.0050595454, + 0.20712197, + -0.12284579, + -0.103947915, + 0.15963243, + 0.079586536, + 0.046046272, + 0.12228099, + -0.15825438, + -0.014796799, + 0.07996475, + 0.05532982, + -0.08836719, + -0.002184571, + -0.04528829, + -0.057088953, + 0.10949727, + 0.08618899, + 0.040242728, + -0.041859094, + -0.08223082, + -0.104844384, + -0.101053454, + -0.031028843, + -0.05574357, + 0.13322599, + 0.088604, + 0.107746825, + -0.14654899, + 0.07539338, + 0.013862365, + -0.09069706, + 0.1796719, + -0.016659677, + -0.04956159, + 0.27165785, + -0.11160872, + -0.2989888, + 0.09245948, + -0.098325185, + -0.24016048, + 0.52027804, + -0.064231955, + -0.005032319, + -0.035204574, + -0.054530594, + 0.0322089, + 0.18308656, + 0.08390766, + 0.043010466, + -0.15488197, + 0.08767366, + 0.004138853, + -0.045949485, + -0.1160313, + -0.07836478, + -0.0026241415, + -0.19997384, + -0.03938704, + -0.084648326, + -0.06306966, + -0.0952212, + 0.049785048, + -0.20149958, + -0.015529836, + 0.04424018, + 0.21384019, + -0.059389725, + 0.17041788, + 0.20170075, + 0.08455694, + -0.22512946, + 0.14936431, + 0.06936717, + 0.18910527, + 0.004311936, + -0.11028445, + 0.24718592, + -0.016151149, + 0.052220903, + -0.028108425, + 0.10092346, + -0.08499343, + -0.094193466, + 0.18906602, + -0.061153974, + 0.3099695, + 0.062192697, + 0.23875803, + 0.21906589, + 0.10406714, + 0.12670133, + 0.030697094, + -0.01277904, + -0.1554076, + -0.033657804, + -0.020216823, + 0.1222573, + 0.13814458, + 0.13875861, + -0.06178161, + 0.08828033, + -0.03528705, + 0.113277875, + 0.07364842, + -0.032665182, + 0.121137686, + 0.050435707, + -0.08710971, + -0.04304125, + -0.12949078, + 0.28858903, + 0.026131703, + 0.06545478, + -0.22433525, + 0.03651575, + -0.0020827467, + 0.10167747, + 0.052595302, + 0.03274926, + 0.0072248536, + 0.15257488, + -0.19761868, + -0.099838115, + -0.21286489, + -0.050637424, + 0.004828881, + -0.03982375, + -0.0035783183, + -0.03838706, + 0.13523293, + 0.061031763, + -0.0036069269, + 0.24715978, + 0.30907214, + -0.1487525, + 0.02132856, + 0.016107956, + 0.047291737, + -0.19320416, + 0.05628742, + 0.014700451, + -0.011124475, + -0.09308549, + 0.11523371, + -0.2443315, + 0.2313448, + -0.25128835, + 0.04598335, + 0.040417917, + -0.09665347, + 0.04653424, + -0.07673263, + 0.12003437, + -0.017339345, + -0.103531435, + 0.022593237, + -0.09782897, + 0.061050087, + -0.13742255, + -0.047381733, + -0.119925104, + 0.2250597, + 0.10950321, + -0.0006340932, + 0.15462278, + 0.16510288, + 0.26863286, + -0.105883054, + -0.047063254, + 0.028948596, + 0.17239384, + 0.056860864, + 0.017096903, + -0.04228211, + -0.29253602, + -0.16056961, + 0.036930505, + 0.024778143, + -0.17934024, + -0.057446398, + -0.053578038, + 0.049655244, + 0.18780835, + -0.14032418, + 0.05718126, + -0.050660625, + -0.07751246, + 0.1379321, + 0.031953067, + -0.11338463, + -0.06786446, + -0.033356477, + 0.20826721, + 0.09957642, + 0.17668569, + 0.037627928, + -0.08140153, + 0.19429025, + 0.17275688, + -0.016087506, + 0.010439479, + 0.26191953, + 0.16416778, + 0.046684586, + 0.014290053, + 0.27286097, + 0.054265194, + -0.11745537, + 0.014185265, + -0.05908016, + 0.020058561, + -0.08973265, + -0.19595727, + -0.08870827, + 0.07308811, + -0.17287685, + -0.1589863, + -0.09653189, + 0.0048071207, + -0.13724232, + -0.08037632, + -0.047368392, + -0.16322568, + -0.033112556, + -0.13488308, + 0.15020229, + 0.1550236, + 0.10108509, + 0.040011637, + -0.20094907, + -0.090874165, + -0.0939116, + 0.080748454, + 0.115656935, + -0.03451729, + -0.13090292, + -0.13617615, + 0.0730974, + 0.122894086, + -0.038970716, + 0.07054392, + -0.06039169, + 0.02978956, + 0.1455779, + -0.077636614, + -0.24305566, + -0.10163222, + -0.10691119, + 0.037590254, + -0.034741804, + -0.06074967, + 0.09297616, + 0.14142899, + -0.11809402, + -0.15412627, + 0.065455005, + 0.09679129, + 0.019191982, + 0.20682472, + -0.20068286, + 0.0781541, + 0.23031051, + -0.18984596, + -0.13075766, + -0.01675578, + -0.0005947731, + -0.07893368, + -0.090689294, + 0.016939832, + -0.22322744, + -0.20472436, + 0.22921742, + -0.09677777, + -0.08641651, + 0.10703374, + 0.02100416, + -0.056980975, + 0.028346613, + -0.14496057, + 0.010396989, + 0.07659849, + 0.07763536, + -0.019762618, + -0.009585714, + 0.14585073, + -0.21997699, + 0.058942523, + -0.05442393, + -0.05415349, + -0.2622914, + 0.0067495466, + 0.036460627, + -0.15851912, + -0.14385591, + 0.15922575, + 0.039899103, + 0.078442976, + 0.08507526, + 0.18382412, + 0.22244017, + -0.03963535, + 0.06646716, + -0.032181084, + 0.005002472, + -0.11370807, + 0.16499059, + -0.09635935, + -0.19694255, + 0.021960784, + -0.27661255, + -0.05737264, + 0.1593896, + 0.09378417, + 0.047629766, + 0.01862993, + 0.14267302, + 0.062311556, + 0.057066027, + -0.0026629304, + -0.0600522, + -0.07380065, + 0.039031584, + -0.16306129, + 0.029224617, + 0.10198643, + -0.050818797, + 0.08438627, + 0.03716007, + -0.0443205, + -0.09043356, + 0.06818091, + 0.056742877, + -0.09950669, + 0.07628545, + -0.106138945, + 0.051655952, + -0.11948614, + 0.10950823, + -0.042758968, + 0.14772765, + -0.07351131, + -0.039129984, + -0.13518056, + -0.010262115, + -0.0797974, + -0.22309522, + -0.041619502, + 0.017691836, + -0.26448753, + 0.048773687, + -0.0430489, + 0.116527185, + -0.092469476, + -0.102719575, + 0.19830416, + -0.0104989875, + -0.108391374, + -0.09932554, + 0.07718134, + 0.02247461, + -0.051388126, + 0.090667345, + -0.28471762, + 0.044905305, + 0.17018011, + 0.0883001, + 0.22816132, + -0.12958865, + 0.16366321, + -0.07649908, + -0.04187491, + 0.031642366, + 0.02797218, + -0.015719814, + 0.17807175, + 0.25526682, + -0.023230469, + -0.051920697, + 0.066116184, + -0.2540213, + 0.033616856, + 0.06835817, + 0.10677274, + -0.14640501, + 0.053824387, + -0.08425958, + 0.06938902, + 0.18427739, + 0.050563883, + -0.12735263, + 0.070671864, + -0.26665702, + 0.11795721, + -0.02624768, + -0.09283501, + 0.024224505, + -0.20036814, + 0.007591098, + 0.13092197, + -0.038981788, + 0.009218034, + 0.036291324, + 0.03673069, + -0.14297684, + 0.06404598, + -0.04193293, + 0.03271318, + -0.015589479, + -0.011117657, + 0.27253702, + 0.1462861, + 0.003484278, + 0.0281606, + 0.015206004, + -0.015842091, + -0.16562876, + 0.10613876, + 0.07942983, + -0.027591115, + 0.01765117, + -0.15868017, + -0.06422185, + -0.16085465, + 0.055914916, + -0.17279945, + -0.08020601, + 0.43271828, + -0.2222178, + -0.35625243, + -0.17985259, + 0.1014227, + -0.054314714, + -0.049593076, + -0.30154017, + 0.09971668, + 0.09626297, + -0.05056628, + 0.057815276, + -0.2127709, + 0.07285609, + -0.08156592, + -0.18285358, + 0.0055787605, + 0.050411053, + -0.04962786, + 0.034054108, + 0.025982443, + 0.042589404, + -0.024576876, + -0.090059645, + -0.30343422, + 0.035683475, + 0.056787435, + -0.18377149, + -0.07379626, + -0.06505526, + 0.044090565, + 0.119362965, + -0.08053844, + 0.21147929, + -0.03585316, + -0.044990774, + -0.04109666, + 0.08407934, + 0.23574571, + -0.2137305, + -0.10426127, + -0.01860227, + 0.12616, + -0.0708649, + 0.022942567, + 0.011244215, + 0.031471517, + 0.0930586, + 0.03214131, + 0.32868665, + 0.03207292, + 0.08603695, + 0.057994302, + -0.13777441, + 0.09182316, + 0.014774016, + 0.021573544, + 0.0076492485, + 0.17600806, + -0.1502997, + -0.24296235, + 0.20674296, + 0.04510429, + 0.22502612, + 0.031383082, + -0.061237697, + 0.050558675, + -0.06667626, + 0.021187605, + -0.09565701, + -0.03821771, + 0.16841638, + -0.100745626, + -0.05147383, + -0.14504951, + 0.019862775, + 0.12780803, + 0.004019054, + 0.03778424, + 0.014132394, + 0.12381858, + -0.04886466, + 0.20842083, + -0.052135132, + 0.031716947, + 0.11628498, + -0.11497015, + -0.17629582, + -0.070079945, + -0.11253226, + -0.07588835, + 0.038207293, + -0.27067733, + -0.13882011, + -0.107785426, + 0.018694866, + 0.08630753, + -0.09765054, + -0.05338821, + -0.14067116 + ], + [ + 0.08710954, + -0.17130332, + 0.18518627, + 0.034484398, + -0.015136474, + -0.11925981, + 0.13832627, + 0.34316385, + 0.119799584, + 0.098551616, + -0.038712684, + 0.07383681, + 0.06289237, + -0.03994811, + -0.16898105, + 0.10380005, + 0.054381933, + -0.01971661, + 0.13839778, + 0.16946474, + -0.13109885, + -0.06390203, + 0.061225582, + 0.030190721, + -0.23004332, + 0.067906775, + -0.18641347, + 0.058921356, + -0.14940989, + -0.14585148, + -0.05028953, + 0.23240112, + 0.053929616, + 0.03085122, + 0.035810877, + -0.06495588, + 0.04786252, + 0.04149577, + 0.0872867, + -0.3250427, + -0.04962545, + 0.11174253, + 0.002483554, + 0.038309716, + 0.034978088, + -0.04172436, + 0.1876067, + -0.07495243, + -0.027371367, + 0.14645548, + -0.033528112, + -0.06964625, + -0.055938557, + 0.17258711, + 0.043243863, + -0.13666941, + -0.050095692, + -0.17020623, + 0.23184434, + 0.053598404, + 0.011893593, + -0.13508283, + 0.10040416, + -0.019216675, + 0.2241509, + -0.23771235, + -0.060662683, + 0.09703199, + -0.112910956, + -0.18161695, + -0.06245686, + -0.33224294, + -0.041275404, + 0.034993168, + 0.075077124, + 0.069832034, + 0.12258471, + 0.037918355, + -0.035581246, + -0.03809896, + 0.106061384, + 0.037776574, + 0.2677748, + 0.017203985, + -0.19279915, + -0.060687363, + 0.1294681, + 0.0033134802, + 0.20178464, + -0.40706864, + 0.034403134, + 0.14862113, + -0.1569005, + -0.06778971, + -0.08919004, + 0.06502744, + 0.20957167, + -0.07431113, + -0.009827447, + 0.14637196, + -0.043831594, + 0.20640296, + -0.12957036, + -0.10145491, + 0.21949573, + -0.1981791, + -0.114104114, + 0.057041492, + -0.14220239, + -0.06584, + 0.08439022, + -0.13847621, + -0.0043671746, + -0.1280747, + -0.2209941, + -0.06541769, + 0.098690294, + 0.12322519, + 0.3025779, + 0.110712335, + -0.13243516, + 0.07160524, + 0.17707202, + 0.11139699, + 0.1694134, + -0.078767486, + -0.13430579, + -0.24052751, + -0.0995972, + -0.22442292, + -0.14696601, + 0.011035744, + -0.005659949, + -0.02199578, + 0.058117013, + -0.022728508, + -0.05242452, + -0.06867182, + -0.23656839, + 0.037021764, + 0.14837083, + -0.30180052, + -0.021341268, + -0.00328619, + -0.14210492, + 0.34642032, + -0.080925204, + 0.025348207, + 0.040403865, + -0.08036848, + -0.026028905, + -0.13547285, + -0.13658845, + 0.08929891, + 0.17939393, + -0.16283756, + 0.20934752, + 0.2804383, + -0.042361386, + 0.0037535643, + -0.08561987, + 0.2610427, + -0.16887006, + -0.0804164, + -0.016786562, + -0.0086533185, + 0.10278585, + 0.21424574, + -0.06778377, + -0.07107834, + -0.023322714, + 0.038586013, + 0.15721613, + 0.010754087, + 0.02862506, + -0.043462764, + 0.2998912, + -0.058243856, + 0.11280774, + 0.02401249, + -0.10704567, + -0.0711847, + 0.22480705, + 0.009597078, + 0.10986745, + -0.18179896, + 0.1457943, + -0.42035487, + -0.06941299, + 0.033767574, + -0.015188636, + -0.14821082, + 0.048311736, + -0.009381438, + 0.058467768, + 0.22949363, + -0.17324351, + 0.09156386, + 0.09434916, + -0.10727511, + 0.17464036, + 0.1562835, + -0.12607798, + 0.0289634, + -0.14662074, + 0.16115622, + 0.006686656, + -0.31932178, + 0.08756386, + 0.30337483, + 0.009069734, + 0.16290994, + 0.2164364, + -0.099484086, + 0.13662401, + -0.13326345, + 0.081423216, + 0.16968766, + 0.0050415867, + 0.15819815, + -0.022486081, + -0.17130947, + -0.18132602, + -0.34165516, + 0.15751578, + 0.09290999, + 0.11265368, + -0.04221395, + -0.10326879, + -0.21709505, + 0.075194344, + 0.16578491, + -0.0228574, + 0.0064968187, + -0.24290843, + -0.17301916, + 0.0010360496, + -0.08306041, + 0.13103732, + -0.059812766, + 0.05592717, + -0.14495893, + -0.1183925, + 0.17915757, + -0.027477909, + -0.1789974, + -0.18129501, + -0.14337839, + 0.01016965, + -0.13785207, + -0.00971535, + 0.11592945, + 0.11714097, + -0.22951867, + 0.16646625, + -0.0031756428, + 0.06898733, + 0.015582822, + 0.11184965, + -0.11426219, + 0.15530753, + -0.12028912, + -0.08832732, + -0.0884876, + 0.01425523, + 0.010757544, + -0.041930813, + -0.056576736, + 0.030607315, + 0.13479832, + -0.07811483, + 0.058088824, + 0.00008467862, + 0.024450693, + 0.17910321, + 0.047229327, + -0.18732254, + -0.045666203, + 0.12912987, + -0.09150285, + 0.03890589, + -0.23575307, + 0.029016903, + -0.040441103, + -0.107373886, + -0.10694674, + 0.036835346, + -0.20357336, + -0.057427242, + 0.037563592, + -0.07291938, + -0.05247231, + 0.016445952, + 0.101862565, + -0.2436902, + -0.24760891, + -0.15459998, + 0.019506909, + -0.21173312, + -0.087886855, + -0.37426487, + -0.07298867, + -0.059791256, + -0.15710446, + -0.08189467, + 0.026206193, + 0.22261252, + -0.24138017, + 0.00890099, + 0.075444795, + 0.030475946, + 0.024316594, + 0.018440438, + -0.024992568, + -0.026872009, + 0.108047344, + 0.11162974, + 0.18837136, + -0.0804132, + -0.32997715, + -0.021764146, + 0.08130553, + -0.1064667, + -0.08348454, + -0.058926456, + 0.123180114, + -0.049904585, + 0.21330495, + 0.08938906, + -0.054655578, + 0.21257395, + 0.05705831, + -0.040726066, + 0.14231788, + 0.023860166, + 0.021737259, + -0.08625983, + -0.046179447, + -0.22169155, + -0.096731246, + 0.07239878, + -0.002620855, + -0.13451672, + -0.0035110759, + -0.100999646, + 0.045851093, + 0.115883544, + -0.040573798, + -0.0038128404, + -0.120611295, + 0.12036302, + 0.14173028, + -0.020041298, + 0.10767743, + 0.05669824, + -0.21955228, + -0.027810711, + 0.2384975, + 0.20769784, + 0.058436036, + 0.13167174, + -0.03997825, + 0.06490375, + -0.39682728, + -0.009131579, + 0.0038182035, + -0.062273886, + 0.06298824, + -0.055486213, + 0.07763187, + 0.18734314, + -0.31007335, + 0.10445774, + 0.0037420318, + 0.011656824, + 0.06310175, + 0.28920949, + 0.046282753, + -0.17289883, + 0.08784476, + -0.28481385, + 0.110921256, + 0.17077385, + -0.11537249, + 0.0025526553, + 0.06728107, + 0.03187741, + 0.24450828, + 0.013957283, + -0.18511797, + -0.28993872, + 0.18045323, + 0.19787051, + -0.08506221, + -0.071446285, + -0.116508596, + -0.19188306, + 0.17366874, + 0.13747092, + -0.030285724, + 0.24382776, + -0.16537562, + 0.17491636, + 0.11145762, + 0.0145614585, + -0.078121655, + -0.15256482, + 0.023452919, + -0.03198221, + 0.15302968, + -0.007858731, + 0.081439614, + -0.03475655, + 0.048131734, + 0.04107332, + -0.18774486, + -0.06102128, + -0.080510825, + 0.14107823, + 0.20489639, + -0.07205382, + 0.09343803, + -0.0017656386, + 0.15036091, + 0.37663803, + -0.0154301, + 0.10145759, + 0.04513303, + -0.15015142, + 0.23778017, + -0.17112422, + -0.011182279, + -0.1409076, + -0.106038846, + -0.020715188, + -0.013257168, + -0.01857237, + 0.10767726, + 0.018119281, + 0.017791973, + -0.185015, + -0.025156049, + -0.31205723, + 0.3476952, + 0.056714073, + -0.13445053, + 0.049857687, + 0.077027015, + -0.086866625, + -0.23081313, + -0.07399999, + -0.07555021, + -0.0629564, + 0.076873794, + 0.19624448, + 0.16596378, + -0.038202465, + 0.1258459, + 0.24286169, + 0.011493793, + 0.007160016, + -0.029425204, + 0.16966641, + 0.12608604, + 0.023404934, + -0.025459873, + -0.011254301, + 0.13206586, + -0.019274868, + 0.09385937, + -0.010656316, + -0.065400034, + -0.14098474, + 0.03497467, + 0.1471968, + -0.08455206, + -0.14741223, + 0.070851095, + 0.021002669, + -0.19233584, + -0.016548818, + 0.035071168, + 0.015510567, + -0.07111342, + -0.0008209783, + -0.17425494, + -0.22124435, + 0.073263645, + 0.14981963, + 0.20124836, + 0.053815912, + -0.013117615, + -0.06897596, + -0.025157409, + 0.008126072, + 0.06966679, + 0.19344465, + 0.091212906, + -0.14537519, + 0.2256169, + 0.05493476, + -0.16982616, + -0.2294222, + -0.020211222, + -0.30311102, + 0.20485239, + -0.0036436268, + -0.38836393, + 0.113643125, + -0.00074193213, + 0.014368115, + 0.16776425, + 0.16274397, + -0.24023412, + -0.0065946416, + -0.08178381, + -0.062067654, + 0.050150722, + 0.08572824, + -0.049099766, + 0.038218405, + 0.13753733, + -0.27165624, + -0.06660632, + -0.0031024336, + 0.00077022106, + 0.109839894, + -0.13052694, + 0.12741303, + -0.02386286, + 0.027174694, + -0.056120556, + -0.22025867, + -0.055497155, + 0.26162866, + 0.05429381, + 0.022575926, + 0.14617692, + -0.48583928, + -0.026855517, + -0.035127908, + -0.0994388, + 0.06336697, + -0.0013796594, + -0.18111397, + -0.11681097, + 0.05168338, + 0.09935661, + -0.13242848, + -0.25558785, + 0.02271768, + 0.15886833, + 0.1357752, + 0.020422582, + 0.05690636, + 0.10379993, + 0.015206449, + 0.042851932, + 0.14122868, + 0.03441443, + 0.17019148, + 0.09627194, + -0.11068253, + -0.17213245, + 0.10069426, + -0.16084917, + -0.20603392, + 0.1537688, + 0.18053167, + 0.012052561, + -0.14493415, + 0.009886556, + -0.100698516, + 0.13298142, + -0.0011233576, + 0.09706364, + 0.11539116, + 0.08948318, + -0.16341476, + 0.076917924, + 0.08169475, + -0.095494755, + -0.15150273, + -0.17631614, + -0.04698625, + -0.10497289, + -0.1551321, + 0.19398408, + -0.04757996, + 0.14968508, + 0.14501125, + 0.3260735, + -0.09190358, + -0.09226908, + -0.29598865, + -0.24046373, + -0.014669314, + -0.064815916, + -0.13214196, + 0.092982225, + -0.06129808, + -0.020856455, + 0.07223847, + 0.15493158, + 0.045643926, + 0.16655073, + 0.07079807, + 0.04770307, + -0.27394435, + -0.017774954, + -0.26278344, + 0.060371034, + -0.1546492, + -0.05711425, + -0.14442088, + 0.10931744, + 0.013118476, + 0.20232604, + -0.21293196, + 0.028680857, + 0.065254785, + 0.10172164, + 0.19185011, + -0.033591248, + -0.009484618, + -0.12940541, + -0.05190162, + -0.0575119, + -0.11933636, + 0.08953299, + -0.30165416, + -0.37290353, + -0.023061203, + -0.08670423, + -0.028334, + 0.06946971, + 0.21275698, + 0.17013383, + -0.08542582, + 0.045555767, + -0.015354397, + -0.1544775, + 0.347188, + -0.1362879, + 0.031593885, + -0.063385, + 0.07919194, + 0.03388464, + -0.07173834, + 0.038540523, + -0.07981687, + -0.09306369, + 0.15257466, + 0.11125348, + 0.03917901, + 0.02519694, + -0.0038859278, + -0.033267844, + -0.24917944, + -0.23705544, + -0.10430534, + -0.05478128, + -0.061792918, + -0.38342047, + -0.08392769, + 0.0018478394, + 0.06320872, + -0.00330005, + -0.09136997, + -0.14779916, + 0.018767918, + -0.17030604, + 0.48196173, + 0.07686573, + 0.013626745, + -0.021895628, + 0.28598508, + 0.13888015, + 0.11149308, + -0.13487104, + -0.23114635, + 0.2573732, + -0.059814, + 0.14756495, + 0.064012565, + -0.08390431, + -0.08256339, + 0.17762811, + 0.065454476, + -0.017735574, + -0.18665063, + -0.0832109, + -0.13875094, + -0.120969504, + 0.034954272, + -0.022674376, + -0.15303893, + -0.0941168, + -0.07137932, + -0.1681534, + -0.17633656, + -0.059532795, + -0.113158494, + 0.006298596, + -0.09926128, + 0.0132873, + 0.13816498, + 0.023504218, + -0.09607285, + -0.013130624, + 0.15760157, + 0.14307156, + -0.058194783, + 0.13315538, + 0.17142235, + 0.014921406, + -0.09991027, + 0.08165643, + -0.03796723, + 0.0028694954, + 0.37064257, + 0.13076137, + -0.09495863, + 0.048868448, + -0.30687767, + 0.058660306, + 0.02056709, + -0.021638418, + -0.02370854, + -0.16527307, + -0.0034045777, + -0.11812108, + -0.052942898, + 0.01853251, + 0.04409421, + 0.08230125, + -0.072519235, + -0.08816045, + -0.01623577, + 0.017485151, + -0.023712784, + -0.17598683, + -0.036672812, + 0.52452826, + -0.030806748, + 0.18347956, + -0.040979996, + -0.03811666, + 0.25729805, + -0.048534125, + -0.2373455, + 0.14856055, + 0.084020026, + -0.048066158, + 0.18652984, + -0.056927446, + -0.29441887, + 0.08345022, + -0.0005996005, + 0.031345766, + 0.27583638, + 0.13197714, + -0.04237792, + 0.24010932, + 0.12292464, + -0.0576011, + 0.006550551, + 0.104579575, + 0.40900844, + -0.18993206, + -0.0060352357, + 0.021329733, + -0.122469254, + 0.05516141, + -0.10217497, + 0.17811313, + 0.22834168, + -0.0727938, + -0.06289365, + -0.0612119, + 0.0885646, + 0.08026745, + -0.0053653144, + 0.18809816, + 0.0068571805, + 0.004589373, + 0.26332837, + 0.19965443, + 0.023766935, + 0.075609, + 0.022701783, + 0.24352224, + -0.12722453, + -0.0043050097, + -0.22488694, + -0.04397989, + -0.12078576, + -0.30110332, + 0.07128283, + -0.16407505, + 0.04377039, + -0.016103752, + 0.2800282, + -0.11904939, + 0.20880961, + -0.014415037, + -0.24259716, + 0.2187837, + 0.018473664, + 0.05934518, + -0.12305252, + 0.23674639, + -0.026103668, + -0.1798324, + -0.21792579, + -0.0977618, + -0.050768767, + -0.016194953, + 0.0853139, + -0.049470134, + 0.045779597, + 0.090835765, + 0.049913734, + -0.11232327, + -0.21321629, + 0.0716266, + -0.003955598, + 0.22519064, + -0.009941131, + 0.19688989, + 0.008872793, + -0.02055702, + 0.09081416, + -0.188783, + 0.11419467, + -0.16248907, + -0.044919066, + -0.07653471, + 0.12145175, + 0.1500241, + 0.13012478, + -0.057526883, + 0.21240865, + 0.105857186, + -0.076245934, + -0.02195677, + 0.06403661, + -0.23577379, + -0.12655307, + -0.11207305, + 0.045876216, + 0.06294753, + -0.16967238, + -0.16886263, + 0.20803007, + -0.071986, + -0.14971863, + -0.039650787, + 0.18356499, + -0.12795182, + 0.2708319, + -0.036815282, + 0.049104527, + -0.122806, + 0.01948398, + -0.07615092, + -0.20949979, + 0.20418935, + 0.07240039, + 0.11423065, + -0.027510138, + 0.17900626, + -0.118062556, + -0.19533488, + 0.3006342, + 0.13511322, + 0.053908553, + 0.20639807, + -0.10189352, + -0.05439522, + -0.17225397, + 0.1793962, + -0.13950582, + 0.17308019, + 0.10287988, + -0.13179243, + -0.11794216, + 0.06293473, + -0.085337915, + 0.11593874, + -0.014016872, + -0.15914717, + 0.08646496, + -0.22406147, + 0.2338852, + 0.17168793, + -0.066047914, + -0.006837371, + 0.0618302, + -0.053112675, + -0.12019638, + 0.03951125, + -0.1434713, + 0.05452687, + -0.16260725, + -0.115297765, + 0.124994054, + -0.24352738, + 0.0137317935, + -0.019007314, + 0.0733735, + 0.0527939, + 0.014726399, + 0.07390042, + 0.006591537, + -0.11127264, + -0.02154871, + -0.030611224, + 0.068587966, + -0.05893293, + 0.23582877, + -0.0012231775, + 0.09105914, + 0.10211014, + -0.2383829, + 0.2864423, + 0.14851955, + 0.19865727, + -0.059792183, + 0.06848801, + 0.13465886, + 0.1286977, + 0.0067498614, + -0.04016777, + 0.20123987, + 0.061112855, + 0.18267728, + -0.071847156, + -0.11303111, + 0.3798, + 0.11107034, + 0.036331534, + -0.009910368, + -0.06711268, + -0.030826552, + 0.00086975336, + 0.011719601, + 0.13172266, + 0.3593029, + 0.036299367, + -0.106546186, + 0.10521198, + 0.07050957, + -0.0057226345, + -0.10432761, + -0.12922813, + 0.059869226, + -0.19951992, + -0.25756156, + -0.220191, + 0.022794224, + 0.028878445, + -0.11174812, + 0.19299944, + 0.13134119, + -0.017544553, + -0.028695328, + 0.25374243, + 0.15130068, + -0.10371478, + -0.06168496, + -0.07604035, + 0.028159369, + -0.20027314, + -0.1225441, + -0.15416595, + -0.060561605, + 0.0145160835, + -0.17459767, + 0.32457855, + 0.15947826, + 0.04655447, + -0.25514564, + 0.08199647, + 0.11665995, + -0.21419841, + -0.14407481, + -0.28173584, + -0.05530346, + 0.11907159, + -0.03749379, + 0.13516708, + -0.122134365, + -0.07007785, + 0.03407505, + -0.18017942, + -0.009898455, + 0.11660031, + 0.2261957, + -0.13030855, + -0.08201036, + -0.037830215, + 0.023683347, + -0.052337818, + -0.08290516, + 0.049878456, + -0.105940744, + -0.03673183, + -0.04835046, + -0.14271665, + -0.03498139, + -0.025459593, + 0.22258753, + 0.07466624, + 0.028863654, + -0.032301944, + 0.22064988, + 0.15382591, + 0.08489334, + 0.06317825, + 0.06779215, + 0.037101578, + 0.10427194, + 0.0017950593 + ], + [ + 0.09922909, + 0.19329362, + -0.089205086, + 0.0902681, + 0.020661248, + 0.055564135, + -0.12960309, + -0.14453314, + -0.057020947, + 0.00055327086, + -0.04725846, + 0.041075412, + -0.074861415, + 0.05147266, + -0.025786527, + 0.014583264, + -0.020930862, + 0.092485875, + -0.110546716, + -0.10858244, + 0.03829249, + -0.0043910975, + 0.007389824, + -0.13461338, + -0.05783951, + 0.034733385, + 0.020813826, + -0.053271953, + -0.115472406, + -0.042540982, + -0.068776384, + -0.15169284, + 0.056827094, + 0.017661918, + -0.1840523, + 0.021772237, + -0.12447468, + 0.07815046, + 0.03433609, + 0.21081972, + 0.10052267, + 0.09689305, + 0.171697, + 0.07907441, + -0.14405127, + -0.039743524, + -0.16300403, + 0.16113685, + 0.13933033, + -0.16585381, + -0.029853264, + 0.0029296472, + 0.07432966, + -0.042239234, + 0.0012049297, + -0.1007821, + 0.04139244, + 0.08449067, + -0.26597312, + -0.267746, + 0.06668256, + 0.046715535, + 0.09555873, + -0.05757943, + -0.24467292, + 0.1794307, + -0.07938974, + 0.029747458, + 0.028291203, + 0.024331363, + 0.10153997, + 0.16175944, + -0.042946056, + -0.021827083, + 0.06844339, + -0.18942854, + -0.13707602, + -0.06688024, + 0.07008942, + 0.1516177, + -0.032231364, + -0.012634971, + -0.15755424, + -0.1480545, + 0.11373915, + -0.10380098, + 0.025902417, + 0.02154159, + -0.12452926, + 0.36185467, + 0.028103083, + -0.16448122, + 0.21683855, + 0.036937624, + 0.12640364, + 0.049958993, + 0.0033274589, + 0.23469786, + -0.073469125, + -0.111896284, + 0.026556036, + -0.121715285, + -0.13547362, + 0.0043646568, + -0.06723719, + 0.13112622, + 0.061854582, + 0.00827685, + 0.18235683, + 0.056204185, + 0.21077286, + 0.03347793, + -0.06673879, + 0.08930961, + 0.14287215, + -0.019558305, + -0.023028698, + -0.04125462, + -0.13874297, + -0.17873141, + -0.06033782, + -0.046079062, + -0.21422128, + 0.009756932, + -0.0037566265, + -0.011301864, + -0.031679, + 0.105446085, + -0.027635487, + 0.15984066, + 0.0028714947, + -0.062425885, + -0.08742873, + 0.027317606, + 0.091146655, + -0.07750772, + 0.00001997856, + -0.04935356, + 0.1724698, + -0.0016211232, + -0.031620953, + 0.18991055, + -0.102658816, + -0.0569686, + 0.14355817, + -0.02570266, + 0.08205178, + 0.029490678, + -0.09032867, + 0.08632978, + 0.11434846, + 0.050010186, + 0.032725327, + 0.12887496, + -0.22924386, + 0.23863578, + -0.18638536, + -0.11488565, + -0.056229144, + -0.13855264, + 0.030596541, + -0.23371081, + 0.21031708, + 0.12080758, + -0.053702675, + -0.077903844, + -0.06315739, + -0.3252722, + 0.046635125, + 0.23390658, + -0.11890019, + 0.0398968, + 0.06454974, + 0.03364276, + 0.052762296, + 0.012723345, + -0.18457334, + 0.1559928, + -0.030700224, + -0.07028264, + 0.03958868, + -0.21390094, + 0.10622579, + -0.08809487, + -0.08636894, + 0.2845554, + 0.026649011, + 0.09251134, + 0.00845717, + 0.063250445, + -0.091051705, + -0.042655118, + 0.067618415, + -0.042166475, + -0.07808972, + -0.17881471, + 0.091531634, + -0.05500939, + -0.06215904, + -0.15168588, + -0.21878763, + -0.010851546, + -0.03942697, + 0.009794777, + 0.071002096, + 0.030673252, + -0.0036948097, + 0.15150587, + 0.026615286, + -0.071532965, + 0.14270544, + 0.031761274, + 0.019514496, + 0.18437156, + -0.0869769, + 0.24016027, + -0.15644383, + -0.21562234, + -0.028324183, + 0.01982025, + 0.23802343, + 0.023614297, + 0.06620521, + 0.20037079, + 0.030448142, + 0.038082566, + 0.19502579, + 0.07739141, + -0.05200885, + 0.06163384, + -0.055865686, + -0.12154154, + -0.045388855, + 0.07095126, + 0.0066120788, + 0.0042130393, + 0.016833203, + -0.01450416, + -0.05324384, + 0.020177407, + 0.1351836, + 0.0006332288, + 0.17686254, + 0.08819308, + -0.042174343, + 0.03003325, + -0.046552762, + 0.14691816, + -0.029027943, + 0.030122029, + 0.10378191, + 0.07062717, + -0.035100482, + 0.2663682, + -0.21149327, + 0.0045717624, + -0.1489291, + 0.09539345, + -0.0056779007, + 0.06230959, + 0.011345457, + -0.022593074, + 0.004680072, + 0.03946722, + -0.07103394, + 0.1076795, + -0.04525275, + 0.08875799, + -0.056481507, + -0.07084696, + 0.08718119, + -0.010572039, + -0.23335618, + 0.022337815, + -0.21947427, + -0.033690974, + 0.10553345, + -0.021017643, + 0.039848566, + 0.10650752, + -0.023180155, + 0.09168934, + -0.100681156, + -0.002023562, + 0.017897502, + 0.014689954, + 0.21349426, + 0.13252033, + 0.16629137, + -0.12498875, + 0.029900944, + 0.034571122, + -0.1183231, + -0.08556117, + 0.18236583, + -0.030042706, + 0.21529557, + 0.013976496, + 0.13808198, + 0.08218435, + 0.03897052, + -0.0309562, + -0.103948675, + 0.06192161, + 0.0003125374, + 0.06724196, + -0.2442015, + 0.09110316, + 0.070472196, + -0.05605174, + 0.026148817, + 0.026922356, + 0.03239555, + -0.10931005, + -0.079498544, + -0.05027006, + -0.14949176, + 0.05897984, + -0.026259027, + 0.33303934, + 0.021910945, + 0.0052683633, + -0.006941438, + -0.1482114, + 0.03904678, + -0.088498645, + 0.051959783, + -0.052535605, + 0.053497687, + 0.044573534, + -0.14639854, + 0.04488398, + 0.18228793, + -0.24426037, + 0.064073786, + -0.13068326, + -0.07520704, + 0.168466, + 0.080884494, + -0.15019079, + -0.22186117, + -0.09016358, + -0.024420097, + 0.011978521, + 0.07390882, + -0.17250529, + 0.020350983, + 0.1312874, + -0.12515071, + -0.091161884, + -0.14904639, + 0.012078115, + 0.020656077, + 0.03169497, + -0.09332102, + 0.17395759, + 0.051100943, + -0.0682836, + -0.12042831, + -0.042630278, + -0.15859638, + -0.038498797, + -0.02163074, + 0.037809588, + -0.051014822, + 0.15335885, + -0.07023838, + -0.059480634, + -0.13480279, + 0.0022430932, + -0.05400732, + 0.11481892, + -0.03943057, + 0.03376113, + 0.029365588, + -0.01985839, + 0.06586234, + 0.17098652, + 0.010993923, + 0.07071217, + 0.15091984, + 0.07246314, + -0.07753422, + -0.014183399, + 0.05115523, + -0.06792704, + -0.028603341, + 0.042584494, + 0.0427936, + -0.046991456, + -0.012732995, + -0.093626425, + -0.025454156, + 0.118326806, + 0.01725479, + 0.019332752, + 0.022475427, + -0.046046827, + -0.14654095, + -0.08640246, + -0.03239433, + 0.11038407, + -0.016438928, + 0.038493652, + -0.00961758, + 0.03227294, + 0.018387197, + 0.0011755466, + -0.022134272, + 0.0013270652, + -0.032188423, + 0.0975892, + -0.1276902, + 0.12248453, + 0.08241598, + 0.09593749, + -0.05755952, + 0.03662516, + -0.09083649, + 0.17452618, + 0.09512618, + 0.058438238, + 0.06198956, + -0.18931474, + -0.26352504, + 0.060267664, + -0.06953952, + 0.047736462, + 0.035634864, + -0.05121662, + 0.13543147, + 0.16854906, + 0.08127721, + 0.1311628, + -0.026975583, + -0.044239435, + -0.09666469, + -0.025346389, + 0.014149978, + 0.06538364, + 0.14216997, + -0.08482117, + 0.01596345, + -0.28832352, + 0.069122784, + -0.030430982, + -0.07955898, + 0.010084634, + -0.06679984, + -0.034068897, + 0.09917545, + 0.005522545, + 0.23021385, + -0.053861145, + -0.20458226, + 0.031469673, + 0.07417353, + -0.111911826, + -0.20756844, + -0.28046608, + -0.019240063, + 0.14704527, + 0.050318588, + -0.027690839, + 0.0433427, + -0.021377675, + 0.122723065, + -0.08607106, + -0.13047236, + -0.1480793, + -0.0017197174, + 0.207719, + 0.11094812, + 0.094099835, + 0.07618005, + 0.049016297, + 0.08265005, + 0.11717951, + -0.06636636, + -0.052570045, + -0.16610357, + -0.2623675, + -0.12448414, + -0.04355849, + -0.055372536, + 0.074382015, + 0.15266834, + -0.20153268, + 0.0153543055, + -0.1258622, + 0.19902235, + -0.029608008, + -0.040371638, + 0.010904064, + 0.2229205, + 0.061830718, + 0.06002709, + -0.08615332, + -0.06174688, + -0.102771245, + 0.049013123, + 0.010193833, + 0.14983569, + -0.025835007, + 0.18083006, + -0.04659801, + 0.018440437, + 0.30839917, + -0.16897294, + 0.08910914, + -0.012183556, + -0.122475974, + -0.10565502, + 0.13073695, + -0.07959442, + 0.041541137, + 0.10392677, + 0.032080103, + 0.019116145, + 0.1501532, + -0.007189581, + -0.081793725, + 0.13843022, + -0.021997655, + -0.10464448, + 0.14577876, + -0.18523328, + 0.11666456, + -0.03671598, + -0.15580918, + -0.0105150165, + 0.10568207, + -0.05139898, + 0.167106, + 0.037360042, + -0.1467535, + 0.07574925, + 0.09398352, + -0.03443882, + 0.09108746, + 0.04016023, + 0.13464329, + -0.008836393, + -0.030829234, + -0.0019725922, + 0.03290327, + -0.09469384, + 0.055525407, + 0.04475484, + 0.25144902, + 0.023231843, + -0.042837147, + -0.057410453, + -0.017798437, + 0.14306754, + -0.06441287, + 0.069441855, + 0.020217743, + -0.046007477, + -0.08419948, + 0.031308196, + -0.08074212, + 0.06644234, + -0.041707832, + -0.04130455, + -0.009568846, + 0.004333094, + -0.23832574, + 0.0445979, + -0.1614168, + 0.07556689, + -0.1989918, + 0.09638659, + -0.022696951, + -0.24606256, + -0.08091246, + -0.16761199, + -0.06137476, + 0.06249124, + 0.008075503, + 0.012498684, + -0.093571134, + 0.21265674, + -0.13365346, + -0.015206457, + -0.13378516, + -0.0639338, + -0.2980513, + -0.07912139, + -0.18038224, + -0.13223216, + -0.17066601, + 0.12556143, + 0.11248599, + 0.17373152, + 0.10231915, + -0.12408944, + 0.12659758, + 0.043861717, + -0.057695147, + -0.051979873, + -0.05279198, + -0.039573293, + -0.27606925, + -0.096079595, + -0.07944101, + -0.027756765, + 0.08178766, + -0.014644812, + -0.008356735, + 0.19732872, + 0.16396426, + 0.118133426, + 0.059197, + 0.04274339, + -0.16191266, + -0.04586772, + -0.2095507, + 0.060357083, + 0.1689379, + 0.034583356, + 0.11114319, + -0.14121269, + 0.02876236, + 0.04930838, + 0.13298379, + 0.090288736, + -0.0777211, + 0.058304645, + -0.085926, + 0.054494467, + 0.0638312, + 0.17181379, + 0.065375715, + 0.012226054, + -0.116761416, + -0.01955284, + -0.22642134, + 0.07072532, + -0.034431268, + 0.10843991, + 0.039243765, + -0.10285648, + -0.095056795, + 0.21969451, + 0.017401649, + -0.11960973, + 0.06276881, + 0.025204107, + 0.038192134, + -0.04021747, + 0.11040308, + -0.049043182, + -0.13384683, + 0.05864996, + -0.08624704, + 0.14130844, + 0.08064963, + 0.36910453, + 0.011995751, + -0.00519787, + 0.055415392, + -0.09282995, + 0.21831799, + -0.18470502, + 0.10403515, + -0.016145466, + -0.02564856, + -0.081023864, + 0.090938374, + -0.035864808, + 0.21258816, + -0.18942566, + 0.083703935, + -0.050557304, + -0.002882522, + -0.10664499, + -0.081433766, + -0.05791506, + 0.085215755, + 0.043338016, + -0.11704907, + 0.002632758, + -0.09690419, + 0.013499867, + -0.054027792, + 0.05061038, + -0.06424348, + 0.0024100016, + 0.051092062, + -0.02161671, + -0.016365506, + -0.037934802, + 0.08334156, + 0.046447176, + -0.17161596, + -0.019718044, + 0.11020432, + 0.06093988, + -0.093766004, + 0.012168797, + 0.012848215, + 0.09886849, + -0.2791596, + 0.044996105, + 0.10416806, + -0.15235019, + 0.03557597, + 0.07601429, + 0.102863245, + 0.038355656, + -0.05436331, + -0.01489334, + 0.03972165, + -0.0124360835, + 0.081610486, + 0.09510311, + 0.055585824, + 0.1183436, + 0.04449894, + -0.20741712, + -0.09764877, + 0.22984175, + -0.19907053, + 0.15185396, + -0.15974532, + -0.060578734, + 0.22258739, + 0.11458275, + 0.2591845, + -0.0773439, + 0.0024641708, + 0.087460205, + 0.11237043, + 0.09208186, + -0.15539859, + -0.050374884, + 0.12713112, + -0.054308232, + 0.04290657, + -0.0060767806, + 0.030408895, + 0.1143094, + -0.28147283, + 0.13243896, + -0.076568425, + 0.0033897345, + 0.072858445, + -0.19654825, + -0.044442017, + 0.09591671, + -0.030466663, + 0.07010624, + -0.017388823, + -0.28332117, + 0.03773549, + 0.08759413, + 0.11723249, + -0.07755449, + -0.26165622, + -0.08599037, + -0.0012194687, + 0.05913376, + -0.23951457, + -0.043991003, + 0.1482903, + -0.02349037, + 0.11864776, + -0.20428394, + -0.0392854, + 0.10281299, + 0.06508676, + 0.01543562, + -0.07616559, + 0.15915586, + -0.20645969, + -0.08338114, + 0.06239686, + -0.01370483, + -0.016423445, + -0.06880195, + -0.07068177, + -0.1404855, + 0.03187874, + -0.065799706, + 0.049834546, + -0.20917483, + 0.06263697, + -0.030516472, + -0.11206959, + 0.1358173, + -0.09966633, + -0.03200116, + -0.03559406, + 0.14644402, + -0.041095372, + -0.063038506, + 0.07866325, + -0.031647455, + 0.097607866, + -0.011589293, + 0.011101283, + -0.1663201, + -0.045941252, + -0.11245013, + 0.21135762, + 0.22063637, + 0.057828937, + 0.03889893, + -0.21873465, + 0.029268285, + -0.28437626, + 0.007473741, + 0.037159376, + 0.1556142, + 0.0406958, + 0.053431734, + 0.07624715, + -0.011513231, + 0.010438517, + 0.117281705, + -0.120060414, + -0.15190017, + 0.16314197, + 0.12882996, + -0.10878672, + 0.0482761, + -0.13475692, + -0.05823976, + -0.25363272, + 0.09063389, + -0.055728428, + 0.015324757, + 0.1371272, + 0.005291453, + 0.052980922, + 0.08767807, + -0.07119296, + -0.047940455, + -0.11089401, + 0.0050558276, + 0.06778891, + -0.13261117, + 0.117238015, + 0.117865466, + 0.004264933, + 0.2004508, + 0.18700008, + 0.16960195, + -0.004454132, + 0.046593163, + 0.03977202, + -0.028631704, + 0.17936163, + -0.09963885, + 0.17131142, + 0.07253733, + 0.01717618, + -0.13217688, + 0.037284497, + 0.013885482, + -0.008089996, + -0.21928462, + 0.034505904, + -0.24764518, + 0.20573963, + 0.045836683, + -0.12769024, + -0.030525493, + -0.14587297, + -0.00046203224, + -0.16328663, + -0.06000922, + -0.059931837, + -0.27740368, + -0.08319252, + -0.12002472, + 0.047623277, + 0.06827662, + -0.013962914, + 0.06548123, + -0.032991156, + 0.08568136, + -0.08882054, + -0.17226891, + -0.052484937, + 0.067378275, + 0.06441795, + 0.01466604, + 0.15071812, + -0.10394031, + 0.18539482, + 0.0063700625, + 0.19983687, + -0.03351717, + -0.17927904, + -0.0648741, + 0.045819163, + -0.07104821, + 0.016821377, + 0.083465725, + 0.1034657, + 0.07942522, + -0.012593937, + 0.12989402, + 0.13088727, + -0.113876365, + -0.029009558, + -0.1600178, + 0.015523028, + -0.101534136, + -0.06799992, + 0.0011157023, + 0.09172839, + -0.11273034, + 0.031842764, + 0.04913983, + 0.012960052, + 0.09009213, + 0.12315478, + -0.07497427, + -0.05469171, + 0.08174025, + -0.021904139, + -0.19433546, + -0.0642246, + 0.20773315, + -0.018804684, + -0.04163051, + -0.014173299, + -0.08506588, + 0.17284273, + -0.10646656, + -0.056095254, + -0.15067372, + -0.118928105, + 0.030093474, + -0.0010089284, + 0.19459692, + -0.1969463, + -0.11664909, + -0.08674267, + 0.059538268, + 0.033058587, + 0.0048441044, + -0.04345918, + 0.012857262, + -0.041662965, + -0.055868775, + -0.07198284, + 0.04975875, + 0.07855949, + 0.003286673, + 0.07077788, + 0.06023691, + 0.0098650325, + 0.020669239, + -0.011959251, + 0.29341477, + 0.26518163, + 0.018302424, + -0.11295785, + -0.123997554, + 0.020731002, + -0.027079932, + 0.036146816, + -0.09746471, + -0.18287756, + -0.17424327, + 0.09247057, + 0.030213475, + -0.017018242, + -0.060300678, + -0.1284134, + 0.090471216, + 0.06812903, + 0.002567259, + 0.12325836, + 0.08277443, + -0.33935252, + -0.18105176, + -0.05420371, + 0.0791375, + 0.06830327, + 0.1263024, + 0.0074555012, + 0.09897055, + 0.05670974, + 0.023920402, + -0.057833895, + -0.013064893, + -0.06849094, + 0.1009468, + 0.16573484, + 0.0041426737, + 0.011762998, + 0.110644124, + -0.06512651, + -0.08114622, + 0.11044569, + -0.04579765, + 0.033811156, + -0.06146758, + 0.038205404, + -0.040913455, + -0.0010137758, + -0.10248012, + 0.08886698, + 0.016633533, + 0.02643175, + 0.14995141, + 0.20175526, + -0.15250771, + 0.037865993, + 0.047024727, + -0.0059053525, + 0.050027758, + -0.015005728, + 0.02289207, + -0.08187314, + -0.15409966, + 0.060548972, + -0.050883785, + 0.13887611 + ] + ], + "bias": [ + 0.15158902, + 0.03748021, + -0.18906918 + ], + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.json new file mode 100644 index 0000000..56b2262 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.json @@ -0,0 +1,560 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 199, + "accuracy": 0.7889447, + "confusion": [ + [ + 71, + 24, + 4 + ], + [ + 0, + 65, + 0 + ], + [ + 5, + 9, + 21 + ] + ], + "tool_precision": 0.93421054, + "tool_recall": 0.7171717, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.14285715, + "memory_false_abstain_rate": 0.0, + "tool_count": 99, + "memory_count": 65, + "abstain_count": 35, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.27311695, + "p_memory": 0.5416255, + "p_abstain": 0.18525758 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.421286, + "p_memory": 0.37514666, + "p_abstain": 0.20356733 + }, + { + "query_id": "holdout_tools_006", + "text": "which crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38631028, + "p_memory": 0.43678588, + "p_abstain": 0.17690389 + }, + { + "query_id": "holdout_tools_020", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4345455, + "p_memory": 0.41529375, + "p_abstain": 0.15016073 + }, + { + "query_id": "holdout_abs_001", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.58328974, + "p_memory": 0.2393619, + "p_abstain": 0.17734835 + }, + { + "query_id": "holdout_abs_003", + "text": "fix this pls", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5047037, + "p_memory": 0.19701391, + "p_abstain": 0.29828238 + }, + { + "query_id": "holdout_abs_004", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.23068972, + "p_memory": 0.55771327, + "p_abstain": 0.21159701 + }, + { + "query_id": "holdout_abs_006", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44023973, + "p_memory": 0.2884513, + "p_abstain": 0.27130887 + }, + { + "query_id": "holdout_abs_009", + "text": "what's the newest rust stable version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.45300436, + "p_memory": 0.28370288, + "p_abstain": 0.26329285 + }, + { + "query_id": "holdout_abs_010", + "text": "is gemma 4 better than llama 3.1? (check online)", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.22349517, + "p_memory": 0.5678281, + "p_abstain": 0.20867671 + }, + { + "query_id": "holdout_abs_011", + "text": "what's my home address?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.54286367, + "p_memory": 0.17845435, + "p_abstain": 0.278682 + }, + { + "query_id": "holdout_abs_013", + "text": "what are the openai api keys on the server? show me", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5330327, + "p_memory": 0.18131717, + "p_abstain": 0.28565007 + }, + { + "query_id": "holdout_abs_015", + "text": "今日の株価ニュースまとめて", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5705767, + "p_memory": 0.19191976, + "p_abstain": 0.2375035 + }, + { + "query_id": "holdout_abs_017", + "text": "what are my saved browser passwords?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.48939604, + "p_memory": 0.28273803, + "p_abstain": 0.22786595 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.32584342, + "p_memory": 0.4522681, + "p_abstain": 0.2218885 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47907954, + "p_memory": 0.32599244, + "p_abstain": 0.19492812 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42581502, + "p_memory": 0.2707809, + "p_abstain": 0.3034041 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48411453, + "p_memory": 0.33207738, + "p_abstain": 0.18380813 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41490114, + "p_memory": 0.39232263, + "p_abstain": 0.19277626 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.498076, + "p_memory": 0.2635602, + "p_abstain": 0.23836374 + }, + { + "query_id": "shell_test_019", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.48745245, + "p_memory": 0.17580202, + "p_abstain": 0.33674553 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44542202, + "p_memory": 0.17025512, + "p_abstain": 0.38432285 + }, + { + "query_id": "shell_test_030", + "text": "journalctl -u klbr — what does it say?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49682242, + "p_memory": 0.21907358, + "p_abstain": 0.28410402 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49529323, + "p_memory": 0.33562484, + "p_abstain": 0.16908191 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47192854, + "p_memory": 0.29691777, + "p_abstain": 0.2311537 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.35084134, + "p_memory": 0.5222145, + "p_abstain": 0.12694421 + }, + { + "query_id": "rf_test_010", + "text": "what's in the .gitignore for klbr?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.43999156, + "p_memory": 0.21865375, + "p_abstain": 0.3413547 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4496186, + "p_memory": 0.29366884, + "p_abstain": 0.25671256 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46442676, + "p_memory": 0.39125812, + "p_abstain": 0.14431503 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3966753, + "p_memory": 0.3473245, + "p_abstain": 0.2560003 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4669734, + "p_memory": 0.30586943, + "p_abstain": 0.22715719 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4187118, + "p_memory": 0.4667664, + "p_abstain": 0.114521824 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4648565, + "p_memory": 0.34376398, + "p_abstain": 0.19137956 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4916422, + "p_memory": 0.2993488, + "p_abstain": 0.20900898 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4452266, + "p_memory": 0.38029882, + "p_abstain": 0.17447457 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48173755, + "p_memory": 0.22332487, + "p_abstain": 0.2949376 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49144813, + "p_memory": 0.2733501, + "p_abstain": 0.2352018 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44083598, + "p_memory": 0.38097113, + "p_abstain": 0.17819287 + }, + { + "query_id": "abs_test_004", + "text": "what's the best programming language?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36859047, + "p_memory": 0.31743553, + "p_abstain": 0.313974 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.37783423, + "p_memory": 0.4140439, + "p_abstain": 0.20812187 + }, + { + "query_id": "abs_test_009", + "text": "どのモデルが今一番スコアが高いですか — ベンチマークサイトで", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36733645, + "p_memory": 0.37445688, + "p_abstain": 0.25820664 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38499674, + "p_memory": 0.4603757, + "p_abstain": 0.15462756 + } + ] + }, + "holdout_metrics": { + "total": 60, + "accuracy": 0.8, + "confusion": [ + [ + 18, + 2, + 0 + ], + [ + 0, + 20, + 0 + ], + [ + 5, + 5, + 10 + ] + ], + "tool_precision": 0.7826087, + "tool_recall": 0.9, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.25, + "memory_false_abstain_rate": 0.0, + "tool_count": 20, + "memory_count": 20, + "abstain_count": 20, + "misclassified": [ + { + "query_id": "holdout_tools_006", + "text": "which crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38631028, + "p_memory": 0.43678588, + "p_abstain": 0.17690389 + }, + { + "query_id": "holdout_tools_020", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4345455, + "p_memory": 0.41529375, + "p_abstain": 0.15016073 + }, + { + "query_id": "holdout_abs_001", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.58328974, + "p_memory": 0.2393619, + "p_abstain": 0.17734835 + }, + { + "query_id": "holdout_abs_003", + "text": "fix this pls", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5047037, + "p_memory": 0.19701391, + "p_abstain": 0.29828238 + }, + { + "query_id": "holdout_abs_004", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.23068972, + "p_memory": 0.55771327, + "p_abstain": 0.21159701 + }, + { + "query_id": "holdout_abs_006", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.44023973, + "p_memory": 0.2884513, + "p_abstain": 0.27130887 + }, + { + "query_id": "holdout_abs_009", + "text": "what's the newest rust stable version today?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.45300436, + "p_memory": 0.28370288, + "p_abstain": 0.26329285 + }, + { + "query_id": "holdout_abs_010", + "text": "is gemma 4 better than llama 3.1? (check online)", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.22349517, + "p_memory": 0.5678281, + "p_abstain": 0.20867671 + }, + { + "query_id": "holdout_abs_011", + "text": "what's my home address?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.54286367, + "p_memory": 0.17845435, + "p_abstain": 0.278682 + }, + { + "query_id": "holdout_abs_013", + "text": "what are the openai api keys on the server? show me", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5330327, + "p_memory": 0.18131717, + "p_abstain": 0.28565007 + }, + { + "query_id": "holdout_abs_015", + "text": "今日の株価ニュースまとめて", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5705767, + "p_memory": 0.19191976, + "p_abstain": 0.2375035 + }, + { + "query_id": "holdout_abs_017", + "text": "what are my saved browser passwords?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.48939604, + "p_memory": 0.28273803, + "p_abstain": 0.22786595 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.md new file mode 100644 index 0000000..9ea11cd --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-4/router_report.md @@ -0,0 +1,88 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-holdout-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.500` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `199` +- tools: `99` memory: `65` abstain: `35` +- accuracy: `0.7889` +- tools precision: `0.9342` +- tools recall: `0.7172` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.1429` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 71 | 24 | 4 | +| memory | 0 | 65 | 0 | +| abstain | 5 | 9 | 21 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.27 | 0.54 | 0.19 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.42 | 0.38 | 0.20 | +| `holdout_tools_006` | which crate defines the LlmClient type? | `Tools` | `Memory` | 0.39 | 0.44 | 0.18 | +| `holdout_tools_020` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.43 | 0.42 | 0.15 | +| `holdout_abs_001` | what's the latest NVIDIA driver version today? | `Abstain` | `Tools` | 0.58 | 0.24 | 0.18 | +| `holdout_abs_003` | fix this pls | `Abstain` | `Tools` | 0.50 | 0.20 | 0.30 | +| `holdout_abs_004` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.23 | 0.56 | 0.21 | +| `holdout_abs_006` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.44 | 0.29 | 0.27 | +| `holdout_abs_009` | what's the newest rust stable version today? | `Abstain` | `Memory` | 0.45 | 0.28 | 0.26 | +| `holdout_abs_010` | is gemma 4 better than llama 3.1? (check online) | `Abstain` | `Memory` | 0.22 | 0.57 | 0.21 | +| `holdout_abs_011` | what's my home address? | `Abstain` | `Tools` | 0.54 | 0.18 | 0.28 | +| `holdout_abs_013` | what are the openai api keys on the server? show me | `Abstain` | `Tools` | 0.53 | 0.18 | 0.29 | +| `holdout_abs_015` | 今日の株価ニュースまとめて | `Abstain` | `Tools` | 0.57 | 0.19 | 0.24 | +| `holdout_abs_017` | what are my saved browser passwords? | `Abstain` | `Memory` | 0.49 | 0.28 | 0.23 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.33 | 0.45 | 0.22 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.48 | 0.33 | 0.19 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Abstain` | 0.43 | 0.27 | 0.30 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.48 | 0.33 | 0.18 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.41 | 0.39 | 0.19 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Memory` | 0.50 | 0.26 | 0.24 | +| `shell_test_019` | is the klbr tui currently connected to the daemon socket? | `Tools` | `Abstain` | 0.49 | 0.18 | 0.34 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.45 | 0.17 | 0.38 | +| `shell_test_030` | journalctl -u klbr — what does it say? | `Tools` | `Memory` | 0.50 | 0.22 | 0.28 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.50 | 0.34 | 0.17 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.47 | 0.30 | 0.23 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.35 | 0.52 | 0.13 | +| `rf_test_010` | what's in the .gitignore for klbr? | `Tools` | `Abstain` | 0.44 | 0.22 | 0.34 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.45 | 0.29 | 0.26 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.46 | 0.39 | 0.14 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.40 | 0.35 | 0.26 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.47 | 0.31 | 0.23 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.42 | 0.47 | 0.11 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Tools` | `Memory` | 0.46 | 0.34 | 0.19 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.49 | 0.30 | 0.21 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.45 | 0.38 | 0.17 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.48 | 0.22 | 0.29 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.49 | 0.27 | 0.24 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.44 | 0.38 | 0.18 | +| `abs_test_004` | what's the best programming language? | `Abstain` | `Memory` | 0.37 | 0.32 | 0.31 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.38 | 0.41 | 0.21 | +| `abs_test_009` | どのモデルが今一番スコアが高いですか — ベンチマークサイトで | `Abstain` | `Memory` | 0.37 | 0.37 | 0.26 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.38 | 0.46 | 0.15 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `60` +- tools precision: `0.7826` +- tools recall: `0.9000` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.2500` +- accuracy: `0.8000` diff --git a/benchmarks/models/router/linear/out-router-linear-iter-5/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-5/router_model.json new file mode 100644 index 0000000..1fce175 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-5/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.16052741, + 0.0024608371, + -0.031318825, + -0.09885255, + -0.10179112, + 0.058994167, + 0.109656274, + -0.2224836, + -0.16442509, + -0.08725672, + 0.018937778, + -0.21723568, + -0.01847783, + 0.05206266, + 0.14480045, + -0.13029885, + -0.06268626, + 0.008815729, + -0.08768228, + -0.018255629, + 0.15967207, + -0.0049391207, + -0.02540304, + 0.049357686, + 0.06391954, + 0.040043183, + 0.09474887, + -0.037570562, + 0.17600152, + 0.19785954, + -0.09910671, + -0.14270158, + -0.0536535, + -0.047332756, + 0.18022339, + 0.0676487, + 0.13717946, + 0.027378082, + -0.10548278, + 0.066528544, + -0.19403522, + -0.11643189, + -0.08065489, + -0.03752704, + 0.033599664, + 0.15706381, + 0.013865536, + -0.08501725, + -0.020272262, + -0.018441383, + -0.03735178, + -0.016081626, + -0.10534383, + -0.17559953, + -0.0019625858, + 0.15813902, + -0.14031114, + 0.06402496, + 0.13181971, + 0.17281957, + -0.012040283, + 0.14102454, + -0.051492248, + 0.1004284, + -0.20263205, + 0.029880801, + 0.12380502, + -0.07871024, + 0.099203266, + 0.16148943, + -0.09668969, + 0.12815538, + 0.003585216, + -0.030985428, + -0.14598812, + 0.1132824, + 0.10472477, + -0.029968526, + -0.063032724, + 0.014652918, + -0.02877283, + 0.052562434, + -0.13108331, + 0.0002965604, + 0.11512386, + 0.09318654, + -0.10961809, + -0.0880418, + -0.09860604, + 0.07382556, + -0.1037337, + -0.038589653, + -0.09008774, + -0.01911863, + -0.14737788, + -0.009485854, + -0.047335766, + 0.016980678, + 0.084372506, + 0.052771546, + -0.044290844, + -0.028756851, + 0.2607931, + 0.08380605, + -0.064346865, + 0.06569313, + -0.041412737, + -0.096951924, + -0.12520541, + -0.014805983, + -0.24914601, + 0.086238, + -0.002006227, + 0.10598944, + 0.12087416, + 0.03508919, + -0.051487036, + -0.1256758, + -0.12843262, + -0.018472746, + 0.17818493, + -0.1824178, + 0.085565396, + -0.0069503593, + -0.06915642, + 0.078164116, + 0.22044943, + 0.095329665, + 0.09443477, + 0.009476721, + 0.12377651, + 0.09528795, + 0.112927206, + 0.1125022, + -0.09308766, + -0.010651261, + 0.091823, + 0.047559917, + 0.07709087, + -0.106162645, + -0.15692854, + 0.072903305, + 0.045788582, + 0.0375331, + 0.06844259, + -0.23813765, + 0.013259943, + -0.06379869, + 0.024160216, + 0.032886725, + -0.18673082, + 0.01760028, + 0.19031574, + -0.11544211, + 0.105799034, + -0.02177165, + -0.065577194, + -0.0861514, + 0.1342203, + 0.18299834, + 0.056894757, + -0.014077486, + 0.15178883, + -0.11575148, + 0.009013221, + 0.13157651, + -0.0039246664, + 0.108594656, + 0.015797623, + -0.13545142, + 0.0977668, + 0.046627443, + -0.14986962, + -0.100400895, + -0.06745776, + -0.02255909, + -0.116668254, + -0.09205319, + -0.042751074, + 0.02285744, + 0.05999009, + 0.105419904, + -0.2971868, + 0.15153843, + 0.013010975, + -0.058407262, + -0.07143607, + 0.32557562, + 0.055834014, + -0.10003547, + 0.14418238, + 0.124668226, + -0.03373238, + 0.03690689, + -0.008434, + -0.043183107, + 0.052016493, + -0.088068604, + -0.14013946, + 0.053377382, + 0.1353628, + -0.122651346, + 0.06593491, + -0.010691284, + 0.049608894, + -0.14469606, + -0.11539988, + 0.26940665, + -0.12557648, + -0.33065188, + -0.16942522, + -0.16879891, + -0.17476796, + -0.11808337, + 0.006112452, + -0.074556805, + 0.05563614, + -0.10752109, + 0.0045725466, + -0.100444764, + -0.19990551, + 0.1623679, + 0.070562355, + 0.17294201, + -0.21465039, + -0.15566683, + -0.22320299, + -0.040394336, + 0.096149504, + 0.12600791, + -0.071590565, + -0.14021218, + 0.07090477, + -0.05439689, + 0.23978162, + 0.19791496, + 0.14561944, + 0.09421892, + 0.03198014, + -0.019724434, + -0.17524067, + 0.11801276, + -0.034325983, + -0.12414058, + 0.09539344, + 0.04428776, + 0.1991088, + 0.031654123, + -0.04807285, + 0.04592151, + -0.07569178, + -0.031496607, + -0.19537716, + 0.060411256, + -0.051872216, + 0.090447225, + 0.08494578, + -0.11747185, + -0.028199665, + 0.1347343, + -0.17381737, + 0.19282167, + 0.08678283, + 0.034531005, + -0.043074735, + -0.118435755, + 0.021840902, + 0.08782734, + -0.028497253, + -0.09325137, + -0.037644356, + -0.10620848, + 0.20808154, + 0.059732296, + 0.033074785, + 0.009470372, + 0.104916535, + 0.066376306, + -0.010363118, + 0.05408072, + 0.02767956, + 0.12599848, + 0.016272616, + -0.10300154, + 0.06984461, + 0.093451455, + -0.3713836, + 0.004608827, + -0.06669234, + 0.1191322, + -0.029813163, + 0.06783095, + 0.053586982, + 0.018094638, + -0.074333526, + 0.19786923, + -0.019449225, + 0.02383677, + 0.004300856, + 0.10008391, + 0.22801672, + 0.12678932, + 0.11370807, + 0.16604814, + 0.088361025, + 0.05435657, + 0.027251301, + -0.0097401505, + -0.10017743, + -0.0076766806, + -0.07074346, + 0.023101224, + 0.045202877, + 0.075106, + 0.1932022, + 0.034920372, + 0.10798186, + -0.15182951, + 0.046124, + 0.08082057, + -0.011904392, + -0.03646725, + 0.06683557, + 0.10610709, + 0.015073168, + -0.027114002, + 0.0211207, + -0.14800651, + -0.16112563, + 0.042969227, + 0.05496639, + -0.10502971, + -0.10001824, + 0.13661686, + -0.07500956, + 0.06874437, + 0.0835452, + -0.10677995, + 0.11066851, + 0.14947426, + 0.05686178, + -0.045194548, + 0.080891475, + 0.01653106, + -0.03289933, + 0.013714271, + -0.12062449, + -0.10942656, + 0.0427117, + 0.11709176, + -0.025273317, + -0.051796433, + 0.0050489707, + -0.1257773, + -0.017385142, + 0.12721983, + -0.10977477, + -0.23171797, + -0.020706484, + -0.010048814, + 0.0075350627, + -0.0067544025, + -0.0440331, + 0.3260202, + -0.00035211392, + -0.039470658, + 0.08292158, + -0.093218766, + 0.09687666, + 0.014780582, + -0.08518228, + 0.080687046, + -0.013215859, + 0.010666921, + 0.043733794, + -0.08879815, + -0.23352464, + -0.19190405, + 0.21334182, + -0.20023906, + -0.04070634, + -0.16439508, + -0.0065315836, + 0.09033729, + -0.096911885, + -0.061966095, + -0.03798265, + -0.17271575, + -0.04042787, + 0.16066734, + 0.19571932, + -0.005219358, + -0.113288976, + -0.034409754, + -0.02542398, + 0.091005854, + 0.075512506, + -0.12603755, + 0.003550829, + 0.05454221, + -0.1700984, + 0.08618596, + 0.07662489, + -0.12044079, + 0.12438776, + 0.0038605034, + -0.008291778, + -0.02776508, + -0.012168801, + -0.0791805, + 0.05979986, + -0.05944031, + 0.16329224, + -0.32967898, + 0.08408068, + 0.013467381, + 0.056141563, + -0.039574765, + 0.01251687, + -0.333109, + -0.10213542, + -0.2410506, + 0.024430476, + 0.02588703, + -0.14144824, + 0.0058720787, + -0.0798089, + -0.017106313, + -0.0020736381, + -0.06813275, + 0.11289762, + -0.19290096, + 0.14954773, + -0.048712995, + 0.09767048, + 0.17014039, + 0.010858046, + -0.06397144, + 0.098602, + -0.026944358, + -0.06880838, + 0.09306078, + 0.21228854, + -0.052566614, + -0.16931634, + 0.1503215, + -0.04413507, + -0.050339848, + 0.070244834, + 0.28390154, + 0.04381359, + 0.13148732, + -0.09735818, + -0.04281441, + -0.0002529125, + -0.12293158, + 0.055222724, + -0.0026420944, + 0.014094278, + 0.055280052, + 0.07080589, + -0.06374293, + -0.21383622, + -0.07480419, + -0.06326204, + -0.050647177, + -0.12127862, + -0.088789135, + 0.07309716, + 0.02734736, + -0.023602324, + -0.11849017, + 0.0862599, + -0.100177184, + -0.20296523, + 0.056530718, + -0.04380203, + -0.07311151, + 0.17730682, + 0.12751189, + 0.059371542, + 0.15086243, + 0.14834693, + 0.05420559, + -0.005883509, + 0.037433855, + 0.03104376, + 0.017708058, + -0.11753717, + -0.092178345, + -0.13057369, + 0.079070516, + 0.13699917, + -0.023019383, + -0.16708986, + -0.100313716, + -0.11022881, + 0.033850983, + 0.17981607, + -0.05736247, + -0.009717411, + 0.06720382, + -0.09542911, + 0.021875495, + 0.13131075, + -0.10874953, + 0.0064090267, + 0.08441985, + -0.010587874, + 0.04712112, + -0.06715897, + -0.0367435, + 0.02208662, + 0.030317329, + 0.033193227, + -0.121694334, + 0.04879346, + -0.045048017, + -0.030299243, + -0.01951495, + -0.04660699, + -0.08691534, + 0.015517739, + 0.07925178, + 0.055090293, + -0.16836199, + -0.020131057, + -0.04979672, + -0.080895, + 0.2045888, + -0.035188764, + 0.04068433, + 0.14953531, + -0.21293955, + -0.18716255, + 0.038786113, + -0.06314729, + -0.13646829, + 0.4301075, + -0.054831143, + 0.11402921, + 0.0715126, + -0.015407559, + 0.021623263, + 0.08860939, + 0.04332614, + 0.0063768667, + -0.1433278, + 0.11752632, + -0.15465412, + -0.08700072, + -0.0845954, + -0.14791308, + 0.0016860976, + -0.09356121, + 0.06972774, + -0.14896198, + -0.0330842, + -0.044193294, + 0.057510998, + -0.1900679, + -0.10587522, + 0.056306608, + 0.11176626, + -0.030755477, + 0.052004084, + 0.10115147, + 0.009432866, + -0.16124472, + 0.19203115, + 0.072125755, + 0.19657056, + 0.000823032, + -0.05501959, + 0.25273466, + 0.037036214, + 0.0799104, + 0.015897552, + 0.05778206, + -0.017158741, + -0.068570286, + 0.03771934, + -0.07160096, + 0.12563969, + 0.09058254, + 0.19363588, + 0.21729888, + -0.038404334, + 0.007869045, + -0.11452937, + -0.061573993, + -0.09937147, + -0.092896424, + -0.022444807, + 0.084605426, + 0.042577196, + 0.091019176, + 0.0687685, + 0.11391002, + -0.023444116, + 0.056105986, + 0.0010575949, + 0.058704697, + 0.13065207, + 0.18417926, + -0.17100154, + -0.03459482, + -0.13954142, + 0.1697018, + -0.032380745, + 0.11538436, + -0.23546107, + 0.13714324, + 0.047174696, + 0.1799499, + -0.039026696, + 0.020465225, + 0.09494173, + 0.26512316, + -0.07517377, + -0.043898042, + -0.13932166, + 0.0056566624, + -0.0055164094, + -0.043773327, + -0.0541095, + -0.0054946276, + 0.0663163, + 0.0021626875, + 0.10122027, + 0.06443506, + 0.2846765, + -0.28031296, + 0.08613889, + 0.04016044, + -0.045890335, + -0.11568628, + 0.09650608, + -0.0396269, + -0.019264355, + -0.12604448, + 0.12022761, + -0.18100314, + 0.1449683, + -0.19298455, + 0.0029647453, + 0.01889965, + -0.0340142, + 0.02557537, + -0.053900447, + 0.102941334, + 0.040459722, + -0.11036412, + -0.05180269, + -0.021873292, + 0.055194244, + -0.18999355, + -0.053315975, + -0.07193985, + 0.13219415, + 0.03263697, + 0.07894841, + 0.027257426, + 0.17079633, + 0.33656633, + -0.09741153, + -0.0012406677, + -0.062253032, + 0.122720554, + -0.07428574, + 0.092065744, + -0.12949777, + -0.2637853, + -0.17236178, + 0.11274971, + -0.0036404934, + -0.067255475, + -0.01703726, + -0.024414714, + -0.08555217, + 0.11511529, + -0.10132069, + 0.035678104, + -0.03773041, + -0.11198654, + 0.09750184, + 0.11933404, + -0.017337881, + -0.0275599, + -0.056142204, + 0.2870939, + 0.025516802, + 0.089016005, + 0.042186953, + 0.061288327, + 0.23220468, + 0.1913539, + 0.041872956, + -0.029348254, + 0.2022436, + 0.24744327, + 0.00085464993, + -0.06835739, + 0.21693708, + 0.03833265, + -0.11809685, + -0.094088, + -0.08797381, + 0.03583436, + -0.03725489, + -0.12992965, + -0.046520386, + 0.08877341, + -0.23196267, + -0.1416181, + -0.14281443, + 0.0051557235, + -0.083149076, + -0.030370584, + 0.028675005, + -0.069152854, + 0.022219311, + -0.24649718, + 0.16162385, + 0.13550395, + 0.24505398, + 0.11737749, + -0.26714596, + 0.039750718, + 0.036444116, + 0.19190054, + 0.25507984, + -0.05611311, + -0.03598602, + -0.00091825356, + 0.035975944, + 0.005179897, + -0.05993038, + 0.04453667, + -0.06822254, + -0.0034093673, + 0.041618973, + 0.009030745, + -0.16429444, + -0.10991389, + -0.21974932, + 0.05839352, + -0.12792662, + -0.17765494, + 0.06688134, + 0.16218579, + -0.13164958, + -0.059599277, + 0.08895215, + 0.102052465, + -0.045741614, + 0.15715373, + -0.26308244, + 0.051839665, + 0.16174427, + -0.13277116, + -0.13938846, + -0.082957774, + -0.03518575, + -0.16200525, + -0.1170968, + 0.090260565, + -0.10743151, + -0.20015334, + 0.19283876, + 0.02739746, + -0.0678554, + -0.0289476, + -0.024540577, + -0.15412445, + 0.028864484, + -0.033722255, + 0.037424758, + 0.04879276, + 0.048878994, + 0.07719014, + -0.027368383, + 0.1159599, + -0.097892486, + 0.015980033, + -0.0061608437, + -0.035205483, + -0.20321424, + -0.08789502, + 0.016100816, + 0.017145542, + -0.17215213, + 0.17112683, + -0.002088437, + 0.13785838, + 0.047853924, + 0.23892085, + 0.16270262, + 0.030570636, + 0.1013973, + -0.060039762, + -0.09836197, + -0.111120716, + 0.12632947, + -0.08306903, + -0.12863602, + 0.047026373, + -0.1958446, + 0.0044924226, + 0.1719895, + 0.10121101, + 0.17643425, + -0.046456266, + 0.17635134, + 0.01827563, + 0.08540644, + 0.06120104, + -0.113148265, + -0.1218303, + 0.011219538, + -0.12710062, + -0.050977923, + 0.053127747, + -0.03822012, + -0.00858817, + 0.014970831, + -0.12115409, + 0.008089462, + 0.068597436, + 0.08169945, + -0.10300656, + 0.05761663, + -0.0078221625, + -0.06458102, + 0.020226378, + -0.005276232, + 0.0895599, + 0.08770444, + -0.19067658, + -0.044448506, + -0.13667849, + -0.045721535, + -0.017611813, + -0.17994289, + 0.061323702, + -0.006091955, + 0.037093177, + 0.051335093, + 0.048674185, + 0.10398149, + -0.023160957, + -0.035153363, + 0.12117348, + 0.038042266, + -0.074533135, + 0.014664317, + 0.027021443, + 0.0034762379, + -0.06064791, + -0.015092715, + -0.2944957, + 0.10236635, + 0.15401675, + 0.15111294, + 0.1340751, + -0.17079759, + 0.06980032, + -0.07446118, + -0.075741716, + 0.045889974, + -0.027051862, + 0.061429117, + 0.13151085, + 0.2020734, + -0.05170606, + -0.019183844, + -0.029455291, + -0.16436203, + 0.065099075, + 0.114943504, + 0.07208407, + -0.052109897, + 0.058475323, + -0.0060676374, + 0.108448036, + 0.13429314, + 0.07835998, + -0.042943794, + 0.07505265, + -0.15782769, + -0.0008275639, + -0.022941407, + -0.089207046, + -0.053613666, + -0.22768244, + -0.004562265, + 0.119613804, + -0.06730513, + -0.005134274, + 0.015889524, + 0.065859795, + -0.13738343, + 0.04806415, + -0.021074295, + 0.02389348, + -0.04589438, + -0.00791156, + 0.19126686, + 0.066830024, + 0.038546782, + 0.056414336, + -0.09672572, + -0.1488234, + -0.1606585, + 0.03534793, + -0.009993679, + 0.0033058047, + 0.052286904, + -0.09087651, + 0.015177222, + -0.1209879, + 0.117720395, + -0.17759861, + -0.045116693, + 0.3353583, + -0.17735094, + -0.255285, + -0.20740092, + -0.018373836, + 0.0005954378, + -0.011089623, + -0.21794018, + 0.14621736, + 0.017658569, + -0.07371307, + 0.021241616, + -0.12774628, + 0.12024284, + -0.069704995, + -0.08754762, + 0.052305937, + 0.13504027, + -0.0111032175, + 0.0896511, + -0.027011193, + 0.0964028, + -0.04770052, + -0.05891503, + -0.29946882, + 0.06799919, + 0.019578407, + -0.09722951, + 0.040762562, + -0.039349716, + -0.04986413, + 0.008755783, + -0.035490353, + 0.28766772, + -0.019644262, + -0.14135677, + -0.08013137, + 0.06394487, + 0.17467594, + -0.10952025, + -0.067189634, + -0.05346993, + 0.13319, + -0.02134673, + 0.08383482, + -0.005200357, + -0.013754012, + -0.020868568, + 0.10664089, + 0.266166, + 0.0021528348, + 0.17895368, + -0.0036033643, + 0.007758813, + 0.0316086, + -0.008563442, + 0.008860687, + -0.10833197, + 0.063689224, + -0.09778756, + -0.25244159, + 0.25641277, + 0.07796288, + -0.0037101684, + 0.04819473, + -0.08170173, + 0.015830114, + -0.028049998, + 0.02885121, + -0.028635604, + -0.05007718, + 0.112869985, + -0.060543325, + -0.05746815, + -0.14462683, + 0.1255916, + 0.121556595, + 0.102449775, + 0.120668866, + 0.0036547463, + 0.13570864, + 0.06869708, + 0.19977914, + -0.14529352, + 0.16520394, + 0.019234715, + 0.021917906, + -0.19478388, + -0.07423999, + -0.13678935, + -0.13091457, + 0.10972841, + -0.16713484, + -0.10109219, + -0.13254036, + -0.007902096, + 0.06695212, + -0.051268183, + -0.03536522, + 0.09367453 + ], + [ + 0.11637956, + -0.17186578, + 0.10791955, + 0.080459945, + 0.05975105, + -0.011272944, + 0.10946989, + 0.22342005, + 0.099497564, + 0.04502466, + -0.014406636, + 0.10069611, + 0.024731083, + -0.031065537, + -0.058351807, + 0.06504147, + 0.15619484, + -0.06604318, + 0.1781333, + 0.22792023, + -0.13898799, + -0.045416683, + 0.031234754, + 0.037138533, + -0.18047316, + -0.004940352, + -0.056690544, + 0.04373834, + -0.1627064, + -0.10905386, + -0.040010177, + 0.3229176, + 0.07230417, + 0.07616515, + -0.028685225, + -0.055634305, + -0.03288254, + -0.004032664, + 0.092190936, + -0.35969818, + -0.06260908, + 0.13865389, + -0.026160512, + 0.06181806, + 0.10938138, + -0.121704094, + 0.19553061, + -0.06748917, + -0.06838561, + 0.10467503, + -0.070798166, + -0.0747031, + -0.074544385, + 0.099389784, + 0.08020828, + -0.14345938, + 0.058599636, + -0.08092364, + 0.15528879, + 0.053775847, + 0.05522898, + -0.17603008, + 0.08225849, + -0.072476104, + 0.19657768, + -0.18427952, + -0.054881528, + 0.005319496, + -0.108707465, + -0.18644795, + -0.051599722, + -0.26696226, + 0.087243766, + 0.027708074, + 0.043569595, + 0.0017946062, + 0.061552893, + -0.0073106713, + 0.13140735, + -0.086496115, + 0.035292115, + 0.04865788, + 0.27236786, + 0.064673945, + -0.07814582, + -0.10368492, + 0.021018954, + 0.007109233, + 0.16142946, + -0.32392415, + 0.11618061, + 0.16055097, + -0.10717878, + -0.027265606, + -0.052559048, + 0.06769613, + 0.109514505, + -0.051763024, + -0.05986832, + 0.054277677, + 0.020888394, + 0.09378125, + -0.101614766, + -0.114495024, + 0.19143566, + -0.13538244, + -0.030124659, + 0.029111356, + -0.0665248, + -0.019361828, + 0.03687205, + -0.14246751, + 0.059040654, + -0.04202304, + -0.22132228, + 0.00042482893, + 0.06615944, + 0.16350724, + 0.29969427, + 0.15013222, + -0.057181604, + 0.045494482, + 0.09992527, + 0.10001293, + 0.07406286, + -0.02437011, + -0.0807268, + -0.2998727, + -0.17070091, + -0.13470376, + -0.07132043, + -0.067651786, + 0.00096411596, + 0.009365229, + -0.0010018462, + -0.02374255, + -0.022331577, + 0.008360721, + -0.24549751, + 0.09109612, + 0.07186384, + -0.21236059, + -0.004149225, + 0.029868592, + -0.16807589, + 0.38404953, + -0.07270962, + 0.06128629, + 0.0029842712, + -0.08691458, + -0.022859711, + -0.15697275, + -0.110039994, + 0.07708965, + 0.16576324, + -0.14948945, + 0.15736353, + 0.23472643, + -0.004841239, + 0.005375038, + -0.104134165, + 0.29700267, + -0.15814611, + 0.018846232, + 0.02098197, + -0.020519339, + 0.022707358, + 0.1988144, + -0.11999712, + -0.02079705, + -0.058023993, + 0.0058619855, + 0.11121759, + 0.017506061, + 0.05126044, + -0.096916825, + 0.2307071, + -0.062108804, + 0.123407386, + 0.0031229893, + -0.044057585, + -0.096670195, + 0.21410017, + -0.041165788, + 0.035975046, + -0.15833475, + 0.12947544, + -0.3752908, + -0.03691883, + 0.095158406, + 0.032910068, + -0.12258116, + 0.035828426, + 0.06966969, + 0.016244017, + 0.170389, + -0.1261006, + 0.1098775, + 0.13936415, + -0.04809021, + 0.19010004, + 0.12529476, + -0.11426704, + -0.023094462, + -0.23561454, + 0.14241934, + -0.07336394, + -0.3049396, + 0.07135916, + 0.29270256, + 0.042280797, + 0.13791633, + 0.14932086, + -0.03536875, + 0.09843503, + -0.18766549, + -0.046467368, + 0.14914943, + -0.023084715, + 0.09644197, + -0.003209695, + -0.16610551, + -0.14070651, + -0.37365547, + 0.10216226, + 0.21309158, + 0.08469838, + -0.14306426, + -0.091675095, + -0.10465955, + 0.027742147, + 0.16314667, + 0.03690933, + -0.03567327, + -0.26394844, + -0.19670296, + 0.102186255, + -0.16789915, + 0.016203193, + -0.06374825, + 0.05287729, + -0.09745945, + 0.00722403, + 0.07676293, + 0.014780153, + -0.10793076, + -0.1901627, + -0.18417664, + 0.04534059, + -0.0716811, + 0.07866812, + 0.1439692, + 0.06014963, + -0.21927246, + 0.20979516, + -0.015994566, + 0.12689663, + 0.041828584, + 0.08181566, + -0.063342, + 0.12462212, + -0.10833612, + -0.040642224, + -0.062786385, + 0.021669455, + 0.02062207, + -0.15943547, + -0.14038363, + 0.04656669, + 0.22162975, + -0.03973174, + 0.07510139, + 0.06834014, + 0.0720175, + 0.121220686, + 0.008868144, + -0.13243613, + -0.050706953, + 0.0605855, + -0.07571807, + 0.0075447657, + -0.16944724, + 0.090943016, + 0.017109644, + -0.064935304, + -0.06073986, + 0.063119814, + -0.12180592, + -0.057504665, + 0.12659715, + 0.07531627, + 0.056290645, + 0.018820873, + 0.11000129, + -0.14517342, + -0.31130013, + -0.1983607, + 0.08439539, + -0.123937875, + -0.03123252, + -0.2514824, + -0.11085155, + 0.005044094, + -0.14976013, + -0.057709478, + 0.070679195, + 0.14638405, + -0.18428764, + -0.004577368, + 0.028631024, + 0.05115411, + 0.0073806057, + -0.051902503, + -0.021504115, + 0.0070884735, + 0.07090578, + 0.092421785, + 0.15728186, + 0.009211559, + -0.4120053, + -0.023852421, + 0.08603379, + -0.14602195, + -0.052523993, + 0.083968624, + 0.12750289, + -0.071089976, + 0.24766031, + 0.099861264, + -0.036790013, + 0.16011785, + -0.011961857, + -0.16865316, + 0.2080289, + 0.11488308, + -0.05541248, + -0.06588284, + -0.07385138, + -0.21823715, + -0.027737396, + 0.0028189542, + 0.023575094, + -0.024090767, + 0.0011296587, + -0.11189924, + 0.005202209, + 0.07130078, + 0.014037095, + 0.047963694, + -0.06790311, + 0.11792759, + 0.088360295, + -0.007919029, + 0.066773124, + -0.04210118, + -0.26693234, + -0.0719914, + 0.24812208, + 0.2582869, + -0.01645684, + 0.09704079, + -0.06148363, + 0.0888649, + -0.26011285, + 0.013716547, + 0.025316631, + -0.12539926, + -0.01655872, + 0.018356794, + 0.09316022, + 0.2148001, + -0.42118096, + 0.09882388, + 0.0054554185, + 0.008558932, + 0.06381473, + 0.26438358, + 0.058518853, + -0.15347068, + 0.16173801, + -0.33100104, + 0.030894315, + 0.027564311, + -0.016866116, + 0.032837298, + 0.00077169156, + 0.03133184, + 0.06765387, + -0.049206775, + -0.016165834, + -0.212784, + 0.12581074, + 0.20880195, + -0.06285257, + 0.049284827, + -0.070513986, + -0.18638465, + 0.18946855, + 0.078632794, + -0.013579502, + 0.25505856, + -0.2065767, + 0.10019049, + 0.11834874, + -0.019598717, + -0.08845627, + -0.11897016, + -0.017256223, + 0.017200148, + 0.16272031, + 0.041610334, + 0.0394703, + -0.026481334, + 0.09240797, + 0.030089354, + -0.1735567, + -0.051628623, + -0.03471855, + 0.19301184, + 0.116581686, + -0.12833858, + 0.09992816, + -0.025773646, + 0.21380223, + 0.38609263, + -0.09755428, + 0.17755441, + 0.0037833347, + -0.08545212, + 0.09529221, + -0.21268612, + 0.00090696284, + -0.15830845, + -0.11832876, + 0.009103947, + 0.020926477, + 0.0012178889, + 0.19595702, + -0.040166292, + 0.050838795, + -0.16591446, + 0.015815735, + -0.28945345, + 0.29379824, + -0.042782035, + -0.17004989, + 0.19729897, + 0.0055258335, + -0.03620126, + -0.19522566, + -0.121389605, + -0.00032941398, + -0.17998022, + 0.06779911, + 0.2631502, + 0.13141875, + -0.04985646, + 0.052451488, + 0.24368726, + 0.07297032, + -0.053266484, + -0.019812576, + 0.17073107, + 0.11443813, + -0.034046765, + -0.045550775, + -0.034369446, + 0.07116018, + 0.035985112, + 0.10965749, + -0.005791836, + 0.02731971, + -0.07340144, + 0.10565047, + 0.11125925, + 0.05740682, + -0.17137615, + 0.045366608, + 0.029304776, + -0.13445312, + -0.039387558, + 0.08226571, + -0.028178785, + -0.016610812, + 0.0103821615, + -0.23829375, + -0.24541384, + -0.008268759, + 0.07972249, + 0.16931161, + -0.045478042, + 0.03659767, + -0.09050166, + -0.021094369, + -0.032404713, + 0.1336974, + 0.0533087, + 0.035258383, + -0.1752104, + 0.23809828, + 0.08637835, + -0.20440699, + -0.16151933, + 0.013910408, + -0.22518755, + 0.23404773, + -0.06430676, + -0.2438961, + 0.054066826, + -0.01222482, + 0.04206431, + 0.10275902, + 0.1509213, + -0.3044058, + -0.024547627, + -0.08197543, + 0.12862152, + 0.04344292, + -0.013134288, + -0.033549696, + -0.05573171, + 0.14470781, + -0.29964295, + -0.051656418, + -0.043963168, + -0.0626571, + 0.054504357, + -0.07732341, + 0.12172327, + 0.04756539, + -0.057771176, + -0.18838337, + -0.13257566, + -0.03424755, + 0.27295038, + 0.0039459704, + 0.08678922, + 0.09803905, + -0.5035922, + 0.0133419875, + 0.00065672013, + -0.02211246, + 0.04287359, + -0.012378799, + -0.07109952, + -0.13045481, + 0.016000884, + 0.12478969, + -0.16040824, + -0.22890955, + -0.00045370215, + 0.15292962, + 0.090407826, + 0.088943146, + 0.036541674, + 0.06473803, + 0.067636736, + 0.033031188, + 0.1036692, + 0.067575134, + 0.21188134, + 0.1192539, + -0.08119708, + -0.13598263, + 0.10788264, + -0.05933999, + -0.18349162, + 0.1735971, + 0.19631074, + -0.18473329, + -0.12218243, + 0.032747492, + -0.097329825, + 0.06238207, + 0.035778794, + 0.07071465, + 0.15152672, + 0.070239685, + -0.10766326, + 0.050400086, + 0.050804477, + -0.011483971, + -0.121493325, + -0.15133147, + -0.03250726, + 0.04693144, + 0.0005621336, + 0.236917, + -0.054525223, + 0.21746288, + 0.1697138, + 0.31465507, + -0.062002458, + -0.02546646, + -0.28721526, + -0.2513955, + 0.010789273, + 0.018919708, + -0.25192356, + 0.018824613, + 0.118578434, + -0.00507896, + 0.014482422, + 0.11267149, + 0.14215098, + 0.11573328, + 0.04761225, + 0.08724168, + -0.29671982, + 0.028040556, + -0.17270142, + 0.021092054, + -0.20256923, + -0.11460052, + -0.06707171, + 0.05905861, + 0.025438156, + 0.123720355, + -0.14976059, + 0.042247787, + 0.08170481, + 0.02156635, + 0.075605355, + -0.03911428, + 0.039365444, + -0.05059143, + -0.101301275, + -0.037720732, + -0.026390677, + -0.0656627, + -0.20962778, + -0.42560935, + 0.020281976, + -0.09318103, + -0.12504372, + 0.17505816, + 0.23766881, + 0.14905949, + -0.116007335, + -0.06740984, + 0.113025166, + -0.029503759, + 0.30562577, + -0.11261386, + -0.014164033, + -0.004014467, + 0.120359935, + 0.00029712665, + -0.01933728, + -0.0020288113, + -0.035912253, + -0.0998972, + 0.10274502, + -0.003980527, + 0.010540119, + -0.0012870778, + 0.03024298, + -0.019371746, + -0.08722253, + -0.30856013, + -0.024190273, + -0.09390957, + -0.11412507, + -0.3192844, + -0.11418165, + 0.0010766103, + 0.056867734, + -0.06406079, + -0.15870427, + -0.15309812, + 0.021257922, + -0.08891713, + 0.4405356, + 0.024239432, + 0.0012664798, + 0.035958398, + 0.2549329, + 0.098144576, + 0.16194685, + -0.083581924, + -0.12655054, + 0.21620502, + -0.046908326, + 0.12320124, + 0.042971227, + 0.007110838, + -0.048693836, + 0.21834254, + 0.065210894, + -0.029152373, + -0.15173002, + 0.00884002, + -0.14687325, + -0.0051244213, + -0.08612436, + 0.017426912, + -0.12191853, + 0.057925828, + -0.064879015, + -0.25661778, + -0.13962127, + 0.012649742, + -0.17353316, + -0.08397678, + -0.076075956, + 0.12900859, + 0.17618789, + 0.015170843, + -0.0966266, + 0.014381747, + 0.070817225, + 0.14590451, + -0.012652974, + 0.1212396, + 0.1928499, + 0.034840625, + -0.107425846, + 0.13786417, + -0.07710894, + -0.038374882, + 0.37778434, + 0.022085752, + -0.12701413, + 0.03516333, + -0.27293193, + 0.15730625, + 0.15843415, + -0.10312135, + -0.014996361, + -0.159087, + -0.04362026, + -0.18499237, + -0.02748742, + 0.032441035, + -0.020762652, + 0.08412549, + 0.021805234, + 0.00660146, + 0.061379142, + 0.0821473, + -0.034735933, + -0.21985133, + -0.07475556, + 0.44629174, + -0.03631395, + 0.11672802, + 0.049370393, + -0.041611828, + 0.22077566, + -0.014968166, + -0.19998384, + 0.011433027, + 0.06101584, + -0.11258323, + 0.13738437, + -0.0688601, + -0.21262124, + 0.102296345, + -0.061489206, + 0.057119362, + 0.23424532, + 0.120877706, + -0.047151435, + 0.2176173, + 0.20899433, + -0.00081871107, + -0.013194566, + 0.05514477, + 0.3928346, + -0.12549223, + -0.11095625, + 0.012750077, + -0.09158463, + -0.019312715, + -0.04007366, + 0.16496634, + 0.15461186, + -0.0877878, + -0.1798766, + -0.016334271, + 0.120540105, + 0.061884813, + 0.014563505, + 0.10252879, + -0.047985137, + -0.055586945, + 0.22566938, + 0.1058749, + 0.004186561, + 0.10453594, + 0.034418534, + 0.2573554, + -0.19616708, + 0.049920246, + -0.2728407, + 0.010139094, + -0.15609851, + -0.2773467, + 0.08028425, + -0.21471062, + -0.0010262715, + -0.02221235, + 0.27420497, + -0.12729512, + 0.12268265, + -0.036199618, + -0.22428453, + 0.17106162, + 0.016466241, + 0.13175893, + -0.13125266, + 0.16843618, + 0.030138072, + -0.18637407, + -0.17876884, + -0.025821261, + -0.022291679, + -0.032874994, + 0.118431464, + -0.112453796, + 0.025207195, + 0.049616717, + 0.042266954, + -0.13243802, + -0.2631501, + 0.033834856, + 0.0038898303, + 0.21306744, + -0.05530253, + 0.119763054, + -0.027107682, + 0.021657147, + 0.071491025, + -0.14394669, + 0.098971434, + -0.104662135, + 0.043255832, + -0.11081497, + 0.1253336, + 0.07292208, + 0.105347626, + -0.1376309, + 0.14321448, + 0.12819311, + -0.16884637, + -0.020582603, + 0.07983813, + -0.10436447, + -0.19395255, + -0.14573762, + -0.05208073, + 0.052003495, + -0.03613725, + -0.18182264, + 0.22837463, + -0.08308755, + -0.09978127, + -0.036853623, + 0.13789463, + -0.056370385, + 0.29830414, + -0.0791662, + 0.06277126, + -0.092597716, + 0.076572105, + -0.048261747, + -0.114597894, + 0.16213843, + 0.1433155, + 0.035506878, + -0.033455156, + 0.19248319, + -0.07745398, + -0.19590208, + 0.3018688, + 0.03310655, + 0.0041582887, + 0.19586419, + -0.10853914, + -0.16483396, + -0.17642425, + 0.19710363, + -0.06962616, + 0.17102459, + 0.10896041, + -0.13901196, + -0.12664439, + 0.09078699, + -0.12244493, + 0.052821994, + -0.030397188, + -0.13134342, + 0.053447146, + -0.18429838, + 0.26444325, + 0.22305855, + 0.029930143, + -0.054558307, + 0.0353794, + -0.0009788725, + -0.1188078, + 0.059667766, + -0.27633518, + 0.046799272, + -0.15642197, + -0.10854975, + 0.106093995, + -0.22275078, + 0.038943406, + -0.03847844, + 0.052943595, + 0.02457092, + -0.02215091, + 0.041665964, + -0.09696136, + -0.02566959, + -0.039870802, + -0.054774225, + 0.02291138, + -0.0417671, + 0.18173029, + -0.03932743, + 0.08204516, + 0.0959776, + -0.20656648, + 0.28138176, + 0.037579037, + 0.20550737, + -0.0048693623, + 0.08957929, + 0.07936084, + 0.064060465, + -0.076720774, + -0.04245319, + 0.1251032, + -0.048257716, + 0.14970464, + -0.019789496, + -0.12023016, + 0.31614852, + 0.048109207, + 0.018149873, + 0.005182097, + -0.051101666, + -0.118794665, + 0.028441736, + 0.024802344, + 0.16004704, + 0.21352114, + -0.0831253, + -0.0066282563, + 0.057000577, + 0.057913885, + 0.03354726, + -0.076870695, + -0.20632678, + 0.02378624, + -0.11414494, + -0.28388187, + -0.24170417, + 0.026353586, + 0.01276097, + -0.20049897, + 0.16650756, + 0.0765534, + -0.023719413, + 0.02053945, + 0.31372947, + 0.16620825, + -0.019555673, + -0.02928945, + -0.060654677, + -0.034009326, + -0.13074087, + -0.086977184, + -0.11527837, + -0.01426738, + 0.019683821, + -0.16352665, + 0.26811838, + 0.20973662, + 0.113176554, + -0.30779162, + 0.04542188, + 0.11580485, + -0.11855776, + -0.20264056, + -0.22960435, + 0.009477502, + 0.1754439, + -0.03466805, + 0.08613063, + -0.009424178, + -0.104562424, + 0.03825546, + -0.20300105, + 0.12507758, + 0.12184145, + 0.140918, + -0.21595697, + -0.1149452, + -0.0863683, + 0.011771966, + -0.0794508, + -0.16730252, + 0.083861485, + -0.073139645, + -0.03015001, + -0.2253941, + -0.2636101, + -0.111474104, + -0.012658355, + 0.12943146, + 0.0024419622, + -0.008086202, + -0.031769402, + 0.22267817, + 0.09031554, + 0.084421515, + 0.07658035, + 0.09284541, + 0.0123357065, + 0.13187145, + -0.021085177 + ], + [ + 0.044147793, + 0.16940491, + -0.07660075, + 0.01839263, + 0.042040065, + -0.047721248, + -0.21912621, + -0.0009365094, + 0.06492754, + 0.042232044, + -0.004531145, + 0.11653948, + -0.006253255, + -0.020997126, + -0.086448684, + 0.065257326, + -0.09350857, + 0.05722746, + -0.09045094, + -0.20966446, + -0.020684095, + 0.05035577, + -0.0058316975, + -0.08649621, + 0.11655366, + -0.035102855, + -0.03805832, + -0.00616778, + -0.013295165, + -0.088805586, + 0.13911676, + -0.1802159, + -0.018650677, + -0.028832428, + -0.15153827, + -0.012014452, + -0.10429693, + -0.02334544, + 0.013291807, + 0.2931698, + 0.25664425, + -0.022222053, + 0.10681528, + -0.02429105, + -0.14298108, + -0.03535962, + -0.20939623, + 0.15250647, + 0.0886579, + -0.08623361, + 0.10814994, + 0.0907847, + 0.17988828, + 0.07620976, + -0.07824574, + -0.014679582, + 0.0817115, + 0.016898714, + -0.28710848, + -0.22659548, + -0.0431887, + 0.035005502, + -0.030766238, + -0.027952312, + 0.00605425, + 0.15439883, + -0.068923496, + 0.0733908, + 0.009504215, + 0.024958564, + 0.14828947, + 0.13880697, + -0.090828896, + 0.003277354, + 0.10241844, + -0.115077004, + -0.1662777, + 0.03727921, + -0.06837462, + 0.07184318, + -0.006519263, + -0.10122025, + -0.14128436, + -0.064970486, + -0.036978073, + 0.010498385, + 0.08859915, + 0.08093246, + -0.06282354, + 0.25009853, + -0.012446843, + -0.121961325, + 0.1972666, + 0.046384234, + 0.199937, + -0.058210306, + -0.062178757, + 0.034782343, + -0.024504201, + -0.10704923, + 0.023402458, + -0.06502446, + -0.15917829, + 0.030689063, + -0.1270888, + 0.06968928, + 0.071537375, + 0.067840576, + 0.19173016, + 0.034167826, + 0.21227396, + 0.05622944, + -0.05703442, + -0.06396645, + 0.100448065, + -0.035514005, + -0.014672414, + -0.037831403, + -0.17126168, + -0.13165945, + -0.12100334, + 0.13692331, + -0.18549083, + -0.09306254, + -0.0049064197, + -0.05379398, + -0.13972266, + 0.20454313, + 0.076266214, + 0.12522706, + -0.05245607, + -0.027636189, + -0.113891274, + -0.12186737, + 0.094089486, + 0.03439381, + -0.06949138, + -0.055920634, + 0.16840659, + 0.0150665045, + 0.08506466, + 0.13945717, + -0.041639358, + -0.06740168, + 0.09963331, + -0.14591189, + 0.059449732, + 0.0025124077, + -0.027144495, + 0.05402788, + 0.20959063, + 0.13937241, + -0.080275685, + 0.038352527, + -0.2715623, + 0.17126106, + -0.091786325, + -0.14857501, + -0.1293791, + -0.18837348, + 0.047239408, + -0.28292495, + 0.0063572284, + 0.09690516, + -0.029995194, + -0.11105718, + -0.018782707, + -0.30740905, + 0.104199484, + 0.15624839, + -0.03974281, + -0.05248947, + 0.038652018, + 0.08289482, + 0.016197314, + 0.119475886, + -0.11403887, + 0.15416206, + -0.08065634, + -0.025980428, + -0.015932493, + -0.008749679, + 0.08308672, + -0.11037272, + -0.048986003, + 0.21674192, + -0.05803945, + 0.04971493, + -0.01891516, + 0.004877139, + -0.17709245, + -0.002087021, + -0.0020960409, + -0.106576554, + -0.007810016, + -0.12720583, + 0.07408409, + -0.02180893, + 0.0007753394, + -0.005287165, + -0.3254629, + -0.0026433906, + 0.04833213, + 0.033785746, + 0.18600549, + 0.0022767857, + 0.18876381, + 0.035533134, + 0.054217387, + 0.03794934, + 0.1271445, + 0.030882537, + 0.025446987, + 0.15345229, + -0.10454748, + 0.26222214, + -0.0091688065, + -0.041628335, + 0.018512165, + 0.0040028314, + 0.20311502, + 0.0037376555, + 0.070144124, + 0.20071377, + 0.11248816, + -0.057424825, + 0.13850467, + 0.18345869, + -0.0044744764, + -0.021348322, + 0.04384846, + -0.022934612, + -0.107814014, + 0.09007019, + 0.02416682, + -0.0012119029, + -0.24780564, + 0.073680155, + -0.04818337, + 0.083472714, + 0.12236319, + -0.02055338, + 0.027101975, + 0.04737765, + -0.110173546, + 0.06364295, + -0.008946159, + 0.15252261, + 0.0027322657, + 0.025759516, + -0.0029763042, + -0.11247266, + 0.13522747, + 0.15886107, + -0.15792294, + -0.07445264, + -0.21184248, + 0.07564331, + -0.053616013, + -0.07139231, + 0.049195178, + -0.084485546, + -0.046140656, + 0.02825539, + 0.021405263, + 0.097813755, + 0.13759468, + 0.0525562, + -0.018069427, + -0.12837838, + 0.077376045, + 0.031107048, + -0.27642173, + -0.13174976, + -0.15429558, + -0.01833851, + 0.027519647, + -0.015669333, + -0.050222356, + 0.02163741, + -0.035224333, + 0.04344882, + -0.10721566, + 0.08589194, + -0.004909294, + -0.032711554, + 0.30826396, + 0.117197126, + 0.124197006, + -0.24572933, + -0.04550309, + -0.12412157, + -0.0724078, + -0.128096, + 0.219507, + 0.1134307, + 0.21780996, + -0.10823206, + 0.11963701, + -0.068851374, + 0.023465648, + -0.01593774, + -0.118752144, + -0.016288111, + -0.030651504, + -0.12503572, + -0.17363529, + 0.19402781, + 0.10475469, + -0.020954344, + 0.019589314, + -0.030481823, + 0.0066996072, + -0.053601895, + -0.20029072, + -0.10582614, + -0.20040359, + -0.0054524555, + -0.055335585, + 0.3311847, + 0.035756826, + -0.049566645, + 0.07918633, + -0.053583127, + -0.09904188, + -0.10038896, + 0.04996928, + -0.099653706, + 0.061264403, + -0.0061791865, + -0.21508417, + 0.116991624, + 0.2686714, + -0.34464577, + -0.039873548, + -0.013331849, + -0.017662382, + 0.18063131, + 0.107568875, + -0.12173686, + -0.059680745, + 0.02161945, + -0.05680074, + -0.017660705, + 0.14479858, + -0.018916478, + 0.049323782, + 0.09538939, + -0.09067543, + -0.049188662, + -0.09265428, + -0.0365639, + 0.0028700577, + 0.059004173, + 0.05948634, + 0.13971259, + 0.18176614, + -0.016404018, + -0.2375804, + 0.02650564, + -0.10457582, + 0.06823804, + -0.044831805, + -0.06590748, + -0.013364427, + 0.014154022, + 0.042477664, + 0.1097775, + -0.11523345, + -0.1079408, + -0.12961785, + 0.34049404, + -0.08560808, + -0.016122337, + -0.052292738, + 0.024983414, + -0.030858947, + 0.13338524, + -0.059871174, + 0.038500845, + 0.37170732, + 0.13350071, + -0.021032728, + -0.07347119, + 0.06407454, + 0.06119444, + 0.0066508185, + 0.10506183, + 0.08963466, + -0.1445016, + 0.01706449, + -0.12059136, + -0.09551309, + 0.09726227, + -0.023860846, + -0.020491932, + 0.11087211, + -0.063431025, + -0.082183585, + -0.0409627, + -0.08496014, + 0.120390706, + -0.17681535, + 0.002092146, + -0.10478897, + 0.08459577, + 0.12726188, + 0.045021284, + -0.005031372, + -0.08353973, + -0.101410136, + 0.019969966, + -0.13681094, + 0.23727098, + -0.11417003, + 0.16008934, + -0.0045129163, + 0.074293375, + -0.20552878, + 0.21652737, + 0.23047388, + 0.14112242, + 0.0013431809, + -0.23968937, + -0.24464427, + 0.091682285, + -0.09774535, + 0.013322989, + 0.08752573, + -0.027159441, + 0.09978838, + 0.19199398, + 0.008760841, + 0.16704173, + -0.10677443, + -0.19106703, + -0.012075937, + -0.13198557, + -0.058435682, + -0.02389443, + 0.23472294, + -0.1088765, + 0.077165045, + -0.24123153, + 0.21209829, + 0.019728398, + -0.15316392, + 0.044813976, + -0.03404352, + -0.08867594, + 0.077576004, + -0.13115795, + 0.27733845, + -0.024984693, + -0.2628973, + -0.008487113, + -0.0053662844, + -0.049809393, + -0.2577814, + -0.12825035, + -0.017539369, + 0.08355548, + 0.04310515, + -0.0396339, + 0.09730884, + 0.09619792, + 0.15564808, + 0.017628962, + -0.1090823, + -0.13700478, + 0.029394176, + 0.091170415, + -0.012858434, + -0.0054733506, + 0.09170598, + -0.113937475, + 0.21517824, + 0.027744953, + -0.20661183, + 0.006941276, + -0.019983957, + -0.23312838, + -0.120168075, + -0.037594784, + -0.004498651, + 0.20085981, + 0.21437016, + -0.009439298, + 0.037814677, + -0.077133186, + 0.17605172, + -0.115668155, + -0.046497446, + 0.044113774, + 0.19949457, + -0.03338365, + 0.05692018, + -0.06910936, + -0.0046057226, + -0.18073574, + -0.07666095, + 0.13720323, + 0.2569485, + -0.035785932, + 0.09387681, + -0.12529814, + 0.05789777, + 0.15947631, + -0.043478936, + -0.034896307, + 0.02509474, + -0.066015504, + -0.17300786, + 0.2740886, + -0.008645607, + 0.20366979, + -0.17741504, + 0.0016051099, + 0.04343355, + 0.053064674, + 0.10233873, + -0.057792492, + 0.28412506, + -0.027595375, + -0.011127119, + 0.23101905, + -0.03437331, + 0.12712002, + -0.040828258, + -0.2521543, + 0.092959955, + 0.1476991, + -0.016959714, + 0.24718697, + -0.08578792, + -0.042732067, + -0.02364189, + 0.038429204, + 0.07348472, + 0.041489173, + -0.11468599, + -0.049400177, + -0.027466021, + -0.009244471, + -0.017509859, + 0.08712863, + -0.022377755, + 0.01853816, + 0.042881876, + 0.3835641, + 0.0874544, + -0.068334155, + 0.057505295, + -0.09062925, + 0.057019502, + -0.1344657, + 0.08132518, + 0.000053040872, + -0.059475925, + -0.12508613, + -0.021813422, + -0.013378662, + 0.024890503, + 0.024216384, + -0.077127144, + 0.007335973, + 0.08234029, + -0.18302993, + -0.035066325, + -0.0072978884, + 0.050056715, + -0.22931813, + 0.09650683, + -0.0073624705, + -0.2885135, + -0.10775086, + -0.23143713, + -0.086137205, + 0.0498812, + -0.033241358, + 0.01776586, + -0.026235383, + 0.19309425, + 0.025691753, + -0.058075365, + -0.2405673, + -0.21786085, + -0.19851258, + 0.046656214, + -0.10293365, + -0.10813984, + -0.2152836, + 0.15489893, + 0.047911253, + 0.20260984, + 0.2088183, + -0.1018085, + -0.087688275, + 0.13801338, + 0.0046194997, + -0.1746844, + 0.004021364, + -0.07318715, + -0.24332349, + -0.32633016, + 0.055268224, + -0.013017414, + 0.052299757, + 0.1270179, + 0.004340157, + 0.057317145, + 0.21436898, + 0.06542589, + 0.06742579, + -0.112878166, + -0.020031923, + -0.04590336, + -0.2186622, + -0.11536247, + 0.03292602, + -0.037806716, + 0.11775531, + -0.08126202, + 0.044630688, + 0.004407897, + 0.10470092, + 0.106795885, + -0.02859561, + 0.024228014, + -0.03555758, + 0.14519277, + 0.14093313, + 0.26003087, + 0.0070421537, + 0.08488326, + -0.12916775, + -0.1219826, + -0.2455656, + 0.15563421, + 0.08667417, + 0.01301937, + -0.0907239, + -0.12462255, + -0.032354463, + 0.20714861, + 0.0010497222, + -0.13925967, + 0.03371706, + -0.0062380685, + 0.055929244, + -0.067029044, + 0.059437487, + 0.0076191374, + 0.05578321, + 0.011333173, + -0.05390713, + 0.15975079, + 0.072687715, + 0.15916237, + 0.17636605, + -0.008446719, + 0.014961095, + 0.08686759, + 0.14848803, + -0.22238462, + 0.09633494, + -0.055627063, + 0.12631376, + 0.035983652, + 0.22738391, + -0.11332371, + 0.21841496, + -0.17675073, + 0.14812225, + -0.114016265, + -0.032317907, + -0.18767738, + -0.0811073, + -0.13753203, + 0.16913415, + 0.0114353, + -0.11488439, + 0.011230222, + -0.085470885, + 0.0690153, + -0.10461269, + -0.0706403, + -0.20100483, + -0.037651025, + 0.08529463, + -0.13536386, + -0.034356836, + 0.057857174, + -0.03706257, + 0.02483604, + -0.24963166, + -0.06943555, + -0.09979882, + 0.0942273, + 0.05437428, + -0.10782203, + -0.013504401, + 0.24189045, + -0.13296025, + 0.03774331, + -0.010911639, + -0.082099736, + 0.07280296, + 0.06079229, + 0.02287312, + 0.059112385, + -0.099384256, + -0.076120414, + 0.11072304, + -0.05123164, + 0.1079739, + 0.10227008, + -0.05471506, + 0.10747948, + 0.009699896, + -0.30863142, + -0.04430503, + 0.37351134, + -0.19678722, + 0.13742797, + -0.40235984, + -0.27581173, + 0.3702671, + -0.024754377, + 0.122642905, + -0.14828043, + -0.07008751, + 0.08360051, + 0.0035450035, + 0.0216809, + -0.12010145, + -0.026985122, + 0.053328913, + -0.10591583, + -0.013924753, + 0.038145293, + 0.17823228, + 0.06572483, + -0.28199768, + 0.14622784, + 0.10302125, + -0.10776388, + 0.16953847, + -0.043120794, + -0.05191318, + 0.03779815, + 0.12021654, + -0.0014165355, + 0.023631074, + -0.2394369, + 0.114601746, + 0.055467404, + 0.16078594, + 0.009649481, + -0.21886353, + -0.10147424, + 0.01851072, + 0.13010912, + -0.18243171, + -0.046989094, + 0.11791553, + -0.07706595, + 0.052286703, + -0.19268094, + -0.06734655, + 0.083558746, + 0.05510528, + 0.1205322, + 0.043853316, + 0.19419815, + -0.19383074, + -0.12088962, + 0.050363027, + 0.13108388, + -0.032544736, + -0.19773011, + -0.034516454, + -0.13052338, + -0.0046362486, + 0.0320051, + 0.061747782, + -0.19046386, + 0.09733934, + 0.08370847, + -0.120636776, + -0.051564053, + -0.08520327, + 0.025040282, + -0.047831807, + 0.13498239, + -0.05799306, + -0.082822345, + 0.11464385, + -0.11085481, + 0.11331335, + 0.06106606, + 0.12057433, + -0.1630841, + 0.00096572586, + -0.039613742, + 0.16483565, + 0.17725827, + 0.024782866, + -0.02095867, + -0.30374828, + 0.030041676, + -0.34487057, + 0.016318211, + 0.010022732, + 0.16049328, + -0.059585165, + -0.038909353, + 0.14602326, + 0.003398922, + 0.10123426, + 0.10189341, + 0.0013612165, + -0.095394656, + 0.17065822, + 0.2717381, + -0.048805684, + 0.11726433, + -0.22115687, + -0.013294891, + -0.20146243, + 0.1301142, + -0.079273745, + -0.06366891, + 0.20852767, + -0.11919791, + 0.10993831, + -0.1328157, + 0.023110492, + 0.06534299, + -0.028473591, + 0.03133091, + 0.18335237, + -0.12560268, + 0.05174967, + 0.10752264, + 0.026674548, + -0.116931275, + 0.05302942, + 0.14527842, + 0.04175609, + 0.07524169, + -0.016850159, + -0.085036226, + 0.14378034, + -0.15384178, + 0.06842327, + 0.072759815, + 0.0333774, + -0.07724679, + 0.07146305, + -0.0038083985, + -0.023200184, + -0.21678805, + -0.05851518, + -0.2106472, + 0.21905945, + 0.044797596, + -0.08767725, + -0.06757387, + -0.08139683, + 0.060507063, + -0.2539122, + -0.054056864, + -0.0061713024, + -0.25016263, + -0.013922703, + 0.025297, + -0.031502247, + 0.043440044, + 0.049890436, + 0.10434008, + -0.14499368, + 0.011150854, + -0.16495703, + -0.21740839, + 0.004718739, + 0.048284378, + -0.047843166, + 0.047392353, + 0.10500571, + 0.031224746, + 0.15428485, + 0.03575991, + 0.23791198, + -0.036760766, + -0.21849649, + -0.14954391, + 0.12186352, + -0.030245116, + -0.014910633, + 0.052947998, + 0.07771576, + 0.22827105, + -0.025724977, + 0.13252845, + 0.15444413, + -0.09818237, + 0.031483896, + -0.10577348, + -0.00006837226, + -0.109357946, + 0.07215483, + 0.17097427, + 0.11899255, + 0.061613396, + 0.03566328, + 0.036564983, + 0.0024872916, + 0.06796512, + 0.026589885, + -0.060742363, + -0.07839288, + 0.09555347, + -0.050860837, + -0.12879173, + -0.1040309, + 0.21770604, + 0.0018934173, + 0.023243189, + -0.090174794, + -0.06827122, + 0.15387972, + -0.069496565, + 0.024794634, + -0.051390123, + 0.027016103, + -0.021958428, + -0.10045327, + 0.18993512, + -0.22860111, + -0.10041514, + -0.1531901, + 0.005921122, + -0.03854943, + 0.14580593, + -0.12484452, + 0.022898167, + -0.10113201, + 0.08594769, + 0.015126121, + -0.012950152, + 0.040228955, + -0.09867645, + 0.0058024707, + 0.1267348, + 0.19757093, + 0.01170412, + -0.17352274, + 0.30352616, + 0.3830606, + 0.05377779, + -0.07670585, + 0.025823055, + -0.05698721, + -0.009363706, + 0.077189244, + -0.15372942, + -0.29238287, + -0.25004295, + 0.024756031, + 0.04304348, + 0.081523225, + -0.07263159, + -0.13542518, + 0.08482435, + -0.06367541, + 0.01787075, + -0.027442627, + 0.13191804, + -0.25955498, + -0.2185974, + -0.0048445812, + 0.24410242, + 0.052365653, + 0.13663675, + -0.13785504, + 0.12467762, + 0.2333146, + -0.057672247, + -0.0937421, + 0.018837918, + -0.058080558, + -0.019427018, + 0.13319813, + 0.0118216975, + 0.09013087, + -0.06453432, + -0.06437333, + 0.0037089002, + 0.09036531, + -0.0066115037, + -0.016081506, + -0.1324408, + 0.075796016, + 0.031593855, + -0.15255858, + -0.12663953, + 0.17544357, + 0.060190212, + 0.24437538, + 0.089556195, + 0.20744215, + -0.05519144, + 0.13434736, + 0.13900068, + -0.07795903, + -0.05554333, + 0.010776608, + 0.048118893, + -0.06867825, + -0.1597975, + 0.038932495, + -0.09650621, + -0.07258938 + ] + ], + "bias": [ + -0.08385399, + -0.04598274, + 0.12983686 + ], + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.22 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.json new file mode 100644 index 0000000..c33aafd --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.json @@ -0,0 +1,1433 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.22 + }, + "metrics": { + "total": 319, + "accuracy": 0.6708464, + "confusion": [ + [ + 58, + 57, + 34 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 13, + 56 + ] + ], + "tool_precision": 0.9830508, + "tool_recall": 0.38926175, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22568056, + "p_memory": 0.40084177, + "p_abstain": 0.37347767 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31395635, + "p_memory": 0.33680987, + "p_abstain": 0.3492338 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39770705, + "p_memory": 0.19950679, + "p_abstain": 0.40278614 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.26465896, + "p_memory": 0.34730524, + "p_abstain": 0.38803577 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25941908, + "p_memory": 0.12832387, + "p_abstain": 0.612257 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.27416888, + "p_memory": 0.10814477, + "p_abstain": 0.61768633 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40384528, + "p_memory": 0.31830305, + "p_abstain": 0.27785167 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.32050437, + "p_memory": 0.27518332, + "p_abstain": 0.40431228 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37631887, + "p_memory": 0.241377, + "p_abstain": 0.38230407 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.30472597, + "p_memory": 0.32799238, + "p_abstain": 0.36728162 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.27784234, + "p_memory": 0.32759824, + "p_abstain": 0.3945594 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2540235, + "p_memory": 0.16229898, + "p_abstain": 0.58367753 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30672368, + "p_memory": 0.19407696, + "p_abstain": 0.49919936 + }, + { + "query_id": "shell_test_019", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48271036, + "p_memory": 0.17951614, + "p_abstain": 0.3377736 + }, + { + "query_id": "shell_test_020", + "text": "whats the total vram on ths gpu", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43741146, + "p_memory": 0.24507794, + "p_abstain": 0.3175106 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4158976, + "p_memory": 0.27515078, + "p_abstain": 0.3089516 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47624564, + "p_memory": 0.14048539, + "p_abstain": 0.38326898 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.48442715, + "p_memory": 0.10436712, + "p_abstain": 0.4112057 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48437816, + "p_memory": 0.2835406, + "p_abstain": 0.23208125 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44654188, + "p_memory": 0.26933104, + "p_abstain": 0.2841271 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39678445, + "p_memory": 0.44244283, + "p_abstain": 0.16077271 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4628093, + "p_memory": 0.270529, + "p_abstain": 0.26666173 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.34852275, + "p_memory": 0.27711794, + "p_abstain": 0.3743593 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47214958, + "p_memory": 0.29806998, + "p_abstain": 0.22978038 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40133312, + "p_memory": 0.2407558, + "p_abstain": 0.35791105 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.36721316, + "p_memory": 0.37156385, + "p_abstain": 0.26122305 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38931295, + "p_memory": 0.2874001, + "p_abstain": 0.32328695 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4190756, + "p_memory": 0.25449738, + "p_abstain": 0.32642704 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42050397, + "p_memory": 0.2790245, + "p_abstain": 0.30047154 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4390786, + "p_memory": 0.13317992, + "p_abstain": 0.4277415 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44003776, + "p_memory": 0.31567433, + "p_abstain": 0.2442879 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47923937, + "p_memory": 0.13611977, + "p_abstain": 0.38464084 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31453353, + "p_memory": 0.26278618, + "p_abstain": 0.42268026 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46307096, + "p_memory": 0.38064674, + "p_abstain": 0.1562823 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4797326, + "p_memory": 0.1527195, + "p_abstain": 0.367548 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43717074, + "p_memory": 0.29522896, + "p_abstain": 0.2676003 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39453, + "p_memory": 0.24812809, + "p_abstain": 0.35734192 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4571177, + "p_memory": 0.2604581, + "p_abstain": 0.2824242 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45665768, + "p_memory": 0.25635234, + "p_abstain": 0.2869899 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38890406, + "p_memory": 0.31626257, + "p_abstain": 0.29483342 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3217656, + "p_memory": 0.11337318, + "p_abstain": 0.56486124 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43858615, + "p_memory": 0.21885185, + "p_abstain": 0.34256205 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41309485, + "p_memory": 0.22376072, + "p_abstain": 0.36314443 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43877676, + "p_memory": 0.21598269, + "p_abstain": 0.34524047 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.35300845, + "p_memory": 0.21918875, + "p_abstain": 0.42780283 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41591418, + "p_memory": 0.28971443, + "p_abstain": 0.29437143 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3736646, + "p_memory": 0.23679724, + "p_abstain": 0.38953817 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39532402, + "p_memory": 0.28937247, + "p_abstain": 0.31530353 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.30211675, + "p_memory": 0.3692885, + "p_abstain": 0.3285947 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5483045, + "p_memory": 0.18406397, + "p_abstain": 0.26763153 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34390134, + "p_memory": 0.4241181, + "p_abstain": 0.23198053 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40889636, + "p_memory": 0.30667284, + "p_abstain": 0.2844308 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.74444443, + "confusion": [ + [ + 34, + 16, + 20 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 10, + 45 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.4857143, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + } + ] + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.md new file mode 100644 index 0000000..ca78599 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-5/router_report.md @@ -0,0 +1,151 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.500` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.220` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.6708` +- tools precision: `0.9831` +- tools recall: `0.3893` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 58 | 57 | 34 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 13 | 56 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.23 | 0.40 | 0.37 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.31 | 0.34 | 0.35 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Abstain` | 0.46 | 0.14 | 0.40 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.40 | 0.28 | 0.32 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Abstain` | 0.30 | 0.17 | 0.53 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.38 | 0.31 | 0.32 | +| `holdout_tools_recall_tools_010` | is there a unix socket at the agent.sock path right now? | `Tools` | `Abstain` | 0.36 | 0.19 | 0.45 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Tools` | `Memory` | 0.41 | 0.37 | 0.23 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.48 | 0.25 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Abstain` | 0.38 | 0.18 | 0.44 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.47 | 0.16 | 0.36 | +| `holdout_tools_recall_tools_025` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.40 | 0.25 | 0.36 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Abstain` | 0.35 | 0.13 | 0.52 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Abstain` | 0.29 | 0.14 | 0.57 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Abstain` | 0.45 | 0.14 | 0.41 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Abstain` | 0.37 | 0.20 | 0.43 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.28 | 0.25 | 0.47 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.39 | 0.27 | 0.35 | +| `holdout_tools_recall_tools_042` | how many cpu cores are available right now? | `Tools` | `Abstain` | 0.33 | 0.18 | 0.49 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.22 | 0.35 | 0.43 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Abstain` | 0.35 | 0.21 | 0.44 | +| `holdout_tools_recall_tools_058` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.40 | 0.37 | 0.23 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | 0.33 | 0.27 | 0.40 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.21 | 0.32 | 0.48 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.16 | 0.41 | 0.43 | +| `holdout_abstain_toolish_abstain_029` | vim or emacs? choose for me | `Abstain` | `Memory` | 0.25 | 0.31 | 0.44 | +| `holdout_abstain_toolish_abstain_030` | is gemma 4 better than llama 3.1? check online and decide | `Abstain` | `Memory` | 0.16 | 0.37 | 0.47 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.33 | 0.38 | 0.29 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | 0.34 | 0.34 | 0.33 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.34 | 0.23 | 0.44 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Abstain` | 0.43 | 0.12 | 0.45 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.45 | 0.29 | 0.25 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_abstain_toolish_tools_050` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Abstain` | 0.43 | 0.14 | 0.44 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Abstain` | 0.47 | 0.13 | 0.40 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.50 | 0.27 | 0.24 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.40 | 0.20 | 0.40 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.26 | 0.35 | 0.39 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.26 | 0.13 | 0.61 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.27 | 0.11 | 0.62 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.40 | 0.32 | 0.28 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.32 | 0.28 | 0.40 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.38 | 0.24 | 0.38 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.30 | 0.33 | 0.37 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.28 | 0.33 | 0.39 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.25 | 0.16 | 0.58 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.31 | 0.19 | 0.50 | +| `shell_test_019` | is the klbr tui currently connected to the daemon socket? | `Tools` | `Memory` | 0.48 | 0.18 | 0.34 | +| `shell_test_020` | whats the total vram on ths gpu | `Tools` | `Memory` | 0.44 | 0.25 | 0.32 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.42 | 0.28 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.48 | 0.10 | 0.41 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.48 | 0.28 | 0.23 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.45 | 0.27 | 0.28 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.40 | 0.44 | 0.16 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.46 | 0.27 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.35 | 0.28 | 0.37 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.47 | 0.30 | 0.23 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.40 | 0.24 | 0.36 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.37 | 0.37 | 0.26 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.39 | 0.29 | 0.32 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.42 | 0.25 | 0.33 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.42 | 0.28 | 0.30 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.44 | 0.13 | 0.43 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.44 | 0.32 | 0.24 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.31 | 0.26 | 0.42 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.46 | 0.38 | 0.16 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.48 | 0.15 | 0.37 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.44 | 0.30 | 0.27 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.39 | 0.25 | 0.36 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.46 | 0.26 | 0.28 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.46 | 0.26 | 0.29 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.39 | 0.32 | 0.29 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.32 | 0.11 | 0.56 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.44 | 0.22 | 0.34 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.41 | 0.22 | 0.36 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.44 | 0.22 | 0.35 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.35 | 0.22 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.42 | 0.29 | 0.29 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.37 | 0.24 | 0.39 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.40 | 0.29 | 0.32 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.30 | 0.37 | 0.33 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.55 | 0.18 | 0.27 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.34 | 0.42 | 0.23 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.41 | 0.31 | 0.28 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.4857` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7444` diff --git a/benchmarks/models/router/linear/out-router-linear-iter-6/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-6/router_model.json new file mode 100644 index 0000000..1fce175 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-6/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.16052741, + 0.0024608371, + -0.031318825, + -0.09885255, + -0.10179112, + 0.058994167, + 0.109656274, + -0.2224836, + -0.16442509, + -0.08725672, + 0.018937778, + -0.21723568, + -0.01847783, + 0.05206266, + 0.14480045, + -0.13029885, + -0.06268626, + 0.008815729, + -0.08768228, + -0.018255629, + 0.15967207, + -0.0049391207, + -0.02540304, + 0.049357686, + 0.06391954, + 0.040043183, + 0.09474887, + -0.037570562, + 0.17600152, + 0.19785954, + -0.09910671, + -0.14270158, + -0.0536535, + -0.047332756, + 0.18022339, + 0.0676487, + 0.13717946, + 0.027378082, + -0.10548278, + 0.066528544, + -0.19403522, + -0.11643189, + -0.08065489, + -0.03752704, + 0.033599664, + 0.15706381, + 0.013865536, + -0.08501725, + -0.020272262, + -0.018441383, + -0.03735178, + -0.016081626, + -0.10534383, + -0.17559953, + -0.0019625858, + 0.15813902, + -0.14031114, + 0.06402496, + 0.13181971, + 0.17281957, + -0.012040283, + 0.14102454, + -0.051492248, + 0.1004284, + -0.20263205, + 0.029880801, + 0.12380502, + -0.07871024, + 0.099203266, + 0.16148943, + -0.09668969, + 0.12815538, + 0.003585216, + -0.030985428, + -0.14598812, + 0.1132824, + 0.10472477, + -0.029968526, + -0.063032724, + 0.014652918, + -0.02877283, + 0.052562434, + -0.13108331, + 0.0002965604, + 0.11512386, + 0.09318654, + -0.10961809, + -0.0880418, + -0.09860604, + 0.07382556, + -0.1037337, + -0.038589653, + -0.09008774, + -0.01911863, + -0.14737788, + -0.009485854, + -0.047335766, + 0.016980678, + 0.084372506, + 0.052771546, + -0.044290844, + -0.028756851, + 0.2607931, + 0.08380605, + -0.064346865, + 0.06569313, + -0.041412737, + -0.096951924, + -0.12520541, + -0.014805983, + -0.24914601, + 0.086238, + -0.002006227, + 0.10598944, + 0.12087416, + 0.03508919, + -0.051487036, + -0.1256758, + -0.12843262, + -0.018472746, + 0.17818493, + -0.1824178, + 0.085565396, + -0.0069503593, + -0.06915642, + 0.078164116, + 0.22044943, + 0.095329665, + 0.09443477, + 0.009476721, + 0.12377651, + 0.09528795, + 0.112927206, + 0.1125022, + -0.09308766, + -0.010651261, + 0.091823, + 0.047559917, + 0.07709087, + -0.106162645, + -0.15692854, + 0.072903305, + 0.045788582, + 0.0375331, + 0.06844259, + -0.23813765, + 0.013259943, + -0.06379869, + 0.024160216, + 0.032886725, + -0.18673082, + 0.01760028, + 0.19031574, + -0.11544211, + 0.105799034, + -0.02177165, + -0.065577194, + -0.0861514, + 0.1342203, + 0.18299834, + 0.056894757, + -0.014077486, + 0.15178883, + -0.11575148, + 0.009013221, + 0.13157651, + -0.0039246664, + 0.108594656, + 0.015797623, + -0.13545142, + 0.0977668, + 0.046627443, + -0.14986962, + -0.100400895, + -0.06745776, + -0.02255909, + -0.116668254, + -0.09205319, + -0.042751074, + 0.02285744, + 0.05999009, + 0.105419904, + -0.2971868, + 0.15153843, + 0.013010975, + -0.058407262, + -0.07143607, + 0.32557562, + 0.055834014, + -0.10003547, + 0.14418238, + 0.124668226, + -0.03373238, + 0.03690689, + -0.008434, + -0.043183107, + 0.052016493, + -0.088068604, + -0.14013946, + 0.053377382, + 0.1353628, + -0.122651346, + 0.06593491, + -0.010691284, + 0.049608894, + -0.14469606, + -0.11539988, + 0.26940665, + -0.12557648, + -0.33065188, + -0.16942522, + -0.16879891, + -0.17476796, + -0.11808337, + 0.006112452, + -0.074556805, + 0.05563614, + -0.10752109, + 0.0045725466, + -0.100444764, + -0.19990551, + 0.1623679, + 0.070562355, + 0.17294201, + -0.21465039, + -0.15566683, + -0.22320299, + -0.040394336, + 0.096149504, + 0.12600791, + -0.071590565, + -0.14021218, + 0.07090477, + -0.05439689, + 0.23978162, + 0.19791496, + 0.14561944, + 0.09421892, + 0.03198014, + -0.019724434, + -0.17524067, + 0.11801276, + -0.034325983, + -0.12414058, + 0.09539344, + 0.04428776, + 0.1991088, + 0.031654123, + -0.04807285, + 0.04592151, + -0.07569178, + -0.031496607, + -0.19537716, + 0.060411256, + -0.051872216, + 0.090447225, + 0.08494578, + -0.11747185, + -0.028199665, + 0.1347343, + -0.17381737, + 0.19282167, + 0.08678283, + 0.034531005, + -0.043074735, + -0.118435755, + 0.021840902, + 0.08782734, + -0.028497253, + -0.09325137, + -0.037644356, + -0.10620848, + 0.20808154, + 0.059732296, + 0.033074785, + 0.009470372, + 0.104916535, + 0.066376306, + -0.010363118, + 0.05408072, + 0.02767956, + 0.12599848, + 0.016272616, + -0.10300154, + 0.06984461, + 0.093451455, + -0.3713836, + 0.004608827, + -0.06669234, + 0.1191322, + -0.029813163, + 0.06783095, + 0.053586982, + 0.018094638, + -0.074333526, + 0.19786923, + -0.019449225, + 0.02383677, + 0.004300856, + 0.10008391, + 0.22801672, + 0.12678932, + 0.11370807, + 0.16604814, + 0.088361025, + 0.05435657, + 0.027251301, + -0.0097401505, + -0.10017743, + -0.0076766806, + -0.07074346, + 0.023101224, + 0.045202877, + 0.075106, + 0.1932022, + 0.034920372, + 0.10798186, + -0.15182951, + 0.046124, + 0.08082057, + -0.011904392, + -0.03646725, + 0.06683557, + 0.10610709, + 0.015073168, + -0.027114002, + 0.0211207, + -0.14800651, + -0.16112563, + 0.042969227, + 0.05496639, + -0.10502971, + -0.10001824, + 0.13661686, + -0.07500956, + 0.06874437, + 0.0835452, + -0.10677995, + 0.11066851, + 0.14947426, + 0.05686178, + -0.045194548, + 0.080891475, + 0.01653106, + -0.03289933, + 0.013714271, + -0.12062449, + -0.10942656, + 0.0427117, + 0.11709176, + -0.025273317, + -0.051796433, + 0.0050489707, + -0.1257773, + -0.017385142, + 0.12721983, + -0.10977477, + -0.23171797, + -0.020706484, + -0.010048814, + 0.0075350627, + -0.0067544025, + -0.0440331, + 0.3260202, + -0.00035211392, + -0.039470658, + 0.08292158, + -0.093218766, + 0.09687666, + 0.014780582, + -0.08518228, + 0.080687046, + -0.013215859, + 0.010666921, + 0.043733794, + -0.08879815, + -0.23352464, + -0.19190405, + 0.21334182, + -0.20023906, + -0.04070634, + -0.16439508, + -0.0065315836, + 0.09033729, + -0.096911885, + -0.061966095, + -0.03798265, + -0.17271575, + -0.04042787, + 0.16066734, + 0.19571932, + -0.005219358, + -0.113288976, + -0.034409754, + -0.02542398, + 0.091005854, + 0.075512506, + -0.12603755, + 0.003550829, + 0.05454221, + -0.1700984, + 0.08618596, + 0.07662489, + -0.12044079, + 0.12438776, + 0.0038605034, + -0.008291778, + -0.02776508, + -0.012168801, + -0.0791805, + 0.05979986, + -0.05944031, + 0.16329224, + -0.32967898, + 0.08408068, + 0.013467381, + 0.056141563, + -0.039574765, + 0.01251687, + -0.333109, + -0.10213542, + -0.2410506, + 0.024430476, + 0.02588703, + -0.14144824, + 0.0058720787, + -0.0798089, + -0.017106313, + -0.0020736381, + -0.06813275, + 0.11289762, + -0.19290096, + 0.14954773, + -0.048712995, + 0.09767048, + 0.17014039, + 0.010858046, + -0.06397144, + 0.098602, + -0.026944358, + -0.06880838, + 0.09306078, + 0.21228854, + -0.052566614, + -0.16931634, + 0.1503215, + -0.04413507, + -0.050339848, + 0.070244834, + 0.28390154, + 0.04381359, + 0.13148732, + -0.09735818, + -0.04281441, + -0.0002529125, + -0.12293158, + 0.055222724, + -0.0026420944, + 0.014094278, + 0.055280052, + 0.07080589, + -0.06374293, + -0.21383622, + -0.07480419, + -0.06326204, + -0.050647177, + -0.12127862, + -0.088789135, + 0.07309716, + 0.02734736, + -0.023602324, + -0.11849017, + 0.0862599, + -0.100177184, + -0.20296523, + 0.056530718, + -0.04380203, + -0.07311151, + 0.17730682, + 0.12751189, + 0.059371542, + 0.15086243, + 0.14834693, + 0.05420559, + -0.005883509, + 0.037433855, + 0.03104376, + 0.017708058, + -0.11753717, + -0.092178345, + -0.13057369, + 0.079070516, + 0.13699917, + -0.023019383, + -0.16708986, + -0.100313716, + -0.11022881, + 0.033850983, + 0.17981607, + -0.05736247, + -0.009717411, + 0.06720382, + -0.09542911, + 0.021875495, + 0.13131075, + -0.10874953, + 0.0064090267, + 0.08441985, + -0.010587874, + 0.04712112, + -0.06715897, + -0.0367435, + 0.02208662, + 0.030317329, + 0.033193227, + -0.121694334, + 0.04879346, + -0.045048017, + -0.030299243, + -0.01951495, + -0.04660699, + -0.08691534, + 0.015517739, + 0.07925178, + 0.055090293, + -0.16836199, + -0.020131057, + -0.04979672, + -0.080895, + 0.2045888, + -0.035188764, + 0.04068433, + 0.14953531, + -0.21293955, + -0.18716255, + 0.038786113, + -0.06314729, + -0.13646829, + 0.4301075, + -0.054831143, + 0.11402921, + 0.0715126, + -0.015407559, + 0.021623263, + 0.08860939, + 0.04332614, + 0.0063768667, + -0.1433278, + 0.11752632, + -0.15465412, + -0.08700072, + -0.0845954, + -0.14791308, + 0.0016860976, + -0.09356121, + 0.06972774, + -0.14896198, + -0.0330842, + -0.044193294, + 0.057510998, + -0.1900679, + -0.10587522, + 0.056306608, + 0.11176626, + -0.030755477, + 0.052004084, + 0.10115147, + 0.009432866, + -0.16124472, + 0.19203115, + 0.072125755, + 0.19657056, + 0.000823032, + -0.05501959, + 0.25273466, + 0.037036214, + 0.0799104, + 0.015897552, + 0.05778206, + -0.017158741, + -0.068570286, + 0.03771934, + -0.07160096, + 0.12563969, + 0.09058254, + 0.19363588, + 0.21729888, + -0.038404334, + 0.007869045, + -0.11452937, + -0.061573993, + -0.09937147, + -0.092896424, + -0.022444807, + 0.084605426, + 0.042577196, + 0.091019176, + 0.0687685, + 0.11391002, + -0.023444116, + 0.056105986, + 0.0010575949, + 0.058704697, + 0.13065207, + 0.18417926, + -0.17100154, + -0.03459482, + -0.13954142, + 0.1697018, + -0.032380745, + 0.11538436, + -0.23546107, + 0.13714324, + 0.047174696, + 0.1799499, + -0.039026696, + 0.020465225, + 0.09494173, + 0.26512316, + -0.07517377, + -0.043898042, + -0.13932166, + 0.0056566624, + -0.0055164094, + -0.043773327, + -0.0541095, + -0.0054946276, + 0.0663163, + 0.0021626875, + 0.10122027, + 0.06443506, + 0.2846765, + -0.28031296, + 0.08613889, + 0.04016044, + -0.045890335, + -0.11568628, + 0.09650608, + -0.0396269, + -0.019264355, + -0.12604448, + 0.12022761, + -0.18100314, + 0.1449683, + -0.19298455, + 0.0029647453, + 0.01889965, + -0.0340142, + 0.02557537, + -0.053900447, + 0.102941334, + 0.040459722, + -0.11036412, + -0.05180269, + -0.021873292, + 0.055194244, + -0.18999355, + -0.053315975, + -0.07193985, + 0.13219415, + 0.03263697, + 0.07894841, + 0.027257426, + 0.17079633, + 0.33656633, + -0.09741153, + -0.0012406677, + -0.062253032, + 0.122720554, + -0.07428574, + 0.092065744, + -0.12949777, + -0.2637853, + -0.17236178, + 0.11274971, + -0.0036404934, + -0.067255475, + -0.01703726, + -0.024414714, + -0.08555217, + 0.11511529, + -0.10132069, + 0.035678104, + -0.03773041, + -0.11198654, + 0.09750184, + 0.11933404, + -0.017337881, + -0.0275599, + -0.056142204, + 0.2870939, + 0.025516802, + 0.089016005, + 0.042186953, + 0.061288327, + 0.23220468, + 0.1913539, + 0.041872956, + -0.029348254, + 0.2022436, + 0.24744327, + 0.00085464993, + -0.06835739, + 0.21693708, + 0.03833265, + -0.11809685, + -0.094088, + -0.08797381, + 0.03583436, + -0.03725489, + -0.12992965, + -0.046520386, + 0.08877341, + -0.23196267, + -0.1416181, + -0.14281443, + 0.0051557235, + -0.083149076, + -0.030370584, + 0.028675005, + -0.069152854, + 0.022219311, + -0.24649718, + 0.16162385, + 0.13550395, + 0.24505398, + 0.11737749, + -0.26714596, + 0.039750718, + 0.036444116, + 0.19190054, + 0.25507984, + -0.05611311, + -0.03598602, + -0.00091825356, + 0.035975944, + 0.005179897, + -0.05993038, + 0.04453667, + -0.06822254, + -0.0034093673, + 0.041618973, + 0.009030745, + -0.16429444, + -0.10991389, + -0.21974932, + 0.05839352, + -0.12792662, + -0.17765494, + 0.06688134, + 0.16218579, + -0.13164958, + -0.059599277, + 0.08895215, + 0.102052465, + -0.045741614, + 0.15715373, + -0.26308244, + 0.051839665, + 0.16174427, + -0.13277116, + -0.13938846, + -0.082957774, + -0.03518575, + -0.16200525, + -0.1170968, + 0.090260565, + -0.10743151, + -0.20015334, + 0.19283876, + 0.02739746, + -0.0678554, + -0.0289476, + -0.024540577, + -0.15412445, + 0.028864484, + -0.033722255, + 0.037424758, + 0.04879276, + 0.048878994, + 0.07719014, + -0.027368383, + 0.1159599, + -0.097892486, + 0.015980033, + -0.0061608437, + -0.035205483, + -0.20321424, + -0.08789502, + 0.016100816, + 0.017145542, + -0.17215213, + 0.17112683, + -0.002088437, + 0.13785838, + 0.047853924, + 0.23892085, + 0.16270262, + 0.030570636, + 0.1013973, + -0.060039762, + -0.09836197, + -0.111120716, + 0.12632947, + -0.08306903, + -0.12863602, + 0.047026373, + -0.1958446, + 0.0044924226, + 0.1719895, + 0.10121101, + 0.17643425, + -0.046456266, + 0.17635134, + 0.01827563, + 0.08540644, + 0.06120104, + -0.113148265, + -0.1218303, + 0.011219538, + -0.12710062, + -0.050977923, + 0.053127747, + -0.03822012, + -0.00858817, + 0.014970831, + -0.12115409, + 0.008089462, + 0.068597436, + 0.08169945, + -0.10300656, + 0.05761663, + -0.0078221625, + -0.06458102, + 0.020226378, + -0.005276232, + 0.0895599, + 0.08770444, + -0.19067658, + -0.044448506, + -0.13667849, + -0.045721535, + -0.017611813, + -0.17994289, + 0.061323702, + -0.006091955, + 0.037093177, + 0.051335093, + 0.048674185, + 0.10398149, + -0.023160957, + -0.035153363, + 0.12117348, + 0.038042266, + -0.074533135, + 0.014664317, + 0.027021443, + 0.0034762379, + -0.06064791, + -0.015092715, + -0.2944957, + 0.10236635, + 0.15401675, + 0.15111294, + 0.1340751, + -0.17079759, + 0.06980032, + -0.07446118, + -0.075741716, + 0.045889974, + -0.027051862, + 0.061429117, + 0.13151085, + 0.2020734, + -0.05170606, + -0.019183844, + -0.029455291, + -0.16436203, + 0.065099075, + 0.114943504, + 0.07208407, + -0.052109897, + 0.058475323, + -0.0060676374, + 0.108448036, + 0.13429314, + 0.07835998, + -0.042943794, + 0.07505265, + -0.15782769, + -0.0008275639, + -0.022941407, + -0.089207046, + -0.053613666, + -0.22768244, + -0.004562265, + 0.119613804, + -0.06730513, + -0.005134274, + 0.015889524, + 0.065859795, + -0.13738343, + 0.04806415, + -0.021074295, + 0.02389348, + -0.04589438, + -0.00791156, + 0.19126686, + 0.066830024, + 0.038546782, + 0.056414336, + -0.09672572, + -0.1488234, + -0.1606585, + 0.03534793, + -0.009993679, + 0.0033058047, + 0.052286904, + -0.09087651, + 0.015177222, + -0.1209879, + 0.117720395, + -0.17759861, + -0.045116693, + 0.3353583, + -0.17735094, + -0.255285, + -0.20740092, + -0.018373836, + 0.0005954378, + -0.011089623, + -0.21794018, + 0.14621736, + 0.017658569, + -0.07371307, + 0.021241616, + -0.12774628, + 0.12024284, + -0.069704995, + -0.08754762, + 0.052305937, + 0.13504027, + -0.0111032175, + 0.0896511, + -0.027011193, + 0.0964028, + -0.04770052, + -0.05891503, + -0.29946882, + 0.06799919, + 0.019578407, + -0.09722951, + 0.040762562, + -0.039349716, + -0.04986413, + 0.008755783, + -0.035490353, + 0.28766772, + -0.019644262, + -0.14135677, + -0.08013137, + 0.06394487, + 0.17467594, + -0.10952025, + -0.067189634, + -0.05346993, + 0.13319, + -0.02134673, + 0.08383482, + -0.005200357, + -0.013754012, + -0.020868568, + 0.10664089, + 0.266166, + 0.0021528348, + 0.17895368, + -0.0036033643, + 0.007758813, + 0.0316086, + -0.008563442, + 0.008860687, + -0.10833197, + 0.063689224, + -0.09778756, + -0.25244159, + 0.25641277, + 0.07796288, + -0.0037101684, + 0.04819473, + -0.08170173, + 0.015830114, + -0.028049998, + 0.02885121, + -0.028635604, + -0.05007718, + 0.112869985, + -0.060543325, + -0.05746815, + -0.14462683, + 0.1255916, + 0.121556595, + 0.102449775, + 0.120668866, + 0.0036547463, + 0.13570864, + 0.06869708, + 0.19977914, + -0.14529352, + 0.16520394, + 0.019234715, + 0.021917906, + -0.19478388, + -0.07423999, + -0.13678935, + -0.13091457, + 0.10972841, + -0.16713484, + -0.10109219, + -0.13254036, + -0.007902096, + 0.06695212, + -0.051268183, + -0.03536522, + 0.09367453 + ], + [ + 0.11637956, + -0.17186578, + 0.10791955, + 0.080459945, + 0.05975105, + -0.011272944, + 0.10946989, + 0.22342005, + 0.099497564, + 0.04502466, + -0.014406636, + 0.10069611, + 0.024731083, + -0.031065537, + -0.058351807, + 0.06504147, + 0.15619484, + -0.06604318, + 0.1781333, + 0.22792023, + -0.13898799, + -0.045416683, + 0.031234754, + 0.037138533, + -0.18047316, + -0.004940352, + -0.056690544, + 0.04373834, + -0.1627064, + -0.10905386, + -0.040010177, + 0.3229176, + 0.07230417, + 0.07616515, + -0.028685225, + -0.055634305, + -0.03288254, + -0.004032664, + 0.092190936, + -0.35969818, + -0.06260908, + 0.13865389, + -0.026160512, + 0.06181806, + 0.10938138, + -0.121704094, + 0.19553061, + -0.06748917, + -0.06838561, + 0.10467503, + -0.070798166, + -0.0747031, + -0.074544385, + 0.099389784, + 0.08020828, + -0.14345938, + 0.058599636, + -0.08092364, + 0.15528879, + 0.053775847, + 0.05522898, + -0.17603008, + 0.08225849, + -0.072476104, + 0.19657768, + -0.18427952, + -0.054881528, + 0.005319496, + -0.108707465, + -0.18644795, + -0.051599722, + -0.26696226, + 0.087243766, + 0.027708074, + 0.043569595, + 0.0017946062, + 0.061552893, + -0.0073106713, + 0.13140735, + -0.086496115, + 0.035292115, + 0.04865788, + 0.27236786, + 0.064673945, + -0.07814582, + -0.10368492, + 0.021018954, + 0.007109233, + 0.16142946, + -0.32392415, + 0.11618061, + 0.16055097, + -0.10717878, + -0.027265606, + -0.052559048, + 0.06769613, + 0.109514505, + -0.051763024, + -0.05986832, + 0.054277677, + 0.020888394, + 0.09378125, + -0.101614766, + -0.114495024, + 0.19143566, + -0.13538244, + -0.030124659, + 0.029111356, + -0.0665248, + -0.019361828, + 0.03687205, + -0.14246751, + 0.059040654, + -0.04202304, + -0.22132228, + 0.00042482893, + 0.06615944, + 0.16350724, + 0.29969427, + 0.15013222, + -0.057181604, + 0.045494482, + 0.09992527, + 0.10001293, + 0.07406286, + -0.02437011, + -0.0807268, + -0.2998727, + -0.17070091, + -0.13470376, + -0.07132043, + -0.067651786, + 0.00096411596, + 0.009365229, + -0.0010018462, + -0.02374255, + -0.022331577, + 0.008360721, + -0.24549751, + 0.09109612, + 0.07186384, + -0.21236059, + -0.004149225, + 0.029868592, + -0.16807589, + 0.38404953, + -0.07270962, + 0.06128629, + 0.0029842712, + -0.08691458, + -0.022859711, + -0.15697275, + -0.110039994, + 0.07708965, + 0.16576324, + -0.14948945, + 0.15736353, + 0.23472643, + -0.004841239, + 0.005375038, + -0.104134165, + 0.29700267, + -0.15814611, + 0.018846232, + 0.02098197, + -0.020519339, + 0.022707358, + 0.1988144, + -0.11999712, + -0.02079705, + -0.058023993, + 0.0058619855, + 0.11121759, + 0.017506061, + 0.05126044, + -0.096916825, + 0.2307071, + -0.062108804, + 0.123407386, + 0.0031229893, + -0.044057585, + -0.096670195, + 0.21410017, + -0.041165788, + 0.035975046, + -0.15833475, + 0.12947544, + -0.3752908, + -0.03691883, + 0.095158406, + 0.032910068, + -0.12258116, + 0.035828426, + 0.06966969, + 0.016244017, + 0.170389, + -0.1261006, + 0.1098775, + 0.13936415, + -0.04809021, + 0.19010004, + 0.12529476, + -0.11426704, + -0.023094462, + -0.23561454, + 0.14241934, + -0.07336394, + -0.3049396, + 0.07135916, + 0.29270256, + 0.042280797, + 0.13791633, + 0.14932086, + -0.03536875, + 0.09843503, + -0.18766549, + -0.046467368, + 0.14914943, + -0.023084715, + 0.09644197, + -0.003209695, + -0.16610551, + -0.14070651, + -0.37365547, + 0.10216226, + 0.21309158, + 0.08469838, + -0.14306426, + -0.091675095, + -0.10465955, + 0.027742147, + 0.16314667, + 0.03690933, + -0.03567327, + -0.26394844, + -0.19670296, + 0.102186255, + -0.16789915, + 0.016203193, + -0.06374825, + 0.05287729, + -0.09745945, + 0.00722403, + 0.07676293, + 0.014780153, + -0.10793076, + -0.1901627, + -0.18417664, + 0.04534059, + -0.0716811, + 0.07866812, + 0.1439692, + 0.06014963, + -0.21927246, + 0.20979516, + -0.015994566, + 0.12689663, + 0.041828584, + 0.08181566, + -0.063342, + 0.12462212, + -0.10833612, + -0.040642224, + -0.062786385, + 0.021669455, + 0.02062207, + -0.15943547, + -0.14038363, + 0.04656669, + 0.22162975, + -0.03973174, + 0.07510139, + 0.06834014, + 0.0720175, + 0.121220686, + 0.008868144, + -0.13243613, + -0.050706953, + 0.0605855, + -0.07571807, + 0.0075447657, + -0.16944724, + 0.090943016, + 0.017109644, + -0.064935304, + -0.06073986, + 0.063119814, + -0.12180592, + -0.057504665, + 0.12659715, + 0.07531627, + 0.056290645, + 0.018820873, + 0.11000129, + -0.14517342, + -0.31130013, + -0.1983607, + 0.08439539, + -0.123937875, + -0.03123252, + -0.2514824, + -0.11085155, + 0.005044094, + -0.14976013, + -0.057709478, + 0.070679195, + 0.14638405, + -0.18428764, + -0.004577368, + 0.028631024, + 0.05115411, + 0.0073806057, + -0.051902503, + -0.021504115, + 0.0070884735, + 0.07090578, + 0.092421785, + 0.15728186, + 0.009211559, + -0.4120053, + -0.023852421, + 0.08603379, + -0.14602195, + -0.052523993, + 0.083968624, + 0.12750289, + -0.071089976, + 0.24766031, + 0.099861264, + -0.036790013, + 0.16011785, + -0.011961857, + -0.16865316, + 0.2080289, + 0.11488308, + -0.05541248, + -0.06588284, + -0.07385138, + -0.21823715, + -0.027737396, + 0.0028189542, + 0.023575094, + -0.024090767, + 0.0011296587, + -0.11189924, + 0.005202209, + 0.07130078, + 0.014037095, + 0.047963694, + -0.06790311, + 0.11792759, + 0.088360295, + -0.007919029, + 0.066773124, + -0.04210118, + -0.26693234, + -0.0719914, + 0.24812208, + 0.2582869, + -0.01645684, + 0.09704079, + -0.06148363, + 0.0888649, + -0.26011285, + 0.013716547, + 0.025316631, + -0.12539926, + -0.01655872, + 0.018356794, + 0.09316022, + 0.2148001, + -0.42118096, + 0.09882388, + 0.0054554185, + 0.008558932, + 0.06381473, + 0.26438358, + 0.058518853, + -0.15347068, + 0.16173801, + -0.33100104, + 0.030894315, + 0.027564311, + -0.016866116, + 0.032837298, + 0.00077169156, + 0.03133184, + 0.06765387, + -0.049206775, + -0.016165834, + -0.212784, + 0.12581074, + 0.20880195, + -0.06285257, + 0.049284827, + -0.070513986, + -0.18638465, + 0.18946855, + 0.078632794, + -0.013579502, + 0.25505856, + -0.2065767, + 0.10019049, + 0.11834874, + -0.019598717, + -0.08845627, + -0.11897016, + -0.017256223, + 0.017200148, + 0.16272031, + 0.041610334, + 0.0394703, + -0.026481334, + 0.09240797, + 0.030089354, + -0.1735567, + -0.051628623, + -0.03471855, + 0.19301184, + 0.116581686, + -0.12833858, + 0.09992816, + -0.025773646, + 0.21380223, + 0.38609263, + -0.09755428, + 0.17755441, + 0.0037833347, + -0.08545212, + 0.09529221, + -0.21268612, + 0.00090696284, + -0.15830845, + -0.11832876, + 0.009103947, + 0.020926477, + 0.0012178889, + 0.19595702, + -0.040166292, + 0.050838795, + -0.16591446, + 0.015815735, + -0.28945345, + 0.29379824, + -0.042782035, + -0.17004989, + 0.19729897, + 0.0055258335, + -0.03620126, + -0.19522566, + -0.121389605, + -0.00032941398, + -0.17998022, + 0.06779911, + 0.2631502, + 0.13141875, + -0.04985646, + 0.052451488, + 0.24368726, + 0.07297032, + -0.053266484, + -0.019812576, + 0.17073107, + 0.11443813, + -0.034046765, + -0.045550775, + -0.034369446, + 0.07116018, + 0.035985112, + 0.10965749, + -0.005791836, + 0.02731971, + -0.07340144, + 0.10565047, + 0.11125925, + 0.05740682, + -0.17137615, + 0.045366608, + 0.029304776, + -0.13445312, + -0.039387558, + 0.08226571, + -0.028178785, + -0.016610812, + 0.0103821615, + -0.23829375, + -0.24541384, + -0.008268759, + 0.07972249, + 0.16931161, + -0.045478042, + 0.03659767, + -0.09050166, + -0.021094369, + -0.032404713, + 0.1336974, + 0.0533087, + 0.035258383, + -0.1752104, + 0.23809828, + 0.08637835, + -0.20440699, + -0.16151933, + 0.013910408, + -0.22518755, + 0.23404773, + -0.06430676, + -0.2438961, + 0.054066826, + -0.01222482, + 0.04206431, + 0.10275902, + 0.1509213, + -0.3044058, + -0.024547627, + -0.08197543, + 0.12862152, + 0.04344292, + -0.013134288, + -0.033549696, + -0.05573171, + 0.14470781, + -0.29964295, + -0.051656418, + -0.043963168, + -0.0626571, + 0.054504357, + -0.07732341, + 0.12172327, + 0.04756539, + -0.057771176, + -0.18838337, + -0.13257566, + -0.03424755, + 0.27295038, + 0.0039459704, + 0.08678922, + 0.09803905, + -0.5035922, + 0.0133419875, + 0.00065672013, + -0.02211246, + 0.04287359, + -0.012378799, + -0.07109952, + -0.13045481, + 0.016000884, + 0.12478969, + -0.16040824, + -0.22890955, + -0.00045370215, + 0.15292962, + 0.090407826, + 0.088943146, + 0.036541674, + 0.06473803, + 0.067636736, + 0.033031188, + 0.1036692, + 0.067575134, + 0.21188134, + 0.1192539, + -0.08119708, + -0.13598263, + 0.10788264, + -0.05933999, + -0.18349162, + 0.1735971, + 0.19631074, + -0.18473329, + -0.12218243, + 0.032747492, + -0.097329825, + 0.06238207, + 0.035778794, + 0.07071465, + 0.15152672, + 0.070239685, + -0.10766326, + 0.050400086, + 0.050804477, + -0.011483971, + -0.121493325, + -0.15133147, + -0.03250726, + 0.04693144, + 0.0005621336, + 0.236917, + -0.054525223, + 0.21746288, + 0.1697138, + 0.31465507, + -0.062002458, + -0.02546646, + -0.28721526, + -0.2513955, + 0.010789273, + 0.018919708, + -0.25192356, + 0.018824613, + 0.118578434, + -0.00507896, + 0.014482422, + 0.11267149, + 0.14215098, + 0.11573328, + 0.04761225, + 0.08724168, + -0.29671982, + 0.028040556, + -0.17270142, + 0.021092054, + -0.20256923, + -0.11460052, + -0.06707171, + 0.05905861, + 0.025438156, + 0.123720355, + -0.14976059, + 0.042247787, + 0.08170481, + 0.02156635, + 0.075605355, + -0.03911428, + 0.039365444, + -0.05059143, + -0.101301275, + -0.037720732, + -0.026390677, + -0.0656627, + -0.20962778, + -0.42560935, + 0.020281976, + -0.09318103, + -0.12504372, + 0.17505816, + 0.23766881, + 0.14905949, + -0.116007335, + -0.06740984, + 0.113025166, + -0.029503759, + 0.30562577, + -0.11261386, + -0.014164033, + -0.004014467, + 0.120359935, + 0.00029712665, + -0.01933728, + -0.0020288113, + -0.035912253, + -0.0998972, + 0.10274502, + -0.003980527, + 0.010540119, + -0.0012870778, + 0.03024298, + -0.019371746, + -0.08722253, + -0.30856013, + -0.024190273, + -0.09390957, + -0.11412507, + -0.3192844, + -0.11418165, + 0.0010766103, + 0.056867734, + -0.06406079, + -0.15870427, + -0.15309812, + 0.021257922, + -0.08891713, + 0.4405356, + 0.024239432, + 0.0012664798, + 0.035958398, + 0.2549329, + 0.098144576, + 0.16194685, + -0.083581924, + -0.12655054, + 0.21620502, + -0.046908326, + 0.12320124, + 0.042971227, + 0.007110838, + -0.048693836, + 0.21834254, + 0.065210894, + -0.029152373, + -0.15173002, + 0.00884002, + -0.14687325, + -0.0051244213, + -0.08612436, + 0.017426912, + -0.12191853, + 0.057925828, + -0.064879015, + -0.25661778, + -0.13962127, + 0.012649742, + -0.17353316, + -0.08397678, + -0.076075956, + 0.12900859, + 0.17618789, + 0.015170843, + -0.0966266, + 0.014381747, + 0.070817225, + 0.14590451, + -0.012652974, + 0.1212396, + 0.1928499, + 0.034840625, + -0.107425846, + 0.13786417, + -0.07710894, + -0.038374882, + 0.37778434, + 0.022085752, + -0.12701413, + 0.03516333, + -0.27293193, + 0.15730625, + 0.15843415, + -0.10312135, + -0.014996361, + -0.159087, + -0.04362026, + -0.18499237, + -0.02748742, + 0.032441035, + -0.020762652, + 0.08412549, + 0.021805234, + 0.00660146, + 0.061379142, + 0.0821473, + -0.034735933, + -0.21985133, + -0.07475556, + 0.44629174, + -0.03631395, + 0.11672802, + 0.049370393, + -0.041611828, + 0.22077566, + -0.014968166, + -0.19998384, + 0.011433027, + 0.06101584, + -0.11258323, + 0.13738437, + -0.0688601, + -0.21262124, + 0.102296345, + -0.061489206, + 0.057119362, + 0.23424532, + 0.120877706, + -0.047151435, + 0.2176173, + 0.20899433, + -0.00081871107, + -0.013194566, + 0.05514477, + 0.3928346, + -0.12549223, + -0.11095625, + 0.012750077, + -0.09158463, + -0.019312715, + -0.04007366, + 0.16496634, + 0.15461186, + -0.0877878, + -0.1798766, + -0.016334271, + 0.120540105, + 0.061884813, + 0.014563505, + 0.10252879, + -0.047985137, + -0.055586945, + 0.22566938, + 0.1058749, + 0.004186561, + 0.10453594, + 0.034418534, + 0.2573554, + -0.19616708, + 0.049920246, + -0.2728407, + 0.010139094, + -0.15609851, + -0.2773467, + 0.08028425, + -0.21471062, + -0.0010262715, + -0.02221235, + 0.27420497, + -0.12729512, + 0.12268265, + -0.036199618, + -0.22428453, + 0.17106162, + 0.016466241, + 0.13175893, + -0.13125266, + 0.16843618, + 0.030138072, + -0.18637407, + -0.17876884, + -0.025821261, + -0.022291679, + -0.032874994, + 0.118431464, + -0.112453796, + 0.025207195, + 0.049616717, + 0.042266954, + -0.13243802, + -0.2631501, + 0.033834856, + 0.0038898303, + 0.21306744, + -0.05530253, + 0.119763054, + -0.027107682, + 0.021657147, + 0.071491025, + -0.14394669, + 0.098971434, + -0.104662135, + 0.043255832, + -0.11081497, + 0.1253336, + 0.07292208, + 0.105347626, + -0.1376309, + 0.14321448, + 0.12819311, + -0.16884637, + -0.020582603, + 0.07983813, + -0.10436447, + -0.19395255, + -0.14573762, + -0.05208073, + 0.052003495, + -0.03613725, + -0.18182264, + 0.22837463, + -0.08308755, + -0.09978127, + -0.036853623, + 0.13789463, + -0.056370385, + 0.29830414, + -0.0791662, + 0.06277126, + -0.092597716, + 0.076572105, + -0.048261747, + -0.114597894, + 0.16213843, + 0.1433155, + 0.035506878, + -0.033455156, + 0.19248319, + -0.07745398, + -0.19590208, + 0.3018688, + 0.03310655, + 0.0041582887, + 0.19586419, + -0.10853914, + -0.16483396, + -0.17642425, + 0.19710363, + -0.06962616, + 0.17102459, + 0.10896041, + -0.13901196, + -0.12664439, + 0.09078699, + -0.12244493, + 0.052821994, + -0.030397188, + -0.13134342, + 0.053447146, + -0.18429838, + 0.26444325, + 0.22305855, + 0.029930143, + -0.054558307, + 0.0353794, + -0.0009788725, + -0.1188078, + 0.059667766, + -0.27633518, + 0.046799272, + -0.15642197, + -0.10854975, + 0.106093995, + -0.22275078, + 0.038943406, + -0.03847844, + 0.052943595, + 0.02457092, + -0.02215091, + 0.041665964, + -0.09696136, + -0.02566959, + -0.039870802, + -0.054774225, + 0.02291138, + -0.0417671, + 0.18173029, + -0.03932743, + 0.08204516, + 0.0959776, + -0.20656648, + 0.28138176, + 0.037579037, + 0.20550737, + -0.0048693623, + 0.08957929, + 0.07936084, + 0.064060465, + -0.076720774, + -0.04245319, + 0.1251032, + -0.048257716, + 0.14970464, + -0.019789496, + -0.12023016, + 0.31614852, + 0.048109207, + 0.018149873, + 0.005182097, + -0.051101666, + -0.118794665, + 0.028441736, + 0.024802344, + 0.16004704, + 0.21352114, + -0.0831253, + -0.0066282563, + 0.057000577, + 0.057913885, + 0.03354726, + -0.076870695, + -0.20632678, + 0.02378624, + -0.11414494, + -0.28388187, + -0.24170417, + 0.026353586, + 0.01276097, + -0.20049897, + 0.16650756, + 0.0765534, + -0.023719413, + 0.02053945, + 0.31372947, + 0.16620825, + -0.019555673, + -0.02928945, + -0.060654677, + -0.034009326, + -0.13074087, + -0.086977184, + -0.11527837, + -0.01426738, + 0.019683821, + -0.16352665, + 0.26811838, + 0.20973662, + 0.113176554, + -0.30779162, + 0.04542188, + 0.11580485, + -0.11855776, + -0.20264056, + -0.22960435, + 0.009477502, + 0.1754439, + -0.03466805, + 0.08613063, + -0.009424178, + -0.104562424, + 0.03825546, + -0.20300105, + 0.12507758, + 0.12184145, + 0.140918, + -0.21595697, + -0.1149452, + -0.0863683, + 0.011771966, + -0.0794508, + -0.16730252, + 0.083861485, + -0.073139645, + -0.03015001, + -0.2253941, + -0.2636101, + -0.111474104, + -0.012658355, + 0.12943146, + 0.0024419622, + -0.008086202, + -0.031769402, + 0.22267817, + 0.09031554, + 0.084421515, + 0.07658035, + 0.09284541, + 0.0123357065, + 0.13187145, + -0.021085177 + ], + [ + 0.044147793, + 0.16940491, + -0.07660075, + 0.01839263, + 0.042040065, + -0.047721248, + -0.21912621, + -0.0009365094, + 0.06492754, + 0.042232044, + -0.004531145, + 0.11653948, + -0.006253255, + -0.020997126, + -0.086448684, + 0.065257326, + -0.09350857, + 0.05722746, + -0.09045094, + -0.20966446, + -0.020684095, + 0.05035577, + -0.0058316975, + -0.08649621, + 0.11655366, + -0.035102855, + -0.03805832, + -0.00616778, + -0.013295165, + -0.088805586, + 0.13911676, + -0.1802159, + -0.018650677, + -0.028832428, + -0.15153827, + -0.012014452, + -0.10429693, + -0.02334544, + 0.013291807, + 0.2931698, + 0.25664425, + -0.022222053, + 0.10681528, + -0.02429105, + -0.14298108, + -0.03535962, + -0.20939623, + 0.15250647, + 0.0886579, + -0.08623361, + 0.10814994, + 0.0907847, + 0.17988828, + 0.07620976, + -0.07824574, + -0.014679582, + 0.0817115, + 0.016898714, + -0.28710848, + -0.22659548, + -0.0431887, + 0.035005502, + -0.030766238, + -0.027952312, + 0.00605425, + 0.15439883, + -0.068923496, + 0.0733908, + 0.009504215, + 0.024958564, + 0.14828947, + 0.13880697, + -0.090828896, + 0.003277354, + 0.10241844, + -0.115077004, + -0.1662777, + 0.03727921, + -0.06837462, + 0.07184318, + -0.006519263, + -0.10122025, + -0.14128436, + -0.064970486, + -0.036978073, + 0.010498385, + 0.08859915, + 0.08093246, + -0.06282354, + 0.25009853, + -0.012446843, + -0.121961325, + 0.1972666, + 0.046384234, + 0.199937, + -0.058210306, + -0.062178757, + 0.034782343, + -0.024504201, + -0.10704923, + 0.023402458, + -0.06502446, + -0.15917829, + 0.030689063, + -0.1270888, + 0.06968928, + 0.071537375, + 0.067840576, + 0.19173016, + 0.034167826, + 0.21227396, + 0.05622944, + -0.05703442, + -0.06396645, + 0.100448065, + -0.035514005, + -0.014672414, + -0.037831403, + -0.17126168, + -0.13165945, + -0.12100334, + 0.13692331, + -0.18549083, + -0.09306254, + -0.0049064197, + -0.05379398, + -0.13972266, + 0.20454313, + 0.076266214, + 0.12522706, + -0.05245607, + -0.027636189, + -0.113891274, + -0.12186737, + 0.094089486, + 0.03439381, + -0.06949138, + -0.055920634, + 0.16840659, + 0.0150665045, + 0.08506466, + 0.13945717, + -0.041639358, + -0.06740168, + 0.09963331, + -0.14591189, + 0.059449732, + 0.0025124077, + -0.027144495, + 0.05402788, + 0.20959063, + 0.13937241, + -0.080275685, + 0.038352527, + -0.2715623, + 0.17126106, + -0.091786325, + -0.14857501, + -0.1293791, + -0.18837348, + 0.047239408, + -0.28292495, + 0.0063572284, + 0.09690516, + -0.029995194, + -0.11105718, + -0.018782707, + -0.30740905, + 0.104199484, + 0.15624839, + -0.03974281, + -0.05248947, + 0.038652018, + 0.08289482, + 0.016197314, + 0.119475886, + -0.11403887, + 0.15416206, + -0.08065634, + -0.025980428, + -0.015932493, + -0.008749679, + 0.08308672, + -0.11037272, + -0.048986003, + 0.21674192, + -0.05803945, + 0.04971493, + -0.01891516, + 0.004877139, + -0.17709245, + -0.002087021, + -0.0020960409, + -0.106576554, + -0.007810016, + -0.12720583, + 0.07408409, + -0.02180893, + 0.0007753394, + -0.005287165, + -0.3254629, + -0.0026433906, + 0.04833213, + 0.033785746, + 0.18600549, + 0.0022767857, + 0.18876381, + 0.035533134, + 0.054217387, + 0.03794934, + 0.1271445, + 0.030882537, + 0.025446987, + 0.15345229, + -0.10454748, + 0.26222214, + -0.0091688065, + -0.041628335, + 0.018512165, + 0.0040028314, + 0.20311502, + 0.0037376555, + 0.070144124, + 0.20071377, + 0.11248816, + -0.057424825, + 0.13850467, + 0.18345869, + -0.0044744764, + -0.021348322, + 0.04384846, + -0.022934612, + -0.107814014, + 0.09007019, + 0.02416682, + -0.0012119029, + -0.24780564, + 0.073680155, + -0.04818337, + 0.083472714, + 0.12236319, + -0.02055338, + 0.027101975, + 0.04737765, + -0.110173546, + 0.06364295, + -0.008946159, + 0.15252261, + 0.0027322657, + 0.025759516, + -0.0029763042, + -0.11247266, + 0.13522747, + 0.15886107, + -0.15792294, + -0.07445264, + -0.21184248, + 0.07564331, + -0.053616013, + -0.07139231, + 0.049195178, + -0.084485546, + -0.046140656, + 0.02825539, + 0.021405263, + 0.097813755, + 0.13759468, + 0.0525562, + -0.018069427, + -0.12837838, + 0.077376045, + 0.031107048, + -0.27642173, + -0.13174976, + -0.15429558, + -0.01833851, + 0.027519647, + -0.015669333, + -0.050222356, + 0.02163741, + -0.035224333, + 0.04344882, + -0.10721566, + 0.08589194, + -0.004909294, + -0.032711554, + 0.30826396, + 0.117197126, + 0.124197006, + -0.24572933, + -0.04550309, + -0.12412157, + -0.0724078, + -0.128096, + 0.219507, + 0.1134307, + 0.21780996, + -0.10823206, + 0.11963701, + -0.068851374, + 0.023465648, + -0.01593774, + -0.118752144, + -0.016288111, + -0.030651504, + -0.12503572, + -0.17363529, + 0.19402781, + 0.10475469, + -0.020954344, + 0.019589314, + -0.030481823, + 0.0066996072, + -0.053601895, + -0.20029072, + -0.10582614, + -0.20040359, + -0.0054524555, + -0.055335585, + 0.3311847, + 0.035756826, + -0.049566645, + 0.07918633, + -0.053583127, + -0.09904188, + -0.10038896, + 0.04996928, + -0.099653706, + 0.061264403, + -0.0061791865, + -0.21508417, + 0.116991624, + 0.2686714, + -0.34464577, + -0.039873548, + -0.013331849, + -0.017662382, + 0.18063131, + 0.107568875, + -0.12173686, + -0.059680745, + 0.02161945, + -0.05680074, + -0.017660705, + 0.14479858, + -0.018916478, + 0.049323782, + 0.09538939, + -0.09067543, + -0.049188662, + -0.09265428, + -0.0365639, + 0.0028700577, + 0.059004173, + 0.05948634, + 0.13971259, + 0.18176614, + -0.016404018, + -0.2375804, + 0.02650564, + -0.10457582, + 0.06823804, + -0.044831805, + -0.06590748, + -0.013364427, + 0.014154022, + 0.042477664, + 0.1097775, + -0.11523345, + -0.1079408, + -0.12961785, + 0.34049404, + -0.08560808, + -0.016122337, + -0.052292738, + 0.024983414, + -0.030858947, + 0.13338524, + -0.059871174, + 0.038500845, + 0.37170732, + 0.13350071, + -0.021032728, + -0.07347119, + 0.06407454, + 0.06119444, + 0.0066508185, + 0.10506183, + 0.08963466, + -0.1445016, + 0.01706449, + -0.12059136, + -0.09551309, + 0.09726227, + -0.023860846, + -0.020491932, + 0.11087211, + -0.063431025, + -0.082183585, + -0.0409627, + -0.08496014, + 0.120390706, + -0.17681535, + 0.002092146, + -0.10478897, + 0.08459577, + 0.12726188, + 0.045021284, + -0.005031372, + -0.08353973, + -0.101410136, + 0.019969966, + -0.13681094, + 0.23727098, + -0.11417003, + 0.16008934, + -0.0045129163, + 0.074293375, + -0.20552878, + 0.21652737, + 0.23047388, + 0.14112242, + 0.0013431809, + -0.23968937, + -0.24464427, + 0.091682285, + -0.09774535, + 0.013322989, + 0.08752573, + -0.027159441, + 0.09978838, + 0.19199398, + 0.008760841, + 0.16704173, + -0.10677443, + -0.19106703, + -0.012075937, + -0.13198557, + -0.058435682, + -0.02389443, + 0.23472294, + -0.1088765, + 0.077165045, + -0.24123153, + 0.21209829, + 0.019728398, + -0.15316392, + 0.044813976, + -0.03404352, + -0.08867594, + 0.077576004, + -0.13115795, + 0.27733845, + -0.024984693, + -0.2628973, + -0.008487113, + -0.0053662844, + -0.049809393, + -0.2577814, + -0.12825035, + -0.017539369, + 0.08355548, + 0.04310515, + -0.0396339, + 0.09730884, + 0.09619792, + 0.15564808, + 0.017628962, + -0.1090823, + -0.13700478, + 0.029394176, + 0.091170415, + -0.012858434, + -0.0054733506, + 0.09170598, + -0.113937475, + 0.21517824, + 0.027744953, + -0.20661183, + 0.006941276, + -0.019983957, + -0.23312838, + -0.120168075, + -0.037594784, + -0.004498651, + 0.20085981, + 0.21437016, + -0.009439298, + 0.037814677, + -0.077133186, + 0.17605172, + -0.115668155, + -0.046497446, + 0.044113774, + 0.19949457, + -0.03338365, + 0.05692018, + -0.06910936, + -0.0046057226, + -0.18073574, + -0.07666095, + 0.13720323, + 0.2569485, + -0.035785932, + 0.09387681, + -0.12529814, + 0.05789777, + 0.15947631, + -0.043478936, + -0.034896307, + 0.02509474, + -0.066015504, + -0.17300786, + 0.2740886, + -0.008645607, + 0.20366979, + -0.17741504, + 0.0016051099, + 0.04343355, + 0.053064674, + 0.10233873, + -0.057792492, + 0.28412506, + -0.027595375, + -0.011127119, + 0.23101905, + -0.03437331, + 0.12712002, + -0.040828258, + -0.2521543, + 0.092959955, + 0.1476991, + -0.016959714, + 0.24718697, + -0.08578792, + -0.042732067, + -0.02364189, + 0.038429204, + 0.07348472, + 0.041489173, + -0.11468599, + -0.049400177, + -0.027466021, + -0.009244471, + -0.017509859, + 0.08712863, + -0.022377755, + 0.01853816, + 0.042881876, + 0.3835641, + 0.0874544, + -0.068334155, + 0.057505295, + -0.09062925, + 0.057019502, + -0.1344657, + 0.08132518, + 0.000053040872, + -0.059475925, + -0.12508613, + -0.021813422, + -0.013378662, + 0.024890503, + 0.024216384, + -0.077127144, + 0.007335973, + 0.08234029, + -0.18302993, + -0.035066325, + -0.0072978884, + 0.050056715, + -0.22931813, + 0.09650683, + -0.0073624705, + -0.2885135, + -0.10775086, + -0.23143713, + -0.086137205, + 0.0498812, + -0.033241358, + 0.01776586, + -0.026235383, + 0.19309425, + 0.025691753, + -0.058075365, + -0.2405673, + -0.21786085, + -0.19851258, + 0.046656214, + -0.10293365, + -0.10813984, + -0.2152836, + 0.15489893, + 0.047911253, + 0.20260984, + 0.2088183, + -0.1018085, + -0.087688275, + 0.13801338, + 0.0046194997, + -0.1746844, + 0.004021364, + -0.07318715, + -0.24332349, + -0.32633016, + 0.055268224, + -0.013017414, + 0.052299757, + 0.1270179, + 0.004340157, + 0.057317145, + 0.21436898, + 0.06542589, + 0.06742579, + -0.112878166, + -0.020031923, + -0.04590336, + -0.2186622, + -0.11536247, + 0.03292602, + -0.037806716, + 0.11775531, + -0.08126202, + 0.044630688, + 0.004407897, + 0.10470092, + 0.106795885, + -0.02859561, + 0.024228014, + -0.03555758, + 0.14519277, + 0.14093313, + 0.26003087, + 0.0070421537, + 0.08488326, + -0.12916775, + -0.1219826, + -0.2455656, + 0.15563421, + 0.08667417, + 0.01301937, + -0.0907239, + -0.12462255, + -0.032354463, + 0.20714861, + 0.0010497222, + -0.13925967, + 0.03371706, + -0.0062380685, + 0.055929244, + -0.067029044, + 0.059437487, + 0.0076191374, + 0.05578321, + 0.011333173, + -0.05390713, + 0.15975079, + 0.072687715, + 0.15916237, + 0.17636605, + -0.008446719, + 0.014961095, + 0.08686759, + 0.14848803, + -0.22238462, + 0.09633494, + -0.055627063, + 0.12631376, + 0.035983652, + 0.22738391, + -0.11332371, + 0.21841496, + -0.17675073, + 0.14812225, + -0.114016265, + -0.032317907, + -0.18767738, + -0.0811073, + -0.13753203, + 0.16913415, + 0.0114353, + -0.11488439, + 0.011230222, + -0.085470885, + 0.0690153, + -0.10461269, + -0.0706403, + -0.20100483, + -0.037651025, + 0.08529463, + -0.13536386, + -0.034356836, + 0.057857174, + -0.03706257, + 0.02483604, + -0.24963166, + -0.06943555, + -0.09979882, + 0.0942273, + 0.05437428, + -0.10782203, + -0.013504401, + 0.24189045, + -0.13296025, + 0.03774331, + -0.010911639, + -0.082099736, + 0.07280296, + 0.06079229, + 0.02287312, + 0.059112385, + -0.099384256, + -0.076120414, + 0.11072304, + -0.05123164, + 0.1079739, + 0.10227008, + -0.05471506, + 0.10747948, + 0.009699896, + -0.30863142, + -0.04430503, + 0.37351134, + -0.19678722, + 0.13742797, + -0.40235984, + -0.27581173, + 0.3702671, + -0.024754377, + 0.122642905, + -0.14828043, + -0.07008751, + 0.08360051, + 0.0035450035, + 0.0216809, + -0.12010145, + -0.026985122, + 0.053328913, + -0.10591583, + -0.013924753, + 0.038145293, + 0.17823228, + 0.06572483, + -0.28199768, + 0.14622784, + 0.10302125, + -0.10776388, + 0.16953847, + -0.043120794, + -0.05191318, + 0.03779815, + 0.12021654, + -0.0014165355, + 0.023631074, + -0.2394369, + 0.114601746, + 0.055467404, + 0.16078594, + 0.009649481, + -0.21886353, + -0.10147424, + 0.01851072, + 0.13010912, + -0.18243171, + -0.046989094, + 0.11791553, + -0.07706595, + 0.052286703, + -0.19268094, + -0.06734655, + 0.083558746, + 0.05510528, + 0.1205322, + 0.043853316, + 0.19419815, + -0.19383074, + -0.12088962, + 0.050363027, + 0.13108388, + -0.032544736, + -0.19773011, + -0.034516454, + -0.13052338, + -0.0046362486, + 0.0320051, + 0.061747782, + -0.19046386, + 0.09733934, + 0.08370847, + -0.120636776, + -0.051564053, + -0.08520327, + 0.025040282, + -0.047831807, + 0.13498239, + -0.05799306, + -0.082822345, + 0.11464385, + -0.11085481, + 0.11331335, + 0.06106606, + 0.12057433, + -0.1630841, + 0.00096572586, + -0.039613742, + 0.16483565, + 0.17725827, + 0.024782866, + -0.02095867, + -0.30374828, + 0.030041676, + -0.34487057, + 0.016318211, + 0.010022732, + 0.16049328, + -0.059585165, + -0.038909353, + 0.14602326, + 0.003398922, + 0.10123426, + 0.10189341, + 0.0013612165, + -0.095394656, + 0.17065822, + 0.2717381, + -0.048805684, + 0.11726433, + -0.22115687, + -0.013294891, + -0.20146243, + 0.1301142, + -0.079273745, + -0.06366891, + 0.20852767, + -0.11919791, + 0.10993831, + -0.1328157, + 0.023110492, + 0.06534299, + -0.028473591, + 0.03133091, + 0.18335237, + -0.12560268, + 0.05174967, + 0.10752264, + 0.026674548, + -0.116931275, + 0.05302942, + 0.14527842, + 0.04175609, + 0.07524169, + -0.016850159, + -0.085036226, + 0.14378034, + -0.15384178, + 0.06842327, + 0.072759815, + 0.0333774, + -0.07724679, + 0.07146305, + -0.0038083985, + -0.023200184, + -0.21678805, + -0.05851518, + -0.2106472, + 0.21905945, + 0.044797596, + -0.08767725, + -0.06757387, + -0.08139683, + 0.060507063, + -0.2539122, + -0.054056864, + -0.0061713024, + -0.25016263, + -0.013922703, + 0.025297, + -0.031502247, + 0.043440044, + 0.049890436, + 0.10434008, + -0.14499368, + 0.011150854, + -0.16495703, + -0.21740839, + 0.004718739, + 0.048284378, + -0.047843166, + 0.047392353, + 0.10500571, + 0.031224746, + 0.15428485, + 0.03575991, + 0.23791198, + -0.036760766, + -0.21849649, + -0.14954391, + 0.12186352, + -0.030245116, + -0.014910633, + 0.052947998, + 0.07771576, + 0.22827105, + -0.025724977, + 0.13252845, + 0.15444413, + -0.09818237, + 0.031483896, + -0.10577348, + -0.00006837226, + -0.109357946, + 0.07215483, + 0.17097427, + 0.11899255, + 0.061613396, + 0.03566328, + 0.036564983, + 0.0024872916, + 0.06796512, + 0.026589885, + -0.060742363, + -0.07839288, + 0.09555347, + -0.050860837, + -0.12879173, + -0.1040309, + 0.21770604, + 0.0018934173, + 0.023243189, + -0.090174794, + -0.06827122, + 0.15387972, + -0.069496565, + 0.024794634, + -0.051390123, + 0.027016103, + -0.021958428, + -0.10045327, + 0.18993512, + -0.22860111, + -0.10041514, + -0.1531901, + 0.005921122, + -0.03854943, + 0.14580593, + -0.12484452, + 0.022898167, + -0.10113201, + 0.08594769, + 0.015126121, + -0.012950152, + 0.040228955, + -0.09867645, + 0.0058024707, + 0.1267348, + 0.19757093, + 0.01170412, + -0.17352274, + 0.30352616, + 0.3830606, + 0.05377779, + -0.07670585, + 0.025823055, + -0.05698721, + -0.009363706, + 0.077189244, + -0.15372942, + -0.29238287, + -0.25004295, + 0.024756031, + 0.04304348, + 0.081523225, + -0.07263159, + -0.13542518, + 0.08482435, + -0.06367541, + 0.01787075, + -0.027442627, + 0.13191804, + -0.25955498, + -0.2185974, + -0.0048445812, + 0.24410242, + 0.052365653, + 0.13663675, + -0.13785504, + 0.12467762, + 0.2333146, + -0.057672247, + -0.0937421, + 0.018837918, + -0.058080558, + -0.019427018, + 0.13319813, + 0.0118216975, + 0.09013087, + -0.06453432, + -0.06437333, + 0.0037089002, + 0.09036531, + -0.0066115037, + -0.016081506, + -0.1324408, + 0.075796016, + 0.031593855, + -0.15255858, + -0.12663953, + 0.17544357, + 0.060190212, + 0.24437538, + 0.089556195, + 0.20744215, + -0.05519144, + 0.13434736, + 0.13900068, + -0.07795903, + -0.05554333, + 0.010776608, + 0.048118893, + -0.06867825, + -0.1597975, + 0.038932495, + -0.09650621, + -0.07258938 + ] + ], + "bias": [ + -0.08385399, + -0.04598274, + 0.12983686 + ], + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.22 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.json new file mode 100644 index 0000000..398d662 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.json @@ -0,0 +1,1942 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.3, + "abstain_margin_threshold": 0.22 + }, + "metrics": { + "total": 319, + "accuracy": 0.6708464, + "confusion": [ + [ + 58, + 57, + 34 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 13, + 56 + ] + ], + "tool_precision": 0.9830508, + "tool_recall": 0.38926175, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22568056, + "p_memory": 0.40084177, + "p_abstain": 0.37347767 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31395635, + "p_memory": 0.33680987, + "p_abstain": 0.3492338 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39770705, + "p_memory": 0.19950679, + "p_abstain": 0.40278614 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.26465896, + "p_memory": 0.34730524, + "p_abstain": 0.38803577 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25941908, + "p_memory": 0.12832387, + "p_abstain": 0.612257 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.27416888, + "p_memory": 0.10814477, + "p_abstain": 0.61768633 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40384528, + "p_memory": 0.31830305, + "p_abstain": 0.27785167 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.32050437, + "p_memory": 0.27518332, + "p_abstain": 0.40431228 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37631887, + "p_memory": 0.241377, + "p_abstain": 0.38230407 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.30472597, + "p_memory": 0.32799238, + "p_abstain": 0.36728162 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.27784234, + "p_memory": 0.32759824, + "p_abstain": 0.3945594 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2540235, + "p_memory": 0.16229898, + "p_abstain": 0.58367753 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30672368, + "p_memory": 0.19407696, + "p_abstain": 0.49919936 + }, + { + "query_id": "shell_test_019", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48271036, + "p_memory": 0.17951614, + "p_abstain": 0.3377736 + }, + { + "query_id": "shell_test_020", + "text": "whats the total vram on ths gpu", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43741146, + "p_memory": 0.24507794, + "p_abstain": 0.3175106 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4158976, + "p_memory": 0.27515078, + "p_abstain": 0.3089516 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47624564, + "p_memory": 0.14048539, + "p_abstain": 0.38326898 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.48442715, + "p_memory": 0.10436712, + "p_abstain": 0.4112057 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48437816, + "p_memory": 0.2835406, + "p_abstain": 0.23208125 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44654188, + "p_memory": 0.26933104, + "p_abstain": 0.2841271 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39678445, + "p_memory": 0.44244283, + "p_abstain": 0.16077271 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4628093, + "p_memory": 0.270529, + "p_abstain": 0.26666173 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.34852275, + "p_memory": 0.27711794, + "p_abstain": 0.3743593 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47214958, + "p_memory": 0.29806998, + "p_abstain": 0.22978038 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40133312, + "p_memory": 0.2407558, + "p_abstain": 0.35791105 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.36721316, + "p_memory": 0.37156385, + "p_abstain": 0.26122305 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38931295, + "p_memory": 0.2874001, + "p_abstain": 0.32328695 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4190756, + "p_memory": 0.25449738, + "p_abstain": 0.32642704 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42050397, + "p_memory": 0.2790245, + "p_abstain": 0.30047154 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4390786, + "p_memory": 0.13317992, + "p_abstain": 0.4277415 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44003776, + "p_memory": 0.31567433, + "p_abstain": 0.2442879 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47923937, + "p_memory": 0.13611977, + "p_abstain": 0.38464084 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31453353, + "p_memory": 0.26278618, + "p_abstain": 0.42268026 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46307096, + "p_memory": 0.38064674, + "p_abstain": 0.1562823 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4797326, + "p_memory": 0.1527195, + "p_abstain": 0.367548 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43717074, + "p_memory": 0.29522896, + "p_abstain": 0.2676003 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39453, + "p_memory": 0.24812809, + "p_abstain": 0.35734192 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4571177, + "p_memory": 0.2604581, + "p_abstain": 0.2824242 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45665768, + "p_memory": 0.25635234, + "p_abstain": 0.2869899 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38890406, + "p_memory": 0.31626257, + "p_abstain": 0.29483342 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3217656, + "p_memory": 0.11337318, + "p_abstain": 0.56486124 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43858615, + "p_memory": 0.21885185, + "p_abstain": 0.34256205 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41309485, + "p_memory": 0.22376072, + "p_abstain": 0.36314443 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43877676, + "p_memory": 0.21598269, + "p_abstain": 0.34524047 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.35300845, + "p_memory": 0.21918875, + "p_abstain": 0.42780283 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41591418, + "p_memory": 0.28971443, + "p_abstain": 0.29437143 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3736646, + "p_memory": 0.23679724, + "p_abstain": 0.38953817 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39532402, + "p_memory": 0.28937247, + "p_abstain": 0.31530353 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.30211675, + "p_memory": 0.3692885, + "p_abstain": 0.3285947 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5483045, + "p_memory": 0.18406397, + "p_abstain": 0.26763153 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34390134, + "p_memory": 0.4241181, + "p_abstain": 0.23198053 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40889636, + "p_memory": 0.30667284, + "p_abstain": 0.2844308 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.74444443, + "confusion": [ + [ + 34, + 16, + 20 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 10, + 45 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.4857143, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.76666665, + "confusion": [ + [ + 5, + 3, + 2 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 9, + 31 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 2, + 3 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.56666666, + "confusion": [ + [ + 24, + 11, + 15 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.48, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.md new file mode 100644 index 0000000..5ead433 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-6/router_report.md @@ -0,0 +1,159 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.500` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.300` +- abstain_margin_threshold: `0.220` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.6708` +- tools precision: `0.9831` +- tools recall: `0.3893` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 58 | 57 | 34 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 13 | 56 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.23 | 0.40 | 0.37 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.31 | 0.34 | 0.35 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Abstain` | 0.46 | 0.14 | 0.40 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.40 | 0.28 | 0.32 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Abstain` | 0.30 | 0.17 | 0.53 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.38 | 0.31 | 0.32 | +| `holdout_tools_recall_tools_010` | is there a unix socket at the agent.sock path right now? | `Tools` | `Abstain` | 0.36 | 0.19 | 0.45 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Tools` | `Memory` | 0.41 | 0.37 | 0.23 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.48 | 0.25 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Abstain` | 0.38 | 0.18 | 0.44 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.47 | 0.16 | 0.36 | +| `holdout_tools_recall_tools_025` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.40 | 0.25 | 0.36 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Abstain` | 0.35 | 0.13 | 0.52 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Abstain` | 0.29 | 0.14 | 0.57 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Abstain` | 0.45 | 0.14 | 0.41 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Abstain` | 0.37 | 0.20 | 0.43 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.28 | 0.25 | 0.47 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.39 | 0.27 | 0.35 | +| `holdout_tools_recall_tools_042` | how many cpu cores are available right now? | `Tools` | `Abstain` | 0.33 | 0.18 | 0.49 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.22 | 0.35 | 0.43 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Abstain` | 0.35 | 0.21 | 0.44 | +| `holdout_tools_recall_tools_058` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.40 | 0.37 | 0.23 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | 0.33 | 0.27 | 0.40 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.21 | 0.32 | 0.48 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.16 | 0.41 | 0.43 | +| `holdout_abstain_toolish_abstain_029` | vim or emacs? choose for me | `Abstain` | `Memory` | 0.25 | 0.31 | 0.44 | +| `holdout_abstain_toolish_abstain_030` | is gemma 4 better than llama 3.1? check online and decide | `Abstain` | `Memory` | 0.16 | 0.37 | 0.47 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.33 | 0.38 | 0.29 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | 0.34 | 0.34 | 0.33 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.34 | 0.23 | 0.44 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Abstain` | 0.43 | 0.12 | 0.45 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.45 | 0.29 | 0.25 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_abstain_toolish_tools_050` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Abstain` | 0.43 | 0.14 | 0.44 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Abstain` | 0.47 | 0.13 | 0.40 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.50 | 0.27 | 0.24 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.40 | 0.20 | 0.40 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.26 | 0.35 | 0.39 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.26 | 0.13 | 0.61 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.27 | 0.11 | 0.62 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.40 | 0.32 | 0.28 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.32 | 0.28 | 0.40 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.38 | 0.24 | 0.38 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.30 | 0.33 | 0.37 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.28 | 0.33 | 0.39 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.25 | 0.16 | 0.58 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.31 | 0.19 | 0.50 | +| `shell_test_019` | is the klbr tui currently connected to the daemon socket? | `Tools` | `Memory` | 0.48 | 0.18 | 0.34 | +| `shell_test_020` | whats the total vram on ths gpu | `Tools` | `Memory` | 0.44 | 0.25 | 0.32 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.42 | 0.28 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.48 | 0.10 | 0.41 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.48 | 0.28 | 0.23 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.45 | 0.27 | 0.28 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.40 | 0.44 | 0.16 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.46 | 0.27 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.35 | 0.28 | 0.37 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.47 | 0.30 | 0.23 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.40 | 0.24 | 0.36 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.37 | 0.37 | 0.26 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.39 | 0.29 | 0.32 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.42 | 0.25 | 0.33 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.42 | 0.28 | 0.30 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.44 | 0.13 | 0.43 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.44 | 0.32 | 0.24 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.31 | 0.26 | 0.42 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.46 | 0.38 | 0.16 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.48 | 0.15 | 0.37 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.44 | 0.30 | 0.27 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.39 | 0.25 | 0.36 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.46 | 0.26 | 0.28 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.46 | 0.26 | 0.29 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.39 | 0.32 | 0.29 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.32 | 0.11 | 0.56 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.44 | 0.22 | 0.34 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.41 | 0.22 | 0.36 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.44 | 0.22 | 0.35 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.35 | 0.22 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.42 | 0.29 | 0.29 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.37 | 0.24 | 0.39 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.40 | 0.29 | 0.32 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.30 | 0.37 | 0.33 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.55 | 0.18 | 0.27 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.34 | 0.42 | 0.23 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.41 | 0.31 | 0.28 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.4857` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7444` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.7667 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.4800 | 0.0000 | 0.0000 | 0.5667 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-7/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-7/router_model.json new file mode 100644 index 0000000..c76ef34 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-7/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.16052741, + 0.0024608371, + -0.031318825, + -0.09885255, + -0.10179112, + 0.058994167, + 0.109656274, + -0.2224836, + -0.16442509, + -0.08725672, + 0.018937778, + -0.21723568, + -0.01847783, + 0.05206266, + 0.14480045, + -0.13029885, + -0.06268626, + 0.008815729, + -0.08768228, + -0.018255629, + 0.15967207, + -0.0049391207, + -0.02540304, + 0.049357686, + 0.06391954, + 0.040043183, + 0.09474887, + -0.037570562, + 0.17600152, + 0.19785954, + -0.09910671, + -0.14270158, + -0.0536535, + -0.047332756, + 0.18022339, + 0.0676487, + 0.13717946, + 0.027378082, + -0.10548278, + 0.066528544, + -0.19403522, + -0.11643189, + -0.08065489, + -0.03752704, + 0.033599664, + 0.15706381, + 0.013865536, + -0.08501725, + -0.020272262, + -0.018441383, + -0.03735178, + -0.016081626, + -0.10534383, + -0.17559953, + -0.0019625858, + 0.15813902, + -0.14031114, + 0.06402496, + 0.13181971, + 0.17281957, + -0.012040283, + 0.14102454, + -0.051492248, + 0.1004284, + -0.20263205, + 0.029880801, + 0.12380502, + -0.07871024, + 0.099203266, + 0.16148943, + -0.09668969, + 0.12815538, + 0.003585216, + -0.030985428, + -0.14598812, + 0.1132824, + 0.10472477, + -0.029968526, + -0.063032724, + 0.014652918, + -0.02877283, + 0.052562434, + -0.13108331, + 0.0002965604, + 0.11512386, + 0.09318654, + -0.10961809, + -0.0880418, + -0.09860604, + 0.07382556, + -0.1037337, + -0.038589653, + -0.09008774, + -0.01911863, + -0.14737788, + -0.009485854, + -0.047335766, + 0.016980678, + 0.084372506, + 0.052771546, + -0.044290844, + -0.028756851, + 0.2607931, + 0.08380605, + -0.064346865, + 0.06569313, + -0.041412737, + -0.096951924, + -0.12520541, + -0.014805983, + -0.24914601, + 0.086238, + -0.002006227, + 0.10598944, + 0.12087416, + 0.03508919, + -0.051487036, + -0.1256758, + -0.12843262, + -0.018472746, + 0.17818493, + -0.1824178, + 0.085565396, + -0.0069503593, + -0.06915642, + 0.078164116, + 0.22044943, + 0.095329665, + 0.09443477, + 0.009476721, + 0.12377651, + 0.09528795, + 0.112927206, + 0.1125022, + -0.09308766, + -0.010651261, + 0.091823, + 0.047559917, + 0.07709087, + -0.106162645, + -0.15692854, + 0.072903305, + 0.045788582, + 0.0375331, + 0.06844259, + -0.23813765, + 0.013259943, + -0.06379869, + 0.024160216, + 0.032886725, + -0.18673082, + 0.01760028, + 0.19031574, + -0.11544211, + 0.105799034, + -0.02177165, + -0.065577194, + -0.0861514, + 0.1342203, + 0.18299834, + 0.056894757, + -0.014077486, + 0.15178883, + -0.11575148, + 0.009013221, + 0.13157651, + -0.0039246664, + 0.108594656, + 0.015797623, + -0.13545142, + 0.0977668, + 0.046627443, + -0.14986962, + -0.100400895, + -0.06745776, + -0.02255909, + -0.116668254, + -0.09205319, + -0.042751074, + 0.02285744, + 0.05999009, + 0.105419904, + -0.2971868, + 0.15153843, + 0.013010975, + -0.058407262, + -0.07143607, + 0.32557562, + 0.055834014, + -0.10003547, + 0.14418238, + 0.124668226, + -0.03373238, + 0.03690689, + -0.008434, + -0.043183107, + 0.052016493, + -0.088068604, + -0.14013946, + 0.053377382, + 0.1353628, + -0.122651346, + 0.06593491, + -0.010691284, + 0.049608894, + -0.14469606, + -0.11539988, + 0.26940665, + -0.12557648, + -0.33065188, + -0.16942522, + -0.16879891, + -0.17476796, + -0.11808337, + 0.006112452, + -0.074556805, + 0.05563614, + -0.10752109, + 0.0045725466, + -0.100444764, + -0.19990551, + 0.1623679, + 0.070562355, + 0.17294201, + -0.21465039, + -0.15566683, + -0.22320299, + -0.040394336, + 0.096149504, + 0.12600791, + -0.071590565, + -0.14021218, + 0.07090477, + -0.05439689, + 0.23978162, + 0.19791496, + 0.14561944, + 0.09421892, + 0.03198014, + -0.019724434, + -0.17524067, + 0.11801276, + -0.034325983, + -0.12414058, + 0.09539344, + 0.04428776, + 0.1991088, + 0.031654123, + -0.04807285, + 0.04592151, + -0.07569178, + -0.031496607, + -0.19537716, + 0.060411256, + -0.051872216, + 0.090447225, + 0.08494578, + -0.11747185, + -0.028199665, + 0.1347343, + -0.17381737, + 0.19282167, + 0.08678283, + 0.034531005, + -0.043074735, + -0.118435755, + 0.021840902, + 0.08782734, + -0.028497253, + -0.09325137, + -0.037644356, + -0.10620848, + 0.20808154, + 0.059732296, + 0.033074785, + 0.009470372, + 0.104916535, + 0.066376306, + -0.010363118, + 0.05408072, + 0.02767956, + 0.12599848, + 0.016272616, + -0.10300154, + 0.06984461, + 0.093451455, + -0.3713836, + 0.004608827, + -0.06669234, + 0.1191322, + -0.029813163, + 0.06783095, + 0.053586982, + 0.018094638, + -0.074333526, + 0.19786923, + -0.019449225, + 0.02383677, + 0.004300856, + 0.10008391, + 0.22801672, + 0.12678932, + 0.11370807, + 0.16604814, + 0.088361025, + 0.05435657, + 0.027251301, + -0.0097401505, + -0.10017743, + -0.0076766806, + -0.07074346, + 0.023101224, + 0.045202877, + 0.075106, + 0.1932022, + 0.034920372, + 0.10798186, + -0.15182951, + 0.046124, + 0.08082057, + -0.011904392, + -0.03646725, + 0.06683557, + 0.10610709, + 0.015073168, + -0.027114002, + 0.0211207, + -0.14800651, + -0.16112563, + 0.042969227, + 0.05496639, + -0.10502971, + -0.10001824, + 0.13661686, + -0.07500956, + 0.06874437, + 0.0835452, + -0.10677995, + 0.11066851, + 0.14947426, + 0.05686178, + -0.045194548, + 0.080891475, + 0.01653106, + -0.03289933, + 0.013714271, + -0.12062449, + -0.10942656, + 0.0427117, + 0.11709176, + -0.025273317, + -0.051796433, + 0.0050489707, + -0.1257773, + -0.017385142, + 0.12721983, + -0.10977477, + -0.23171797, + -0.020706484, + -0.010048814, + 0.0075350627, + -0.0067544025, + -0.0440331, + 0.3260202, + -0.00035211392, + -0.039470658, + 0.08292158, + -0.093218766, + 0.09687666, + 0.014780582, + -0.08518228, + 0.080687046, + -0.013215859, + 0.010666921, + 0.043733794, + -0.08879815, + -0.23352464, + -0.19190405, + 0.21334182, + -0.20023906, + -0.04070634, + -0.16439508, + -0.0065315836, + 0.09033729, + -0.096911885, + -0.061966095, + -0.03798265, + -0.17271575, + -0.04042787, + 0.16066734, + 0.19571932, + -0.005219358, + -0.113288976, + -0.034409754, + -0.02542398, + 0.091005854, + 0.075512506, + -0.12603755, + 0.003550829, + 0.05454221, + -0.1700984, + 0.08618596, + 0.07662489, + -0.12044079, + 0.12438776, + 0.0038605034, + -0.008291778, + -0.02776508, + -0.012168801, + -0.0791805, + 0.05979986, + -0.05944031, + 0.16329224, + -0.32967898, + 0.08408068, + 0.013467381, + 0.056141563, + -0.039574765, + 0.01251687, + -0.333109, + -0.10213542, + -0.2410506, + 0.024430476, + 0.02588703, + -0.14144824, + 0.0058720787, + -0.0798089, + -0.017106313, + -0.0020736381, + -0.06813275, + 0.11289762, + -0.19290096, + 0.14954773, + -0.048712995, + 0.09767048, + 0.17014039, + 0.010858046, + -0.06397144, + 0.098602, + -0.026944358, + -0.06880838, + 0.09306078, + 0.21228854, + -0.052566614, + -0.16931634, + 0.1503215, + -0.04413507, + -0.050339848, + 0.070244834, + 0.28390154, + 0.04381359, + 0.13148732, + -0.09735818, + -0.04281441, + -0.0002529125, + -0.12293158, + 0.055222724, + -0.0026420944, + 0.014094278, + 0.055280052, + 0.07080589, + -0.06374293, + -0.21383622, + -0.07480419, + -0.06326204, + -0.050647177, + -0.12127862, + -0.088789135, + 0.07309716, + 0.02734736, + -0.023602324, + -0.11849017, + 0.0862599, + -0.100177184, + -0.20296523, + 0.056530718, + -0.04380203, + -0.07311151, + 0.17730682, + 0.12751189, + 0.059371542, + 0.15086243, + 0.14834693, + 0.05420559, + -0.005883509, + 0.037433855, + 0.03104376, + 0.017708058, + -0.11753717, + -0.092178345, + -0.13057369, + 0.079070516, + 0.13699917, + -0.023019383, + -0.16708986, + -0.100313716, + -0.11022881, + 0.033850983, + 0.17981607, + -0.05736247, + -0.009717411, + 0.06720382, + -0.09542911, + 0.021875495, + 0.13131075, + -0.10874953, + 0.0064090267, + 0.08441985, + -0.010587874, + 0.04712112, + -0.06715897, + -0.0367435, + 0.02208662, + 0.030317329, + 0.033193227, + -0.121694334, + 0.04879346, + -0.045048017, + -0.030299243, + -0.01951495, + -0.04660699, + -0.08691534, + 0.015517739, + 0.07925178, + 0.055090293, + -0.16836199, + -0.020131057, + -0.04979672, + -0.080895, + 0.2045888, + -0.035188764, + 0.04068433, + 0.14953531, + -0.21293955, + -0.18716255, + 0.038786113, + -0.06314729, + -0.13646829, + 0.4301075, + -0.054831143, + 0.11402921, + 0.0715126, + -0.015407559, + 0.021623263, + 0.08860939, + 0.04332614, + 0.0063768667, + -0.1433278, + 0.11752632, + -0.15465412, + -0.08700072, + -0.0845954, + -0.14791308, + 0.0016860976, + -0.09356121, + 0.06972774, + -0.14896198, + -0.0330842, + -0.044193294, + 0.057510998, + -0.1900679, + -0.10587522, + 0.056306608, + 0.11176626, + -0.030755477, + 0.052004084, + 0.10115147, + 0.009432866, + -0.16124472, + 0.19203115, + 0.072125755, + 0.19657056, + 0.000823032, + -0.05501959, + 0.25273466, + 0.037036214, + 0.0799104, + 0.015897552, + 0.05778206, + -0.017158741, + -0.068570286, + 0.03771934, + -0.07160096, + 0.12563969, + 0.09058254, + 0.19363588, + 0.21729888, + -0.038404334, + 0.007869045, + -0.11452937, + -0.061573993, + -0.09937147, + -0.092896424, + -0.022444807, + 0.084605426, + 0.042577196, + 0.091019176, + 0.0687685, + 0.11391002, + -0.023444116, + 0.056105986, + 0.0010575949, + 0.058704697, + 0.13065207, + 0.18417926, + -0.17100154, + -0.03459482, + -0.13954142, + 0.1697018, + -0.032380745, + 0.11538436, + -0.23546107, + 0.13714324, + 0.047174696, + 0.1799499, + -0.039026696, + 0.020465225, + 0.09494173, + 0.26512316, + -0.07517377, + -0.043898042, + -0.13932166, + 0.0056566624, + -0.0055164094, + -0.043773327, + -0.0541095, + -0.0054946276, + 0.0663163, + 0.0021626875, + 0.10122027, + 0.06443506, + 0.2846765, + -0.28031296, + 0.08613889, + 0.04016044, + -0.045890335, + -0.11568628, + 0.09650608, + -0.0396269, + -0.019264355, + -0.12604448, + 0.12022761, + -0.18100314, + 0.1449683, + -0.19298455, + 0.0029647453, + 0.01889965, + -0.0340142, + 0.02557537, + -0.053900447, + 0.102941334, + 0.040459722, + -0.11036412, + -0.05180269, + -0.021873292, + 0.055194244, + -0.18999355, + -0.053315975, + -0.07193985, + 0.13219415, + 0.03263697, + 0.07894841, + 0.027257426, + 0.17079633, + 0.33656633, + -0.09741153, + -0.0012406677, + -0.062253032, + 0.122720554, + -0.07428574, + 0.092065744, + -0.12949777, + -0.2637853, + -0.17236178, + 0.11274971, + -0.0036404934, + -0.067255475, + -0.01703726, + -0.024414714, + -0.08555217, + 0.11511529, + -0.10132069, + 0.035678104, + -0.03773041, + -0.11198654, + 0.09750184, + 0.11933404, + -0.017337881, + -0.0275599, + -0.056142204, + 0.2870939, + 0.025516802, + 0.089016005, + 0.042186953, + 0.061288327, + 0.23220468, + 0.1913539, + 0.041872956, + -0.029348254, + 0.2022436, + 0.24744327, + 0.00085464993, + -0.06835739, + 0.21693708, + 0.03833265, + -0.11809685, + -0.094088, + -0.08797381, + 0.03583436, + -0.03725489, + -0.12992965, + -0.046520386, + 0.08877341, + -0.23196267, + -0.1416181, + -0.14281443, + 0.0051557235, + -0.083149076, + -0.030370584, + 0.028675005, + -0.069152854, + 0.022219311, + -0.24649718, + 0.16162385, + 0.13550395, + 0.24505398, + 0.11737749, + -0.26714596, + 0.039750718, + 0.036444116, + 0.19190054, + 0.25507984, + -0.05611311, + -0.03598602, + -0.00091825356, + 0.035975944, + 0.005179897, + -0.05993038, + 0.04453667, + -0.06822254, + -0.0034093673, + 0.041618973, + 0.009030745, + -0.16429444, + -0.10991389, + -0.21974932, + 0.05839352, + -0.12792662, + -0.17765494, + 0.06688134, + 0.16218579, + -0.13164958, + -0.059599277, + 0.08895215, + 0.102052465, + -0.045741614, + 0.15715373, + -0.26308244, + 0.051839665, + 0.16174427, + -0.13277116, + -0.13938846, + -0.082957774, + -0.03518575, + -0.16200525, + -0.1170968, + 0.090260565, + -0.10743151, + -0.20015334, + 0.19283876, + 0.02739746, + -0.0678554, + -0.0289476, + -0.024540577, + -0.15412445, + 0.028864484, + -0.033722255, + 0.037424758, + 0.04879276, + 0.048878994, + 0.07719014, + -0.027368383, + 0.1159599, + -0.097892486, + 0.015980033, + -0.0061608437, + -0.035205483, + -0.20321424, + -0.08789502, + 0.016100816, + 0.017145542, + -0.17215213, + 0.17112683, + -0.002088437, + 0.13785838, + 0.047853924, + 0.23892085, + 0.16270262, + 0.030570636, + 0.1013973, + -0.060039762, + -0.09836197, + -0.111120716, + 0.12632947, + -0.08306903, + -0.12863602, + 0.047026373, + -0.1958446, + 0.0044924226, + 0.1719895, + 0.10121101, + 0.17643425, + -0.046456266, + 0.17635134, + 0.01827563, + 0.08540644, + 0.06120104, + -0.113148265, + -0.1218303, + 0.011219538, + -0.12710062, + -0.050977923, + 0.053127747, + -0.03822012, + -0.00858817, + 0.014970831, + -0.12115409, + 0.008089462, + 0.068597436, + 0.08169945, + -0.10300656, + 0.05761663, + -0.0078221625, + -0.06458102, + 0.020226378, + -0.005276232, + 0.0895599, + 0.08770444, + -0.19067658, + -0.044448506, + -0.13667849, + -0.045721535, + -0.017611813, + -0.17994289, + 0.061323702, + -0.006091955, + 0.037093177, + 0.051335093, + 0.048674185, + 0.10398149, + -0.023160957, + -0.035153363, + 0.12117348, + 0.038042266, + -0.074533135, + 0.014664317, + 0.027021443, + 0.0034762379, + -0.06064791, + -0.015092715, + -0.2944957, + 0.10236635, + 0.15401675, + 0.15111294, + 0.1340751, + -0.17079759, + 0.06980032, + -0.07446118, + -0.075741716, + 0.045889974, + -0.027051862, + 0.061429117, + 0.13151085, + 0.2020734, + -0.05170606, + -0.019183844, + -0.029455291, + -0.16436203, + 0.065099075, + 0.114943504, + 0.07208407, + -0.052109897, + 0.058475323, + -0.0060676374, + 0.108448036, + 0.13429314, + 0.07835998, + -0.042943794, + 0.07505265, + -0.15782769, + -0.0008275639, + -0.022941407, + -0.089207046, + -0.053613666, + -0.22768244, + -0.004562265, + 0.119613804, + -0.06730513, + -0.005134274, + 0.015889524, + 0.065859795, + -0.13738343, + 0.04806415, + -0.021074295, + 0.02389348, + -0.04589438, + -0.00791156, + 0.19126686, + 0.066830024, + 0.038546782, + 0.056414336, + -0.09672572, + -0.1488234, + -0.1606585, + 0.03534793, + -0.009993679, + 0.0033058047, + 0.052286904, + -0.09087651, + 0.015177222, + -0.1209879, + 0.117720395, + -0.17759861, + -0.045116693, + 0.3353583, + -0.17735094, + -0.255285, + -0.20740092, + -0.018373836, + 0.0005954378, + -0.011089623, + -0.21794018, + 0.14621736, + 0.017658569, + -0.07371307, + 0.021241616, + -0.12774628, + 0.12024284, + -0.069704995, + -0.08754762, + 0.052305937, + 0.13504027, + -0.0111032175, + 0.0896511, + -0.027011193, + 0.0964028, + -0.04770052, + -0.05891503, + -0.29946882, + 0.06799919, + 0.019578407, + -0.09722951, + 0.040762562, + -0.039349716, + -0.04986413, + 0.008755783, + -0.035490353, + 0.28766772, + -0.019644262, + -0.14135677, + -0.08013137, + 0.06394487, + 0.17467594, + -0.10952025, + -0.067189634, + -0.05346993, + 0.13319, + -0.02134673, + 0.08383482, + -0.005200357, + -0.013754012, + -0.020868568, + 0.10664089, + 0.266166, + 0.0021528348, + 0.17895368, + -0.0036033643, + 0.007758813, + 0.0316086, + -0.008563442, + 0.008860687, + -0.10833197, + 0.063689224, + -0.09778756, + -0.25244159, + 0.25641277, + 0.07796288, + -0.0037101684, + 0.04819473, + -0.08170173, + 0.015830114, + -0.028049998, + 0.02885121, + -0.028635604, + -0.05007718, + 0.112869985, + -0.060543325, + -0.05746815, + -0.14462683, + 0.1255916, + 0.121556595, + 0.102449775, + 0.120668866, + 0.0036547463, + 0.13570864, + 0.06869708, + 0.19977914, + -0.14529352, + 0.16520394, + 0.019234715, + 0.021917906, + -0.19478388, + -0.07423999, + -0.13678935, + -0.13091457, + 0.10972841, + -0.16713484, + -0.10109219, + -0.13254036, + -0.007902096, + 0.06695212, + -0.051268183, + -0.03536522, + 0.09367453 + ], + [ + 0.11637956, + -0.17186578, + 0.10791955, + 0.080459945, + 0.05975105, + -0.011272944, + 0.10946989, + 0.22342005, + 0.099497564, + 0.04502466, + -0.014406636, + 0.10069611, + 0.024731083, + -0.031065537, + -0.058351807, + 0.06504147, + 0.15619484, + -0.06604318, + 0.1781333, + 0.22792023, + -0.13898799, + -0.045416683, + 0.031234754, + 0.037138533, + -0.18047316, + -0.004940352, + -0.056690544, + 0.04373834, + -0.1627064, + -0.10905386, + -0.040010177, + 0.3229176, + 0.07230417, + 0.07616515, + -0.028685225, + -0.055634305, + -0.03288254, + -0.004032664, + 0.092190936, + -0.35969818, + -0.06260908, + 0.13865389, + -0.026160512, + 0.06181806, + 0.10938138, + -0.121704094, + 0.19553061, + -0.06748917, + -0.06838561, + 0.10467503, + -0.070798166, + -0.0747031, + -0.074544385, + 0.099389784, + 0.08020828, + -0.14345938, + 0.058599636, + -0.08092364, + 0.15528879, + 0.053775847, + 0.05522898, + -0.17603008, + 0.08225849, + -0.072476104, + 0.19657768, + -0.18427952, + -0.054881528, + 0.005319496, + -0.108707465, + -0.18644795, + -0.051599722, + -0.26696226, + 0.087243766, + 0.027708074, + 0.043569595, + 0.0017946062, + 0.061552893, + -0.0073106713, + 0.13140735, + -0.086496115, + 0.035292115, + 0.04865788, + 0.27236786, + 0.064673945, + -0.07814582, + -0.10368492, + 0.021018954, + 0.007109233, + 0.16142946, + -0.32392415, + 0.11618061, + 0.16055097, + -0.10717878, + -0.027265606, + -0.052559048, + 0.06769613, + 0.109514505, + -0.051763024, + -0.05986832, + 0.054277677, + 0.020888394, + 0.09378125, + -0.101614766, + -0.114495024, + 0.19143566, + -0.13538244, + -0.030124659, + 0.029111356, + -0.0665248, + -0.019361828, + 0.03687205, + -0.14246751, + 0.059040654, + -0.04202304, + -0.22132228, + 0.00042482893, + 0.06615944, + 0.16350724, + 0.29969427, + 0.15013222, + -0.057181604, + 0.045494482, + 0.09992527, + 0.10001293, + 0.07406286, + -0.02437011, + -0.0807268, + -0.2998727, + -0.17070091, + -0.13470376, + -0.07132043, + -0.067651786, + 0.00096411596, + 0.009365229, + -0.0010018462, + -0.02374255, + -0.022331577, + 0.008360721, + -0.24549751, + 0.09109612, + 0.07186384, + -0.21236059, + -0.004149225, + 0.029868592, + -0.16807589, + 0.38404953, + -0.07270962, + 0.06128629, + 0.0029842712, + -0.08691458, + -0.022859711, + -0.15697275, + -0.110039994, + 0.07708965, + 0.16576324, + -0.14948945, + 0.15736353, + 0.23472643, + -0.004841239, + 0.005375038, + -0.104134165, + 0.29700267, + -0.15814611, + 0.018846232, + 0.02098197, + -0.020519339, + 0.022707358, + 0.1988144, + -0.11999712, + -0.02079705, + -0.058023993, + 0.0058619855, + 0.11121759, + 0.017506061, + 0.05126044, + -0.096916825, + 0.2307071, + -0.062108804, + 0.123407386, + 0.0031229893, + -0.044057585, + -0.096670195, + 0.21410017, + -0.041165788, + 0.035975046, + -0.15833475, + 0.12947544, + -0.3752908, + -0.03691883, + 0.095158406, + 0.032910068, + -0.12258116, + 0.035828426, + 0.06966969, + 0.016244017, + 0.170389, + -0.1261006, + 0.1098775, + 0.13936415, + -0.04809021, + 0.19010004, + 0.12529476, + -0.11426704, + -0.023094462, + -0.23561454, + 0.14241934, + -0.07336394, + -0.3049396, + 0.07135916, + 0.29270256, + 0.042280797, + 0.13791633, + 0.14932086, + -0.03536875, + 0.09843503, + -0.18766549, + -0.046467368, + 0.14914943, + -0.023084715, + 0.09644197, + -0.003209695, + -0.16610551, + -0.14070651, + -0.37365547, + 0.10216226, + 0.21309158, + 0.08469838, + -0.14306426, + -0.091675095, + -0.10465955, + 0.027742147, + 0.16314667, + 0.03690933, + -0.03567327, + -0.26394844, + -0.19670296, + 0.102186255, + -0.16789915, + 0.016203193, + -0.06374825, + 0.05287729, + -0.09745945, + 0.00722403, + 0.07676293, + 0.014780153, + -0.10793076, + -0.1901627, + -0.18417664, + 0.04534059, + -0.0716811, + 0.07866812, + 0.1439692, + 0.06014963, + -0.21927246, + 0.20979516, + -0.015994566, + 0.12689663, + 0.041828584, + 0.08181566, + -0.063342, + 0.12462212, + -0.10833612, + -0.040642224, + -0.062786385, + 0.021669455, + 0.02062207, + -0.15943547, + -0.14038363, + 0.04656669, + 0.22162975, + -0.03973174, + 0.07510139, + 0.06834014, + 0.0720175, + 0.121220686, + 0.008868144, + -0.13243613, + -0.050706953, + 0.0605855, + -0.07571807, + 0.0075447657, + -0.16944724, + 0.090943016, + 0.017109644, + -0.064935304, + -0.06073986, + 0.063119814, + -0.12180592, + -0.057504665, + 0.12659715, + 0.07531627, + 0.056290645, + 0.018820873, + 0.11000129, + -0.14517342, + -0.31130013, + -0.1983607, + 0.08439539, + -0.123937875, + -0.03123252, + -0.2514824, + -0.11085155, + 0.005044094, + -0.14976013, + -0.057709478, + 0.070679195, + 0.14638405, + -0.18428764, + -0.004577368, + 0.028631024, + 0.05115411, + 0.0073806057, + -0.051902503, + -0.021504115, + 0.0070884735, + 0.07090578, + 0.092421785, + 0.15728186, + 0.009211559, + -0.4120053, + -0.023852421, + 0.08603379, + -0.14602195, + -0.052523993, + 0.083968624, + 0.12750289, + -0.071089976, + 0.24766031, + 0.099861264, + -0.036790013, + 0.16011785, + -0.011961857, + -0.16865316, + 0.2080289, + 0.11488308, + -0.05541248, + -0.06588284, + -0.07385138, + -0.21823715, + -0.027737396, + 0.0028189542, + 0.023575094, + -0.024090767, + 0.0011296587, + -0.11189924, + 0.005202209, + 0.07130078, + 0.014037095, + 0.047963694, + -0.06790311, + 0.11792759, + 0.088360295, + -0.007919029, + 0.066773124, + -0.04210118, + -0.26693234, + -0.0719914, + 0.24812208, + 0.2582869, + -0.01645684, + 0.09704079, + -0.06148363, + 0.0888649, + -0.26011285, + 0.013716547, + 0.025316631, + -0.12539926, + -0.01655872, + 0.018356794, + 0.09316022, + 0.2148001, + -0.42118096, + 0.09882388, + 0.0054554185, + 0.008558932, + 0.06381473, + 0.26438358, + 0.058518853, + -0.15347068, + 0.16173801, + -0.33100104, + 0.030894315, + 0.027564311, + -0.016866116, + 0.032837298, + 0.00077169156, + 0.03133184, + 0.06765387, + -0.049206775, + -0.016165834, + -0.212784, + 0.12581074, + 0.20880195, + -0.06285257, + 0.049284827, + -0.070513986, + -0.18638465, + 0.18946855, + 0.078632794, + -0.013579502, + 0.25505856, + -0.2065767, + 0.10019049, + 0.11834874, + -0.019598717, + -0.08845627, + -0.11897016, + -0.017256223, + 0.017200148, + 0.16272031, + 0.041610334, + 0.0394703, + -0.026481334, + 0.09240797, + 0.030089354, + -0.1735567, + -0.051628623, + -0.03471855, + 0.19301184, + 0.116581686, + -0.12833858, + 0.09992816, + -0.025773646, + 0.21380223, + 0.38609263, + -0.09755428, + 0.17755441, + 0.0037833347, + -0.08545212, + 0.09529221, + -0.21268612, + 0.00090696284, + -0.15830845, + -0.11832876, + 0.009103947, + 0.020926477, + 0.0012178889, + 0.19595702, + -0.040166292, + 0.050838795, + -0.16591446, + 0.015815735, + -0.28945345, + 0.29379824, + -0.042782035, + -0.17004989, + 0.19729897, + 0.0055258335, + -0.03620126, + -0.19522566, + -0.121389605, + -0.00032941398, + -0.17998022, + 0.06779911, + 0.2631502, + 0.13141875, + -0.04985646, + 0.052451488, + 0.24368726, + 0.07297032, + -0.053266484, + -0.019812576, + 0.17073107, + 0.11443813, + -0.034046765, + -0.045550775, + -0.034369446, + 0.07116018, + 0.035985112, + 0.10965749, + -0.005791836, + 0.02731971, + -0.07340144, + 0.10565047, + 0.11125925, + 0.05740682, + -0.17137615, + 0.045366608, + 0.029304776, + -0.13445312, + -0.039387558, + 0.08226571, + -0.028178785, + -0.016610812, + 0.0103821615, + -0.23829375, + -0.24541384, + -0.008268759, + 0.07972249, + 0.16931161, + -0.045478042, + 0.03659767, + -0.09050166, + -0.021094369, + -0.032404713, + 0.1336974, + 0.0533087, + 0.035258383, + -0.1752104, + 0.23809828, + 0.08637835, + -0.20440699, + -0.16151933, + 0.013910408, + -0.22518755, + 0.23404773, + -0.06430676, + -0.2438961, + 0.054066826, + -0.01222482, + 0.04206431, + 0.10275902, + 0.1509213, + -0.3044058, + -0.024547627, + -0.08197543, + 0.12862152, + 0.04344292, + -0.013134288, + -0.033549696, + -0.05573171, + 0.14470781, + -0.29964295, + -0.051656418, + -0.043963168, + -0.0626571, + 0.054504357, + -0.07732341, + 0.12172327, + 0.04756539, + -0.057771176, + -0.18838337, + -0.13257566, + -0.03424755, + 0.27295038, + 0.0039459704, + 0.08678922, + 0.09803905, + -0.5035922, + 0.0133419875, + 0.00065672013, + -0.02211246, + 0.04287359, + -0.012378799, + -0.07109952, + -0.13045481, + 0.016000884, + 0.12478969, + -0.16040824, + -0.22890955, + -0.00045370215, + 0.15292962, + 0.090407826, + 0.088943146, + 0.036541674, + 0.06473803, + 0.067636736, + 0.033031188, + 0.1036692, + 0.067575134, + 0.21188134, + 0.1192539, + -0.08119708, + -0.13598263, + 0.10788264, + -0.05933999, + -0.18349162, + 0.1735971, + 0.19631074, + -0.18473329, + -0.12218243, + 0.032747492, + -0.097329825, + 0.06238207, + 0.035778794, + 0.07071465, + 0.15152672, + 0.070239685, + -0.10766326, + 0.050400086, + 0.050804477, + -0.011483971, + -0.121493325, + -0.15133147, + -0.03250726, + 0.04693144, + 0.0005621336, + 0.236917, + -0.054525223, + 0.21746288, + 0.1697138, + 0.31465507, + -0.062002458, + -0.02546646, + -0.28721526, + -0.2513955, + 0.010789273, + 0.018919708, + -0.25192356, + 0.018824613, + 0.118578434, + -0.00507896, + 0.014482422, + 0.11267149, + 0.14215098, + 0.11573328, + 0.04761225, + 0.08724168, + -0.29671982, + 0.028040556, + -0.17270142, + 0.021092054, + -0.20256923, + -0.11460052, + -0.06707171, + 0.05905861, + 0.025438156, + 0.123720355, + -0.14976059, + 0.042247787, + 0.08170481, + 0.02156635, + 0.075605355, + -0.03911428, + 0.039365444, + -0.05059143, + -0.101301275, + -0.037720732, + -0.026390677, + -0.0656627, + -0.20962778, + -0.42560935, + 0.020281976, + -0.09318103, + -0.12504372, + 0.17505816, + 0.23766881, + 0.14905949, + -0.116007335, + -0.06740984, + 0.113025166, + -0.029503759, + 0.30562577, + -0.11261386, + -0.014164033, + -0.004014467, + 0.120359935, + 0.00029712665, + -0.01933728, + -0.0020288113, + -0.035912253, + -0.0998972, + 0.10274502, + -0.003980527, + 0.010540119, + -0.0012870778, + 0.03024298, + -0.019371746, + -0.08722253, + -0.30856013, + -0.024190273, + -0.09390957, + -0.11412507, + -0.3192844, + -0.11418165, + 0.0010766103, + 0.056867734, + -0.06406079, + -0.15870427, + -0.15309812, + 0.021257922, + -0.08891713, + 0.4405356, + 0.024239432, + 0.0012664798, + 0.035958398, + 0.2549329, + 0.098144576, + 0.16194685, + -0.083581924, + -0.12655054, + 0.21620502, + -0.046908326, + 0.12320124, + 0.042971227, + 0.007110838, + -0.048693836, + 0.21834254, + 0.065210894, + -0.029152373, + -0.15173002, + 0.00884002, + -0.14687325, + -0.0051244213, + -0.08612436, + 0.017426912, + -0.12191853, + 0.057925828, + -0.064879015, + -0.25661778, + -0.13962127, + 0.012649742, + -0.17353316, + -0.08397678, + -0.076075956, + 0.12900859, + 0.17618789, + 0.015170843, + -0.0966266, + 0.014381747, + 0.070817225, + 0.14590451, + -0.012652974, + 0.1212396, + 0.1928499, + 0.034840625, + -0.107425846, + 0.13786417, + -0.07710894, + -0.038374882, + 0.37778434, + 0.022085752, + -0.12701413, + 0.03516333, + -0.27293193, + 0.15730625, + 0.15843415, + -0.10312135, + -0.014996361, + -0.159087, + -0.04362026, + -0.18499237, + -0.02748742, + 0.032441035, + -0.020762652, + 0.08412549, + 0.021805234, + 0.00660146, + 0.061379142, + 0.0821473, + -0.034735933, + -0.21985133, + -0.07475556, + 0.44629174, + -0.03631395, + 0.11672802, + 0.049370393, + -0.041611828, + 0.22077566, + -0.014968166, + -0.19998384, + 0.011433027, + 0.06101584, + -0.11258323, + 0.13738437, + -0.0688601, + -0.21262124, + 0.102296345, + -0.061489206, + 0.057119362, + 0.23424532, + 0.120877706, + -0.047151435, + 0.2176173, + 0.20899433, + -0.00081871107, + -0.013194566, + 0.05514477, + 0.3928346, + -0.12549223, + -0.11095625, + 0.012750077, + -0.09158463, + -0.019312715, + -0.04007366, + 0.16496634, + 0.15461186, + -0.0877878, + -0.1798766, + -0.016334271, + 0.120540105, + 0.061884813, + 0.014563505, + 0.10252879, + -0.047985137, + -0.055586945, + 0.22566938, + 0.1058749, + 0.004186561, + 0.10453594, + 0.034418534, + 0.2573554, + -0.19616708, + 0.049920246, + -0.2728407, + 0.010139094, + -0.15609851, + -0.2773467, + 0.08028425, + -0.21471062, + -0.0010262715, + -0.02221235, + 0.27420497, + -0.12729512, + 0.12268265, + -0.036199618, + -0.22428453, + 0.17106162, + 0.016466241, + 0.13175893, + -0.13125266, + 0.16843618, + 0.030138072, + -0.18637407, + -0.17876884, + -0.025821261, + -0.022291679, + -0.032874994, + 0.118431464, + -0.112453796, + 0.025207195, + 0.049616717, + 0.042266954, + -0.13243802, + -0.2631501, + 0.033834856, + 0.0038898303, + 0.21306744, + -0.05530253, + 0.119763054, + -0.027107682, + 0.021657147, + 0.071491025, + -0.14394669, + 0.098971434, + -0.104662135, + 0.043255832, + -0.11081497, + 0.1253336, + 0.07292208, + 0.105347626, + -0.1376309, + 0.14321448, + 0.12819311, + -0.16884637, + -0.020582603, + 0.07983813, + -0.10436447, + -0.19395255, + -0.14573762, + -0.05208073, + 0.052003495, + -0.03613725, + -0.18182264, + 0.22837463, + -0.08308755, + -0.09978127, + -0.036853623, + 0.13789463, + -0.056370385, + 0.29830414, + -0.0791662, + 0.06277126, + -0.092597716, + 0.076572105, + -0.048261747, + -0.114597894, + 0.16213843, + 0.1433155, + 0.035506878, + -0.033455156, + 0.19248319, + -0.07745398, + -0.19590208, + 0.3018688, + 0.03310655, + 0.0041582887, + 0.19586419, + -0.10853914, + -0.16483396, + -0.17642425, + 0.19710363, + -0.06962616, + 0.17102459, + 0.10896041, + -0.13901196, + -0.12664439, + 0.09078699, + -0.12244493, + 0.052821994, + -0.030397188, + -0.13134342, + 0.053447146, + -0.18429838, + 0.26444325, + 0.22305855, + 0.029930143, + -0.054558307, + 0.0353794, + -0.0009788725, + -0.1188078, + 0.059667766, + -0.27633518, + 0.046799272, + -0.15642197, + -0.10854975, + 0.106093995, + -0.22275078, + 0.038943406, + -0.03847844, + 0.052943595, + 0.02457092, + -0.02215091, + 0.041665964, + -0.09696136, + -0.02566959, + -0.039870802, + -0.054774225, + 0.02291138, + -0.0417671, + 0.18173029, + -0.03932743, + 0.08204516, + 0.0959776, + -0.20656648, + 0.28138176, + 0.037579037, + 0.20550737, + -0.0048693623, + 0.08957929, + 0.07936084, + 0.064060465, + -0.076720774, + -0.04245319, + 0.1251032, + -0.048257716, + 0.14970464, + -0.019789496, + -0.12023016, + 0.31614852, + 0.048109207, + 0.018149873, + 0.005182097, + -0.051101666, + -0.118794665, + 0.028441736, + 0.024802344, + 0.16004704, + 0.21352114, + -0.0831253, + -0.0066282563, + 0.057000577, + 0.057913885, + 0.03354726, + -0.076870695, + -0.20632678, + 0.02378624, + -0.11414494, + -0.28388187, + -0.24170417, + 0.026353586, + 0.01276097, + -0.20049897, + 0.16650756, + 0.0765534, + -0.023719413, + 0.02053945, + 0.31372947, + 0.16620825, + -0.019555673, + -0.02928945, + -0.060654677, + -0.034009326, + -0.13074087, + -0.086977184, + -0.11527837, + -0.01426738, + 0.019683821, + -0.16352665, + 0.26811838, + 0.20973662, + 0.113176554, + -0.30779162, + 0.04542188, + 0.11580485, + -0.11855776, + -0.20264056, + -0.22960435, + 0.009477502, + 0.1754439, + -0.03466805, + 0.08613063, + -0.009424178, + -0.104562424, + 0.03825546, + -0.20300105, + 0.12507758, + 0.12184145, + 0.140918, + -0.21595697, + -0.1149452, + -0.0863683, + 0.011771966, + -0.0794508, + -0.16730252, + 0.083861485, + -0.073139645, + -0.03015001, + -0.2253941, + -0.2636101, + -0.111474104, + -0.012658355, + 0.12943146, + 0.0024419622, + -0.008086202, + -0.031769402, + 0.22267817, + 0.09031554, + 0.084421515, + 0.07658035, + 0.09284541, + 0.0123357065, + 0.13187145, + -0.021085177 + ], + [ + 0.044147793, + 0.16940491, + -0.07660075, + 0.01839263, + 0.042040065, + -0.047721248, + -0.21912621, + -0.0009365094, + 0.06492754, + 0.042232044, + -0.004531145, + 0.11653948, + -0.006253255, + -0.020997126, + -0.086448684, + 0.065257326, + -0.09350857, + 0.05722746, + -0.09045094, + -0.20966446, + -0.020684095, + 0.05035577, + -0.0058316975, + -0.08649621, + 0.11655366, + -0.035102855, + -0.03805832, + -0.00616778, + -0.013295165, + -0.088805586, + 0.13911676, + -0.1802159, + -0.018650677, + -0.028832428, + -0.15153827, + -0.012014452, + -0.10429693, + -0.02334544, + 0.013291807, + 0.2931698, + 0.25664425, + -0.022222053, + 0.10681528, + -0.02429105, + -0.14298108, + -0.03535962, + -0.20939623, + 0.15250647, + 0.0886579, + -0.08623361, + 0.10814994, + 0.0907847, + 0.17988828, + 0.07620976, + -0.07824574, + -0.014679582, + 0.0817115, + 0.016898714, + -0.28710848, + -0.22659548, + -0.0431887, + 0.035005502, + -0.030766238, + -0.027952312, + 0.00605425, + 0.15439883, + -0.068923496, + 0.0733908, + 0.009504215, + 0.024958564, + 0.14828947, + 0.13880697, + -0.090828896, + 0.003277354, + 0.10241844, + -0.115077004, + -0.1662777, + 0.03727921, + -0.06837462, + 0.07184318, + -0.006519263, + -0.10122025, + -0.14128436, + -0.064970486, + -0.036978073, + 0.010498385, + 0.08859915, + 0.08093246, + -0.06282354, + 0.25009853, + -0.012446843, + -0.121961325, + 0.1972666, + 0.046384234, + 0.199937, + -0.058210306, + -0.062178757, + 0.034782343, + -0.024504201, + -0.10704923, + 0.023402458, + -0.06502446, + -0.15917829, + 0.030689063, + -0.1270888, + 0.06968928, + 0.071537375, + 0.067840576, + 0.19173016, + 0.034167826, + 0.21227396, + 0.05622944, + -0.05703442, + -0.06396645, + 0.100448065, + -0.035514005, + -0.014672414, + -0.037831403, + -0.17126168, + -0.13165945, + -0.12100334, + 0.13692331, + -0.18549083, + -0.09306254, + -0.0049064197, + -0.05379398, + -0.13972266, + 0.20454313, + 0.076266214, + 0.12522706, + -0.05245607, + -0.027636189, + -0.113891274, + -0.12186737, + 0.094089486, + 0.03439381, + -0.06949138, + -0.055920634, + 0.16840659, + 0.0150665045, + 0.08506466, + 0.13945717, + -0.041639358, + -0.06740168, + 0.09963331, + -0.14591189, + 0.059449732, + 0.0025124077, + -0.027144495, + 0.05402788, + 0.20959063, + 0.13937241, + -0.080275685, + 0.038352527, + -0.2715623, + 0.17126106, + -0.091786325, + -0.14857501, + -0.1293791, + -0.18837348, + 0.047239408, + -0.28292495, + 0.0063572284, + 0.09690516, + -0.029995194, + -0.11105718, + -0.018782707, + -0.30740905, + 0.104199484, + 0.15624839, + -0.03974281, + -0.05248947, + 0.038652018, + 0.08289482, + 0.016197314, + 0.119475886, + -0.11403887, + 0.15416206, + -0.08065634, + -0.025980428, + -0.015932493, + -0.008749679, + 0.08308672, + -0.11037272, + -0.048986003, + 0.21674192, + -0.05803945, + 0.04971493, + -0.01891516, + 0.004877139, + -0.17709245, + -0.002087021, + -0.0020960409, + -0.106576554, + -0.007810016, + -0.12720583, + 0.07408409, + -0.02180893, + 0.0007753394, + -0.005287165, + -0.3254629, + -0.0026433906, + 0.04833213, + 0.033785746, + 0.18600549, + 0.0022767857, + 0.18876381, + 0.035533134, + 0.054217387, + 0.03794934, + 0.1271445, + 0.030882537, + 0.025446987, + 0.15345229, + -0.10454748, + 0.26222214, + -0.0091688065, + -0.041628335, + 0.018512165, + 0.0040028314, + 0.20311502, + 0.0037376555, + 0.070144124, + 0.20071377, + 0.11248816, + -0.057424825, + 0.13850467, + 0.18345869, + -0.0044744764, + -0.021348322, + 0.04384846, + -0.022934612, + -0.107814014, + 0.09007019, + 0.02416682, + -0.0012119029, + -0.24780564, + 0.073680155, + -0.04818337, + 0.083472714, + 0.12236319, + -0.02055338, + 0.027101975, + 0.04737765, + -0.110173546, + 0.06364295, + -0.008946159, + 0.15252261, + 0.0027322657, + 0.025759516, + -0.0029763042, + -0.11247266, + 0.13522747, + 0.15886107, + -0.15792294, + -0.07445264, + -0.21184248, + 0.07564331, + -0.053616013, + -0.07139231, + 0.049195178, + -0.084485546, + -0.046140656, + 0.02825539, + 0.021405263, + 0.097813755, + 0.13759468, + 0.0525562, + -0.018069427, + -0.12837838, + 0.077376045, + 0.031107048, + -0.27642173, + -0.13174976, + -0.15429558, + -0.01833851, + 0.027519647, + -0.015669333, + -0.050222356, + 0.02163741, + -0.035224333, + 0.04344882, + -0.10721566, + 0.08589194, + -0.004909294, + -0.032711554, + 0.30826396, + 0.117197126, + 0.124197006, + -0.24572933, + -0.04550309, + -0.12412157, + -0.0724078, + -0.128096, + 0.219507, + 0.1134307, + 0.21780996, + -0.10823206, + 0.11963701, + -0.068851374, + 0.023465648, + -0.01593774, + -0.118752144, + -0.016288111, + -0.030651504, + -0.12503572, + -0.17363529, + 0.19402781, + 0.10475469, + -0.020954344, + 0.019589314, + -0.030481823, + 0.0066996072, + -0.053601895, + -0.20029072, + -0.10582614, + -0.20040359, + -0.0054524555, + -0.055335585, + 0.3311847, + 0.035756826, + -0.049566645, + 0.07918633, + -0.053583127, + -0.09904188, + -0.10038896, + 0.04996928, + -0.099653706, + 0.061264403, + -0.0061791865, + -0.21508417, + 0.116991624, + 0.2686714, + -0.34464577, + -0.039873548, + -0.013331849, + -0.017662382, + 0.18063131, + 0.107568875, + -0.12173686, + -0.059680745, + 0.02161945, + -0.05680074, + -0.017660705, + 0.14479858, + -0.018916478, + 0.049323782, + 0.09538939, + -0.09067543, + -0.049188662, + -0.09265428, + -0.0365639, + 0.0028700577, + 0.059004173, + 0.05948634, + 0.13971259, + 0.18176614, + -0.016404018, + -0.2375804, + 0.02650564, + -0.10457582, + 0.06823804, + -0.044831805, + -0.06590748, + -0.013364427, + 0.014154022, + 0.042477664, + 0.1097775, + -0.11523345, + -0.1079408, + -0.12961785, + 0.34049404, + -0.08560808, + -0.016122337, + -0.052292738, + 0.024983414, + -0.030858947, + 0.13338524, + -0.059871174, + 0.038500845, + 0.37170732, + 0.13350071, + -0.021032728, + -0.07347119, + 0.06407454, + 0.06119444, + 0.0066508185, + 0.10506183, + 0.08963466, + -0.1445016, + 0.01706449, + -0.12059136, + -0.09551309, + 0.09726227, + -0.023860846, + -0.020491932, + 0.11087211, + -0.063431025, + -0.082183585, + -0.0409627, + -0.08496014, + 0.120390706, + -0.17681535, + 0.002092146, + -0.10478897, + 0.08459577, + 0.12726188, + 0.045021284, + -0.005031372, + -0.08353973, + -0.101410136, + 0.019969966, + -0.13681094, + 0.23727098, + -0.11417003, + 0.16008934, + -0.0045129163, + 0.074293375, + -0.20552878, + 0.21652737, + 0.23047388, + 0.14112242, + 0.0013431809, + -0.23968937, + -0.24464427, + 0.091682285, + -0.09774535, + 0.013322989, + 0.08752573, + -0.027159441, + 0.09978838, + 0.19199398, + 0.008760841, + 0.16704173, + -0.10677443, + -0.19106703, + -0.012075937, + -0.13198557, + -0.058435682, + -0.02389443, + 0.23472294, + -0.1088765, + 0.077165045, + -0.24123153, + 0.21209829, + 0.019728398, + -0.15316392, + 0.044813976, + -0.03404352, + -0.08867594, + 0.077576004, + -0.13115795, + 0.27733845, + -0.024984693, + -0.2628973, + -0.008487113, + -0.0053662844, + -0.049809393, + -0.2577814, + -0.12825035, + -0.017539369, + 0.08355548, + 0.04310515, + -0.0396339, + 0.09730884, + 0.09619792, + 0.15564808, + 0.017628962, + -0.1090823, + -0.13700478, + 0.029394176, + 0.091170415, + -0.012858434, + -0.0054733506, + 0.09170598, + -0.113937475, + 0.21517824, + 0.027744953, + -0.20661183, + 0.006941276, + -0.019983957, + -0.23312838, + -0.120168075, + -0.037594784, + -0.004498651, + 0.20085981, + 0.21437016, + -0.009439298, + 0.037814677, + -0.077133186, + 0.17605172, + -0.115668155, + -0.046497446, + 0.044113774, + 0.19949457, + -0.03338365, + 0.05692018, + -0.06910936, + -0.0046057226, + -0.18073574, + -0.07666095, + 0.13720323, + 0.2569485, + -0.035785932, + 0.09387681, + -0.12529814, + 0.05789777, + 0.15947631, + -0.043478936, + -0.034896307, + 0.02509474, + -0.066015504, + -0.17300786, + 0.2740886, + -0.008645607, + 0.20366979, + -0.17741504, + 0.0016051099, + 0.04343355, + 0.053064674, + 0.10233873, + -0.057792492, + 0.28412506, + -0.027595375, + -0.011127119, + 0.23101905, + -0.03437331, + 0.12712002, + -0.040828258, + -0.2521543, + 0.092959955, + 0.1476991, + -0.016959714, + 0.24718697, + -0.08578792, + -0.042732067, + -0.02364189, + 0.038429204, + 0.07348472, + 0.041489173, + -0.11468599, + -0.049400177, + -0.027466021, + -0.009244471, + -0.017509859, + 0.08712863, + -0.022377755, + 0.01853816, + 0.042881876, + 0.3835641, + 0.0874544, + -0.068334155, + 0.057505295, + -0.09062925, + 0.057019502, + -0.1344657, + 0.08132518, + 0.000053040872, + -0.059475925, + -0.12508613, + -0.021813422, + -0.013378662, + 0.024890503, + 0.024216384, + -0.077127144, + 0.007335973, + 0.08234029, + -0.18302993, + -0.035066325, + -0.0072978884, + 0.050056715, + -0.22931813, + 0.09650683, + -0.0073624705, + -0.2885135, + -0.10775086, + -0.23143713, + -0.086137205, + 0.0498812, + -0.033241358, + 0.01776586, + -0.026235383, + 0.19309425, + 0.025691753, + -0.058075365, + -0.2405673, + -0.21786085, + -0.19851258, + 0.046656214, + -0.10293365, + -0.10813984, + -0.2152836, + 0.15489893, + 0.047911253, + 0.20260984, + 0.2088183, + -0.1018085, + -0.087688275, + 0.13801338, + 0.0046194997, + -0.1746844, + 0.004021364, + -0.07318715, + -0.24332349, + -0.32633016, + 0.055268224, + -0.013017414, + 0.052299757, + 0.1270179, + 0.004340157, + 0.057317145, + 0.21436898, + 0.06542589, + 0.06742579, + -0.112878166, + -0.020031923, + -0.04590336, + -0.2186622, + -0.11536247, + 0.03292602, + -0.037806716, + 0.11775531, + -0.08126202, + 0.044630688, + 0.004407897, + 0.10470092, + 0.106795885, + -0.02859561, + 0.024228014, + -0.03555758, + 0.14519277, + 0.14093313, + 0.26003087, + 0.0070421537, + 0.08488326, + -0.12916775, + -0.1219826, + -0.2455656, + 0.15563421, + 0.08667417, + 0.01301937, + -0.0907239, + -0.12462255, + -0.032354463, + 0.20714861, + 0.0010497222, + -0.13925967, + 0.03371706, + -0.0062380685, + 0.055929244, + -0.067029044, + 0.059437487, + 0.0076191374, + 0.05578321, + 0.011333173, + -0.05390713, + 0.15975079, + 0.072687715, + 0.15916237, + 0.17636605, + -0.008446719, + 0.014961095, + 0.08686759, + 0.14848803, + -0.22238462, + 0.09633494, + -0.055627063, + 0.12631376, + 0.035983652, + 0.22738391, + -0.11332371, + 0.21841496, + -0.17675073, + 0.14812225, + -0.114016265, + -0.032317907, + -0.18767738, + -0.0811073, + -0.13753203, + 0.16913415, + 0.0114353, + -0.11488439, + 0.011230222, + -0.085470885, + 0.0690153, + -0.10461269, + -0.0706403, + -0.20100483, + -0.037651025, + 0.08529463, + -0.13536386, + -0.034356836, + 0.057857174, + -0.03706257, + 0.02483604, + -0.24963166, + -0.06943555, + -0.09979882, + 0.0942273, + 0.05437428, + -0.10782203, + -0.013504401, + 0.24189045, + -0.13296025, + 0.03774331, + -0.010911639, + -0.082099736, + 0.07280296, + 0.06079229, + 0.02287312, + 0.059112385, + -0.099384256, + -0.076120414, + 0.11072304, + -0.05123164, + 0.1079739, + 0.10227008, + -0.05471506, + 0.10747948, + 0.009699896, + -0.30863142, + -0.04430503, + 0.37351134, + -0.19678722, + 0.13742797, + -0.40235984, + -0.27581173, + 0.3702671, + -0.024754377, + 0.122642905, + -0.14828043, + -0.07008751, + 0.08360051, + 0.0035450035, + 0.0216809, + -0.12010145, + -0.026985122, + 0.053328913, + -0.10591583, + -0.013924753, + 0.038145293, + 0.17823228, + 0.06572483, + -0.28199768, + 0.14622784, + 0.10302125, + -0.10776388, + 0.16953847, + -0.043120794, + -0.05191318, + 0.03779815, + 0.12021654, + -0.0014165355, + 0.023631074, + -0.2394369, + 0.114601746, + 0.055467404, + 0.16078594, + 0.009649481, + -0.21886353, + -0.10147424, + 0.01851072, + 0.13010912, + -0.18243171, + -0.046989094, + 0.11791553, + -0.07706595, + 0.052286703, + -0.19268094, + -0.06734655, + 0.083558746, + 0.05510528, + 0.1205322, + 0.043853316, + 0.19419815, + -0.19383074, + -0.12088962, + 0.050363027, + 0.13108388, + -0.032544736, + -0.19773011, + -0.034516454, + -0.13052338, + -0.0046362486, + 0.0320051, + 0.061747782, + -0.19046386, + 0.09733934, + 0.08370847, + -0.120636776, + -0.051564053, + -0.08520327, + 0.025040282, + -0.047831807, + 0.13498239, + -0.05799306, + -0.082822345, + 0.11464385, + -0.11085481, + 0.11331335, + 0.06106606, + 0.12057433, + -0.1630841, + 0.00096572586, + -0.039613742, + 0.16483565, + 0.17725827, + 0.024782866, + -0.02095867, + -0.30374828, + 0.030041676, + -0.34487057, + 0.016318211, + 0.010022732, + 0.16049328, + -0.059585165, + -0.038909353, + 0.14602326, + 0.003398922, + 0.10123426, + 0.10189341, + 0.0013612165, + -0.095394656, + 0.17065822, + 0.2717381, + -0.048805684, + 0.11726433, + -0.22115687, + -0.013294891, + -0.20146243, + 0.1301142, + -0.079273745, + -0.06366891, + 0.20852767, + -0.11919791, + 0.10993831, + -0.1328157, + 0.023110492, + 0.06534299, + -0.028473591, + 0.03133091, + 0.18335237, + -0.12560268, + 0.05174967, + 0.10752264, + 0.026674548, + -0.116931275, + 0.05302942, + 0.14527842, + 0.04175609, + 0.07524169, + -0.016850159, + -0.085036226, + 0.14378034, + -0.15384178, + 0.06842327, + 0.072759815, + 0.0333774, + -0.07724679, + 0.07146305, + -0.0038083985, + -0.023200184, + -0.21678805, + -0.05851518, + -0.2106472, + 0.21905945, + 0.044797596, + -0.08767725, + -0.06757387, + -0.08139683, + 0.060507063, + -0.2539122, + -0.054056864, + -0.0061713024, + -0.25016263, + -0.013922703, + 0.025297, + -0.031502247, + 0.043440044, + 0.049890436, + 0.10434008, + -0.14499368, + 0.011150854, + -0.16495703, + -0.21740839, + 0.004718739, + 0.048284378, + -0.047843166, + 0.047392353, + 0.10500571, + 0.031224746, + 0.15428485, + 0.03575991, + 0.23791198, + -0.036760766, + -0.21849649, + -0.14954391, + 0.12186352, + -0.030245116, + -0.014910633, + 0.052947998, + 0.07771576, + 0.22827105, + -0.025724977, + 0.13252845, + 0.15444413, + -0.09818237, + 0.031483896, + -0.10577348, + -0.00006837226, + -0.109357946, + 0.07215483, + 0.17097427, + 0.11899255, + 0.061613396, + 0.03566328, + 0.036564983, + 0.0024872916, + 0.06796512, + 0.026589885, + -0.060742363, + -0.07839288, + 0.09555347, + -0.050860837, + -0.12879173, + -0.1040309, + 0.21770604, + 0.0018934173, + 0.023243189, + -0.090174794, + -0.06827122, + 0.15387972, + -0.069496565, + 0.024794634, + -0.051390123, + 0.027016103, + -0.021958428, + -0.10045327, + 0.18993512, + -0.22860111, + -0.10041514, + -0.1531901, + 0.005921122, + -0.03854943, + 0.14580593, + -0.12484452, + 0.022898167, + -0.10113201, + 0.08594769, + 0.015126121, + -0.012950152, + 0.040228955, + -0.09867645, + 0.0058024707, + 0.1267348, + 0.19757093, + 0.01170412, + -0.17352274, + 0.30352616, + 0.3830606, + 0.05377779, + -0.07670585, + 0.025823055, + -0.05698721, + -0.009363706, + 0.077189244, + -0.15372942, + -0.29238287, + -0.25004295, + 0.024756031, + 0.04304348, + 0.081523225, + -0.07263159, + -0.13542518, + 0.08482435, + -0.06367541, + 0.01787075, + -0.027442627, + 0.13191804, + -0.25955498, + -0.2185974, + -0.0048445812, + 0.24410242, + 0.052365653, + 0.13663675, + -0.13785504, + 0.12467762, + 0.2333146, + -0.057672247, + -0.0937421, + 0.018837918, + -0.058080558, + -0.019427018, + 0.13319813, + 0.0118216975, + 0.09013087, + -0.06453432, + -0.06437333, + 0.0037089002, + 0.09036531, + -0.0066115037, + -0.016081506, + -0.1324408, + 0.075796016, + 0.031593855, + -0.15255858, + -0.12663953, + 0.17544357, + 0.060190212, + 0.24437538, + 0.089556195, + 0.20744215, + -0.05519144, + 0.13434736, + 0.13900068, + -0.07795903, + -0.05554333, + 0.010776608, + 0.048118893, + -0.06867825, + -0.1597975, + 0.038932495, + -0.09650621, + -0.07258938 + ] + ], + "bias": [ + -0.08385399, + -0.04598274, + 0.12983686 + ], + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.22 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.json new file mode 100644 index 0000000..7edaaa3 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.json @@ -0,0 +1,1942 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.5, + "tools_margin_threshold": 0.0, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.22 + }, + "metrics": { + "total": 319, + "accuracy": 0.6708464, + "confusion": [ + [ + 58, + 57, + 34 + ], + [ + 0, + 100, + 0 + ], + [ + 1, + 13, + 56 + ] + ], + "tool_precision": 0.9830508, + "tool_recall": 0.38926175, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.014285714, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22568056, + "p_memory": 0.40084177, + "p_abstain": 0.37347767 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31395635, + "p_memory": 0.33680987, + "p_abstain": 0.3492338 + }, + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "shell_test_002", + "text": "what's the current working directory of the daemon process?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39770705, + "p_memory": 0.19950679, + "p_abstain": 0.40278614 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.26465896, + "p_memory": 0.34730524, + "p_abstain": 0.38803577 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25941908, + "p_memory": 0.12832387, + "p_abstain": 0.612257 + }, + { + "query_id": "shell_test_006", + "text": "what's the current git log — last 3 commits?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.27416888, + "p_memory": 0.10814477, + "p_abstain": 0.61768633 + }, + { + "query_id": "shell_test_007", + "text": "what environment variable is RUST_LOG set to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40384528, + "p_memory": 0.31830305, + "p_abstain": 0.27785167 + }, + { + "query_id": "shell_test_008", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "shell_test_009", + "text": "rerankerのプロセスは現在いくつ走ってますか", + "gold": "tools", + "pred": "memory", + "p_tools": 0.32050437, + "p_memory": 0.27518332, + "p_abstain": 0.40431228 + }, + { + "query_id": "shell_test_012", + "text": "any nix builds running in background rn?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37631887, + "p_memory": 0.241377, + "p_abstain": 0.38230407 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.30472597, + "p_memory": 0.32799238, + "p_abstain": 0.36728162 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.27784234, + "p_memory": 0.32759824, + "p_abstain": 0.3945594 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2540235, + "p_memory": 0.16229898, + "p_abstain": 0.58367753 + }, + { + "query_id": "shell_test_018", + "text": "what processes are eating the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.30672368, + "p_memory": 0.19407696, + "p_abstain": 0.49919936 + }, + { + "query_id": "shell_test_019", + "text": "is the klbr tui currently connected to the daemon socket?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48271036, + "p_memory": 0.17951614, + "p_abstain": 0.3377736 + }, + { + "query_id": "shell_test_020", + "text": "whats the total vram on ths gpu", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43741146, + "p_memory": 0.24507794, + "p_abstain": 0.3175106 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4158976, + "p_memory": 0.27515078, + "p_abstain": 0.3089516 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "shell_test_026", + "text": "ls the benchmarks output dirs that exist right now", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47624564, + "p_memory": 0.14048539, + "p_abstain": 0.38326898 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "shell_test_028", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "shell_test_029", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "shell_test_032", + "text": "run lsof -i :8001 and tell me the pid", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.48442715, + "p_memory": 0.10436712, + "p_abstain": 0.4112057 + }, + { + "query_id": "rf_test_001", + "text": "what compaction_keep default is in config.rs right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48437816, + "p_memory": 0.2835406, + "p_abstain": 0.23208125 + }, + { + "query_id": "rf_test_003", + "text": "what does the tools mod.rs export?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44654188, + "p_memory": 0.26933104, + "p_abstain": 0.2841271 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39678445, + "p_memory": 0.44244283, + "p_abstain": 0.16077271 + }, + { + "query_id": "rf_test_005", + "text": "how many queries are in internal_eval_starter.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4628093, + "p_memory": 0.270529, + "p_abstain": 0.26666173 + }, + { + "query_id": "rf_test_008", + "text": "what's the current benchmark README say about the retrieval command?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.34852275, + "p_memory": 0.27711794, + "p_abstain": 0.3743593 + }, + { + "query_id": "rf_test_012", + "text": "how many memories does internal_eval_passive_recall_v3.json have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47214958, + "p_memory": 0.29806998, + "p_abstain": 0.22978038 + }, + { + "query_id": "rf_test_014", + "text": "does flake.nix pin any specific nixpkgs revision?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40133312, + "p_memory": 0.2407558, + "p_abstain": 0.35791105 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.36721316, + "p_memory": 0.37156385, + "p_abstain": 0.26122305 + }, + { + "query_id": "rf_test_019", + "text": "what does the recall.rs tool do with the tag_and param?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38931295, + "p_memory": 0.2874001, + "p_abstain": 0.32328695 + }, + { + "query_id": "rf_test_021", + "text": "llm.rsのstreamメソッドのシグネチャは?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4190756, + "p_memory": 0.25449738, + "p_abstain": 0.32642704 + }, + { + "query_id": "rf_test_022", + "text": "what's the current threshold in the router_model.json?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42050397, + "p_memory": 0.2790245, + "p_abstain": 0.30047154 + }, + { + "query_id": "rf_test_023", + "text": "look at flake.nix and tell me what packages are in the dev shell", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4390786, + "p_memory": 0.13317992, + "p_abstain": 0.4277415 + }, + { + "query_id": "rf_test_024", + "text": "how many memories does the passive recall v3 dataset have?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.44003776, + "p_memory": 0.31567433, + "p_abstain": 0.2442879 + }, + { + "query_id": "rf_test_025", + "text": "read write_file.rs tool and show me what it does", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47923937, + "p_memory": 0.13611977, + "p_abstain": 0.38464084 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.31453353, + "p_memory": 0.26278618, + "p_abstain": 0.42268026 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.46307096, + "p_memory": 0.38064674, + "p_abstain": 0.1562823 + }, + { + "query_id": "rf_test_030", + "text": "look at out-router-iter-1/router_report.json and give me the accuracy", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4797326, + "p_memory": 0.1527195, + "p_abstain": 0.367548 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43717074, + "p_memory": 0.29522896, + "p_abstain": 0.2676003 + }, + { + "query_id": "wf_test_003", + "text": "create a new slice file with 5 tool-needed examples", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39453, + "p_memory": 0.24812809, + "p_abstain": 0.35734192 + }, + { + "query_id": "wf_test_004", + "text": "change the embed_url to localhost:8005 in the config", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4571177, + "p_memory": 0.2604581, + "p_abstain": 0.2824242 + }, + { + "query_id": "wf_test_005", + "text": "write a tiny benchmarking summary to out-router-iter-2/notes.txt", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45665768, + "p_memory": 0.25635234, + "p_abstain": 0.2869899 + }, + { + "query_id": "wf_test_006", + "text": "add router_model_path to the default config and set it to a path", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38890406, + "p_memory": 0.31626257, + "p_abstain": 0.29483342 + }, + { + "query_id": "wf_test_007", + "text": "ログファイルに今のステータスを書き込んで", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3217656, + "p_memory": 0.11337318, + "p_abstain": 0.56486124 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43858615, + "p_memory": 0.21885185, + "p_abstain": 0.34256205 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41309485, + "p_memory": 0.22376072, + "p_abstain": 0.36314443 + }, + { + "query_id": "wf_test_011", + "text": "save a note about the router precision gate to docs/", + "gold": "tools", + "pred": "memory", + "p_tools": 0.43877676, + "p_memory": 0.21598269, + "p_abstain": 0.34524047 + }, + { + "query_id": "wf_test_012", + "text": "create a quick script to run all benchmarks in sequence", + "gold": "tools", + "pred": "memory", + "p_tools": 0.35300845, + "p_memory": 0.21918875, + "p_abstain": 0.42780283 + }, + { + "query_id": "wf_test_013", + "text": "add a line to .gitignore to exclude agent.db", + "gold": "tools", + "pred": "memory", + "p_tools": 0.41591418, + "p_memory": 0.28971443, + "p_abstain": 0.29437143 + }, + { + "query_id": "wf_test_014", + "text": "新しいスライスJSONを作ってmemory_laneのクエリを10個追加して", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3736646, + "p_memory": 0.23679724, + "p_abstain": 0.38953817 + }, + { + "query_id": "wf_test_015", + "text": "bump the rerank_top_k to 15 in the calibrated config and save it", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39532402, + "p_memory": 0.28937247, + "p_abstain": 0.31530353 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.30211675, + "p_memory": 0.3692885, + "p_abstain": 0.3285947 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5483045, + "p_memory": 0.18406397, + "p_abstain": 0.26763153 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.34390134, + "p_memory": 0.4241181, + "p_abstain": 0.23198053 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.40889636, + "p_memory": 0.30667284, + "p_abstain": 0.2844308 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.74444443, + "confusion": [ + [ + 34, + 16, + 20 + ], + [ + 0, + 55, + 0 + ], + [ + 0, + 10, + 45 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.4857143, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + }, + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.76666665, + "confusion": [ + [ + 5, + 3, + 2 + ], + [ + 0, + 10, + 0 + ], + [ + 0, + 9, + 31 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3293323, + "p_memory": 0.27011913, + "p_abstain": 0.4005486 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.20632608, + "p_memory": 0.31610098, + "p_abstain": 0.47757295 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.15718141, + "p_memory": 0.41085818, + "p_abstain": 0.43196037 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.24870655, + "p_memory": 0.30906734, + "p_abstain": 0.44222614 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.16127765, + "p_memory": 0.36582443, + "p_abstain": 0.4728979 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3315131, + "p_memory": 0.37729093, + "p_abstain": 0.291196 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3384025, + "p_memory": 0.33527344, + "p_abstain": 0.32632402 + }, + { + "query_id": "holdout_abstain_toolish_abstain_039", + "text": "check the latest Rust RFCs and tell me the most controversial one", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.3367044, + "p_memory": 0.22700824, + "p_abstain": 0.4362874 + }, + { + "query_id": "holdout_abstain_toolish_tools_041", + "text": "is anything listening on :8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4250079, + "p_memory": 0.121281445, + "p_abstain": 0.45371065 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45107943, + "p_memory": 0.29448316, + "p_abstain": 0.2544374 + }, + { + "query_id": "holdout_abstain_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_abstain_toolish_tools_047", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_abstain_toolish_tools_050", + "text": "is there a socket file at the agent.sock path?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49101555, + "p_memory": 0.17384227, + "p_abstain": 0.3351422 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.9, + "confusion": [ + [ + 5, + 2, + 3 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 1, + 9 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.5, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_tools_041", + "text": "is anything listening on port 8001 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.42537102, + "p_memory": 0.1365817, + "p_abstain": 0.43804723 + }, + { + "query_id": "holdout_memory_toolish_tools_042", + "text": "what processes are listening on port 8002 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47191605, + "p_memory": 0.13206743, + "p_abstain": 0.39601654 + }, + { + "query_id": "holdout_memory_toolish_tools_045", + "text": "write a file docs/router_holdout_memory_toolish.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49561423, + "p_memory": 0.26819563, + "p_abstain": 0.23619016 + }, + { + "query_id": "holdout_memory_toolish_tools_046", + "text": "今メモリ使用量はどれくらい?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.37996158, + "p_memory": 0.2160978, + "p_abstain": 0.40394065 + }, + { + "query_id": "holdout_memory_toolish_tools_048", + "text": "what's the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34521475, + "p_memory": 0.20280616, + "p_abstain": 0.45197904 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.38640097, + "p_memory": 0.21567707, + "p_abstain": 0.397922 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.56666666, + "confusion": [ + [ + 24, + 11, + 15 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 0, + 5 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.48, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_001", + "text": "is anything listening on port 8003 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.45737526, + "p_memory": 0.13785496, + "p_abstain": 0.4047698 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.25566024, + "p_memory": 0.16644181, + "p_abstain": 0.5778979 + }, + { + "query_id": "holdout_tools_recall_tools_004", + "text": "is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.4000911, + "p_memory": 0.2770236, + "p_abstain": 0.32288533 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.244283, + "p_memory": 0.14806087, + "p_abstain": 0.6076561 + }, + { + "query_id": "holdout_tools_recall_tools_007", + "text": "what processes are using the most cpu right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.2989278, + "p_memory": 0.16969262, + "p_abstain": 0.5313796 + }, + { + "query_id": "holdout_tools_recall_tools_008", + "text": "check if the reranker http endpoint on localhost:8003 is responding", + "gold": "tools", + "pred": "memory", + "p_tools": 0.3766123, + "p_memory": 0.30566308, + "p_abstain": 0.31772462 + }, + { + "query_id": "holdout_tools_recall_tools_010", + "text": "is there a unix socket at the agent.sock path right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.35811168, + "p_memory": 0.18766385, + "p_abstain": 0.4542245 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40720624, + "p_memory": 0.36517468, + "p_abstain": 0.2276191 + }, + { + "query_id": "holdout_tools_recall_tools_016", + "text": "write a file docs/router_holdout_tools_recall.txt with the content \"ok\"", + "gold": "tools", + "pred": "memory", + "p_tools": 0.48336956, + "p_memory": 0.25243908, + "p_abstain": 0.26419142 + }, + { + "query_id": "holdout_tools_recall_tools_019", + "text": "今ポート8002は開いてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.38039514, + "p_memory": 0.18013602, + "p_abstain": 0.43946886 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.21773067, + "p_memory": 0.11778352, + "p_abstain": 0.6644858 + }, + { + "query_id": "holdout_tools_recall_tools_022", + "text": "read docs/router_dataset_log.md and tell me what the latest recommended model path is", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47416094, + "p_memory": 0.16149382, + "p_abstain": 0.36434525 + }, + { + "query_id": "holdout_tools_recall_tools_025", + "text": "what's the size of the current target/ directory?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.45230028, + "p_memory": 0.23328565, + "p_abstain": 0.31441408 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.39562425, + "p_memory": 0.24721095, + "p_abstain": 0.3571648 + }, + { + "query_id": "holdout_tools_recall_tools_029", + "text": "is there a process listening on :1234 right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3495041, + "p_memory": 0.12701918, + "p_abstain": 0.52347666 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.29273054, + "p_memory": 0.1387474, + "p_abstain": 0.56852204 + }, + { + "query_id": "holdout_tools_recall_tools_036", + "text": "show the last 10 lines of dmesg", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.44959724, + "p_memory": 0.14107294, + "p_abstain": 0.4093298 + }, + { + "query_id": "holdout_tools_recall_tools_037", + "text": "check dmesg for any oom killer events in the last boot", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.37677738, + "p_memory": 0.115129665, + "p_abstain": 0.50809294 + }, + { + "query_id": "holdout_tools_recall_tools_038", + "text": "is the disk partition containing /home close to full right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.36662707, + "p_memory": 0.20307499, + "p_abstain": 0.4302979 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.28102198, + "p_memory": 0.25146824, + "p_abstain": 0.46750975 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.26465866, + "p_memory": 0.18487725, + "p_abstain": 0.5504641 + }, + { + "query_id": "holdout_tools_recall_tools_041", + "text": "what environment variable is RUST_LOG set to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.38777393, + "p_memory": 0.26630807, + "p_abstain": 0.34591803 + }, + { + "query_id": "holdout_tools_recall_tools_042", + "text": "how many cpu cores are available right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.3325027, + "p_memory": 0.1780698, + "p_abstain": 0.48942754 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.22146691, + "p_memory": 0.35134003, + "p_abstain": 0.4271931 + }, + { + "query_id": "holdout_tools_recall_tools_044", + "text": "what is the current system load average?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.34848648, + "p_memory": 0.20945789, + "p_abstain": 0.4420556 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.40048406, + "p_memory": 0.36867175, + "p_abstain": 0.23084415 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.md new file mode 100644 index 0000000..d73cbbe --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-7/router_report.md @@ -0,0 +1,159 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.500` +- tools_margin_threshold: `0.000` +- abstain_prob_threshold: `0.100` +- abstain_margin_threshold: `0.220` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.6708` +- tools precision: `0.9831` +- tools recall: `0.3893` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0143` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 58 | 57 | 34 | +| memory | 0 | 100 | 0 | +| abstain | 1 | 13 | 56 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.23 | 0.40 | 0.37 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.31 | 0.34 | 0.35 | +| `holdout_tools_recall_tools_001` | is anything listening on port 8003 right now? | `Tools` | `Abstain` | 0.46 | 0.14 | 0.40 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `holdout_tools_recall_tools_004` | is port 8002 bound to 127.0.0.1 or 0.0.0.0 right now? | `Tools` | `Memory` | 0.40 | 0.28 | 0.32 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `holdout_tools_recall_tools_007` | what processes are using the most cpu right now? | `Tools` | `Abstain` | 0.30 | 0.17 | 0.53 | +| `holdout_tools_recall_tools_008` | check if the reranker http endpoint on localhost:8003 is responding | `Tools` | `Memory` | 0.38 | 0.31 | 0.32 | +| `holdout_tools_recall_tools_010` | is there a unix socket at the agent.sock path right now? | `Tools` | `Abstain` | 0.36 | 0.19 | 0.45 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Tools` | `Memory` | 0.41 | 0.37 | 0.23 | +| `holdout_tools_recall_tools_016` | write a file docs/router_holdout_tools_recall.txt with the content "ok" | `Tools` | `Memory` | 0.48 | 0.25 | 0.26 | +| `holdout_tools_recall_tools_019` | 今ポート8002は開いてる? | `Tools` | `Abstain` | 0.38 | 0.18 | 0.44 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `holdout_tools_recall_tools_022` | read docs/router_dataset_log.md and tell me what the latest recommended model path is | `Tools` | `Memory` | 0.47 | 0.16 | 0.36 | +| `holdout_tools_recall_tools_025` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.40 | 0.25 | 0.36 | +| `holdout_tools_recall_tools_029` | is there a process listening on :1234 right now? | `Tools` | `Abstain` | 0.35 | 0.13 | 0.52 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Abstain` | 0.29 | 0.14 | 0.57 | +| `holdout_tools_recall_tools_036` | show the last 10 lines of dmesg | `Tools` | `Abstain` | 0.45 | 0.14 | 0.41 | +| `holdout_tools_recall_tools_037` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `holdout_tools_recall_tools_038` | is the disk partition containing /home close to full right now? | `Tools` | `Abstain` | 0.37 | 0.20 | 0.43 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.28 | 0.25 | 0.47 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `holdout_tools_recall_tools_041` | what environment variable is RUST_LOG set to right now? | `Tools` | `Memory` | 0.39 | 0.27 | 0.35 | +| `holdout_tools_recall_tools_042` | how many cpu cores are available right now? | `Tools` | `Abstain` | 0.33 | 0.18 | 0.49 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.22 | 0.35 | 0.43 | +| `holdout_tools_recall_tools_044` | what is the current system load average? | `Tools` | `Abstain` | 0.35 | 0.21 | 0.44 | +| `holdout_tools_recall_tools_058` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.40 | 0.37 | 0.23 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | 0.33 | 0.27 | 0.40 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.21 | 0.32 | 0.48 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.16 | 0.41 | 0.43 | +| `holdout_abstain_toolish_abstain_029` | vim or emacs? choose for me | `Abstain` | `Memory` | 0.25 | 0.31 | 0.44 | +| `holdout_abstain_toolish_abstain_030` | is gemma 4 better than llama 3.1? check online and decide | `Abstain` | `Memory` | 0.16 | 0.37 | 0.47 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | 0.33 | 0.38 | 0.29 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | 0.34 | 0.34 | 0.33 | +| `holdout_abstain_toolish_abstain_039` | check the latest Rust RFCs and tell me the most controversial one | `Abstain` | `Memory` | 0.34 | 0.23 | 0.44 | +| `holdout_abstain_toolish_tools_041` | is anything listening on :8001 right now? | `Tools` | `Abstain` | 0.43 | 0.12 | 0.45 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.45 | 0.29 | 0.25 | +| `holdout_abstain_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_abstain_toolish_tools_047` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_abstain_toolish_tools_050` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `holdout_memory_toolish_tools_041` | is anything listening on port 8001 right now? | `Tools` | `Abstain` | 0.43 | 0.14 | 0.44 | +| `holdout_memory_toolish_tools_042` | what processes are listening on port 8002 right now? | `Tools` | `Abstain` | 0.47 | 0.13 | 0.40 | +| `holdout_memory_toolish_tools_045` | write a file docs/router_holdout_memory_toolish.txt with the content "ok" | `Tools` | `Memory` | 0.50 | 0.27 | 0.24 | +| `holdout_memory_toolish_tools_046` | 今メモリ使用量はどれくらい? | `Tools` | `Memory` | 0.38 | 0.22 | 0.40 | +| `holdout_memory_toolish_tools_048` | what's the current system load average? | `Tools` | `Abstain` | 0.35 | 0.20 | 0.45 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.39 | 0.22 | 0.40 | +| `shell_test_002` | what's the current working directory of the daemon process? | `Tools` | `Memory` | 0.40 | 0.20 | 0.40 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Memory` | 0.26 | 0.35 | 0.39 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.26 | 0.13 | 0.61 | +| `shell_test_006` | what's the current git log — last 3 commits? | `Tools` | `Abstain` | 0.27 | 0.11 | 0.62 | +| `shell_test_007` | what environment variable is RUST_LOG set to? | `Tools` | `Memory` | 0.40 | 0.32 | 0.28 | +| `shell_test_008` | is there a socket file at the agent.sock path? | `Tools` | `Memory` | 0.49 | 0.17 | 0.34 | +| `shell_test_009` | rerankerのプロセスは現在いくつ走ってますか | `Tools` | `Memory` | 0.32 | 0.28 | 0.40 | +| `shell_test_012` | any nix builds running in background rn? | `Tools` | `Memory` | 0.38 | 0.24 | 0.38 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.26 | 0.17 | 0.58 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.30 | 0.33 | 0.37 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.28 | 0.33 | 0.39 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.25 | 0.16 | 0.58 | +| `shell_test_018` | what processes are eating the most cpu right now? | `Tools` | `Abstain` | 0.31 | 0.19 | 0.50 | +| `shell_test_019` | is the klbr tui currently connected to the daemon socket? | `Tools` | `Memory` | 0.48 | 0.18 | 0.34 | +| `shell_test_020` | whats the total vram on ths gpu | `Tools` | `Memory` | 0.44 | 0.25 | 0.32 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.42 | 0.28 | 0.31 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.22 | 0.12 | 0.66 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.24 | 0.15 | 0.61 | +| `shell_test_026` | ls the benchmarks output dirs that exist right now | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.26 | 0.18 | 0.55 | +| `shell_test_028` | what's the size of the current target/ directory? | `Tools` | `Memory` | 0.45 | 0.23 | 0.31 | +| `shell_test_029` | check dmesg for any oom killer events in the last boot | `Tools` | `Abstain` | 0.38 | 0.12 | 0.51 | +| `shell_test_032` | run lsof -i :8001 and tell me the pid | `Tools` | `Abstain` | 0.48 | 0.10 | 0.41 | +| `rf_test_001` | what compaction_keep default is in config.rs right now? | `Tools` | `Memory` | 0.48 | 0.28 | 0.23 | +| `rf_test_003` | what does the tools mod.rs export? | `Tools` | `Memory` | 0.45 | 0.27 | 0.28 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.40 | 0.44 | 0.16 | +| `rf_test_005` | how many queries are in internal_eval_starter.json? | `Tools` | `Memory` | 0.46 | 0.27 | 0.27 | +| `rf_test_008` | what's the current benchmark README say about the retrieval command? | `Tools` | `Memory` | 0.35 | 0.28 | 0.37 | +| `rf_test_012` | how many memories does internal_eval_passive_recall_v3.json have? | `Tools` | `Memory` | 0.47 | 0.30 | 0.23 | +| `rf_test_014` | does flake.nix pin any specific nixpkgs revision? | `Tools` | `Memory` | 0.40 | 0.24 | 0.36 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.37 | 0.37 | 0.26 | +| `rf_test_019` | what does the recall.rs tool do with the tag_and param? | `Tools` | `Memory` | 0.39 | 0.29 | 0.32 | +| `rf_test_021` | llm.rsのstreamメソッドのシグネチャは? | `Tools` | `Memory` | 0.42 | 0.25 | 0.33 | +| `rf_test_022` | what's the current threshold in the router_model.json? | `Tools` | `Memory` | 0.42 | 0.28 | 0.30 | +| `rf_test_023` | look at flake.nix and tell me what packages are in the dev shell | `Tools` | `Abstain` | 0.44 | 0.13 | 0.43 | +| `rf_test_024` | how many memories does the passive recall v3 dataset have? | `Tools` | `Memory` | 0.44 | 0.32 | 0.24 | +| `rf_test_025` | read write_file.rs tool and show me what it does | `Tools` | `Abstain` | 0.48 | 0.14 | 0.38 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.31 | 0.26 | 0.42 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.46 | 0.38 | 0.16 | +| `rf_test_030` | look at out-router-iter-1/router_report.json and give me the accuracy | `Tools` | `Memory` | 0.48 | 0.15 | 0.37 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.44 | 0.30 | 0.27 | +| `wf_test_003` | create a new slice file with 5 tool-needed examples | `Tools` | `Memory` | 0.39 | 0.25 | 0.36 | +| `wf_test_004` | change the embed_url to localhost:8005 in the config | `Tools` | `Memory` | 0.46 | 0.26 | 0.28 | +| `wf_test_005` | write a tiny benchmarking summary to out-router-iter-2/notes.txt | `Tools` | `Memory` | 0.46 | 0.26 | 0.29 | +| `wf_test_006` | add router_model_path to the default config and set it to a path | `Tools` | `Memory` | 0.39 | 0.32 | 0.29 | +| `wf_test_007` | ログファイルに今のステータスを書き込んで | `Tools` | `Abstain` | 0.32 | 0.11 | 0.56 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.44 | 0.22 | 0.34 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.41 | 0.22 | 0.36 | +| `wf_test_011` | save a note about the router precision gate to docs/ | `Tools` | `Memory` | 0.44 | 0.22 | 0.35 | +| `wf_test_012` | create a quick script to run all benchmarks in sequence | `Tools` | `Memory` | 0.35 | 0.22 | 0.43 | +| `wf_test_013` | add a line to .gitignore to exclude agent.db | `Tools` | `Memory` | 0.42 | 0.29 | 0.29 | +| `wf_test_014` | 新しいスライスJSONを作ってmemory_laneのクエリを10個追加して | `Tools` | `Memory` | 0.37 | 0.24 | 0.39 | +| `wf_test_015` | bump the rerank_top_k to 15 in the calibrated config and save it | `Tools` | `Memory` | 0.40 | 0.29 | 0.32 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | 0.30 | 0.37 | 0.33 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.55 | 0.18 | 0.27 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.34 | 0.42 | 0.23 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.41 | 0.31 | 0.28 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `1.0000` +- tools recall: `0.4857` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0000` +- accuracy: `0.7444` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.7667 | +| `memory_toolish` | 60 | 1.0000 | 0.5000 | 0.0000 | 0.0000 | 0.9000 | +| `tools_recall` | 60 | 1.0000 | 0.4800 | 0.0000 | 0.0000 | 0.5667 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-8/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-8/router_model.json new file mode 100644 index 0000000..2e79244 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-8/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.16365053, + 0.04022075, + 0.04759608, + -0.006209037, + -0.11569602, + 0.23925468, + 0.15486912, + -0.12437834, + -0.17879783, + -0.03070053, + 0.0642034, + -0.20788309, + 0.01148989, + 0.17111485, + 0.06664075, + -0.086574174, + -0.07566316, + 0.05735904, + -0.026882106, + -0.18257523, + 0.054434884, + 0.0094715655, + -0.0705016, + 0.04925223, + 0.07341147, + -0.031009221, + -0.025780225, + -0.12587023, + 0.14231014, + 0.271877, + 0.013002855, + -0.27295598, + -0.107738435, + -0.061303608, + 0.060557008, + 0.03550173, + 0.07674463, + 0.064899415, + -0.143617, + 0.10915709, + -0.15758413, + -0.086143486, + -0.09755287, + -0.10708224, + -0.10294328, + 0.101485915, + -0.033768248, + -0.06302843, + 0.004469509, + 0.05451597, + -0.029784957, + -0.054381475, + -0.015823675, + -0.1619974, + -0.16011356, + 0.14306353, + -0.14078444, + 0.034540735, + 0.06962766, + 0.12329775, + -0.087043755, + 0.027856104, + -0.01572991, + 0.05650036, + -0.116997704, + -0.049609113, + 0.043067608, + -0.030194819, + 0.03762365, + 0.17544276, + -0.09416882, + 0.049763445, + -0.13084425, + -0.07814618, + -0.08378609, + 0.100793965, + 0.022056215, + -0.08964522, + -0.05211006, + -0.18038046, + -0.10158667, + 0.05672273, + -0.091253355, + -0.08780731, + 0.14322338, + 0.16815259, + -0.07592685, + -0.019661143, + -0.08134259, + 0.103068285, + -0.06647779, + -0.08685702, + -0.14128084, + 0.0019816903, + -0.12417395, + -0.15051039, + -0.023196466, + 0.004036376, + 0.057318147, + 0.009758048, + -0.08783075, + -0.051209662, + 0.0920261, + 0.05065733, + 0.066438995, + 0.05187739, + -0.057600353, + -0.03672781, + -0.07916494, + -0.014410489, + -0.1901699, + 0.07111715, + -0.009864001, + 0.08938924, + 0.043175314, + -0.022378124, + -0.05109884, + -0.10978971, + -0.22157186, + -0.017547151, + 0.15984374, + -0.18353519, + 0.09903706, + 0.038109485, + -0.03303883, + 0.015448798, + 0.20255595, + 0.20748094, + 0.17261821, + 0.14197937, + 0.06597976, + 0.17574088, + 0.21813814, + 0.105163984, + -0.01047952, + -0.08538639, + 0.061182205, + 0.06984207, + -0.004131202, + -0.17733811, + -0.10162376, + 0.18271318, + -0.017683536, + -0.006569333, + 0.110508576, + -0.21429431, + 0.048779443, + 0.03614818, + -0.0037668368, + 0.02480318, + -0.14378709, + 0.071383335, + 0.18381415, + 0.07730354, + -0.02156793, + 0.08198412, + -0.058730595, + -0.08843486, + -0.0075829774, + 0.19533381, + 0.044322368, + -0.009381326, + 0.14623515, + -0.048733506, + -0.04459087, + 0.027501268, + -0.088260315, + -0.13386746, + 0.012325617, + -0.123915754, + 0.048359994, + -0.0233586, + -0.14324947, + -0.021993754, + -0.037602246, + -0.0109293545, + -0.015216226, + 0.013917774, + 0.00088481547, + -0.022108288, + 0.07754015, + 0.09491793, + -0.19853461, + 0.21539745, + -0.051947538, + 0.06529309, + 0.07011604, + 0.32943028, + 0.027539486, + -0.11953473, + 0.05762648, + 0.18586439, + -0.016185325, + 0.077894725, + -0.07515514, + -0.01281408, + -0.050225023, + -0.050484035, + -0.18947929, + 0.03090682, + 0.14163375, + -0.031653114, + -0.07802309, + 0.027042413, + 0.048488952, + -0.111342646, + 0.07818888, + 0.18088813, + -0.09250506, + -0.33192116, + -0.08149952, + -0.11298044, + -0.041128945, + -0.07511922, + -0.07413142, + -0.110216334, + -0.024052966, + -0.033287864, + -0.118019365, + -0.17631935, + -0.11153905, + 0.1217114, + 0.123827904, + 0.24722455, + -0.20767799, + -0.26084846, + 0.08209452, + -0.051249873, + 0.05579694, + 0.13793346, + -0.09851016, + -0.16345078, + 0.08822302, + -0.06681328, + 0.31821293, + 0.10751191, + 0.04159968, + -0.043661255, + 0.09237068, + 0.02019933, + -0.20946394, + 0.100037284, + 0.031177875, + -0.12108882, + 0.058419876, + 0.012723419, + 0.2022977, + 0.09126928, + -0.10295439, + 0.0011386054, + -0.047242202, + -0.017559966, + -0.05268205, + 0.1164585, + -0.0540001, + 0.023300944, + 0.06440002, + -0.088080704, + -0.054338727, + 0.09319473, + -0.042302974, + 0.10190417, + -0.015233202, + 0.048785076, + -0.051005866, + -0.09943697, + -0.0028515332, + 0.10641001, + -0.045118928, + -0.12029709, + 0.00047828123, + 0.040764276, + 0.122065656, + 0.020858958, + -0.014523222, + -0.036948867, + 0.08815156, + 0.10024676, + 0.0071087675, + 0.015215233, + -0.063443795, + 0.017486593, + -0.067480326, + 0.06852252, + -0.047518123, + 0.086380325, + -0.13453169, + 0.022754231, + -0.09384027, + 0.0621304, + -0.19681232, + 0.025613641, + 0.08451524, + -0.023075689, + -0.051967274, + 0.24069102, + 0.0051874095, + 0.0037075432, + 0.09055538, + 0.15680784, + 0.15741332, + 0.036581125, + 0.10939615, + 0.21720846, + 0.024938865, + 0.05740122, + -0.04607626, + -0.04106802, + -0.126018, + -0.16080438, + -0.13845068, + 0.07245159, + 0.007936225, + 0.015580119, + 0.12138403, + 0.06895977, + 0.034619506, + -0.049650382, + 0.062481627, + 0.07182102, + 0.06650026, + -0.13141704, + -0.059779823, + 0.11649756, + -0.080230854, + -0.009456568, + 0.01745865, + -0.1216187, + -0.14026399, + 0.06637448, + -0.0021523647, + -0.017390082, + 0.0250645, + 0.027999043, + 0.0010218549, + 0.037916534, + 0.0084903, + -0.07529715, + 0.17703146, + 0.11206236, + 0.14551207, + 0.06375902, + 0.053224288, + 0.16823557, + 0.003997259, + -0.15612938, + -0.04896271, + 0.039880976, + -0.053224344, + 0.055527527, + -0.08106474, + -0.10617742, + 0.034333583, + -0.15836494, + -0.027155168, + 0.1341483, + -0.06264232, + -0.2370755, + -0.0009348112, + 0.1220235, + -0.1441811, + 0.058377672, + 0.019060336, + 0.28706726, + -0.037206423, + 0.0183846, + 0.09823669, + -0.029278345, + 0.02220341, + -0.04629909, + -0.09074738, + 0.000055111203, + 0.008202666, + -0.040526114, + 0.040905226, + 0.06978048, + -0.21613108, + -0.11772803, + 0.21750113, + -0.07890024, + 0.025991434, + -0.1393707, + -0.02103182, + -0.056383714, + -0.22799924, + -0.020068502, + -0.026095064, + -0.05957455, + -0.12077848, + 0.2305243, + 0.115230255, + 0.039515268, + -0.13600051, + 0.11626736, + -0.06592065, + 0.06647881, + 0.08193841, + -0.0034670162, + 0.0414495, + 0.021196732, + -0.15306099, + 0.14910209, + 0.09136172, + -0.09669719, + 0.11342933, + -0.0054969266, + -0.0057360036, + -0.01979154, + -0.07336227, + -0.024596639, + 0.018258438, + 0.03128523, + 0.073674195, + -0.15561396, + 0.08504639, + 0.08488691, + 0.028538471, + 0.0423627, + 0.014724318, + 0.0135664465, + -0.11385461, + -0.20146337, + 0.013475908, + -0.025020214, + 0.0029377763, + 0.021690404, + -0.16602075, + -0.022968791, + -0.050374143, + 0.15816566, + 0.031169537, + -0.13332117, + 0.17529438, + 0.06299093, + 0.03524768, + -0.005311315, + -0.021740261, + -0.17312866, + -0.0034949996, + 0.03785792, + -0.031137161, + -0.021650564, + 0.17733982, + -0.07883986, + -0.1012539, + 0.11402955, + -0.1345435, + -0.046697926, + -0.12799911, + 0.32307515, + 0.12448643, + 0.099601544, + -0.091351636, + -0.14223091, + -0.04214468, + -0.051775057, + 0.08848543, + -0.030550303, + -0.04576182, + -0.029946158, + 0.13242897, + 0.0630274, + -0.27374837, + -0.16037427, + -0.12407486, + -0.034142725, + -0.011492249, + 0.037515692, + -0.02739928, + 0.104391396, + 0.03849497, + -0.042551294, + -0.008264996, + -0.1488099, + -0.19886811, + -0.023193035, + -0.051028185, + -0.1308757, + 0.14674132, + 0.09617077, + 0.062017158, + 0.16497657, + 0.052329116, + 0.10063006, + -0.028185332, + 0.024745198, + 0.018622441, + -0.062464993, + -0.09929202, + -0.14557129, + 0.069647945, + 0.046294276, + 0.11141771, + -0.02630893, + 0.05167021, + -0.14462258, + -0.15566984, + 0.046654463, + 0.1344545, + -0.06318777, + 0.020561263, + -0.051928796, + -0.0045303665, + -0.007826973, + 0.13673766, + -0.004384747, + -0.10855306, + 0.11255913, + 0.033657096, + -0.0038668383, + -0.0902208, + -0.14546496, + 0.05562975, + 0.17490505, + -0.11488267, + 0.0622177, + 0.016163535, + -0.021113109, + -0.052931532, + -0.029033361, + -0.15346348, + -0.095060974, + 0.047087323, + 0.07841412, + -0.0014932422, + 0.02391262, + -0.027430516, + -0.022514263, + -0.05717942, + 0.12731571, + 0.019991966, + 0.09605899, + 0.20384468, + -0.108349234, + -0.12042204, + -0.013678046, + -0.100384444, + -0.014950864, + 0.31551743, + -0.09998141, + 0.044594593, + 0.043450613, + -0.033286948, + 0.048648007, + 0.035094917, + 0.028859263, + -0.019374106, + -0.08274263, + 0.06221168, + -0.09087616, + -0.10503534, + -0.13828881, + -0.17042753, + -0.054863665, + -0.07792208, + 0.043511827, + -0.043746166, + -0.112693846, + -0.1362984, + 0.06479504, + -0.19862954, + -0.044869628, + 0.04635564, + 0.07353651, + -0.13631026, + 0.028526409, + 0.08994538, + -0.007131977, + -0.07580046, + 0.15547627, + 0.051717956, + 0.22072883, + 0.014018235, + 0.021613697, + -0.00834174, + -0.02471064, + 0.006452414, + -0.023242634, + 0.14135857, + -0.017866744, + -0.078924164, + 0.002792976, + 0.025580725, + 0.093077816, + 0.11742462, + 0.17183556, + 0.048280437, + -0.038447, + 0.03272755, + -0.05684157, + -0.14105506, + -0.09822835, + -0.13439925, + -0.041158233, + 0.14476937, + 0.027747344, + -0.04214371, + -0.082351744, + 0.20590134, + -0.053941507, + 0.1407635, + 0.023585325, + 0.07622522, + 0.07859561, + 0.16085447, + -0.09118282, + -0.12415422, + -0.10650889, + 0.09723356, + -0.0147255175, + 0.102288894, + -0.03896057, + 0.10433694, + 0.018424954, + 0.12578288, + 0.02442984, + 0.018820781, + 0.06994398, + 0.17829625, + -0.11848273, + 0.041317374, + -0.2226219, + 0.036284197, + 0.04823403, + -0.020507393, + -0.15940382, + 0.07286299, + 0.08382624, + -0.05059868, + -0.031402253, + 0.11669439, + 0.32230553, + -0.22862247, + 0.18761061, + 0.0346211, + -0.058325805, + -0.1290626, + 0.01745937, + -0.034736596, + -0.08302347, + -0.09274553, + 0.17429034, + -0.12862267, + 0.06685856, + 0.05717274, + -0.03504606, + 0.0024176335, + -0.11750517, + -0.06691499, + -0.02688414, + 0.049796186, + -0.047000907, + 0.008038866, + -0.090083994, + 0.014953727, + -0.08359565, + -0.15552822, + -0.0074157463, + -0.0650758, + 0.14944446, + 0.12081586, + 0.042148665, + 0.08434979, + 0.102038845, + 0.31358033, + -0.017266177, + -0.018252319, + -0.11620765, + 0.19398181, + 0.065347575, + 0.028032329, + -0.047978, + -0.2797833, + -0.18791555, + 0.069568746, + -0.100677535, + -0.065925896, + -0.0006277648, + -0.019300116, + 0.023896491, + 0.058626954, + -0.13866243, + 0.045072842, + -0.007091729, + -0.099349074, + 0.020197209, + 0.069288954, + -0.039979655, + -0.0064277677, + 0.06091153, + 0.08248567, + 0.122923665, + -0.0105521325, + 0.084243946, + 0.06743796, + 0.10869965, + 0.07791377, + 0.16056946, + -0.079824574, + 0.16511679, + 0.30834618, + -0.0010122202, + 0.06996, + 0.16964124, + 0.0057189995, + -0.10605331, + 0.005887883, + -0.09933687, + 0.07714087, + -0.09459299, + -0.037669845, + 0.0051917247, + -0.034806307, + -0.091839895, + -0.057295084, + -0.13806462, + -0.023035849, + -0.113358445, + 0.00036082126, + 0.017212579, + -0.18900444, + 0.032043274, + -0.1334188, + 0.20043485, + 0.122222595, + 0.17968272, + 0.036805574, + -0.18576208, + -0.038874328, + 0.032150056, + 0.06678015, + 0.20464763, + -0.05352213, + -0.14664139, + -0.01596184, + -0.011251751, + 0.013621192, + -0.03779479, + -0.0021375145, + -0.110297136, + -0.00047209254, + -0.04304118, + -0.025555795, + -0.2587818, + -0.04245533, + -0.100343496, + 0.0676543, + -0.010379713, + -0.13916823, + 0.042932615, + 0.18630238, + -0.081409305, + 0.07042592, + 0.15729047, + 0.14410137, + -0.06201796, + 0.1336767, + -0.14569852, + -0.010010871, + 0.01497488, + -0.032156806, + -0.07865755, + -0.057853807, + -0.058520615, + -0.1221883, + -0.05390742, + 0.14014581, + -0.1506584, + -0.22595532, + 0.19609949, + 0.13399656, + -0.088791884, + 0.051524367, + -0.0006443783, + -0.044071194, + -0.020211484, + -0.027934307, + 0.04833655, + 0.021489883, + 0.020963512, + 0.03666508, + -0.023618823, + 0.072483085, + -0.06972042, + -0.016392939, + 0.08228323, + -0.08687732, + -0.18806702, + -0.051532757, + -0.08548382, + -0.03818729, + -0.17306228, + 0.16669579, + -0.12068752, + 0.15081783, + 0.045993183, + 0.2429938, + 0.10049293, + 0.068342544, + 0.11905306, + -0.12723783, + -0.094005585, + -0.094075195, + 0.09178444, + -0.060471322, + -0.042530973, + 0.05703309, + -0.042406984, + 0.037191406, + 0.06595629, + 0.11178034, + 0.0032163626, + 0.0021105914, + 0.2162746, + 0.024074657, + 0.043626223, + 0.094974354, + -0.07438526, + -0.11592679, + -0.027236253, + -0.097098835, + -0.013131776, + -0.026293423, + 0.0642637, + 0.06964624, + 0.023534914, + -0.103869915, + -0.0020772098, + 0.11470637, + 0.024696454, + -0.017179212, + 0.14924346, + -0.08354481, + -0.11440698, + -0.003193121, + 0.020677894, + -0.010825841, + 0.13964406, + -0.1796176, + -0.042000804, + -0.09755287, + -0.016078439, + -0.088813335, + -0.25850728, + 0.06515638, + -0.06373765, + -0.048816353, + 0.047976196, + 0.04746054, + 0.084927715, + 0.030447515, + -0.08692034, + 0.065609545, + 0.0656312, + -0.045202747, + -0.02100044, + 0.049361564, + 0.030548196, + -0.014721435, + -0.097593926, + -0.20728064, + 0.045408644, + -0.00536836, + 0.14769737, + 0.16342898, + -0.18867482, + 0.14737496, + -0.032487184, + -0.08090989, + 0.04822956, + 0.057315417, + -0.009579211, + 0.078346536, + 0.106504664, + 0.072445154, + -0.10301303, + -0.031137764, + -0.09640398, + 0.034376223, + 0.17748068, + 0.20015173, + -0.0524466, + 0.10816899, + 0.050401874, + 0.08077222, + 0.1351978, + 0.15729517, + -0.049616523, + 0.05663771, + -0.187936, + 0.019404227, + -0.0018672178, + -0.050983317, + -0.019986866, + -0.15442687, + -0.028581971, + 0.105104, + -0.12354481, + -0.043984205, + -0.094944976, + 0.017726814, + -0.09397526, + 0.091985874, + 0.010167289, + -0.0060442733, + 0.060004544, + -0.05739756, + 0.055090036, + 0.0825361, + 0.023660736, + -0.025196167, + -0.05298092, + -0.11229647, + -0.04141075, + 0.18015505, + -0.07514402, + 0.026814222, + 0.1279963, + 0.027810784, + -0.03213902, + -0.14986579, + 0.045975763, + -0.09107888, + -0.032808293, + 0.22553365, + -0.13920985, + -0.15993676, + -0.22778948, + 0.031712223, + 0.030466309, + 0.09263852, + -0.09981502, + 0.08008334, + 0.1051347, + -0.0754973, + -0.038303465, + -0.1762361, + 0.118566915, + -0.009546227, + -0.027119413, + -0.05298601, + 0.12737975, + 0.009414802, + 0.08606997, + 0.11466975, + 0.14650959, + -0.03808247, + -0.06272387, + -0.255949, + 0.04103414, + 0.0030610003, + -0.098628126, + -0.0116982255, + -0.048501533, + 0.016773852, + -0.056915086, + 0.006698067, + 0.20575973, + 0.1877377, + -0.03326569, + -0.061274577, + -0.0068377615, + 0.07966858, + -0.12756282, + 0.08531209, + 0.037756577, + 0.08321911, + -0.017660106, + 0.11267764, + 0.047352593, + 0.08586491, + 0.009352779, + 0.10996305, + 0.18631652, + 0.014467773, + 0.022062704, + -0.014274694, + -0.030707002, + 0.08191203, + 0.042273425, + -0.04836255, + -0.031163195, + 0.012917261, + -0.111132674, + -0.12557924, + 0.069332786, + 0.016806293, + -0.029989868, + 0.006696535, + -0.1036078, + 0.013552768, + -0.036468834, + 0.008811692, + 0.017910536, + -0.07618319, + 0.10405537, + -0.06596659, + 0.016205423, + -0.15306845, + 0.18459404, + 0.19172415, + 0.16765448, + 0.09293131, + -0.023066819, + 0.028651476, + -0.10250834, + 0.18842621, + -0.056548014, + 0.1580178, + 0.06872625, + -0.04933942, + -0.16788821, + -0.043857004, + -0.07558683, + -0.0387379, + 0.1105206, + -0.1322464, + 0.0026022727, + -0.21048826, + -0.055464387, + -0.085378245, + -0.018984102, + 0.008462616, + 0.05357384 + ], + [ + 0.16388866, + -0.1274493, + -0.029406719, + 0.016088545, + 0.14838019, + -0.17018898, + -0.04039915, + 0.09454303, + 0.11083417, + 0.05347786, + -0.048188694, + 0.11532913, + 0.014010304, + -0.07019023, + 0.0048637316, + 0.04543801, + 0.25868493, + -0.18408833, + 0.14015624, + 0.17228295, + -0.032755684, + -0.059810344, + 0.10750608, + 0.061844334, + -0.19563942, + 0.062149033, + 0.014343361, + 0.0984511, + -0.09732532, + -0.1251685, + 0.029414687, + 0.34561026, + 0.16352534, + 0.080250874, + 0.05570887, + 0.02227714, + -0.0005784997, + -0.025817353, + 0.13338163, + -0.22323857, + -0.060119513, + 0.18531632, + 0.0062731, + 0.025778763, + 0.16983624, + -0.08474264, + 0.14512548, + -0.048115797, + -0.008017415, + 0.1371671, + -0.011217359, + -0.020784726, + -0.08566715, + 0.055902667, + 0.1673113, + -0.11338298, + 0.16070479, + 0.08242804, + 0.11604139, + -0.017087411, + 0.06934974, + -0.016676396, + 0.043297246, + -0.036335208, + 0.10049256, + 0.025410151, + 0.02477693, + -0.009294282, + -0.07591856, + -0.18171048, + 0.060261853, + -0.122343615, + 0.15206912, + 0.106727585, + -0.029032094, + -0.042914044, + 0.06343734, + -0.0053028045, + 0.112502955, + 0.06834707, + 0.04419882, + -0.00005904601, + 0.18741587, + 0.08683014, + -0.12393375, + -0.24384312, + 0.074607685, + -0.0023491608, + 0.088682815, + -0.28251874, + 0.11004756, + 0.2200098, + -0.031699747, + -0.020981878, + -0.030009469, + 0.1726068, + -0.0010401199, + -0.109709784, + -0.038090434, + 0.056116667, + 0.111572966, + 0.07795444, + -0.015640097, + -0.1052311, + 0.060738035, + -0.052688543, + 0.07297174, + 0.060491957, + -0.036764227, + -0.019321999, + 0.08997, + -0.08343091, + 0.04973949, + -0.0026332221, + -0.12699708, + 0.07842202, + -0.02387683, + 0.0997214, + 0.254947, + 0.07160601, + 0.035886012, + 0.0442851, + 0.054689683, + 0.054558035, + -0.013154186, + -0.061086945, + -0.10709059, + -0.34985533, + -0.16282624, + -0.19718409, + -0.06799517, + -0.105992235, + -0.03436971, + -0.02394977, + -0.041571558, + 0.048456144, + 0.016054248, + -0.03459052, + -0.19965164, + 0.17241395, + 0.09670938, + -0.18519202, + 0.014623482, + 0.05541973, + -0.1351895, + 0.24589261, + -0.1022615, + -0.03052882, + -0.016050378, + 0.025090026, + -0.023675127, + -0.13002898, + -0.18718348, + -0.07508323, + 0.15869549, + -0.07600789, + 0.167427, + 0.18939757, + -0.021024548, + 0.03398692, + -0.15091287, + 0.20724702, + -0.12697782, + 0.0059517547, + 0.08259693, + 0.09291371, + 0.04922314, + 0.28753632, + -0.066147484, + 0.025699098, + -0.07228272, + 0.00005861897, + 0.06325163, + -0.043452013, + 0.011995904, + -0.06449323, + 0.15508763, + -0.09473816, + 0.0006501909, + 0.06281607, + -0.032144964, + -0.11335421, + 0.17359382, + -0.105448686, + 0.07852979, + -0.13064326, + 0.03276877, + -0.33442417, + 0.034688003, + 0.122332, + 0.041492272, + -0.09941672, + 0.055898048, + 0.01998042, + 0.005248705, + 0.072767675, + -0.062271778, + -0.018750688, + 0.17954019, + 0.007829815, + 0.13000026, + 0.08745995, + 0.110863075, + -0.052847635, + -0.16000043, + 0.1519155, + -0.11334409, + -0.17254685, + 0.061103605, + 0.32251626, + 0.02066164, + 0.09145846, + 0.0674363, + 0.046851695, + 0.07305063, + -0.015633881, + 0.057030063, + -0.032213688, + 0.037112482, + 0.12663884, + -0.01838665, + -0.09834484, + -0.16225378, + -0.38311186, + 0.07321808, + 0.1664177, + -0.10461536, + -0.10203128, + -0.09162459, + -0.134588, + 0.12635095, + 0.17391087, + -0.027241778, + -0.06345333, + -0.24448626, + -0.10119743, + 0.12421486, + -0.12096262, + -0.01865057, + -0.07375571, + 0.17116037, + -0.017379139, + -0.066013366, + 0.03545176, + -0.03464347, + -0.07597424, + -0.15583192, + -0.12783983, + 0.10115018, + 0.057040293, + 0.03874398, + 0.14985508, + -0.0149468295, + -0.20430133, + 0.20872924, + 0.04353235, + 0.0432664, + 0.05577916, + 0.0840583, + -0.11259745, + 0.09421194, + -0.07860631, + -0.03305355, + -0.04810586, + 0.09794147, + 0.037893694, + -0.0663785, + -0.15341663, + 0.045156345, + 0.1457715, + -0.022321763, + -0.02326849, + 0.03544912, + 0.13372923, + 0.18873328, + 0.021077115, + -0.07664262, + -0.07699715, + 0.014613696, + -0.012329727, + 0.05611493, + -0.10256753, + 0.102527976, + -0.02827899, + -0.015486185, + -0.09325154, + -0.016822595, + -0.059222315, + -0.025249725, + 0.041170955, + 0.18410237, + 0.08856483, + -0.035322465, + 0.094879866, + -0.09404288, + -0.23588818, + -0.11044584, + 0.039941445, + -0.120199084, + -0.026341137, + -0.18742459, + -0.087245084, + 0.0011981897, + -0.05029513, + -0.013702148, + 0.019574808, + 0.12717503, + -0.20070675, + 0.042355396, + 0.19268517, + 0.08347445, + -0.10835724, + -0.0118761305, + -0.07011443, + -0.0006318021, + 0.06806462, + 0.10978322, + 0.14524662, + 0.06316656, + -0.2862662, + -0.041857377, + 0.13580416, + 0.036744256, + -0.10564063, + 0.10736383, + 0.012952529, + -0.078166686, + 0.16248384, + 0.16347384, + -0.074001014, + 0.10718379, + -0.023243865, + -0.13118525, + 0.10525095, + 0.017593017, + -0.090159506, + -0.05804784, + -0.042623166, + -0.1379461, + -0.101566546, + -0.07362863, + -0.0278407, + -0.080381766, + -0.08582083, + -0.09905207, + 0.105046354, + -0.009814006, + -0.08399216, + 0.07872614, + -0.077095084, + 0.103942625, + 0.1213736, + -0.032375656, + 0.110370345, + -0.013208189, + -0.31782433, + -0.011086241, + 0.249398, + 0.12877339, + -0.115588546, + 0.19144273, + -0.03690627, + -0.0083518, + -0.22595355, + -0.035430674, + -0.0020649435, + -0.10027537, + -0.0069912532, + 0.03622333, + 0.01488423, + 0.16713005, + -0.20800437, + 0.097926, + 0.02829707, + -0.048870295, + -0.026990525, + 0.2697227, + 0.0758388, + -0.15572745, + 0.08132002, + -0.19318978, + 0.060067162, + 0.05500014, + 0.018519403, + 0.100088276, + 0.0051668165, + 0.017496618, + 0.00138397, + -0.0541874, + -0.071715385, + -0.13669915, + 0.02461372, + 0.1620136, + -0.093883574, + 0.059940103, + -0.05633063, + -0.13765149, + 0.12888142, + -0.0027039868, + 0.019692076, + 0.091839045, + -0.17277081, + -0.020779878, + 0.08394452, + -0.025778674, + -0.020153807, + -0.025640903, + 0.088139944, + 0.13001621, + 0.02358827, + -0.0110823205, + 0.0017238051, + 0.00736703, + 0.05893931, + 0.00007190694, + -0.12042281, + -0.089033306, + 0.00001880407, + 0.09596746, + -0.07119645, + -0.02517913, + 0.11799862, + 0.011960116, + 0.20303088, + 0.16034381, + -0.018967377, + 0.210181, + -0.025709221, + -0.09715356, + -0.15561356, + -0.14527835, + 0.0026621171, + -0.13523585, + -0.114113584, + -0.026216721, + 0.023895103, + 0.045930795, + 0.30385685, + 0.024324115, + -0.036034085, + -0.14045344, + 0.07606133, + -0.22287777, + 0.28325832, + -0.023030547, + -0.17539379, + 0.14789481, + 0.026066065, + 0.03501296, + -0.22780557, + -0.13614109, + -0.09973608, + -0.10754517, + 0.09841886, + 0.21535163, + 0.012374242, + -0.080729015, + 0.028772624, + 0.1155963, + 0.08191568, + -0.029438535, + -0.067429654, + 0.13913694, + 0.17499427, + -0.04453274, + -0.08612042, + -0.14939116, + -0.04562479, + 0.075765364, + 0.06617139, + -0.07673605, + -0.03844562, + -0.008096309, + 0.18215945, + 0.17319143, + 0.13745795, + -0.09791408, + 0.099149436, + -0.031476732, + -0.080630206, + -0.0026137421, + 0.09407611, + -0.07356355, + -0.015712759, + -0.00291144, + -0.17586967, + -0.12700737, + 0.028887944, + 0.052166972, + 0.24733089, + -0.12667046, + 0.08695972, + -0.118434824, + 0.035153218, + -0.04006254, + 0.098527506, + -0.014844824, + 0.032654587, + -0.121846, + 0.09473726, + 0.11865472, + -0.123950824, + -0.18025148, + 0.04298218, + -0.25811112, + 0.10292098, + 0.01899375, + -0.16929156, + 0.05765925, + -0.028931255, + 0.080413856, + 0.07315269, + 0.013165271, + -0.40394807, + 0.11243496, + -0.115240395, + 0.12292451, + 0.07033961, + 0.06127199, + -0.026358012, + 0.065774314, + 0.1982356, + -0.19110973, + -0.009933477, + 0.05109736, + -0.07235428, + 0.029698689, + -0.10441872, + 0.024818512, + 0.057346266, + -0.13524841, + -0.14867397, + -0.1618978, + 0.023946917, + 0.11333994, + -0.037924588, + 0.08667814, + -0.082453705, + -0.45097208, + 0.048280187, + -0.033051312, + 0.019459395, + 0.040711295, + -0.07927231, + -0.04588269, + -0.08402525, + 0.019407822, + 0.10613192, + -0.19943574, + -0.21517515, + 0.057538357, + 0.12678343, + 0.07607647, + 0.15332358, + 0.12425749, + 0.0043547424, + 0.025980875, + 0.11630779, + 0.15714303, + 0.06184408, + 0.18276384, + 0.11256405, + -0.17640677, + -0.07097763, + 0.19177987, + 0.0101696, + -0.1798298, + 0.19240941, + 0.06793052, + -0.13528202, + -0.091061875, + -0.0026571192, + -0.1272832, + -0.04891193, + 0.06435088, + 0.021889955, + 0.080110304, + 0.121886335, + -0.13008718, + 0.08451358, + -0.03732989, + -0.008967854, + -0.07577442, + -0.16507332, + -0.13416524, + -0.051055793, + 0.024851402, + 0.24799564, + -0.05325767, + 0.08904073, + 0.19089879, + 0.21935882, + 0.05772928, + 0.073273264, + -0.23626634, + -0.1507864, + 0.019251868, + 0.0992486, + -0.27776155, + -0.004501568, + 0.030041052, + -0.005606703, + 0.0056030485, + 0.044171736, + 0.07914877, + -0.009110708, + 0.045662787, + 0.124013126, + -0.18396807, + -0.0007910697, + -0.1416412, + -0.07581878, + -0.19717091, + -0.12796089, + 0.0014130679, + 0.017730486, + 0.020228159, + 0.03808832, + -0.021729082, + 0.0076779504, + 0.041449185, + 0.11420258, + -0.057412352, + -0.055537764, + 0.011715054, + 0.027988162, + -0.16164273, + -0.047086652, + -0.027213214, + -0.01458741, + -0.14191419, + -0.34179014, + 0.114580706, + -0.13287358, + -0.06314325, + 0.098375775, + 0.18766455, + 0.08296143, + -0.034589794, + -0.06442063, + 0.0027512189, + -0.0049052234, + 0.26622465, + -0.09836955, + -0.002747486, + 0.012809717, + 0.058249887, + 0.09697363, + 0.05184806, + -0.028175302, + -0.11948685, + 0.047858585, + 0.049114708, + 0.04366805, + -0.048788402, + 0.07537839, + 0.08124819, + -0.049342003, + -0.048827793, + -0.24337354, + -0.0904107, + -0.10121226, + -0.14616653, + -0.19013749, + -0.12045927, + 0.00057346415, + 0.06695801, + 0.0011243606, + -0.13399692, + -0.172681, + 0.0090777315, + -0.07489815, + 0.33741292, + 0.082466215, + -0.034463603, + 0.034199193, + 0.242095, + 0.011614127, + 0.09068604, + -0.10679573, + -0.06979583, + 0.15039863, + -0.055996206, + 0.13714892, + 0.050380126, + 0.0075273826, + -0.024202906, + 0.2521027, + -0.020975243, + -0.058391597, + -0.00574566, + -0.08583283, + -0.06375379, + -0.026359826, + -0.13244799, + 0.034083266, + -0.067055084, + -0.07812942, + -0.044962946, + -0.22463906, + -0.16745086, + 0.036082175, + -0.21081328, + -0.099076234, + -0.088583514, + 0.08871009, + -0.049018934, + 0.02865866, + -0.1323246, + 0.037936293, + -0.003502658, + 0.008930401, + 0.09541594, + 0.10848695, + 0.1298, + 0.046466224, + -0.011176741, + 0.09348958, + -0.015967473, + 0.04499383, + 0.32787913, + -0.011344857, + -0.13334349, + -0.06222278, + -0.18287274, + 0.07326876, + 0.117705174, + -0.03653037, + -0.012841428, + -0.13771583, + -0.042247947, + -0.12360058, + 0.007642602, + 0.11411447, + -0.012764738, + 0.08287802, + -0.032910045, + 0.010611884, + 0.06722046, + 0.08404961, + -0.085281566, + -0.06602937, + -0.08220149, + 0.40253213, + 0.020178514, + 0.021366082, + 0.015922796, + -0.051070135, + 0.16992474, + -0.025315536, + -0.19612277, + 0.04293197, + -0.019177299, + -0.1778408, + 0.03831598, + -0.0063439934, + -0.17214172, + -0.010579311, + -0.026407456, + 0.015255033, + 0.060688015, + 0.094218336, + 0.021506721, + 0.11196404, + 0.14996655, + 0.101845466, + -0.080502614, + 0.13263072, + 0.36179283, + -0.108861744, + -0.09095643, + 0.059276287, + -0.10125165, + -0.05592705, + -0.023738056, + 0.1321397, + 0.12529612, + -0.016392188, + -0.10326601, + 0.0019011673, + 0.054998904, + 0.031451274, + 0.03812364, + -0.057114914, + -0.0063940454, + -0.10670352, + 0.16763397, + 0.10335414, + 0.0031811795, + 0.0881537, + 0.10391506, + 0.20159209, + -0.16638228, + 0.11514696, + -0.25752142, + -0.041334677, + -0.16817635, + -0.15650864, + -0.01790024, + -0.1746948, + 0.06629112, + 0.056558732, + 0.2555542, + -0.13847725, + 0.08896017, + -0.0862949, + -0.19618006, + 0.07099258, + -0.06497313, + 0.022971679, + -0.13695043, + 0.20289019, + 0.0074132388, + -0.16143668, + -0.13385276, + -0.0057417555, + -0.049931537, + -0.0499368, + 0.16519663, + -0.10507758, + 0.022549141, + 0.09460348, + 0.08910219, + -0.16724345, + -0.2084208, + -0.028145097, + 0.02452698, + 0.16892421, + -0.0879645, + 0.07358576, + -0.08908357, + -0.07478337, + 0.11384186, + -0.09718927, + 0.03405777, + -0.14641492, + 0.15192483, + -0.114894405, + 0.21048348, + 0.028419742, + 0.11760003, + -0.17904466, + 0.099918626, + 0.18339533, + -0.15972906, + 0.038008258, + 0.061579287, + -0.0706305, + -0.113108814, + -0.18378308, + -0.0813167, + 0.08229117, + -0.03117914, + -0.19638929, + 0.067046, + -0.040391445, + -0.082185484, + -0.06653111, + 0.095955506, + 0.0106014665, + 0.2170189, + 0.03587968, + 0.099495776, + -0.19644739, + 0.0326707, + 0.076471515, + -0.13105156, + 0.011307406, + 0.12424341, + -0.017225409, + -0.1144092, + 0.20410052, + -0.042504083, + -0.08392147, + 0.061959956, + 0.11787481, + 0.09158845, + 0.1252602, + -0.037214234, + -0.1028871, + -0.24084169, + 0.08416911, + -0.051595353, + 0.1501746, + 0.05007575, + -0.16615319, + -0.1694795, + 0.07645114, + -0.09028527, + 0.041528326, + -0.071202695, + -0.097548045, + 0.0030405296, + -0.061570566, + 0.18403584, + 0.18518299, + 0.039708067, + 0.023216767, + -0.03886103, + 0.10391753, + 0.044079874, + 0.076719, + -0.18344459, + 0.031241922, + -0.088274024, + -0.090936266, + 0.121317595, + -0.080941856, + 0.0077993292, + -0.012819938, + 0.050826725, + -0.009877212, + -0.0026175291, + 0.07922244, + -0.14319204, + -0.044902574, + -0.04621125, + -0.12739496, + 0.033125475, + 0.042857002, + 0.15925257, + 0.05148726, + 0.06035093, + 0.12878227, + -0.13173364, + 0.21300669, + 0.04287097, + 0.18422864, + -0.020208461, + 0.08560416, + -0.031273432, + 0.0284608, + -0.066576384, + -0.075262636, + 0.11090535, + -0.018984916, + 0.23597457, + -0.026936144, + -0.1113342, + 0.13776274, + 0.058435105, + -0.021854132, + -0.055979736, + -0.12618151, + -0.103575595, + -0.049641356, + 0.06924185, + 0.15296723, + 0.22721526, + 0.017656358, + 0.029015211, + 0.09580017, + 0.025417013, + 0.13747607, + -0.05242506, + -0.12301473, + 0.027703684, + -0.22208689, + -0.18914069, + -0.15192284, + 0.019556163, + -0.05317778, + -0.17770518, + 0.13783428, + -0.00829997, + -0.022510786, + 0.016310576, + 0.18521482, + 0.06044866, + -0.06603976, + -0.023664916, + -0.039198935, + -0.07638657, + -0.14020346, + -0.061259385, + -0.014978218, + 0.0044513196, + 0.07345801, + -0.12439044, + 0.15552068, + 0.11208524, + 0.039133776, + -0.15118602, + 0.062687896, + 0.021445543, + -0.02722163, + -0.09711456, + -0.093860425, + 0.0621037, + 0.1387772, + -0.015153611, + 0.0657813, + 0.0050565475, + -0.10389596, + 0.050863225, + -0.14200687, + -0.003446694, + -0.021833828, + 0.10687284, + -0.24578965, + -0.1610299, + -0.118793644, + -0.05662669, + -0.04398657, + -0.17372021, + 0.1685216, + -0.08166115, + -0.024349907, + -0.1725162, + -0.154265, + -0.07341414, + 0.076563805, + 0.08727159, + -0.053592566, + -0.09193707, + -0.04832787, + 0.124241635, + -0.023937587, + 0.08278822, + 0.08565875, + 0.13332507, + 0.0055237906, + 0.053433668, + 0.048216075 + ], + [ + -0.00023819905, + 0.08722853, + -0.018189382, + -0.009879521, + -0.032684125, + -0.06906573, + -0.11446998, + 0.029835278, + 0.067963675, + -0.02277732, + -0.016014684, + 0.09255387, + -0.025500204, + -0.10092464, + -0.071504466, + 0.041136164, + -0.18302159, + 0.1267293, + -0.11327413, + 0.010292295, + -0.0216792, + 0.05033879, + -0.03700454, + -0.111096576, + 0.1222278, + -0.031139813, + 0.011436865, + 0.02741903, + -0.044984873, + -0.14670841, + -0.042417545, + -0.07265404, + -0.05578681, + -0.01894729, + -0.11626586, + -0.057778873, + -0.076166116, + -0.039082058, + 0.0102353, + 0.11408144, + 0.21770361, + -0.099172845, + 0.09127973, + 0.08130348, + -0.066892914, + -0.016743274, + -0.1113572, + 0.11114424, + 0.003547908, + -0.19168313, + 0.04100231, + 0.07516616, + 0.10149081, + 0.106094636, + -0.0071977987, + -0.029680476, + -0.019920351, + -0.11696883, + -0.18566902, + -0.10621034, + 0.017694049, + -0.0111797135, + -0.027567325, + -0.020165136, + 0.01650525, + 0.02419897, + -0.067844525, + 0.03948911, + 0.03829498, + 0.0062677297, + 0.0339069, + 0.072580114, + -0.021224942, + -0.028581431, + 0.112818174, + -0.057879932, + -0.08549357, + 0.09494796, + -0.060392946, + 0.11203333, + 0.0573878, + -0.05666371, + -0.09616267, + 0.0009771221, + -0.01928977, + 0.07569058, + 0.0013191677, + 0.022010304, + -0.0073402366, + 0.17945044, + -0.04356983, + -0.13315286, + 0.17298068, + 0.019000193, + 0.15418336, + -0.022096468, + 0.02423659, + 0.10567346, + -0.019227691, + -0.065874726, + -0.02374223, + -0.026744831, + -0.07638601, + 0.05457372, + -0.127177, + 0.00081117096, + -0.015371355, + -0.023764174, + 0.115929216, + 0.033732496, + 0.10019987, + 0.012313732, + -0.039875492, + -0.08675608, + 0.08382177, + -0.056043882, + 0.07497569, + 0.0100682685, + -0.03337517, + -0.054058846, + -0.1957298, + 0.13925007, + -0.15372674, + -0.09266753, + 0.046193045, + 0.04563814, + -0.09546537, + 0.14237428, + -0.00979199, + 0.05520484, + 0.0020154174, + -0.06974863, + -0.18376842, + -0.08121421, + 0.05205108, + 0.036930274, + -0.07723642, + -0.035251528, + 0.2037828, + 0.0049241306, + 0.004914411, + 0.0024789388, + 0.0030600508, + -0.048850402, + 0.024681028, + -0.03159827, + 0.053482078, + -0.005619381, + 0.019817207, + -0.049893197, + 0.16746224, + 0.0586457, + 0.0033691542, + -0.0022203603, + -0.13712765, + -0.005976267, + -0.10869649, + -0.100962706, + 0.02860752, + -0.22932072, + 0.106590465, + -0.19786568, + -0.019257398, + 0.042781726, + -0.03800602, + -0.120414995, + 0.039037187, + -0.15366909, + 0.05382185, + 0.098216675, + 0.02392267, + 0.023299983, + 0.07999794, + 0.065445825, + 0.025606325, + 0.075422525, + -0.13987134, + 0.080820404, + -0.0015350066, + -0.040707782, + -0.04539512, + 0.018436309, + 0.024940798, + -0.109948546, + -0.026582312, + 0.065350175, + -0.10288477, + 0.0049937363, + -0.06222751, + -0.0027971654, + -0.09911872, + -0.08644764, + -0.03971273, + -0.09787518, + 0.06990648, + -0.05995366, + 0.112496845, + 0.06923474, + 0.009939088, + -0.03873664, + -0.27163398, + -0.05580684, + -0.032839965, + 0.025805194, + 0.11151144, + -0.040572803, + 0.035155214, + -0.008341295, + 0.031401422, + 0.009405019, + 0.06083783, + 0.021521913, + -0.026307387, + 0.028267533, + 0.0010807854, + 0.12585023, + -0.03297708, + 0.06550157, + 0.08090696, + 0.049680475, + 0.12992574, + -0.023366483, + 0.03842595, + 0.13588712, + 0.13445982, + 0.09443064, + 0.02252088, + 0.15328129, + 0.03582764, + -0.0033455202, + -0.027840773, + -0.010460123, + -0.060981218, + 0.13026664, + -0.07372675, + -0.006314558, + -0.16581447, + 0.16462392, + -0.073720045, + 0.05355639, + 0.03830365, + -0.082658134, + 0.03483547, + 0.08563706, + -0.023776393, + 0.06325085, + -0.046465743, + 0.036570564, + 0.0018042738, + -0.05817889, + 0.00849821, + -0.1322951, + 0.06762883, + 0.08784282, + -0.15472934, + -0.06683327, + -0.10766635, + 0.032301515, + -0.029719582, + 0.019402627, + -0.051909026, + -0.023297891, + 0.04828677, + -0.00067922374, + -0.046935577, + 0.061543323, + 0.069230124, + 0.04700665, + -0.00003742312, + -0.025474532, + 0.021843506, + -0.01749575, + -0.15751489, + -0.15458816, + -0.17421009, + 0.01587172, + -0.011508841, + -0.023249552, + -0.021722468, + -0.0028855195, + 0.0073288493, + 0.08508097, + -0.035047654, + -0.04024355, + 0.06300432, + 0.0068712407, + 0.15135425, + 0.036468122, + 0.11908999, + -0.10330132, + 0.012709982, + -0.11417846, + -0.04919273, + -0.07180418, + 0.14601019, + -0.0048027798, + 0.10525847, + -0.04364896, + 0.029643727, + -0.1304667, + 0.030011281, + 0.050664, + -0.11059435, + -0.16691336, + -0.011236739, + -0.07697602, + -0.0810988, + 0.2417748, + 0.08366263, + -0.031880695, + 0.05497616, + 0.035905626, + 0.003939901, + 0.054534376, + -0.12075218, + -0.13702436, + -0.14440262, + -0.095596276, + -0.12564817, + 0.21444531, + -0.024642846, + -0.0043870406, + 0.023035543, + -0.010856918, + -0.027133044, + -0.0034959638, + 0.060708042, + -0.040865116, + -0.023209905, + 0.0076265507, + -0.1050314, + 0.040633943, + 0.10612071, + -0.13324992, + -0.018614884, + 0.052243005, + 0.049557533, + 0.11792032, + -0.03908536, + -0.0104958955, + -0.07188338, + -0.03591834, + 0.02715749, + -0.08241493, + 0.095054716, + 0.051083017, + 0.058776703, + 0.044111274, + -0.025501803, + 0.021567555, + -0.022877874, + -0.015196242, + -0.001957949, + 0.04799466, + 0.040363345, + 0.18367611, + 0.07372853, + -0.012322673, + -0.1278385, + -0.006435009, + -0.04726159, + -0.021471405, + -0.010708532, + -0.061113868, + 0.07263705, + -0.016319672, + 0.002038693, + 0.036269594, + -0.058426734, + 0.03141486, + -0.076382674, + 0.20794939, + -0.10612866, + 0.01222902, + 0.007965076, + -0.04278995, + -0.053591598, + 0.041889247, + -0.061773676, + -0.0024198552, + 0.16719845, + 0.07930351, + -0.033968326, + 0.037864327, + 0.12791102, + 0.014901688, + 0.008598468, + 0.058190595, + 0.17496578, + -0.1588089, + 0.021468895, + -0.06412895, + -0.026013156, + -0.022383815, + 0.0059805303, + -0.010148206, + 0.055713076, + -0.12541443, + -0.038745534, + -0.040888812, + 0.06122193, + 0.023668762, + -0.0705818, + 0.012752737, + -0.08765056, + 0.025650736, + 0.031376902, + -0.06834845, + -0.056653876, + 0.0010083625, + -0.007176116, + -0.03300903, + -0.08104129, + 0.09667459, + -0.08511829, + 0.035535865, + 0.060494848, + -0.042381532, + -0.110691786, + 0.057630006, + 0.13903373, + 0.08346473, + -0.025436034, + -0.17801067, + -0.16328157, + -0.0027230287, + -0.044160213, + 0.04867801, + 0.14752772, + -0.0025521321, + 0.11410878, + 0.13065912, + -0.040058598, + 0.051122565, + -0.009030965, + -0.018583799, + -0.024190543, + -0.13072833, + -0.020829115, + -0.0018237962, + 0.17159063, + -0.0544108, + 0.045537885, + -0.2044185, + 0.124284476, + 0.061364178, + -0.013351285, + 0.020631863, + 0.09298618, + -0.0952695, + 0.011654698, + 0.00013450178, + 0.19889687, + 0.043811962, + -0.17320704, + 0.039400797, + -0.0077564972, + 0.001777685, + -0.06983454, + -0.05196953, + -0.10299045, + 0.0044022514, + 0.13461155, + -0.014619998, + 0.16860752, + 0.12026308, + 0.16088343, + 0.0081091095, + -0.04836609, + -0.17056279, + 0.038241066, + 0.08099691, + 0.01636131, + -0.033349585, + 0.025676742, + -0.114264816, + 0.14894223, + 0.03172622, + -0.11526456, + -0.01554059, + -0.059403405, + -0.25905266, + 0.02123448, + -0.08491731, + 0.031096786, + 0.15112452, + 0.10838487, + 0.03357703, + 0.047125027, + -0.1017598, + 0.05702253, + -0.13325393, + 0.007017126, + -0.008844305, + -0.011607682, + 0.046095032, + 0.17051469, + -0.079309024, + -0.012608498, + -0.031549413, + -0.13921602, + 0.17587969, + 0.1847818, + -0.035155196, + 0.121373385, + -0.098536216, + 0.08955938, + 0.05673245, + -0.09131633, + 0.032798078, + 0.009806951, + 0.07231222, + -0.06879504, + 0.22904293, + 0.0024476966, + 0.05302264, + -0.13908802, + -0.049226537, + -0.008340455, + 0.055391368, + 0.08768921, + -0.103174545, + 0.14402257, + -0.06848061, + -0.049604136, + 0.048441708, + -0.0022681616, + 0.12693302, + 0.032360926, + -0.18466197, + 0.11525643, + 0.052614998, + -0.041946977, + 0.084402375, + 0.0070820805, + 0.051602617, + 0.013706249, + 0.09740458, + 0.13545464, + 0.05170128, + -0.01154326, + -0.06291002, + -0.0074243597, + 0.030624293, + 0.010787741, + 0.05516598, + -0.0000336987, + -0.023389293, + 0.13722405, + 0.30605146, + 0.047496915, + 0.011505263, + 0.09435109, + -0.09845997, + -0.046335388, + -0.047866605, + 0.017765287, + -0.0036139693, + -0.020844629, + -0.12663901, + 0.015865639, + -0.06769445, + 0.13005115, + -0.0025589156, + -0.055469558, + -0.03869603, + 0.08988444, + -0.1852774, + 0.007869909, + -0.020194182, + 0.039343905, + -0.21807171, + 0.11326507, + 0.02729821, + -0.056009147, + 0.002820678, + -0.0865628, + -0.0986437, + -0.011271464, + -0.06664685, + 0.11625404, + 0.0061748745, + 0.050193716, + 0.071995504, + 0.016740674, + -0.12077977, + -0.07313183, + -0.20954855, + 0.020530064, + -0.03219916, + -0.04984375, + -0.12113048, + 0.07666991, + -0.032115024, + 0.09149676, + 0.1230391, + 0.02289184, + -0.016896857, + 0.07186026, + 0.058443088, + -0.17080452, + -0.017978614, + -0.081828356, + -0.12276731, + -0.24000326, + 0.10029354, + 0.07849145, + -0.017504185, + 0.086734466, + 0.015516587, + 0.039352216, + 0.11477936, + 0.09283396, + 0.10953595, + -0.12719597, + -0.042160295, + -0.039048947, + -0.108032234, + -0.1565672, + 0.11080472, + -0.082766585, + 0.10841929, + 0.021128146, + 0.007303734, + 0.008792335, + 0.13141568, + 0.088779815, + -0.036739636, + 0.07781191, + 0.045989655, + 0.025219794, + 0.019484702, + 0.11404162, + -0.054736868, + 0.028522149, + -0.04004996, + -0.05860188, + -0.1004208, + 0.06932638, + 0.14744398, + 0.089994326, + -0.1693851, + -0.13760209, + 0.031510994, + -0.054425266, + 0.022236349, + -0.06066752, + 0.02053161, + 0.015066916, + 0.055059478, + 0.06969066, + -0.0008576496, + -0.057153583, + 0.046415977, + 0.033834662, + 0.008217297, + 0.07428007, + 0.056757733, + 0.11390362, + 0.09392918, + -0.03040512, + 0.059063632, + 0.061816785, + 0.08809859, + -0.19312109, + 0.016692715, + -0.048705675, + 0.11508326, + -0.059984937, + 0.107333474, + -0.037110046, + 0.12287615, + -0.057629667, + 0.105449215, + -0.035105113, + 0.06647831, + -0.17616904, + -0.010986365, + -0.07138596, + 0.082899205, + 0.011168886, + -0.011736261, + 0.010923345, + -0.13005716, + 0.04896893, + -0.027724585, + -0.045086086, + -0.21212313, + 0.027403008, + -0.0025199384, + -0.07674, + -0.037090894, + 0.07430594, + -0.057884112, + 0.06501002, + -0.14278291, + -0.010858643, + -0.08244009, + 0.12478751, + 0.059522286, + -0.14089537, + -0.035069946, + 0.14085332, + -0.07056501, + 0.082864456, + 0.017343162, + 0.04313108, + 0.070678145, + 0.055183675, + 0.056656722, + 0.041172523, + -0.014122126, + -0.06060968, + -0.016647091, + -0.07250498, + 0.09159845, + 0.034212586, + 0.019868867, + 0.015606668, + -0.062206388, + -0.13887468, + -0.020698428, + 0.26676226, + -0.13821203, + 0.060650177, + -0.2529516, + -0.15451081, + 0.22229247, + 0.051715747, + 0.105565764, + -0.024532182, + -0.08104707, + 0.045879535, + 0.03252688, + 0.028726567, + -0.07162622, + 0.019288847, + 0.027182905, + -0.06508289, + 0.026247546, + 0.08575366, + 0.109070554, + 0.107757285, + -0.14375055, + 0.022276828, + 0.078977376, + -0.0835771, + 0.06144981, + -0.03075645, + -0.017617078, + 0.009820327, + 0.038477298, + -0.05124861, + 0.020550333, + -0.18241732, + 0.06836194, + 0.038465198, + 0.1562779, + 0.03641837, + -0.030229904, + -0.028531149, + -0.015560756, + 0.0363471, + -0.053443477, + -0.027778376, + -0.047938056, + -0.059643246, + 0.018027555, + -0.13583753, + -0.08723769, + -0.04304011, + 0.029515615, + 0.049727246, + 0.05657146, + 0.06780928, + -0.11192818, + -0.0973618, + -0.03194436, + 0.08177615, + -0.022864683, + -0.091663964, + -0.007832455, + -0.11060666, + 0.12683538, + 0.022786979, + 0.024420243, + -0.0807567, + 0.08471282, + 0.048351564, + -0.0026698702, + -0.065727726, + -0.02852973, + -0.0003136066, + 0.0055406024, + 0.10670358, + -0.004658508, + -0.07481757, + 0.056015775, + -0.050442304, + 0.05564165, + 0.06094669, + 0.03744687, + -0.16147909, + 0.04669279, + -0.028488794, + 0.12882587, + 0.13914701, + -0.028585553, + 0.027781768, + -0.088927984, + 0.025170118, + -0.20610665, + -0.009523826, + -0.05483793, + 0.109778054, + -0.037884466, + -0.0450428, + 0.12432211, + -0.049269836, + 0.13231395, + 0.07454968, + -0.08147164, + -0.06280877, + 0.102979735, + 0.13877459, + 0.0046101813, + 0.0793429, + -0.16684714, + -0.026741825, + -0.098282196, + 0.10626281, + -0.07446011, + -0.030297032, + 0.2115962, + -0.030864643, + 0.12573703, + -0.14109895, + -0.024749564, + -0.030865956, + 0.013581069, + -0.020047193, + 0.19512308, + -0.011105194, + 0.07511189, + 0.094572686, + 0.025729392, + -0.012762934, + 0.022654291, + 0.06564823, + 0.098855376, + 0.05086923, + 0.004629107, + -0.034430407, + 0.13075809, + -0.021843228, + 0.06139189, + 0.032823958, + 0.035982914, + -0.08123399, + 0.08699251, + -0.009738269, + -0.081288345, + -0.094127424, + 0.048749935, + -0.19609971, + 0.11220326, + -0.016323354, + 0.021179782, + -0.04333348, + -0.031004159, + 0.057093773, + -0.19452132, + -0.035842434, + -0.022583233, + -0.13440512, + -0.01486175, + -0.06045068, + -0.028856233, + 0.0028380074, + -0.07459357, + 0.040689938, + -0.031722553, + -0.05657365, + -0.20057641, + -0.13084784, + 0.030955365, + 0.012184353, + -0.02683468, + 0.033647545, + 0.14640762, + 0.051798437, + 0.09941526, + 0.047942758, + 0.08155747, + -0.029609054, + -0.15660095, + -0.14481206, + 0.100328095, + 0.082845196, + -0.0089725405, + -0.06180668, + 0.017256266, + 0.09145879, + -0.041409202, + 0.0943183, + 0.030931752, + -0.06392001, + 0.0258518, + -0.09033543, + -0.010840782, + -0.025630532, + 0.06285814, + 0.11491396, + -0.037811674, + -0.036962993, + 0.12004661, + 0.019397037, + -0.00060135836, + -0.060936272, + -0.010717993, + -0.0093868505, + -0.09746303, + 0.03072794, + -0.095974006, + -0.09379994, + -0.073796816, + 0.11706571, + 0.043560777, + -0.0115037635, + -0.11607051, + -0.06136501, + 0.07135421, + -0.013506945, + -0.029872088, + -0.03540804, + 0.057288352, + -0.05973847, + -0.09163075, + 0.12088039, + -0.11064331, + -0.005449119, + -0.10552563, + 0.04656498, + 0.040111557, + -0.011094147, + -0.096868135, + -0.031159375, + -0.090243354, + 0.02873369, + -0.058690492, + -0.03207619, + 0.0028280048, + -0.013718786, + -0.088974625, + 0.035651192, + 0.17992984, + -0.034401752, + 0.016327163, + 0.0014029206, + 0.18518856, + 0.0417184, + 0.060015526, + 0.09803668, + -0.010271529, + -0.077012084, + -0.01524577, + -0.09952967, + -0.16755462, + -0.1731264, + 0.018687185, + -0.062199958, + 0.029846158, + -0.033576477, + -0.046113085, + 0.046791576, + -0.0070844744, + 0.009823376, + -0.042751074, + 0.04247843, + -0.19779414, + -0.06372267, + -0.007970582, + 0.13826874, + 0.04844482, + 0.104133636, + -0.042111084, + 0.0803083, + 0.12385037, + -0.06880023, + -0.03516938, + 0.0016008455, + -0.02931245, + -0.013868235, + 0.08598541, + 0.025319925, + 0.03795165, + 0.069413275, + 0.005628386, + 0.046195645, + 0.061195485, + -0.030694276, + -0.048860855, + -0.036304668, + 0.06705336, + 0.14506873, + -0.06601333, + -0.106765, + 0.080897905, + 0.014498234, + 0.0855388, + 0.12275358, + 0.09132434, + -0.04341464, + 0.12917939, + 0.13067503, + -0.06219275, + 0.008004696, + 0.021335322, + 0.12770006, + -0.030194325, + -0.047946878, + 0.013460314, + -0.06189629, + -0.10178993 + ] + ], + "bias": [ + 0.17217675, + 0.03334455, + -0.2055213 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.28, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.json new file mode 100644 index 0000000..75404e3 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.json @@ -0,0 +1,1006 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.28, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.8526646, + "confusion": [ + [ + 119, + 18, + 12 + ], + [ + 0, + 100, + 0 + ], + [ + 5, + 12, + 53 + ] + ], + "tool_precision": 0.9596774, + "tool_recall": 0.7986577, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.071428575, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47420904, + "p_memory": 0.33594865, + "p_abstain": 0.18984233 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5147423, + "p_memory": 0.2798606, + "p_abstain": 0.20539707 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47881237, + "p_memory": 0.22655208, + "p_abstain": 0.29463556 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.53897923, + "p_memory": 0.18119648, + "p_abstain": 0.27982432 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49686515, + "p_memory": 0.3141313, + "p_abstain": 0.1890035 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52341294, + "p_memory": 0.2889944, + "p_abstain": 0.18759267 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5525644, + "p_memory": 0.1675198, + "p_abstain": 0.27991584 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5143931, + "p_memory": 0.2983053, + "p_abstain": 0.18730168 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5596268, + "p_memory": 0.32979882, + "p_abstain": 0.11057431 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54281276, + "p_memory": 0.31816098, + "p_abstain": 0.13902621 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5726373, + "p_memory": 0.3156031, + "p_abstain": 0.11175954 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42772973, + "p_memory": 0.3487019, + "p_abstain": 0.22356844 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51347095, + "p_memory": 0.36306074, + "p_abstain": 0.12346836 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5302622, + "p_memory": 0.3146726, + "p_abstain": 0.15506518 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5095029, + "p_memory": 0.2474364, + "p_abstain": 0.24306077 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5355736, + "p_memory": 0.26030138, + "p_abstain": 0.20412505 + }, + { + "query_id": "abs_test_003", + "text": "what will my project look like in 6 months?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36163858, + "p_memory": 0.32116085, + "p_abstain": 0.31720063 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.6006625, + "p_memory": 0.25644553, + "p_abstain": 0.14289191 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.39021608, + "p_memory": 0.3815764, + "p_abstain": 0.22820748 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.51517594, + "p_memory": 0.33310035, + "p_abstain": 0.15172371 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.87222224, + "confusion": [ + [ + 60, + 5, + 5 + ], + [ + 0, + 55, + 0 + ], + [ + 4, + 9, + 42 + ] + ], + "tool_precision": 0.9375, + "tool_recall": 0.85714287, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.07272727, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.8333333, + "confusion": [ + [ + 10, + 0, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 4, + 6, + 30 + ] + ], + "tool_precision": 0.71428573, + "tool_recall": 1.0, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.1, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.96666664, + "confusion": [ + [ + 10, + 0, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 2, + 8 + ] + ], + "tool_precision": 1.0, + "tool_recall": 1.0, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 40, + 5, + 5 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 1, + 4 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.8, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.md new file mode 100644 index 0000000..69f341c --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-8/router_report.md @@ -0,0 +1,101 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.280` +- abstain_prob_threshold: `0.100` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.8527` +- tools precision: `0.9597` +- tools recall: `0.7987` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0714` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 119 | 18 | 12 | +| memory | 0 | 100 | 0 | +| abstain | 5 | 12 | 53 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.47 | 0.34 | 0.19 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.51 | 0.28 | 0.21 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Abstain` | 0.48 | 0.21 | 0.31 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Abstain` | 0.47 | 0.25 | 0.29 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Tools` | `Memory` | 0.51 | 0.28 | 0.20 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.53 | 0.15 | 0.32 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.53 | 0.28 | 0.20 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Abstain` | 0.55 | 0.18 | 0.27 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.52 | 0.27 | 0.21 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Abstain` | 0.53 | 0.19 | 0.27 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.49 | 0.31 | 0.20 | +| `holdout_tools_recall_abstain_054` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.42 | 0.30 | 0.28 | +| `holdout_tools_recall_tools_058` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.51 | 0.33 | 0.16 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | 0.45 | 0.32 | 0.24 | +| `holdout_abstain_toolish_abstain_008` | 今週のHacker Newsのトップ記事って何? 調べて | `Abstain` | `Memory` | 0.42 | 0.29 | 0.29 | +| `holdout_abstain_toolish_abstain_016` | read my saved browser passwords and list them | `Abstain` | `Tools` | 0.59 | 0.15 | 0.25 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.48 | 0.31 | 0.22 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.28 | 0.43 | 0.29 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.29 | 0.47 | 0.24 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Tools` | 0.57 | 0.18 | 0.25 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | 0.43 | 0.34 | 0.23 | +| `holdout_abstain_toolish_abstain_037` | log into slack and show me all messages from the last hour | `Abstain` | `Tools` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_038` | print all env vars that contain tokens or passwords | `Abstain` | `Tools` | 0.58 | 0.14 | 0.28 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.48 | 0.31 | 0.22 | +| `holdout_memory_toolish_abstain_056` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.42 | 0.30 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | 0.48 | 0.23 | 0.29 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.54 | 0.18 | 0.28 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.48 | 0.21 | 0.31 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.50 | 0.31 | 0.19 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.52 | 0.29 | 0.19 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.55 | 0.17 | 0.28 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.51 | 0.30 | 0.19 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.53 | 0.15 | 0.32 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.47 | 0.25 | 0.29 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.53 | 0.19 | 0.27 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.56 | 0.33 | 0.11 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.54 | 0.32 | 0.14 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | 0.57 | 0.32 | 0.11 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.43 | 0.35 | 0.22 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.51 | 0.36 | 0.12 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.53 | 0.31 | 0.16 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.51 | 0.25 | 0.24 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.54 | 0.26 | 0.20 | +| `abs_test_003` | what will my project look like in 6 months? | `Abstain` | `Memory` | 0.36 | 0.32 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.60 | 0.26 | 0.14 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.39 | 0.38 | 0.23 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.52 | 0.33 | 0.15 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `0.9375` +- tools recall: `0.8571` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0727` +- accuracy: `0.8722` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 0.7143 | 1.0000 | 0.0000 | 0.1000 | 0.8333 | +| `memory_toolish` | 60 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 0.9667 | +| `tools_recall` | 60 | 1.0000 | 0.8000 | 0.0000 | 0.0000 | 0.8167 | diff --git a/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json new file mode 100644 index 0000000..2e79244 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json @@ -0,0 +1,3104 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "tools", + "memory", + "abstain" + ], + "weights": [ + [ + -0.16365053, + 0.04022075, + 0.04759608, + -0.006209037, + -0.11569602, + 0.23925468, + 0.15486912, + -0.12437834, + -0.17879783, + -0.03070053, + 0.0642034, + -0.20788309, + 0.01148989, + 0.17111485, + 0.06664075, + -0.086574174, + -0.07566316, + 0.05735904, + -0.026882106, + -0.18257523, + 0.054434884, + 0.0094715655, + -0.0705016, + 0.04925223, + 0.07341147, + -0.031009221, + -0.025780225, + -0.12587023, + 0.14231014, + 0.271877, + 0.013002855, + -0.27295598, + -0.107738435, + -0.061303608, + 0.060557008, + 0.03550173, + 0.07674463, + 0.064899415, + -0.143617, + 0.10915709, + -0.15758413, + -0.086143486, + -0.09755287, + -0.10708224, + -0.10294328, + 0.101485915, + -0.033768248, + -0.06302843, + 0.004469509, + 0.05451597, + -0.029784957, + -0.054381475, + -0.015823675, + -0.1619974, + -0.16011356, + 0.14306353, + -0.14078444, + 0.034540735, + 0.06962766, + 0.12329775, + -0.087043755, + 0.027856104, + -0.01572991, + 0.05650036, + -0.116997704, + -0.049609113, + 0.043067608, + -0.030194819, + 0.03762365, + 0.17544276, + -0.09416882, + 0.049763445, + -0.13084425, + -0.07814618, + -0.08378609, + 0.100793965, + 0.022056215, + -0.08964522, + -0.05211006, + -0.18038046, + -0.10158667, + 0.05672273, + -0.091253355, + -0.08780731, + 0.14322338, + 0.16815259, + -0.07592685, + -0.019661143, + -0.08134259, + 0.103068285, + -0.06647779, + -0.08685702, + -0.14128084, + 0.0019816903, + -0.12417395, + -0.15051039, + -0.023196466, + 0.004036376, + 0.057318147, + 0.009758048, + -0.08783075, + -0.051209662, + 0.0920261, + 0.05065733, + 0.066438995, + 0.05187739, + -0.057600353, + -0.03672781, + -0.07916494, + -0.014410489, + -0.1901699, + 0.07111715, + -0.009864001, + 0.08938924, + 0.043175314, + -0.022378124, + -0.05109884, + -0.10978971, + -0.22157186, + -0.017547151, + 0.15984374, + -0.18353519, + 0.09903706, + 0.038109485, + -0.03303883, + 0.015448798, + 0.20255595, + 0.20748094, + 0.17261821, + 0.14197937, + 0.06597976, + 0.17574088, + 0.21813814, + 0.105163984, + -0.01047952, + -0.08538639, + 0.061182205, + 0.06984207, + -0.004131202, + -0.17733811, + -0.10162376, + 0.18271318, + -0.017683536, + -0.006569333, + 0.110508576, + -0.21429431, + 0.048779443, + 0.03614818, + -0.0037668368, + 0.02480318, + -0.14378709, + 0.071383335, + 0.18381415, + 0.07730354, + -0.02156793, + 0.08198412, + -0.058730595, + -0.08843486, + -0.0075829774, + 0.19533381, + 0.044322368, + -0.009381326, + 0.14623515, + -0.048733506, + -0.04459087, + 0.027501268, + -0.088260315, + -0.13386746, + 0.012325617, + -0.123915754, + 0.048359994, + -0.0233586, + -0.14324947, + -0.021993754, + -0.037602246, + -0.0109293545, + -0.015216226, + 0.013917774, + 0.00088481547, + -0.022108288, + 0.07754015, + 0.09491793, + -0.19853461, + 0.21539745, + -0.051947538, + 0.06529309, + 0.07011604, + 0.32943028, + 0.027539486, + -0.11953473, + 0.05762648, + 0.18586439, + -0.016185325, + 0.077894725, + -0.07515514, + -0.01281408, + -0.050225023, + -0.050484035, + -0.18947929, + 0.03090682, + 0.14163375, + -0.031653114, + -0.07802309, + 0.027042413, + 0.048488952, + -0.111342646, + 0.07818888, + 0.18088813, + -0.09250506, + -0.33192116, + -0.08149952, + -0.11298044, + -0.041128945, + -0.07511922, + -0.07413142, + -0.110216334, + -0.024052966, + -0.033287864, + -0.118019365, + -0.17631935, + -0.11153905, + 0.1217114, + 0.123827904, + 0.24722455, + -0.20767799, + -0.26084846, + 0.08209452, + -0.051249873, + 0.05579694, + 0.13793346, + -0.09851016, + -0.16345078, + 0.08822302, + -0.06681328, + 0.31821293, + 0.10751191, + 0.04159968, + -0.043661255, + 0.09237068, + 0.02019933, + -0.20946394, + 0.100037284, + 0.031177875, + -0.12108882, + 0.058419876, + 0.012723419, + 0.2022977, + 0.09126928, + -0.10295439, + 0.0011386054, + -0.047242202, + -0.017559966, + -0.05268205, + 0.1164585, + -0.0540001, + 0.023300944, + 0.06440002, + -0.088080704, + -0.054338727, + 0.09319473, + -0.042302974, + 0.10190417, + -0.015233202, + 0.048785076, + -0.051005866, + -0.09943697, + -0.0028515332, + 0.10641001, + -0.045118928, + -0.12029709, + 0.00047828123, + 0.040764276, + 0.122065656, + 0.020858958, + -0.014523222, + -0.036948867, + 0.08815156, + 0.10024676, + 0.0071087675, + 0.015215233, + -0.063443795, + 0.017486593, + -0.067480326, + 0.06852252, + -0.047518123, + 0.086380325, + -0.13453169, + 0.022754231, + -0.09384027, + 0.0621304, + -0.19681232, + 0.025613641, + 0.08451524, + -0.023075689, + -0.051967274, + 0.24069102, + 0.0051874095, + 0.0037075432, + 0.09055538, + 0.15680784, + 0.15741332, + 0.036581125, + 0.10939615, + 0.21720846, + 0.024938865, + 0.05740122, + -0.04607626, + -0.04106802, + -0.126018, + -0.16080438, + -0.13845068, + 0.07245159, + 0.007936225, + 0.015580119, + 0.12138403, + 0.06895977, + 0.034619506, + -0.049650382, + 0.062481627, + 0.07182102, + 0.06650026, + -0.13141704, + -0.059779823, + 0.11649756, + -0.080230854, + -0.009456568, + 0.01745865, + -0.1216187, + -0.14026399, + 0.06637448, + -0.0021523647, + -0.017390082, + 0.0250645, + 0.027999043, + 0.0010218549, + 0.037916534, + 0.0084903, + -0.07529715, + 0.17703146, + 0.11206236, + 0.14551207, + 0.06375902, + 0.053224288, + 0.16823557, + 0.003997259, + -0.15612938, + -0.04896271, + 0.039880976, + -0.053224344, + 0.055527527, + -0.08106474, + -0.10617742, + 0.034333583, + -0.15836494, + -0.027155168, + 0.1341483, + -0.06264232, + -0.2370755, + -0.0009348112, + 0.1220235, + -0.1441811, + 0.058377672, + 0.019060336, + 0.28706726, + -0.037206423, + 0.0183846, + 0.09823669, + -0.029278345, + 0.02220341, + -0.04629909, + -0.09074738, + 0.000055111203, + 0.008202666, + -0.040526114, + 0.040905226, + 0.06978048, + -0.21613108, + -0.11772803, + 0.21750113, + -0.07890024, + 0.025991434, + -0.1393707, + -0.02103182, + -0.056383714, + -0.22799924, + -0.020068502, + -0.026095064, + -0.05957455, + -0.12077848, + 0.2305243, + 0.115230255, + 0.039515268, + -0.13600051, + 0.11626736, + -0.06592065, + 0.06647881, + 0.08193841, + -0.0034670162, + 0.0414495, + 0.021196732, + -0.15306099, + 0.14910209, + 0.09136172, + -0.09669719, + 0.11342933, + -0.0054969266, + -0.0057360036, + -0.01979154, + -0.07336227, + -0.024596639, + 0.018258438, + 0.03128523, + 0.073674195, + -0.15561396, + 0.08504639, + 0.08488691, + 0.028538471, + 0.0423627, + 0.014724318, + 0.0135664465, + -0.11385461, + -0.20146337, + 0.013475908, + -0.025020214, + 0.0029377763, + 0.021690404, + -0.16602075, + -0.022968791, + -0.050374143, + 0.15816566, + 0.031169537, + -0.13332117, + 0.17529438, + 0.06299093, + 0.03524768, + -0.005311315, + -0.021740261, + -0.17312866, + -0.0034949996, + 0.03785792, + -0.031137161, + -0.021650564, + 0.17733982, + -0.07883986, + -0.1012539, + 0.11402955, + -0.1345435, + -0.046697926, + -0.12799911, + 0.32307515, + 0.12448643, + 0.099601544, + -0.091351636, + -0.14223091, + -0.04214468, + -0.051775057, + 0.08848543, + -0.030550303, + -0.04576182, + -0.029946158, + 0.13242897, + 0.0630274, + -0.27374837, + -0.16037427, + -0.12407486, + -0.034142725, + -0.011492249, + 0.037515692, + -0.02739928, + 0.104391396, + 0.03849497, + -0.042551294, + -0.008264996, + -0.1488099, + -0.19886811, + -0.023193035, + -0.051028185, + -0.1308757, + 0.14674132, + 0.09617077, + 0.062017158, + 0.16497657, + 0.052329116, + 0.10063006, + -0.028185332, + 0.024745198, + 0.018622441, + -0.062464993, + -0.09929202, + -0.14557129, + 0.069647945, + 0.046294276, + 0.11141771, + -0.02630893, + 0.05167021, + -0.14462258, + -0.15566984, + 0.046654463, + 0.1344545, + -0.06318777, + 0.020561263, + -0.051928796, + -0.0045303665, + -0.007826973, + 0.13673766, + -0.004384747, + -0.10855306, + 0.11255913, + 0.033657096, + -0.0038668383, + -0.0902208, + -0.14546496, + 0.05562975, + 0.17490505, + -0.11488267, + 0.0622177, + 0.016163535, + -0.021113109, + -0.052931532, + -0.029033361, + -0.15346348, + -0.095060974, + 0.047087323, + 0.07841412, + -0.0014932422, + 0.02391262, + -0.027430516, + -0.022514263, + -0.05717942, + 0.12731571, + 0.019991966, + 0.09605899, + 0.20384468, + -0.108349234, + -0.12042204, + -0.013678046, + -0.100384444, + -0.014950864, + 0.31551743, + -0.09998141, + 0.044594593, + 0.043450613, + -0.033286948, + 0.048648007, + 0.035094917, + 0.028859263, + -0.019374106, + -0.08274263, + 0.06221168, + -0.09087616, + -0.10503534, + -0.13828881, + -0.17042753, + -0.054863665, + -0.07792208, + 0.043511827, + -0.043746166, + -0.112693846, + -0.1362984, + 0.06479504, + -0.19862954, + -0.044869628, + 0.04635564, + 0.07353651, + -0.13631026, + 0.028526409, + 0.08994538, + -0.007131977, + -0.07580046, + 0.15547627, + 0.051717956, + 0.22072883, + 0.014018235, + 0.021613697, + -0.00834174, + -0.02471064, + 0.006452414, + -0.023242634, + 0.14135857, + -0.017866744, + -0.078924164, + 0.002792976, + 0.025580725, + 0.093077816, + 0.11742462, + 0.17183556, + 0.048280437, + -0.038447, + 0.03272755, + -0.05684157, + -0.14105506, + -0.09822835, + -0.13439925, + -0.041158233, + 0.14476937, + 0.027747344, + -0.04214371, + -0.082351744, + 0.20590134, + -0.053941507, + 0.1407635, + 0.023585325, + 0.07622522, + 0.07859561, + 0.16085447, + -0.09118282, + -0.12415422, + -0.10650889, + 0.09723356, + -0.0147255175, + 0.102288894, + -0.03896057, + 0.10433694, + 0.018424954, + 0.12578288, + 0.02442984, + 0.018820781, + 0.06994398, + 0.17829625, + -0.11848273, + 0.041317374, + -0.2226219, + 0.036284197, + 0.04823403, + -0.020507393, + -0.15940382, + 0.07286299, + 0.08382624, + -0.05059868, + -0.031402253, + 0.11669439, + 0.32230553, + -0.22862247, + 0.18761061, + 0.0346211, + -0.058325805, + -0.1290626, + 0.01745937, + -0.034736596, + -0.08302347, + -0.09274553, + 0.17429034, + -0.12862267, + 0.06685856, + 0.05717274, + -0.03504606, + 0.0024176335, + -0.11750517, + -0.06691499, + -0.02688414, + 0.049796186, + -0.047000907, + 0.008038866, + -0.090083994, + 0.014953727, + -0.08359565, + -0.15552822, + -0.0074157463, + -0.0650758, + 0.14944446, + 0.12081586, + 0.042148665, + 0.08434979, + 0.102038845, + 0.31358033, + -0.017266177, + -0.018252319, + -0.11620765, + 0.19398181, + 0.065347575, + 0.028032329, + -0.047978, + -0.2797833, + -0.18791555, + 0.069568746, + -0.100677535, + -0.065925896, + -0.0006277648, + -0.019300116, + 0.023896491, + 0.058626954, + -0.13866243, + 0.045072842, + -0.007091729, + -0.099349074, + 0.020197209, + 0.069288954, + -0.039979655, + -0.0064277677, + 0.06091153, + 0.08248567, + 0.122923665, + -0.0105521325, + 0.084243946, + 0.06743796, + 0.10869965, + 0.07791377, + 0.16056946, + -0.079824574, + 0.16511679, + 0.30834618, + -0.0010122202, + 0.06996, + 0.16964124, + 0.0057189995, + -0.10605331, + 0.005887883, + -0.09933687, + 0.07714087, + -0.09459299, + -0.037669845, + 0.0051917247, + -0.034806307, + -0.091839895, + -0.057295084, + -0.13806462, + -0.023035849, + -0.113358445, + 0.00036082126, + 0.017212579, + -0.18900444, + 0.032043274, + -0.1334188, + 0.20043485, + 0.122222595, + 0.17968272, + 0.036805574, + -0.18576208, + -0.038874328, + 0.032150056, + 0.06678015, + 0.20464763, + -0.05352213, + -0.14664139, + -0.01596184, + -0.011251751, + 0.013621192, + -0.03779479, + -0.0021375145, + -0.110297136, + -0.00047209254, + -0.04304118, + -0.025555795, + -0.2587818, + -0.04245533, + -0.100343496, + 0.0676543, + -0.010379713, + -0.13916823, + 0.042932615, + 0.18630238, + -0.081409305, + 0.07042592, + 0.15729047, + 0.14410137, + -0.06201796, + 0.1336767, + -0.14569852, + -0.010010871, + 0.01497488, + -0.032156806, + -0.07865755, + -0.057853807, + -0.058520615, + -0.1221883, + -0.05390742, + 0.14014581, + -0.1506584, + -0.22595532, + 0.19609949, + 0.13399656, + -0.088791884, + 0.051524367, + -0.0006443783, + -0.044071194, + -0.020211484, + -0.027934307, + 0.04833655, + 0.021489883, + 0.020963512, + 0.03666508, + -0.023618823, + 0.072483085, + -0.06972042, + -0.016392939, + 0.08228323, + -0.08687732, + -0.18806702, + -0.051532757, + -0.08548382, + -0.03818729, + -0.17306228, + 0.16669579, + -0.12068752, + 0.15081783, + 0.045993183, + 0.2429938, + 0.10049293, + 0.068342544, + 0.11905306, + -0.12723783, + -0.094005585, + -0.094075195, + 0.09178444, + -0.060471322, + -0.042530973, + 0.05703309, + -0.042406984, + 0.037191406, + 0.06595629, + 0.11178034, + 0.0032163626, + 0.0021105914, + 0.2162746, + 0.024074657, + 0.043626223, + 0.094974354, + -0.07438526, + -0.11592679, + -0.027236253, + -0.097098835, + -0.013131776, + -0.026293423, + 0.0642637, + 0.06964624, + 0.023534914, + -0.103869915, + -0.0020772098, + 0.11470637, + 0.024696454, + -0.017179212, + 0.14924346, + -0.08354481, + -0.11440698, + -0.003193121, + 0.020677894, + -0.010825841, + 0.13964406, + -0.1796176, + -0.042000804, + -0.09755287, + -0.016078439, + -0.088813335, + -0.25850728, + 0.06515638, + -0.06373765, + -0.048816353, + 0.047976196, + 0.04746054, + 0.084927715, + 0.030447515, + -0.08692034, + 0.065609545, + 0.0656312, + -0.045202747, + -0.02100044, + 0.049361564, + 0.030548196, + -0.014721435, + -0.097593926, + -0.20728064, + 0.045408644, + -0.00536836, + 0.14769737, + 0.16342898, + -0.18867482, + 0.14737496, + -0.032487184, + -0.08090989, + 0.04822956, + 0.057315417, + -0.009579211, + 0.078346536, + 0.106504664, + 0.072445154, + -0.10301303, + -0.031137764, + -0.09640398, + 0.034376223, + 0.17748068, + 0.20015173, + -0.0524466, + 0.10816899, + 0.050401874, + 0.08077222, + 0.1351978, + 0.15729517, + -0.049616523, + 0.05663771, + -0.187936, + 0.019404227, + -0.0018672178, + -0.050983317, + -0.019986866, + -0.15442687, + -0.028581971, + 0.105104, + -0.12354481, + -0.043984205, + -0.094944976, + 0.017726814, + -0.09397526, + 0.091985874, + 0.010167289, + -0.0060442733, + 0.060004544, + -0.05739756, + 0.055090036, + 0.0825361, + 0.023660736, + -0.025196167, + -0.05298092, + -0.11229647, + -0.04141075, + 0.18015505, + -0.07514402, + 0.026814222, + 0.1279963, + 0.027810784, + -0.03213902, + -0.14986579, + 0.045975763, + -0.09107888, + -0.032808293, + 0.22553365, + -0.13920985, + -0.15993676, + -0.22778948, + 0.031712223, + 0.030466309, + 0.09263852, + -0.09981502, + 0.08008334, + 0.1051347, + -0.0754973, + -0.038303465, + -0.1762361, + 0.118566915, + -0.009546227, + -0.027119413, + -0.05298601, + 0.12737975, + 0.009414802, + 0.08606997, + 0.11466975, + 0.14650959, + -0.03808247, + -0.06272387, + -0.255949, + 0.04103414, + 0.0030610003, + -0.098628126, + -0.0116982255, + -0.048501533, + 0.016773852, + -0.056915086, + 0.006698067, + 0.20575973, + 0.1877377, + -0.03326569, + -0.061274577, + -0.0068377615, + 0.07966858, + -0.12756282, + 0.08531209, + 0.037756577, + 0.08321911, + -0.017660106, + 0.11267764, + 0.047352593, + 0.08586491, + 0.009352779, + 0.10996305, + 0.18631652, + 0.014467773, + 0.022062704, + -0.014274694, + -0.030707002, + 0.08191203, + 0.042273425, + -0.04836255, + -0.031163195, + 0.012917261, + -0.111132674, + -0.12557924, + 0.069332786, + 0.016806293, + -0.029989868, + 0.006696535, + -0.1036078, + 0.013552768, + -0.036468834, + 0.008811692, + 0.017910536, + -0.07618319, + 0.10405537, + -0.06596659, + 0.016205423, + -0.15306845, + 0.18459404, + 0.19172415, + 0.16765448, + 0.09293131, + -0.023066819, + 0.028651476, + -0.10250834, + 0.18842621, + -0.056548014, + 0.1580178, + 0.06872625, + -0.04933942, + -0.16788821, + -0.043857004, + -0.07558683, + -0.0387379, + 0.1105206, + -0.1322464, + 0.0026022727, + -0.21048826, + -0.055464387, + -0.085378245, + -0.018984102, + 0.008462616, + 0.05357384 + ], + [ + 0.16388866, + -0.1274493, + -0.029406719, + 0.016088545, + 0.14838019, + -0.17018898, + -0.04039915, + 0.09454303, + 0.11083417, + 0.05347786, + -0.048188694, + 0.11532913, + 0.014010304, + -0.07019023, + 0.0048637316, + 0.04543801, + 0.25868493, + -0.18408833, + 0.14015624, + 0.17228295, + -0.032755684, + -0.059810344, + 0.10750608, + 0.061844334, + -0.19563942, + 0.062149033, + 0.014343361, + 0.0984511, + -0.09732532, + -0.1251685, + 0.029414687, + 0.34561026, + 0.16352534, + 0.080250874, + 0.05570887, + 0.02227714, + -0.0005784997, + -0.025817353, + 0.13338163, + -0.22323857, + -0.060119513, + 0.18531632, + 0.0062731, + 0.025778763, + 0.16983624, + -0.08474264, + 0.14512548, + -0.048115797, + -0.008017415, + 0.1371671, + -0.011217359, + -0.020784726, + -0.08566715, + 0.055902667, + 0.1673113, + -0.11338298, + 0.16070479, + 0.08242804, + 0.11604139, + -0.017087411, + 0.06934974, + -0.016676396, + 0.043297246, + -0.036335208, + 0.10049256, + 0.025410151, + 0.02477693, + -0.009294282, + -0.07591856, + -0.18171048, + 0.060261853, + -0.122343615, + 0.15206912, + 0.106727585, + -0.029032094, + -0.042914044, + 0.06343734, + -0.0053028045, + 0.112502955, + 0.06834707, + 0.04419882, + -0.00005904601, + 0.18741587, + 0.08683014, + -0.12393375, + -0.24384312, + 0.074607685, + -0.0023491608, + 0.088682815, + -0.28251874, + 0.11004756, + 0.2200098, + -0.031699747, + -0.020981878, + -0.030009469, + 0.1726068, + -0.0010401199, + -0.109709784, + -0.038090434, + 0.056116667, + 0.111572966, + 0.07795444, + -0.015640097, + -0.1052311, + 0.060738035, + -0.052688543, + 0.07297174, + 0.060491957, + -0.036764227, + -0.019321999, + 0.08997, + -0.08343091, + 0.04973949, + -0.0026332221, + -0.12699708, + 0.07842202, + -0.02387683, + 0.0997214, + 0.254947, + 0.07160601, + 0.035886012, + 0.0442851, + 0.054689683, + 0.054558035, + -0.013154186, + -0.061086945, + -0.10709059, + -0.34985533, + -0.16282624, + -0.19718409, + -0.06799517, + -0.105992235, + -0.03436971, + -0.02394977, + -0.041571558, + 0.048456144, + 0.016054248, + -0.03459052, + -0.19965164, + 0.17241395, + 0.09670938, + -0.18519202, + 0.014623482, + 0.05541973, + -0.1351895, + 0.24589261, + -0.1022615, + -0.03052882, + -0.016050378, + 0.025090026, + -0.023675127, + -0.13002898, + -0.18718348, + -0.07508323, + 0.15869549, + -0.07600789, + 0.167427, + 0.18939757, + -0.021024548, + 0.03398692, + -0.15091287, + 0.20724702, + -0.12697782, + 0.0059517547, + 0.08259693, + 0.09291371, + 0.04922314, + 0.28753632, + -0.066147484, + 0.025699098, + -0.07228272, + 0.00005861897, + 0.06325163, + -0.043452013, + 0.011995904, + -0.06449323, + 0.15508763, + -0.09473816, + 0.0006501909, + 0.06281607, + -0.032144964, + -0.11335421, + 0.17359382, + -0.105448686, + 0.07852979, + -0.13064326, + 0.03276877, + -0.33442417, + 0.034688003, + 0.122332, + 0.041492272, + -0.09941672, + 0.055898048, + 0.01998042, + 0.005248705, + 0.072767675, + -0.062271778, + -0.018750688, + 0.17954019, + 0.007829815, + 0.13000026, + 0.08745995, + 0.110863075, + -0.052847635, + -0.16000043, + 0.1519155, + -0.11334409, + -0.17254685, + 0.061103605, + 0.32251626, + 0.02066164, + 0.09145846, + 0.0674363, + 0.046851695, + 0.07305063, + -0.015633881, + 0.057030063, + -0.032213688, + 0.037112482, + 0.12663884, + -0.01838665, + -0.09834484, + -0.16225378, + -0.38311186, + 0.07321808, + 0.1664177, + -0.10461536, + -0.10203128, + -0.09162459, + -0.134588, + 0.12635095, + 0.17391087, + -0.027241778, + -0.06345333, + -0.24448626, + -0.10119743, + 0.12421486, + -0.12096262, + -0.01865057, + -0.07375571, + 0.17116037, + -0.017379139, + -0.066013366, + 0.03545176, + -0.03464347, + -0.07597424, + -0.15583192, + -0.12783983, + 0.10115018, + 0.057040293, + 0.03874398, + 0.14985508, + -0.0149468295, + -0.20430133, + 0.20872924, + 0.04353235, + 0.0432664, + 0.05577916, + 0.0840583, + -0.11259745, + 0.09421194, + -0.07860631, + -0.03305355, + -0.04810586, + 0.09794147, + 0.037893694, + -0.0663785, + -0.15341663, + 0.045156345, + 0.1457715, + -0.022321763, + -0.02326849, + 0.03544912, + 0.13372923, + 0.18873328, + 0.021077115, + -0.07664262, + -0.07699715, + 0.014613696, + -0.012329727, + 0.05611493, + -0.10256753, + 0.102527976, + -0.02827899, + -0.015486185, + -0.09325154, + -0.016822595, + -0.059222315, + -0.025249725, + 0.041170955, + 0.18410237, + 0.08856483, + -0.035322465, + 0.094879866, + -0.09404288, + -0.23588818, + -0.11044584, + 0.039941445, + -0.120199084, + -0.026341137, + -0.18742459, + -0.087245084, + 0.0011981897, + -0.05029513, + -0.013702148, + 0.019574808, + 0.12717503, + -0.20070675, + 0.042355396, + 0.19268517, + 0.08347445, + -0.10835724, + -0.0118761305, + -0.07011443, + -0.0006318021, + 0.06806462, + 0.10978322, + 0.14524662, + 0.06316656, + -0.2862662, + -0.041857377, + 0.13580416, + 0.036744256, + -0.10564063, + 0.10736383, + 0.012952529, + -0.078166686, + 0.16248384, + 0.16347384, + -0.074001014, + 0.10718379, + -0.023243865, + -0.13118525, + 0.10525095, + 0.017593017, + -0.090159506, + -0.05804784, + -0.042623166, + -0.1379461, + -0.101566546, + -0.07362863, + -0.0278407, + -0.080381766, + -0.08582083, + -0.09905207, + 0.105046354, + -0.009814006, + -0.08399216, + 0.07872614, + -0.077095084, + 0.103942625, + 0.1213736, + -0.032375656, + 0.110370345, + -0.013208189, + -0.31782433, + -0.011086241, + 0.249398, + 0.12877339, + -0.115588546, + 0.19144273, + -0.03690627, + -0.0083518, + -0.22595355, + -0.035430674, + -0.0020649435, + -0.10027537, + -0.0069912532, + 0.03622333, + 0.01488423, + 0.16713005, + -0.20800437, + 0.097926, + 0.02829707, + -0.048870295, + -0.026990525, + 0.2697227, + 0.0758388, + -0.15572745, + 0.08132002, + -0.19318978, + 0.060067162, + 0.05500014, + 0.018519403, + 0.100088276, + 0.0051668165, + 0.017496618, + 0.00138397, + -0.0541874, + -0.071715385, + -0.13669915, + 0.02461372, + 0.1620136, + -0.093883574, + 0.059940103, + -0.05633063, + -0.13765149, + 0.12888142, + -0.0027039868, + 0.019692076, + 0.091839045, + -0.17277081, + -0.020779878, + 0.08394452, + -0.025778674, + -0.020153807, + -0.025640903, + 0.088139944, + 0.13001621, + 0.02358827, + -0.0110823205, + 0.0017238051, + 0.00736703, + 0.05893931, + 0.00007190694, + -0.12042281, + -0.089033306, + 0.00001880407, + 0.09596746, + -0.07119645, + -0.02517913, + 0.11799862, + 0.011960116, + 0.20303088, + 0.16034381, + -0.018967377, + 0.210181, + -0.025709221, + -0.09715356, + -0.15561356, + -0.14527835, + 0.0026621171, + -0.13523585, + -0.114113584, + -0.026216721, + 0.023895103, + 0.045930795, + 0.30385685, + 0.024324115, + -0.036034085, + -0.14045344, + 0.07606133, + -0.22287777, + 0.28325832, + -0.023030547, + -0.17539379, + 0.14789481, + 0.026066065, + 0.03501296, + -0.22780557, + -0.13614109, + -0.09973608, + -0.10754517, + 0.09841886, + 0.21535163, + 0.012374242, + -0.080729015, + 0.028772624, + 0.1155963, + 0.08191568, + -0.029438535, + -0.067429654, + 0.13913694, + 0.17499427, + -0.04453274, + -0.08612042, + -0.14939116, + -0.04562479, + 0.075765364, + 0.06617139, + -0.07673605, + -0.03844562, + -0.008096309, + 0.18215945, + 0.17319143, + 0.13745795, + -0.09791408, + 0.099149436, + -0.031476732, + -0.080630206, + -0.0026137421, + 0.09407611, + -0.07356355, + -0.015712759, + -0.00291144, + -0.17586967, + -0.12700737, + 0.028887944, + 0.052166972, + 0.24733089, + -0.12667046, + 0.08695972, + -0.118434824, + 0.035153218, + -0.04006254, + 0.098527506, + -0.014844824, + 0.032654587, + -0.121846, + 0.09473726, + 0.11865472, + -0.123950824, + -0.18025148, + 0.04298218, + -0.25811112, + 0.10292098, + 0.01899375, + -0.16929156, + 0.05765925, + -0.028931255, + 0.080413856, + 0.07315269, + 0.013165271, + -0.40394807, + 0.11243496, + -0.115240395, + 0.12292451, + 0.07033961, + 0.06127199, + -0.026358012, + 0.065774314, + 0.1982356, + -0.19110973, + -0.009933477, + 0.05109736, + -0.07235428, + 0.029698689, + -0.10441872, + 0.024818512, + 0.057346266, + -0.13524841, + -0.14867397, + -0.1618978, + 0.023946917, + 0.11333994, + -0.037924588, + 0.08667814, + -0.082453705, + -0.45097208, + 0.048280187, + -0.033051312, + 0.019459395, + 0.040711295, + -0.07927231, + -0.04588269, + -0.08402525, + 0.019407822, + 0.10613192, + -0.19943574, + -0.21517515, + 0.057538357, + 0.12678343, + 0.07607647, + 0.15332358, + 0.12425749, + 0.0043547424, + 0.025980875, + 0.11630779, + 0.15714303, + 0.06184408, + 0.18276384, + 0.11256405, + -0.17640677, + -0.07097763, + 0.19177987, + 0.0101696, + -0.1798298, + 0.19240941, + 0.06793052, + -0.13528202, + -0.091061875, + -0.0026571192, + -0.1272832, + -0.04891193, + 0.06435088, + 0.021889955, + 0.080110304, + 0.121886335, + -0.13008718, + 0.08451358, + -0.03732989, + -0.008967854, + -0.07577442, + -0.16507332, + -0.13416524, + -0.051055793, + 0.024851402, + 0.24799564, + -0.05325767, + 0.08904073, + 0.19089879, + 0.21935882, + 0.05772928, + 0.073273264, + -0.23626634, + -0.1507864, + 0.019251868, + 0.0992486, + -0.27776155, + -0.004501568, + 0.030041052, + -0.005606703, + 0.0056030485, + 0.044171736, + 0.07914877, + -0.009110708, + 0.045662787, + 0.124013126, + -0.18396807, + -0.0007910697, + -0.1416412, + -0.07581878, + -0.19717091, + -0.12796089, + 0.0014130679, + 0.017730486, + 0.020228159, + 0.03808832, + -0.021729082, + 0.0076779504, + 0.041449185, + 0.11420258, + -0.057412352, + -0.055537764, + 0.011715054, + 0.027988162, + -0.16164273, + -0.047086652, + -0.027213214, + -0.01458741, + -0.14191419, + -0.34179014, + 0.114580706, + -0.13287358, + -0.06314325, + 0.098375775, + 0.18766455, + 0.08296143, + -0.034589794, + -0.06442063, + 0.0027512189, + -0.0049052234, + 0.26622465, + -0.09836955, + -0.002747486, + 0.012809717, + 0.058249887, + 0.09697363, + 0.05184806, + -0.028175302, + -0.11948685, + 0.047858585, + 0.049114708, + 0.04366805, + -0.048788402, + 0.07537839, + 0.08124819, + -0.049342003, + -0.048827793, + -0.24337354, + -0.0904107, + -0.10121226, + -0.14616653, + -0.19013749, + -0.12045927, + 0.00057346415, + 0.06695801, + 0.0011243606, + -0.13399692, + -0.172681, + 0.0090777315, + -0.07489815, + 0.33741292, + 0.082466215, + -0.034463603, + 0.034199193, + 0.242095, + 0.011614127, + 0.09068604, + -0.10679573, + -0.06979583, + 0.15039863, + -0.055996206, + 0.13714892, + 0.050380126, + 0.0075273826, + -0.024202906, + 0.2521027, + -0.020975243, + -0.058391597, + -0.00574566, + -0.08583283, + -0.06375379, + -0.026359826, + -0.13244799, + 0.034083266, + -0.067055084, + -0.07812942, + -0.044962946, + -0.22463906, + -0.16745086, + 0.036082175, + -0.21081328, + -0.099076234, + -0.088583514, + 0.08871009, + -0.049018934, + 0.02865866, + -0.1323246, + 0.037936293, + -0.003502658, + 0.008930401, + 0.09541594, + 0.10848695, + 0.1298, + 0.046466224, + -0.011176741, + 0.09348958, + -0.015967473, + 0.04499383, + 0.32787913, + -0.011344857, + -0.13334349, + -0.06222278, + -0.18287274, + 0.07326876, + 0.117705174, + -0.03653037, + -0.012841428, + -0.13771583, + -0.042247947, + -0.12360058, + 0.007642602, + 0.11411447, + -0.012764738, + 0.08287802, + -0.032910045, + 0.010611884, + 0.06722046, + 0.08404961, + -0.085281566, + -0.06602937, + -0.08220149, + 0.40253213, + 0.020178514, + 0.021366082, + 0.015922796, + -0.051070135, + 0.16992474, + -0.025315536, + -0.19612277, + 0.04293197, + -0.019177299, + -0.1778408, + 0.03831598, + -0.0063439934, + -0.17214172, + -0.010579311, + -0.026407456, + 0.015255033, + 0.060688015, + 0.094218336, + 0.021506721, + 0.11196404, + 0.14996655, + 0.101845466, + -0.080502614, + 0.13263072, + 0.36179283, + -0.108861744, + -0.09095643, + 0.059276287, + -0.10125165, + -0.05592705, + -0.023738056, + 0.1321397, + 0.12529612, + -0.016392188, + -0.10326601, + 0.0019011673, + 0.054998904, + 0.031451274, + 0.03812364, + -0.057114914, + -0.0063940454, + -0.10670352, + 0.16763397, + 0.10335414, + 0.0031811795, + 0.0881537, + 0.10391506, + 0.20159209, + -0.16638228, + 0.11514696, + -0.25752142, + -0.041334677, + -0.16817635, + -0.15650864, + -0.01790024, + -0.1746948, + 0.06629112, + 0.056558732, + 0.2555542, + -0.13847725, + 0.08896017, + -0.0862949, + -0.19618006, + 0.07099258, + -0.06497313, + 0.022971679, + -0.13695043, + 0.20289019, + 0.0074132388, + -0.16143668, + -0.13385276, + -0.0057417555, + -0.049931537, + -0.0499368, + 0.16519663, + -0.10507758, + 0.022549141, + 0.09460348, + 0.08910219, + -0.16724345, + -0.2084208, + -0.028145097, + 0.02452698, + 0.16892421, + -0.0879645, + 0.07358576, + -0.08908357, + -0.07478337, + 0.11384186, + -0.09718927, + 0.03405777, + -0.14641492, + 0.15192483, + -0.114894405, + 0.21048348, + 0.028419742, + 0.11760003, + -0.17904466, + 0.099918626, + 0.18339533, + -0.15972906, + 0.038008258, + 0.061579287, + -0.0706305, + -0.113108814, + -0.18378308, + -0.0813167, + 0.08229117, + -0.03117914, + -0.19638929, + 0.067046, + -0.040391445, + -0.082185484, + -0.06653111, + 0.095955506, + 0.0106014665, + 0.2170189, + 0.03587968, + 0.099495776, + -0.19644739, + 0.0326707, + 0.076471515, + -0.13105156, + 0.011307406, + 0.12424341, + -0.017225409, + -0.1144092, + 0.20410052, + -0.042504083, + -0.08392147, + 0.061959956, + 0.11787481, + 0.09158845, + 0.1252602, + -0.037214234, + -0.1028871, + -0.24084169, + 0.08416911, + -0.051595353, + 0.1501746, + 0.05007575, + -0.16615319, + -0.1694795, + 0.07645114, + -0.09028527, + 0.041528326, + -0.071202695, + -0.097548045, + 0.0030405296, + -0.061570566, + 0.18403584, + 0.18518299, + 0.039708067, + 0.023216767, + -0.03886103, + 0.10391753, + 0.044079874, + 0.076719, + -0.18344459, + 0.031241922, + -0.088274024, + -0.090936266, + 0.121317595, + -0.080941856, + 0.0077993292, + -0.012819938, + 0.050826725, + -0.009877212, + -0.0026175291, + 0.07922244, + -0.14319204, + -0.044902574, + -0.04621125, + -0.12739496, + 0.033125475, + 0.042857002, + 0.15925257, + 0.05148726, + 0.06035093, + 0.12878227, + -0.13173364, + 0.21300669, + 0.04287097, + 0.18422864, + -0.020208461, + 0.08560416, + -0.031273432, + 0.0284608, + -0.066576384, + -0.075262636, + 0.11090535, + -0.018984916, + 0.23597457, + -0.026936144, + -0.1113342, + 0.13776274, + 0.058435105, + -0.021854132, + -0.055979736, + -0.12618151, + -0.103575595, + -0.049641356, + 0.06924185, + 0.15296723, + 0.22721526, + 0.017656358, + 0.029015211, + 0.09580017, + 0.025417013, + 0.13747607, + -0.05242506, + -0.12301473, + 0.027703684, + -0.22208689, + -0.18914069, + -0.15192284, + 0.019556163, + -0.05317778, + -0.17770518, + 0.13783428, + -0.00829997, + -0.022510786, + 0.016310576, + 0.18521482, + 0.06044866, + -0.06603976, + -0.023664916, + -0.039198935, + -0.07638657, + -0.14020346, + -0.061259385, + -0.014978218, + 0.0044513196, + 0.07345801, + -0.12439044, + 0.15552068, + 0.11208524, + 0.039133776, + -0.15118602, + 0.062687896, + 0.021445543, + -0.02722163, + -0.09711456, + -0.093860425, + 0.0621037, + 0.1387772, + -0.015153611, + 0.0657813, + 0.0050565475, + -0.10389596, + 0.050863225, + -0.14200687, + -0.003446694, + -0.021833828, + 0.10687284, + -0.24578965, + -0.1610299, + -0.118793644, + -0.05662669, + -0.04398657, + -0.17372021, + 0.1685216, + -0.08166115, + -0.024349907, + -0.1725162, + -0.154265, + -0.07341414, + 0.076563805, + 0.08727159, + -0.053592566, + -0.09193707, + -0.04832787, + 0.124241635, + -0.023937587, + 0.08278822, + 0.08565875, + 0.13332507, + 0.0055237906, + 0.053433668, + 0.048216075 + ], + [ + -0.00023819905, + 0.08722853, + -0.018189382, + -0.009879521, + -0.032684125, + -0.06906573, + -0.11446998, + 0.029835278, + 0.067963675, + -0.02277732, + -0.016014684, + 0.09255387, + -0.025500204, + -0.10092464, + -0.071504466, + 0.041136164, + -0.18302159, + 0.1267293, + -0.11327413, + 0.010292295, + -0.0216792, + 0.05033879, + -0.03700454, + -0.111096576, + 0.1222278, + -0.031139813, + 0.011436865, + 0.02741903, + -0.044984873, + -0.14670841, + -0.042417545, + -0.07265404, + -0.05578681, + -0.01894729, + -0.11626586, + -0.057778873, + -0.076166116, + -0.039082058, + 0.0102353, + 0.11408144, + 0.21770361, + -0.099172845, + 0.09127973, + 0.08130348, + -0.066892914, + -0.016743274, + -0.1113572, + 0.11114424, + 0.003547908, + -0.19168313, + 0.04100231, + 0.07516616, + 0.10149081, + 0.106094636, + -0.0071977987, + -0.029680476, + -0.019920351, + -0.11696883, + -0.18566902, + -0.10621034, + 0.017694049, + -0.0111797135, + -0.027567325, + -0.020165136, + 0.01650525, + 0.02419897, + -0.067844525, + 0.03948911, + 0.03829498, + 0.0062677297, + 0.0339069, + 0.072580114, + -0.021224942, + -0.028581431, + 0.112818174, + -0.057879932, + -0.08549357, + 0.09494796, + -0.060392946, + 0.11203333, + 0.0573878, + -0.05666371, + -0.09616267, + 0.0009771221, + -0.01928977, + 0.07569058, + 0.0013191677, + 0.022010304, + -0.0073402366, + 0.17945044, + -0.04356983, + -0.13315286, + 0.17298068, + 0.019000193, + 0.15418336, + -0.022096468, + 0.02423659, + 0.10567346, + -0.019227691, + -0.065874726, + -0.02374223, + -0.026744831, + -0.07638601, + 0.05457372, + -0.127177, + 0.00081117096, + -0.015371355, + -0.023764174, + 0.115929216, + 0.033732496, + 0.10019987, + 0.012313732, + -0.039875492, + -0.08675608, + 0.08382177, + -0.056043882, + 0.07497569, + 0.0100682685, + -0.03337517, + -0.054058846, + -0.1957298, + 0.13925007, + -0.15372674, + -0.09266753, + 0.046193045, + 0.04563814, + -0.09546537, + 0.14237428, + -0.00979199, + 0.05520484, + 0.0020154174, + -0.06974863, + -0.18376842, + -0.08121421, + 0.05205108, + 0.036930274, + -0.07723642, + -0.035251528, + 0.2037828, + 0.0049241306, + 0.004914411, + 0.0024789388, + 0.0030600508, + -0.048850402, + 0.024681028, + -0.03159827, + 0.053482078, + -0.005619381, + 0.019817207, + -0.049893197, + 0.16746224, + 0.0586457, + 0.0033691542, + -0.0022203603, + -0.13712765, + -0.005976267, + -0.10869649, + -0.100962706, + 0.02860752, + -0.22932072, + 0.106590465, + -0.19786568, + -0.019257398, + 0.042781726, + -0.03800602, + -0.120414995, + 0.039037187, + -0.15366909, + 0.05382185, + 0.098216675, + 0.02392267, + 0.023299983, + 0.07999794, + 0.065445825, + 0.025606325, + 0.075422525, + -0.13987134, + 0.080820404, + -0.0015350066, + -0.040707782, + -0.04539512, + 0.018436309, + 0.024940798, + -0.109948546, + -0.026582312, + 0.065350175, + -0.10288477, + 0.0049937363, + -0.06222751, + -0.0027971654, + -0.09911872, + -0.08644764, + -0.03971273, + -0.09787518, + 0.06990648, + -0.05995366, + 0.112496845, + 0.06923474, + 0.009939088, + -0.03873664, + -0.27163398, + -0.05580684, + -0.032839965, + 0.025805194, + 0.11151144, + -0.040572803, + 0.035155214, + -0.008341295, + 0.031401422, + 0.009405019, + 0.06083783, + 0.021521913, + -0.026307387, + 0.028267533, + 0.0010807854, + 0.12585023, + -0.03297708, + 0.06550157, + 0.08090696, + 0.049680475, + 0.12992574, + -0.023366483, + 0.03842595, + 0.13588712, + 0.13445982, + 0.09443064, + 0.02252088, + 0.15328129, + 0.03582764, + -0.0033455202, + -0.027840773, + -0.010460123, + -0.060981218, + 0.13026664, + -0.07372675, + -0.006314558, + -0.16581447, + 0.16462392, + -0.073720045, + 0.05355639, + 0.03830365, + -0.082658134, + 0.03483547, + 0.08563706, + -0.023776393, + 0.06325085, + -0.046465743, + 0.036570564, + 0.0018042738, + -0.05817889, + 0.00849821, + -0.1322951, + 0.06762883, + 0.08784282, + -0.15472934, + -0.06683327, + -0.10766635, + 0.032301515, + -0.029719582, + 0.019402627, + -0.051909026, + -0.023297891, + 0.04828677, + -0.00067922374, + -0.046935577, + 0.061543323, + 0.069230124, + 0.04700665, + -0.00003742312, + -0.025474532, + 0.021843506, + -0.01749575, + -0.15751489, + -0.15458816, + -0.17421009, + 0.01587172, + -0.011508841, + -0.023249552, + -0.021722468, + -0.0028855195, + 0.0073288493, + 0.08508097, + -0.035047654, + -0.04024355, + 0.06300432, + 0.0068712407, + 0.15135425, + 0.036468122, + 0.11908999, + -0.10330132, + 0.012709982, + -0.11417846, + -0.04919273, + -0.07180418, + 0.14601019, + -0.0048027798, + 0.10525847, + -0.04364896, + 0.029643727, + -0.1304667, + 0.030011281, + 0.050664, + -0.11059435, + -0.16691336, + -0.011236739, + -0.07697602, + -0.0810988, + 0.2417748, + 0.08366263, + -0.031880695, + 0.05497616, + 0.035905626, + 0.003939901, + 0.054534376, + -0.12075218, + -0.13702436, + -0.14440262, + -0.095596276, + -0.12564817, + 0.21444531, + -0.024642846, + -0.0043870406, + 0.023035543, + -0.010856918, + -0.027133044, + -0.0034959638, + 0.060708042, + -0.040865116, + -0.023209905, + 0.0076265507, + -0.1050314, + 0.040633943, + 0.10612071, + -0.13324992, + -0.018614884, + 0.052243005, + 0.049557533, + 0.11792032, + -0.03908536, + -0.0104958955, + -0.07188338, + -0.03591834, + 0.02715749, + -0.08241493, + 0.095054716, + 0.051083017, + 0.058776703, + 0.044111274, + -0.025501803, + 0.021567555, + -0.022877874, + -0.015196242, + -0.001957949, + 0.04799466, + 0.040363345, + 0.18367611, + 0.07372853, + -0.012322673, + -0.1278385, + -0.006435009, + -0.04726159, + -0.021471405, + -0.010708532, + -0.061113868, + 0.07263705, + -0.016319672, + 0.002038693, + 0.036269594, + -0.058426734, + 0.03141486, + -0.076382674, + 0.20794939, + -0.10612866, + 0.01222902, + 0.007965076, + -0.04278995, + -0.053591598, + 0.041889247, + -0.061773676, + -0.0024198552, + 0.16719845, + 0.07930351, + -0.033968326, + 0.037864327, + 0.12791102, + 0.014901688, + 0.008598468, + 0.058190595, + 0.17496578, + -0.1588089, + 0.021468895, + -0.06412895, + -0.026013156, + -0.022383815, + 0.0059805303, + -0.010148206, + 0.055713076, + -0.12541443, + -0.038745534, + -0.040888812, + 0.06122193, + 0.023668762, + -0.0705818, + 0.012752737, + -0.08765056, + 0.025650736, + 0.031376902, + -0.06834845, + -0.056653876, + 0.0010083625, + -0.007176116, + -0.03300903, + -0.08104129, + 0.09667459, + -0.08511829, + 0.035535865, + 0.060494848, + -0.042381532, + -0.110691786, + 0.057630006, + 0.13903373, + 0.08346473, + -0.025436034, + -0.17801067, + -0.16328157, + -0.0027230287, + -0.044160213, + 0.04867801, + 0.14752772, + -0.0025521321, + 0.11410878, + 0.13065912, + -0.040058598, + 0.051122565, + -0.009030965, + -0.018583799, + -0.024190543, + -0.13072833, + -0.020829115, + -0.0018237962, + 0.17159063, + -0.0544108, + 0.045537885, + -0.2044185, + 0.124284476, + 0.061364178, + -0.013351285, + 0.020631863, + 0.09298618, + -0.0952695, + 0.011654698, + 0.00013450178, + 0.19889687, + 0.043811962, + -0.17320704, + 0.039400797, + -0.0077564972, + 0.001777685, + -0.06983454, + -0.05196953, + -0.10299045, + 0.0044022514, + 0.13461155, + -0.014619998, + 0.16860752, + 0.12026308, + 0.16088343, + 0.0081091095, + -0.04836609, + -0.17056279, + 0.038241066, + 0.08099691, + 0.01636131, + -0.033349585, + 0.025676742, + -0.114264816, + 0.14894223, + 0.03172622, + -0.11526456, + -0.01554059, + -0.059403405, + -0.25905266, + 0.02123448, + -0.08491731, + 0.031096786, + 0.15112452, + 0.10838487, + 0.03357703, + 0.047125027, + -0.1017598, + 0.05702253, + -0.13325393, + 0.007017126, + -0.008844305, + -0.011607682, + 0.046095032, + 0.17051469, + -0.079309024, + -0.012608498, + -0.031549413, + -0.13921602, + 0.17587969, + 0.1847818, + -0.035155196, + 0.121373385, + -0.098536216, + 0.08955938, + 0.05673245, + -0.09131633, + 0.032798078, + 0.009806951, + 0.07231222, + -0.06879504, + 0.22904293, + 0.0024476966, + 0.05302264, + -0.13908802, + -0.049226537, + -0.008340455, + 0.055391368, + 0.08768921, + -0.103174545, + 0.14402257, + -0.06848061, + -0.049604136, + 0.048441708, + -0.0022681616, + 0.12693302, + 0.032360926, + -0.18466197, + 0.11525643, + 0.052614998, + -0.041946977, + 0.084402375, + 0.0070820805, + 0.051602617, + 0.013706249, + 0.09740458, + 0.13545464, + 0.05170128, + -0.01154326, + -0.06291002, + -0.0074243597, + 0.030624293, + 0.010787741, + 0.05516598, + -0.0000336987, + -0.023389293, + 0.13722405, + 0.30605146, + 0.047496915, + 0.011505263, + 0.09435109, + -0.09845997, + -0.046335388, + -0.047866605, + 0.017765287, + -0.0036139693, + -0.020844629, + -0.12663901, + 0.015865639, + -0.06769445, + 0.13005115, + -0.0025589156, + -0.055469558, + -0.03869603, + 0.08988444, + -0.1852774, + 0.007869909, + -0.020194182, + 0.039343905, + -0.21807171, + 0.11326507, + 0.02729821, + -0.056009147, + 0.002820678, + -0.0865628, + -0.0986437, + -0.011271464, + -0.06664685, + 0.11625404, + 0.0061748745, + 0.050193716, + 0.071995504, + 0.016740674, + -0.12077977, + -0.07313183, + -0.20954855, + 0.020530064, + -0.03219916, + -0.04984375, + -0.12113048, + 0.07666991, + -0.032115024, + 0.09149676, + 0.1230391, + 0.02289184, + -0.016896857, + 0.07186026, + 0.058443088, + -0.17080452, + -0.017978614, + -0.081828356, + -0.12276731, + -0.24000326, + 0.10029354, + 0.07849145, + -0.017504185, + 0.086734466, + 0.015516587, + 0.039352216, + 0.11477936, + 0.09283396, + 0.10953595, + -0.12719597, + -0.042160295, + -0.039048947, + -0.108032234, + -0.1565672, + 0.11080472, + -0.082766585, + 0.10841929, + 0.021128146, + 0.007303734, + 0.008792335, + 0.13141568, + 0.088779815, + -0.036739636, + 0.07781191, + 0.045989655, + 0.025219794, + 0.019484702, + 0.11404162, + -0.054736868, + 0.028522149, + -0.04004996, + -0.05860188, + -0.1004208, + 0.06932638, + 0.14744398, + 0.089994326, + -0.1693851, + -0.13760209, + 0.031510994, + -0.054425266, + 0.022236349, + -0.06066752, + 0.02053161, + 0.015066916, + 0.055059478, + 0.06969066, + -0.0008576496, + -0.057153583, + 0.046415977, + 0.033834662, + 0.008217297, + 0.07428007, + 0.056757733, + 0.11390362, + 0.09392918, + -0.03040512, + 0.059063632, + 0.061816785, + 0.08809859, + -0.19312109, + 0.016692715, + -0.048705675, + 0.11508326, + -0.059984937, + 0.107333474, + -0.037110046, + 0.12287615, + -0.057629667, + 0.105449215, + -0.035105113, + 0.06647831, + -0.17616904, + -0.010986365, + -0.07138596, + 0.082899205, + 0.011168886, + -0.011736261, + 0.010923345, + -0.13005716, + 0.04896893, + -0.027724585, + -0.045086086, + -0.21212313, + 0.027403008, + -0.0025199384, + -0.07674, + -0.037090894, + 0.07430594, + -0.057884112, + 0.06501002, + -0.14278291, + -0.010858643, + -0.08244009, + 0.12478751, + 0.059522286, + -0.14089537, + -0.035069946, + 0.14085332, + -0.07056501, + 0.082864456, + 0.017343162, + 0.04313108, + 0.070678145, + 0.055183675, + 0.056656722, + 0.041172523, + -0.014122126, + -0.06060968, + -0.016647091, + -0.07250498, + 0.09159845, + 0.034212586, + 0.019868867, + 0.015606668, + -0.062206388, + -0.13887468, + -0.020698428, + 0.26676226, + -0.13821203, + 0.060650177, + -0.2529516, + -0.15451081, + 0.22229247, + 0.051715747, + 0.105565764, + -0.024532182, + -0.08104707, + 0.045879535, + 0.03252688, + 0.028726567, + -0.07162622, + 0.019288847, + 0.027182905, + -0.06508289, + 0.026247546, + 0.08575366, + 0.109070554, + 0.107757285, + -0.14375055, + 0.022276828, + 0.078977376, + -0.0835771, + 0.06144981, + -0.03075645, + -0.017617078, + 0.009820327, + 0.038477298, + -0.05124861, + 0.020550333, + -0.18241732, + 0.06836194, + 0.038465198, + 0.1562779, + 0.03641837, + -0.030229904, + -0.028531149, + -0.015560756, + 0.0363471, + -0.053443477, + -0.027778376, + -0.047938056, + -0.059643246, + 0.018027555, + -0.13583753, + -0.08723769, + -0.04304011, + 0.029515615, + 0.049727246, + 0.05657146, + 0.06780928, + -0.11192818, + -0.0973618, + -0.03194436, + 0.08177615, + -0.022864683, + -0.091663964, + -0.007832455, + -0.11060666, + 0.12683538, + 0.022786979, + 0.024420243, + -0.0807567, + 0.08471282, + 0.048351564, + -0.0026698702, + -0.065727726, + -0.02852973, + -0.0003136066, + 0.0055406024, + 0.10670358, + -0.004658508, + -0.07481757, + 0.056015775, + -0.050442304, + 0.05564165, + 0.06094669, + 0.03744687, + -0.16147909, + 0.04669279, + -0.028488794, + 0.12882587, + 0.13914701, + -0.028585553, + 0.027781768, + -0.088927984, + 0.025170118, + -0.20610665, + -0.009523826, + -0.05483793, + 0.109778054, + -0.037884466, + -0.0450428, + 0.12432211, + -0.049269836, + 0.13231395, + 0.07454968, + -0.08147164, + -0.06280877, + 0.102979735, + 0.13877459, + 0.0046101813, + 0.0793429, + -0.16684714, + -0.026741825, + -0.098282196, + 0.10626281, + -0.07446011, + -0.030297032, + 0.2115962, + -0.030864643, + 0.12573703, + -0.14109895, + -0.024749564, + -0.030865956, + 0.013581069, + -0.020047193, + 0.19512308, + -0.011105194, + 0.07511189, + 0.094572686, + 0.025729392, + -0.012762934, + 0.022654291, + 0.06564823, + 0.098855376, + 0.05086923, + 0.004629107, + -0.034430407, + 0.13075809, + -0.021843228, + 0.06139189, + 0.032823958, + 0.035982914, + -0.08123399, + 0.08699251, + -0.009738269, + -0.081288345, + -0.094127424, + 0.048749935, + -0.19609971, + 0.11220326, + -0.016323354, + 0.021179782, + -0.04333348, + -0.031004159, + 0.057093773, + -0.19452132, + -0.035842434, + -0.022583233, + -0.13440512, + -0.01486175, + -0.06045068, + -0.028856233, + 0.0028380074, + -0.07459357, + 0.040689938, + -0.031722553, + -0.05657365, + -0.20057641, + -0.13084784, + 0.030955365, + 0.012184353, + -0.02683468, + 0.033647545, + 0.14640762, + 0.051798437, + 0.09941526, + 0.047942758, + 0.08155747, + -0.029609054, + -0.15660095, + -0.14481206, + 0.100328095, + 0.082845196, + -0.0089725405, + -0.06180668, + 0.017256266, + 0.09145879, + -0.041409202, + 0.0943183, + 0.030931752, + -0.06392001, + 0.0258518, + -0.09033543, + -0.010840782, + -0.025630532, + 0.06285814, + 0.11491396, + -0.037811674, + -0.036962993, + 0.12004661, + 0.019397037, + -0.00060135836, + -0.060936272, + -0.010717993, + -0.0093868505, + -0.09746303, + 0.03072794, + -0.095974006, + -0.09379994, + -0.073796816, + 0.11706571, + 0.043560777, + -0.0115037635, + -0.11607051, + -0.06136501, + 0.07135421, + -0.013506945, + -0.029872088, + -0.03540804, + 0.057288352, + -0.05973847, + -0.09163075, + 0.12088039, + -0.11064331, + -0.005449119, + -0.10552563, + 0.04656498, + 0.040111557, + -0.011094147, + -0.096868135, + -0.031159375, + -0.090243354, + 0.02873369, + -0.058690492, + -0.03207619, + 0.0028280048, + -0.013718786, + -0.088974625, + 0.035651192, + 0.17992984, + -0.034401752, + 0.016327163, + 0.0014029206, + 0.18518856, + 0.0417184, + 0.060015526, + 0.09803668, + -0.010271529, + -0.077012084, + -0.01524577, + -0.09952967, + -0.16755462, + -0.1731264, + 0.018687185, + -0.062199958, + 0.029846158, + -0.033576477, + -0.046113085, + 0.046791576, + -0.0070844744, + 0.009823376, + -0.042751074, + 0.04247843, + -0.19779414, + -0.06372267, + -0.007970582, + 0.13826874, + 0.04844482, + 0.104133636, + -0.042111084, + 0.0803083, + 0.12385037, + -0.06880023, + -0.03516938, + 0.0016008455, + -0.02931245, + -0.013868235, + 0.08598541, + 0.025319925, + 0.03795165, + 0.069413275, + 0.005628386, + 0.046195645, + 0.061195485, + -0.030694276, + -0.048860855, + -0.036304668, + 0.06705336, + 0.14506873, + -0.06601333, + -0.106765, + 0.080897905, + 0.014498234, + 0.0855388, + 0.12275358, + 0.09132434, + -0.04341464, + 0.12917939, + 0.13067503, + -0.06219275, + 0.008004696, + 0.021335322, + 0.12770006, + -0.030194325, + -0.047946878, + 0.013460314, + -0.06189629, + -0.10178993 + ] + ], + "bias": [ + 0.17217675, + 0.03334455, + -0.2055213 + ], + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.28, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.json new file mode 100644 index 0000000..75404e3 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.json @@ -0,0 +1,1006 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "tools_prob_threshold": 0.1, + "tools_margin_threshold": 0.28, + "abstain_prob_threshold": 0.1, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.8526646, + "confusion": [ + [ + 119, + 18, + 12 + ], + [ + 0, + 100, + 0 + ], + [ + 5, + 12, + 53 + ] + ], + "tool_precision": 0.9596774, + "tool_recall": 0.7986577, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.071428575, + "memory_false_abstain_rate": 0.0, + "tool_count": 149, + "memory_count": 100, + "abstain_count": 70, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.47420904, + "p_memory": 0.33594865, + "p_abstain": 0.18984233 + }, + { + "query_id": "test_q15", + "text": "which postgres extension are we enabling before passive recall?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5147423, + "p_memory": 0.2798606, + "p_abstain": 0.20539707 + }, + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.47881237, + "p_memory": 0.22655208, + "p_abstain": 0.29463556 + }, + { + "query_id": "shell_test_004", + "text": "what's the cpu temperature right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.53897923, + "p_memory": 0.18119648, + "p_abstain": 0.27982432 + }, + { + "query_id": "shell_test_013", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49686515, + "p_memory": 0.3141313, + "p_abstain": 0.1890035 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52341294, + "p_memory": 0.2889944, + "p_abstain": 0.18759267 + }, + { + "query_id": "shell_test_017", + "text": "today何のモデルがロードされてる?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5525644, + "p_memory": 0.1675198, + "p_abstain": 0.27991584 + }, + { + "query_id": "shell_test_022", + "text": "what ip address is localhost:8002 actually bound to?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5143931, + "p_memory": 0.2983053, + "p_abstain": 0.18730168 + }, + { + "query_id": "shell_test_024", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "shell_test_025", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "shell_test_027", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5596268, + "p_memory": 0.32979882, + "p_abstain": 0.11057431 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.54281276, + "p_memory": 0.31816098, + "p_abstain": 0.13902621 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5726373, + "p_memory": 0.3156031, + "p_abstain": 0.11175954 + }, + { + "query_id": "rf_test_026", + "text": "what warnings did the last cargo check produce?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.42772973, + "p_memory": 0.3487019, + "p_abstain": 0.22356844 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51347095, + "p_memory": 0.36306074, + "p_abstain": 0.12346836 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5302622, + "p_memory": 0.3146726, + "p_abstain": 0.15506518 + }, + { + "query_id": "wf_test_009", + "text": "update the dataset_id in internal_eval_starter.json to v3", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5095029, + "p_memory": 0.2474364, + "p_abstain": 0.24306077 + }, + { + "query_id": "wf_test_010", + "text": "write the sweep results summary to a txt file in the output dir", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5355736, + "p_memory": 0.26030138, + "p_abstain": 0.20412505 + }, + { + "query_id": "abs_test_003", + "text": "what will my project look like in 6 months?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.36163858, + "p_memory": 0.32116085, + "p_abstain": 0.31720063 + }, + { + "query_id": "abs_test_010", + "text": "when will the klbr project be finished?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.6006625, + "p_memory": 0.25644553, + "p_abstain": 0.14289191 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.39021608, + "p_memory": 0.3815764, + "p_abstain": 0.22820748 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.51517594, + "p_memory": 0.33310035, + "p_abstain": 0.15172371 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.87222224, + "confusion": [ + [ + 60, + 5, + 5 + ], + [ + 0, + 55, + 0 + ], + [ + 4, + 9, + 42 + ] + ], + "tool_precision": 0.9375, + "tool_recall": 0.85714287, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.07272727, + "memory_false_abstain_rate": 0.0, + "tool_count": 70, + "memory_count": 55, + "abstain_count": 55, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.8333333, + "confusion": [ + [ + 10, + 0, + 0 + ], + [ + 0, + 10, + 0 + ], + [ + 4, + 6, + 30 + ] + ], + "tool_precision": 0.71428573, + "tool_recall": 1.0, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.1, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 10, + "abstain_count": 40, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4472603, + "p_memory": 0.31608102, + "p_abstain": 0.23665877 + }, + { + "query_id": "holdout_abstain_toolish_abstain_008", + "text": "今週のHacker Newsのトップ記事って何? 調べて", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41855448, + "p_memory": 0.29148448, + "p_abstain": 0.289961 + }, + { + "query_id": "holdout_abstain_toolish_abstain_016", + "text": "read my saved browser passwords and list them", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5942878, + "p_memory": 0.1527169, + "p_abstain": 0.2529953 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28040448, + "p_memory": 0.42840877, + "p_abstain": 0.29118672 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.28884652, + "p_memory": 0.47185266, + "p_abstain": 0.23930086 + }, + { + "query_id": "holdout_abstain_toolish_abstain_031", + "text": "what's the latest NVIDIA driver version today?", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.56835616, + "p_memory": 0.18172053, + "p_abstain": 0.24992327 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.42986798, + "p_memory": 0.3391081, + "p_abstain": 0.2310239 + }, + { + "query_id": "holdout_abstain_toolish_abstain_037", + "text": "log into slack and show me all messages from the last hour", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.59388983, + "p_memory": 0.1271276, + "p_abstain": 0.2789826 + }, + { + "query_id": "holdout_abstain_toolish_abstain_038", + "text": "print all env vars that contain tokens or passwords", + "gold": "abstain", + "pred": "tools", + "p_tools": 0.5769879, + "p_memory": 0.14078811, + "p_abstain": 0.28222397 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.96666664, + "confusion": [ + [ + 10, + 0, + 0 + ], + [ + 0, + 40, + 0 + ], + [ + 0, + 2, + 8 + ] + ], + "tool_precision": 1.0, + "tool_recall": 1.0, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 10, + "memory_count": 40, + "abstain_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.4772462, + "p_memory": 0.30548796, + "p_abstain": 0.21726589 + }, + { + "query_id": "holdout_memory_toolish_abstain_056", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.81666666, + "confusion": [ + [ + 40, + 5, + 5 + ], + [ + 0, + 5, + 0 + ], + [ + 0, + 1, + 4 + ] + ], + "tool_precision": 1.0, + "tool_recall": 0.8, + "memory_false_tool_rate": 0.0, + "abstain_false_tool_rate": 0.0, + "memory_false_abstain_rate": 0.0, + "tool_count": 50, + "memory_count": 5, + "abstain_count": 5, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_003", + "text": "what's the current hostname?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.4834218, + "p_memory": 0.20741296, + "p_abstain": 0.30916515 + }, + { + "query_id": "holdout_tools_recall_tools_005", + "text": "how much swap is being used right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.46816638, + "p_memory": 0.24633291, + "p_abstain": 0.28550074 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.5135842, + "p_memory": 0.2836179, + "p_abstain": 0.20279795 + }, + { + "query_id": "holdout_tools_recall_tools_021", + "text": "what's the current git stash list?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5306845, + "p_memory": 0.1522484, + "p_abstain": 0.31706712 + }, + { + "query_id": "holdout_tools_recall_tools_026", + "text": "what ip address is localhost:8002 actually bound to right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52769524, + "p_memory": 0.27700135, + "p_abstain": 0.1953034 + }, + { + "query_id": "holdout_tools_recall_tools_030", + "text": "what's the current cpu temperature?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5463929, + "p_memory": 0.18182917, + "p_abstain": 0.27177787 + }, + { + "query_id": "holdout_tools_recall_tools_039", + "text": "what is the network interface with the highest tx bytes right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.52403766, + "p_memory": 0.26615608, + "p_abstain": 0.20980622 + }, + { + "query_id": "holdout_tools_recall_tools_040", + "text": "is port 1234 open right now?", + "gold": "tools", + "pred": "abstain", + "p_tools": 0.5339573, + "p_memory": 0.19223545, + "p_abstain": 0.27380726 + }, + { + "query_id": "holdout_tools_recall_tools_043", + "text": "what gpu am i using for the reranker right now?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.49325278, + "p_memory": 0.30790615, + "p_abstain": 0.19884107 + }, + { + "query_id": "holdout_tools_recall_abstain_054", + "text": "今週のHacker Newsのトップ記事って何?", + "gold": "abstain", + "pred": "memory", + "p_tools": 0.41718727, + "p_memory": 0.30132157, + "p_abstain": 0.28149116 + }, + { + "query_id": "holdout_tools_recall_tools_058", + "text": "what is the default value of compaction_keep in Config::default()?", + "gold": "tools", + "pred": "memory", + "p_tools": 0.51008576, + "p_memory": 0.32511467, + "p_abstain": 0.16479962 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md new file mode 100644 index 0000000..69f341c --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md @@ -0,0 +1,101 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-memory-lane-v3+router-read-file-repo-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- tools_prob_threshold: `0.100` +- tools_margin_threshold: `0.280` +- abstain_prob_threshold: `0.100` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- tools: `149` memory: `100` abstain: `70` +- accuracy: `0.8527` +- tools precision: `0.9597` +- tools recall: `0.7987` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0714` +- memory false-abstain rate (memory→abstain): `0.0000` + +## Confusion Matrix (gold x pred) + +| gold \ pred | tools | memory | abstain | +|---|---:|---:|---:| +| tools | 119 | 18 | 12 | +| memory | 0 | 100 | 0 | +| abstain | 5 | 12 | 53 | + +## Misclassified Queries + +| id | text | gold | pred | p_tools | p_memory | p_abstain | +|---|---|---|---|---:|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Tools` | `Memory` | 0.47 | 0.34 | 0.19 | +| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.51 | 0.28 | 0.21 | +| `holdout_tools_recall_tools_003` | what's the current hostname? | `Tools` | `Abstain` | 0.48 | 0.21 | 0.31 | +| `holdout_tools_recall_tools_005` | how much swap is being used right now? | `Tools` | `Abstain` | 0.47 | 0.25 | 0.29 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Tools` | `Memory` | 0.51 | 0.28 | 0.20 | +| `holdout_tools_recall_tools_021` | what's the current git stash list? | `Tools` | `Abstain` | 0.53 | 0.15 | 0.32 | +| `holdout_tools_recall_tools_026` | what ip address is localhost:8002 actually bound to right now? | `Tools` | `Memory` | 0.53 | 0.28 | 0.20 | +| `holdout_tools_recall_tools_030` | what's the current cpu temperature? | `Tools` | `Abstain` | 0.55 | 0.18 | 0.27 | +| `holdout_tools_recall_tools_039` | what is the network interface with the highest tx bytes right now? | `Tools` | `Memory` | 0.52 | 0.27 | 0.21 | +| `holdout_tools_recall_tools_040` | is port 1234 open right now? | `Tools` | `Abstain` | 0.53 | 0.19 | 0.27 | +| `holdout_tools_recall_tools_043` | what gpu am i using for the reranker right now? | `Tools` | `Memory` | 0.49 | 0.31 | 0.20 | +| `holdout_tools_recall_abstain_054` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.42 | 0.30 | 0.28 | +| `holdout_tools_recall_tools_058` | what is the default value of compaction_keep in Config::default()? | `Tools` | `Memory` | 0.51 | 0.33 | 0.16 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | 0.45 | 0.32 | 0.24 | +| `holdout_abstain_toolish_abstain_008` | 今週のHacker Newsのトップ記事って何? 調べて | `Abstain` | `Memory` | 0.42 | 0.29 | 0.29 | +| `holdout_abstain_toolish_abstain_016` | read my saved browser passwords and list them | `Abstain` | `Tools` | 0.59 | 0.15 | 0.25 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.48 | 0.31 | 0.22 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | 0.28 | 0.43 | 0.29 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | 0.29 | 0.47 | 0.24 | +| `holdout_abstain_toolish_abstain_031` | what's the latest NVIDIA driver version today? | `Abstain` | `Tools` | 0.57 | 0.18 | 0.25 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | 0.43 | 0.34 | 0.23 | +| `holdout_abstain_toolish_abstain_037` | log into slack and show me all messages from the last hour | `Abstain` | `Tools` | 0.59 | 0.13 | 0.28 | +| `holdout_abstain_toolish_abstain_038` | print all env vars that contain tokens or passwords | `Abstain` | `Tools` | 0.58 | 0.14 | 0.28 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | 0.48 | 0.31 | 0.22 | +| `holdout_memory_toolish_abstain_056` | 今週のHacker Newsのトップ記事って何? | `Abstain` | `Memory` | 0.42 | 0.30 | 0.28 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Tools` | `Abstain` | 0.48 | 0.23 | 0.29 | +| `shell_test_004` | what's the cpu temperature right now? | `Tools` | `Abstain` | 0.54 | 0.18 | 0.28 | +| `shell_test_013` | what's the current hostname? | `Tools` | `Abstain` | 0.48 | 0.21 | 0.31 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Tools` | `Memory` | 0.50 | 0.31 | 0.19 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Tools` | `Memory` | 0.52 | 0.29 | 0.19 | +| `shell_test_017` | today何のモデルがロードされてる? | `Tools` | `Abstain` | 0.55 | 0.17 | 0.28 | +| `shell_test_022` | what ip address is localhost:8002 actually bound to? | `Tools` | `Memory` | 0.51 | 0.30 | 0.19 | +| `shell_test_024` | what's the current git stash list? | `Tools` | `Abstain` | 0.53 | 0.15 | 0.32 | +| `shell_test_025` | how much swap is being used right now? | `Tools` | `Abstain` | 0.47 | 0.25 | 0.29 | +| `shell_test_027` | is port 1234 open right now? | `Tools` | `Abstain` | 0.53 | 0.19 | 0.27 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Tools` | `Memory` | 0.56 | 0.33 | 0.11 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Tools` | `Memory` | 0.54 | 0.32 | 0.14 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Tools` | `Memory` | 0.57 | 0.32 | 0.11 | +| `rf_test_026` | what warnings did the last cargo check produce? | `Tools` | `Memory` | 0.43 | 0.35 | 0.22 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Tools` | `Memory` | 0.51 | 0.36 | 0.12 | +| `wf_test_002` | write the router precision result to a markdown file | `Tools` | `Memory` | 0.53 | 0.31 | 0.16 | +| `wf_test_009` | update the dataset_id in internal_eval_starter.json to v3 | `Tools` | `Memory` | 0.51 | 0.25 | 0.24 | +| `wf_test_010` | write the sweep results summary to a txt file in the output dir | `Tools` | `Memory` | 0.54 | 0.26 | 0.20 | +| `abs_test_003` | what will my project look like in 6 months? | `Abstain` | `Memory` | 0.36 | 0.32 | 0.32 | +| `abs_test_010` | when will the klbr project be finished? | `Abstain` | `Tools` | 0.60 | 0.26 | 0.14 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | 0.39 | 0.38 | 0.23 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | 0.52 | 0.33 | 0.15 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- tools precision: `0.9375` +- tools recall: `0.8571` +- memory false-tools rate (memory→tools): `0.0000` +- abstain false-tools rate (abstain→tools): `0.0727` +- accuracy: `0.8722` + +## Holdout Metrics By Bucket + +| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy | +|---|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 0.7143 | 1.0000 | 0.0000 | 0.1000 | 0.8333 | +| `memory_toolish` | 60 | 1.0000 | 1.0000 | 0.0000 | 0.0000 | 0.9667 | +| `tools_recall` | 60 | 1.0000 | 0.8000 | 0.0000 | 0.0000 | 0.8167 | diff --git a/benchmarks/run_all.py b/benchmarks/run_all.py new file mode 100755 index 0000000..a3a572f --- /dev/null +++ b/benchmarks/run_all.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +import argparse +import datetime as dt +import json +import os +import re +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +BENCH = ROOT / "benchmarks" + + +def sh(args, *, cwd=ROOT, env=None): + subprocess.run(args, cwd=cwd, env=env, check=True) + + +def git_rev(): + try: + out = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT) + return out.decode("utf-8", "replace").strip() + except Exception: + return None + + +METRIC_RE = re.compile(r"^- (?P[^:]+): `(?P[^`]+)`\s*$") + + +def parse_metrics(report_md_path: Path): + metrics = {} + for line in report_md_path.read_text("utf-8", errors="replace").splitlines(): + m = METRIC_RE.match(line) + if not m: + continue + # Keep the first occurrence; later blocks might repeat keys with different contexts. + metrics.setdefault(m.group("key"), m.group("value")) + return metrics + + +def write_summary(run_dir: Path, sections): + lines = [] + lines.append(f"# KLBR Benchmark Suite Run") + lines.append("") + lines.append(f"- run_dir: `{run_dir.relative_to(ROOT)}`") + lines.append(f"- timestamp_utc: `{dt.datetime.now(dt.timezone.utc).isoformat()}`") + rev = git_rev() + if rev: + lines.append(f"- git_rev: `{rev}`") + lines.append("") + + for title, rel_out_dir, report_rel, metrics in sections: + lines.append(f"## {title}") + lines.append("") + lines.append(f"- output: `{rel_out_dir}`") + lines.append(f"- report: `{report_rel}`") + lines.append("") + # Keep this short: only the metrics people always look for. + keep = [ + "Final decision accuracy", + "Coverage", + "Selective accuracy", + "No-hit false answers", + "Activation accuracy", + "Activation precision", + "Activation recall", + "No-recall false activation rate", + "tools recall", + "tools precision", + "router accuracy", + "called any tool rate (given routed-to-tools)", + "Embedding latency p50/p95 ms", + "Rerank latency p50/p95 ms", + ] + for k in keep: + if k in metrics: + lines.append(f"- {k}: `{metrics[k]}`") + lines.append("") + + (run_dir / "summary.md").write_text("\n".join(lines) + "\n", "utf-8") + + +def main(): + parser = argparse.ArgumentParser(description="Run the standard KLBR benchmark suite into one folder.") + parser.add_argument( + "--run-id", + default=None, + help="Run id. Default: UTC timestamp like 2026-04-24_120501Z", + ) + parser.add_argument( + "--out-root", + default=str(BENCH / "runs"), + help="Root output directory (default: benchmarks/runs).", + ) + parser.add_argument( + "--llm-url", + default=os.environ.get("KLBR_BENCH_LLM_URL", "http://localhost:8001"), + help="Chat LLM base URL for tools-lane (default: $KLBR_BENCH_LLM_URL or http://localhost:8001).", + ) + parser.add_argument( + "--active-dataset", + default=str(BENCH / "inputs" / "datasets" / "internal_eval_starter.json"), + ) + parser.add_argument( + "--passive-dataset", + default=str(BENCH / "inputs" / "datasets" / "internal_eval_passive_recall_v3.json"), + ) + parser.add_argument( + "--config-dev", + default=str(BENCH / "inputs" / "configs" / "mvp_rerank_support_calibrated.json"), + ) + parser.add_argument( + "--config-test", + default=str(BENCH / "inputs" / "configs" / "mvp_rerank_support_calibrated_test.json"), + ) + parser.add_argument( + "--router-model", + default=str( + BENCH + / "models" + / "router" + / "linear" + / "out-router-linear-iter-9" + / "router_model.json" + ), + ) + args = parser.parse_args() + + run_id = args.run_id + if not run_id: + run_id = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%d_%H%M%SZ") + + out_root = Path(args.out_root).resolve() + run_dir = out_root / run_id + run_dir.mkdir(parents=True, exist_ok=False) + + manifest = { + "run_id": run_id, + "timestamp_utc": dt.datetime.now(dt.timezone.utc).isoformat(), + "git_rev": git_rev(), + "paths": { + "active_dataset": str(Path(args.active_dataset)), + "passive_dataset": str(Path(args.passive_dataset)), + "config_dev": str(Path(args.config_dev)), + "config_test": str(Path(args.config_test)), + "router_model": str(Path(args.router_model)), + }, + "endpoints": {"llm_url": args.llm_url}, + "commands": [], + } + + def run_step(name, cmd, out_subdir): + out_dir = run_dir / out_subdir + cmd = [str(x) for x in cmd] + manifest["commands"].append({"name": name, "cmd": cmd, "out_dir": str(out_dir)}) + sh(cmd) + return out_dir + + retrieval_dev_dir = run_step( + "retrieval-dev", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "retrieval", + args.active_dataset, + args.config_dev, + run_dir / "retrieval-dev", + ], + "retrieval-dev", + ) + retrieval_test_dir = run_step( + "retrieval-test", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "retrieval", + args.active_dataset, + args.config_test, + run_dir / "retrieval-test", + ], + "retrieval-test", + ) + passive_dev_dir = run_step( + "passive-dev", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "passive-recall", + args.passive_dataset, + args.config_dev, + run_dir / "passive-dev", + ], + "passive-dev", + ) + passive_test_dir = run_step( + "passive-test", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "passive-recall", + args.passive_dataset, + args.config_test, + run_dir / "passive-test", + ], + "passive-test", + ) + tools_dev_dir = run_step( + "tools-lane-dev", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "tools-lane", + args.active_dataset, + args.config_dev, + args.router_model, + run_dir / "tools-lane-dev", + args.llm_url, + ], + "tools-lane-dev", + ) + tools_test_dir = run_step( + "tools-lane-test", + [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "tools-lane", + args.active_dataset, + args.config_test, + args.router_model, + run_dir / "tools-lane-test", + args.llm_url, + ], + "tools-lane-test", + ) + + (run_dir / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", "utf-8") + + sections = [] + for title, out_dir in [ + ("Active Recall (Retrieval) — Dev", retrieval_dev_dir), + ("Active Recall (Retrieval) — Test", retrieval_test_dir), + ("Passive Recall — Dev", passive_dev_dir), + ("Passive Recall — Test", passive_test_dir), + ("Tools Lane — Dev", tools_dev_dir), + ("Tools Lane — Test", tools_test_dir), + ]: + report = out_dir / "report.md" + metrics = parse_metrics(report) if report.exists() else {} + sections.append( + ( + title, + str(out_dir.relative_to(ROOT)), + str(report.relative_to(ROOT)), + metrics, + ) + ) + write_summary(run_dir, sections) + + print(str(run_dir.relative_to(ROOT))) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/runs/2026-04-24_suite_smoke/manifest.json b/benchmarks/runs/2026-04-24_suite_smoke/manifest.json new file mode 100644 index 0000000..b4928cb --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/manifest.json @@ -0,0 +1,117 @@ +{ + "run_id": "2026-04-24_suite_smoke", + "timestamp_utc": "2026-04-24T20:21:04.591023+00:00", + "git_rev": "da2b58ac63fcb33a4f80e7773ba8fcf14f334f8b", + "paths": { + "active_dataset": "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_starter.json", + "passive_dataset": "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json", + "config_dev": "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json", + "config_test": "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json", + "router_model": "/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json" + }, + "endpoints": { + "llm_url": "http://localhost:8001" + }, + "commands": [ + { + "name": "retrieval-dev", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "retrieval", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_starter.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev" + }, + { + "name": "retrieval-test", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "retrieval", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_starter.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test" + }, + { + "name": "passive-dev", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "passive-recall", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/passive-dev" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/passive-dev" + }, + { + "name": "passive-test", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "passive-recall", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/passive-test" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/passive-test" + }, + { + "name": "tools-lane-dev", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "tools-lane", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_starter.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated.json", + "/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev", + "http://localhost:8001" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev" + }, + { + "name": "tools-lane-test", + "cmd": [ + "cargo", + "run", + "-q", + "-p", + "klbr-bench", + "--", + "tools-lane", + "/home/mayer/proj/klbr/benchmarks/inputs/datasets/internal_eval_starter.json", + "/home/mayer/proj/klbr/benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json", + "/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json", + "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test", + "http://localhost:8001" + ], + "out_dir": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test" + } + ] +} diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.json new file mode 100644 index 0000000..4ac8225 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.json @@ -0,0 +1,55 @@ +{ + "experiment": { + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 + }, + "dataset_id": "internal-eval-passive-recall-v3", + "evaluated_split": "dev", + "ingest_count": 44, + "ingest_latency_ms": 1105, + "metrics": { + "query_count": 21, + "should_recall_count": 16, + "should_not_recall_count": 5, + "activation_accuracy": 0.95238096, + "activation_precision": 1.0, + "activation_recall": 0.9375, + "no_recall_false_activation_rate": 0.0, + "support_precision_at_k_mean": 0.84444445, + "support_recall_at_k_mean": 0.7708333, + "recall_rate": 0.71428573, + "candidate_set_size_mean": 14.285714, + "time_window_hit_rate": 0.8125, + "retrieval_latency_p50_ms": 0, + "retrieval_latency_p95_ms": 1, + "embedding_latency_p50_ms": 10, + "embedding_latency_p95_ms": 13, + "rerank_latency_p50_ms": 78, + "rerank_latency_p95_ms": 283 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.md b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.md new file mode 100644 index 0000000..5b5c494 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.md @@ -0,0 +1,30 @@ +# Passive Recall Benchmark Report + +dataset: `internal-eval-passive-recall-v3` +experiment: `mvp-rerank-support-calibrated` +split: `dev` +queries: `21` +memories ingested: `44` +ingest latency ms: `1105` +window expansions: `8` + +## Metrics + +- Should-recall queries: `16` +- Should-not-recall queries: `5` +- Activation accuracy: `0.9524` +- Activation precision: `1.0000` +- Activation recall: `0.9375` +- No-recall false activation rate: `0.0000` +- Support precision@k mean: `0.8444` +- Support recall@k mean: `0.7708` +- Recall rate: `0.7143` +- Candidate set size mean: `14.29` +- Time-window hit rate: `0.8125` +- Retrieval latency p50/p95 ms: `0` / `1` +- Embedding latency p50/p95 ms: `10` / `13` +- Rerank latency p50/p95 ms: `78` / `283` + +## False Activations + +(none) diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/resolved_config.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/resolved_config.json new file mode 100644 index 0000000..5b834f0 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/resolved_config.json @@ -0,0 +1,29 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/traces.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/traces.json new file mode 100644 index 0000000..38f6fcd --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-dev/traces.json @@ -0,0 +1,7638 @@ +[ + { + "query_id": "pr_dev_1", + "query_text": "still working on the benchmark scaffolding and retrieval plumbing stuff today.", + "query_category": "passive recall / progress update", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 105, + "score": 0.14846128, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": 0.4260432, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 103, + "score": 0.48613524, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 107, + "score": 0.4989556, + "rank": 4, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 104, + "score": 0.5053553, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 105, + "score": 2.523844, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": -7.8910065, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 107, + "score": -8.81143, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": -10.769505, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": -11.007328, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 55, + "first_stage_top1_distance": 0.14846128, + "rerank_top1_raw_score": 2.523844, + "rerank_top2_raw_score": -7.8910065, + "rerank_margin_raw": 10.41485, + "support_top1_score": 0.70067513, + "candidate_memory_ids": [ + 105, + 106, + 107 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 105 + ], + "no_hit": false, + "first_stage_top1_distance": 0.14846128, + "rerank_top1_raw_score": 2.523844, + "rerank_top2_raw_score": -7.8910065, + "rerank_margin_raw": 10.41485, + "support_top1_score": 0.70067513, + "activation_decision": "recall", + "recalled_memory_ids": [ + 105 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 55, + "passed": true + }, + { + "query_id": "pr_dev_2", + "query_text": "remind me which reranker model we settled on", + "query_category": "passive recall / elliptical request", + "interaction_mode": "request", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 103, + "score": 0.36606687, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": 0.42811048, + "rank": 2, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 107, + "score": 0.45934892, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 106, + "score": 0.5047166, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": 0.52607626, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "reranked_candidates": [ + { + "memory_id": 104, + "score": -2.7245376, + "rank": 1, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 103, + "score": -5.6391196, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 107, + "score": -6.42451, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 106, + "score": -9.314201, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": -10.261274, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 38, + "first_stage_top1_distance": 0.36606687, + "rerank_top1_raw_score": -2.7245376, + "rerank_top2_raw_score": -5.6391196, + "rerank_margin_raw": 2.914582, + "support_top1_score": 0.108208016, + "candidate_memory_ids": [ + 104, + 103, + 107 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 104 + ], + "no_hit": false, + "first_stage_top1_distance": 0.36606687, + "rerank_top1_raw_score": -2.7245376, + "rerank_top2_raw_score": -5.6391196, + "rerank_margin_raw": 2.914582, + "support_top1_score": 0.108208016, + "activation_decision": "recall", + "recalled_memory_ids": [ + 104, + 103, + 107 + ], + "activation_correct": true, + "support_precision_at_k": 0.33333334, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 38, + "passed": true + }, + { + "query_id": "pr_dev_3", + "query_text": "i'm back in zed again for klbr work.", + "query_category": "passive recall / declarative state cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 105, + "score": 0.45327115, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": 0.51761144, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 103, + "score": 0.5304637, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": 0.53205335, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 107, + "score": 0.5464099, + "rank": 5, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "reranked_candidates": [ + { + "memory_id": 105, + "score": -3.0331912, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 107, + "score": -5.927005, + "rank": 2, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": -5.9693375, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 106, + "score": -6.45352, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 104, + "score": -7.043628, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 39, + "first_stage_top1_distance": 0.45327115, + "rerank_top1_raw_score": -3.0331912, + "rerank_top2_raw_score": -5.927005, + "rerank_margin_raw": 2.8938136, + "support_top1_score": 0.047247257, + "candidate_memory_ids": [ + 105, + 107, + 103 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 101 + ], + "no_hit": false, + "first_stage_top1_distance": 0.45327115, + "rerank_top1_raw_score": -3.0331912, + "rerank_top2_raw_score": -5.927005, + "rerank_margin_raw": 2.8938136, + "support_top1_score": 0.047247257, + "activation_decision": "recall", + "recalled_memory_ids": [ + 105, + 107, + 103 + ], + "activation_correct": true, + "support_precision_at_k": 0.0, + "support_recall_at_k": 0.0, + "time_window_hit": false, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 39, + "passed": true + }, + { + "query_id": "pr_dev_4", + "query_text": "reranker on 8003", + "query_category": "passive recall / service endpoint cue", + "interaction_mode": "fragment", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 103, + "score": 0.25671482, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": 0.3848204, + "rank": 2, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 107, + "score": 0.5143768, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 105, + "score": 0.52240455, + "rank": 4, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": 0.5286351, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + } + ], + "reranked_candidates": [ + { + "memory_id": 103, + "score": 4.059761, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": -2.4767005, + "rank": 2, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 107, + "score": -6.8454924, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 106, + "score": -9.435911, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": -9.835778, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 36, + "first_stage_top1_distance": 0.25671482, + "rerank_top1_raw_score": 4.059761, + "rerank_top2_raw_score": -2.4767005, + "rerank_margin_raw": 6.536462, + "support_top1_score": 0.8703012, + "candidate_memory_ids": [ + 103, + 104, + 107 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 103 + ], + "no_hit": false, + "first_stage_top1_distance": 0.25671482, + "rerank_top1_raw_score": 4.059761, + "rerank_top2_raw_score": -2.4767005, + "rerank_margin_raw": 6.536462, + "support_top1_score": 0.8703012, + "activation_decision": "recall", + "recalled_memory_ids": [ + 103 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 36, + "passed": true + }, + { + "query_id": "pr_dev_5", + "query_text": "keeping chat, embeddings, and rerank split out seems like the cleanest setup.", + "query_category": "passive recall / multi-evidence state", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 107, + "score": 0.3548236, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 105, + "score": 0.47242278, + "rank": 2, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 103, + "score": 0.4780714, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 106, + "score": 0.4880166, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 104, + "score": 0.51332104, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 107, + "score": -0.5802778, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": -8.166855, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": -9.53215, + "rank": 3, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 105, + "score": -10.065486, + "rank": 4, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": -10.664398, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 42, + "first_stage_top1_distance": 0.3548236, + "rerank_top1_raw_score": -0.5802778, + "rerank_top2_raw_score": -8.166855, + "rerank_margin_raw": 7.586577, + "support_top1_score": 0.30296415, + "candidate_memory_ids": [ + 107, + 103, + 104 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 107, + 102, + 103 + ], + "no_hit": false, + "first_stage_top1_distance": 0.3548236, + "rerank_top1_raw_score": -0.5802778, + "rerank_top2_raw_score": -8.166855, + "rerank_margin_raw": 7.586577, + "support_top1_score": 0.30296415, + "activation_decision": "recall", + "recalled_memory_ids": [ + 107 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 0.33333334, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 42, + "passed": true + }, + { + "query_id": "pr_dev_6", + "query_text": "lol yeah that makes sense", + "query_category": "passive recall / no recall social chatter", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 109, + "score": 0.48238832, + "rank": 1, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 113, + "score": 0.52612734, + "rank": 2, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 108, + "score": 0.5352837, + "rank": 3, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": 0.5501019, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 107, + "score": 0.56153, + "rank": 5, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 111, + "score": 0.5691928, + "rank": 6, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 139, + "score": 0.5748621, + "rank": 7, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 124, + "score": 0.5777582, + "rank": 8, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 112, + "score": 0.58770764, + "rank": 9, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 125, + "score": 0.59436, + "rank": 10, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 101, + "score": 0.5947859, + "rank": 11, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 110, + "score": 0.60012656, + "rank": 12, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 128, + "score": 0.60199404, + "rank": 13, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 103, + "score": 0.60449404, + "rank": 14, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 127, + "score": 0.6058165, + "rank": 15, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 105, + "score": 0.61623305, + "rank": 16, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 114, + "score": 0.6200714, + "rank": 17, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 102, + "score": 0.62481713, + "rank": 18, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 115, + "score": 0.632601, + "rank": 19, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 126, + "score": 0.63343495, + "rank": 20, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + } + ], + "reranked_candidates": [ + { + "memory_id": 107, + "score": -9.250978, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 113, + "score": -9.439176, + "rank": 2, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 124, + "score": -9.600663, + "rank": 3, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 112, + "score": -10.135542, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 111, + "score": -10.172331, + "rank": 5, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 139, + "score": -10.19146, + "rank": 6, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 109, + "score": -10.277372, + "rank": 7, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 106, + "score": -10.30883, + "rank": 8, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 108, + "score": -10.4052305, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 125, + "score": -10.942574, + "rank": 10, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 43, + "first_stage_top1_distance": 0.5501019, + "rerank_top1_raw_score": -9.240962, + "rerank_top2_raw_score": -10.310344, + "rerank_margin_raw": 1.0693817, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 107, + 106, + 103 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 47, + "first_stage_top1_distance": 0.5352837, + "rerank_top1_raw_score": -9.250326, + "rerank_top2_raw_score": -10.3095665, + "rerank_margin_raw": 1.0592403, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 107, + 106, + 103 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 77, + "first_stage_top1_distance": 0.5352837, + "rerank_top1_raw_score": -9.237688, + "rerank_top2_raw_score": -10.188411, + "rerank_margin_raw": 0.9507227, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 107, + 139, + 106 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 92, + "first_stage_top1_distance": 0.48238832, + "rerank_top1_raw_score": -9.250978, + "rerank_top2_raw_score": -9.439176, + "rerank_margin_raw": 0.18819714, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 107, + 113, + 124 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.48238832, + "rerank_top1_raw_score": -9.250978, + "rerank_top2_raw_score": -9.439176, + "rerank_margin_raw": 0.18819714, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 259, + "passed": true + }, + { + "query_id": "pr_dev_7", + "query_text": "i'm kind of tired today", + "query_category": "passive recall / no recall generic status", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 105, + "score": 0.45717758, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 101, + "score": 0.48101574, + "rank": 2, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 126, + "score": 0.48887217, + "rank": 3, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 106, + "score": 0.49363613, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 127, + "score": 0.52134585, + "rank": 5, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 124, + "score": 0.5265149, + "rank": 6, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 112, + "score": 0.53765035, + "rank": 7, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 123, + "score": 0.5430762, + "rank": 8, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 109, + "score": 0.55469763, + "rank": 9, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 108, + "score": 0.5556853, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 139, + "score": 0.5569287, + "rank": 11, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 103, + "score": 0.5605443, + "rank": 12, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 110, + "score": 0.5692084, + "rank": 13, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": 0.57124007, + "rank": 14, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 104, + "score": 0.57587206, + "rank": 15, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 113, + "score": 0.5793928, + "rank": 16, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 115, + "score": 0.5843036, + "rank": 17, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 107, + "score": 0.5887744, + "rank": 18, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 114, + "score": 0.5908557, + "rank": 19, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 125, + "score": 0.6057254, + "rank": 20, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + } + ], + "reranked_candidates": [ + { + "memory_id": 109, + "score": -10.975371, + "rank": 1, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 101, + "score": -11.0026455, + "rank": 2, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 127, + "score": -11.028177, + "rank": 3, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 106, + "score": -11.034254, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 126, + "score": -11.035046, + "rank": 5, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 124, + "score": -11.039471, + "rank": 6, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": -11.039745, + "rank": 7, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 112, + "score": -11.041565, + "rank": 8, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 105, + "score": -11.042013, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 108, + "score": -11.042695, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 37, + "first_stage_top1_distance": 0.45717758, + "rerank_top1_raw_score": -11.034687, + "rerank_top2_raw_score": -11.0383625, + "rerank_margin_raw": 0.0036754608, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 106, + 107, + 103 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 46, + "first_stage_top1_distance": 0.45717758, + "rerank_top1_raw_score": -11.0026455, + "rerank_top2_raw_score": -11.034056, + "rerank_margin_raw": 0.031410217, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 101, + 106, + 107 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 64, + "first_stage_top1_distance": 0.45717758, + "rerank_top1_raw_score": -11.0026455, + "rerank_top2_raw_score": -11.034056, + "rerank_margin_raw": 0.031410217, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 101, + 106, + 139 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 86, + "first_stage_top1_distance": 0.45717758, + "rerank_top1_raw_score": -10.975371, + "rerank_top2_raw_score": -11.0026455, + "rerank_margin_raw": 0.027274132, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 109, + 101, + 127 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.45717758, + "rerank_top1_raw_score": -10.975371, + "rerank_top2_raw_score": -11.0026455, + "rerank_margin_raw": 0.027274132, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 233, + "passed": true + }, + { + "query_id": "pr_dev_8", + "query_text": "the tui finally shows the stream errors now.", + "query_category": "passive recall / error context cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776860000, + "expected_memory_action": "recall", + "chosen_window_days": 30, + "windows_considered": [ + 7, + 30 + ], + "memories_searched": 8, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 108, + "score": 0.25787687, + "rank": 1, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": 0.51641774, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": 0.5211359, + "rank": 3, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 101, + "score": 0.5212692, + "rank": 4, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 103, + "score": 0.53510857, + "rank": 5, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 107, + "score": 0.55958056, + "rank": 6, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 104, + "score": 0.5717087, + "rank": 7, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 102, + "score": 0.5723226, + "rank": 8, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + } + ], + "reranked_candidates": [ + { + "memory_id": 108, + "score": -3.8927307, + "rank": 1, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": -10.7181835, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 103, + "score": -10.899811, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 105, + "score": -10.916544, + "rank": 4, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 102, + "score": -10.955683, + "rank": 5, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 104, + "score": -11.020746, + "rank": 6, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 101, + "score": -11.023073, + "rank": 7, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 107, + "score": -11.023433, + "rank": 8, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 36, + "first_stage_top1_distance": 0.51641774, + "rerank_top1_raw_score": -10.715274, + "rerank_top2_raw_score": -10.902437, + "rerank_margin_raw": 0.18716335, + "support_top1_score": 0.026947515, + "candidate_memory_ids": [ + 106, + 103, + 105 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 47, + "first_stage_top1_distance": 0.25787687, + "rerank_top1_raw_score": -3.8927307, + "rerank_top2_raw_score": -10.7181835, + "rerank_margin_raw": 6.825453, + "support_top1_score": 0.48884588, + "candidate_memory_ids": [ + 108, + 106, + 103 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 108 + ], + "no_hit": false, + "first_stage_top1_distance": 0.25787687, + "rerank_top1_raw_score": -3.8927307, + "rerank_top2_raw_score": -10.7181835, + "rerank_margin_raw": 6.825453, + "support_top1_score": 0.48884588, + "activation_decision": "recall", + "recalled_memory_ids": [ + 108 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 83, + "passed": true + }, + { + "query_id": "pr_dev_9", + "query_text": "the support feature ended up mattering more than i expected.", + "query_category": "passive recall / calibration result statement", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776864600, + "expected_memory_action": "recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 111, + "score": 0.41771632, + "rank": 1, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 109, + "score": 0.41888648, + "rank": 2, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 106, + "score": 0.4231729, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 125, + "score": 0.4523031, + "rank": 4, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 114, + "score": 0.46942472, + "rank": 5, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 113, + "score": 0.47101742, + "rank": 6, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 110, + "score": 0.48526394, + "rank": 7, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 115, + "score": 0.49010897, + "rank": 8, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 105, + "score": 0.49044204, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 112, + "score": 0.49739444, + "rank": 10, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 139, + "score": 0.5119235, + "rank": 11, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 126, + "score": 0.5119979, + "rank": 12, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 123, + "score": 0.5343915, + "rank": 13, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 124, + "score": 0.53668135, + "rank": 14, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 108, + "score": 0.539206, + "rank": 15, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 107, + "score": 0.54555655, + "rank": 16, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 128, + "score": 0.54715455, + "rank": 17, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 101, + "score": 0.5645046, + "rank": 18, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 103, + "score": 0.5687105, + "rank": 19, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 104, + "score": 0.5740272, + "rank": 20, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 115, + "score": -4.172902, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 111, + "score": -4.2241173, + "rank": 2, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 110, + "score": -5.0673556, + "rank": 3, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 113, + "score": -6.7077436, + "rank": 4, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 109, + "score": -7.8248215, + "rank": 5, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 112, + "score": -7.9311132, + "rank": 6, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 125, + "score": -8.791086, + "rank": 7, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 114, + "score": -9.631502, + "rank": 8, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 106, + "score": -9.697488, + "rank": 9, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": -11.012492, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 57, + "first_stage_top1_distance": 0.41771632, + "rerank_top1_raw_score": -4.224549, + "rerank_top2_raw_score": -5.06748, + "rerank_margin_raw": 0.8429313, + "support_top1_score": 0.13235487, + "candidate_memory_ids": [ + 111, + 110, + 109 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.41771632, + "rerank_top1_raw_score": -4.224549, + "rerank_top2_raw_score": -5.06748, + "rerank_margin_raw": 0.8429313, + "support_top1_score": 0.13235487, + "candidate_memory_ids": [ + 111, + 110, + 109 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.41771632, + "rerank_top1_raw_score": -4.2158523, + "rerank_top2_raw_score": -5.1194525, + "rerank_margin_raw": 0.9036002, + "support_top1_score": 0.13235489, + "candidate_memory_ids": [ + 111, + 110, + 109 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 73, + "first_stage_top1_distance": 0.41771632, + "rerank_top1_raw_score": -4.172902, + "rerank_top2_raw_score": -4.2241173, + "rerank_margin_raw": 0.05121517, + "support_top1_score": 0.16682652, + "candidate_memory_ids": [ + 115, + 111, + 110 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 109, + 110 + ], + "no_hit": false, + "first_stage_top1_distance": 0.41771632, + "rerank_top1_raw_score": -4.172902, + "rerank_top2_raw_score": -4.2241173, + "rerank_margin_raw": 0.05121517, + "support_top1_score": 0.16682652, + "activation_decision": "no_recall", + "activation_correct": false, + "support_recall_at_k": 0.0, + "time_window_hit": true, + "embedding_latency_ms": 13, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 272, + "passed": false + }, + { + "query_id": "pr_dev_10", + "query_text": "i still don't want to solve this with query-type guards.", + "query_category": "passive recall / anti taxonomy cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 12, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 112, + "score": 0.3489337, + "rank": 1, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 106, + "score": 0.5179335, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 111, + "score": 0.5326375, + "rank": 3, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 109, + "score": 0.53496575, + "rank": 4, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 104, + "score": 0.558537, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 115, + "score": 0.5758363, + "rank": 6, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 113, + "score": 0.5821229, + "rank": 7, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 105, + "score": 0.58470786, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 110, + "score": 0.5851202, + "rank": 9, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 114, + "score": 0.5991535, + "rank": 10, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 107, + "score": 0.6015059, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": 0.60416806, + "rank": 12, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + } + ], + "reranked_candidates": [ + { + "memory_id": 112, + "score": -0.92203295, + "rank": 1, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 109, + "score": -10.406666, + "rank": 2, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 113, + "score": -10.7771, + "rank": 3, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 106, + "score": -10.838323, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 115, + "score": -10.952465, + "rank": 5, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 110, + "score": -10.961796, + "rank": 6, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": -10.985658, + "rank": 7, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 114, + "score": -11.010836, + "rank": 8, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 104, + "score": -11.034729, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 105, + "score": -11.038456, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 76, + "first_stage_top1_distance": 0.3489337, + "rerank_top1_raw_score": -0.92203295, + "rerank_top2_raw_score": -10.406666, + "rerank_margin_raw": 9.4846325, + "support_top1_score": 0.34839955, + "candidate_memory_ids": [ + 112, + 109, + 113 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 112 + ], + "no_hit": false, + "first_stage_top1_distance": 0.3489337, + "rerank_top1_raw_score": -0.92203295, + "rerank_top2_raw_score": -10.406666, + "rerank_margin_raw": 9.4846325, + "support_top1_score": 0.34839955, + "activation_decision": "recall", + "recalled_memory_ids": [ + 112 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 76, + "passed": true + }, + { + "query_id": "pr_dev_11", + "query_text": "before tuning passive recall further we needed a bigger eval slice.", + "query_category": "passive recall / planning cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 12, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 114, + "score": 0.19860536, + "rank": 1, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 115, + "score": 0.35062307, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 112, + "score": 0.35855275, + "rank": 3, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 106, + "score": 0.42015427, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 113, + "score": 0.44209558, + "rank": 5, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 109, + "score": 0.45193982, + "rank": 6, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 110, + "score": 0.4816568, + "rank": 7, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 105, + "score": 0.500268, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 104, + "score": 0.5006188, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 111, + "score": 0.5078089, + "rank": 10, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 107, + "score": 0.52255535, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": 0.5560189, + "rank": 12, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + } + ], + "reranked_candidates": [ + { + "memory_id": 114, + "score": 2.4427826, + "rank": 1, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 112, + "score": -4.4696217, + "rank": 2, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 113, + "score": -4.786584, + "rank": 3, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 115, + "score": -5.935573, + "rank": 4, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 110, + "score": -9.105424, + "rank": 5, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 106, + "score": -9.34005, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 109, + "score": -9.768773, + "rank": 7, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 111, + "score": -10.290443, + "rank": 8, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 105, + "score": -11.0245285, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 104, + "score": -11.029362, + "rank": 10, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "first_stage_top1_distance": 0.19860536, + "rerank_top1_raw_score": 2.4427826, + "rerank_top2_raw_score": -4.4696217, + "rerank_margin_raw": 6.912404, + "support_top1_score": 0.4449543, + "candidate_memory_ids": [ + 114, + 112, + 113 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 114 + ], + "no_hit": false, + "first_stage_top1_distance": 0.19860536, + "rerank_top1_raw_score": 2.4427826, + "rerank_top2_raw_score": -4.4696217, + "rerank_margin_raw": 6.912404, + "support_top1_score": 0.4449543, + "activation_decision": "recall", + "recalled_memory_ids": [ + 114 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "passed": true + }, + { + "query_id": "pr_dev_12", + "query_text": "activation seems fine but the recalled set is still kind of noisy.", + "query_category": "passive recall / support quality cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 12, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 115, + "score": 0.38157398, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 106, + "score": 0.39391643, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 114, + "score": 0.4067996, + "rank": 3, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 112, + "score": 0.43709886, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 113, + "score": 0.46453536, + "rank": 5, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 111, + "score": 0.50657415, + "rank": 6, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 105, + "score": 0.5112834, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 110, + "score": 0.5352155, + "rank": 8, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 104, + "score": 0.54909027, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 109, + "score": 0.5536618, + "rank": 10, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 107, + "score": 0.55692333, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": 0.5932268, + "rank": 12, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + } + ], + "reranked_candidates": [ + { + "memory_id": 115, + "score": -2.4796772, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 113, + "score": -2.8599527, + "rank": 2, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 114, + "score": -7.39377, + "rank": 3, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 112, + "score": -7.991495, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 106, + "score": -8.302415, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 110, + "score": -8.879658, + "rank": 6, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": -9.900946, + "rank": 7, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 104, + "score": -10.575184, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 109, + "score": -10.773544, + "rank": 9, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 105, + "score": -10.991126, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "first_stage_top1_distance": 0.38157398, + "rerank_top1_raw_score": -2.4796772, + "rerank_top2_raw_score": -2.8599527, + "rerank_margin_raw": 0.3802755, + "support_top1_score": 0.4202165, + "candidate_memory_ids": [ + 115, + 113, + 114 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 115 + ], + "no_hit": false, + "first_stage_top1_distance": 0.38157398, + "rerank_top1_raw_score": -2.4796772, + "rerank_top2_raw_score": -2.8599527, + "rerank_margin_raw": 0.3802755, + "support_top1_score": 0.4202165, + "activation_decision": "recall", + "recalled_memory_ids": [ + 115, + 113, + 114 + ], + "activation_correct": true, + "support_precision_at_k": 0.33333334, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "passed": true + }, + { + "query_id": "pr_dev_13", + "query_text": "memory use really is activation first and then trust.", + "query_category": "passive recall / policy framing cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 12, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 113, + "score": 0.16599256, + "rank": 1, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 115, + "score": 0.40730947, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 114, + "score": 0.44174564, + "rank": 3, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 112, + "score": 0.45426202, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 109, + "score": 0.49106807, + "rank": 5, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 107, + "score": 0.4926756, + "rank": 6, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 105, + "score": 0.5110403, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": 0.52678853, + "rank": 8, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 111, + "score": 0.5484854, + "rank": 9, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 110, + "score": 0.5486244, + "rank": 10, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 104, + "score": 0.5489534, + "rank": 11, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 103, + "score": 0.5752128, + "rank": 12, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + } + ], + "reranked_candidates": [ + { + "memory_id": 113, + "score": 4.5863523, + "rank": 1, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 115, + "score": -4.9490814, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 109, + "score": -8.74253, + "rank": 3, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 110, + "score": -8.88356, + "rank": 4, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 112, + "score": -9.807703, + "rank": 5, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 114, + "score": -10.037568, + "rank": 6, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 111, + "score": -10.047054, + "rank": 7, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 107, + "score": -10.319635, + "rank": 8, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 105, + "score": -10.875392, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 106, + "score": -10.918894, + "rank": 10, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.16599256, + "rerank_top1_raw_score": 4.5863523, + "rerank_top2_raw_score": -4.9490814, + "rerank_margin_raw": 9.535434, + "support_top1_score": 0.7570637, + "candidate_memory_ids": [ + 113, + 115, + 109 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 113 + ], + "no_hit": false, + "first_stage_top1_distance": 0.16599256, + "rerank_top1_raw_score": 4.5863523, + "rerank_top2_raw_score": -4.9490814, + "rerank_margin_raw": 9.535434, + "support_top1_score": 0.7570637, + "activation_decision": "recall", + "recalled_memory_ids": [ + 113 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "passed": true + }, + { + "query_id": "pr_dev_14", + "query_text": "that color palette is nice tbh", + "query_category": "passive recall / no recall off topic opinion", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 139, + "score": 0.52440083, + "rank": 1, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 105, + "score": 0.533294, + "rank": 2, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 108, + "score": 0.5480815, + "rank": 3, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 109, + "score": 0.5557009, + "rank": 4, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 101, + "score": 0.5624205, + "rank": 5, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 127, + "score": 0.5659032, + "rank": 6, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 125, + "score": 0.56805813, + "rank": 7, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 106, + "score": 0.5681991, + "rank": 8, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 124, + "score": 0.57817626, + "rank": 9, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": 0.5827557, + "rank": 10, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 107, + "score": 0.5836867, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 126, + "score": 0.5973433, + "rank": 12, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 104, + "score": 0.6011347, + "rank": 13, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 112, + "score": 0.6108132, + "rank": 14, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 128, + "score": 0.61399806, + "rank": 15, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 114, + "score": 0.6150427, + "rank": 16, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 102, + "score": 0.6176349, + "rank": 17, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 113, + "score": 0.62739396, + "rank": 18, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 103, + "score": 0.6345879, + "rank": 19, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 110, + "score": 0.63821244, + "rank": 20, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + } + ], + "reranked_candidates": [ + { + "memory_id": 127, + "score": -10.569054, + "rank": 1, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 123, + "score": -10.78887, + "rank": 2, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 139, + "score": -10.819296, + "rank": 3, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 125, + "score": -10.987976, + "rank": 4, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 109, + "score": -10.998049, + "rank": 5, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 101, + "score": -11.018277, + "rank": 6, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 124, + "score": -11.024796, + "rank": 7, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 106, + "score": -11.035976, + "rank": 8, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 108, + "score": -11.036461, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 105, + "score": -11.038323, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 81, + "first_stage_top1_distance": 0.533294, + "rerank_top1_raw_score": -10.598901, + "rerank_top2_raw_score": -10.733767, + "rerank_margin_raw": 0.13486576, + "support_top1_score": 0.03457107, + "candidate_memory_ids": [ + 104, + 110, + 109 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 15, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.533294, + "rerank_top1_raw_score": -10.606691, + "rerank_top2_raw_score": -10.996923, + "rerank_margin_raw": 0.3902321, + "support_top1_score": 0.03457107, + "candidate_memory_ids": [ + 104, + 109, + 101 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 68, + "first_stage_top1_distance": 0.52440083, + "rerank_top1_raw_score": -10.605501, + "rerank_top2_raw_score": -10.819296, + "rerank_margin_raw": 0.21379471, + "support_top1_score": 0.034571074, + "candidate_memory_ids": [ + 104, + 139, + 109 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 69, + "first_stage_top1_distance": 0.52440083, + "rerank_top1_raw_score": -10.569054, + "rerank_top2_raw_score": -10.78887, + "rerank_margin_raw": 0.21981621, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 127, + 123, + 139 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.52440083, + "rerank_top1_raw_score": -10.569054, + "rerank_top2_raw_score": -10.78887, + "rerank_margin_raw": 0.21981621, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 290, + "passed": true + }, + { + "query_id": "pr_dev_15", + "query_text": "lol fair enough", + "query_category": "passive recall / no recall short acknowledgement", + "interaction_mode": "fragment", + "split": "dev", + "namespace": "default", + "reference_time": 1776871800, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 108, + "score": 0.5503299, + "rank": 1, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": 0.5568734, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 109, + "score": 0.5625887, + "rank": 3, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 111, + "score": 0.56691384, + "rank": 4, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 110, + "score": 0.5837023, + "rank": 5, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 125, + "score": 0.588933, + "rank": 6, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 105, + "score": 0.60687333, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 103, + "score": 0.6073603, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 102, + "score": 0.60820043, + "rank": 9, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 139, + "score": 0.6091462, + "rank": 10, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 112, + "score": 0.61472094, + "rank": 11, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 107, + "score": 0.61491406, + "rank": 12, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 114, + "score": 0.6222656, + "rank": 13, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 113, + "score": 0.62728447, + "rank": 14, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 101, + "score": 0.6283264, + "rank": 15, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 126, + "score": 0.635069, + "rank": 16, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 124, + "score": 0.6380831, + "rank": 17, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": 0.6433791, + "rank": 18, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 127, + "score": 0.6494651, + "rank": 19, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 115, + "score": 0.6571832, + "rank": 20, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + } + ], + "reranked_candidates": [ + { + "memory_id": 110, + "score": -8.199463, + "rank": 1, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": -8.383871, + "rank": 2, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 108, + "score": -9.31638, + "rank": 3, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": -9.789129, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 125, + "score": -10.017516, + "rank": 5, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 109, + "score": -10.184971, + "rank": 6, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 139, + "score": -10.564805, + "rank": 7, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 103, + "score": -10.620922, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 102, + "score": -10.745512, + "rank": 9, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 105, + "score": -10.883545, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 67, + "first_stage_top1_distance": 0.5568734, + "rerank_top1_raw_score": -8.203095, + "rerank_top2_raw_score": -8.382244, + "rerank_margin_raw": 0.17914867, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 110, + 111, + 106 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 15, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 65, + "first_stage_top1_distance": 0.5503299, + "rerank_top1_raw_score": -8.210517, + "rerank_top2_raw_score": -8.383871, + "rerank_margin_raw": 0.17335415, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 110, + 111, + 108 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 66, + "first_stage_top1_distance": 0.5503299, + "rerank_top1_raw_score": -8.210517, + "rerank_top2_raw_score": -8.383871, + "rerank_margin_raw": 0.17335415, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 110, + 111, + 108 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 66, + "first_stage_top1_distance": 0.5503299, + "rerank_top1_raw_score": -8.199463, + "rerank_top2_raw_score": -8.383871, + "rerank_margin_raw": 0.18440819, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 110, + 111, + 108 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.5503299, + "rerank_top1_raw_score": -8.199463, + "rerank_top2_raw_score": -8.383871, + "rerank_margin_raw": 0.18440819, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 6, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 264, + "passed": true + }, + { + "query_id": "pr_dev_16", + "query_text": "before we switched embeddings the dimension was 1024.", + "query_category": "passive recall / temporal ordering cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776910000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 16, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 123, + "score": 0.28528404, + "rank": 1, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 124, + "score": 0.43967694, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 126, + "score": 0.49571198, + "rank": 3, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 114, + "score": 0.50962335, + "rank": 4, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 106, + "score": 0.5180775, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 107, + "score": 0.5547039, + "rank": 6, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 103, + "score": 0.55667996, + "rank": 7, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 105, + "score": 0.57140315, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 109, + "score": 0.5733623, + "rank": 9, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 104, + "score": 0.574305, + "rank": 10, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 115, + "score": 0.5882881, + "rank": 11, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 125, + "score": 0.58838975, + "rank": 12, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 113, + "score": 0.590513, + "rank": 13, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 111, + "score": 0.5967205, + "rank": 14, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 110, + "score": 0.5991497, + "rank": 15, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 112, + "score": 0.62005794, + "rank": 16, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + } + ], + "reranked_candidates": [ + { + "memory_id": 123, + "score": 1.802, + "rank": 1, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 124, + "score": -6.317837, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 126, + "score": -8.093594, + "rank": 3, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 114, + "score": -8.101568, + "rank": 4, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 107, + "score": -10.146215, + "rank": 5, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 106, + "score": -10.984112, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 103, + "score": -11.015821, + "rank": 7, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 109, + "score": -11.02097, + "rank": 8, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 104, + "score": -11.033529, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 105, + "score": -11.035505, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 92, + "first_stage_top1_distance": 0.28528404, + "rerank_top1_raw_score": 1.802, + "rerank_top2_raw_score": -6.317837, + "rerank_margin_raw": 8.119837, + "support_top1_score": 0.74552685, + "candidate_memory_ids": [ + 123, + 124, + 126 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 123, + 102 + ], + "no_hit": false, + "first_stage_top1_distance": 0.28528404, + "rerank_top1_raw_score": 1.802, + "rerank_top2_raw_score": -6.317837, + "rerank_margin_raw": 8.119837, + "support_top1_score": 0.74552685, + "activation_decision": "recall", + "recalled_memory_ids": [ + 123 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 0.5, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 92, + "passed": true + }, + { + "query_id": "pr_dev_17", + "query_text": "the other day i moved back to neovim.", + "query_category": "passive recall / ambiguous time reference", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776910000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 16, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 126, + "score": 0.36542523, + "rank": 1, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 124, + "score": 0.4476359, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 105, + "score": 0.4713468, + "rank": 3, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 123, + "score": 0.49429852, + "rank": 4, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 103, + "score": 0.50187373, + "rank": 5, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 106, + "score": 0.5118531, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 114, + "score": 0.51558876, + "rank": 7, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 125, + "score": 0.535044, + "rank": 8, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 104, + "score": 0.5387982, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 110, + "score": 0.5624677, + "rank": 10, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 107, + "score": 0.5694328, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 113, + "score": 0.5814004, + "rank": 12, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 109, + "score": 0.58591914, + "rank": 13, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 112, + "score": 0.5887184, + "rank": 14, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 111, + "score": 0.5899497, + "rank": 15, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 115, + "score": 0.63056, + "rank": 16, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + } + ], + "reranked_candidates": [ + { + "memory_id": 126, + "score": -0.23175268, + "rank": 1, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 124, + "score": -8.8337755, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": -9.615074, + "rank": 3, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 103, + "score": -10.791116, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 110, + "score": -10.8611555, + "rank": 5, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 114, + "score": -10.974151, + "rank": 6, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 106, + "score": -10.994492, + "rank": 7, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 105, + "score": -11.014104, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 125, + "score": -11.022516, + "rank": 9, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 104, + "score": -11.027432, + "rank": 10, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 78, + "first_stage_top1_distance": 0.36542523, + "rerank_top1_raw_score": -0.23175268, + "rerank_top2_raw_score": -8.8337755, + "rerank_margin_raw": 8.602023, + "support_top1_score": 0.43414816, + "candidate_memory_ids": [ + 126, + 124, + 123 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 126 + ], + "no_hit": false, + "first_stage_top1_distance": 0.36542523, + "rerank_top1_raw_score": -0.23175268, + "rerank_top2_raw_score": -8.8337755, + "rerank_margin_raw": 8.602023, + "support_top1_score": 0.43414816, + "activation_decision": "recall", + "recalled_memory_ids": [ + 126 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 78, + "passed": true + }, + { + "query_id": "pr_dev_18", + "query_text": "i'm not using zed anymore, switched back to neovim.", + "query_category": "passive recall / update signal", + "interaction_mode": "update", + "split": "dev", + "namespace": "default", + "reference_time": 1776910000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 16, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 126, + "score": 0.2442069, + "rank": 1, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 124, + "score": 0.46534574, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": 0.47866488, + "rank": 3, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 112, + "score": 0.53056103, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 111, + "score": 0.53669417, + "rank": 5, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 106, + "score": 0.54705733, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 125, + "score": 0.5481124, + "rank": 7, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 103, + "score": 0.5552136, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 114, + "score": 0.55925995, + "rank": 9, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 105, + "score": 0.56204486, + "rank": 10, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 104, + "score": 0.5710908, + "rank": 11, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 110, + "score": 0.5917262, + "rank": 12, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 109, + "score": 0.5959132, + "rank": 13, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 107, + "score": 0.61837375, + "rank": 14, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 113, + "score": 0.6215014, + "rank": 15, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 115, + "score": 0.63838047, + "rank": 16, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + } + ], + "reranked_candidates": [ + { + "memory_id": 126, + "score": 2.413997, + "rank": 1, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 123, + "score": -7.2684717, + "rank": 2, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 124, + "score": -9.3898945, + "rank": 3, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 111, + "score": -10.000078, + "rank": 4, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 106, + "score": -10.672592, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 125, + "score": -10.772844, + "rank": 6, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 105, + "score": -10.948254, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 114, + "score": -10.966532, + "rank": 8, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 103, + "score": -10.977399, + "rank": 9, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 112, + "score": -10.998682, + "rank": 10, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "first_stage_top1_distance": 0.2442069, + "rerank_top1_raw_score": 2.413997, + "rerank_top2_raw_score": -7.2684717, + "rerank_margin_raw": 9.682468, + "support_top1_score": 0.50222796, + "candidate_memory_ids": [ + 126, + 123, + 124 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 101, + 126 + ], + "no_hit": false, + "first_stage_top1_distance": 0.2442069, + "rerank_top1_raw_score": 2.413997, + "rerank_top2_raw_score": -7.2684717, + "rerank_margin_raw": 9.682468, + "support_top1_score": 0.50222796, + "activation_decision": "recall", + "recalled_memory_ids": [ + 126 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 0.5, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "passed": true + }, + { + "query_id": "pr_dev_19", + "query_text": "my internet connection keeps dropping and i have no idea why.", + "query_category": "passive recall / no recall false technical cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776910000, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 124, + "score": 0.4964862, + "rank": 1, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 108, + "score": 0.50520575, + "rank": 2, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 103, + "score": 0.5166926, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 106, + "score": 0.51696074, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 123, + "score": 0.51941574, + "rank": 5, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 107, + "score": 0.5365547, + "rank": 6, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 126, + "score": 0.5381036, + "rank": 7, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 105, + "score": 0.5528072, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 101, + "score": 0.5591252, + "rank": 9, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 110, + "score": 0.56114525, + "rank": 10, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": 0.5720798, + "rank": 11, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + }, + { + "memory_id": 115, + "score": 0.57404923, + "rank": 12, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 139, + "score": 0.5773858, + "rank": 13, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 128, + "score": 0.5802127, + "rank": 14, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 104, + "score": 0.5825337, + "rank": 15, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 127, + "score": 0.58758104, + "rank": 16, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 102, + "score": 0.595356, + "rank": 17, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 114, + "score": 0.59760773, + "rank": 18, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 112, + "score": 0.59909225, + "rank": 19, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 125, + "score": 0.6103208, + "rank": 20, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + } + ], + "reranked_candidates": [ + { + "memory_id": 123, + "score": -8.742855, + "rank": 1, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 103, + "score": -9.366026, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 110, + "score": -10.801086, + "rank": 3, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 106, + "score": -11.002831, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 126, + "score": -11.019142, + "rank": 5, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 101, + "score": -11.0355625, + "rank": 6, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 108, + "score": -11.036682, + "rank": 7, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 124, + "score": -11.041651, + "rank": 8, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 105, + "score": -11.042189, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 107, + "score": -11.043166, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "first_stage_top1_distance": 0.4964862, + "rerank_top1_raw_score": -8.167488, + "rerank_top2_raw_score": -8.769489, + "rerank_margin_raw": 0.6020012, + "support_top1_score": 0.10850572, + "candidate_memory_ids": [ + 111, + 123, + 103 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 19, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.4964862, + "rerank_top1_raw_score": -8.742855, + "rerank_top2_raw_score": -9.366026, + "rerank_margin_raw": 0.62317085, + "support_top1_score": 0.0909178, + "candidate_memory_ids": [ + 123, + 103, + 110 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 20, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.4964862, + "rerank_top1_raw_score": -8.742855, + "rerank_top2_raw_score": -9.366026, + "rerank_margin_raw": 0.62317085, + "support_top1_score": 0.090917796, + "candidate_memory_ids": [ + 123, + 103, + 110 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.4964862, + "rerank_top1_raw_score": -8.742855, + "rerank_top2_raw_score": -9.366026, + "rerank_margin_raw": 0.62317085, + "support_top1_score": 0.090917796, + "candidate_memory_ids": [ + 123, + 103, + 110 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.4964862, + "rerank_top1_raw_score": -8.742855, + "rerank_top2_raw_score": -9.366026, + "rerank_margin_raw": 0.62317085, + "support_top1_score": 0.090917796, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 283, + "passed": true + }, + { + "query_id": "pr_dev_20", + "query_text": "we should prioritize the abstention and update patterns from LongMemEval.", + "query_category": "passive recall / public dataset extraction cue", + "interaction_mode": "statement", + "split": "dev", + "namespace": "default", + "reference_time": 1776992000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 17, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 128, + "score": 0.19135273, + "rank": 1, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 114, + "score": 0.39633477, + "rank": 2, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 113, + "score": 0.415785, + "rank": 3, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 112, + "score": 0.42824835, + "rank": 4, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 127, + "score": 0.43995023, + "rank": 5, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 124, + "score": 0.4472975, + "rank": 6, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 107, + "score": 0.46309906, + "rank": 7, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 125, + "score": 0.4727841, + "rank": 8, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 104, + "score": 0.4851827, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 106, + "score": 0.48577523, + "rank": 10, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 109, + "score": 0.48739845, + "rank": 11, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 115, + "score": 0.5115558, + "rank": 12, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m4", + "text": "the first passive-recall starter run had perfect activation, but support precision was much weaker than support recall." + }, + { + "memory_id": 123, + "score": 0.519381, + "rank": 13, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 126, + "score": 0.52035403, + "rank": 14, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 103, + "score": 0.5300884, + "rank": 15, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 110, + "score": 0.53179383, + "rank": 16, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 111, + "score": 0.5497722, + "rank": 17, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + } + ], + "reranked_candidates": [ + { + "memory_id": 128, + "score": 3.9398613, + "rank": 1, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 113, + "score": -6.6776056, + "rank": 2, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 114, + "score": -6.815131, + "rank": 3, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 127, + "score": -6.9176803, + "rank": 4, + "event_time": 1776978000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr11:m1", + "text": "started extracting temporal and ambiguity cue patterns from TemporalMemoryDataset into KLBR-style passive items." + }, + { + "memory_id": 125, + "score": -8.620837, + "rank": 5, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 107, + "score": -9.344268, + "rank": 6, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 124, + "score": -9.745716, + "rank": 7, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 112, + "score": -9.836691, + "rank": 8, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 106, + "score": -10.505291, + "rank": 9, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 104, + "score": -10.811256, + "rank": 10, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 101, + "first_stage_top1_distance": 0.19135273, + "rerank_top1_raw_score": 3.9398613, + "rerank_top2_raw_score": -6.6776056, + "rerank_margin_raw": 10.617467, + "support_top1_score": 0.7739787, + "candidate_memory_ids": [ + 128, + 113, + 114 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 128 + ], + "no_hit": false, + "first_stage_top1_distance": 0.19135273, + "rerank_top1_raw_score": 3.9398613, + "rerank_top2_raw_score": -6.6776056, + "rerank_margin_raw": 10.617467, + "support_top1_score": 0.7739787, + "activation_decision": "recall", + "recalled_memory_ids": [ + 128 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 101, + "passed": true + }, + { + "query_id": "pr_dev_21", + "query_text": "what was the decision on the frontend rewrite back in january?", + "query_category": "passive recall / long gap temporal reference", + "interaction_mode": "request", + "split": "dev", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 90, + "windows_considered": [ + 7, + 30, + 90 + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 139, + "score": 0.34117627, + "rank": 1, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 124, + "score": 0.53152025, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 103, + "score": 0.53572977, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 101, + "score": 0.5663592, + "rank": 4, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 126, + "score": 0.57492584, + "rank": 5, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 123, + "score": 0.60597086, + "rank": 6, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 114, + "score": 0.60894185, + "rank": 7, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 108, + "score": 0.61608756, + "rank": 8, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 106, + "score": 0.62186825, + "rank": 9, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 128, + "score": 0.6228112, + "rank": 10, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + }, + { + "memory_id": 104, + "score": 0.62644565, + "rank": 11, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:pr2:m2", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 107, + "score": 0.629497, + "rank": 12, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:pr4:m1", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 113, + "score": 0.6320094, + "rank": 13, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr7:m2", + "text": "memory use should be treated as two linked decisions: activation first, then trust in the recalled support." + }, + { + "memory_id": 110, + "score": 0.633762, + "rank": 14, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m2", + "text": "a focused support sweep found the best operating point near rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 105, + "score": 0.6377164, + "rank": 15, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:pr3:m1", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 112, + "score": 0.6549413, + "rank": 16, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr7:m1", + "text": "passive recall should reuse generic evidence support rather than brittle query-type guards." + }, + { + "memory_id": 125, + "score": 0.65858084, + "rank": 17, + "event_time": 1776898800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:benchmarking" + ], + "source_ref": "session:pr10:m3", + "text": "the nomic model runs faster than bge-m3 and the recall quality difference on our eval set was negligible." + }, + { + "memory_id": 109, + "score": 0.6641365, + "rank": 18, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 102, + "score": 0.6729333, + "rank": 19, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:pr1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 111, + "score": 0.688813, + "rank": 20, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr6:m3", + "text": "the support-assisted operating point removed no-hit false answers, but answerable coverage dropped compared with the looser balanced mode." + } + ], + "reranked_candidates": [ + { + "memory_id": 139, + "score": 2.986772, + "rank": 1, + "event_time": 1772000000, + "tags": [ + "project:frontend", + "topic:planning" + ], + "source_ref": "session:pr15:m1", + "text": "january planning: we decided to skip the rust rewrite of the web frontend because the typescript version is stable enough." + }, + { + "memory_id": 124, + "score": -7.370745, + "rank": 2, + "event_time": 1776895200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:migration" + ], + "source_ref": "session:pr10:m2", + "text": "the embed dim change required migrating the db schema and re-indexing all stored memories." + }, + { + "memory_id": 123, + "score": -8.46657, + "rank": 3, + "event_time": 1776891600, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:model-update" + ], + "source_ref": "session:pr10:m1", + "text": "switched the embedding model from bge-m3 to nomic-embed-text-v1.5, dropping dimension from 1024 to 768." + }, + { + "memory_id": 101, + "score": -9.003717, + "rank": 4, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 126, + "score": -9.642012, + "rank": 5, + "event_time": 1776906000, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:pr10:m4", + "text": "switched back to neovim for klbr work; zed was slow on the larger files." + }, + { + "memory_id": 106, + "score": -10.129065, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:pr3:m2", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 114, + "score": -10.148338, + "rank": 7, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr7:m3", + "text": "before doing more passive recall tuning, the eval dataset was expanded and the retrieval benchmarks were rerun." + }, + { + "memory_id": 103, + "score": -10.3236685, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:pr2:m1", + "text": "the reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 108, + "score": -10.767101, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:pr5:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 128, + "score": -11.007427, + "rank": 10, + "event_time": 1776981600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:abstention" + ], + "source_ref": "session:pr11:m2", + "text": "LongMemEval abstention and knowledge-update items are the highest priority patterns to import next." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 16, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 76, + "first_stage_top1_distance": 0.53152025, + "rerank_top1_raw_score": -7.3623652, + "rerank_top2_raw_score": -8.468954, + "rerank_margin_raw": 1.1065888, + "support_top1_score": 0.018212482, + "candidate_memory_ids": [ + 124, + 123, + 113 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 21, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 76, + "first_stage_top1_distance": 0.53152025, + "rerank_top1_raw_score": -7.3712034, + "rerank_top2_raw_score": -8.454106, + "rerank_margin_raw": 1.0829029, + "support_top1_score": 0.01821248, + "candidate_memory_ids": [ + 124, + 123, + 101 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.34117627, + "rerank_top1_raw_score": 2.986772, + "rerank_top2_raw_score": -7.370745, + "rerank_margin_raw": 10.357517, + "support_top1_score": 0.679178, + "candidate_memory_ids": [ + 139, + 124, + 123 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 139 + ], + "no_hit": false, + "first_stage_top1_distance": 0.34117627, + "rerank_top1_raw_score": 2.986772, + "rerank_top2_raw_score": -7.370745, + "rerank_margin_raw": 10.357517, + "support_top1_score": 0.679178, + "activation_decision": "recall", + "recalled_memory_ids": [ + 139 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": false, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 224, + "passed": true + } +] \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.json new file mode 100644 index 0000000..db3eaaa --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.json @@ -0,0 +1,55 @@ +{ + "experiment": { + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Same operating point as mvp_rerank_support_calibrated.json but evaluated on the test split.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 + }, + "dataset_id": "internal-eval-passive-recall-v3", + "evaluated_split": "test", + "ingest_count": 44, + "ingest_latency_ms": 1041, + "metrics": { + "query_count": 21, + "should_recall_count": 16, + "should_not_recall_count": 5, + "activation_accuracy": 0.9047619, + "activation_precision": 0.9375, + "activation_recall": 0.9375, + "no_recall_false_activation_rate": 0.2, + "support_precision_at_k_mean": 0.875, + "support_recall_at_k_mean": 0.90625, + "recall_rate": 0.7619048, + "candidate_set_size_mean": 14.904762, + "time_window_hit_rate": 0.75, + "retrieval_latency_p50_ms": 0, + "retrieval_latency_p95_ms": 0, + "embedding_latency_p50_ms": 10, + "embedding_latency_p95_ms": 11, + "rerank_latency_p50_ms": 85, + "rerank_latency_p95_ms": 295 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.md b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.md new file mode 100644 index 0000000..a840b2c --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.md @@ -0,0 +1,30 @@ +# Passive Recall Benchmark Report + +dataset: `internal-eval-passive-recall-v3` +experiment: `mvp-rerank-support-calibrated` +split: `test` +queries: `21` +memories ingested: `44` +ingest latency ms: `1041` +window expansions: `8` + +## Metrics + +- Should-recall queries: `16` +- Should-not-recall queries: `5` +- Activation accuracy: `0.9048` +- Activation precision: `0.9375` +- Activation recall: `0.9375` +- No-recall false activation rate: `0.2000` +- Support precision@k mean: `0.8750` +- Support recall@k mean: `0.9062` +- Recall rate: `0.7619` +- Candidate set size mean: `14.90` +- Time-window hit rate: `0.7500` +- Retrieval latency p50/p95 ms: `0` / `0` +- Embedding latency p50/p95 ms: `10` / `11` +- Rerank latency p50/p95 ms: `85` / `295` + +## False Activations + +- `pr_test_14`: query=`working on the dotfiles syncer today, not touching klbr.` top_score=`Some(-2.4893126)` margin=`Some(2.1447616)` support=`Some(0.38702488)` recalled=`131, 118, 130` diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-test/resolved_config.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/resolved_config.json new file mode 100644 index 0000000..73f58cf --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/resolved_config.json @@ -0,0 +1,29 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Same operating point as mvp_rerank_support_calibrated.json but evaluated on the test split.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/passive-test/traces.json b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/traces.json new file mode 100644 index 0000000..7b94489 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/passive-test/traces.json @@ -0,0 +1,7829 @@ +[ + { + "query_id": "pr_test_1", + "query_text": "the bigger eval rerun still liked the support-calibrated point.", + "query_category": "passive recall / rerun result cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 118, + "score": 0.26975697, + "rank": 1, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": 0.42495716, + "rank": 2, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 119, + "score": 0.44446933, + "rank": 3, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 116, + "score": 0.4468426, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 120, + "score": 0.46479923, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.49372727, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.5323551, + "rank": 7, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "reranked_candidates": [ + { + "memory_id": 118, + "score": -0.33192647, + "rank": 1, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": -5.549051, + "rank": 2, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 116, + "score": -6.4039316, + "rank": 3, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 120, + "score": -6.8044524, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 119, + "score": -7.604869, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 117, + "score": -7.8854847, + "rank": 6, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 121, + "score": -8.271024, + "rank": 7, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 74, + "first_stage_top1_distance": 0.26975697, + "rerank_top1_raw_score": -0.33192647, + "rerank_top2_raw_score": -5.549051, + "rerank_margin_raw": 5.2171245, + "support_top1_score": 0.6269295, + "candidate_memory_ids": [ + 118, + 122, + 116 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 118 + ], + "no_hit": false, + "first_stage_top1_distance": 0.26975697, + "rerank_top1_raw_score": -0.33192647, + "rerank_top2_raw_score": -5.549051, + "rerank_margin_raw": 5.2171245, + "support_top1_score": 0.6269295, + "activation_decision": "recall", + "recalled_memory_ids": [ + 118 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 74, + "passed": true + }, + { + "query_id": "pr_test_2", + "query_text": "the expanded main eval set is way less toy now.", + "query_category": "passive recall / expanded dataset cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 116, + "score": 0.4024449, + "rank": 1, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 119, + "score": 0.43645483, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": 0.46261328, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 120, + "score": 0.48185176, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": 0.5022454, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 118, + "score": 0.5335996, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": 0.5686452, + "rank": 7, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + } + ], + "reranked_candidates": [ + { + "memory_id": 116, + "score": -0.74899757, + "rank": 1, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 117, + "score": -5.296856, + "rank": 2, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 118, + "score": -6.6251144, + "rank": 3, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 120, + "score": -7.3718276, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": -8.099231, + "rank": 5, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -10.458679, + "rank": 6, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 122, + "score": -11.036381, + "rank": 7, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 52, + "first_stage_top1_distance": 0.4024449, + "rerank_top1_raw_score": -0.74899757, + "rerank_top2_raw_score": -5.296856, + "rerank_margin_raw": 4.547858, + "support_top1_score": 0.5960946, + "candidate_memory_ids": [ + 116, + 117, + 118 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 116, + 117 + ], + "no_hit": false, + "first_stage_top1_distance": 0.4024449, + "rerank_top1_raw_score": -0.74899757, + "rerank_top2_raw_score": -5.296856, + "rerank_margin_raw": 4.547858, + "support_top1_score": 0.5960946, + "activation_decision": "recall", + "recalled_memory_ids": [ + 116, + 117 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 52, + "passed": true + }, + { + "query_id": "pr_test_3", + "query_text": "what should the next passive recall pass focus on again", + "query_category": "passive recall / planning request", + "interaction_mode": "request", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 119, + "score": 0.24583209, + "rank": 1, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": 0.34915566, + "rank": 2, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.39016694, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 122, + "score": 0.47859013, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 116, + "score": 0.581673, + "rank": 5, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 118, + "score": 0.5955089, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 117, + "score": 0.63182425, + "rank": 7, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "reranked_candidates": [ + { + "memory_id": 119, + "score": 4.659873, + "rank": 1, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": -0.3037232, + "rank": 2, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 120, + "score": -1.4605634, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 122, + "score": -4.47011, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": -9.496445, + "rank": 5, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": -10.469751, + "rank": 6, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 117, + "score": -10.806492, + "rank": 7, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 53, + "first_stage_top1_distance": 0.24583209, + "rerank_top1_raw_score": 4.659873, + "rerank_top2_raw_score": -0.3037232, + "rerank_margin_raw": 4.9635963, + "support_top1_score": 0.7733434, + "candidate_memory_ids": [ + 119, + 121, + 120 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 119 + ], + "no_hit": false, + "first_stage_top1_distance": 0.24583209, + "rerank_top1_raw_score": 4.659873, + "rerank_top2_raw_score": -0.3037232, + "rerank_margin_raw": 4.9635963, + "support_top1_score": 0.7733434, + "activation_decision": "recall", + "recalled_memory_ids": [ + 119, + 121 + ], + "activation_correct": true, + "support_precision_at_k": 0.5, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 53, + "passed": true + }, + { + "query_id": "pr_test_4", + "query_text": "we still need more no-recall and update-style passive items.", + "query_category": "passive recall / dataset curation cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 120, + "score": 0.18304151, + "rank": 1, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 119, + "score": 0.3342418, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": 0.3399567, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.42628014, + "rank": 4, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": 0.4980023, + "rank": 5, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": 0.49933022, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": 0.52588785, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "reranked_candidates": [ + { + "memory_id": 120, + "score": 4.0356965, + "rank": 1, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": -4.091538, + "rank": 2, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -4.246844, + "rank": 3, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 122, + "score": -4.7814636, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 117, + "score": -9.001568, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 118, + "score": -10.52603, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": -10.631227, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 52, + "first_stage_top1_distance": 0.18304151, + "rerank_top1_raw_score": 4.0356965, + "rerank_top2_raw_score": -4.091538, + "rerank_margin_raw": 8.127234, + "support_top1_score": 0.76253176, + "candidate_memory_ids": [ + 120, + 121, + 119 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 120 + ], + "no_hit": false, + "first_stage_top1_distance": 0.18304151, + "rerank_top1_raw_score": 4.0356965, + "rerank_top2_raw_score": -4.091538, + "rerank_margin_raw": 8.127234, + "support_top1_score": 0.76253176, + "activation_decision": "recall", + "recalled_memory_ids": [ + 120 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 52, + "passed": true + }, + { + "query_id": "pr_test_5", + "query_text": "the passive benchmark should stop being dev-only.", + "query_category": "passive recall / split planning cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 121, + "score": 0.1902144, + "rank": 1, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": 0.2974158, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": 0.34008956, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 122, + "score": 0.4878118, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 117, + "score": 0.52123713, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 118, + "score": 0.5439208, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": 0.5652727, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "reranked_candidates": [ + { + "memory_id": 121, + "score": 5.001939, + "rank": 1, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -2.7153468, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": -3.8707933, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": -7.418685, + "rank": 4, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": -7.8907585, + "rank": 5, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": -9.447876, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": -10.70613, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 51, + "first_stage_top1_distance": 0.1902144, + "rerank_top1_raw_score": 5.001939, + "rerank_top2_raw_score": -2.7153468, + "rerank_margin_raw": 7.7172856, + "support_top1_score": 0.7729362, + "candidate_memory_ids": [ + 121, + 119, + 120 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 121 + ], + "no_hit": false, + "first_stage_top1_distance": 0.1902144, + "rerank_top1_raw_score": 5.001939, + "rerank_top2_raw_score": -2.7153468, + "rerank_margin_raw": 7.7172856, + "support_top1_score": 0.7729362, + "activation_decision": "recall", + "recalled_memory_ids": [ + 121 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 51, + "passed": true + }, + { + "query_id": "pr_test_6", + "query_text": "support precision was only around point three nine", + "query_category": "passive recall / metric fragment", + "interaction_mode": "fragment", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 122, + "score": 0.31442833, + "rank": 1, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 119, + "score": 0.44376856, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 118, + "score": 0.4898635, + "rank": 3, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 121, + "score": 0.5289463, + "rank": 4, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.54861724, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 120, + "score": 0.5983036, + "rank": 6, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 116, + "score": 0.61963713, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "reranked_candidates": [ + { + "memory_id": 122, + "score": -0.70682245, + "rank": 1, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": -3.6919541, + "rank": 2, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 119, + "score": -7.8712416, + "rank": 3, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 117, + "score": -8.788115, + "rank": 4, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 116, + "score": -9.30529, + "rank": 5, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 121, + "score": -10.597439, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 120, + "score": -10.60754, + "rank": 7, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 50, + "first_stage_top1_distance": 0.31442833, + "rerank_top1_raw_score": -0.70682245, + "rerank_top2_raw_score": -3.6919541, + "rerank_margin_raw": 2.9851317, + "support_top1_score": 0.4921964, + "candidate_memory_ids": [ + 122, + 118, + 119 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 122 + ], + "no_hit": false, + "first_stage_top1_distance": 0.31442833, + "rerank_top1_raw_score": -0.70682245, + "rerank_top2_raw_score": -3.6919541, + "rerank_margin_raw": 2.9851317, + "support_top1_score": 0.4921964, + "activation_decision": "recall", + "recalled_memory_ids": [ + 122, + 118 + ], + "activation_correct": true, + "support_precision_at_k": 0.5, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 50, + "passed": true + }, + { + "query_id": "pr_test_7", + "query_text": "i need coffee lol", + "query_category": "passive recall / no recall ambient need", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 131, + "score": 0.52220976, + "rank": 1, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 120, + "score": 0.530589, + "rank": 2, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 142, + "score": 0.53926677, + "rank": 3, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 141, + "score": 0.5540633, + "rank": 4, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 144, + "score": 0.56627715, + "rank": 5, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 134, + "score": 0.5745435, + "rank": 6, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 143, + "score": 0.58491695, + "rank": 7, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 140, + "score": 0.5901284, + "rank": 8, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 119, + "score": 0.5916828, + "rank": 9, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 117, + "score": 0.5958793, + "rank": 10, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 133, + "score": 0.60068, + "rank": 11, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 121, + "score": 0.60231555, + "rank": 12, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 135, + "score": 0.60371244, + "rank": 13, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 130, + "score": 0.61947227, + "rank": 14, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 137, + "score": 0.6214979, + "rank": 15, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 138, + "score": 0.62200904, + "rank": 16, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 116, + "score": 0.6320429, + "rank": 17, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 118, + "score": 0.6395415, + "rank": 18, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 129, + "score": 0.64426744, + "rank": 19, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 136, + "score": 0.65023124, + "rank": 20, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + } + ], + "reranked_candidates": [ + { + "memory_id": 144, + "score": -10.902485, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 134, + "score": -11.01281, + "rank": 2, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 141, + "score": -11.029923, + "rank": 3, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 143, + "score": -11.0349045, + "rank": 4, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 142, + "score": -11.036159, + "rank": 5, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 120, + "score": -11.0383625, + "rank": 6, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 131, + "score": -11.040049, + "rank": 7, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 117, + "score": -11.040413, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 119, + "score": -11.042373, + "rank": 9, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 140, + "score": -11.042789, + "rank": 10, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 50, + "first_stage_top1_distance": 0.530589, + "rerank_top1_raw_score": -11.035286, + "rerank_top2_raw_score": -11.037974, + "rerank_margin_raw": 0.002688408, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 121, + 120, + 117 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 48, + "first_stage_top1_distance": 0.530589, + "rerank_top1_raw_score": -11.034807, + "rerank_top2_raw_score": -11.038296, + "rerank_margin_raw": 0.0034885406, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 121, + 120, + 117 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 103, + "first_stage_top1_distance": 0.530589, + "rerank_top1_raw_score": -10.902358, + "rerank_top2_raw_score": -11.030611, + "rerank_margin_raw": 0.12825298, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 144, + 141, + 143 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 81, + "first_stage_top1_distance": 0.52220976, + "rerank_top1_raw_score": -10.902485, + "rerank_top2_raw_score": -11.01281, + "rerank_margin_raw": 0.11032486, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 144, + 134, + 141 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.52220976, + "rerank_top1_raw_score": -10.902485, + "rerank_top2_raw_score": -11.01281, + "rerank_margin_raw": 0.11032486, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 282, + "passed": true + }, + { + "query_id": "pr_test_8", + "query_text": "that sounds annoying", + "query_category": "passive recall / no recall empathy", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 120, + "score": 0.5138862, + "rank": 1, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.54651433, + "rank": 2, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 142, + "score": 0.5465306, + "rank": 3, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 135, + "score": 0.54714406, + "rank": 4, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 131, + "score": 0.563793, + "rank": 5, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 137, + "score": 0.5819503, + "rank": 6, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 119, + "score": 0.5827255, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 138, + "score": 0.58588326, + "rank": 8, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 133, + "score": 0.5879414, + "rank": 9, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 129, + "score": 0.5955586, + "rank": 10, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 116, + "score": 0.59814084, + "rank": 11, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 136, + "score": 0.6018096, + "rank": 12, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 144, + "score": 0.61066186, + "rank": 13, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 134, + "score": 0.6129111, + "rank": 14, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 143, + "score": 0.61328876, + "rank": 15, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 141, + "score": 0.6208509, + "rank": 16, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 118, + "score": 0.6229422, + "rank": 17, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 140, + "score": 0.62974733, + "rank": 18, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 132, + "score": 0.63359165, + "rank": 19, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 122, + "score": 0.63400495, + "rank": 20, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + } + ], + "reranked_candidates": [ + { + "memory_id": 121, + "score": -9.866693, + "rank": 1, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 138, + "score": -10.0395355, + "rank": 2, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 120, + "score": -10.677531, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 137, + "score": -10.866402, + "rank": 4, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 119, + "score": -10.917327, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 129, + "score": -10.948622, + "rank": 6, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 131, + "score": -10.959998, + "rank": 7, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 135, + "score": -10.962893, + "rank": 8, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 133, + "score": -11.013199, + "rank": 9, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 142, + "score": -11.029854, + "rank": 10, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 51, + "first_stage_top1_distance": 0.5138862, + "rerank_top1_raw_score": -9.8523445, + "rerank_top2_raw_score": -10.682026, + "rerank_margin_raw": 0.8296814, + "support_top1_score": 0.0543785, + "candidate_memory_ids": [ + 121, + 120, + 117 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 47, + "first_stage_top1_distance": 0.5138862, + "rerank_top1_raw_score": -9.8523445, + "rerank_top2_raw_score": -10.68136, + "rerank_margin_raw": 0.82901573, + "support_top1_score": 0.0543785, + "candidate_memory_ids": [ + 121, + 120, + 117 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 69, + "first_stage_top1_distance": 0.5138862, + "rerank_top1_raw_score": -9.855167, + "rerank_top2_raw_score": -10.6218815, + "rerank_margin_raw": 0.7667141, + "support_top1_score": 0.0543785, + "candidate_memory_ids": [ + 121, + 140, + 120 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 89, + "first_stage_top1_distance": 0.5138862, + "rerank_top1_raw_score": -9.866693, + "rerank_top2_raw_score": -10.0395355, + "rerank_margin_raw": 0.17284298, + "support_top1_score": 0.0543785, + "candidate_memory_ids": [ + 121, + 138, + 120 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.5138862, + "rerank_top1_raw_score": -9.866693, + "rerank_top2_raw_score": -10.0395355, + "rerank_margin_raw": 0.17284298, + "support_top1_score": 0.0543785, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 7, + "retrieval_latency_ms": 1, + "rerank_latency_ms": 256, + "passed": true + }, + { + "query_id": "pr_test_9", + "query_text": "the rerun made the calibration feel more real because the dataset got bigger and the same point stayed best.", + "query_category": "passive recall / multi-evidence calibration cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 7, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 118, + "score": 0.27944446, + "rank": 1, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": 0.41981244, + "rank": 2, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 120, + "score": 0.44239527, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 122, + "score": 0.49403238, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 119, + "score": 0.49726564, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": 0.5066859, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.51626945, + "rank": 7, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "reranked_candidates": [ + { + "memory_id": 118, + "score": -1.0538187, + "rank": 1, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 116, + "score": -6.325289, + "rank": 2, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 117, + "score": -6.56876, + "rank": 3, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": -6.642009, + "rank": 4, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 120, + "score": -7.805112, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": -9.356337, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -9.573716, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 58, + "first_stage_top1_distance": 0.27944446, + "rerank_top1_raw_score": -1.0538187, + "rerank_top2_raw_score": -6.325289, + "rerank_margin_raw": 5.27147, + "support_top1_score": 0.48441973, + "candidate_memory_ids": [ + 118, + 116, + 117 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 116, + 118 + ], + "no_hit": false, + "first_stage_top1_distance": 0.27944446, + "rerank_top1_raw_score": -1.0538187, + "rerank_top2_raw_score": -6.325289, + "rerank_margin_raw": 5.27147, + "support_top1_score": 0.48441973, + "activation_decision": "recall", + "recalled_memory_ids": [ + 118 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 0.5, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 58, + "passed": true + }, + { + "query_id": "pr_test_10", + "query_text": "brb for a sec", + "query_category": "passive recall / no recall short absence", + "interaction_mode": "fragment", + "split": "test", + "namespace": "default", + "reference_time": 1776884400, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 131, + "score": 0.5199567, + "rank": 1, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 138, + "score": 0.5374882, + "rank": 2, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 144, + "score": 0.5585899, + "rank": 3, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 121, + "score": 0.5589148, + "rank": 4, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 130, + "score": 0.5631373, + "rank": 5, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 119, + "score": 0.57874554, + "rank": 6, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 135, + "score": 0.5788797, + "rank": 7, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 122, + "score": 0.5922484, + "rank": 8, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 142, + "score": 0.59247476, + "rank": 9, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 137, + "score": 0.5925252, + "rank": 10, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 133, + "score": 0.59318614, + "rank": 11, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 120, + "score": 0.59745187, + "rank": 12, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 116, + "score": 0.59959733, + "rank": 13, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 136, + "score": 0.6062269, + "rank": 14, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 129, + "score": 0.6103461, + "rank": 15, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 118, + "score": 0.61539733, + "rank": 16, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 117, + "score": 0.62729305, + "rank": 17, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 134, + "score": 0.6285839, + "rank": 18, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 143, + "score": 0.6387669, + "rank": 19, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 132, + "score": 0.652027, + "rank": 20, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + } + ], + "reranked_candidates": [ + { + "memory_id": 122, + "score": -8.223513, + "rank": 1, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 135, + "score": -8.385631, + "rank": 2, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 138, + "score": -8.431872, + "rank": 3, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 121, + "score": -8.889839, + "rank": 4, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 130, + "score": -9.146656, + "rank": 5, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 119, + "score": -9.465469, + "rank": 6, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 137, + "score": -9.720135, + "rank": 7, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 131, + "score": -10.154036, + "rank": 8, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 142, + "score": -10.455651, + "rank": 9, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 144, + "score": -10.549268, + "rank": 10, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 48, + "first_stage_top1_distance": 0.5589148, + "rerank_top1_raw_score": -8.218206, + "rerank_top2_raw_score": -8.250129, + "rerank_margin_raw": 0.03192234, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 122, + 118, + 116 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 49, + "first_stage_top1_distance": 0.5589148, + "rerank_top1_raw_score": -8.234552, + "rerank_top2_raw_score": -8.250129, + "rerank_margin_raw": 0.015576363, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 122, + 118, + 121 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 69, + "first_stage_top1_distance": 0.5585899, + "rerank_top1_raw_score": -8.227769, + "rerank_top2_raw_score": -8.263083, + "rerank_margin_raw": 0.03531456, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 122, + 118, + 116 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 68, + "first_stage_top1_distance": 0.5199567, + "rerank_top1_raw_score": -8.223513, + "rerank_top2_raw_score": -8.385631, + "rerank_margin_raw": 0.16211796, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 122, + 135, + 138 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.5199567, + "rerank_top1_raw_score": -8.223513, + "rerank_top2_raw_score": -8.385631, + "rerank_margin_raw": 0.16211796, + "support_top1_score": 0.0, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 7, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 234, + "passed": true + }, + { + "query_id": "pr_test_11", + "query_text": "the memory_used annotations in RealMemBench are exactly what we need for support grading.", + "query_category": "passive recall / multi-session back-reference", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1776992000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 9, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 129, + "score": 0.19588023, + "rank": 1, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 130, + "score": 0.44310045, + "rank": 2, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 120, + "score": 0.47775614, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 119, + "score": 0.48139656, + "rank": 4, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 118, + "score": 0.48973, + "rank": 5, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": 0.5017115, + "rank": 6, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 116, + "score": 0.5020365, + "rank": 7, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 121, + "score": 0.502518, + "rank": 8, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.5758947, + "rank": 9, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "reranked_candidates": [ + { + "memory_id": 129, + "score": -2.9934068, + "rank": 1, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 130, + "score": -8.115297, + "rank": 2, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 122, + "score": -8.645108, + "rank": 3, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 116, + "score": -9.268699, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 119, + "score": -9.357105, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 118, + "score": -9.394159, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 120, + "score": -10.095301, + "rank": 7, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": -10.161032, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 121, + "score": -10.322941, + "rank": 9, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.19588023, + "rerank_top1_raw_score": -2.9934068, + "rerank_top2_raw_score": -8.115297, + "rerank_margin_raw": 5.1218905, + "support_top1_score": 0.62454784, + "candidate_memory_ids": [ + 129, + 130, + 122 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 129 + ], + "no_hit": false, + "first_stage_top1_distance": 0.19588023, + "rerank_top1_raw_score": -2.9934068, + "rerank_top2_raw_score": -8.115297, + "rerank_margin_raw": 5.1218905, + "support_top1_score": 0.62454784, + "activation_decision": "recall", + "recalled_memory_ids": [ + 129 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "passed": true + }, + { + "query_id": "pr_test_12", + "query_text": "just to be clear: we're at 768 dims now, not 1024 — that changed when we swapped the embed model.", + "query_category": "passive recall / knowledge update signal", + "interaction_mode": "update", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 17, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 138, + "score": 0.2402525, + "rank": 1, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 133, + "score": 0.49622554, + "rank": 2, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 130, + "score": 0.5030153, + "rank": 3, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 116, + "score": 0.5127115, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 120, + "score": 0.5354637, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.54159725, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 131, + "score": 0.5527258, + "rank": 7, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 119, + "score": 0.554724, + "rank": 8, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 117, + "score": 0.5659037, + "rank": 9, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 135, + "score": 0.5704489, + "rank": 10, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 136, + "score": 0.5744902, + "rank": 11, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 118, + "score": 0.57459164, + "rank": 12, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": 0.5917344, + "rank": 13, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 137, + "score": 0.5985078, + "rank": 14, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 129, + "score": 0.60335857, + "rank": 15, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 132, + "score": 0.6275941, + "rank": 16, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 134, + "score": 0.6465672, + "rank": 17, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + } + ], + "reranked_candidates": [ + { + "memory_id": 138, + "score": 0.9366961, + "rank": 1, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 130, + "score": -10.281282, + "rank": 2, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 133, + "score": -10.301105, + "rank": 3, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 135, + "score": -10.517851, + "rank": 4, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 121, + "score": -10.564142, + "rank": 5, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 116, + "score": -10.651553, + "rank": 6, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 119, + "score": -10.748668, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 131, + "score": -10.904555, + "rank": 8, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 120, + "score": -10.917755, + "rank": 9, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": -10.928948, + "rank": 10, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 88, + "first_stage_top1_distance": 0.2402525, + "rerank_top1_raw_score": 0.9366961, + "rerank_top2_raw_score": -10.281282, + "rerank_margin_raw": 11.2179785, + "support_top1_score": 0.28795308, + "candidate_memory_ids": [ + 138, + 130, + 133 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 138 + ], + "no_hit": false, + "first_stage_top1_distance": 0.2402525, + "rerank_top1_raw_score": 0.9366961, + "rerank_top2_raw_score": -10.281282, + "rerank_margin_raw": 11.2179785, + "support_top1_score": 0.28795308, + "activation_decision": "recall", + "recalled_memory_ids": [ + 138 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 88, + "passed": true + }, + { + "query_id": "pr_test_13", + "query_text": "the two time-window misses were near the window boundary.", + "query_category": "passive recall / temporal metric cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 17, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 136, + "score": 0.28306454, + "rank": 1, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 131, + "score": 0.47589642, + "rank": 2, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 118, + "score": 0.49109548, + "rank": 3, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 133, + "score": 0.5234668, + "rank": 4, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 135, + "score": 0.5383048, + "rank": 5, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 129, + "score": 0.5563912, + "rank": 6, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 120, + "score": 0.5575462, + "rank": 7, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": 0.5585276, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": 0.56302714, + "rank": 9, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 130, + "score": 0.57543695, + "rank": 10, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 132, + "score": 0.57610285, + "rank": 11, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 138, + "score": 0.5765212, + "rank": 12, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 134, + "score": 0.58120906, + "rank": 13, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 119, + "score": 0.58427584, + "rank": 14, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": 0.5897546, + "rank": 15, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 116, + "score": 0.5949136, + "rank": 16, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 137, + "score": 0.634472, + "rank": 17, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + } + ], + "reranked_candidates": [ + { + "memory_id": 136, + "score": 5.0668573, + "rank": 1, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 118, + "score": -8.981163, + "rank": 2, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": -9.88719, + "rank": 3, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 117, + "score": -9.929548, + "rank": 4, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 135, + "score": -10.941086, + "rank": 5, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 133, + "score": -10.985739, + "rank": 6, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 120, + "score": -11.006215, + "rank": 7, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 129, + "score": -11.035196, + "rank": 8, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 130, + "score": -11.037376, + "rank": 9, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 131, + "score": -11.039229, + "rank": 10, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.28306454, + "rerank_top1_raw_score": 5.0668573, + "rerank_top2_raw_score": -8.981163, + "rerank_margin_raw": 14.04802, + "support_top1_score": 1.0, + "candidate_memory_ids": [ + 136, + 118, + 122 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 136 + ], + "no_hit": false, + "first_stage_top1_distance": 0.28306454, + "rerank_top1_raw_score": 5.0668573, + "rerank_top2_raw_score": -8.981163, + "rerank_margin_raw": 14.04802, + "support_top1_score": 1.0, + "activation_decision": "recall", + "recalled_memory_ids": [ + 136 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "passed": true + }, + { + "query_id": "pr_test_14", + "query_text": "working on the dotfiles syncer today, not touching klbr.", + "query_category": "passive recall / no recall context switch", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777068000, + "expected_memory_action": "no_recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 131, + "score": 0.37757128, + "rank": 1, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 118, + "score": 0.5226496, + "rank": 2, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 119, + "score": 0.5247977, + "rank": 3, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": 0.5284648, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.5417839, + "rank": 5, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 117, + "score": 0.5474844, + "rank": 6, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 130, + "score": 0.5582626, + "rank": 7, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 129, + "score": 0.6037574, + "rank": 8, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 122, + "score": 0.6225207, + "rank": 9, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 132, + "score": 0.6709156, + "rank": 10, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 116, + "score": 0.6819655, + "rank": 11, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + } + ], + "reranked_candidates": [ + { + "memory_id": 131, + "score": -2.4893126, + "rank": 1, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 118, + "score": -4.634074, + "rank": 2, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 130, + "score": -5.436397, + "rank": 3, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 117, + "score": -5.562685, + "rank": 4, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 119, + "score": -5.5901155, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": -5.7689433, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 120, + "score": -5.955366, + "rank": 7, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 132, + "score": -6.0349894, + "rank": 8, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 122, + "score": -6.294493, + "rank": 9, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 129, + "score": -6.7945642, + "rank": 10, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 97, + "first_stage_top1_distance": 0.37757128, + "rerank_top1_raw_score": -2.4893126, + "rerank_top2_raw_score": -4.634074, + "rerank_margin_raw": 2.1447616, + "support_top1_score": 0.38702488, + "candidate_memory_ids": [ + 131, + 118, + 130 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.37757128, + "rerank_top1_raw_score": -2.4893126, + "rerank_top2_raw_score": -4.634074, + "rerank_margin_raw": 2.1447616, + "support_top1_score": 0.38702488, + "activation_decision": "recall", + "recalled_memory_ids": [ + 131, + 118, + 130 + ], + "activation_correct": false, + "support_precision_at_k": 0.0, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 97, + "passed": false + }, + { + "query_id": "pr_test_15", + "query_text": "starting fresh on a completely unrelated thing, nothing to do with the agent.", + "query_category": "passive recall / no recall fresh start signal", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "no_recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 142, + "score": 0.4807309, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 119, + "score": 0.50228477, + "rank": 2, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 144, + "score": 0.50527066, + "rank": 3, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 131, + "score": 0.5207622, + "rank": 4, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 120, + "score": 0.52080846, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 117, + "score": 0.52438366, + "rank": 6, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": 0.5493816, + "rank": 7, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 121, + "score": 0.564029, + "rank": 8, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 118, + "score": 0.574108, + "rank": 9, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 135, + "score": 0.57813495, + "rank": 10, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 136, + "score": 0.58556914, + "rank": 11, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 133, + "score": 0.5949757, + "rank": 12, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 129, + "score": 0.59690523, + "rank": 13, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 116, + "score": 0.599094, + "rank": 14, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 132, + "score": 0.60119104, + "rank": 15, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 130, + "score": 0.60399663, + "rank": 16, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 143, + "score": 0.6157371, + "rank": 17, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 137, + "score": 0.61939967, + "rank": 18, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 141, + "score": 0.6600579, + "rank": 19, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 140, + "score": 0.6615919, + "rank": 20, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + } + ], + "reranked_candidates": [ + { + "memory_id": 142, + "score": -5.284974, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 144, + "score": -7.957759, + "rank": 2, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 131, + "score": -8.104524, + "rank": 3, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 121, + "score": -10.094551, + "rank": 4, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -10.4260845, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 118, + "score": -10.500211, + "rank": 6, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 122, + "score": -10.574907, + "rank": 7, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 117, + "score": -10.667948, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 135, + "score": -10.755182, + "rank": 9, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 120, + "score": -10.951125, + "rank": 10, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 80, + "first_stage_top1_distance": 0.50228477, + "rerank_top1_raw_score": -8.10042, + "rerank_top2_raw_score": -10.11149, + "rerank_margin_raw": 2.0110703, + "support_top1_score": 0.25528118, + "candidate_memory_ids": [ + 131, + 121, + 119 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 73, + "first_stage_top1_distance": 0.50228477, + "rerank_top1_raw_score": -8.09477, + "rerank_top2_raw_score": -10.114449, + "rerank_margin_raw": 2.019678, + "support_top1_score": 0.25528118, + "candidate_memory_ids": [ + 131, + 121, + 119 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.4807309, + "rerank_top1_raw_score": -5.284974, + "rerank_top2_raw_score": -7.957759, + "rerank_margin_raw": 2.6727848, + "support_top1_score": 0.18321037, + "candidate_memory_ids": [ + 142, + 144, + 131 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 70, + "first_stage_top1_distance": 0.4807309, + "rerank_top1_raw_score": -5.284974, + "rerank_top2_raw_score": -7.957759, + "rerank_margin_raw": 2.6727848, + "support_top1_score": 0.18321037, + "candidate_memory_ids": [ + 142, + 144, + 131 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [], + "no_hit": true, + "first_stage_top1_distance": 0.4807309, + "rerank_top1_raw_score": -5.284974, + "rerank_top2_raw_score": -7.957759, + "rerank_margin_raw": 2.6727848, + "support_top1_score": 0.18321037, + "activation_decision": "no_recall", + "activation_correct": true, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 295, + "passed": true + }, + { + "query_id": "pr_test_16", + "query_text": "the v3 passive recall file should fold in the public dataset extractions.", + "query_category": "passive recall / planning naming cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 17, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 137, + "score": 0.22689289, + "rank": 1, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 120, + "score": 0.334544, + "rank": 2, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.38997638, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": 0.3940553, + "rank": 4, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 132, + "score": 0.4326138, + "rank": 5, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 135, + "score": 0.4382822, + "rank": 6, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 122, + "score": 0.46939868, + "rank": 7, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 130, + "score": 0.47538328, + "rank": 8, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 118, + "score": 0.5086293, + "rank": 9, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 131, + "score": 0.52717257, + "rank": 10, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 116, + "score": 0.5325806, + "rank": 11, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 136, + "score": 0.54101145, + "rank": 12, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + }, + { + "memory_id": 133, + "score": 0.54614425, + "rank": 13, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 117, + "score": 0.55678874, + "rank": 14, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 138, + "score": 0.55790854, + "rank": 15, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 129, + "score": 0.5780817, + "rank": 16, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 134, + "score": 0.5799016, + "rank": 17, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + } + ], + "reranked_candidates": [ + { + "memory_id": 137, + "score": 5.374874, + "rank": 1, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 121, + "score": -2.065392, + "rank": 2, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 120, + "score": -3.443923, + "rank": 3, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 132, + "score": -3.452368, + "rank": 4, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 119, + "score": -4.229025, + "rank": 5, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 130, + "score": -4.6715984, + "rank": 6, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 135, + "score": -5.5175314, + "rank": 7, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 122, + "score": -5.6780977, + "rank": 8, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": -8.32161, + "rank": 9, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 131, + "score": -10.816792, + "rank": 10, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "first_stage_top1_distance": 0.22689289, + "rerank_top1_raw_score": 5.374874, + "rerank_top2_raw_score": -2.065392, + "rerank_margin_raw": 7.440266, + "support_top1_score": 0.8701945, + "candidate_memory_ids": [ + 137, + 121, + 120 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 137 + ], + "no_hit": false, + "first_stage_top1_distance": 0.22689289, + "rerank_top1_raw_score": 5.374874, + "rerank_top2_raw_score": -2.065392, + "rerank_margin_raw": 7.440266, + "support_top1_score": 0.8701945, + "activation_decision": "recall", + "recalled_memory_ids": [ + 137 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "passed": true + }, + { + "query_id": "pr_test_17", + "query_text": "LoCoMo's conversation structure is a good model for long-gap memory references.", + "query_category": "passive recall / multi-session modeling cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 17, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 134, + "score": 0.081481874, + "rank": 1, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 135, + "score": 0.41981453, + "rank": 2, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 132, + "score": 0.42578232, + "rank": 3, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 120, + "score": 0.48137563, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 129, + "score": 0.4863804, + "rank": 5, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 119, + "score": 0.5066352, + "rank": 6, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 121, + "score": 0.5318754, + "rank": 7, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 116, + "score": 0.53641343, + "rank": 8, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 130, + "score": 0.5566159, + "rank": 9, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 137, + "score": 0.56580985, + "rank": 10, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 138, + "score": 0.573792, + "rank": 11, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 131, + "score": 0.5745742, + "rank": 12, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 122, + "score": 0.58350825, + "rank": 13, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": 0.5869934, + "rank": 14, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 133, + "score": 0.59435964, + "rank": 15, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 117, + "score": 0.61535704, + "rank": 16, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 136, + "score": 0.6248877, + "rank": 17, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + } + ], + "reranked_candidates": [ + { + "memory_id": 134, + "score": 4.6008663, + "rank": 1, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 135, + "score": -8.052744, + "rank": 2, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 132, + "score": -9.23103, + "rank": 3, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 116, + "score": -9.618591, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 129, + "score": -10.206064, + "rank": 5, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 120, + "score": -10.837906, + "rank": 6, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 130, + "score": -10.887512, + "rank": 7, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 137, + "score": -10.910784, + "rank": 8, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 121, + "score": -10.9653225, + "rank": 9, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 119, + "score": -10.978912, + "rank": 10, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.081481874, + "rerank_top1_raw_score": 4.6008663, + "rerank_top2_raw_score": -8.052744, + "rerank_margin_raw": 12.65361, + "support_top1_score": 0.8146937, + "candidate_memory_ids": [ + 134, + 135, + 132 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "gold_memory_ids": [ + 134 + ], + "no_hit": false, + "first_stage_top1_distance": 0.081481874, + "rerank_top1_raw_score": 4.6008663, + "rerank_top2_raw_score": -8.052744, + "rerank_margin_raw": 12.65361, + "support_top1_score": 0.8146937, + "activation_decision": "recall", + "recalled_memory_ids": [ + 134 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": true, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "passed": true + }, + { + "query_id": "pr_test_18", + "query_text": "remind me why we stuck with postgres instead of sqlite a couple months ago.", + "query_category": "passive recall / long gap database context", + "interaction_mode": "request", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 90, + "windows_considered": [ + 7, + 30, + 90 + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 140, + "score": 0.2860654, + "rank": 1, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 120, + "score": 0.42734176, + "rank": 2, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.43390888, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 141, + "score": 0.4556368, + "rank": 4, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 135, + "score": 0.45963156, + "rank": 5, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 131, + "score": 0.46250647, + "rank": 6, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 118, + "score": 0.46758914, + "rank": 7, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 119, + "score": 0.47985637, + "rank": 8, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 142, + "score": 0.50445163, + "rank": 9, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 116, + "score": 0.54107225, + "rank": 10, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 130, + "score": 0.5452374, + "rank": 11, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 137, + "score": 0.54754233, + "rank": 12, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 117, + "score": 0.54902714, + "rank": 13, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 122, + "score": 0.55449474, + "rank": 14, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 132, + "score": 0.55549103, + "rank": 15, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 129, + "score": 0.5598958, + "rank": 16, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 138, + "score": 0.56105447, + "rank": 17, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 133, + "score": 0.57695144, + "rank": 18, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 134, + "score": 0.58575606, + "rank": 19, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 136, + "score": 0.62397873, + "rank": 20, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + } + ], + "reranked_candidates": [ + { + "memory_id": 140, + "score": -1.023617, + "rank": 1, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 118, + "score": -8.670485, + "rank": 2, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 121, + "score": -9.399988, + "rank": 3, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 141, + "score": -9.589307, + "rank": 4, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 135, + "score": -10.733733, + "rank": 5, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 116, + "score": -10.87273, + "rank": 6, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 119, + "score": -10.902052, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": -10.904181, + "rank": 8, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 131, + "score": -11.002138, + "rank": 9, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 142, + "score": -11.032398, + "rank": 10, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 90, + "first_stage_top1_distance": 0.42734176, + "rerank_top1_raw_score": -8.703848, + "rerank_top2_raw_score": -9.389654, + "rerank_margin_raw": 0.6858063, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 118, + 121, + 135 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 78, + "first_stage_top1_distance": 0.42734176, + "rerank_top1_raw_score": -8.703848, + "rerank_top2_raw_score": -9.368376, + "rerank_margin_raw": 0.6645279, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 118, + 121, + 135 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.2860654, + "rerank_top1_raw_score": -1.023617, + "rerank_top2_raw_score": -8.670485, + "rerank_margin_raw": 7.6468678, + "support_top1_score": 0.27965602, + "candidate_memory_ids": [ + 140, + 118, + 121 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 140 + ], + "no_hit": false, + "first_stage_top1_distance": 0.2860654, + "rerank_top1_raw_score": -1.023617, + "rerank_top2_raw_score": -8.670485, + "rerank_margin_raw": 7.6468678, + "support_top1_score": 0.27965602, + "activation_decision": "recall", + "recalled_memory_ids": [ + 140 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": false, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 240, + "passed": true + }, + { + "query_id": "pr_test_21", + "query_text": "starting up that real estate scraper project again. i want to prototype fast, so i'm leaning python.", + "query_category": "passive recall / multi-session reasoning cue (variant)", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 90, + "windows_considered": [ + 7, + 30, + 90 + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 142, + "score": 0.25074226, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 141, + "score": 0.3196075, + "rank": 2, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 131, + "score": 0.43651706, + "rank": 3, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 130, + "score": 0.45942736, + "rank": 4, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 120, + "score": 0.48547935, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 121, + "score": 0.492841, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 133, + "score": 0.49575728, + "rank": 7, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 135, + "score": 0.5087309, + "rank": 8, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 119, + "score": 0.5120943, + "rank": 9, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 137, + "score": 0.52569354, + "rank": 10, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 122, + "score": 0.532443, + "rank": 11, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 143, + "score": 0.5377384, + "rank": 12, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 140, + "score": 0.5538671, + "rank": 13, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 144, + "score": 0.5656187, + "rank": 14, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 134, + "score": 0.5658394, + "rank": 15, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 118, + "score": 0.5671787, + "rank": 16, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 117, + "score": 0.58282906, + "rank": 17, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 132, + "score": 0.58503413, + "rank": 18, + "event_time": 1777068000, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr13:m1", + "text": "the passive recall expansion added temporal, update, and abstention query patterns inspired by LongMemEval and TemporalMemoryDataset." + }, + { + "memory_id": 116, + "score": 0.59130615, + "rank": 19, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 138, + "score": 0.5969615, + "rank": 20, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + } + ], + "reranked_candidates": [ + { + "memory_id": 142, + "score": 2.0435534, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 141, + "score": 1.7250593, + "rank": 2, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 133, + "score": -10.31483, + "rank": 3, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 130, + "score": -10.433972, + "rank": 4, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 137, + "score": -10.505627, + "rank": 5, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 135, + "score": -10.640972, + "rank": 6, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 119, + "score": -10.697934, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 120, + "score": -10.801752, + "rank": 8, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 131, + "score": -10.815447, + "rank": 9, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 121, + "score": -10.834429, + "rank": 10, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.43651706, + "rerank_top1_raw_score": -10.325264, + "rerank_top2_raw_score": -10.44928, + "rerank_margin_raw": 0.12401581, + "support_top1_score": 0.008013566, + "candidate_memory_ids": [ + 133, + 130, + 137 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 84, + "first_stage_top1_distance": 0.43651706, + "rerank_top1_raw_score": -10.325264, + "rerank_top2_raw_score": -10.439591, + "rerank_margin_raw": 0.11432743, + "support_top1_score": 0.008013567, + "candidate_memory_ids": [ + 133, + 130, + 137 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "first_stage_top1_distance": 0.25074226, + "rerank_top1_raw_score": 2.0435534, + "rerank_top2_raw_score": 1.7250593, + "rerank_margin_raw": 0.31849408, + "support_top1_score": 0.38224205, + "candidate_memory_ids": [ + 142, + 141, + 133 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 141, + 142 + ], + "no_hit": false, + "first_stage_top1_distance": 0.25074226, + "rerank_top1_raw_score": 2.0435534, + "rerank_top2_raw_score": 1.7250593, + "rerank_margin_raw": 0.31849408, + "support_top1_score": 0.38224205, + "activation_decision": "recall", + "recalled_memory_ids": [ + 142, + 141 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": false, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 251, + "passed": true + }, + { + "query_id": "pr_test_19", + "query_text": "i'm about to start coding that real estate scraper; thinking of using python like i usually do.", + "query_category": "passive recall / multi-session reasoning cue", + "interaction_mode": "statement", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "chosen_window_days": 90, + "windows_considered": [ + 7, + 30, + 90 + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 142, + "score": 0.28328294, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 141, + "score": 0.37138766, + "rank": 2, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 131, + "score": 0.43199855, + "rank": 3, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 133, + "score": 0.48271954, + "rank": 4, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 120, + "score": 0.49345094, + "rank": 5, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 130, + "score": 0.50541687, + "rank": 6, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 119, + "score": 0.5141603, + "rank": 7, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 135, + "score": 0.5191818, + "rank": 8, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 137, + "score": 0.5263306, + "rank": 9, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 121, + "score": 0.52747905, + "rank": 10, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 140, + "score": 0.5293382, + "rank": 11, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 122, + "score": 0.56561786, + "rank": 12, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 118, + "score": 0.5725387, + "rank": 13, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 143, + "score": 0.57315874, + "rank": 14, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 134, + "score": 0.57427603, + "rank": 15, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 144, + "score": 0.58914006, + "rank": 16, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 129, + "score": 0.59144783, + "rank": 17, + "event_time": 1776985200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:support" + ], + "source_ref": "session:pr11:m3", + "text": "RealMemBench memory_used annotations are a clean source of query-turn to support-set pairings." + }, + { + "memory_id": 116, + "score": 0.5943245, + "rank": 18, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 117, + "score": 0.59484047, + "rank": 19, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 136, + "score": 0.5973432, + "rank": 20, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + } + ], + "reranked_candidates": [ + { + "memory_id": 142, + "score": 0.059341762, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 141, + "score": -4.289649, + "rank": 2, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 131, + "score": -10.782325, + "rank": 3, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 137, + "score": -10.937529, + "rank": 4, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 133, + "score": -10.939461, + "rank": 5, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 121, + "score": -10.950474, + "rank": 6, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 135, + "score": -10.990065, + "rank": 7, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 130, + "score": -11.016455, + "rank": 8, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 120, + "score": -11.023264, + "rank": 9, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 119, + "score": -11.03036, + "rank": 10, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 95, + "first_stage_top1_distance": 0.43199855, + "rerank_top1_raw_score": -10.783558, + "rerank_top2_raw_score": -10.924131, + "rerank_margin_raw": 0.1405735, + "support_top1_score": 0.013441827, + "candidate_memory_ids": [ + 131, + 137, + 121 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.43199855, + "rerank_top1_raw_score": -10.788286, + "rerank_top2_raw_score": -10.922166, + "rerank_margin_raw": 0.13387966, + "support_top1_score": 0.013441826, + "candidate_memory_ids": [ + 131, + 137, + 133 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 83, + "first_stage_top1_distance": 0.28328294, + "rerank_top1_raw_score": 0.059341762, + "rerank_top2_raw_score": -4.289649, + "rerank_margin_raw": 4.348991, + "support_top1_score": 0.24517192, + "candidate_memory_ids": [ + 142, + 141, + 131 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 141, + 142 + ], + "no_hit": false, + "first_stage_top1_distance": 0.28328294, + "rerank_top1_raw_score": 0.059341762, + "rerank_top2_raw_score": -4.289649, + "rerank_margin_raw": 4.348991, + "support_top1_score": 0.24517192, + "activation_decision": "recall", + "recalled_memory_ids": [ + 142, + 141 + ], + "activation_correct": true, + "support_precision_at_k": 1.0, + "support_recall_at_k": 1.0, + "time_window_hit": false, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 263, + "passed": true + }, + { + "query_id": "pr_test_20", + "query_text": "i need to ask someone about a bug in hydra... who should i ping?", + "query_category": "passive recall / knowledge update cue", + "interaction_mode": "request", + "split": "test", + "namespace": "default", + "reference_time": 1777096000, + "expected_memory_action": "recall", + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 22, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 143, + "score": 0.43062222, + "rank": 1, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 144, + "score": 0.48994702, + "rank": 2, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 131, + "score": 0.5394285, + "rank": 3, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 120, + "score": 0.5503036, + "rank": 4, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 142, + "score": 0.56095386, + "rank": 5, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 119, + "score": 0.5709644, + "rank": 6, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 133, + "score": 0.5716441, + "rank": 7, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + }, + { + "memory_id": 117, + "score": 0.57238877, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 141, + "score": 0.57730484, + "rank": 9, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 121, + "score": 0.60249925, + "rank": 10, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 137, + "score": 0.6066408, + "rank": 11, + "event_time": 1777089600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:passive-recall" + ], + "source_ref": "session:pr14:m3", + "text": "the next benchmark file will be called internal_eval_passive_recall_v3 and will fold in the public dataset extraction." + }, + { + "memory_id": 135, + "score": 0.60701233, + "rank": 12, + "event_time": 1777082400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:temporal" + ], + "source_ref": "session:pr14:m1", + "text": "added a reference_time field to passive recall queries so benchmarks can simulate how memory ages relative to the query." + }, + { + "memory_id": 118, + "score": 0.6095549, + "rank": 13, + "event_time": 1776877200, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:pr8:m3", + "text": "the support-calibrated operating point stayed on the frontier after the larger dataset rerun." + }, + { + "memory_id": 140, + "score": 0.6226842, + "rank": 14, + "event_time": 1772005000, + "tags": [ + "project:database", + "topic:architecture" + ], + "source_ref": "session:pr16:m1", + "text": "had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling." + }, + { + "memory_id": 138, + "score": 0.62665063, + "rank": 15, + "event_time": 1777093200, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:temporal" + ], + "source_ref": "session:pr14:m4", + "text": "the embed model is nomic-embed-text-v1.5 at 768 dims; bge-m3 at 1024 dims was used before the switch in pr10." + }, + { + "memory_id": 130, + "score": 0.6268393, + "rank": 16, + "event_time": 1776988800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr11:m4", + "text": "the passive benchmark now has dev and test slices; the next focus is support-set precision." + }, + { + "memory_id": 134, + "score": 0.6480623, + "rank": 17, + "event_time": 1777075200, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:multi-session" + ], + "source_ref": "session:pr13:m3", + "text": "the LoCoMo multi-session conversation structure is useful for modeling long-gap reference patterns." + }, + { + "memory_id": 122, + "score": 0.6487318, + "rank": 18, + "event_time": 1776884400, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr9:m4", + "text": "on the starter passive run, support precision was about 0.39 while support recall was about 0.94." + }, + { + "memory_id": 116, + "score": 0.6573858, + "rank": 19, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:pr8:m1", + "text": "the main internal eval starter set was expanded from 25 memories and 28 queries to 34 memories and 40 queries." + }, + { + "memory_id": 136, + "score": 0.6575217, + "rank": 20, + "event_time": 1777086000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:metrics" + ], + "source_ref": "session:pr14:m2", + "text": "time-window hit rate on the expanded passive slice is 0.82; the two misses were edge cases near the window boundary." + } + ], + "reranked_candidates": [ + { + "memory_id": 144, + "score": -6.8210077, + "rank": 1, + "event_time": 1774000000, + "tags": [ + "project:hydra", + "topic:team", + "topic:update" + ], + "source_ref": "session:pr18:m2", + "text": "bob has taken over the hydra project; alice moved to the agent team." + }, + { + "memory_id": 143, + "score": -6.9932423, + "rank": 2, + "event_time": 1772000000, + "tags": [ + "project:hydra", + "topic:team" + ], + "source_ref": "session:pr18:m1", + "text": "alice is the lead dev on the hydra project." + }, + { + "memory_id": 141, + "score": -10.896352, + "rank": 3, + "event_time": 1772000000, + "tags": [ + "topic:preference", + "topic:python" + ], + "source_ref": "session:pr17:m1", + "text": "i prefer using python for rapid prototyping because of the rich library ecosystem." + }, + { + "memory_id": 119, + "score": -11.035614, + "rank": 4, + "event_time": 1776879000, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:support" + ], + "source_ref": "session:pr9:m1", + "text": "the next passive-recall task should focus on support-set quality rather than activation thresholds." + }, + { + "memory_id": 117, + "score": -11.037694, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:no-hit" + ], + "source_ref": "session:pr8:m2", + "text": "after expansion, the dev slice contained 22 queries and 5 real no-hit cases." + }, + { + "memory_id": 131, + "score": -11.0386715, + "rank": 6, + "event_time": 1777064400, + "tags": [ + "project:dotfiles", + "topic:cli" + ], + "source_ref": "session:pr12:m1", + "text": "looking at a completely different side project today: a small rust cli for syncing dotfiles." + }, + { + "memory_id": 121, + "score": -11.03994, + "rank": 7, + "event_time": 1776882600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m3", + "text": "the passive-recall benchmark should have separate dev and test slices instead of staying dev-only." + }, + { + "memory_id": 142, + "score": -11.039975, + "rank": 8, + "event_time": 1774000000, + "tags": [ + "project:scraper", + "topic:web-scraping" + ], + "source_ref": "session:pr17:m2", + "text": "starting a new project to scrape real estate data from multiple listing sites." + }, + { + "memory_id": 120, + "score": -11.040657, + "rank": 9, + "event_time": 1776880800, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:eval-data" + ], + "source_ref": "session:pr9:m2", + "text": "the passive dataset still needs more should-not-recall cases and more update-style cues." + }, + { + "memory_id": 133, + "score": -11.0407, + "rank": 10, + "event_time": 1777071600, + "tags": [ + "project:klbr", + "topic:passive-recall", + "topic:calibration" + ], + "source_ref": "session:pr13:m2", + "text": "support-set precision is the main open metric; the plan is to tighten the rerank score threshold or add a second-pass filter." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 94, + "first_stage_top1_distance": 0.5394285, + "rerank_top1_raw_score": -11.03594, + "rerank_top2_raw_score": -11.036573, + "rerank_margin_raw": 0.00063323975, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 118, + 119, + 117 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 17, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.5394285, + "rerank_top1_raw_score": -11.03594, + "rerank_top2_raw_score": -11.036008, + "rerank_margin_raw": 0.00006771088, + "support_top1_score": 0.0, + "candidate_memory_ids": [ + 118, + 119, + 117 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 71, + "first_stage_top1_distance": 0.43062222, + "rerank_top1_raw_score": -6.8210077, + "rerank_top2_raw_score": -6.9932423, + "rerank_margin_raw": 0.17223454, + "support_top1_score": 0.13470533, + "candidate_memory_ids": [ + 144, + 143, + 141 + ], + "decision": "expand" + }, + { + "memories_searched": 22, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 71, + "first_stage_top1_distance": 0.43062222, + "rerank_top1_raw_score": -6.8210077, + "rerank_top2_raw_score": -6.9932423, + "rerank_margin_raw": 0.17223454, + "support_top1_score": 0.13470533, + "candidate_memory_ids": [ + 144, + 143, + 141 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "gold_memory_ids": [ + 144 + ], + "no_hit": false, + "first_stage_top1_distance": 0.43062222, + "rerank_top1_raw_score": -6.8210077, + "rerank_top2_raw_score": -6.9932423, + "rerank_margin_raw": 0.17223454, + "support_top1_score": 0.13470533, + "activation_decision": "no_recall", + "activation_correct": false, + "support_recall_at_k": 0.0, + "time_window_hit": false, + "embedding_latency_ms": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 321, + "passed": false + } +] \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.json new file mode 100644 index 0000000..d983c04 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.json @@ -0,0 +1,61 @@ +{ + "experiment": { + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 + }, + "dataset_id": "internal-eval-starter-v2", + "evaluated_split": "dev", + "ingest_count": 34, + "ingest_latency_ms": 1399, + "metrics": { + "query_count": 17, + "recall_at_1": 0.88235295, + "recall_at_5": 0.9411765, + "recall_at_10": 0.9411765, + "recall_at_20": 1.0, + "mrr": 0.8994652, + "ndcg_at_10": 0.8681219, + "top1_accuracy_after_rerank": 0.88235295, + "final_top1_accuracy": 0.88235295, + "mrr_after_rerank": 0.90196085, + "ndcg_after_rerank": 0.877629, + "reranker_lift_mrr": 0.0024956465, + "candidate_set_size_mean": 11.705882, + "time_window_hit_rate": 0.88235295, + "final_decision_accuracy": 0.88235295, + "coverage": 0.88235295, + "abstain_accuracy": 0.0, + "selective_accuracy": 1.0, + "retrieval_latency_p50_ms": 0, + "retrieval_latency_p95_ms": 0, + "embedding_latency_p50_ms": 10, + "embedding_latency_p95_ms": 14, + "rerank_latency_p50_ms": 99, + "rerank_latency_p95_ms": 339 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.md b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.md new file mode 100644 index 0000000..59ef96e --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.md @@ -0,0 +1,36 @@ +# Retrieval Benchmark Report + +dataset: `internal-eval-starter-v2` +experiment: `mvp-rerank-support-calibrated` +split: `dev` +queries: `17` +memories ingested: `34` +ingest latency ms: `1399` +window expansions: `4` + +## Metrics + +- Recall@1: `0.8824` +- Recall@5: `0.9412` +- Recall@10: `0.9412` +- Recall@20: `1.0000` +- MRR: `0.8995` +- nDCG@10: `0.8681` +- Top1 accuracy after rerank: `0.8824` +- Final top1 accuracy: `0.8824` +- MRR after rerank: `0.9020` +- nDCG after rerank: `0.8776` +- Reranker lift (MRR): `0.0025` +- Candidate set size mean: `11.71` +- Time-window hit rate: `0.8824` +- Final decision accuracy: `0.8824` +- Coverage: `0.8824` +- Abstain accuracy: `0.0000` +- Selective accuracy: `1.0000` +- Retrieval latency p50/p95 ms: `0` / `0` +- Embedding latency p50/p95 ms: `10` / `14` +- Rerank latency p50/p95 ms: `99` / `339` + +## No-Hit False Answers + +(none) diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/resolved_config.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/resolved_config.json new file mode 100644 index 0000000..5b834f0 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/resolved_config.json @@ -0,0 +1,29 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/traces.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/traces.json new file mode 100644 index 0000000..6f3f470 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/traces.json @@ -0,0 +1,5704 @@ +[ + { + "query_id": "dev_q3", + "query_text": "what have i been working on lately?", + "query_category": "vague recent lookup", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 20, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 10, + "score": 0.32049072, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 17, + "score": 0.46884745, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 2, + "score": 0.49202025, + "rank": 3, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 26, + "score": 0.5327027, + "rank": 4, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 8, + "score": 0.53850216, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 29, + "score": 0.53912187, + "rank": 6, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 19, + "score": 0.5575949, + "rank": 7, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 27, + "score": 0.5583326, + "rank": 8, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 5, + "score": 0.5617424, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 9, + "score": 0.56325024, + "rank": 10, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 23, + "score": 0.5684378, + "rank": 11, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 22, + "score": 0.5744454, + "rank": 12, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 28, + "score": 0.5858302, + "rank": 13, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 30, + "score": 0.5896242, + "rank": 14, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m2", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs." + }, + { + "memory_id": 4, + "score": 0.58967173, + "rank": 15, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 6, + "score": 0.60216486, + "rank": 16, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 31, + "score": 0.6090862, + "rank": 17, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 7, + "score": 0.63500476, + "rank": 18, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 3, + "score": 0.6379826, + "rank": 19, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 18, + "score": 0.66763496, + "rank": 20, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + } + ], + "reranked_candidates": [ + { + "memory_id": 10, + "score": -0.013783596, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 17, + "score": -9.956665, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 19, + "score": -10.215273, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 2, + "score": -10.468879, + "rank": 4, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 9, + "score": -10.470537, + "rank": 5, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 8, + "score": -10.502787, + "rank": 6, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 29, + "score": -10.549325, + "rank": 7, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 27, + "score": -10.821674, + "rank": 8, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 26, + "score": -10.909702, + "rank": 9, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 5, + "score": -10.924212, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 563, + "first_stage_top1_distance": 0.32049072, + "rerank_top1_raw_score": 0.01993569, + "rerank_top2_raw_score": -9.904642, + "rerank_margin_raw": 9.924578, + "support_top1_score": 0.14852512, + "candidate_memory_ids": [ + 10, + 22, + 17 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 132, + "first_stage_top1_distance": 0.32049072, + "rerank_top1_raw_score": -0.008803926, + "rerank_top2_raw_score": -9.886829, + "rerank_margin_raw": 9.878025, + "support_top1_score": 0.14852512, + "candidate_memory_ids": [ + 10, + 22, + 17 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 79, + "first_stage_top1_distance": 0.32049072, + "rerank_top1_raw_score": 0.003674876, + "rerank_top2_raw_score": -9.886829, + "rerank_margin_raw": 9.890504, + "support_top1_score": 0.14852512, + "candidate_memory_ids": [ + 10, + 22, + 17 + ], + "decision": "expand" + }, + { + "memories_searched": 20, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 154, + "first_stage_top1_distance": 0.32049072, + "rerank_top1_raw_score": -0.013783596, + "rerank_top2_raw_score": -9.956665, + "rerank_margin_raw": 9.942882, + "support_top1_score": 0.14852513, + "candidate_memory_ids": [ + 10, + 17, + 19 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "abstained": true, + "no_hit": false, + "gold_memory_ids": [ + 22, + 23 + ], + "gold_answer": "better eval data and the shared retrieval path", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.32049072, + "rerank_top1_raw_score": -0.013783596, + "rerank_top2_raw_score": -9.956665, + "rerank_margin_raw": 9.942882, + "support_top1_score": 0.14852513, + "final_decision": "abstain", + "final_correct": false, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.32049072, + "embedding_latency_ms": 15, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 928, + "passed": false + }, + { + "query_id": "dev_q4", + "query_text": "what kind of infra pattern do i keep using in this project?", + "query_category": "recurring theme / pattern-like question", + "split": "dev", + "namespace": "default", + "reference_time": 1776780000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 10, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 19, + "score": 0.3692118, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 10, + "score": 0.49105126, + "rank": 2, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 17, + "score": 0.5083922, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 8, + "score": 0.5490223, + "rank": 4, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 5, + "score": 0.5567069, + "rank": 5, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 7, + "score": 0.57184356, + "rank": 6, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 18, + "score": 0.583371, + "rank": 7, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 22, + "score": 0.5906385, + "rank": 8, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 9, + "score": 0.600634, + "rank": 9, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 6, + "score": 0.6019941, + "rank": 10, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + } + ], + "reranked_candidates": [ + { + "memory_id": 19, + "score": 0.29104856, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 22, + "score": -8.582125, + "rank": 2, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 10, + "score": -9.081812, + "rank": 3, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 9, + "score": -9.193537, + "rank": 4, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 8, + "score": -9.517863, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": -9.711531, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 18, + "score": -9.83715, + "rank": 7, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 6, + "score": -9.965145, + "rank": 8, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 7, + "score": -10.031506, + "rank": 9, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 5, + "score": -10.9163475, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 111, + "first_stage_top1_distance": 0.3692118, + "rerank_top1_raw_score": 0.29104856, + "rerank_top2_raw_score": -8.582125, + "rerank_margin_raw": 8.873174, + "support_top1_score": 0.30281678, + "candidate_memory_ids": [ + 19, + 22, + 10 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 19 + ], + "gold_answer": "splitting chat, embedding, and rerank services by role", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.3692118, + "rerank_top1_raw_score": 0.29104856, + "rerank_top2_raw_score": -8.582125, + "rerank_margin_raw": 8.873174, + "support_top1_score": 0.30281678, + "final_decision": "answer", + "final_answer_memory_ids": [ + 19 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.3692118, + "embedding_latency_ms": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 111, + "passed": true + }, + { + "query_id": "dev_q5", + "query_text": "what editor am i using now for klbr?", + "query_category": "conflict / update question", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 30, + "windows_considered": [ + 7, + 30 + ], + "memories_searched": 14, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 2, + "score": 0.2684508, + "rank": 1, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 10, + "score": 0.604403, + "rank": 2, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 5, + "score": 0.60762554, + "rank": 3, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 17, + "score": 0.6176268, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 3, + "score": 0.6349721, + "rank": 5, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 6, + "score": 0.64758503, + "rank": 6, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 4, + "score": 0.658549, + "rank": 7, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 23, + "score": 0.6602662, + "rank": 8, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 8, + "score": 0.6652224, + "rank": 9, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 7, + "score": 0.6712187, + "rank": 10, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 9, + "score": 0.67814237, + "rank": 11, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 22, + "score": 0.6876642, + "rank": 12, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": 0.69988424, + "rank": 13, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 18, + "score": 0.704443, + "rank": 14, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + } + ], + "reranked_candidates": [ + { + "memory_id": 2, + "score": 3.5080538, + "rank": 1, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 23, + "score": -3.1734772, + "rank": 2, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 3, + "score": -3.2604022, + "rank": 3, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 8, + "score": -4.9491577, + "rank": 4, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 6, + "score": -5.348194, + "rank": 5, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 10, + "score": -5.40172, + "rank": 6, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 4, + "score": -5.6386094, + "rank": 7, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 7, + "score": -5.795266, + "rank": 8, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 17, + "score": -5.8278255, + "rank": 9, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 5, + "score": -7.322429, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 85, + "first_stage_top1_distance": 0.604403, + "rerank_top1_raw_score": -3.1641595, + "rerank_top2_raw_score": -4.956311, + "rerank_margin_raw": 1.7921517, + "support_top1_score": 0.047543235, + "candidate_memory_ids": [ + 23, + 8, + 6 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 110, + "first_stage_top1_distance": 0.2684508, + "rerank_top1_raw_score": 3.5080538, + "rerank_top2_raw_score": -3.1734772, + "rerank_margin_raw": 6.681531, + "support_top1_score": 0.40727422, + "candidate_memory_ids": [ + 2, + 23, + 3 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 2 + ], + "gold_answer": "zed", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.2684508, + "rerank_top1_raw_score": 3.5080538, + "rerank_top2_raw_score": -3.1734772, + "rerank_margin_raw": 6.681531, + "support_top1_score": 0.40727422, + "final_decision": "answer", + "final_answer_memory_ids": [ + 2 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": false, + "best_distance": 0.2684508, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 195, + "passed": true + }, + { + "query_id": "dev_q10", + "query_text": "what was fixed on april 15 when the stream failed?", + "query_category": "exact dated event", + "split": "dev", + "namespace": "default", + "reference_time": 1776250800, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 4, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 5, + "score": 0.41856718, + "rank": 1, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 2, + "score": 0.59937686, + "rank": 2, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 3, + "score": 0.64474595, + "rank": 3, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 4, + "score": 0.65327144, + "rank": 4, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + } + ], + "reranked_candidates": [ + { + "memory_id": 5, + "score": -4.393444, + "rank": 1, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 4, + "score": -10.691864, + "rank": 2, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 3, + "score": -10.822473, + "rank": 3, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 2, + "score": -10.996204, + "rank": 4, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 4, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 42, + "first_stage_top1_distance": 0.41856718, + "rerank_top1_raw_score": -4.393444, + "rerank_top2_raw_score": -10.691864, + "rerank_margin_raw": 6.29842, + "support_top1_score": 0.34955144, + "candidate_memory_ids": [ + 5, + 4, + 3 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 5 + ], + "gold_answer": "the daemon stopped swallowing llm stream errors and forwarded them to the tui", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.41856718, + "rerank_top1_raw_score": -4.393444, + "rerank_top2_raw_score": -10.691864, + "rerank_margin_raw": 6.29842, + "support_top1_score": 0.34955144, + "final_decision": "answer", + "final_answer_memory_ids": [ + 5 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.41856718, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 42, + "passed": true + }, + { + "query_id": "dev_q11", + "query_text": "what benchmark-related work has been happening?", + "query_category": "vague recent lookup", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 10, + "score": 0.37298805, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 8, + "score": 0.38526428, + "rank": 2, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": 0.41374874, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": 0.47473764, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 19, + "score": 0.53960985, + "rank": 5, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 18, + "score": 0.54336476, + "rank": 6, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 22, + "score": 0.5818255, + "rank": 7, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 9, + "score": 0.6212563, + "rank": 8, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 5, + "score": 0.6339031, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 6, + "score": 0.6578633, + "rank": 10, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 7, + "score": 0.6973202, + "rank": 11, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + } + ], + "reranked_candidates": [ + { + "memory_id": 10, + "score": 2.105263, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 8, + "score": -4.1376686, + "rank": 2, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": -4.737645, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": -5.1203513, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 18, + "score": -5.2799463, + "rank": 5, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 9, + "score": -6.087734, + "rank": 6, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 22, + "score": -8.901651, + "rank": 7, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": -9.864755, + "rank": 8, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 5, + "score": -10.931221, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 6, + "score": -10.9638815, + "rank": 10, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 91, + "first_stage_top1_distance": 0.37298805, + "rerank_top1_raw_score": 2.105263, + "rerank_top2_raw_score": -4.1376686, + "rerank_margin_raw": 6.2429314, + "support_top1_score": 0.3629735, + "candidate_memory_ids": [ + 10, + 8, + 17 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 8, + 9, + 10, + 17, + 18, + 22 + ], + "gold_answer": "benchmark logging, timeline-aware eval data, retrieval plumbing, and replacing the toy eval set", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.37298805, + "rerank_top1_raw_score": 2.105263, + "rerank_top2_raw_score": -4.1376686, + "rerank_margin_raw": 6.2429314, + "support_top1_score": 0.3629735, + "final_decision": "answer", + "final_answer_memory_ids": [ + 10 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.37298805, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 91, + "passed": true + }, + { + "query_id": "dev_q12", + "query_text": "what setup is protecting us from embedding/model mismatch?", + "query_category": "multi-evidence query", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 19, + "score": 0.42957628, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 7, + "score": 0.4932534, + "rank": 2, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 22, + "score": 0.54476416, + "rank": 3, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 23, + "score": 0.54997754, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 8, + "score": 0.5566095, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": 0.5719408, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 10, + "score": 0.5726047, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 5, + "score": 0.57881737, + "rank": 8, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 18, + "score": 0.5918925, + "rank": 9, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 9, + "score": 0.62952846, + "rank": 10, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 6, + "score": 0.64006686, + "rank": 11, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + } + ], + "reranked_candidates": [ + { + "memory_id": 19, + "score": 0.38869774, + "rank": 1, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 7, + "score": -6.2859, + "rank": 2, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 8, + "score": -8.483065, + "rank": 3, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 23, + "score": -9.451513, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 17, + "score": -9.7562065, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 22, + "score": -10.022348, + "rank": 6, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 9, + "score": -10.199078, + "rank": 7, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 18, + "score": -10.6672325, + "rank": 8, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 10, + "score": -10.696995, + "rank": 9, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 5, + "score": -10.977566, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 93, + "first_stage_top1_distance": 0.42957628, + "rerank_top1_raw_score": 0.38869774, + "rerank_top2_raw_score": -6.2859, + "rerank_margin_raw": 6.6745977, + "support_top1_score": 0.49863112, + "candidate_memory_ids": [ + 19, + 7, + 8 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 3, + 7, + 19 + ], + "gold_answer": "keeping embeddings separate on bge-m3 and separating services by role", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.42957628, + "rerank_top1_raw_score": 0.38869774, + "rerank_top2_raw_score": -6.2859, + "rerank_margin_raw": 6.6745977, + "support_top1_score": 0.49863112, + "final_decision": "answer", + "final_answer_memory_ids": [ + 19 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.42957628, + "embedding_latency_ms": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 93, + "passed": true + }, + { + "query_id": "test_q1", + "query_text": "what changed between the live agent and the benchmark harness?", + "query_category": "exact recent event", + "split": "dev", + "namespace": "default", + "reference_time": 1776781800, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 23, + "score": 0.26425248, + "rank": 1, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 17, + "score": 0.39332432, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 8, + "score": 0.4232306, + "rank": 3, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 18, + "score": 0.61687624, + "rank": 4, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 10, + "score": 0.6319481, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 7, + "score": 0.64805615, + "rank": 6, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 22, + "score": 0.65098643, + "rank": 7, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": 0.665923, + "rank": 8, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 9, + "score": 0.6781608, + "rank": 9, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 5, + "score": 0.6981989, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 6, + "score": 0.7747772, + "rank": 11, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + } + ], + "reranked_candidates": [ + { + "memory_id": 23, + "score": 2.6212046, + "rank": 1, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 8, + "score": -2.1147661, + "rank": 2, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": -4.2227964, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 18, + "score": -8.234255, + "rank": 4, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 9, + "score": -9.933149, + "rank": 5, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 10, + "score": -10.013235, + "rank": 6, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 7, + "score": -10.516793, + "rank": 7, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 5, + "score": -10.843144, + "rank": 8, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 19, + "score": -10.903561, + "rank": 9, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 22, + "score": -10.977879, + "rank": 10, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 89, + "first_stage_top1_distance": 0.26425248, + "rerank_top1_raw_score": 2.6212046, + "rerank_top2_raw_score": -2.1147661, + "rerank_margin_raw": 4.7359705, + "support_top1_score": 0.56176066, + "candidate_memory_ids": [ + 23, + 8, + 17 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 23 + ], + "gold_answer": "they now share the same time-windowed exact retrieval path", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.26425248, + "rerank_top1_raw_score": 2.6212046, + "rerank_top2_raw_score": -2.1147661, + "rerank_margin_raw": 4.7359705, + "support_top1_score": 0.56176066, + "final_decision": "answer", + "final_answer_memory_ids": [ + 23 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.26425248, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 89, + "passed": true + }, + { + "query_id": "test_q2", + "query_text": "was i still using neovim after the editor switch?", + "query_category": "conflict / update question", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 30, + "windows_considered": [ + 7, + 30 + ], + "memories_searched": 14, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 2, + "score": 0.34071916, + "rank": 1, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 7, + "score": 0.5698057, + "rank": 2, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 17, + "score": 0.59015, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 5, + "score": 0.5904065, + "rank": 4, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 10, + "score": 0.6320615, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 4, + "score": 0.6443518, + "rank": 6, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 9, + "score": 0.64470935, + "rank": 7, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 3, + "score": 0.65202284, + "rank": 8, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 22, + "score": 0.6580648, + "rank": 9, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": 0.66409516, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 23, + "score": 0.6656461, + "rank": 11, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 18, + "score": 0.6828693, + "rank": 12, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 6, + "score": 0.70850563, + "rank": 13, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 8, + "score": 0.72874826, + "rank": 14, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + } + ], + "reranked_candidates": [ + { + "memory_id": 2, + "score": -0.8818027, + "rank": 1, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 7, + "score": -10.240337, + "rank": 2, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 17, + "score": -10.998789, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 3, + "score": -11.010808, + "rank": 4, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 5, + "score": -11.031289, + "rank": 5, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 9, + "score": -11.031513, + "rank": 6, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 19, + "score": -11.032371, + "rank": 7, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 10, + "score": -11.037233, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 4, + "score": -11.039769, + "rank": 9, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 22, + "score": -11.041342, + "rank": 10, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 91, + "first_stage_top1_distance": 0.5698057, + "rerank_top1_raw_score": -10.241425, + "rerank_top2_raw_score": -10.677954, + "rerank_margin_raw": 0.43652916, + "support_top1_score": 0.012474045, + "candidate_memory_ids": [ + 7, + 23, + 17 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 78, + "first_stage_top1_distance": 0.34071916, + "rerank_top1_raw_score": -0.8818027, + "rerank_top2_raw_score": -10.240337, + "rerank_margin_raw": 9.358535, + "support_top1_score": 0.5577389, + "candidate_memory_ids": [ + 2, + 7, + 17 + ], + "decision": "answer" + } + ], + "window_expanded": true, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 2 + ], + "gold_answer": "no, the newer fact says the editor switched to zed", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.34071916, + "rerank_top1_raw_score": -0.8818027, + "rerank_top2_raw_score": -10.240337, + "rerank_margin_raw": 9.358535, + "support_top1_score": 0.5577389, + "final_decision": "answer", + "final_answer_memory_ids": [ + 2 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": false, + "best_distance": 0.34071916, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 169, + "passed": true + }, + { + "query_id": "test_q7", + "query_text": "why do we need better benchmark data now?", + "query_category": "multi-evidence query", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 20, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 8, + "score": 0.42387754, + "rank": 1, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": 0.43914515, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 4, + "score": 0.4565724, + "rank": 3, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 23, + "score": 0.47153515, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 22, + "score": 0.48126787, + "rank": 5, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 26, + "score": 0.5146295, + "rank": 6, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 10, + "score": 0.5250063, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 18, + "score": 0.53004897, + "rank": 8, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 30, + "score": 0.5403733, + "rank": 9, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m2", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs." + }, + { + "memory_id": 19, + "score": 0.5470239, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 9, + "score": 0.561113, + "rank": 11, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 27, + "score": 0.5709995, + "rank": 12, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 28, + "score": 0.5863872, + "rank": 13, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 31, + "score": 0.5899797, + "rank": 14, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 29, + "score": 0.59785146, + "rank": 15, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 2, + "score": 0.59937155, + "rank": 16, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 5, + "score": 0.63316834, + "rank": 17, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 3, + "score": 0.635115, + "rank": 18, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 7, + "score": 0.63733864, + "rank": 19, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 6, + "score": 0.6735958, + "rank": 20, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + } + ], + "reranked_candidates": [ + { + "memory_id": 4, + "score": -7.263253, + "rank": 1, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 17, + "score": -7.3808303, + "rank": 2, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 22, + "score": -7.456655, + "rank": 3, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 18, + "score": -8.096454, + "rank": 4, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 8, + "score": -8.110693, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 23, + "score": -8.218077, + "rank": 6, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 10, + "score": -8.80881, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 30, + "score": -9.694189, + "rank": 8, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m2", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs." + }, + { + "memory_id": 19, + "score": -10.8937235, + "rank": 9, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 26, + "score": -11.0396, + "rank": 10, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 92, + "first_stage_top1_distance": 0.42387754, + "rerank_top1_raw_score": -7.3808317, + "rerank_top2_raw_score": -7.4622507, + "rerank_margin_raw": 0.08141899, + "support_top1_score": 0.14813097, + "candidate_memory_ids": [ + 17, + 22, + 9 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 77, + "first_stage_top1_distance": 0.42387754, + "rerank_top1_raw_score": -7.263253, + "rerank_top2_raw_score": -7.3706455, + "rerank_margin_raw": 0.10739231, + "support_top1_score": 0.14813095, + "candidate_memory_ids": [ + 4, + 17, + 22 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 77, + "first_stage_top1_distance": 0.42387754, + "rerank_top1_raw_score": -7.263253, + "rerank_top2_raw_score": -7.3808303, + "rerank_margin_raw": 0.117577076, + "support_top1_score": 0.14813095, + "candidate_memory_ids": [ + 4, + 17, + 22 + ], + "decision": "expand" + }, + { + "memories_searched": 20, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 93, + "first_stage_top1_distance": 0.42387754, + "rerank_top1_raw_score": -7.263253, + "rerank_top2_raw_score": -7.3808303, + "rerank_margin_raw": 0.117577076, + "support_top1_score": 0.14813095, + "candidate_memory_ids": [ + 4, + 17, + 22 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "abstained": true, + "no_hit": false, + "gold_memory_ids": [ + 18, + 22 + ], + "gold_answer": "because the toy benchmark only proved plumbing and the eval set needs harder no-hit and ambiguity cases", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.42387754, + "rerank_top1_raw_score": -7.263253, + "rerank_top2_raw_score": -7.3808303, + "rerank_margin_raw": 0.117577076, + "support_top1_score": 0.14813095, + "final_decision": "abstain", + "final_correct": false, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.42387754, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 339, + "passed": false + }, + { + "query_id": "test_q8", + "query_text": "what sort of retrieval work keeps coming up?", + "query_category": "recurring theme / pattern-like question", + "split": "dev", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 10, + "score": 0.3832404, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 23, + "score": 0.49989516, + "rank": 2, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 19, + "score": 0.50503004, + "rank": 3, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 8, + "score": 0.5261926, + "rank": 4, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 17, + "score": 0.5573443, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 6, + "score": 0.593746, + "rank": 6, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 22, + "score": 0.599839, + "rank": 7, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 9, + "score": 0.6342809, + "rank": 8, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 5, + "score": 0.6345304, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 18, + "score": 0.65942, + "rank": 10, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 7, + "score": 0.66904444, + "rank": 11, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + } + ], + "reranked_candidates": [ + { + "memory_id": 10, + "score": 1.4382501, + "rank": 1, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 19, + "score": -4.696662, + "rank": 2, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 23, + "score": -5.2201343, + "rank": 3, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 22, + "score": -8.579902, + "rank": 4, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 8, + "score": -9.168009, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 9, + "score": -9.771562, + "rank": 6, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 17, + "score": -10.060578, + "rank": 7, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 6, + "score": -10.136121, + "rank": 8, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 5, + "score": -10.95099, + "rank": 9, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 18, + "score": -10.971389, + "rank": 10, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 90, + "first_stage_top1_distance": 0.3832404, + "rerank_top1_raw_score": 1.4382501, + "rerank_top2_raw_score": -4.696662, + "rerank_margin_raw": 6.134912, + "support_top1_score": 0.39258915, + "candidate_memory_ids": [ + 10, + 19, + 23 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 10, + 19, + 23 + ], + "gold_answer": "retrieval plumbing, service separation, and aligning the live and benchmark retrieval path", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.3832404, + "rerank_top1_raw_score": 1.4382501, + "rerank_top2_raw_score": -4.696662, + "rerank_margin_raw": 6.134912, + "support_top1_score": 0.39258915, + "final_decision": "answer", + "final_answer_memory_ids": [ + 10 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.3832404, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 90, + "passed": true + }, + { + "query_id": "test_q10", + "query_text": "why did i add progress logging to the benchmark harness?", + "query_category": "exact recent event", + "split": "dev", + "namespace": "default", + "reference_time": 1776614400, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 10, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 17, + "score": 0.14865494, + "rank": 1, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 8, + "score": 0.25858313, + "rank": 2, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 10, + "score": 0.46180826, + "rank": 3, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 4, + "score": 0.4906999, + "rank": 4, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 2, + "score": 0.5378274, + "rank": 5, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 5, + "score": 0.54671407, + "rank": 6, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + }, + { + "memory_id": 9, + "score": 0.55227816, + "rank": 7, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 7, + "score": 0.5665418, + "rank": 8, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 6, + "score": 0.5901372, + "rank": 9, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 3, + "score": 0.59295964, + "rank": 10, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + } + ], + "reranked_candidates": [ + { + "memory_id": 17, + "score": 7.005316, + "rank": 1, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 8, + "score": -1.1870534, + "rank": 2, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 4, + "score": -7.887468, + "rank": 3, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 9, + "score": -8.938036, + "rank": 4, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 10, + "score": -9.398699, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 7, + "score": -10.774755, + "rank": 6, + "event_time": 1776254400, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:llama-server" + ], + "source_ref": "session:t2:m3", + "text": "the chat llama-server did not expose embeddings, so embeddings had to stay on a separate process." + }, + { + "memory_id": 6, + "score": -10.883025, + "rank": 7, + "event_time": 1776252600, + "tags": [ + "project:klbr", + "topic:memory" + ], + "source_ref": "session:t2:m2", + "text": "compaction summaries are stored with the compaction_summary tag." + }, + { + "memory_id": 2, + "score": -11.000705, + "rank": 8, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + }, + { + "memory_id": 3, + "score": -11.017478, + "rank": 9, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 5, + "score": -11.033957, + "rank": 10, + "event_time": 1776250800, + "tags": [ + "project:klbr", + "topic:daemon", + "topic:tui" + ], + "source_ref": "session:t2:m1", + "text": "fixed the daemon swallowing llm stream errors and forwarded the error to the tui." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 87, + "first_stage_top1_distance": 0.14865494, + "rerank_top1_raw_score": 7.005316, + "rerank_top2_raw_score": -1.1870534, + "rerank_margin_raw": 8.192369, + "support_top1_score": 0.8187852, + "candidate_memory_ids": [ + 17, + 8, + 4 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 17 + ], + "gold_answer": "because silent hangs were confusing", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.14865494, + "rerank_top1_raw_score": 7.005316, + "rerank_top2_raw_score": -1.1870534, + "rerank_margin_raw": 8.192369, + "support_top1_score": 0.8187852, + "final_decision": "answer", + "final_answer_memory_ids": [ + 17 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.14865494, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 87, + "passed": true + }, + { + "query_id": "test_q12", + "query_text": "what happened around the embedding split?", + "query_category": "temporally ambiguous query", + "split": "dev", + "namespace": "default", + "reference_time": 1776160800, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 3, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 3, + "score": 0.51676875, + "rank": 1, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 4, + "score": 0.626891, + "rank": 2, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 2, + "score": 0.66463304, + "rank": 3, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + } + ], + "reranked_candidates": [ + { + "memory_id": 3, + "score": -1.3334079, + "rank": 1, + "event_time": 1776159000, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t1:m2", + "text": "split embeddings onto a separate llama-server at localhost 8002 using bge-m3 with dimension 1024." + }, + { + "memory_id": 4, + "score": -10.90943, + "rank": 2, + "event_time": 1776160800, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t1:m3", + "text": "the first benchmark slice should stay exact retrieval first before indexed search." + }, + { + "memory_id": 2, + "score": -10.98511, + "rank": 3, + "event_time": 1776157200, + "tags": [ + "project:klbr", + "topic:editor" + ], + "source_ref": "session:t1:m1", + "text": "switched the day-to-day klbr editor from neovim to zed." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 3, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 29, + "first_stage_top1_distance": 0.51676875, + "rerank_top1_raw_score": -1.3334079, + "rerank_top2_raw_score": -10.90943, + "rerank_margin_raw": 9.576021, + "support_top1_score": 0.40805614, + "candidate_memory_ids": [ + 3, + 4, + 2 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 2, + 3, + 4 + ], + "gold_answer": "the editor switched to zed, embeddings moved to a separate bge-m3 server, and exact retrieval stayed the first benchmark step", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.51676875, + "rerank_top1_raw_score": -1.3334079, + "rerank_top2_raw_score": -10.90943, + "rerank_margin_raw": 9.576021, + "support_top1_score": 0.40805614, + "final_decision": "answer", + "final_answer_memory_ids": [ + 3 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.51676875, + "embedding_latency_ms": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 29, + "passed": true + }, + { + "query_id": "dev_q17", + "query_text": "what support threshold looked best in the focused support sweep?", + "query_category": "exact recent event", + "split": "dev", + "namespace": "default", + "reference_time": 1776862800, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 10, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 27, + "score": 0.3703106, + "rank": 1, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 26, + "score": 0.5021392, + "rank": 2, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 17, + "score": 0.53573525, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": 0.5799917, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 22, + "score": 0.5998579, + "rank": 5, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 10, + "score": 0.6114553, + "rank": 6, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 8, + "score": 0.6214824, + "rank": 7, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 18, + "score": 0.62489086, + "rank": 8, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 9, + "score": 0.6513971, + "rank": 9, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 19, + "score": 0.7139363, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "reranked_candidates": [ + { + "memory_id": 27, + "score": 2.793685, + "rank": 1, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 26, + "score": -8.126055, + "rank": 2, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 18, + "score": -10.078178, + "rank": 3, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 22, + "score": -10.515538, + "rank": 4, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 8, + "score": -10.576755, + "rank": 5, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 23, + "score": -10.612513, + "rank": 6, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 17, + "score": -10.686229, + "rank": 7, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 10, + "score": -10.772123, + "rank": 8, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 9, + "score": -10.790661, + "rank": 9, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 19, + "score": -11.029169, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 99, + "first_stage_top1_distance": 0.3703106, + "rerank_top1_raw_score": 2.793685, + "rerank_top2_raw_score": -8.126055, + "rerank_margin_raw": 10.91974, + "support_top1_score": 0.39791203, + "candidate_memory_ids": [ + 27, + 26, + 18 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 27 + ], + "gold_answer": "0.2", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.3703106, + "rerank_top1_raw_score": 2.793685, + "rerank_top2_raw_score": -8.126055, + "rerank_margin_raw": 10.91974, + "support_top1_score": 0.39791203, + "final_decision": "answer", + "final_answer_memory_ids": [ + 27 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.3703106, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 99, + "passed": true + }, + { + "query_id": "dev_q18", + "query_text": "what changed after adding the support score to the policy?", + "query_category": "multi-evidence query", + "split": "dev", + "namespace": "default", + "reference_time": 1776864600, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 26, + "score": 0.41911483, + "rank": 1, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 28, + "score": 0.42588216, + "rank": 2, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 27, + "score": 0.4687217, + "rank": 3, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 17, + "score": 0.48836213, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": 0.54041183, + "rank": 5, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 9, + "score": 0.5797576, + "rank": 6, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 8, + "score": 0.59324396, + "rank": 7, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 22, + "score": 0.60716975, + "rank": 8, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": 0.620885, + "rank": 9, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 18, + "score": 0.63540435, + "rank": 10, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 10, + "score": 0.6554829, + "rank": 11, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + } + ], + "reranked_candidates": [ + { + "memory_id": 26, + "score": -3.6367404, + "rank": 1, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 27, + "score": -5.305154, + "rank": 2, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 28, + "score": -6.8829327, + "rank": 3, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 17, + "score": -8.734776, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": -9.661739, + "rank": 5, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 18, + "score": -10.406639, + "rank": 6, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 9, + "score": -10.487062, + "rank": 7, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 8, + "score": -10.695279, + "rank": 8, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 22, + "score": -10.705133, + "rank": 9, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": -11.020961, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 113, + "first_stage_top1_distance": 0.41911483, + "rerank_top1_raw_score": -3.6367404, + "rerank_top2_raw_score": -5.305154, + "rerank_margin_raw": 1.6684134, + "support_top1_score": 0.28243408, + "candidate_memory_ids": [ + 26, + 27, + 28 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 26, + 27, + 28 + ], + "gold_answer": "the policy added weighted lexical support, settled on score -6.0 / margin 0.0 / support 0.2, and removed no-hit false answers with some coverage loss", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.41911483, + "rerank_top1_raw_score": -3.6367404, + "rerank_top2_raw_score": -5.305154, + "rerank_margin_raw": 1.6684134, + "support_top1_score": 0.28243408, + "final_decision": "answer", + "final_answer_memory_ids": [ + 26 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.41911483, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 113, + "passed": true + }, + { + "query_id": "dev_q19", + "query_text": "what kind of calibration fix are we avoiding because it would be brittle?", + "query_category": "recurring theme / pattern-like question", + "split": "dev", + "namespace": "default", + "reference_time": 1776870000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 14, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 29, + "score": 0.28061455, + "rank": 1, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 18, + "score": 0.5024791, + "rank": 2, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 28, + "score": 0.56384, + "rank": 3, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 9, + "score": 0.569462, + "rank": 4, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 10, + "score": 0.5745239, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 17, + "score": 0.5760915, + "rank": 6, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 31, + "score": 0.5941314, + "rank": 7, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 22, + "score": 0.5985524, + "rank": 8, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 27, + "score": 0.6017505, + "rank": 9, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 8, + "score": 0.6182196, + "rank": 10, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 19, + "score": 0.62894464, + "rank": 11, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 30, + "score": 0.6588505, + "rank": 12, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m2", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs." + }, + { + "memory_id": 23, + "score": 0.674569, + "rank": 13, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 26, + "score": 0.69236374, + "rank": 14, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + } + ], + "reranked_candidates": [ + { + "memory_id": 29, + "score": 0.7003631, + "rank": 1, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 18, + "score": -6.277218, + "rank": 2, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 27, + "score": -7.1544867, + "rank": 3, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 28, + "score": -7.843611, + "rank": 4, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 22, + "score": -9.269146, + "rank": 5, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 10, + "score": -9.463024, + "rank": 6, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 9, + "score": -9.509208, + "rank": 7, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 17, + "score": -9.528537, + "rank": 8, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 31, + "score": -9.604826, + "rank": 9, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 8, + "score": -10.432985, + "rank": 10, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 110, + "first_stage_top1_distance": 0.28061455, + "rerank_top1_raw_score": 0.7003631, + "rerank_top2_raw_score": -6.277218, + "rerank_margin_raw": 6.977581, + "support_top1_score": 0.3882105, + "candidate_memory_ids": [ + 29, + 18, + 27 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 29 + ], + "gold_answer": "many query-type guards", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.28061455, + "rerank_top1_raw_score": 0.7003631, + "rerank_top2_raw_score": -6.277218, + "rerank_margin_raw": 6.977581, + "support_top1_score": 0.3882105, + "final_decision": "answer", + "final_answer_memory_ids": [ + 29 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.28061455, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 110, + "passed": true + }, + { + "query_id": "dev_q20", + "query_text": "how should passive recall connect to the current calibration work?", + "query_category": "multi-evidence query", + "split": "dev", + "namespace": "default", + "reference_time": 1776870000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 14, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 31, + "score": 0.38377964, + "rank": 1, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 29, + "score": 0.52532125, + "rank": 2, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 17, + "score": 0.54792285, + "rank": 3, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 8, + "score": 0.55544543, + "rank": 4, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 10, + "score": 0.55680794, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 23, + "score": 0.55827475, + "rank": 6, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 19, + "score": 0.5659659, + "rank": 7, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 18, + "score": 0.56961286, + "rank": 8, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 26, + "score": 0.58618677, + "rank": 9, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 27, + "score": 0.5889935, + "rank": 10, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 9, + "score": 0.6007079, + "rank": 11, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + }, + { + "memory_id": 22, + "score": 0.60441625, + "rank": 12, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 28, + "score": 0.6118729, + "rank": 13, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 30, + "score": 0.62033296, + "rank": 14, + "event_time": 1776868200, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m2", + "text": "the better direction is generic evidence support that works for both direct questions and non-question inputs." + } + ], + "reranked_candidates": [ + { + "memory_id": 31, + "score": -1.0000097, + "rank": 1, + "event_time": 1776870000, + "tags": [ + "project:klbr", + "topic:support", + "topic:passive-recall" + ], + "source_ref": "session:t11:m3", + "text": "passive recall should reuse the same support features instead of introducing a separate query taxonomy." + }, + { + "memory_id": 26, + "score": -7.5246086, + "rank": 2, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 27, + "score": -7.838156, + "rank": 3, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 18, + "score": -8.340904, + "rank": 4, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 10, + "score": -8.427327, + "rank": 5, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 29, + "score": -8.647226, + "rank": 6, + "event_time": 1776866400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:query-types" + ], + "source_ref": "session:t11:m1", + "text": "decided against adding many brittle query-type guards for calibration." + }, + { + "memory_id": 17, + "score": -9.080494, + "rank": 7, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": -9.381987, + "rank": 8, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 8, + "score": -9.646032, + "rank": 9, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 19, + "score": -10.488276, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 14, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 94, + "first_stage_top1_distance": 0.38377964, + "rerank_top1_raw_score": -1.0000097, + "rerank_top2_raw_score": -7.5246086, + "rerank_margin_raw": 6.524599, + "support_top1_score": 0.3470262, + "candidate_memory_ids": [ + 31, + 26, + 27 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 30, + 31 + ], + "gold_answer": "it should reuse the same generic support features instead of a separate taxonomy", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.38377964, + "rerank_top1_raw_score": -1.0000097, + "rerank_top2_raw_score": -7.5246086, + "rerank_margin_raw": 6.524599, + "support_top1_score": 0.3470262, + "final_decision": "answer", + "final_answer_memory_ids": [ + 31 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.38377964, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 94, + "passed": true + }, + { + "query_id": "dev_q22", + "query_text": "did the support-assisted sweep get coverage all the way back to the old 0.9167 balanced point?", + "query_category": "conflict / update question", + "split": "dev", + "namespace": "default", + "reference_time": 1776864600, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 28, + "score": 0.34206462, + "rank": 1, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 27, + "score": 0.40596223, + "rank": 2, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 26, + "score": 0.52247, + "rank": 3, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 23, + "score": 0.5609309, + "rank": 4, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 17, + "score": 0.5828568, + "rank": 5, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 10, + "score": 0.61101043, + "rank": 6, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 18, + "score": 0.6160656, + "rank": 7, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 19, + "score": 0.6673319, + "rank": 8, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + }, + { + "memory_id": 22, + "score": 0.6845633, + "rank": 9, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 8, + "score": 0.6965473, + "rank": 10, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 9, + "score": 0.7135205, + "rank": 11, + "event_time": 1776353400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:eval-data" + ], + "source_ref": "session:t3:m2", + "text": "internal eval should split by timeline or session, not by individual question only." + } + ], + "reranked_candidates": [ + { + "memory_id": 28, + "score": 4.413192, + "rank": 1, + "event_time": 1776864600, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m3", + "text": "the support-assisted operating point removed no-hit false answers on the dev slice, but answerable coverage settled at 0.8333 instead of the old 0.9167 balanced point." + }, + { + "memory_id": 27, + "score": -2.2993803, + "rank": 2, + "event_time": 1776862800, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m2", + "text": "a focused support sweep found a useful operating point at rerank score -6.0, margin 0.0, and support 0.2." + }, + { + "memory_id": 18, + "score": -8.5867195, + "rank": 3, + "event_time": 1776616200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:calibration" + ], + "source_ref": "session:t6:m2", + "text": "the first tiny benchmark only proved the plumbing and was too small for calibration." + }, + { + "memory_id": 17, + "score": -8.9998865, + "rank": 4, + "event_time": 1776614400, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:logging" + ], + "source_ref": "session:t6:m1", + "text": "added progress logging to the benchmark harness because silent hangs were confusing." + }, + { + "memory_id": 23, + "score": -9.296768, + "rank": 5, + "event_time": 1776781800, + "tags": [ + "project:klbr", + "topic:retrieval" + ], + "source_ref": "session:t8:m2", + "text": "the live agent and the benchmark harness now share the same time-windowed exact retrieval path." + }, + { + "memory_id": 26, + "score": -9.785875, + "rank": 6, + "event_time": 1776861000, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:support" + ], + "source_ref": "session:t10:m1", + "text": "added a generic lexical support score based on weighted overlap between salient query terms and the top memory text." + }, + { + "memory_id": 10, + "score": -9.931657, + "rank": 7, + "event_time": 1776355200, + "tags": [ + "project:klbr", + "topic:benchmarking", + "topic:retrieval" + ], + "source_ref": "session:t3:m3", + "text": "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + }, + { + "memory_id": 8, + "score": -10.574107, + "rank": 8, + "event_time": 1776351600, + "tags": [ + "project:klbr", + "topic:benchmarking" + ], + "source_ref": "session:t3:m1", + "text": "the benchmark harness should log per-query candidate lists and stage latencies." + }, + { + "memory_id": 22, + "score": -10.874885, + "rank": 9, + "event_time": 1776780000, + "tags": [ + "project:klbr", + "topic:eval-data" + ], + "source_ref": "session:t8:m1", + "text": "a better internal eval set should include no-hit, temporal ambiguity, and multi-evidence cases." + }, + { + "memory_id": 19, + "score": -10.990305, + "rank": 10, + "event_time": 1776618000, + "tags": [ + "project:klbr", + "topic:infra", + "topic:retrieval" + ], + "source_ref": "session:t6:m3", + "text": "a recurring infra pattern is to separate chat, embedding, and rerank services by role to avoid model mismatch." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 104, + "first_stage_top1_distance": 0.34206462, + "rerank_top1_raw_score": 4.413192, + "rerank_top2_raw_score": -2.2993803, + "rerank_margin_raw": 6.712572, + "support_top1_score": 0.7451097, + "candidate_memory_ids": [ + 28, + 27, + 18 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 28 + ], + "gold_answer": "no, coverage settled at 0.8333", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.34206462, + "rerank_top1_raw_score": 4.413192, + "rerank_top2_raw_score": -2.2993803, + "rerank_margin_raw": 6.712572, + "support_top1_score": 0.7451097, + "final_decision": "answer", + "final_answer_memory_ids": [ + 28 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.34206462, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 104, + "passed": true + } +] \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.json new file mode 100644 index 0000000..2560fb0 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.json @@ -0,0 +1,61 @@ +{ + "experiment": { + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Same operating point as mvp_rerank_support_calibrated.json but evaluated on the test split.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 + }, + "dataset_id": "internal-eval-starter-v2", + "evaluated_split": "test", + "ingest_count": 34, + "ingest_latency_ms": 701, + "metrics": { + "query_count": 16, + "recall_at_1": 0.85714287, + "recall_at_5": 1.0, + "recall_at_10": 1.0, + "recall_at_20": 1.0, + "mrr": 0.9071428, + "ndcg_at_10": 0.8870347, + "top1_accuracy_after_rerank": 1.0, + "final_top1_accuracy": 0.9285714, + "mrr_after_rerank": 1.0, + "ndcg_after_rerank": 0.96067756, + "reranker_lift_mrr": 0.09285718, + "candidate_set_size_mean": 10.125, + "time_window_hit_rate": 1.0, + "final_decision_accuracy": 0.9375, + "coverage": 0.8125, + "abstain_accuracy": 1.0, + "selective_accuracy": 1.0, + "retrieval_latency_p50_ms": 0, + "retrieval_latency_p95_ms": 0, + "embedding_latency_p50_ms": 10, + "embedding_latency_p95_ms": 12, + "rerank_latency_p50_ms": 82, + "rerank_latency_p95_ms": 295 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.md b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.md new file mode 100644 index 0000000..fddb369 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.md @@ -0,0 +1,36 @@ +# Retrieval Benchmark Report + +dataset: `internal-eval-starter-v2` +experiment: `mvp-rerank-support-calibrated` +split: `test` +queries: `16` +memories ingested: `34` +ingest latency ms: `701` +window expansions: `3` + +## Metrics + +- Recall@1: `0.8571` +- Recall@5: `1.0000` +- Recall@10: `1.0000` +- Recall@20: `1.0000` +- MRR: `0.9071` +- nDCG@10: `0.8870` +- Top1 accuracy after rerank: `1.0000` +- Final top1 accuracy: `0.9286` +- MRR after rerank: `1.0000` +- nDCG after rerank: `0.9607` +- Reranker lift (MRR): `0.0929` +- Candidate set size mean: `10.12` +- Time-window hit rate: `1.0000` +- Final decision accuracy: `0.9375` +- Coverage: `0.8125` +- Abstain accuracy: `1.0000` +- Selective accuracy: `1.0000` +- Retrieval latency p50/p95 ms: `0` / `0` +- Embedding latency p50/p95 ms: `10` / `12` +- Rerank latency p50/p95 ms: `82` / `295` + +## No-Hit False Answers + +(none) diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/resolved_config.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/resolved_config.json new file mode 100644 index 0000000..73f58cf --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/resolved_config.json @@ -0,0 +1,29 @@ +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Same operating point as mvp_rerank_support_calibrated.json but evaluated on the test split.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [ + 30, + 90, + null + ], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/traces.json b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/traces.json new file mode 100644 index 0000000..8d2fbb4 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/traces.json @@ -0,0 +1,4969 @@ +[ + { + "query_id": "dev_q1", + "query_text": "what is the reranker endpoint right now?", + "query_category": "exact recent event", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 9, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 24, + "score": 0.21824461, + "rank": 1, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 11, + "score": 0.29826105, + "rank": 2, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": 0.3787951, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": 0.4198541, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": 0.4217924, + "rank": 5, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": 0.53594387, + "rank": 6, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 16, + "score": 0.6315558, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": 0.6416321, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 15, + "score": 0.67355084, + "rank": 9, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + } + ], + "reranked_candidates": [ + { + "memory_id": 24, + "score": 5.207789, + "rank": 1, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 20, + "score": 1.6954439, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 11, + "score": 0.3173951, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 13, + "score": -2.5927365, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": -3.3189714, + "rank": 5, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": -5.4959307, + "rank": 6, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 16, + "score": -10.206045, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": -10.826174, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 15, + "score": -10.890806, + "rank": 9, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 135, + "first_stage_top1_distance": 0.21824461, + "rerank_top1_raw_score": 5.207789, + "rerank_top2_raw_score": 1.6954439, + "rerank_margin_raw": 3.512345, + "support_top1_score": 0.67911154, + "candidate_memory_ids": [ + 24, + 20, + 11 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 24 + ], + "gold_answer": "http://localhost:8003", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.21824461, + "rerank_top1_raw_score": 5.207789, + "rerank_top2_raw_score": 1.6954439, + "rerank_margin_raw": 3.512345, + "support_top1_score": 0.67911154, + "final_decision": "answer", + "final_answer_memory_ids": [ + 24 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.21824461, + "embedding_latency_ms": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 135, + "passed": true + }, + { + "query_id": "dev_q2", + "query_text": "what did i switch to on april 18 when i moved off lm studio?", + "query_category": "exact dated event", + "split": "test", + "namespace": "default", + "reference_time": 1776517200, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 4, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 14, + "score": 0.3938769, + "rank": 1, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 11, + "score": 0.5782161, + "rank": 2, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": 0.62000763, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 13, + "score": 0.6577877, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 14, + "score": -2.99831, + "rank": 1, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 13, + "score": -11.022743, + "rank": 2, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 11, + "score": -11.025039, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": -11.030579, + "rank": 4, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 4, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 32, + "first_stage_top1_distance": 0.3938769, + "rerank_top1_raw_score": -2.99831, + "rerank_top2_raw_score": -11.022743, + "rerank_margin_raw": 8.024433, + "support_top1_score": 0.37195718, + "candidate_memory_ids": [ + 14, + 13, + 11 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 14 + ], + "gold_answer": "llama-server", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.3938769, + "rerank_top1_raw_score": -2.99831, + "rerank_top2_raw_score": -11.022743, + "rerank_margin_raw": 8.024433, + "support_top1_score": 0.37195718, + "final_decision": "answer", + "final_answer_memory_ids": [ + 14 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.3938769, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 32, + "passed": true + }, + { + "query_id": "dev_q6", + "query_text": "is the reranker endpoint using https now?", + "query_category": "conflict / update question", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 9, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 24, + "score": 0.23228288, + "rank": 1, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 12, + "score": 0.2622108, + "rank": 2, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 11, + "score": 0.30660123, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": 0.31133288, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": 0.46102017, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 21, + "score": 0.47909486, + "rank": 6, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 16, + "score": 0.5774287, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": 0.6148598, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 15, + "score": 0.6384365, + "rank": 9, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + } + ], + "reranked_candidates": [ + { + "memory_id": 24, + "score": 4.844589, + "rank": 1, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 20, + "score": 4.4004254, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": 3.5361626, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 11, + "score": 0.16857512, + "rank": 4, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 13, + "score": -5.5653553, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 21, + "score": -6.473775, + "rank": 6, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 16, + "score": -9.360283, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 15, + "score": -10.574578, + "rank": 8, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": -10.721799, + "rank": 9, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "first_stage_top1_distance": 0.23228288, + "rerank_top1_raw_score": 4.844589, + "rerank_top2_raw_score": 4.4004254, + "rerank_margin_raw": 0.4441638, + "support_top1_score": 0.70476305, + "candidate_memory_ids": [ + 24, + 20, + 12 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 12, + 24 + ], + "gold_answer": "no, it should use plain http and the current endpoint is http://localhost:8003", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.23228288, + "rerank_top1_raw_score": 4.844589, + "rerank_top2_raw_score": 4.4004254, + "rerank_margin_raw": 0.4441638, + "support_top1_score": 0.70476305, + "final_decision": "answer", + "final_answer_memory_ids": [ + 24 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.23228288, + "embedding_latency_ms": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 72, + "passed": true + }, + { + "query_id": "dev_q7", + "query_text": "what happened around the migration?", + "query_category": "temporally ambiguous query", + "split": "test", + "namespace": "default", + "reference_time": 1776519000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 5, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 15, + "score": 0.5008143, + "rank": 1, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": 0.6915206, + "rank": 2, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 12, + "score": 0.7378313, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 11, + "score": 0.75587124, + "rank": 4, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 13, + "score": 0.8044558, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 15, + "score": 1.1136857, + "rank": 1, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": -8.237752, + "rank": 2, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 11, + "score": -11.033457, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 13, + "score": -11.036493, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": -11.044441, + "rank": 5, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 5, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 39, + "first_stage_top1_distance": 0.5008143, + "rerank_top1_raw_score": 1.1136857, + "rerank_top2_raw_score": -8.237752, + "rerank_margin_raw": 9.351438, + "support_top1_score": 0.6102845, + "candidate_memory_ids": [ + 15, + 14, + 11 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 14, + 15, + 16 + ], + "gold_answer": "chat moved to llama-server, auto model discovery was added, and embeddings stayed separate on bge-m3", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.5008143, + "rerank_top1_raw_score": 1.1136857, + "rerank_top2_raw_score": -8.237752, + "rerank_margin_raw": 9.351438, + "support_top1_score": 0.6102845, + "final_decision": "answer", + "final_answer_memory_ids": [ + 15 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.5008143, + "embedding_latency_ms": 6, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 39, + "passed": true + }, + { + "query_id": "dev_q8", + "query_text": "why did the benchmark latency improve?", + "query_category": "multi-evidence query", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 9, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 20, + "score": 0.34632146, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 21, + "score": 0.46931833, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 11, + "score": 0.48832768, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": 0.4890368, + "rank": 4, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 24, + "score": 0.51704735, + "rank": 5, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 16, + "score": 0.5461181, + "rank": 6, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 15, + "score": 0.54766715, + "rank": 7, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": 0.55123526, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 13, + "score": 0.583761, + "rank": 9, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 20, + "score": -4.2785473, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 21, + "score": -6.0401225, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 15, + "score": -10.748125, + "rank": 3, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 24, + "score": -10.901875, + "rank": 4, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 13, + "score": -10.936829, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": -10.938702, + "rank": 6, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 16, + "score": -10.948033, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 11, + "score": -10.977427, + "rank": 8, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 14, + "score": -11.006992, + "rank": 9, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 71, + "first_stage_top1_distance": 0.34632146, + "rerank_top1_raw_score": -4.2785473, + "rerank_top2_raw_score": -6.0401225, + "rerank_margin_raw": 1.7615752, + "support_top1_score": 0.2741226, + "candidate_memory_ids": [ + 20, + 21, + 15 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 20, + 24 + ], + "gold_answer": "because the reranker endpoint switched to plain http and the model was warm", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.34632146, + "rerank_top1_raw_score": -4.2785473, + "rerank_top2_raw_score": -6.0401225, + "rerank_margin_raw": 1.7615752, + "support_top1_score": 0.2741226, + "final_decision": "answer", + "final_answer_memory_ids": [ + 20 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.34632146, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 71, + "passed": true + }, + { + "query_id": "dev_q9", + "query_text": "what postgres version am i running for klbr?", + "query_category": "no-hit query", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 13, + "score": 0.55336356, + "rank": 1, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 16, + "score": 0.56840855, + "rank": 2, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 24, + "score": 0.5717842, + "rank": 3, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 12, + "score": 0.5768131, + "rank": 4, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 11, + "score": 0.5796458, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 25, + "score": 0.59437966, + "rank": 6, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 14, + "score": 0.5957218, + "rank": 7, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 34, + "score": 0.6207646, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 20, + "score": 0.6295264, + "rank": 9, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 21, + "score": 0.637861, + "rank": 10, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 32, + "score": 0.6580149, + "rank": 11, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": 0.68314207, + "rank": 12, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 15, + "score": 0.68807095, + "rank": 13, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + } + ], + "reranked_candidates": [ + { + "memory_id": 13, + "score": -2.058997, + "rank": 1, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 16, + "score": -3.992416, + "rank": 2, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 11, + "score": -5.038967, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 25, + "score": -5.0817056, + "rank": 4, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 24, + "score": -5.282342, + "rank": 5, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 34, + "score": -5.534534, + "rank": 6, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 12, + "score": -5.589077, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 20, + "score": -5.7388763, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 14, + "score": -5.868785, + "rank": 9, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 21, + "score": -6.130826, + "rank": 10, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 77, + "first_stage_top1_distance": 0.55336356, + "rerank_top1_raw_score": -2.0575826, + "rerank_top2_raw_score": -4.002022, + "rerank_margin_raw": 1.9444392, + "support_top1_score": 0.023680523, + "candidate_memory_ids": [ + 13, + 16, + 11 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 65, + "first_stage_top1_distance": 0.55336356, + "rerank_top1_raw_score": -2.058997, + "rerank_top2_raw_score": -3.992416, + "rerank_margin_raw": 1.933419, + "support_top1_score": 0.023680523, + "candidate_memory_ids": [ + 13, + 16, + 11 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 64, + "first_stage_top1_distance": 0.55336356, + "rerank_top1_raw_score": -2.058997, + "rerank_top2_raw_score": -3.992416, + "rerank_margin_raw": 1.933419, + "support_top1_score": 0.02368052, + "candidate_memory_ids": [ + 13, + 16, + 11 + ], + "decision": "expand" + }, + { + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 93, + "first_stage_top1_distance": 0.55336356, + "rerank_top1_raw_score": -2.058997, + "rerank_top2_raw_score": -3.992416, + "rerank_margin_raw": 1.933419, + "support_top1_score": 0.02368052, + "candidate_memory_ids": [ + 13, + 16, + 11 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "abstained": true, + "no_hit": true, + "gold_memory_ids": [], + "first_stage_topk_contains_gold": false, + "first_stage_top1_distance": 0.55336356, + "rerank_top1_raw_score": -2.058997, + "rerank_top2_raw_score": -3.992416, + "rerank_margin_raw": 1.933419, + "support_top1_score": 0.02368052, + "final_decision": "abstain", + "final_correct": true, + "matched_gold": false, + "time_window_hit": false, + "best_distance": 0.55336356, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 299, + "passed": true + }, + { + "query_id": "test_q3", + "query_text": "what happened around the latency fix?", + "query_category": "temporally ambiguous query", + "split": "test", + "namespace": "default", + "reference_time": 1776688200, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 8, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 20, + "score": 0.47658592, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 11, + "score": 0.55802435, + "rank": 2, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 16, + "score": 0.5615119, + "rank": 3, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": 0.56483173, + "rank": 4, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 15, + "score": 0.5682094, + "rank": 5, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 12, + "score": 0.59718174, + "rank": 6, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": 0.61211073, + "rank": 7, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 13, + "score": 0.71054864, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "reranked_candidates": [ + { + "memory_id": 20, + "score": -4.509659, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 15, + "score": -7.122124, + "rank": 2, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 21, + "score": -10.517637, + "rank": 3, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 16, + "score": -10.695452, + "rank": 4, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": -10.700682, + "rank": 5, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 11, + "score": -10.952301, + "rank": 6, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": -10.960333, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 13, + "score": -11.027676, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 61, + "first_stage_top1_distance": 0.47658592, + "rerank_top1_raw_score": -4.509659, + "rerank_top2_raw_score": -7.122124, + "rerank_margin_raw": 2.6124654, + "support_top1_score": 0.3087778, + "candidate_memory_ids": [ + 20, + 15, + 21 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 20, + 21 + ], + "gold_answer": "latency improved after moving to http and a remaining bug was the wrong abstention rule", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.47658592, + "rerank_top1_raw_score": -4.509659, + "rerank_top2_raw_score": -7.122124, + "rerank_margin_raw": 2.6124654, + "support_top1_score": 0.3087778, + "final_decision": "answer", + "final_answer_memory_ids": [ + 20 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.47658592, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 61, + "passed": true + }, + { + "query_id": "test_q4", + "query_text": "which browser am i using to inspect the benchmark output?", + "query_category": "no-hit query", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 12, + "score": 0.43304205, + "rank": 1, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 24, + "score": 0.43905824, + "rank": 2, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 11, + "score": 0.4803093, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": 0.48766696, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 25, + "score": 0.48778892, + "rank": 5, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 13, + "score": 0.49560863, + "rank": 6, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 34, + "score": 0.5042389, + "rank": 7, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 21, + "score": 0.5293069, + "rank": 8, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 32, + "score": 0.5556946, + "rank": 9, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 16, + "score": 0.56365263, + "rank": 10, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": 0.5935675, + "rank": 11, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 33, + "score": 0.62041926, + "rank": 12, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 15, + "score": 0.64443076, + "rank": 13, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + } + ], + "reranked_candidates": [ + { + "memory_id": 32, + "score": -9.000448, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 20, + "score": -9.027144, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": -9.091831, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": -9.160467, + "rank": 4, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 34, + "score": -9.716939, + "rank": 5, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 13, + "score": -9.90797, + "rank": 6, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 24, + "score": -9.9791765, + "rank": 7, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 16, + "score": -10.530354, + "rank": 8, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 25, + "score": -10.817764, + "rank": 9, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 11, + "score": -10.877481, + "rank": 10, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "first_stage_top1_distance": 0.43304205, + "rerank_top1_raw_score": -9.027144, + "rerank_top2_raw_score": -9.091831, + "rerank_margin_raw": 0.064686775, + "support_top1_score": 0.12108205, + "candidate_memory_ids": [ + 20, + 12, + 21 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 68, + "first_stage_top1_distance": 0.43304205, + "rerank_top1_raw_score": -9.027144, + "rerank_top2_raw_score": -9.091831, + "rerank_margin_raw": 0.064686775, + "support_top1_score": 0.12108208, + "candidate_memory_ids": [ + 20, + 12, + 21 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 65, + "first_stage_top1_distance": 0.43304205, + "rerank_top1_raw_score": -9.027144, + "rerank_top2_raw_score": -9.091831, + "rerank_margin_raw": 0.064686775, + "support_top1_score": 0.12108207, + "candidate_memory_ids": [ + 20, + 12, + 21 + ], + "decision": "expand" + }, + { + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 80, + "first_stage_top1_distance": 0.43304205, + "rerank_top1_raw_score": -9.000448, + "rerank_top2_raw_score": -9.027144, + "rerank_margin_raw": 0.026696205, + "support_top1_score": 0.24071752, + "candidate_memory_ids": [ + 32, + 20, + 12 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "abstained": true, + "no_hit": true, + "gold_memory_ids": [], + "first_stage_topk_contains_gold": false, + "first_stage_top1_distance": 0.43304205, + "rerank_top1_raw_score": -9.000448, + "rerank_top2_raw_score": -9.027144, + "rerank_margin_raw": 0.026696205, + "support_top1_score": 0.24071752, + "final_decision": "abstain", + "final_correct": true, + "matched_gold": false, + "time_window_hit": false, + "best_distance": 0.43304205, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 295, + "passed": true + }, + { + "query_id": "test_q5", + "query_text": "what reranker did i decide to use on april 17?", + "query_category": "exact dated event", + "split": "test", + "namespace": "default", + "reference_time": 1776425400, + "windows_considered": [ + 7, + 30, + 90, + null + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 11, + "score": 0.37341732, + "rank": 1, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 24, + "score": 0.40523702, + "rank": 2, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 12, + "score": 0.41267246, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 20, + "score": 0.4146363, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": 0.41660315, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 21, + "score": 0.49884838, + "rank": 6, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 32, + "score": 0.51526046, + "rank": 7, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": 0.54314053, + "rank": 8, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 34, + "score": 0.5573089, + "rank": 9, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 14, + "score": 0.58860075, + "rank": 10, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 15, + "score": 0.6171034, + "rank": 11, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 25, + "score": 0.6384069, + "rank": 12, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 16, + "score": 0.64733934, + "rank": 13, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + } + ], + "reranked_candidates": [ + { + "memory_id": 13, + "score": -2.0152168, + "rank": 1, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 21, + "score": -4.7012267, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 12, + "score": -5.164671, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 20, + "score": -5.3125772, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 11, + "score": -5.944386, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 24, + "score": -6.1211977, + "rank": 6, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 32, + "score": -10.555868, + "rank": 7, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 34, + "score": -10.729806, + "rank": 8, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 33, + "score": -10.829969, + "rank": 9, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 14, + "score": -10.965754, + "rank": 10, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 3, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 25, + "first_stage_top1_distance": 0.37341732, + "rerank_top1_raw_score": -2.0136237, + "rerank_top2_raw_score": -5.153698, + "rerank_margin_raw": 3.1400743, + "support_top1_score": 0.10586866, + "candidate_memory_ids": [ + 13, + 12, + 11 + ], + "decision": "expand" + }, + { + "window_days": 30, + "memories_searched": 3, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 23, + "first_stage_top1_distance": 0.37341732, + "rerank_top1_raw_score": -2.0136237, + "rerank_top2_raw_score": -5.153698, + "rerank_margin_raw": 3.1400743, + "support_top1_score": 0.10586868, + "candidate_memory_ids": [ + 13, + 12, + 11 + ], + "decision": "expand" + }, + { + "window_days": 90, + "memories_searched": 3, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 23, + "first_stage_top1_distance": 0.37341732, + "rerank_top1_raw_score": -2.0136237, + "rerank_top2_raw_score": -5.153698, + "rerank_margin_raw": 3.1400743, + "support_top1_score": 0.10586867, + "candidate_memory_ids": [ + 13, + 12, + 11 + ], + "decision": "expand" + }, + { + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 102, + "first_stage_top1_distance": 0.37341732, + "rerank_top1_raw_score": -2.0152168, + "rerank_top2_raw_score": -4.7012267, + "rerank_margin_raw": 2.68601, + "support_top1_score": 0.10586867, + "candidate_memory_ids": [ + 13, + 21, + 12 + ], + "decision": "abstain" + } + ], + "window_expanded": true, + "abstained": true, + "no_hit": false, + "gold_memory_ids": [ + 13 + ], + "gold_answer": "bge-reranker-v2-m3", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.37341732, + "rerank_top1_raw_score": -2.0152168, + "rerank_top2_raw_score": -4.7012267, + "rerank_margin_raw": 2.68601, + "support_top1_score": 0.10586867, + "final_decision": "abstain", + "final_correct": false, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.37341732, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 173, + "passed": false + }, + { + "query_id": "test_q6", + "query_text": "what changed in model serving around llama-server?", + "query_category": "vague recent lookup", + "split": "test", + "namespace": "default", + "reference_time": 1776852000, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 9, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 14, + "score": 0.20254111, + "rank": 1, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 16, + "score": 0.4988569, + "rank": 2, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 20, + "score": 0.509845, + "rank": 3, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 15, + "score": 0.54394186, + "rank": 4, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 11, + "score": 0.580803, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 24, + "score": 0.63381386, + "rank": 6, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 12, + "score": 0.65555215, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 13, + "score": 0.6842468, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 21, + "score": 0.7125449, + "rank": 9, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + } + ], + "reranked_candidates": [ + { + "memory_id": 15, + "score": 1.8137314, + "rank": 1, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": 1.5220034, + "rank": 2, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 16, + "score": -5.8803244, + "rank": 3, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 20, + "score": -8.690493, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": -10.527811, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 11, + "score": -10.580921, + "rank": 6, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": -10.632627, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": -10.742384, + "rank": 8, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 24, + "score": -10.809497, + "rank": 9, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "first_stage_top1_distance": 0.20254111, + "rerank_top1_raw_score": 1.8137314, + "rerank_top2_raw_score": 1.5220034, + "rerank_margin_raw": 0.29172802, + "support_top1_score": 0.22384623, + "candidate_memory_ids": [ + 15, + 14, + 16 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 14, + 15, + 16 + ], + "gold_answer": "chat moved to llama-server, auto model discovery was added, and embeddings remained separate", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.20254111, + "rerank_top1_raw_score": 1.8137314, + "rerank_top2_raw_score": 1.5220034, + "rerank_margin_raw": 0.29172802, + "support_top1_score": 0.22384623, + "final_decision": "answer", + "final_answer_memory_ids": [ + 15 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.20254111, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 82, + "passed": true + }, + { + "query_id": "test_q11", + "query_text": "what was wrong with the benchmark after the reranker got fast?", + "query_category": "conflict / update question", + "split": "test", + "namespace": "default", + "reference_time": 1776688200, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 8, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 20, + "score": 0.37488163, + "rank": 1, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 21, + "score": 0.37673914, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 12, + "score": 0.38627172, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 11, + "score": 0.4347251, + "rank": 4, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 13, + "score": 0.49489146, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 16, + "score": 0.61126906, + "rank": 6, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 15, + "score": 0.6380105, + "rank": 7, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": 0.65780246, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "reranked_candidates": [ + { + "memory_id": 21, + "score": 0.47590742, + "rank": 1, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 20, + "score": -2.7969186, + "rank": 2, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": -6.7498226, + "rank": 3, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 13, + "score": -8.040634, + "rank": 4, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 11, + "score": -8.379541, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 15, + "score": -10.981193, + "rank": 6, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 16, + "score": -10.992662, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": -11.029812, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 8, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 65, + "first_stage_top1_distance": 0.37488163, + "rerank_top1_raw_score": 0.47590742, + "rerank_top2_raw_score": -2.7969186, + "rerank_margin_raw": 3.272826, + "support_top1_score": 0.5172907, + "candidate_memory_ids": [ + 21, + 20, + 12 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 21 + ], + "gold_answer": "it still abstained on a positive query because it used first-stage distance instead of rerank confidence", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.37488163, + "rerank_top1_raw_score": 0.47590742, + "rerank_top2_raw_score": -2.7969186, + "rerank_margin_raw": 3.272826, + "support_top1_score": 0.5172907, + "final_decision": "answer", + "final_answer_memory_ids": [ + 21 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.37488163, + "embedding_latency_ms": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 65, + "passed": true + }, + { + "query_id": "test_q13", + "query_text": "what are we doing before passive recall work?", + "query_category": "exact recent event", + "split": "test", + "namespace": "default", + "reference_time": 1776871800, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 11, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 32, + "score": 0.30882895, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 21, + "score": 0.5285853, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 11, + "score": 0.5750121, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": 0.58136404, + "rank": 4, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": 0.5972656, + "rank": 5, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 24, + "score": 0.6108694, + "rank": 6, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 16, + "score": 0.6125368, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 13, + "score": 0.6125791, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 15, + "score": 0.62170863, + "rank": 9, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 25, + "score": 0.6707506, + "rank": 10, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 14, + "score": 0.672945, + "rank": 11, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "reranked_candidates": [ + { + "memory_id": 32, + "score": 2.4341173, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 16, + "score": -11.022617, + "rank": 2, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 11, + "score": -11.0276985, + "rank": 3, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 15, + "score": -11.032341, + "rank": 4, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 24, + "score": -11.035565, + "rank": 5, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 20, + "score": -11.039518, + "rank": 6, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 25, + "score": -11.039633, + "rank": 7, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 13, + "score": -11.040808, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": -11.040964, + "rank": 9, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": -11.044161, + "rank": 10, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 11, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 86, + "first_stage_top1_distance": 0.30882895, + "rerank_top1_raw_score": 2.4341173, + "rerank_top2_raw_score": -11.022617, + "rerank_margin_raw": 13.456735, + "support_top1_score": 0.6894706, + "candidate_memory_ids": [ + 32, + 16, + 11 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 32 + ], + "gold_answer": "expanding the eval dataset and rerunning the retrieval benchmarks", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.30882895, + "rerank_top1_raw_score": 2.4341173, + "rerank_top2_raw_score": -11.022617, + "rerank_margin_raw": 13.456735, + "support_top1_score": 0.6894706, + "final_decision": "answer", + "final_answer_memory_ids": [ + 32 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.30882895, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 86, + "passed": true + }, + { + "query_id": "test_q14", + "query_text": "why are we expanding the dataset before treating thresholds as final?", + "query_category": "multi-evidence query", + "split": "test", + "namespace": "default", + "reference_time": 1776875400, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 34, + "score": 0.29977006, + "rank": 1, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 32, + "score": 0.43898106, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": 0.4789518, + "rank": 3, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 21, + "score": 0.5315, + "rank": 4, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 11, + "score": 0.5622864, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 15, + "score": 0.5718218, + "rank": 6, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 12, + "score": 0.5830258, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 24, + "score": 0.58842564, + "rank": 8, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 20, + "score": 0.60705435, + "rank": 9, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 16, + "score": 0.6085937, + "rank": 10, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 25, + "score": 0.6190504, + "rank": 11, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 13, + "score": 0.6478834, + "rank": 12, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 14, + "score": 0.6828743, + "rank": 13, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "reranked_candidates": [ + { + "memory_id": 34, + "score": -1.6889277, + "rank": 1, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 32, + "score": -2.930198, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": -8.306936, + "rank": 3, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 21, + "score": -10.220276, + "rank": 4, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 15, + "score": -10.906208, + "rank": 5, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 16, + "score": -10.969531, + "rank": 6, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 11, + "score": -11.025587, + "rank": 7, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 24, + "score": -11.028642, + "rank": 8, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 20, + "score": -11.032233, + "rank": 9, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": -11.0349655, + "rank": 10, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 99, + "first_stage_top1_distance": 0.29977006, + "rerank_top1_raw_score": -1.6889277, + "rerank_top2_raw_score": -2.930198, + "rerank_margin_raw": 1.2412703, + "support_top1_score": 0.33857638, + "candidate_memory_ids": [ + 34, + 32, + 33 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 32, + 33, + 34 + ], + "gold_answer": "because the eval set needed stronger coverage and the thresholds should stay provisional until the larger dataset is benchmarked again", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.29977006, + "rerank_top1_raw_score": -1.6889277, + "rerank_top2_raw_score": -2.930198, + "rerank_margin_raw": 1.2412703, + "support_top1_score": 0.33857638, + "final_decision": "answer", + "final_answer_memory_ids": [ + 34 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.29977006, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 99, + "passed": true + }, + { + "query_id": "test_q16", + "query_text": "what kinds of eval cases did the starter set need more of?", + "query_category": "recurring theme / pattern-like question", + "split": "test", + "namespace": "default", + "reference_time": 1776873600, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 12, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 33, + "score": 0.24168712, + "rank": 1, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 32, + "score": 0.4856398, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 21, + "score": 0.5849088, + "rank": 3, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 15, + "score": 0.62238055, + "rank": 4, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 11, + "score": 0.62328947, + "rank": 5, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 16, + "score": 0.6395925, + "rank": 6, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 20, + "score": 0.6528981, + "rank": 7, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": 0.6603805, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": 0.6671079, + "rank": 9, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 14, + "score": 0.6739335, + "rank": 10, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 24, + "score": 0.7105627, + "rank": 11, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 25, + "score": 0.7659235, + "rank": 12, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + } + ], + "reranked_candidates": [ + { + "memory_id": 33, + "score": 5.935469, + "rank": 1, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 32, + "score": -5.50741, + "rank": 2, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 21, + "score": -8.22163, + "rank": 3, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 15, + "score": -10.223949, + "rank": 4, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 13, + "score": -10.557413, + "rank": 5, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 20, + "score": -10.76824, + "rank": 6, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 16, + "score": -10.868647, + "rank": 7, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 14, + "score": -11.021965, + "rank": 8, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 11, + "score": -11.022623, + "rank": 9, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 12, + "score": -11.029509, + "rank": 10, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 12, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 86, + "first_stage_top1_distance": 0.24168712, + "rerank_top1_raw_score": 5.935469, + "rerank_top2_raw_score": -5.50741, + "rerank_margin_raw": 11.442879, + "support_top1_score": 0.61777955, + "candidate_memory_ids": [ + 33, + 32, + 21 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 33 + ], + "gold_answer": "no-hit, conflict/update, and multi-evidence cases", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.24168712, + "rerank_top1_raw_score": 5.935469, + "rerank_top2_raw_score": -5.50741, + "rerank_margin_raw": 11.442879, + "support_top1_score": 0.61777955, + "final_decision": "answer", + "final_answer_memory_ids": [ + 33 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.24168712, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 86, + "passed": true + }, + { + "query_id": "test_q17", + "query_text": "are the current thresholds final now that we have one sweep?", + "query_category": "conflict / update question", + "split": "test", + "namespace": "default", + "reference_time": 1776875400, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 34, + "score": 0.42431813, + "rank": 1, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 24, + "score": 0.50685287, + "rank": 2, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 25, + "score": 0.55963767, + "rank": 3, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 11, + "score": 0.5625962, + "rank": 4, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 33, + "score": 0.5648034, + "rank": 5, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 20, + "score": 0.5670828, + "rank": 6, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": 0.6027025, + "rank": 7, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 21, + "score": 0.6055334, + "rank": 8, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 32, + "score": 0.60941494, + "rank": 9, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 13, + "score": 0.6245899, + "rank": 10, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 16, + "score": 0.6402204, + "rank": 11, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 15, + "score": 0.6608897, + "rank": 12, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 14, + "score": 0.71607155, + "rank": 13, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + } + ], + "reranked_candidates": [ + { + "memory_id": 34, + "score": -0.5518186, + "rank": 1, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 21, + "score": -9.504051, + "rank": 2, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 32, + "score": -10.390141, + "rank": 3, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": -10.559173, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 24, + "score": -10.9036455, + "rank": 5, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 25, + "score": -10.947115, + "rank": 6, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + }, + { + "memory_id": 13, + "score": -10.98835, + "rank": 7, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 11, + "score": -11.03325, + "rank": 8, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": -11.03633, + "rank": 9, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 12, + "score": -11.0394745, + "rank": 10, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "first_stage_top1_distance": 0.42431813, + "rerank_top1_raw_score": -0.5518186, + "rerank_top2_raw_score": -9.504051, + "rerank_margin_raw": 8.952232, + "support_top1_score": 0.49671164, + "candidate_memory_ids": [ + 34, + 21, + 32 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 34 + ], + "gold_answer": "no, they should remain provisional until the larger dataset is benchmarked again", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.42431813, + "rerank_top1_raw_score": -0.5518186, + "rerank_top2_raw_score": -9.504051, + "rerank_margin_raw": 8.952232, + "support_top1_score": 0.49671164, + "final_decision": "answer", + "final_answer_memory_ids": [ + 34 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.42431813, + "embedding_latency_ms": 9, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "passed": true + }, + { + "query_id": "test_q18", + "query_text": "what was the sequence around the dataset expansion decision?", + "query_category": "temporally ambiguous query", + "split": "test", + "namespace": "default", + "reference_time": 1776875400, + "chosen_window_days": 7, + "windows_considered": [ + 7 + ], + "memories_searched": 13, + "vector_search_mode": "exact", + "top_candidates": [ + { + "memory_id": 32, + "score": 0.4326526, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 33, + "score": 0.48608702, + "rank": 2, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 34, + "score": 0.49828094, + "rank": 3, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 15, + "score": 0.5548509, + "rank": 4, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 21, + "score": 0.6137197, + "rank": 5, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 11, + "score": 0.62564224, + "rank": 6, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": 0.6582935, + "rank": 7, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 13, + "score": 0.66541797, + "rank": 8, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 12, + "score": 0.67650366, + "rank": 9, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + }, + { + "memory_id": 14, + "score": 0.6795111, + "rank": 10, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 16, + "score": 0.67974293, + "rank": 11, + "event_time": 1776520800, + "tags": [ + "project:klbr", + "topic:embeddings" + ], + "source_ref": "session:t5:m3", + "text": "the separate embedding server still uses bge-m3 rather than whatever chat model is currently loaded." + }, + { + "memory_id": 24, + "score": 0.68718433, + "rank": 12, + "event_time": 1776852000, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t9:m1", + "text": "the current reranker endpoint is http://localhost:8003." + }, + { + "memory_id": 25, + "score": 0.6936032, + "rank": 13, + "event_time": 1776853800, + "tags": [ + "project:klbr", + "topic:embeddings", + "topic:networking" + ], + "source_ref": "session:t9:m2", + "text": "the current embedding endpoint is http://localhost:8002." + } + ], + "reranked_candidates": [ + { + "memory_id": 32, + "score": -3.462239, + "rank": 1, + "event_time": 1776871800, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:benchmarking" + ], + "source_ref": "session:t12:m1", + "text": "before passive recall work, the next step was to expand the eval dataset and rerun the retrieval benchmarks." + }, + { + "memory_id": 34, + "score": -5.8266144, + "rank": 2, + "event_time": 1776875400, + "tags": [ + "project:klbr", + "topic:calibration", + "topic:eval-data" + ], + "source_ref": "session:t12:m3", + "text": "the current thresholds should stay provisional until the larger dataset is benchmarked again." + }, + { + "memory_id": 15, + "score": -6.769223, + "rank": 3, + "event_time": 1776519000, + "tags": [ + "project:klbr", + "topic:llama-server", + "topic:models" + ], + "source_ref": "session:t5:m2", + "text": "around the migration, auto model discovery was added for chat and embeddings." + }, + { + "memory_id": 33, + "score": -8.372524, + "rank": 4, + "event_time": 1776873600, + "tags": [ + "project:klbr", + "topic:eval-data", + "topic:calibration" + ], + "source_ref": "session:t12:m2", + "text": "the starter eval set needed stronger coverage in no-hit, conflict/update, and multi-evidence cases." + }, + { + "memory_id": 21, + "score": -9.179924, + "rank": 5, + "event_time": 1776688200, + "tags": [ + "project:klbr", + "topic:abstention", + "topic:reranker" + ], + "source_ref": "session:t7:m2", + "text": "the benchmark still abstained wrongly on one positive query because abstention used first-stage distance instead of rerank confidence." + }, + { + "memory_id": 13, + "score": -10.018687, + "rank": 6, + "event_time": 1776425400, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m3", + "text": "the reranker choice for the mvp is bge-reranker-v2-m3." + }, + { + "memory_id": 11, + "score": -10.405049, + "rank": 7, + "event_time": 1776421800, + "tags": [ + "project:klbr", + "topic:reranker" + ], + "source_ref": "session:t4:m1", + "text": "planned a separate reranker endpoint on localhost 8003." + }, + { + "memory_id": 20, + "score": -10.809007, + "rank": 8, + "event_time": 1776686400, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:latency" + ], + "source_ref": "session:t7:m1", + "text": "rerank latency dropped sharply after switching the reranker endpoint to plain http and using a warmed model." + }, + { + "memory_id": 14, + "score": -10.8813095, + "rank": 9, + "event_time": 1776517200, + "tags": [ + "project:klbr", + "topic:llama-server" + ], + "source_ref": "session:t5:m1", + "text": "moved chat model serving from lm studio to llama-server." + }, + { + "memory_id": 12, + "score": -11.023531, + "rank": 10, + "event_time": 1776423600, + "tags": [ + "project:klbr", + "topic:reranker", + "topic:networking" + ], + "source_ref": "session:t4:m2", + "text": "the cpu-only reranker should use plain http, not https." + } + ], + "window_attempts": [ + { + "window_days": 7, + "memories_searched": 13, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "first_stage_top1_distance": 0.4326526, + "rerank_top1_raw_score": -3.462239, + "rerank_top2_raw_score": -5.8266144, + "rerank_margin_raw": 2.3643754, + "support_top1_score": 0.2793007, + "candidate_memory_ids": [ + 32, + 34, + 15 + ], + "decision": "answer" + } + ], + "window_expanded": false, + "abstained": false, + "no_hit": false, + "gold_memory_ids": [ + 32, + 33, + 34 + ], + "gold_answer": "the next step was to expand the dataset, it needed more hard cases, and thresholds stayed provisional until rerunning the larger benchmark", + "first_stage_topk_contains_gold": true, + "first_stage_top1_distance": 0.4326526, + "rerank_top1_raw_score": -3.462239, + "rerank_top2_raw_score": -5.8266144, + "rerank_margin_raw": 2.3643754, + "support_top1_score": 0.2793007, + "final_decision": "answer", + "final_answer_memory_ids": [ + 32 + ], + "final_correct": true, + "matched_gold": true, + "time_window_hit": true, + "best_distance": 0.4326526, + "embedding_latency_ms": 10, + "retrieval_latency_ms": 0, + "rerank_latency_ms": 75, + "passed": true + } +] \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/summary.md b/benchmarks/runs/2026-04-24_suite_smoke/summary.md new file mode 100644 index 0000000..d1f869d --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/summary.md @@ -0,0 +1,64 @@ +# KLBR Benchmark Suite Run + +- run_dir: `benchmarks/runs/2026-04-24_suite_smoke` +- timestamp_utc: `2026-04-24T20:27:38.334913+00:00` +- git_rev: `da2b58ac63fcb33a4f80e7773ba8fcf14f334f8b` + +## Active Recall (Retrieval) — Dev + +- output: `benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev` +- report: `benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev/report.md` + +- Final decision accuracy: `0.8824` +- Coverage: `0.8824` +- Selective accuracy: `1.0000` + +## Active Recall (Retrieval) — Test + +- output: `benchmarks/runs/2026-04-24_suite_smoke/retrieval-test` +- report: `benchmarks/runs/2026-04-24_suite_smoke/retrieval-test/report.md` + +- Final decision accuracy: `0.9375` +- Coverage: `0.8125` +- Selective accuracy: `1.0000` + +## Passive Recall — Dev + +- output: `benchmarks/runs/2026-04-24_suite_smoke/passive-dev` +- report: `benchmarks/runs/2026-04-24_suite_smoke/passive-dev/report.md` + +- Activation accuracy: `0.9524` +- Activation precision: `1.0000` +- Activation recall: `0.9375` +- No-recall false activation rate: `0.0000` + +## Passive Recall — Test + +- output: `benchmarks/runs/2026-04-24_suite_smoke/passive-test` +- report: `benchmarks/runs/2026-04-24_suite_smoke/passive-test/report.md` + +- Activation accuracy: `0.9048` +- Activation precision: `0.9375` +- Activation recall: `0.9375` +- No-recall false activation rate: `0.2000` + +## Tools Lane — Dev + +- output: `benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev` +- report: `benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.md` + +- tools recall: `0.4000` +- tools precision: `0.6667` +- router accuracy: `0.8182` +- called any tool rate (given routed-to-tools): `0.6667` + +## Tools Lane — Test + +- output: `benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test` +- report: `benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.md` + +- tools recall: `0.0000` +- tools precision: `0.0000` +- router accuracy: `0.8750` +- called any tool rate (given routed-to-tools): `0.0000` + diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.json b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.json new file mode 100644 index 0000000..46a48fe --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.json @@ -0,0 +1,25 @@ +{ + "dataset_id": "internal-eval-starter-v2", + "experiment_id": "mvp-rerank-support-calibrated", + "evaluated_split": "dev", + "llm_url": "http://localhost:8001", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "router_model_path": "/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json", + "write_root": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_writes", + "metrics": { + "query_count": 22, + "gold_tools": 5, + "gold_memory": 17, + "gold_abstain": 0, + "router_accuracy": 0.8181818, + "router_tools_recall": 0.4, + "router_tools_precision": 0.6666667, + "router_memory_false_tools_rate": 0.05882353, + "router_abstain_false_tools_rate": 0.0, + "tools_lane_routed_count": 3, + "tools_lane_called_any_tool_rate": 0.6666667, + "tools_lane_required_tool_hit_rate": null, + "tools_lane_tool_exec_error_rate": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.md b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.md new file mode 100644 index 0000000..691824e --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/report.md @@ -0,0 +1,37 @@ +# Tools Lane Benchmark Report + +dataset: `internal-eval-starter-v2` +experiment: `mvp-rerank-support-calibrated` +split: `Some(Dev)` +llm_url: `http://localhost:8001` +embed_url: `http://localhost:8002` +embed_model: `bge-m3-FP16.gguf` +router_model: `/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json` +write_root: `/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_writes` + +## Router Metrics (Gold = route_label) + +- queries: `22` (tools `5` / memory `17` / abstain `0`) +- router accuracy: `0.8182` +- tools precision: `0.6667` +- tools recall: `0.4000` +- memory→tools rate: `0.0588` +- abstain→tools rate: `0.0000` + +## End-to-End Tool Lane + +- queries routed to tools: `3` +- called any tool rate (given routed-to-tools): `0.6667` +- required tool hit rate (if defined): `n/a` +- tool exec error rate (given routed-to-tools): `0.0000` + +## Traces (Tools Gold or Tools Pred) + +| id | gold | pred | called_tool | required_hit | tools_score | memory_score | abstain_score | kind | +|---|---|---|---:|---:|---:|---:|---:|---| +| `dev_q13` | `Tools` | `Tools` | 1 | n/a | 0.67 | 0.18 | 0.15 | `linear_softmax_v1(prob)` | +| `dev_q14` | `Tools` | `Tools` | 0 | n/a | 0.60 | 0.23 | 0.18 | `linear_softmax_v1(prob)` | +| `dev_q15` | `Tools` | `Memory` | 0 | n/a | 0.55 | 0.29 | 0.17 | `linear_softmax_v1(prob)` | +| `dev_q16` | `Tools` | `Memory` | 0 | n/a | 0.41 | 0.41 | 0.19 | `linear_softmax_v1(prob)` | +| `dev_q21` | `Tools` | `Memory` | 0 | n/a | 0.42 | 0.37 | 0.21 | `linear_softmax_v1(prob)` | +| `dev_q5` | `Memory` | `Tools` | 1 | n/a | 0.59 | 0.26 | 0.15 | `linear_softmax_v1(prob)` | diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_agent.db b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_agent.db new file mode 100644 index 0000000000000000000000000000000000000000..15b081b3be0950e3e58107204d1251712a22a083 GIT binary patch literal 40960 zcmWFz^vNtqRY=P(%1ta$FlG>7U}R))P*7lCU|?ooVBlgv0Colj1{MUDff0#~i^KNjx5aQ_MsSss85K3MrmGVMQTBQa)yScCW?zS8zni|#a&$) z8%@DMQpPQPSUzC|z9AB22T#{cDpPW&emtAaNfKQGCB_NgXDOJ)a$}i7MK?y~O zO~ndMKK@P$Fz0KAbFztNyD~Cl=B1=o6c^@XmZZifmX_p$`5@QfbZWdIJ{NE{s4MEf@hI3$sD6hCv7(rb8odw|N*Z7q$wSByjOP8~%&JtF7va&ZQ#l$o1Y zRH=}iTB)N@mY7qTs*sphsp-hVCLXF!RB8m7h?WFFLZCJ_2dK=7E;Ix6hebN{oEDGQj-lZORd!0q|}rYP^poYlb=`;ZD?R*604As zSzMBsmz)}(n_5zonQWV!U!0kjidrVLJ2A40%gQqLg@c_3D-U2a6VzRhLK7y6r`&{; zP9PFX&cH0rKuNW@ATb$SjDmE5i#-qnRx-c~HuaR$w8YY!5_L>dauO?3i$I#-7`>cD znBrrI#Q<3G4A+n03`|Q4GV}6Mp|u4>AI!%XPB*~RU6Pm%Ey*Ez6)I5|2Ni<48! zQu9jUOEPoeL5I_Nlqvz!DVcfcsl_Npk!un>8OG=4r=;d6;PX^zUUpu7c^)>O!4h+P zN@gxNba1){OTZ%BP?lO$oSC18Ct%C4rG?^>#FA2!xWS#65|c|Z%TlrUp*X*^C^|V$@lqAut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?+(SSD$1ng3776eu05diT&?o?tAhRag zxB+PXpJ8+W#Nn z31HOwqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UiCC1V;P+BO;ndJvkZzqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?!b4!R|3AVLz^L~}Ltr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%&%fX@FL-Tyx#qI%SmqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;3@?8UiCY1V-ooM{wkidVMqmMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(#Do9< Dr~t/dev/null || echo \\\"klbr not found\\\"\"}", + "status": "blocked", + "result": "error: shell blocked by bench sandbox" + }, + { + "name": "shell", + "args": "{\"cmd\":\"ls -R\"}", + "status": "ok", + "result": "exit 0\n.:\nAGENTS.md\nCargo.lock\nCargo.toml\nbenchmarks\ndocs\nflake.lock\nflake.nix\nklbr-bench\nklbr-core\nklbr-daemon\nklbr-ipc\nklbr-tui\nrust-toolchain.toml\ntarget\n\n./benchmarks:\nREADME.md\ninputs\nmodels\nrun_all.py\nruns\n\n./benchmarks/inputs:\nconfigs\ndatasets\nrouter\n\n./benchmarks/inputs/configs:\nmvp_exact_retrieval.json\nmvp_rerank_calibrated_balanced.json\nmvp_rerank_calibrated_conservative.json\nmvp_rerank_support_calibrated.json\nmvp_rerank_support_calibrated_test.json\n\n./benchmarks/inputs/datasets:\ninternal_eval_passive_recall_starter.json\ninternal_eval_passive_recall_v3.json\ninternal_eval_starter.json\ninternal_eval_template.json\n\n./benchmarks/inputs/router:\nrouter_holdout_v1.json\nrouter_holdouts\nrouter_slices\nrouter_tools.json\n\n./benchmarks/inputs/router/router_holdouts:\nholdout_abstain_toolish_v1.json\nholdout_memory_toolish_v1.json\nholdout_tools_recall_v1.json\n\n./benchmarks/inputs/router/router_slices:\nabstain_v1.json\nabstain_v2.json\nabstain_v3.json\ndev_pack_v1.json\nmemory_adversarial_v1.json\nmemory_lane_v1.json\nmemory_lane_v1b.json\nmemory_lane_v2.json\nmemory_lane_v3.json\nread_file_repo_v1.json\nread_file_repo_v1b.json\nread_file_repo_v2.json\nread_file_repo_v3.json\nread_file_repo_v4.json\nshell_state_v1.json\nshell_state_v1b.json\nshell_state_v2.json\nshell_state_v3.json\ntrain_pack_v1.json\nwrite_file_v1.json\n\n./benchmarks/models:\nrouter\n\n./benchmarks/models/router:\ncentroid\nlinear\n\n./benchmarks/models/router/centroid:\nout-router-iter-1\nout-router-iter-2\nout-router-iter-3\nout-router-iter-4\nout-router-iter-5\nout-router-iter-6\nout-router-iter-7\n\n./benchmarks/models/router/centroid/out-router-iter-1:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-2:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-3:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-4:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-5:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-6:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/centroid/out-router-iter-7:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear:\nout-router-linear-iter-1\nout-router-linear-iter-10\nout-router-linear-iter-11\nout-router-linear-iter-12\nout-router-linear-iter-13\nout-router-linear-iter-14\nout-router-linear-iter-2\nout-router-linear-iter-3\nout-router-linear-iter-4\nout-router-linear-iter-5\nout-router-linear-iter-6\nout-router-linear-iter-7\nout-router-linear-iter-8\nout-router-linear-iter-9\n\n./benchmarks/models/router/linear/out-router-linear-iter-1:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-10:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-11:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-12:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-13:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-14:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-2:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-3:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-4:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-5:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-6:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-7:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-8:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/models/router/linear/out-router-linear-iter-9:\nrouter_model.json\nrouter_report.json\nrouter_report.md\n\n./benchmarks/runs:\n2026-04-24_suite_smoke\nlegacy\n\n./benchmarks/runs/2026-04-24_suite_smoke:\npassive-dev\npassive-test\nretrieval-dev\nretrieval-test\ntools-lane-dev\n\n./benchmarks/runs/2026-04-24_suite_smoke/passive-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/2026-04-24_suite_smoke/passive-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/2026-04-24_suite_smoke/retrieval-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/2026-04-24_suite_smoke/retrieval-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev:\ntool_lane_agent.db\ntool_lane_writes\n\n./benchmarks/runs/2026-04-24_suite_smoke/tools-lane-dev/tool_lane_writes:\n\n./benchmarks/runs/legacy:\nout\nout-2026-04-23-end2end-passive-dev\nout-2026-04-23-end2end-passive-test\nout-2026-04-23-end2end-retrieval-dev\nout-2026-04-23-end2end-retrieval-test\nout-2026-04-23-end2end-tools-lane-dev\nout-2026-04-23-end2end-tools-lane-test\nout-2026-04-23-latest-passive-dev\nout-2026-04-23-latest-passive-test\nout-2026-04-23-latest-retrieval-dev\nout-2026-04-23-latest-retrieval-test\nout-2026-04-23-latest-retrieval-test-no-tools\nout-2026-04-23-passive-dev\nout-2026-04-23-passive-test\nout-2026-04-23-retrieval-dev\nout-2026-04-23-retrieval-test\nout-2026-04-23b-passive-dev\nout-2026-04-23b-passive-test\nout-2026-04-23b-retrieval-dev\nout-2026-04-23b-retrieval-test\nout-2026-04-23c-retrieval-dev\nout-2026-04-23c-sweep\nout-2026-04-23d-retrieval-dev\nout-2026-04-23e-retrieval-dev\nout-2026-04-23g-retrieval-dev\nout-2026-04-23h-retrieval-dev\nout-2026-04-23i-retrieval-dev\nout-2026-04-23j-retrieval-dev\nout-2026-04-23k-retrieval-dev\nout-2026-04-23k-retrieval-test\nout-2026-04-23l-retrieval-dev\nout-2026-04-23l-retrieval-test\nout-2026-04-23m-retrieval-test\nout-2026-04-23n-retrieval-test\nout-2026-04-23o-retrieval-dev\nout-2026-04-23p-passive-dev\nout-2026-04-23p-passive-test\nout-2026-04-23p-retrieval-dev\nout-2026-04-23p-retrieval-test\nout-2026-04-23p-router\nout-2026-04-23p-router2\nout-2026-04-23q-retrieval-dev\nout-2026-04-23q-retrieval-test\nout-2026-04-24-rerun-passive-dev\nout-2026-04-24-rerun-passive-test\nout-2026-04-24-rerun-retrieval-dev\nout-2026-04-24-rerun-retrieval-test\nout-2026-04-24-rerun-tools-lane-dev\nout-2026-04-24-rerun-tools-lane-test\nout-expanded-baseline\nout-expanded-support-calibrated\nout-expanded-support-sweep\nout-passive-recall\nout-passive-recall-expanded\nout-passive-recall-v3\nout-passive-recall-v3-final\nout-passive-recall-v3-final-test\nout-passive-recall-v3-gap\nout-passive-recall-v3-gap-test\nout-passive-recall-v3-test\n\n./benchmarks/runs/legacy/out:\nbalanced-inspect\nreport.json\nreport.md\nresolved_config.json\nsupport-trace-inspect\nsweep-coarse\nsweep-coarse-fast\nsweep-fine\nsweep-smoke\nsweep-support-focused\nsweep-support-smoke\ntraces.json\n\n./benchmarks/runs/legacy/out/balanced-inspect:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out/support-trace-inspect:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out/sweep-coarse:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out/sweep-coarse-fast:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out/sweep-fine:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out/sweep-smoke:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out/sweep-support-focused:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out/sweep-support-smoke:\nsweep_report.json\nsweep_report.md\nsweep_results.csv\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-passive-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-passive-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-retrieval-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-retrieval-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-tools-lane-dev:\nreport.json\nreport.md\ntool_lane_agent.db\ntool_lane_writes\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-tools-lane-dev/tool_lane_writes:\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-tools-lane-test:\nreport.json\nreport.md\ntool_lane_agent.db\ntool_lane_writes\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-end2end-tools-lane-test/tool_lane_writes:\n\n./benchmarks/runs/legacy/out-2026-04-23-latest-passive-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-latest-passive-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test-no-tools:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-passive-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-passive-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-retrieval-dev:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23-retrieval-test:\nreport.json\nreport.md\nresolved_config.json\ntraces.json\n\n./benchmarks/runs/legacy/out-2026-04-23b-passive-dev:\nreport.js..." + }, + { + "name": "read_file", + "args": "{\"path\":\"Cargo.toml\"}", + "status": "ok", + "result": "[workspace]\nmembers = [\n \"klbr-core\",\n \"klbr-bench\",\n \"klbr-daemon\",\n \"klbr-ipc\",\n \"klbr-tui\",\n]\nresolver = \"2\"\n" + }, + { + "name": "read_file", + "args": "{\"path\":\"flake.nix\"}", + "status": "ok", + "result": "{\n inputs.parts.url = \"github:hercules-ci/flake-parts\";\n inputs.nixpkgs.url = \"github:nixos/nixpkgs/nixpkgs-unstable\";\n inputs.nci.url = \"github:90-008/nix-cargo-integration\";\n\n outputs =\n inp:\n inp.parts.lib.mkFlake { inputs = inp; } {\n systems = [ \"x86_64-linux\" ];\n imports = [inp.nci.flakeModule];\n perSystem =\n {\n pkgs,\n config,\n ...\n }:\n {\n nci.projects.\"klbr\".path = ./.;\n packages.default = pkgs.callPackage ./default.nix {};\n devShells.default = config.nci.outputs.\"klbr\".devShell.overrideAttrs (old: {\n packages = (old.packages or []) ++ (with pkgs; [\n cargo-outdated\n clang\n wild\n ]);\n });\n };\n };\n}\n" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" .\"}", + "status": "ok", + "result": "exit 0\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-83ea6e6aab9f319f/lib-compact_str.json:{\"rustc\":5391851738765093524,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":2241668132362809309,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,5341227187612151375],[5532778797167691009,\"itoa\",false,922675177516112274],[6400797066282925533,\"ryu\",false,14154448430385616858],[7667230146095136825,\"cfg_if\",false,3918158090290381747],[13785866025199020095,\"static_assertions\",false,6061289999393473978],[14156967978702956262,\"rustversion\",false,15309377919697259106]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-83ea6e6aab9f319f/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-7802b1bda435b451/lib-compact_str.json:{\"rustc\":5391851738765093524,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":15657897354478470176,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,2066080914085243198],[5532778797167691009,\"itoa\",false,4844579086482742857],[6400797066282925533,\"ryu\",false,5662618109667158596],[7667230146095136825,\"cfg_if\",false,14602120158342893136],[13785866025199020095,\"static_assertions\",false,17151640901316189779],[14156967978702956262,\"rustversion\",false,15309377919697259106]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-7802b1bda435b451/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-1f6fb97c26282d55/lib-compact_str.json:{\"rustc\":7458672600737419911,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":2241668132362809309,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,7948244931920883639],[5532778797167691009,\"itoa\",false,15261797462267040700],[6400797066282925533,\"ryu\",false,3515111037861312882],[7667230146095136825,\"cfg_if\",false,8674239339299079555],[13785866025199020095,\"static_assertions\",false,16358668755797807532],[14156967978702956262,\"rustversion\",false,18027498038588910508]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-1f6fb97c26282d55/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-c65e47853e54c7a6/lib-compact_str.json:{\"rustc\":7458672600737419911,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":15657897354478470176,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,48144468780678448],[5532778797167691009,\"itoa\",false,8101464175262059514],[6400797066282925533,\"ryu\",false,3116507050166516951],[7667230146095136825,\"cfg_if\",false,12566202661935525378],[13785866025199020095,\"static_assertions\",false,6410479291781019334],[14156967978702956262,\"rustversion\",false,18027498038588910508]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-c65e47853e54c7a6/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./docs/klbr_rerank_sweep_2026-04-22.md:- `dev_q13`: `what postgres version is klbr using right now?`\n./benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json: \"text\": \"had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling.\",\n./benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json: \"text\": \"remind me why we stuck with postgres instead of sqlite a couple months ago.\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"what postgres version am i running for klbr?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"what postgres version is klbr using right now?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"which postgres extension did i enable for the benchmark database?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/inputs/router/router_holdout_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_holdout_v1.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_slices/abstain_v1.json: \"text\": \"should i use sqlite or postgres for the next project?\",\n./benchmarks/inputs/router/router_slices/abstain_v1.json: \"text\": \"can you log into the remote postgres and show me the schema?\",\n./benchmarks/inputs/router/router_slices/train_pack_v1.json: \"text\": \"log into the remote postgres and show me all tables (you have the password, right?)\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"description\": \"Additional repo-introspection tool queries focused on config/migrations/deps questions that look like 'memory recall' but should use read_file/grep. Includes targeted variants for postgres-extension questions.\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension do we need for vector search? find it in the docs or code\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension are we enabling for passive recall work (exact name)?\",\n./benchmarks/models/router/centroid/out-router-iter-3/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-3/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` |\n./benchmarks/models/router/centroid/out-router-iter-4/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-4/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-5/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-5/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-6/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-6/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.json: \"text\": \"can you log into the remote postgres and show me the schema?\",\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.md:| `abs_test_015` | can you log into the remote postgres and show me the schema? | `Abstain` | `Tools` | 0.70 | 0.59 | 0.62 |\n./benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json: \"text\": \"which postgres extension are we enabling bef..." + }, + { + "name": "shell", + "args": "{\"cmd\":\"psql --version\"}", + "status": "ok", + "result": "exit 127\nstderr:\nsh: line 1: psql: command not found\n" + }, + { + "name": "shell", + "args": "{\"cmd\":\"ls /usr/bin/psql || echo \\\"psql not found\\\"\"}", + "status": "ok", + "result": "exit 0\npsql not found\n\nstderr:\nls: cannot access '/usr/bin/psql': No such file or directory\n" + }, + { + "name": "shell", + "args": "{\"cmd\":\"which psql\"}", + "status": "ok", + "result": "exit 1\nstderr:\nwhich: no psql in (/home/mayer/.codex/tmp/arg0/codex-arg04lItmm:/nix/store/922crn2k3v8yaqk7anps80hba919lnds-ripgrep-15.1.0/bin:/nix/store/gr9l6ql3wg70idpqlqhnfdx81hak22c8-bubblewrap-0.11.0/bin:/nix/store/hb2bs5fg5wkm04x565737qd5nh2hy5nk-gcc-wrapper-15.2.0/bin:/nix/store/lvwga6ivl1d4lnw0zis9ajs0rqx9gp4i-gcc-15.2.0/bin:/nix/store/9hm3jdm6kk4m4ppwjcxz4s1dsdvd6fin-glibc-2.42-51-bin/bin:/nix/store/74sind1d6vf2bfwd7yklg8chsvzqxmmq-coreutils-9.10/bin:/nix/store/x7ikkplbrv5dlihy1bqq32gp6lilkval-binutils-wrapper-2.44/bin:/nix/store/rfp8lhk4dl9syfn64rwb3h3c73426p08-binutils-2.44/bin:/nix/store/m3nqzyycqsnf0l3gslizcx07hd8qmqdh-cargo-outdated-0.19.0/bin:/nix/store/hh6y3s72d21whp6q98h4dh0valxiaw69-clang-wrapper-21.1.8/bin:/nix/store/zb8py6p8qq01nygnnf8jrnkps9nna189-clang-21.1.8/bin:/nix/store/gknk1jw10h0pswl53ps5xj06cmwza52p-binutils-wrapper-2.44/bin:/nix/store/196a70lbnl4xbjycd13xiyf1qw1myf1p-wild-unwrapped-wrapper-0.8.0/bin:/nix/store/v13fslmca4ms1s0pl7w39y8330kzsqvg-wild-unwrapped-0.8.0/bin:/nix/store/g7nx2v7vmx0zmmd9l5d64cdkm4svgmq9-rust-default-1.95.0/bin:/nix/store/sj3f6y3j8m1831l0gqm1bsk1f46jzkfd-patchelf-0.15.2/bin:/nix/store/jlw598mkd1b4d50rwqv6m499h2lrlssj-compiler-rt-libc-21.1.8/bin:/nix/store/74sind1d6vf2bfwd7yklg8chsvzqxmmq-coreutils-9.10/bin:/nix/store/c89zz4vh8v9dbs8169wk8ahwxvrdxgm5-findutils-4.10.0/bin:/nix/store/355mp4ns4042sb5p51rx3ys4mlliiwc5-diffutils-3.12/bin:/nix/store/jpsqy47rdl0j0dvyyzb4kw8gqajw8nx0-gnused-4.9/bin:/nix/store/h6hdbgkfh59np7bi7h8qa76pq27ixz8r-gnugrep-3.12/bin:/nix/store/gg169kyil5vhsg5aqcpagyhs8fwl0r5r-gawk-5.3.2/bin:/nix/store/kvmqv1jqv4792rsihf7yjc5kwk3d8z6x-gnutar-1.35/bin:/nix/store/pnxiz967z73a45f4x7c1icldjaaqmlcp-gzip-1.14/bin:/nix/store/by6npbc13ly5a1zgsqghv7wv5bklyiny-bzip2-1.0.8-bin/bin:/nix/store/1fvcxyhg3i5fvw0j4l8wmyml10dnvm7q-gnumake-4.4.1/bin:/nix/store/v8sa6r6q037ihghxfbwzjj4p59v2x0pv-bash-5.3p9/bin:/nix/store/ca84rbhb0bhwdcci7q51dad20nkbn9xc-patch-2.8/bin:/nix/store/g6mlwdvpg92rchq352ll7jbi0pz7h43r-xz-5.8.2-bin/bin:/nix/store/kjg8z2k0zvsczaij78803g7imlfm1vfb-file-5.45/bin:/home/mayer/proj/klbr/.direnv/bin:/home/mayer/.config/carapace/bin:/home/mayer/.npm-global/bin:/run/wrappers/bin:/home/mayer/.local/share/flatpak/exports/bin:/var/lib/flatpak/exports/bin:/home/mayer/.nix-profile/bin:/nix/profile/bin:/home/mayer/.local/state/nix/profile/bin:/etc/profiles/per-user/mayer/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin)\n" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" klbr-core\"}", + "status": "ok", + "result": "exit 1" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" klbr-daemon\"}", + "status": "ok", + "result": "exit 1" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" klbr-ipc\"}", + "status": "ok", + "result": "exit 1" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" klbr-tui\"}", + "status": "ok", + "result": "exit 1" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" klbr-bench\"}", + "status": "ok", + "result": "exit 1" + }, + { + "name": "shell", + "args": "{\"cmd\":\"grep -r \\\"postgres\\\" .\"}", + "status": "ok", + "result": "exit 0\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-83ea6e6aab9f319f/lib-compact_str.json:{\"rustc\":5391851738765093524,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":2241668132362809309,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,5341227187612151375],[5532778797167691009,\"itoa\",false,922675177516112274],[6400797066282925533,\"ryu\",false,14154448430385616858],[7667230146095136825,\"cfg_if\",false,3918158090290381747],[13785866025199020095,\"static_assertions\",false,6061289999393473978],[14156967978702956262,\"rustversion\",false,15309377919697259106]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-83ea6e6aab9f319f/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-7802b1bda435b451/lib-compact_str.json:{\"rustc\":5391851738765093524,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":15657897354478470176,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,2066080914085243198],[5532778797167691009,\"itoa\",false,4844579086482742857],[6400797066282925533,\"ryu\",false,5662618109667158596],[7667230146095136825,\"cfg_if\",false,14602120158342893136],[13785866025199020095,\"static_assertions\",false,17151640901316189779],[14156967978702956262,\"rustversion\",false,15309377919697259106]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-7802b1bda435b451/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-1f6fb97c26282d55/lib-compact_str.json:{\"rustc\":7458672600737419911,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":2241668132362809309,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,7948244931920883639],[5532778797167691009,\"itoa\",false,15261797462267040700],[6400797066282925533,\"ryu\",false,3515111037861312882],[7667230146095136825,\"cfg_if\",false,8674239339299079555],[13785866025199020095,\"static_assertions\",false,16358668755797807532],[14156967978702956262,\"rustversion\",false,18027498038588910508]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-1f6fb97c26282d55/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./target/x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-c65e47853e54c7a6/lib-compact_str.json:{\"rustc\":7458672600737419911,\"features\":\"[\\\"std\\\"]\",\"declared_features\":\"[\\\"arbitrary\\\", \\\"borsh\\\", \\\"bytes\\\", \\\"default\\\", \\\"diesel\\\", \\\"markup\\\", \\\"proptest\\\", \\\"quickcheck\\\", \\\"rkyv\\\", \\\"serde\\\", \\\"smallvec\\\", \\\"sqlx\\\", \\\"sqlx-mysql\\\", \\\"sqlx-postgres\\\", \\\"sqlx-sqlite\\\", \\\"std\\\", \\\"zeroize\\\"]\",\"target\":7968499388442294171,\"profile\":15657897354478470176,\"path\":5809892735563642802,\"deps\":[[1127187624154154345,\"castaway\",false,48144468780678448],[5532778797167691009,\"itoa\",false,8101464175262059514],[6400797066282925533,\"ryu\",false,3116507050166516951],[7667230146095136825,\"cfg_if\",false,12566202661935525378],[13785866025199020095,\"static_assertions\",false,6410479291781019334],[14156967978702956262,\"rustversion\",false,18027498038588910508]],\"local\":[{\"CheckDepInfo\":{\"dep_info\":\"x86_64-unknown-linux-gnu/debug/.fingerprint/compact_str-c65e47853e54c7a6/dep-lib-compact_str\",\"checksum\":false}}],\"rustflags\":[\"-Clink-arg=--ld-path=wild\"],\"config\":9185878174080762935,\"compile_kind\":13270707523875659407}\n./docs/klbr_rerank_sweep_2026-04-22.md:- `dev_q13`: `what postgres version is klbr using right now?`\n./benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json: \"text\": \"had a long debate about switching from postgres to sqlite, but concluded postgres is still needed for concurrent writer scaling.\",\n./benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json: \"text\": \"remind me why we stuck with postgres instead of sqlite a couple months ago.\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"what postgres version am i running for klbr?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"what postgres version is klbr using right now?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"which postgres extension did i enable for the benchmark database?\",\n./benchmarks/inputs/datasets/internal_eval_starter.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/inputs/router/router_holdout_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_holdout_v1.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_slices/abstain_v1.json: \"text\": \"should i use sqlite or postgres for the next project?\",\n./benchmarks/inputs/router/router_slices/abstain_v1.json: \"text\": \"can you log into the remote postgres and show me the schema?\",\n./benchmarks/inputs/router/router_slices/train_pack_v1.json: \"text\": \"log into the remote postgres and show me all tables (you have the password, right?)\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"tell me the postgres password\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"log into the remote postgres and show me the schema\",\n./benchmarks/inputs/router/router_slices/abstain_v2.json: \"text\": \"postgresのパスワード教えて\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"description\": \"Additional repo-introspection tool queries focused on config/migrations/deps questions that look like 'memory recall' but should use read_file/grep. Includes targeted variants for postgres-extension questions.\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension do we need for vector search? find it in the docs or code\",\n./benchmarks/inputs/router/router_slices/read_file_repo_v4.json: \"text\": \"which postgres extension are we enabling for passive recall work (exact name)?\",\n./benchmarks/models/router/centroid/out-router-iter-3/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-3/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` |\n./benchmarks/models/router/centroid/out-router-iter-4/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-4/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-5/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-5/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-6/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-6/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.json: \"text\": \"which postgres extension are we enabling before passive recall?\",\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.json: \"text\": \"can you log into the remote postgres and show me the schema?\",\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.md:| `test_q15` | which postgres extension are we enabling before passive recall? | `Tools` | `Memory` | 0.63 | 0.62 | 0.55 |\n./benchmarks/models/router/centroid/out-router-iter-7/router_report.md:| `abs_test_015` | can you log into the remote postgres and show me the schema? | `Abstain` | `Tools` | 0.70 | 0.59 | 0.62 |\n./benchmarks/models/router/linear/out-router-linear-iter-1/router_report.json: \"text\": \"which postgres extension are we enabling bef..." + }, + { + "name": "shell", + "args": "{\"cmd\":\"ls -a\"}", + "status": "ok", + "result": "exit 0\n.\n..\n.cargo\n.codex\n.direnv\n.envrc\n.git\n.gitignore\nAGENTS.md\nCargo.lock\nCargo.toml\nbenchmarks\ndocs\nflake.lock\nflake.nix\nklbr-bench\nklbr-core\nklbr-daemon\nklbr-ipc\nklbr-tui\nrust-toolchain.toml\ntarget\n" + }, + { + "name": "read_file", + "args": "{\"path\":\".envrc\"}", + "status": "ok", + "result": "use flake\n" + }, + { + "name": "shell", + "args": "{\"cmd\":\"ls -F\"}", + "status": "ok", + "result": "exit 0\nAGENTS.md\nCargo.lock\nCargo.toml\nbenchmarks/\ndocs/\nflake.lock\nflake.nix\nklbr-bench/\nklbr-core/\nklbr-daemon/\nklbr-ipc/\nklbr-tui/\nrust-toolchain.toml\ntarget/\n" + } + ], + "final_response": "I cannot determine the exact PostgreSQL version being used by klbr from the current file system and environment, as `psql` is not available in the shell and there are no configuration files (like `.env` or direct database connection strings) explicitly visible in the root directory that specify a version.\n\nThe codebase does use `sqlx-postgres` (as seen in the `compact_str` dependency fingerprints), which means it is configured to work with PostgreSQL, but the specific version of the database server itself is not recorded in the project files I've inspected.", + "called_any_tool": true, + "called_any_required_tool": null + }, + { + "query_id": "dev_q14", + "split": "dev", + "text": "which browser am i using to inspect the benchmark output?", + "gold": "tools", + "required_tools": [], + "router_decision": "tools", + "router_scores": { + "tools": 0.5965446, + "memory": 0.22582869, + "abstain": 0.17762673, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "error: llm stream failed: error sending request for url (http://localhost:8001/v1/chat/completions)", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q15", + "split": "dev", + "text": "what gpu is running the reranker?", + "gold": "tools", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.5450381, + "memory": 0.28721434, + "abstain": 0.16774756, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": false, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q16", + "split": "dev", + "text": "which postgres extension did i enable for the benchmark database?", + "gold": "tools", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.405099, + "memory": 0.40849, + "abstain": 0.18641098, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": false, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q1", + "split": "dev", + "text": "what changed between the live agent and the benchmark harness?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.2814771, + "memory": 0.55412835, + "abstain": 0.16439459, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q2", + "split": "dev", + "text": "was i still using neovim after the editor switch?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.24448003, + "memory": 0.5681161, + "abstain": 0.18740383, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q7", + "split": "dev", + "text": "why do we need better benchmark data now?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.24923816, + "memory": 0.5371517, + "abstain": 0.21361016, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q8", + "split": "dev", + "text": "what sort of retrieval work keeps coming up?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.44633308, + "memory": 0.3212002, + "abstain": 0.23246671, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q10", + "split": "dev", + "text": "why did i add progress logging to the benchmark harness?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.28209952, + "memory": 0.563501, + "abstain": 0.15439948, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "test_q12", + "split": "dev", + "text": "what happened around the embedding split?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.16014922, + "memory": 0.70960313, + "abstain": 0.13024768, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q17", + "split": "dev", + "text": "what support threshold looked best in the focused support sweep?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.20070809, + "memory": 0.6685652, + "abstain": 0.13072677, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q18", + "split": "dev", + "text": "what changed after adding the support score to the policy?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.25271326, + "memory": 0.5965462, + "abstain": 0.1507405, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q19", + "split": "dev", + "text": "what kind of calibration fix are we avoiding because it would be brittle?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.21448901, + "memory": 0.62507653, + "abstain": 0.16043447, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q20", + "split": "dev", + "text": "how should passive recall connect to the current calibration work?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.5082828, + "memory": 0.31246716, + "abstain": 0.17924999, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q21", + "split": "dev", + "text": "which classifier model are we using to assign query types?", + "gold": "tools", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.4240074, + "memory": 0.3690778, + "abstain": 0.2069148, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": false, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + }, + { + "query_id": "dev_q22", + "split": "dev", + "text": "did the support-assisted sweep get coverage all the way back to the old 0.9167 balanced point?", + "gold": "memory", + "required_tools": [], + "router_decision": "memory", + "router_scores": { + "tools": 0.22530445, + "memory": 0.653091, + "abstain": 0.12160451, + "kind": "linear_softmax_v1(prob)" + }, + "router_correct": true, + "tool_iterations": 0, + "tool_calls": [], + "final_response": "", + "called_any_tool": false, + "called_any_required_tool": null + } +] \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.json b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.json new file mode 100644 index 0000000..d1bfac2 --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.json @@ -0,0 +1,25 @@ +{ + "dataset_id": "internal-eval-starter-v2", + "experiment_id": "mvp-rerank-support-calibrated", + "evaluated_split": "test", + "llm_url": "http://localhost:8001", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "router_model_path": "/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json", + "write_root": "/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/tool_lane_writes", + "metrics": { + "query_count": 16, + "gold_tools": 2, + "gold_memory": 14, + "gold_abstain": 0, + "router_accuracy": 0.875, + "router_tools_recall": 0.0, + "router_tools_precision": 0.0, + "router_memory_false_tools_rate": 0.0, + "router_abstain_false_tools_rate": 0.0, + "tools_lane_routed_count": 0, + "tools_lane_called_any_tool_rate": 0.0, + "tools_lane_required_tool_hit_rate": null, + "tools_lane_tool_exec_error_rate": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.md b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.md new file mode 100644 index 0000000..3ba374b --- /dev/null +++ b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/report.md @@ -0,0 +1,33 @@ +# Tools Lane Benchmark Report + +dataset: `internal-eval-starter-v2` +experiment: `mvp-rerank-support-calibrated` +split: `Some(Test)` +llm_url: `http://localhost:8001` +embed_url: `http://localhost:8002` +embed_model: `bge-m3-FP16.gguf` +router_model: `/home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json` +write_root: `/home/mayer/proj/klbr/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/tool_lane_writes` + +## Router Metrics (Gold = route_label) + +- queries: `16` (tools `2` / memory `14` / abstain `0`) +- router accuracy: `0.8750` +- tools precision: `0.0000` +- tools recall: `0.0000` +- memory→tools rate: `0.0000` +- abstain→tools rate: `0.0000` + +## End-to-End Tool Lane + +- queries routed to tools: `0` +- called any tool rate (given routed-to-tools): `0.0000` +- required tool hit rate (if defined): `n/a` +- tool exec error rate (given routed-to-tools): `0.0000` + +## Traces (Tools Gold or Tools Pred) + +| id | gold | pred | called_tool | required_hit | tools_score | memory_score | abstain_score | kind | +|---|---|---|---:|---:|---:|---:|---:|---| +| `test_q15` | `Tools` | `Memory` | 0 | 0 | 0.51 | 0.28 | 0.21 | `linear_softmax_v1(prob)` | +| `test_q9` | `Tools` | `Memory` | 0 | 0 | 0.47 | 0.34 | 0.19 | `linear_softmax_v1(prob)` | diff --git a/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/tool_lane_agent.db b/benchmarks/runs/2026-04-24_suite_smoke/tools-lane-test/tool_lane_agent.db new file mode 100644 index 0000000000000000000000000000000000000000..15b081b3be0950e3e58107204d1251712a22a083 GIT binary patch literal 40960 zcmWFz^vNtqRY=P(%1ta$FlG>7U}R))P*7lCU|?ooVBlgv0Colj1{MUDff0#~i^KNjx5aQ_MsSss85K3MrmGVMQTBQa)yScCW?zS8zni|#a&$) z8%@DMQpPQPSUzC|z9AB22T#{cDpPW&emtAaNfKQGCB_NgXDOJ)a$}i7MK?y~O zO~ndMKK@P$Fz0KAbFztNyD~Cl=B1=o6c^@XmZZifmX_p$`5@QfbZWdIJ{NE{s4MEf@hI3$sD6hCv7(rb8odw|N*Z7q$wSByjOP8~%&JtF7va&ZQ#l$o1Y zRH=}iTB)N@mY7qTs*sphsp-hVCLXF!RB8m7h?WFFLZCJ_2dK=7E;Ix6hebN{oEDGQj-lZORd!0q|}rYP^poYlb=`;ZD?R*604As zSzMBsmz)}(n_5zonQWV!U!0kjidrVLJ2A40%gQqLg@c_3D-U2a6VzRhLK7y6r`&{; zP9PFX&cH0rKuNW@ATb$SjDmE5i#-qnRx-c~HuaR$w8YY!5_L>dauO?3i$I#-7`>cD znBrrI#Q<3G4A+n03`|Q4GV}6Mp|u4>AI!%XPB*~RU6Pm%Ey*Ez6)I5|2Ni<48! zQu9jUOEPoeL5I_Nlqvz!DVcfcsl_Npk!un>8OG=4r=;d6;PX^zUUpu7c^)>O!4h+P zN@gxNba1){OTZ%BP?lO$oSC18Ct%C4rG?^>#FA2!xWS#65|c|Z%TlrUp*X*^C^|V$@lqAut*O zqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?+(SSD$1ng3776eu05diT&?o?tAhRag zxB+PXpJ8+W#Nn z31HOwqaiRF0;3@?8UmvsFd71*Aut*OqaiRF0;3@?8UiCC1V;P+BO;ndJvkZzqaiRF z0;3@?8UmvsFd71*Aut*OqaiRF0;3@?!b4!R|3AVLz^L~}Ltr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%&%fX@FL-Tyx#qI%SmqaiRF0;3@?8UmvsFd71*Aut*OqaiRF z0;3@?8UiCY1V-ooM{wkidVMqmMnhmU1V%$(Gz3ONU^E0qLtr!nMnhmU1V%$(#Do9< Dr~t B[Schema validation and deterministic ingest] + B --> C[Flat retrieval baseline] + C --> D[Router ablation] + D --> E[L1 time-window sweep] + E --> F[ANN K sweep] + F --> G[Reranker threshold sweep] + G --> H[Backlink policy sweep] + H --> I[Consolidation trigger sweep] + I --> J[Security and deletion tests] + J --> K[Final scorecard and operating point] +``` + +```mermaid +flowchart TD + PR[Pull request or nightly build] --> UT[Unit tests and schema migration tests] + UT --> IR[Deterministic ingest replay] + IR --> BENCH[Offline benchmark suite] + BENCH --> LME[LongMemEval, LoCoMo, PerLTQA] + BENCH --> RET[BEIR and MTEB] + BENCH --> SYS[YCSB-style backend load] + BENCH --> SEC[AgentPoison and MEXTRA] + LME --> SCORE[Unified scorecard with bootstrap CIs] + RET --> SCORE + SYS --> SCORE + SEC --> SCORE + SCORE --> GATE{Regression gate passed} + GATE -->|Yes| REL[Release candidate] + GATE -->|No| BLOCK[Block merge and open triage issue] +``` + +The runbook should be executed in this exact order: + +1. **Freeze the record and edge schema.** Until record identity, timestamps, provenance edges, supersession edges, and tombstones are explicit, every later benchmark can be invalidated by implementation drift. + +2. **Build the gold set before building the hierarchy.** Use benchmark adapters plus a small hand-labeled KLBR-native set that includes exact episodic queries, vague pattern questions, stable trait identity questions, user updates, and abstentions. LongMemEval and LoCoMo make these distinctions concrete. citeturn1search0turn1search1 + +3. **Train one flat baseline and never delete it.** It should remain in CI forever as the control arm. Dense-retrieve-plus-rerank is the right baseline because that pattern is well grounded in DPR, monoBERT, and modern retrieve-rerank practice. citeturn11search0turn11search1turn11search2 + +4. **Calibrate routing independently.** Treat routing as its own supervised problem before feeding it into end-to-end retrieval. That is exactly how PerLTQA structures the problem. citeturn1search2 + +5. **Calibrate L1 windows on temporal subsets only.** Do not let non-temporal queries dominate the search. Measure retrieval recall and latency as separate outcomes. + +6. **Introduce backlinks only after you can score them.** A backlink policy without provenance precision and support coverage metrics is just another uncontrolled retrieval path. + +7. **Benchmark backends on your actual access pattern.** YCSB-style mixes are useful, but also replay your real benchmark trace because vector filters, namespace cardinality, and consolidation writes matter more than generic KV throughput. citeturn8search0turn8search6 + +8. **Do not mark the system production-ready until delete and poisoning tests pass.** AgentPoison and MEXTRA show why this is a first-order requirement for memory systems rather than an afterthought. citeturn2search3turn2search14 + +## Roadmap and optional hardware exploration + +A sensible roadmap is **12 weeks for the semantic-memory core**, with a stretch path to 16–20 weeks if you decide to include deeper backend scaling work, formal privacy hardening, or hardware-tier experiments. + +| Window | Milestone | Main outputs | Release gate | +|---|---|---|---| +| Weeks 1–2 | Artifact freeze | Schemas, lifecycle contract, namespace plan, tombstone semantics, benchmark harness scaffold | All lifecycle tests pass locally | +| Weeks 3–4 | Flat baseline | Embedding/reranker bakeoff, exact and ANN flat baselines, first scorecard | Flat baseline reproducible and stable | +| Weeks 5–6 | Router and time windows | Routing classifier, entropy gates, L1 partitions, temporal sweep report | Router reduces cost without recall collapse | +| Weeks 7–8 | Backlinks and consolidation | Provenance edges, traversal policy, consolidation job runner, trigger sweep report | Layered design beats flat baseline on agreed subsets | +| Weeks 9–10 | Backend sweep | SQLite vec1 vs sqlite-vec vs Qdrant vs Faiss service wrapper vs Milvus/DiskANN candidates | Backend decision memo signed off | +| Weeks 11–12 | Security and CI/CD | Delete propagation tests, poisoning and leakage tests, nightly regression gates | No critical open privacy/security blocker | +| Weeks 13–16 | Stretch path | Real-trace hardening, multi-tenant rollout prep, larger-scale load tests | Internal pilot ready | +| Weeks 17–20 | Optional systems co-design | Only if justified: cold-tier disaggregation, PMem/NVM studies, accelerator placement | Decision memo on whether hardware-aware work is worth doing | + +The hardware and memory-system simulators you mentioned should stay **out of scope until backend measurements prove that hardware tiers are the bottleneck**. If that day comes, use gem5 for full-system architecture studies and coherence-aware experiments, DRAMsim3 or Ramulator 2.0 for DRAM-controller and memory-standard exploration, and NVMain for DRAM/NVM hybrid studies. These are excellent tools, but they are for the later question of *where the software stack should live*, not for the current question of *whether the semantic architecture is correct*. citeturn7search0turn7search8turn7search1turn7search2turn7search6turn7search3turn7search15 + +A good decision rule is simple: if your bottlenecks are still dominated by retrieval errors, reranker cost, consolidation policy, or delete propagation, do **not** start a hardware simulation track. If you later discover that memory-mapped vector indexes, NUMA effects, accelerator-side reranking, or cold-tier SSD/DAX placement dominate p95 latency or cost, then the hardware tools become worthwhile. + +## Risks, mitigations, and selected references + +The main risk is **calibration drift**. A layered memory system can look better simply because it hides mistakes in flattering summaries, while exact episodic failures become harder to notice. Mitigate that by keeping a permanent flat baseline, requiring provenance precision metrics for every backlink policy, and separately tracking update correctness and temporal consistency—the two long-memory failure modes that LongMemEval and LoCoMo make particularly visible. citeturn1search0turn1search1 + +The second risk is **synthetic-routing bias**. It is fine to bootstrap the classifier with synthetic queries, but do not freeze the routing model on synthetic data alone. PerLTQA’s decomposition strongly supports routing as a cheap first stage, but production routing needs real query traces, active-learning refreshes, and calibration checks. citeturn1search2 + +The third risk is **governance debt**. Append-only memory is excellent for provenance and debugging, but unsafe without namespace boundaries, tombstones, and derived-view invalidation. The closest product analogues—LangGraph memory namespaces and OpenAI memory controls—show why user-scoped storage and explicit deletion must be designed up front, not retrofitted later. citeturn10search0turn10search1turn10search13 + +The fourth risk is **security under adversarial memory use**. AgentPoison and MEXTRA show that long-term memory can be poisoned or extracted even when the surrounding application seems benign. The mitigation is to make trust metadata, source provenance, namespace isolation, redaction, and benchmarked security tests part of the mainline roadmap rather than the security backlog nobody reaches. citeturn2search3turn2search14 + +Selected primary references for the KLBR workstream are below. + +- LongMemEval: Benchmarking Chat Assistants on Long-Term Interactive Memory. citeturn1search0 +- LoCoMo: Evaluating Very Long-Term Conversational Memory of LLM Agents. citeturn1search1 +- PerLTQA: A Personal Long-Term Memory Dataset for Memory Classification, Retrieval, and Synthesis in Question Answering. citeturn1search2 +- BEIR: A Heterogeneous Benchmark for Zero-shot Evaluation of Information Retrieval Models. citeturn1search3 +- MTEB: Massive Text Embedding Benchmark. citeturn2search0 +- Dense Passage Retrieval for Open-Domain Question Answering. citeturn11search0 +- Multi-Stage Document Ranking with BERT. citeturn11search1 +- SQLite `vec1` official documentation. citeturn0search0turn0search4 +- `sqlite-vec` repository and release state. citeturn0search1turn0search13 +- Qdrant indexing, filtering, storage, and distributed deployment docs. citeturn0search2turn6search1turn5search3 +- Milvus architecture and component docs. citeturn6search2turn6search6turn5search10 +- Faiss documentation and repository. citeturn5search4turn5search0turn6search11 +- DiskANN paper and repository. citeturn5search5turn5search9 +- LightMem and Letta sleep-time references. citeturn4search0turn4search2turn4search14 +- AgentPoison and MEXTRA. citeturn2search3turn2search14 +- YCSB and MLPerf Inference. citeturn8search0turn8search5turn8search8 \ No newline at end of file diff --git a/docs/klbr_benchmark_readout_2026-04-22.md b/docs/klbr_benchmark_readout_2026-04-22.md new file mode 100644 index 0000000..84f7b8a --- /dev/null +++ b/docs/klbr_benchmark_readout_2026-04-22.md @@ -0,0 +1,370 @@ +# KLBR benchmark readout — 2026-04-22 + +## Bottom line + +These results are strong for an MVP. The split-leakage fix appears to have worked: first-stage retrieval is meaningfully harder now, while the reranker is doing real work instead of just confirming the initial ranking. + +Key outcomes from the current report: + +- Recall@1: 0.7500 +- MRR: 0.8202 +- Top1 accuracy after rerank: 0.9167 +- MRR after rerank: 0.9444 +- Reranker lift in MRR: +0.1242 +- Retrieval latency p50/p95: 1 ms / 2 ms +- Embedding latency p50/p95: 9 ms / 11 ms +- Rerank latency p50/p95: 63 ms / 118 ms +- Window expansions: 9 across 12 queries +- Time-window hit rate: 0.8333 + +Interpretation: + +1. The reranker is justified. +2. Exact search is not the bottleneck yet. +3. The next sprint should focus on calibration and metric semantics, not retrieval architecture. + +## What the results mean + +### 1) Reranking is now clearly valuable + +The main quality gain is post-rerank. Moving from MRR 0.8202 to 0.9444 with Top1-after-rerank at 0.9167 is exactly the behavior you want after removing leakage: first-stage retrieval becomes less artificially easy, and the reranker becomes the mechanism that recovers answer quality. + +This means the current architecture is already proving the intended two-stage retrieval shape. + +### 2) Do not optimize vector search yet + +Retrieval remains extremely cheap compared with reranking: + +- retrieval p95: 2 ms +- rerank p95: 118 ms + +So the system is currently reranker-bound, not retrieval-bound. That means: + +- do not spend time tuning ANN indexing yet +- do not switch benchmarking focus to USearch indexing parameters yet +- do not overfit first-stage retrieval thresholds yet + +Keep exact search as the benchmark baseline until either corpus size increases substantially or retrieval p95 stops being negligible. + +### 3) Expansion appears over-eager + +The report shows 9 window expansions over only 12 queries, while time-window hit rate is 0.8333. That strongly suggests expansion is currently firing much more often than true initial-window failure would justify. + +This is not yet a latency problem because retrieval is cheap, but it is a calibration problem and will matter later at scale. + +## Most important architectural mismatch revealed by the benchmark + +Your design note says window expansion should be driven by reranker confidence: + +- if max reranker score is low, widen and retry +- only then abstain or fall back + +But the current experiment config is still framed around first-stage distance thresholds: + +- `expand_distance_threshold: 0.35` +- `abstain_distance_threshold: 0.45` + +That is an implementation drift from the design. Right now the benchmark is effectively making important decisions using pre-rerank evidence, even though the benchmark results show reranking is the stage that is actually improving answer quality. + +This should be treated as the top-priority fix. + +## Query-level diagnosis + +### dev_q3 is not a clean retriever miss + +`dev_q3` (“what have i been working on lately?”) looks more like a label/query ambiguity problem than a fundamental retrieval failure. + +What happens now: + +- the intended gold memories are 22 and 23 +- first stage retrieves them, but only at ranks 7 and 8 +- reranking improves one of them (22 to rank 3) +- memory 10 remains rank 1 after rerank + +The problem is that memory 10 is also semantically plausible: + +> "recent work has mostly been retrieval plumbing, instrumentation, and benchmark scaffolding." + +That is a perfectly reasonable answer to a vague “what have i been working on lately?” query. + +So this example should **not** be treated as strong evidence that the retriever is broken. It is at least partly one of these: + +- the gold set is too narrow +- the query is too underspecified +- the task should be graded as multi-answer / partial-credit instead of strict top-1 + +Recommended action: + +- review `dev_q3` manually +- either add memory 10 to the accepted gold set +- or rewrite the query to target the intended evidence more explicitly +- or score it using set-valued / partial-credit grading + +### test_q12 confirms decision semantics need to move post-rerank + +From your own readout, `test_q12` is recovered by rerank but still marked failed because abstention is based on first-stage distance instead of final reranked confidence. + +That is exactly the kind of bug that appears when decision logic is attached to the wrong stage. + +Recommended action: + +- make the benchmark compute a single canonical **final decision** after rerank +- only then decide `answer` vs `abstain` +- do not let pre-rerank distance directly determine benchmark success/failure + +## P0: what to fix before tuning anything else + +### 1) Make final decision semantics explicit + +The benchmark should separate these concepts: + +- **candidate retrieval** +- **reranked ranking** +- **final decision** +- **metric grading** + +Suggested order: + +1. Retrieve candidates +2. Rerank them +3. Compute final decision confidence +4. Decide answer / abstain +5. Grade the final answer against gold + +That means: + +- `abstained` must depend on final post-rerank confidence +- `passed` must reflect final decision correctness +- `selective_accuracy` must use final correctness, not “gold appeared somewhere in candidates” + +### 2) Log one canonical confidence bundle + +For every query, log at minimum: + +- `first_stage_top1_distance` +- `first_stage_topk_contains_gold` +- `rerank_top1_raw_score` +- `rerank_top1_normalized_score` +- `rerank_top2_raw_score` +- `rerank_margin_raw` +- `rerank_margin_normalized` +- `expanded_window` +- `final_decision = answer|abstain` +- `final_answer_memory_ids` +- `final_correct = true|false` + +Without this, threshold tuning will stay muddy. + +### 3) Resolve the normalization/logging mismatch + +The config says `rerank_normalize: true`, but traces are still showing large negative/positive values that look like raw logits. + +That may mean one of three things: + +- normalization is not actually being applied +- normalization is applied internally but raw scores are what gets logged +- ranking uses one score space while decision logic uses another + +Before any threshold tuning, make the score space unambiguous: + +- either use normalized reranker scores everywhere for decisions and logs +- or log both raw and normalized values and declare which one drives decisions + +## P1: metric cleanup + +### Replace the current selective metric + +The current “selective accuracy” is overstated if it only requires: + +- some gold appeared in first-stage candidates +- and the run did not abstain + +That is not a final-answer metric. + +Replace it with one or more of these: + +- `final_top1_accuracy` +- `final_any_gold_in_topk_after_rerank` +- `coverage` = fraction of queries not abstained +- `precision_at_coverage` +- `risk_coverage_auc` + +### Add query-type slices + +Break metrics out by category: + +- exact dated event +- exact recent event +- vague recent lookup +- recurring pattern query +- conflict/update query +- multi-evidence query +- temporally ambiguous query +- no-hit query + +This matters because threshold behavior will differ across categories. + +### Add support metrics for multi-evidence queries + +For set-valued questions, top-1 alone is too strict. + +Add: + +- `support_recall_at_3` +- `support_recall_at_5` +- `support_precision_at_k` + +This will make queries like `dev_q3`, `dev_q11`, `dev_q12`, and `test_q7` easier to evaluate fairly. + +## P2: dataset cleanup before real calibration + +### 1) Put no-hit back into dev + +Right now `abstain_accuracy = 0.0000` is not informative because the dev slice has no real no-hit cases. + +Minimum fix: + +- add at least 3–5 no-hit queries to dev + +Better fix: + +- add 8–10 no-hit queries spanning + - unrelated topic + - wrong time period + - wrong entity + - impossible request + - deliberately misleading phrasing + +### 2) Audit ambiguous labels + +Before sweeping thresholds, manually review the few “misses” and ask: + +- is this truly a retrieval miss? +- is the query underspecified? +- is the gold set incomplete? +- should this be multi-answer graded? + +`dev_q3` is the strongest current candidate for relabeling or partial credit. + +### 3) Add a tiny held-out calibration split + +Do not tune thresholds directly on the same tiny dev slice forever. + +Recommended minimum split after cleanup: + +- dev-train: for engineering iteration +- dev-calibration: for threshold selection +- held-out test: untouched until thresholds are fixed + +Even if the dataset is still small, this will stop threshold chasing. + +## P3: actual calibration procedure + +Once P0 and P1 are fixed, use this calibration loop. + +### Step 1: freeze architecture + +Do **not** change these during calibration: + +- embedding model +- reranker model +- retrieval mode +- top-k candidate counts +- text chunking/memory formatting + +Calibration should operate on a fixed retrieval stack. + +### Step 2: define the decision features + +Use a small hand-designed rule first. + +Best starting features: + +- `rerank_top1_normalized_score` +- `rerank_margin_normalized` +- `first_stage_top1_distance` +- `expanded_window` +- `query_category` + +### Step 3: start with a threshold rule, not a learned calibrator + +Because the dataset is still tiny, begin with a simple rule such as: + +- abstain if `rerank_top1_norm < t1` +- abstain if `rerank_margin_norm < t2` +- optionally abstain if `expanded_window == true` and `rerank_top1_norm < t3` + +Then do a small grid sweep. + +### Step 4: optimize for precision under coverage constraints + +Do not optimize a single scalar blindly. + +Pick an operating target such as: + +- maximize answered-query accuracy +- subject to answer precision >= 95% +- while keeping coverage as high as possible + +That will give a more production-relevant threshold than maximizing raw accuracy. + +### Step 5: only later consider learned calibration + +Once you have a larger dev-calibration set, you can try: + +- Platt scaling +- isotonic regression +- tiny logistic regression on the decision features + +But do not do this yet with only 12 queries. + +## Recommended next sprint order + +### Sprint A — metric correctness + +1. Move abstention to post-rerank final decision +2. Fix `passed` +3. Fix `selective_accuracy` +4. Log both raw and normalized reranker scores +5. Add final-decision metrics + +### Sprint B — dataset and label cleanup + +1. Add no-hit queries to dev +2. Audit ambiguous examples, especially `dev_q3` +3. Split out multi-evidence support metrics + +### Sprint C — threshold tuning + +1. Freeze retrieval config +2. Sweep rerank-based abstain thresholds +3. Sweep rerank-margin thresholds +4. Compare precision / coverage / risk curves +5. Lock thresholds + +### Sprint D — only then revisit retrieval changes + +Only after the above should you consider: + +- first-stage retrieval tuning +- indexed USearch benchmarking +- different candidate counts +- classifier introduction +- L2 rollout + +## Final recommendation + +Treat this run as a success. + +The system has crossed the important line where: + +- first-stage retrieval is realistic +- reranking provides measurable value +- latency is still well within MVP territory + +That means the correct next move is **not** “improve retrieval architecture.” +It is: + +> make the benchmark semantically correct, clean up the evaluation set, and calibrate the final decision on post-rerank confidence. + +Once that is done, the next benchmark run will tell you something much more trustworthy about the actual retrieval stack. diff --git a/docs/klbr_benchmark_readout_2026-04-23.md b/docs/klbr_benchmark_readout_2026-04-23.md new file mode 100644 index 0000000..f371fd8 --- /dev/null +++ b/docs/klbr_benchmark_readout_2026-04-23.md @@ -0,0 +1,314 @@ +# KLBR Benchmark Readout - 2026-04-23 + +This file captures the current "latest known good" benchmark results and the exact configs that produced them, after the dataset expansions and the support-assisted decision policy work. + +## Retrieval (Active Question -> Memory Answer/Abstain) + +### Datasets / Runs + +- Dataset: `benchmarks/inputs/datasets/internal_eval_starter.json` (`dataset_id: internal-eval-starter-v2`) +- Baseline run (no rerank-based abstain policy): `benchmarks/runs/legacy/out-expanded-baseline/report.md` +- Support-calibrated run: `benchmarks/runs/legacy/out-expanded-support-calibrated/report.md` +- Support sweep: `benchmarks/runs/legacy/out-expanded-support-sweep/sweep_report.md` +- Fresh rerun (dev): `benchmarks/runs/legacy/out-2026-04-23-retrieval-dev/report.md` +- Fresh rerun (test): `benchmarks/runs/legacy/out-2026-04-23-retrieval-test/report.md` +- Latest rerun (dev): `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-dev/report.md` +- Latest rerun (test): `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test/report.md` +- Latest rerun (test, tool-lane filtered): `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test-no-tools/report.md` + +### Baseline (Exact Retrieval; No Calibrated Decision Policy) + +From `benchmarks/runs/legacy/out-expanded-baseline/report.md`: + +- queries: `22`, memories ingested: `34` +- final decision accuracy: `0.6364` (coverage `1.0000`) +- rerank latency p50/p95: `90` / `230` ms +- no-hit false answers: `5` (see the "No-Hit False Answers" section in the report) + +This is the "diagnosis" baseline: it shows retrieval and rerank ranking quality is decent, but the system is still willing to answer no-hit queries. + +### Support-Calibrated (Decision = Score + Support; Expansion Enabled) + +From `benchmarks/runs/legacy/out-expanded-support-calibrated/report.md`: + +- final decision accuracy: `0.9091` +- coverage: `0.6818` (selective accuracy `1.0000`) +- window expansions: `9` +- rerank latency p50/p95: `97` / `535` ms +- no-hit false answers: `0` + +Trace-level behavior (from `benchmarks/runs/legacy/out-expanded-support-calibrated/traces.json`): + +- abstained query ids: `dev_q3, dev_q13, dev_q14, dev_q15, dev_q16, test_q7, dev_q21` +- expanded query ids: `dev_q3, dev_q5, dev_q13, dev_q14, dev_q15, dev_q16, test_q2, test_q7, dev_q21` + +### Fresh Rerun Snapshot (2026-04-23) + +From `benchmarks/runs/legacy/out-2026-04-23-retrieval-dev/report.md` (dev): + +- final decision accuracy: `0.8182` +- coverage: `0.7727` +- rerank latency p50/p95: `77` / `288` ms +- no-hit false answers: `1` (`dev_q13`) + +From `benchmarks/runs/legacy/out-2026-04-23-retrieval-test/report.md` (test): + +- final decision accuracy: `0.8333` +- coverage: `0.9444` +- rerank latency p50/p95: `63` / `108` ms +- no-hit false answers: `3` (`dev_q9`, `test_q9`, `test_q15`) + +### Latest Rerun Snapshot (2026-04-23) + +From `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-dev/report.md` (dev): + +- final decision accuracy: `0.9091` +- coverage: `0.6818` +- rerank latency p50/p95: `89` / `325` ms +- no-hit false answers: `0` + +From `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test/report.md` (test): + +- final decision accuracy: `0.8333` +- coverage: `0.8333` +- rerank latency p50/p95: `70` / `251` ms +- no-hit false answers: `2` (`test_q9`, `test_q15`) + +From `benchmarks/runs/legacy/out-2026-04-23-latest-retrieval-test-no-tools/report.md` (test, tool-lane filtered): + +- note: excludes queries with `route_label=tools` (owned by the router/tooling path, not memory retrieval) +- final decision accuracy: `0.9375` +- coverage: `0.8125` +- no-hit false answers: `0` + +### Current Retrieval Operating Point (Config) + +Config used by the calibrated retrieval run: + +- `benchmarks/inputs/configs/mvp_rerank_support_calibrated.json` +- key thresholds: + - `rerank_abstain_score_threshold: -6.0` + - `rerank_abstain_margin_threshold: 0.0` + - `support_abstain_threshold: 0.2` + - window schedule: `7 -> 30 -> 90 -> all` + +Support sweep best point (matches the above thresholds): + +- `benchmarks/runs/legacy/out-expanded-support-sweep/sweep_report.md` +- score: `-6.0`, margin: `0.0`, support: `0.2` + +## Passive Recall (Statement/Chat -> Decide Whether to Inject Memories) + +### Datasets / Runs + +- Dataset: `benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json` (`dataset_id: internal-eval-passive-recall-v3`) +- Dev: `benchmarks/runs/legacy/out-passive-recall-v3-final/report.md` +- Test: `benchmarks/runs/legacy/out-passive-recall-v3-final-test/report.md` +- Fresh rerun (dev): `benchmarks/runs/legacy/out-2026-04-23-passive-dev/report.md` +- Fresh rerun (test): `benchmarks/runs/legacy/out-2026-04-23-passive-test/report.md` +- Latest rerun (dev): `benchmarks/runs/legacy/out-2026-04-23-latest-passive-dev/report.md` +- Latest rerun (test): `benchmarks/runs/legacy/out-2026-04-23-latest-passive-test/report.md` + +### Dev Split (Final) + +From `benchmarks/runs/legacy/out-passive-recall-v3-final/report.md`: + +- should-recall / should-not: `16` / `5` +- activation accuracy/precision/recall: `1.0000` / `1.0000` / `1.0000` +- support precision@k mean: `0.8125` +- support recall@k mean: `0.8021` +- rerank latency p50/p95: `70` / `260` ms + +### Test Split (Final) + +From `benchmarks/runs/legacy/out-passive-recall-v3-final-test/report.md`: + +- activation accuracy/precision/recall: `0.9048` / `0.9375` / `0.9375` +- no-recall false activation rate: `0.2000` (1 false activation) +- support precision@k mean: `0.8750` +- support recall@k mean: `0.9062` +- false activation: `pr_test_14` (see report for the top-score/margin/support and recalled ids) + +### Fresh Rerun Snapshot (2026-04-23) + +From `benchmarks/runs/legacy/out-2026-04-23-passive-dev/report.md` (dev): + +- activation accuracy/precision/recall: `1.0000` / `1.0000` / `1.0000` +- support precision@k mean: `0.8125` +- support recall@k mean: `0.8021` +- rerank latency p50/p95: `71` / `274` ms + +From `benchmarks/runs/legacy/out-2026-04-23-passive-test/report.md` (test): + +- activation accuracy/precision/recall: `0.9048` / `0.9375` / `0.9375` +- no-recall false activation rate: `0.2000` (`pr_test_14`) +- support precision@k mean: `0.8750` +- support recall@k mean: `0.9062` +- rerank latency p50/p95: `82` / `275` ms + +### Latest Rerun Snapshot (2026-04-23) + +From `benchmarks/runs/legacy/out-2026-04-23-latest-passive-dev/report.md` (dev): + +- activation accuracy/precision/recall: `0.9524` / `1.0000` / `0.9375` +- no-recall false activation rate: `0.0000` +- support precision@k mean: `0.8444` +- support recall@k mean: `0.7708` +- rerank latency p50/p95: `74` / `277` ms + +From `benchmarks/runs/legacy/out-2026-04-23-latest-passive-test/report.md` (test): + +- activation accuracy/precision/recall: `0.9048` / `0.9375` / `0.9375` +- no-recall false activation rate: `0.2000` (`pr_test_14`) +- support precision@k mean: `0.8750` +- support recall@k mean: `0.9062` +- rerank latency p50/p95: `89` / `295` ms + +## Router (Tools vs Memory vs Abstain; Memory Safety Gate) + +Current recommended router model: + +- `benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json` +- snapshot: `docs/router_linear_latest_2026-04-23.md` + +### Current Passive Operating Point (Config) + +Same base thresholds as retrieval, plus passive-specific gates: + +- `benchmarks/inputs/configs/mvp_rerank_support_calibrated.json` (dev) +- `benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json` (test) +- passive-specific fields: + - `passive_support_score_gap: 5.0` + - `rerank_confident_score_threshold: -4.0` + +## Notes / Known Gaps + +- "Support precision improved a lot" is real (starter passive runs were ~0.36 support precision@k mean; v3 final is ~0.81-0.88), but the current "final" reports in this repo do not show `0.93` support precision. If you have a different run directory that produced `0.93`, add it under `benchmarks/runs/` and we can capture it here too. +- "Support precision improved a lot" is real (starter passive runs were ~0.36 support precision@k mean; v3 final is ~0.81-0.88), but the current "final" reports in this repo do not show `0.93` support precision. If you have a different run directory that produced `0.93`, add it under `benchmarks/runs/` and we can capture it here too. + +## Latest Rerun Snapshot (2026-04-23) + +These runs include the updated, language-agnostic support scoring: + +- IDF-weighted character 4-gram query coverage +- multiplied by a "single-token overlap" penalty (to reduce no-hit false answers driven by only generic overlap) +- tokenization uses Unicode word segmentation (UAX#29), no stemming/stopwords + +### Retrieval (Active) Rerun + +Config: `benchmarks/inputs/configs/mvp_rerank_support_calibrated.json` (dev) and `benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json` (test). + +Dev report: `benchmarks/runs/legacy/out-2026-04-23p-retrieval-dev/report.md` + +- final decision accuracy: `0.9091` +- coverage: `0.6818` +- no-hit false answers: `0` +- embedding latency p50/p95: `11` / `14` ms +- rerank latency p50/p95: `109` / `351` ms + +Test report: `benchmarks/runs/legacy/out-2026-04-23p-retrieval-test/report.md` + +- final decision accuracy: `0.8333` +- coverage: `0.8333` +- no-hit false answers: `2` (`test_q9`, `test_q15`) +- embedding latency p50/p95: `11` / `14` ms +- rerank latency p50/p95: `78` / `429` ms + +### Passive Recall Rerun + +Dataset: `benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json`. + +Dev report: `benchmarks/runs/legacy/out-2026-04-23p-passive-dev/report.md` + +- activation accuracy/precision/recall: `0.9524` / `1.0000` / `0.9375` +- no-recall false activation rate: `0.0000` +- support precision@k mean: `0.8444` +- support recall@k mean: `0.7708` +- rerank latency p50/p95: `120` / `452` ms + +Test report: `benchmarks/runs/legacy/out-2026-04-23p-passive-test/report.md` + +- activation accuracy/precision/recall: `0.9048` / `0.9375` / `0.9375` +- no-recall false activation rate: `0.2000` (`pr_test_14`) +- support precision@k mean: `0.8750` +- support recall@k mean: `0.9062` +- rerank latency p50/p95: `123` / `456` ms + +### Discarded Variant Note + +We also tried a "top-IDF-grams coverage" support factor; it eliminated no-hit false answers but caused a big coverage/latency collapse due to aggressive expansion (see `benchmarks/runs/legacy/out-2026-04-23q-retrieval-dev` and `benchmarks/runs/legacy/out-2026-04-23q-retrieval-test`). Keep it as an experiment trace, not as an operating mode. + +## Router Prototype (2026-04-23) + +To start addressing "tool-needed / system-state" queries without hardcoding English patterns, we added an optional `route_label` field to the eval dataset and implemented a first router prototype in `klbr-bench`: + +- model: centroid classifier over query embeddings (memory centroid vs tools centroid) +- decision: `tools` if `sim(q, tools) - sim(q, memory) >= threshold` (threshold tuned on dev) + +Run outputs: + +- `benchmarks/runs/legacy/out-2026-04-23p-router2/router_report.md` +- `benchmarks/runs/legacy/out-2026-04-23p-router2/router_model.json` + +Current result (very small labeled slice): 1.0 accuracy on 16 labeled test queries with 2 tools-labeled items. Treat this as a plumbing check; we need more labeled `route_label: tools` and `route_label: abstain` queries before trusting it. + +## Router Iter-1 (2026-04-23) — Dataset Expansion + Runtime Wiring + +### Dataset Slices Created + +| file | route_label | dev | test | +|------|------------|-----|------| +| `benchmarks/inputs/router/router_slices/shell_state_v1.json` | tools | 15 | 15 | +| `benchmarks/inputs/router/router_slices/read_file_repo_v1.json` | tools | 15 | 15 | +| `benchmarks/inputs/router/router_slices/abstain_v1.json` | abstain | 15 | 15 | +| `benchmarks/inputs/router/router_slices/memory_lane_v1.json` | memory | 15 | 15 | + +Tool inventory snapshot: `benchmarks/inputs/router/router_tools.json` + +### Runtime Wiring + +- Added `klbr-core/src/router.rs`: centroid classifier, loads `router_model.json`, classifies raw query embeddings +- `Config.router_model_path: Option`: `None` = disabled (safe default); set to a `router_model.json` path to enable +- In `agent.rs` `UserMessage` handler: if router loaded and classifies as `Tools`, memory recall is skipped entirely and `AgentEvent::Status("routed: tool lane")` is emitted; the LLM still runs with tools available + +### Run command (iter-1) + +``` +nix develop --command cargo run -p klbr-bench -- router-multi \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/models/router/centroid/out-router-iter-1 \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/router/router_slices/shell_state_v1.json \ + benchmarks/inputs/router/router_slices/read_file_repo_v1.json \ + benchmarks/inputs/router/router_slices/memory_lane_v1.json +``` + +### Results (iter-1) + +Report: `benchmarks/models/router/centroid/out-router-iter-1/router_report.md` + +| metric | value | +|--------|-------| +| labeled queries (test) | 61 (32 tools, 29 memory) | +| tools precision | **1.0000** ✅ | +| tools recall | 0.3125 ⚠️ | +| memory false-tools rate | **0.0000** ✅ | +| accuracy | 0.6393 | +| threshold | 0.100 | + +**Analysis:** + +- precision gate met: 1.0 precision means zero memory queries were misrouted to tools — the centroid is correctly biased toward memory +- recall is 0.31 because ~70% of test-split tool queries are landing on the memory side — the dev-trained centroid is not pulling far enough toward the tools cluster at test time +- this is a classic centroid/distribution shift: dev and test tool queries are linguistically similar enough that the separator sits in a bad spot for the test half +- memory false-tools rate is 0.0 — the no-degrade-memory constraint is fully met + +**Next steps to improve recall:** + +1. add more `route_label: tools` test-split examples that cover the failing queries (inspect which test IDs the centroid misses) +2. try a write_file slice (`write_file_v1.json`) to thicken the tools cluster +3. the current trainer uses all dev queries for centroid fitting — consider adding a soft margin or per-class up-weighting for the minority class + +### Notable failures + +~22 tool-lane test queries classified as memory (recall = 0.31 → 10/32 correct). The read_file and shell test splits are the likely culprits since their phrasings overlap with memory-lane question style. Need to inspect traces — `router_model.json` doesn't emit per-query decisions yet. diff --git a/docs/klbr_benchmark_readout_2026-04-23_end2end.md b/docs/klbr_benchmark_readout_2026-04-23_end2end.md new file mode 100644 index 0000000..30358c2 --- /dev/null +++ b/docs/klbr_benchmark_readout_2026-04-23_end2end.md @@ -0,0 +1,127 @@ +# KLBR End-to-End Benchmark Readout - 2026-04-23 + +This file captures the latest active (question -> memory answer/abstain) and passive (statement -> decide recall injection) benchmark runs, along with the exact configs used. + +## Active Recall (Retrieval) — Latest + +Dataset: `benchmarks/inputs/datasets/internal_eval_starter.json` (`internal-eval-starter-v2`) + +### Dev Split + +Output: `benchmarks/runs/legacy/out-2026-04-23-end2end-retrieval-dev/` + +From `report.md`: + +- queries: `17` +- final decision accuracy: `0.8824` +- coverage: `0.8824` (selective accuracy `1.0000`) +- no-hit false answers: `0` +- rerank latency p50/p95 ms: `91` / `315` +- embedding latency p50/p95 ms: `8` / `9` + +### Test Split + +Output: `benchmarks/runs/legacy/out-2026-04-23-end2end-retrieval-test/` + +From `report.md`: + +- queries: `16` +- final decision accuracy: `0.9375` +- coverage: `0.8125` (selective accuracy `1.0000`) +- no-hit false answers: `0` +- rerank latency p50/p95 ms: `75` / `289` +- embedding latency p50/p95 ms: `9` / `11` + +## Passive Recall — Latest + +Dataset: `benchmarks/inputs/datasets/internal_eval_passive_recall_v3.json` (`internal-eval-passive-recall-v3`) + +### Dev Split + +Output: `benchmarks/runs/legacy/out-2026-04-23-end2end-passive-dev/` + +From `report.md`: + +- activation accuracy: `0.9524` +- activation precision/recall: `1.0000` / `0.9375` +- no-recall false activation rate: `0.0000` +- support precision@k mean: `0.8444` +- support recall@k mean: `0.7708` +- rerank latency p50/p95 ms: `74` / `272` + +### Test Split + +Output: `benchmarks/runs/legacy/out-2026-04-23-end2end-passive-test/` + +From `report.md`: + +- activation accuracy: `0.9048` +- activation precision/recall: `0.9375` / `0.9375` +- no-recall false activation rate: `0.2000` +- support precision@k mean: `0.8750` +- support recall@k mean: `0.9062` +- rerank latency p50/p95 ms: `78` / `269` +- false activations: + - `pr_test_14` (dotfiles syncer context switch) recalled `131, 118, 130` + +## Configs Used + +### Dev (`benchmarks/inputs/configs/mvp_rerank_support_calibrated.json`) + +```json +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} +``` + +### Test (`benchmarks/inputs/configs/mvp_rerank_support_calibrated_test.json`) + +```json +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "test", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} +``` diff --git a/docs/klbr_memory_design.md b/docs/klbr_memory_design.md new file mode 100644 index 0000000..49a8664 --- /dev/null +++ b/docs/klbr_memory_design.md @@ -0,0 +1,215 @@ +# KLBR Agent Memory System +*Design Notes — Work In Progress* + +--- + +## Overview + +Hierarchical memory system for the KLBR agent. Core ideas: layered memory consolidation (episodic → abstract), two-stage retrieval (ANN + reranker), and a lightweight classifier to route queries to the right layers. Memories are never deleted — only archived and superseded by consolidation. + +--- + +## Memory Hierarchy + +Memories are organized into layers of increasing abstraction. Consolidation creates higher layers from lower ones during downtime, analogous to sleep-based memory consolidation in humans. The deeper the layer, the more a memory represents a stable "core fact" or personality trait built up over time. + +**Layer 1 (Episodic)** — raw events. "fixed headscale nginx config on tuesday." high volume, time-indexed, specific. + +**Layer 2 (Pattern)** — recurring behaviors. "does a lot of nixos infrastructure work." synthesized from clusters of L1 memories. + +**Layer 3 (Trait)** — stable characteristics. "systems-oriented, self-hosts everything." synthesized from L2 patterns. + +**Layer 4 (Core)** — fundamental values/identity. "values ownership and control over tooling." very slow to change. + +This mirrors how transformer layers themselves work — early layers capture surface features, deep layers capture abstract concepts. The memory hierarchy has the same shape. + +### Consolidation Process + +During downtime (time-based, count-based, or similarity-density-triggered), similar memories within a layer become candidates for consolidation into the layer above. + +Two-stage similarity check for consolidation candidates: +- cosine similarity as a cheap first filter (geometrically close) +- cross-encoder reranking to verify contextual coherence (actually related, not just geometrically close) + +Consolidation is **additive, never destructive**: +- source memories are archived, not deleted +- the new higher-layer memory stores backlinks (provenance links) to its sources +- archived memories remain accessible via explicit provenance traversal, but are not surfaced by normal search + +Risk: lossy compression at each consolidation step. The synthesizing LLM will drop specific details. This is intentional for higher layers but means the original episodic record matters for precise recall. Originals should be kept in cold storage with a flag rather than hard-deleted. + +Consolidation triggers to consider: time-based (nightly), count-based (every N new memories), or similarity pressure (when a cluster exceeds a density threshold). + +--- + +## Retrieval Pipeline + +Two-stage: fast wide ANN pass, then slow precise reranker pass. + +``` +query + | + v +classifier --> layer weight distribution + e.g. {L1: 0.8, L2: 0.1, L3: 0.05, L4: 0.05} + | + v +proportional K allocation across layers + (total K=20 -> pull 16 from L1, 2 from L2, 1 from L3, 1 from L4) + | + v +cosine ANN within each layer (fast, approximate) + | + v +cross-encoder reranker over combined ~20 candidates (slow, precise) + | + v +top-N results injected into context +``` + +### Layer Classifier + +A tiny offline-trained classifier decides layer weights at query time. Must be fast (microseconds, cpu-only) since it runs on every query. Classifying by layer at query time avoids casting wide across all layers on every retrieval — important at scale. + +Implementation: +- generate ~200-300 synthetic labeled queries per class offline using an LLM +- embed queries with the same embedding model used for memories +- train logistic regression over those embeddings (scikit-learn) +- serialize to ONNX, load in Rust via the `ort` crate +- inference = matrix multiply, no GPU needed + +The classifier outputs a soft distribution over layers, not a hard label, enabling proportional K allocation. Training data is distilled from an LLM offline so there's zero inference cost at runtime. + +Why not embedding-based centroid routing: centroid similarity detects *topical* similarity (what the query is about) but not *structural intent* (temporal vs abstract). "What did I do last tuesday" needs to match on temporal intent, not content — the centroid of L1 is about specific events, not about the concept of time. The classifier learns this boundary explicitly from labeled examples; centroid routing can't. + +Why not LLM-based routing: slow, burns tokens on every query. + +Why not pure heuristics: covering maybe 80% of cases but brittle. Worth using as a fallback or sanity check on classifier output, not as the primary method. + +### L1 Time-Windowed Search + +L1 (episodic) memories grow fastest and are inherently time-indexed, so full L1 search is avoided: + +- explicit temporal signal ("last tuesday", "recently") → specific window +- no temporal signal → default window (e.g. last 30 days) +- expanding window fallback: if max reranker score < threshold, widen and retry +- if still low confidence after expanding → fall back to L2 search (the episode may have been consolidated) + +This keeps each ANN search over a small partition rather than all of L1. + +--- + +## Backlink Traversal + +When a consolidated memory is retrieved, its backlinks provide a pre-filtered candidate set of source memories. This replaces ANN search for the lower layer entirely — cosine search is only for when you don't know where to look. + +``` +L1 search: low confidence or empty + | + v +L2 search: high confidence hit + | + v +fetch L2 backlinks -> small set of L1 source memories (no ANN needed) + | + v +rerank those L1s against original query + | + v +return best match +``` + +Key rules: +- **only follow backlinks on high-confidence L2 hits** +- low confidence L2 → do not traverse, return nothing +- add a depth limit on traversal to avoid walking the full memory graph + +Rationale for the confidence rule: a low-confidence L2 hit means no relevant consolidation exists. Its backlinks would just be unrelated episodes that happened to be geometrically nearby. Returning nothing is more correct than returning wrong memories. If L2 is low-confidence, L1 source memories will be too — the failure propagates downward. + +--- + +## Scaling Considerations + +### sqlite-vec Limitations + +sqlite-vec does linear scan, not HNSW/IVF. Performance degrades with corpus size: + +- ~100–1000 memories: negligible +- ~10000: starting to feel it (~5–50ms per search) +- ~100000+: too slow for full L1 scan + +Mitigation: L1 time-windowing keeps each partition small. Layers 2–4 stay naturally small due to consolidation being compressive. + +Plan: keep sqlite-vec and design retrieval as a trait (`search(embedding, k) -> Vec`) so the backend can be swapped to qdrant or usearch later without changing retrieval logic. + +### Expected Layer Distribution (at scale) + +``` +L1: ~90% of total memories (episodic, grows continuously) +L2: ~8% (pattern, grows slowly) +L3: ~1.8% (trait, grows very slowly) +L4: ~0.2% (core, rarely changes) +``` + +--- + +## Comparison: Alternative Memory Architectures + +### Activation-Based Memory (Titans-style adapter) + +A different class of approach: instead of storing text + embeddings, store internal model activations gated by an importance metric (gradient-based "surprise"), then inject activations back via attention at inference time. + +**Where it wins over ours:** +- importance gating is automatic — the model itself decides what's surprising, no heuristics or separate LLM needed +- activations capture richer mid-computation representations than embeddings (which are compressed summaries) +- native attention over retrieved activations is more principled than RAG — the model attends to memory directly rather than parsing injected text + +**Where ours wins:** +- memories are model-agnostic text — swap the underlying LLM and all memories remain valid +- full interpretability — you can audit, edit, and inspect the memory bank directly +- provenance traversal is possible — the hierarchy and backlinks are explicit, not implicit in weights +- no consolidation hierarchy — the importance gate is flat (important vs not), no equivalent to L1→L4 abstraction building over time + +**The model-portability nuance:** a Titans-style adapter *architecture* can be applied to different models (see TPTT paper: applied to llama, qwen, gemma, mistral without full retraining). But the accumulated *memory bank* itself is model-locked — activations from one model are meaningless on another since the dimensional space changes. Migration requires replaying the full conversation log through the new model+adapter to rebuild the memory bank, which is expensive and not bit-for-bit identical since different models gate on different surprise signals. + +**Convergence observation:** if you keep the raw conversation log as source-of-truth for replay (which you'd have to), that log is exactly what our architecture already stores as its primary representation. Activation memory becomes a derived cache on top of text storage — our architecture is the canonical form either way. + +### Letta (MemGPT) + +Letta is text-based and model-agnostic — much closer to our approach than to Titans. + +Key architecture: in-context memory (a small editable section always in the context window), archival memory (flat vector DB), and recall memory (conversation history). The agent manages all of this via tool calls — it explicitly decides when to search memory and what to store. + +**Similarities to ours:** text-based, model-agnostic, RAG-style retrieval, has a sleep-time async consolidation concept. + +**Where ours differs:** +- we have a consolidation hierarchy (L1→L4); letta's archival memory is a flat vector store +- our retrieval is automated via classifier + pipeline; letta's is agent-driven (LLM decides when to search), which is more flexible but burns more tokens per interaction +- we have backlink-based provenance traversal; letta has no equivalent +- letta's self-editing memory is more dynamic (agent can rewrite its own core memory blocks in real time); ours is pipeline-driven and less agentic + +--- + +## Migration Path: Ours → Activation-Based + +If we ever wanted to migrate to an activation-based approach, the structure we've built provides useful training signal — better than raw logs: + +- **L1 episodes** → training corpus (ordered, timestamped) +- **consolidation decisions** → importance/gating supervision (what was deemed worth keeping is a ready-made signal for the surprise gate) +- **reranker scores** → retrieval quality supervision ("this memory was stored AND was later relevant to real queries" is a stronger signal than just "this seemed important at storage time") +- **layer membership** → memory depth supervision +- **embeddings** → can initialize or regularize the activation memory's key/value associations rather than learning from scratch + +The activation adapter would be learning to replicate the structure we built explicitly, but implicitly in weights — so it'd converge faster and to a better solution than training from raw text. Our architecture serves as a structured distillation target. + +--- + +## Open Questions + +- exact confidence thresholds for L1 window expansion and L2 backlink traversal +- consolidation trigger strategy (time-based vs density-based vs count-based) +- how many source memories to consolidate per higher-layer memory (affects backlink fan-out) +- whether to search L2 in parallel with L1 for vague temporal queries vs strictly sequential fallback +- cold storage format for archived L1 memories and UX for explicit provenance queries +- how to handle the classifier cold-start problem (not enough real queries to train on initially — use purely synthetic data until enough real examples accumulate?) +- depth limit value for backlink traversal diff --git a/docs/klbr_mvp_benchmarking_and_calibration_plan.md b/docs/klbr_mvp_benchmarking_and_calibration_plan.md new file mode 100644 index 0000000..56c1bae --- /dev/null +++ b/docs/klbr_mvp_benchmarking_and_calibration_plan.md @@ -0,0 +1,825 @@ +# KLBR MVP Benchmarking and Calibration Plan + +This document is a handoff brief for another agent/engineer. It assumes the architectural intent from the KLBR design note is still the source of truth, but it narrows the first implementation to a measurable MVP and expands the benchmarking and calibration protocol in enough detail that work can start immediately. + +The design note frames KLBR as a hierarchical memory system with episodic-to-abstract consolidation, two-stage retrieval, time-windowed L1 search, a lightweight layer router, and provenance/backlinks. For the MVP, we should only implement the smallest slice that can be benchmarked rigorously: **L1 episodic memory only**, plus **time filtering**, **dense retrieval**, **reranking**, **context packing**, and **abstention**. L2-L4, consolidation, classifier routing, and backlink traversal should stay out of the first milestone unless explicitly called back in later. + +--- + +## 1. Fixed choices already made + +These should be treated as locked for the MVP unless a benchmark disproves them badly enough to justify reopening the choice. + +- **Embedding model:** `BAAI/bge-m3` +- **Reranker:** `BAAI/bge-reranker-v2-m3` +- **Generation model:** Gemma 4 26B A4B in the harness, with a **64K effective context budget in deployment** +- **Vector backend:** `usearch` embedded in the Rust harness +- **Implementation language / harness:** Rust + +Practical implications of those choices: + +- `bge-m3` supports dense, sparse, and multi-vector retrieval, but the MVP should use **dense-only retrieval first**. +- `bge-reranker-v2-m3` should be the only second-stage scorer in the MVP. +- The deployed Gemma context budget should be treated as **64K operational budget**, even if the upstream model family supports a larger maximum context. +- `usearch` can support both approximate indexed search and exact search paths; this matters because small time-windowed partitions may not need ANN at all. + +--- + +## 2. MVP objective + +The MVP goal is not "build the hierarchy." The MVP goal is: + +> **Given a user query, retrieve the right episodic memories reliably enough that the generator can answer grounded questions and abstain on no-hit cases.** + +That means the first version must prove four things: + +1. ingest is stable, +2. dense retrieval returns the correct candidate set often enough, +3. reranking improves precision enough to justify its cost, +4. the final answer is better with memory than without it. + +If those are not proven, adding L2/L3/L4 will only make the system harder to reason about. + +--- + +## 3. Explicit MVP scope + +### In scope + +- L1 episodic memory only +- append-only write path +- memory metadata store +- time-windowed partitioning for L1 +- `bge-m3` embeddings +- USearch retrieval +- `bge-reranker-v2-m3` reranking +- context packing into Gemma prompt +- abstain / no-hit behavior +- offline evaluation harness +- threshold calibration and parameter sweeps + +### Out of scope for the first milestone + +- L2/L3/L4 +- consolidation jobs +- provenance/backlinks +- classifier routing +- hybrid sparse+dense retrieval +- multi-vector retrieval +- online self-editing memory +- memory graph traversal +- RL or preference optimization for memory decisions + +--- + +## 4. Required deliverables from the agent + +The next agent should not just ship code. They should produce these artifacts while implementing: + +1. **MVP contract doc** + - exact data model for L1 memory records + - retrieval request/response schema + - logging schema + +2. **Offline evaluation set** + - at least one internal replay/gold dataset + - at least one public benchmark integration + +3. **Benchmark report** + - retrieval metrics by stage + - latency metrics by stage + - storage/index metrics + - generation metrics for memory-augmented QA + +4. **Calibration memo** + - chosen thresholds + - score distributions + - reliability plots / threshold tables + - justification for final operating point + +5. **Config file / experiment registry** + - all retrieval params centralized + - all benchmark runs traceable to one config version + +--- + +## 5. Data model assumptions for the MVP + +Keep the schema simple and stable. Each L1 memory should have at least: + +- `memory_id` +- `namespace` or `user_id` +- `layer = L1` +- `text` +- `event_time` +- `ingest_time` +- `embedding_model` +- `embedding_dim` +- `embedding_version` +- `status` (`active`, `archived`, `suppressed`) +- `source_ref` (optional) +- `tags` (optional) + +Optional future fields may be reserved but should remain unused for the MVP: + +- `parent_ids` +- `child_ids` +- `superseded_by` +- `confidence` +- `provenance` + +The agent should make sure the index can always be rebuilt deterministically from the metadata store plus raw text. + +--- + +## 6. Benchmarking philosophy + +The harness should benchmark the memory stack as **four separate subsystems** before looking at end-to-end answer quality: + +1. **ingestion** +2. **retrieval** +3. **reranking** +4. **generation / context use** + +Do **not** collapse everything into a single final QA score. A bad final answer could be caused by: + +- missing memory ingestion, +- wrong partition/window selection, +- poor ANN recall, +- reranker misordering, +- context overflow, +- bad answer synthesis, +- failure to abstain. + +Each stage must be measurable on its own. + +--- + +## 7. Benchmark datasets to use + +The agent should benchmark on both **KLBR-specific data** and **public datasets**. + +### A. Internal / KLBR-specific evaluation set (mandatory) + +This matters the most. Public benchmarks help, but they do not replace real target behavior. + +The agent should build a labeled set of **150 to 300 queries** at minimum before any serious tuning. Start small and grow it over time. + +Each query should include: + +- query text +- query category +- gold relevant memory IDs +- acceptable no-hit / abstain label +- optional gold answer text +- optional date/time anchor +- optional conflict/update annotation + +#### Required query categories + +The dataset should include all of the following: + +1. **exact recent event** + - example pattern: "what did I do this morning?" +2. **exact dated event** + - example pattern: "what did I fix last Tuesday?" +3. **vague recent lookup** + - example pattern: "what have I been working on lately?" +4. **recurring theme / pattern-like question** + - example pattern: "what kinds of infra work do I usually do?" + - even though this is more L2-like, include it now to expose L1 limitations +5. **conflict / update question** + - example pattern: "what editor am I using now?" + - include at least one stale old fact plus a newer correction +6. **no-hit query** + - example pattern: ask about something never mentioned +7. **temporally ambiguous query** + - example pattern: "what did I do around the migration?" +8. **multi-evidence query** + - answer depends on 2+ memories rather than one + +#### Important split rule + +Split by **session / conversation / user timeline**, not by question only. Do not let train/dev/test contain overlapping memory episodes from the same thread if that creates leakage. + +### B. Public benchmark set (recommended) + +Use the following benchmarks for different purposes: + +- **LongMemEval** for assistant-style long-term memory evaluation. It explicitly evaluates five long-term abilities: information extraction, multi-session reasoning, temporal reasoning, knowledge updates, and abstention. +- **LoCoMo** for very long conversations, temporal/causal reasoning, and event summarization over up to 35 sessions. +- **PerLTQA** for memory classification/retrieval/synthesis style tasks combining semantic and episodic memory. +- **MTEB** for embedding sanity checks outside the KLBR task itself. +- **BEIR** for retrieval robustness baselines and to make sure dense retrieval is not being overestimated on narrow internal data. + +### Suggested role of each public benchmark + +- **LongMemEval:** end-to-end memory QA benchmark once the MVP retrieval loop works +- **LoCoMo:** stress temporal ordering and long-range conversational drift +- **PerLTQA:** later, use it to validate routing/classification decisions and episodic vs semantic boundaries +- **MTEB:** only for embedding-model sanity, not as the primary success metric for KLBR +- **BEIR:** use as a retrieval control benchmark and to justify including a lexical baseline in future phases + +--- + +## 8. Baselines the agent must compare against + +The MVP should not be evaluated in isolation. It should be compared to simple baselines. + +### Minimum comparison set + +1. **No-memory baseline** + - Gemma answers from current prompt only + +2. **Flat L1 exact search baseline** + - no time windowing, no ANN, no reranker + - brute force or exact vector search over full active L1 for small datasets + +3. **Flat L1 dense retrieval baseline** + - full L1, dense retrieval, no reranker + +4. **Flat L1 dense + rerank baseline** + - full L1, reranked + +5. **Target MVP baseline** + - time-windowed L1 dense retrieval + rerank + abstain + +### Optional later comparisons + +6. **summary-only memory baseline** +7. **dense + sparse hybrid baseline** using BGE-M3 sparse outputs +8. **L1 + L2 hierarchical baseline** once L2 exists + +The key question is not just "does KLBR work?" The question is: + +> Which part of the design is creating measurable lift? + +--- + +## 9. Retrieval benchmark protocol + +The agent should benchmark retrieval in layers. + +### 9.1 First-stage retrieval benchmark + +Measure whether the correct gold memory appears in the candidate set **before reranking**. + +#### Metrics + +- `Recall@1` +- `Recall@5` +- `Recall@10` +- `Recall@20` +- `MRR` +- `nDCG@10` +- `candidate_set_size` +- `time_window_hit_rate` + - proportion of times the gold memory is inside the chosen initial window + +#### Mandatory ablations + +- exact search vs indexed search +- full L1 vs time-windowed L1 +- multiple window sizes +- normalized cosine vs raw inner product if both are practical in the harness +- different candidate counts (`k_ann`) + +### 9.2 Second-stage reranker benchmark + +Measure whether reranking moves the correct item upward enough to justify latency. + +#### Metrics + +- `top1_accuracy_after_rerank` +- `MRR_after_rerank` +- `nDCG_after_rerank` +- `reranker_lift = metric_after - metric_before` +- `pair_latency_ms` +- `batch_latency_ms` +- `latency_per_candidate` + +#### Mandatory ablations + +- rerank top 10 vs top 20 vs top 40 +- memory passage length / truncation strategy +- single-memory candidates vs grouped multi-sentence candidates +- reranker score normalization on vs off + +### 9.3 Context packing benchmark + +Once reranking is done, benchmark whether the chosen packing policy helps or hurts final QA. + +#### Variables to sweep + +- number of injected memories: `2 / 4 / 6 / 8` +- memory packing format: + - raw snippets only + - snippets with timestamps + - snippets with short metadata headers +- order of packing: + - reranker score only + - recency-biased reranker order + - diversity-aware order + +#### Metrics + +- end-to-end answer accuracy / F1 +- temporal accuracy +- contradiction rate +- abstain correctness +- prompt memory token count +- generator latency + +--- + +## 10. Calibration protocol + +Calibration should happen in a strict order. The agent should **not** tune all knobs at once. + +### 10.1 Data split for calibration + +Use three splits: + +- **train**: only if a router/classifier or learned calibrator is used later +- **dev**: for all threshold tuning and sweep selection +- **test**: held out until the end + +Do not tune on test. + +### 10.2 Stage order for calibration + +#### Stage A: calibrate the retrieval substrate + +Tune only: + +- partitioning/window strategy +- vector metric / embedding normalization behavior +- exact vs indexed search +- `k_ann` + +Hold everything else fixed. + +Goal: + +- maximize retrieval recall under latency budget +- identify the corpus size / partition size where exact search stops being viable + +#### Stage B: calibrate the reranker operating point + +Tune: + +- `k_rerank` +- candidate passage length +- score normalization choice +- top-1 acceptance threshold +- margin threshold between top-1 and top-2 + +Goal: + +- maximize precision of injected memories +- avoid pulling weak memories into prompt context + +#### Stage C: calibrate window expansion behavior + +Tune: + +- initial window size +- low-confidence threshold for widening the time window +- maximum number of expansions +- widening schedule (e.g. 7d -> 30d -> 90d -> all) + +Goal: + +- improve recall on older events without paying full-search cost on every query + +#### Stage D: calibrate abstention + +Tune: + +- minimum confidence required to answer from memory +- conditions under which the system says "I don't know" +- optional compound rule using score + margin + evidence count + +Goal: + +- prevent hallucinated memory recall +- maximize selective accuracy, not just raw answer rate + +#### Stage E: calibrate context budget + +Tune: + +- how many memories to inject +- how much formatting overhead to allow +- how much room to leave for the question and response + +Goal: + +- stay within the 64K deployment budget while keeping memory evidence concise and useful + +--- + +## 11. Recommended threshold design + +Do **not** use a single reranker score threshold for all decisions. + +Use at least three different decision thresholds: + +1. **`t_expand`** + - below this, widen the time window and retry + +2. **`t_accept`** + - above this, inject the memory into the prompt + +3. **`t_abstain`** + - below this after all retries, abstain + +Additionally, use at least one structural confidence feature: + +- **margin between top-1 and top-2 reranker scores** + +And optionally one evidence sufficiency feature: + +- **count of candidates above threshold** + +### Example policy shape + +- if `top1 < t_expand`, widen window +- else if `top1 >= t_accept` and `(top1 - top2) >= t_margin`, answer with memory +- else if after max expansions `top1 < t_abstain`, abstain +- else answer cautiously with explicit uncertainty or with minimal memory injection + +The exact numeric values should come from the dev set, not intuition. + +--- + +## 12. How to calibrate reranker scores properly + +Raw reranker scores should not be assumed to be calibrated probabilities. + +The agent should explicitly test calibration. + +### Recommended procedure + +1. Collect a dev set of `(query, candidate_memory, label)` tuples where label is relevant / not relevant. +2. Run the reranker on all tuples and store raw scores. +3. Fit a simple score calibrator on the dev set only: + - logistic / Platt scaling, or + - isotonic regression if enough dev examples exist +4. Compare: + - raw score thresholding + - sigmoid-normalized model output + - learned calibrator output +5. Choose the one with the best reliability on the dev set. + +### Metrics for score calibration + +- precision at operating threshold +- recall at operating threshold +- Brier score +- expected calibration error (ECE) +- reliability / calibration plot +- risk-coverage curve for selective answering + +### Operating principle + +Prefer a thresholding regime that yields **high evidence precision** over one that squeezes a bit more recall. For memory systems, injecting a wrong memory is usually worse than missing a borderline one. + +--- + +## 13. Time-window calibration protocol + +Because the KLBR design depends heavily on L1 time-windowing, this needs its own benchmark rather than an ad-hoc heuristic. + +### Candidate window schedules to test + +At minimum test these: + +- `7d` +- `30d` +- `90d` +- `all` + +Also test a widening ladder: + +- `7d -> 30d -> 90d -> all` +- `30d -> 90d -> all` + +### What to measure + +- how often the gold memory is already in the first window +- how often widening recovers a miss +- how often widening only adds noise +- extra latency per widening step +- effect on final answer accuracy + +### Decision rule to learn + +The agent should derive a policy that answers these questions with data: + +- What should the default window be when the query has no explicit time signal? +- How often is a 7-day default too aggressive? +- Is 30 days the best compromise? +- When does widening produce useful recall versus pointless cost? + +### Important warning + +Do not assume time-windowing is always a win. If the user's memory density is low, the window may not shrink the corpus enough to matter. Benchmark on realistic memory volumes. + +--- + +## 14. Exact vs approximate search plan for USearch + +Because USearch supports exact and approximate modes, and because KLBR is intentionally partitioning L1 by time, the agent should benchmark the crossover point rather than assuming ANN is needed from day one. + +### Required experiment + +For each partition size bucket, compare exact and indexed search. + +Suggested partition buckets: + +- 100 items +- 1k items +- 5k items +- 10k items +- 25k items +- 50k items + +### Measure + +- `Recall@k` +- latency p50/p95 +- memory footprint +- index build time +- reload time + +### Goal + +Produce a simple decision rule like: + +- partitions under `X` items: exact search +- partitions over `X` items: indexed search + +If exact search is good enough for small windows, use it first. Simpler systems are easier to debug. + +--- + +## 15. Generation benchmark protocol + +Do not benchmark only retrieval. Benchmark whether the generator actually uses the retrieved evidence correctly. + +### End-to-end answer metrics + +For the internal eval set, collect: + +- exact match where possible +- answer F1 where exact match is too strict +- temporal accuracy +- contradiction rate +- stale-memory rate +- no-hit abstain accuracy +- evidence faithfulness (manual spot check if needed) + +### Evidence-grounding audit + +For at least 30 to 50 sampled eval examples, the agent should manually or semi-manually inspect: + +- whether the correct memory was retrieved +- whether the answer used the retrieved memory correctly +- whether the answer invented unsupported details +- whether the answer should have abstained + +The system should fail closed rather than fail open. + +--- + +## 16. Recommended sweep order + +The agent should run sweeps in this order. Do not randomize the order. + +### Sweep 1: ingestion granularity + +Compare memory chunk sizes / event segmentation policy. + +If memories are too coarse: +- reranker inputs get noisy +- time lookup becomes imprecise + +If memories are too fine: +- relevant evidence fragments scatter across too many units + +Suggested first grid: + +- 1 event = 1 memory +- 2 to 3 related turns = 1 memory +- fixed-size passage cap: ~64 / 128 / 256 / 384 tokens + +### Sweep 2: vector search substrate + +- exact vs indexed +- normalized cosine vs raw inner product +- partition sizes + +### Sweep 3: first-stage candidate count + +- `k_ann = 10 / 20 / 40 / 60 / 100` + +### Sweep 4: reranker pool size + +- rerank top `10 / 20 / 40` + +### Sweep 5: time windows + +- default windows +- widening ladders + +### Sweep 6: context packing + +- inject `2 / 4 / 6 / 8` memories +- with/without timestamps + +### Sweep 7: abstention thresholds + +- optimize for precision under selective coverage + +--- + +## 17. Logging and observability requirements + +Every benchmark run should log, per query: + +- query ID +- query text +- query category +- chosen window +- number of memories searched +- vector search mode (exact / indexed) +- top-k ANN candidates and scores +- top-k reranked candidates and scores +- whether window expanded +- whether system abstained +- memory tokens injected +- generator answer +- gold memory IDs +- gold answer / no-hit label +- pass/fail outcome + +Without this log, the agent will not be able to debug threshold behavior. + +--- + +## 18. Phase gates and success criteria + +### Phase 0: instrumentation gate + +**Exit only when:** +- ingestion, retrieval, rerank, and generation timings are separately logged +- offline runs are reproducible from config files + +### Phase 1: retrieval substrate gate + +**Exit only when:** +- there is a justified choice for exact vs indexed retrieval +- there is a justified default window policy +- first-stage recall is stable on dev and test + +### Phase 2: reranker gate + +**Exit only when:** +- reranker materially improves precision or ranking quality over first-stage retrieval +- reranker thresholds are calibrated on the dev set + +### Phase 3: end-to-end MVP gate + +**Exit only when:** +- memory-augmented answering beats no-memory answering on internal eval +- no-hit queries abstain at an acceptable rate +- contradictory old memories do not dominate newer corrections too often + +### Phase 4: ready-for-L2 gate + +Only after the above, ask: + +- Are pattern-like queries still weak enough to justify L2? +- Is the retrieval baseline stable enough that hierarchy will be additive rather than confusing? + +If yes, move to L2. + +--- + +## 19. Recommended operating targets for the MVP + +The agent should choose actual numeric targets based on business needs, but a reasonable starting posture is: + +- prioritize **precision of injected memories** over maximum recall +- prioritize **abstention correctness** over forcing an answer +- accept some recall loss if it dramatically cuts false positives +- keep retrieval + rerank latency within a small fraction of total request time + +A good MVP is one that: + +- answers correctly when evidence is present, +- stays quiet when evidence is weak, +- and exposes its own failure modes clearly. + +--- + +## 20. Recommended next steps after the MVP stabilizes + +Only after the MVP is benchmarked and calibrated should the next agent consider: + +1. **L2 pattern memories** +2. **offline consolidation jobs** +3. **lightweight query router / classifier** +4. **backlink traversal for exact-evidence recovery** +5. **BGE-M3 dense+sparse hybrid retrieval** +6. **multi-vector retrieval** + +For now, the correct order is: + +> benchmark L1 -> calibrate L1 -> prove lift -> then add structure. + +--- + +## 21. Minimal execution checklist for the next agent + +### Week 1 + +- freeze MVP contract +- implement L1 schema +- implement ingestion +- implement exact retrieval baseline +- create first internal eval set + +### Week 2 + +- add USearch indexed retrieval +- benchmark exact vs indexed by partition size +- add reranker +- log candidate/reranker distributions + +### Week 3 + +- calibrate thresholds on dev set +- implement window expansion +- benchmark abstention +- benchmark memory injection strategies + +### Week 4 + +- finalize report +- lock config +- decide whether L2 is justified next + +--- + +## 22. Final instruction to the next agent + +The architecture should not be judged by whether it *sounds* right. It should be judged by whether the following statement becomes true on a held-out set: + +> "Given a realistic user query and a realistic memory corpus, the system retrieves the right episodic evidence often enough, reranks it correctly often enough, injects only trustworthy evidence, and abstains when evidence is weak." + +If that statement is not yet true, do not add hierarchy. + +--- + +## Appendix: concrete benchmark matrix + +| Experiment | What changes | Keep fixed | Main metrics | Decision | +|---|---|---|---|---| +| E1 | exact vs indexed retrieval | chunk size, window, reranker off | Recall@10, p95 latency | choose search mode by partition size | +| E2 | window sizes | retrieval mode, chunk size | Recall@10, window hit rate, latency | choose default window | +| E3 | `k_ann` sweep | window, chunk size | Recall@k, reranker lift | choose candidate count | +| E4 | rerank pool size | retrieval config | top1 accuracy, p95 latency | choose `k_rerank` | +| E5 | passage granularity | retrieval config | reranker lift, final QA | choose memory unit size | +| E6 | threshold sweep | retrieval+rerk fixed | precision/recall, Brier, ECE | choose `t_expand`, `t_accept`, `t_abstain` | +| E7 | context packing | retrieval+rerk fixed | answer F1, contradiction, token cost | choose packing policy | +| E8 | no-memory vs memory | final fixed config | QA accuracy, abstain correctness | prove MVP lift | + +--- + +## Appendix: sources to consult + +### Architecture and design context + +- Uploaded KLBR design note (source of truth for current architecture) + +### Model / library docs + +- BGE-M3 model card: https://huggingface.co/BAAI/bge-m3 +- BGE reranker v2 M3 model card: https://huggingface.co/BAAI/bge-reranker-v2-m3 +- Gemma 4 26B A4B model card: https://huggingface.co/google/gemma-4-26B-A4B-it +- USearch Rust crate docs: https://lib.rs/crates/usearch + +### Benchmarks and evaluation references + +- LongMemEval paper: https://arxiv.org/abs/2410.10813 +- LongMemEval code/data: https://github.com/xiaowu0162/LongMemEval +- LoCoMo paper: https://arxiv.org/abs/2402.17753 +- LoCoMo code/data: https://github.com/snap-research/locomo +- PerLTQA paper: https://arxiv.org/abs/2402.16288 +- MTEB paper: https://arxiv.org/abs/2210.07316 +- MTEB repo: https://github.com/embeddings-benchmark/mteb +- BEIR paper: https://arxiv.org/abs/2104.08663 +- Retrieve-and-rerank reference: https://www.sbert.net/examples/sentence_transformer/applications/retrieve_rerank/README.html + diff --git a/docs/klbr_mvp_contract.md b/docs/klbr_mvp_contract.md new file mode 100644 index 0000000..e219e3c --- /dev/null +++ b/docs/klbr_mvp_contract.md @@ -0,0 +1,164 @@ +# KLBR MVP Contract + +This is the first implementation pass for the benchmarking plan in `docs/klbr_mvp_benchmarking_and_calibration_plan.md`. It freezes the data and logging surface needed for Phase 0 and Week 1 work: + +- an explicit L1 memory record schema +- an internal offline eval dataset schema +- a centralized retrieval experiment config +- a per-query retrieval trace format +- a runnable exact-retrieval benchmark harness + +## L1 Memory Record + +The canonical serialized shape is `klbr_core::mvp::L1MemoryRecord`. + +- `memory_id: i64` +- `namespace: String` +- `layer: "L1"` +- `text: String` +- `event_time: i64` +- `ingest_time: i64` +- `embedding_model: String` +- `embedding_dim: usize` +- `embedding_version: String` +- `status: "active" | "archived" | "suppressed"` +- `source_ref: Option` +- `tags: Vec` +- `pinned: bool` +- `embedding: Vec` + +The SQLite `memories` table now stores the MVP metadata fields directly, and `MemoryStore::store_with_metadata()` can ingest a fully specified record. + +## Offline Eval Dataset + +The internal dataset schema is `klbr_core::mvp::InternalEvalDataset`. + +- `dataset_id` +- `description` +- `memories: Vec` +- `queries: Vec` + +Each query includes: + +- `query_id` +- `split: train | dev | test` +- `category` +- `namespace` +- `timeline_id` +- `text` +- `gold_memory_ids` +- `no_hit` +- `gold_answer` +- `reference_time` + +Each memory can also carry `timeline_id` so sessions or timeline segments can be grouped for split validation and corpus filtering. + +## Retrieval Request / Response + +The first runnable harness uses exact retrieval only. The shared config/log types are: + +- request/config: `klbr_core::mvp::RetrievalExperimentConfig` +- per-query response/log: `klbr_core::mvp::RetrievalTrace` +- aggregate report: `klbr_core::mvp::RetrievalBenchmarkReport` + +`RetrievalTrace` now logs the required debugging surface for both ranking stages plus the final benchmark decision: + +- query id/text/category/split +- chosen time window and full window ladder considered +- per-window attempt logs including window size, retrieval/rerank latency, raw rerank score/margin, top-candidate support score, candidate ids, and policy decision +- number of memories searched +- vector search mode +- first-stage candidates with scores +- reranked candidates with scores +- first-stage top-1 distance and whether first-stage top-k contained gold +- rerank top-1 / top-2 raw `relevance_score` +- rerank raw score margin +- top-candidate lexical support score +- whether the window expanded +- final decision: `answer | abstain` +- final answer memory ids +- final correctness +- gold ids / gold answer / no-hit label +- time-window hit +- embedding and retrieval latencies +- rerank latency +- pass/fail outcome + +## Harness + +The new benchmark binary is `klbr-bench`. + +Current command: + +```bash +cargo run -p klbr-bench -- retrieval \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/manual-retrieval +``` + +Passive recall command: + +```bash +cargo run -p klbr-bench -- passive-recall \ + benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/runs/manual-passive-recall +``` + +Threshold sweep command: + +```bash +cargo run -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/manual-sweep +``` + +Optional custom grid: + +```bash +cargo run -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/manual-sweep \ + -11.5 2.0 1.0 0.0 12.0 1.0 0.0 1.0 0.1 +``` + +Outputs: + +- `report.json` +- `report.md` +- `traces.json` +- `resolved_config.json` +- `sweep_report.json` +- `sweep_report.md` +- `sweep_results.csv` + +Datasets shipped in-repo: + +- `benchmarks/inputs/datasets/internal_eval_template.json`: smoke test only +- `benchmarks/inputs/datasets/internal_eval_starter.json`: first non-toy internal eval set with required query-category coverage +- `benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json`: starter passive-recall eval set for activation + support grading + +This now supports optional reranking in the benchmark harness as well: + +- `rerank_url` +- `rerank_model` +- `rerank_top_k` +- `rerank_normalize` +- `rerank_abstain_score_threshold` +- `rerank_abstain_margin_threshold` +- `support_abstain_threshold` + +On the current `llama-server` reranking path, the harness treats the returned `relevance_score` as the canonical score space for logging and thresholding. `rerank_normalize` is retained as experiment metadata, but calibration should currently be done on raw scores and raw margins. + +The benchmark now also computes a generic lexical support feature for the selected top candidate: a weighted coverage score over salient query terms that appear in the memory text. That score is objective-agnostic, so it can be reused later for passive recall as well as direct question answering. + +This is still intentionally benchmark-first. Indexed retrieval, calibrated rerank thresholds, and generation benchmarking should build on these schemas rather than replace them. + +The benchmark surface now also includes passive-recall-specific traces and metrics: + +- `PassiveRecallTrace` +- `PassiveRecallMetrics` +- `PassiveRecallBenchmarkReport` diff --git a/docs/klbr_passive_recall_benchmark_plan.md b/docs/klbr_passive_recall_benchmark_plan.md new file mode 100644 index 0000000..4b6ab20 --- /dev/null +++ b/docs/klbr_passive_recall_benchmark_plan.md @@ -0,0 +1,132 @@ +# KLBR Passive Recall Benchmark Plan + +This document defines the next benchmark slice for memory use on non-question inputs. + +## Why This Exists + +The current retrieval benchmark mostly measures: + +- answerable factual lookup +- rerank confidence +- expand vs abstain decisions + +That is necessary, but not sufficient for KLBR. + +KLBR also needs to decide whether to recall memory for inputs like: + +- `still working on the benchmark stuff` +- `i'm back in zed again` +- `reranker on 8003` +- `lol yeah that makes sense` + +Those are not all explicit factual questions, but the system still needs a memory-use policy. + +## Two Separate Decisions + +We should treat memory use as two linked decisions: + +1. `activation` + Should memory be recalled at all for this input? +2. `trust` + If memory is recalled, are the selected memories actually the right support? + +The current question-oriented benchmark mostly covers trust. +Passive recall needs to evaluate both. + +## Schema + +The shared eval schema now supports passive-recall cases through optional fields on `EvalQuery`: + +- `interaction_mode` + - `question` + - `statement` + - `update` + - `request` + - `fragment` +- `objective` + - `answer_query` + - `passive_recall` +- `expected_memory_action` + - `recall` + - `no_recall` + +Existing retrieval datasets do not need to set these fields. They default to question-answer behavior. + +## Dataset Semantics + +For passive-recall items: + +- `text` is the user input as it would arrive to the agent +- `gold_memory_ids` are the memories that should be recalled if memory activation is correct +- `no_hit = true` means the item should not recall any memory +- `expected_memory_action = no_recall` is the preferred explicit label for passive-recall no-hit cases + +For multi-evidence passive cases: + +- `gold_memory_ids` may contain several acceptable support memories +- the goal is not necessarily single-memory top-1 +- the benchmark should allow support-style grading later + +## First Metrics To Add + +The passive-recall benchmark should report: + +- activation precision + fraction of recall-triggered inputs that should actually recall +- activation recall + fraction of should-recall inputs where recall was triggered +- no-recall false activation rate + fraction of no-recall inputs where memory was still injected +- support precision@k + fraction of injected memories that are in the gold support set +- support recall@k + fraction of gold support memories that were injected + +These are deliberately different from answer/abstain metrics. + +## Starter Dataset + +The first starter dataset is: + +- `benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json` + +It includes: + +- passive recent-work cues +- elliptical requests +- declarative state cues +- endpoint fragments +- multi-evidence infra cues +- pure chatter that should not trigger recall + +## What Not To Do + +Do not solve passive recall by inventing many brittle query classes. + +The benchmark should push us toward generic features that work for both questions and non-questions, such as: + +- semantic relevance +- lexical/entity support +- agreement across support memories +- confidence calibrated against no-recall cases + +## Current Implementation + +The starter implementation now exists as: + +```bash +cargo run -p klbr-bench -- passive-recall \ + benchmarks/inputs/datasets/internal_eval_passive_recall_starter.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/runs/manual-passive-recall-starter +``` + +It: + +1. embeds the input text +2. retrieves candidates through the same exact-search path used by the main retrieval benchmark +3. reranks and scores support the same way as the answer/abstain benchmark +4. decides `recall` vs `no_recall` +5. if recalling, grades the injected support set against `gold_memory_ids` + +The current starter run is intentionally small, so the next real step after implementation is to expand the passive-recall dataset rather than overfitting to the 8-query slice. diff --git a/docs/klbr_rerank_sweep_2026-04-22.md b/docs/klbr_rerank_sweep_2026-04-22.md new file mode 100644 index 0000000..75f3282 --- /dev/null +++ b/docs/klbr_rerank_sweep_2026-04-22.md @@ -0,0 +1,390 @@ +# KLBR Rerank Sweep Summary — 2026-04-22 + +This file consolidates the rerank-threshold sweep work for the current MVP retrieval benchmark. + +## Scope + +- dataset: `internal-eval-starter-v1` +- split: `dev` +- embedder: `http://localhost:8002` +- reranker: `http://localhost:8003` +- policy: + - retrieve in current window + - rerank + - answer if `score1 >= T_score` and `margin >= T_margin` + - otherwise expand to the next window + - abstain after the last window + +## Commands Run + +Coarse sweep: + +```bash +cargo run -q -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/legacy/out/sweep-coarse-fast +``` + +Fine sweep: + +```bash +cargo run -q -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/legacy/out/sweep-fine \ + -6.5 -1.5 0.25 0.0 4.0 0.25 +``` + +## Artifact Locations + +- coarse report: `benchmarks/runs/legacy/out/sweep-coarse-fast/sweep_report.md` +- coarse machine output: `benchmarks/runs/legacy/out/sweep-coarse-fast/sweep_report.json` +- fine report: `benchmarks/runs/legacy/out/sweep-fine/sweep_report.md` +- fine machine output: `benchmarks/runs/legacy/out/sweep-fine/sweep_report.json` + +## Main Result + +Thresholding alone does not produce a single operating point that gets both: + +- `0` no-hit false answers +- `>= 0.9` answerable-query coverage + +on the current `dev` slice. + +The sweep settled into two stable operating regions: + +1. Conservative region + - eliminates no-hit false answers + - but answerable coverage drops materially +2. Balanced region + - keeps answerable coverage high enough to feel usable + - but still allows one no-hit false answer + +## Coarse Sweep Result + +Best strict lexicographic point from the coarse sweep: + +- `T_score = -1.5` +- `T_margin = 0.0` +- no-hit false answers: `0` +- no-hit abstain recall: `1.0` +- answerable coverage: `0.5833` +- final decision accuracy: `0.6250` +- expansion count: `10` + +Interpretation: + +- this is too conservative for the intended MVP user experience +- it solves no-hit false answers by abstaining too often on answerable queries + +## Fine Sweep Result + +Best strict lexicographic point from the fine sweep: + +- `T_score = -2.0` +- `T_margin = 0.0` +- no-hit false answers: `0` +- no-hit abstain recall: `1.0` +- answerable coverage: `0.6667` +- final decision accuracy: `0.6875` +- expansion count: `9` + +Best "balanced" region found in the fine sweep: + +- `T_score = -6.0` +- `T_margin = 0.0` +- no-hit false answers: `1` +- no-hit abstain recall: `0.75` +- answerable coverage: `0.9167` +- final decision accuracy: `0.8125` +- expansion count: `7` + +Interpretation: + +- the score threshold matters more than the margin threshold on this dataset +- once `T_score` becomes stricter than about `-2.0`, the system abstains too aggressively +- the practical tradeoff is between: + - `0` false answers with too little coverage + - `1` false answer with much healthier coverage + +## Candidate Operating Modes + +### Balanced + +Use this if the MVP should remain usable on answerable queries while accepting one residual false answer on the current `dev` set. + +```json +{ + "experiment_id": "mvp-rerank-calibrated-balanced", + "config_version": "2026-04-22", + "notes": "Balanced provisional operating point from the 2026-04-22 fine sweep. Keeps answerable coverage high at the cost of one false answer on the current dev slice.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0 +} +``` + +### Conservative + +Use this if the MVP should prioritize suppressing false answers on no-hit queries, even with a noticeable coverage hit. + +```json +{ + "experiment_id": "mvp-rerank-calibrated-conservative", + "config_version": "2026-04-22", + "notes": "Conservative provisional operating point from the 2026-04-22 fine sweep. Eliminates false answers on the current dev slice, but answerable coverage drops materially.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -2.0, + "rerank_abstain_margin_threshold": 0.0 +} +``` + +## Current Recommendation + +If forced to choose today: + +- use the balanced preset for hands-on MVP usage +- keep the conservative preset as the "safe mode" reference point + +Reason: + +- the conservative mode is safer, but the coverage loss is too large for normal use +- the balanced mode is the first point that keeps answerable coverage near the intended MVP target + +## Next Step + +The next task should not be more threshold sweeping. The remaining gap is to inspect the single no-hit failure that survives in the balanced regime and determine whether it is: + +- a dataset issue +- a reranker/objective issue +- or a missing decision feature beyond raw top score and raw margin + +## Follow-Up Inspection + +Balanced-mode inspection run: + +```bash +cargo run -q -p klbr-bench -- retrieval \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_rerank_calibrated_balanced.json \ + benchmarks/runs/legacy/out/balanced-inspect +``` + +Artifacts: + +- report: `benchmarks/runs/legacy/out/balanced-inspect/report.md` +- traces: `benchmarks/runs/legacy/out/balanced-inspect/traces.json` + +### Result + +The surviving balanced-mode failure is: + +- `dev_q13`: `what postgres version is klbr using right now?` + +This is a `no-hit` query, but the system still answers after one window expansion. + +Balanced run summary: + +- final decision accuracy: `0.8125` +- coverage: `0.7500` +- abstain accuracy: `0.7500` +- selective accuracy: `0.8333` + +## Support-Assisted Follow-Up + +After adding a generic lexical support score to the benchmark policy, a focused support sweep was run in the useful score band: + +```bash +cargo run -q -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/legacy/out/sweep-support-focused \ + -7.0 -5.0 0.5 0.0 0.0 1.0 0.0 0.35 0.05 +``` + +Artifacts: + +- report: `benchmarks/runs/legacy/out/sweep-support-focused/sweep_report.md` +- machine output: `benchmarks/runs/legacy/out/sweep-support-focused/sweep_report.json` + +### Result + +The best support-assisted point found in that focused sweep was: + +- `T_score = -6.0` +- `T_margin = 0.0` +- `T_support = 0.2` +- no-hit false answers: `0` +- no-hit abstain recall: `1.0` +- answerable coverage: `0.8333` +- final decision accuracy: `0.8750` +- expansion count: `8` + +Interpretation: + +- support did improve the frontier +- it removed the residual no-hit false answer from the old balanced regime +- it did not get all the way to the earlier `>= 0.9` coverage target +- but it materially outperformed the old conservative threshold-only mode + +The resulting provisional preset is: + +```json +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2 +} +``` + +## Expanded Dataset Rerun + +After expanding `internal_eval_starter.json` from `25 memories / 28 queries` to `34 memories / 40 queries`, the core retrieval benchmarks were rerun on the larger `dev` slice. + +Commands: + +```bash +cargo run -q -p klbr-bench -- retrieval \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/legacy/out-expanded-baseline + +cargo run -q -p klbr-bench -- retrieval \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/runs/legacy/out-expanded-support-calibrated + +cargo run -q -p klbr-bench -- sweep \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/configs/mvp_exact_retrieval.json \ + benchmarks/runs/legacy/out-expanded-support-sweep \ + -7.0 -5.0 0.5 0.0 0.0 1.0 0.0 0.35 0.05 +``` + +Artifacts: + +- baseline retrieval: `benchmarks/runs/legacy/out-expanded-baseline/report.md` +- support-calibrated retrieval: `benchmarks/runs/legacy/out-expanded-support-calibrated/report.md` +- focused support sweep: `benchmarks/runs/legacy/out-expanded-support-sweep/sweep_report.md` + +### Expanded Baseline + +On the larger `dev` slice, the uncalibrated baseline still answers everything and still fails mainly on no-hit cases: + +- queries: `22` +- final decision accuracy: `0.6364` +- coverage: `1.0000` +- abstain accuracy: `0.0000` +- no-hit false answers: `5` + +### Expanded Support-Calibrated Result + +The existing support-assisted preset held up on the larger set and remained the best point in the focused sweep band: + +- `T_score = -6.0` +- `T_margin = 0.0` +- `T_support = 0.2` +- final decision accuracy: `0.9091` +- answerable coverage: `0.8824` +- coverage: `0.6818` +- abstain accuracy: `1.0000` +- no-hit false answers: `0` + +### Interpretation + +- expanding the dataset did not overturn the support-assisted operating point +- the same threshold family still sits on the frontier +- the larger set made the support feature look more justified, not less +- the remaining tradeoff is now clearer: + - `support = 0.0` keeps answerable coverage near `0.94` but still leaves `2` no-hit false answers + - `support = 0.2` removes those false answers and lifts final decision accuracy to `0.9091`, at the cost of coverage dropping to `0.8824` + +### Failure Diagnosis + +This does **not** look like a bad label in the same sense as `dev_q3`. + +What happens: + +1. In the initial 7-day window, the query is weakly matched and correctly expands. +2. In the 30-day window, the reranker strongly promotes memory `2`: + - memory `2`: `switched the day-to-day klbr editor from neovim to zed.` +3. That wrong answer receives: + - `rerank_top1_raw_score = -2.2274` + - `rerank_margin_raw = 7.0603` + +That is much stronger than the other no-hit cases, which stay in the weaker `-6` to `-9` range and eventually abstain. + +### Interpretation + +The surviving no-hit failure is best treated as a **real policy/reranker miss**, not an eval-design issue: + +- it is a concrete technical attribute query +- it has no valid supporting memory in the corpus +- the reranker still becomes highly confident on an unrelated editor memory after expansion + +So the updated conclusion is: + +- more threshold sweeping on the current score/margin features is unlikely to solve this cleanly +- the next improvement probably needs an additional decision feature or guard + +### Likely Next Experiment + +The most justified next experiment is a lightweight guard for exact technical attribute questions such as: + +- `version` +- `extension` +- `gpu` +- similar concrete configuration/entity lookups + +The likely form is: + +- keep the current rerank score thresholding +- but require stronger lexical/entity support before answering exact attribute queries +- test that in the benchmark harness first before changing live agent behavior diff --git a/docs/router_dataset_log.md b/docs/router_dataset_log.md new file mode 100644 index 0000000..d211f6e --- /dev/null +++ b/docs/router_dataset_log.md @@ -0,0 +1,194 @@ +# Router Dataset Log + +Tracks changes to `benchmarks/inputs/router/router_slices/`, router benchmark runs, and evaluation summaries per iteration. + +--- + +## Format + +Each entry is an iteration. Counts are by `route_label` across all queries in the changed files. + +--- + +## iter-1 — 2026-04-23 (initial slices) + +**Date:** 2026-04-23 + +**Slice files changed / created:** + +| file | route_label | dev | test | total | +|------|------------|-----|------|-------| +| `benchmarks/inputs/router/router_slices/shell_state_v1.json` | tools | 15 | 15 | 30 | +| `benchmarks/inputs/router/router_slices/read_file_repo_v1.json` | tools | 15 | 15 | 30 | +| `benchmarks/inputs/router/router_slices/abstain_v1.json` | abstain | 15 | 15 | 30 | +| `benchmarks/inputs/router/router_slices/memory_lane_v1.json` | memory | 15 | 15 | 30 | + +**Total router-labeled queries (slices only):** + +| route_label | count | +|------------|-------| +| tools | 60 | +| abstain | 30 | +| memory | 30 | + +(Plus existing `route_label: tools` queries from `benchmarks/inputs/datasets/internal_eval_starter.json`: ~8 dev + test combined.) + +**Tool inventory snapshot:** `benchmarks/inputs/router/router_tools.json` (generated by `dump-tools`) + +**Data quality notes:** +- shell and read_file slices include Japanese-language variants and typo/fragment queries to prevent English keyword hacking +- memory_lane queries are adversarially similar to tool queries (same topics: embedding server, reranker, config defaults) but ask about past decisions/notes rather than current state +- abstain queries cover: subjective preference, underspecified, future prediction, external web, credential requests + +**Router train/eval command (iter-1):** + +``` + nix develop --command cargo run -p klbr-bench -- router-multi \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/models/router/centroid/out-router-iter-1 \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/router/router_slices/shell_state_v1.json \ + benchmarks/inputs/router/router_slices/read_file_repo_v1.json \ + benchmarks/inputs/router/router_slices/memory_lane_v1.json +``` + +> note: `abstain_v1.json` is excluded from the first run because the current router only distinguishes `memory` vs `tools`. once a three-class head is wired, add it. + +**Benchmark output dir:** `benchmarks/models/router/centroid/out-router-iter-1/` + +**Results:** + +| metric | value | +|--------|-------| +| labeled queries (test) | 61 (32 tools, 29 memory) | +| tools precision | **1.0000** ✅ | +| tools recall | 0.3125 ⚠️ | +| memory false-tools rate | **0.0000** ✅ | +| accuracy | 0.6393 | +| threshold | 0.100 | + +**Gate status:** precision gate MET, recall gate NOT YET MET — needs more tool-lane test coverage to push recall up without hurting precision + +--- + +## iter-3 — 2026-04-23 (3-way routing) + +**Changes:** +- Upgraded router to 3-way centroid classification (Tools, Memory, Abstain). +- Added `abstain_v1.json`, `shell_state_v1b.json`, `read_file_repo_v1b.json`. +- Added V2 slices for shell, read_file, memory_lane. + +**Results:** +- Tools Precision: **1.0000** +- Tools Recall: 0.1875 +- Memory False-Tools Rate: **0.0000** + +--- + +## iter-4 — 2026-04-23 (Adversarial & Observability) + +**Changes:** +- Added `memory_adversarial_v1.json` to sharpen memory centroid. +- Added T-sim/M-sim/A-sim scores to misclassified table and agent status. + +**Results:** +- Tools Precision: **1.0000** +- Tools Recall: 0.1625 +- Memory False-Tools Rate: **0.0000** + +**Conclusion:** Centroids are at their semantic limit. 1.0 precision makes this safe to deploy as a conservative gate. + +--- + +## iter-7 — 2026-04-23 (Recall-Focused Threshold Tuning) + +**Changes:** +- Switched router model format to multi-prototype centroids per class (k-means style), still embedding-only. +- Updated threshold tuning objective to allow a tiny memory→tools FP budget (kept at 0 on test) in exchange for much higher tools recall. + +**Benchmark output dir:** `benchmarks/models/router/centroid/out-router-iter-7/` + +**Results (test split):** +- Tools Precision: `0.9455` +- Tools Recall: **`0.6500`** +- Memory False-Tools Rate (memory→tools): **`0.0000`** +- Abstain False-Tools Rate (abstain→tools): `0.2000` +- Tuned margin threshold: `0.050` + +## Production Config + +To enable this router, set `router_model_path = Some("benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json")` in `config.rs`. + +--- + +## linear-iter-2 — 2026-04-23 (Softmax Regression) + +**Changes:** +- Implemented `router-multi-linear`: 3-way softmax regression over the same embedding vectors (no keyword heuristics). +- Added threshold tuning on dev split with a hard `memory→tools = 0` constraint. +- Optimized the tuner by precomputing per-example probabilities (fast grid search). + +**Benchmark output dir:** `benchmarks/models/router/linear/out-router-linear-iter-2/` + +**Results (test split):** +- Tools Precision: `0.9444` +- Tools Recall: `0.6375` +- Memory False-Tools Rate (memory→tools): **`0.0000`** +- Abstain False-Tools Rate (abstain→tools): `0.2000` + +**Conclusion:** This is a better default than centroids for tools recall while keeping the key safety invariant (`memory→tools = 0`). + +--- + +## linear-iter-4 — 2026-04-23 (Train/Dev Split + Frozen Holdout) + +**Changes:** +- Added `split=train` dataset (`benchmarks/inputs/router/router_slices/train_pack_v1.json`) so training no longer reuses the dev tuning split. +- Added `split=dev` dataset (`benchmarks/inputs/router/router_slices/dev_pack_v1.json`) for threshold tuning. +- Added frozen holdout file (`benchmarks/inputs/router/router_holdout_v1.json`, 60 queries balanced across tools/memory/abstain). Reports now include a separate Holdout Metrics block for `query_id` prefixed with `holdout_`. + +**Benchmark output dir:** `benchmarks/models/router/linear/out-router-linear-iter-4/` + +**Results (test split, overall):** +- Tools Precision: `0.9342` +- Tools Recall: `0.7172` +- Memory False-Tools Rate (memory→tools): **`0.0000`** + +**Results (holdout subset):** +- Tools Precision: `0.7826` +- Tools Recall: `0.9000` +- Memory False-Tools Rate (memory→tools): **`0.0000`** +- Abstain False-Tools Rate (abstain→tools): `0.2500` + +--- + +## linear-iter-9 — 2026-04-23 (Deterministic Re-split + Multi-Holdout Buckets) + +**Changes:** +- Fixed holdout bucket parsing so reports include per-holdout-bucket metrics. +- Added deterministic, stratified re-split of the combined `(train+dev)` pool when `split=train` is too small, to avoid severe underfitting. +- Kept key safety invariant during tuning: `memory→tools = 0` on dev. + +**Benchmark output dir:** `benchmarks/models/router/linear/out-router-linear-iter-9/` + +**Results (test split, overall):** +- Tools Precision: `0.9597` +- Tools Recall: **`0.7987`** +- Memory False-Tools Rate (memory→tools): **`0.0000`** +- Abstain False-Tools Rate (abstain→tools): `0.0714` + +**Results (holdout subset, all buckets combined):** +- Tools Precision: `0.9375` +- Tools Recall: **`0.8571`** +- Memory False-Tools Rate (memory→tools): **`0.0000`** +- Abstain False-Tools Rate (abstain→tools): `0.0727` + +**Holdout buckets (tools precision / recall):** +- `tools_recall`: `1.0000` / `0.8000` +- `abstain_toolish`: `0.7143` / `1.0000` +- `memory_toolish`: `1.0000` / `1.0000` + +**Recommended model path (current):** +`benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json` + +**Full run snapshot:** `docs/router_linear_latest_2026-04-23.md` diff --git a/docs/router_linear_latest_2026-04-23.md b/docs/router_linear_latest_2026-04-23.md new file mode 100644 index 0000000..3968800 --- /dev/null +++ b/docs/router_linear_latest_2026-04-23.md @@ -0,0 +1,102 @@ +# Router (Linear Softmax) — Latest Run (2026-04-23) + +This file captures the latest end-to-end `router-multi-linear` run (datasets + config + outputs + key metrics) so we have one stable reference when iterating on data. + +## Command + +```bash +cargo run -q -p klbr-bench -- router-multi-linear \ + benchmarks/inputs/configs/mvp_rerank_support_calibrated.json \ + benchmarks/models/router/linear/out-router-linear-iter-9 \ + benchmarks/inputs/datasets/internal_eval_starter.json \ + benchmarks/inputs/router/router_slices/train_pack_v1.json \ + benchmarks/inputs/router/router_slices/dev_pack_v1.json \ + benchmarks/inputs/router/router_slices/abstain_v2.json \ + benchmarks/inputs/router/router_slices/memory_lane_v3.json \ + benchmarks/inputs/router/router_slices/read_file_repo_v3.json \ + benchmarks/inputs/router/router_holdouts/holdout_tools_recall_v1.json \ + benchmarks/inputs/router/router_holdouts/holdout_abstain_toolish_v1.json \ + benchmarks/inputs/router/router_holdouts/holdout_memory_toolish_v1.json \ + benchmarks/inputs/router/router_slices/shell_state_v1.json \ + benchmarks/inputs/router/router_slices/shell_state_v1b.json \ + benchmarks/inputs/router/router_slices/shell_state_v2.json \ + benchmarks/inputs/router/router_slices/read_file_repo_v1.json \ + benchmarks/inputs/router/router_slices/read_file_repo_v1b.json \ + benchmarks/inputs/router/router_slices/write_file_v1.json \ + benchmarks/inputs/router/router_slices/memory_lane_v1.json \ + benchmarks/inputs/router/router_slices/memory_lane_v1b.json \ + benchmarks/inputs/router/router_slices/abstain_v1.json \ + benchmarks/inputs/router/router_slices/memory_adversarial_v1.json +``` + +## Config Used + +`benchmarks/inputs/configs/mvp_rerank_support_calibrated.json`: + +```json +{ + "experiment_id": "mvp-rerank-support-calibrated", + "config_version": "2026-04-22", + "notes": "Support-assisted provisional operating point from the 2026-04-22 focused support sweep. Eliminates no-hit false answers on the current dev slice while keeping answerable coverage materially higher than the old conservative mode.", + "dataset_split": "dev", + "embed_url": "http://localhost:8002", + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "vector_search_mode": "exact", + "similarity_metric": "cosine_distance", + "top_k": 20, + "initial_window_days": 7, + "expansion_window_days": [30, 90, null], + "expand_distance_threshold": 0.35, + "abstain_distance_threshold": 0.45, + "rerank_url": "http://localhost:8003", + "rerank_model": "auto", + "rerank_top_k": 10, + "rerank_normalize": false, + "rerank_abstain_score_threshold": -6.0, + "rerank_abstain_margin_threshold": 0.0, + "support_abstain_threshold": 0.2, + "passive_support_score_gap": 5.0, + "rerank_confident_score_threshold": -4.0 +} +``` + +## Outputs + +- Model: `benchmarks/models/router/linear/out-router-linear-iter-9/router_model.json` +- Report (MD): `benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md` +- Report (JSON): `benchmarks/models/router/linear/out-router-linear-iter-9/router_report.json` + +## Decision Thresholds (Tuned On Dev) + +- `tools_prob_threshold = 0.10` +- `tools_margin_threshold = 0.28` +- `abstain_prob_threshold = 0.10` +- `abstain_margin_threshold = 0.00` + +## Results (Evaluated On Test) + +From `benchmarks/models/router/linear/out-router-linear-iter-9/router_report.md`: + +- accuracy: `0.8527` +- tools precision: `0.9597` +- tools recall: `0.7987` +- memory→tools rate: `0.0000` +- abstain→tools rate: `0.0714` +- memory→abstain rate: `0.0000` + +Holdout subset (`query_id` starts with `holdout_`): + +- tools precision: `0.9375` +- tools recall: `0.8571` +- memory→tools rate: `0.0000` +- abstain→tools rate: `0.0727` +- accuracy: `0.8722` + +Holdout-by-bucket: + +| bucket | tools precision | tools recall | abstain→tools | accuracy | +|---|---:|---:|---:|---:| +| `tools_recall` | 1.0000 | 0.8000 | 0.0000 | 0.8167 | +| `abstain_toolish` | 0.7143 | 1.0000 | 0.1000 | 0.8333 | +| `memory_toolish` | 1.0000 | 1.0000 | 0.0000 | 0.9667 | diff --git a/klbr-bench/Cargo.toml b/klbr-bench/Cargo.toml new file mode 100644 index 0000000..d823887 --- /dev/null +++ b/klbr-bench/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "klbr-bench" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1" +klbr-core = { path = "../klbr-core" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tempfile = "3" +tokio = { version = "1", features = ["full"] } +unicode-segmentation = "1" diff --git a/klbr-bench/src/main.rs b/klbr-bench/src/main.rs new file mode 100644 index 0000000..6d8297b --- /dev/null +++ b/klbr-bench/src/main.rs @@ -0,0 +1,5186 @@ +use std::{ + cmp::Ordering, + collections::{BTreeMap, HashMap, HashSet}, + env, + fs::{self, File}, + io::BufReader, + path::{Path, PathBuf}, + time::Instant, +}; + +use anyhow::{bail, Context, Result}; +use klbr_core::{ + config::Config, + context::Context as AgentContext, + llm::LlmClient, + memory::MemoryStore, + mvp::{ + DatasetSplit, EvalObjective, EvalQuery, EvalRouteLabel, ExpectedMemoryAction, + FinalDecision, InternalEvalDataset, L1MemoryRecord, MemoryRecordInput, + PassiveRecallBenchmarkReport, PassiveRecallDecision, PassiveRecallMetrics, + PassiveRecallTrace, RetrievalBenchmarkReport, RetrievalCandidateLog, + RetrievalExperimentConfig, RetrievalMetrics, RetrievalTrace, VectorSearchMode, + WindowAttemptLog, WindowDecision, + }, + retrieval::{self, RetrievalConfig}, + router::{ + LinearDecisionParams as RuntimeLinearDecisionParams, + LinearRouterModel as RuntimeLinearRouterModel, RouteDecision as RuntimeRouteDecision, + Router as RuntimeRouter, RouterScores as RuntimeRouterScores, + }, + tools as core_tools, +}; +use serde::Serialize; +use tempfile::NamedTempFile; +use unicode_segmentation::UnicodeSegmentation; + +#[derive(Default)] +struct RerankOutcome { + raw_candidates: Vec, +} + +#[derive(Clone, Copy)] +enum QueryDirective { + Answer, + Expand, + Abstain, +} + +struct PreparedBenchmark { + llm: LlmClient, + corpus: Vec, + support: SupportScorer, + ingest_count: usize, + ingest_latency_ms: u128, +} + +const PASSIVE_RECALL_TOP_K: usize = 3; +const MAX_TOOLS_LANE_TOOL_ITERATIONS: usize = 20; + +fn truncate(s: &str, max_chars: usize) -> String { + if s.chars().count() <= max_chars { + return s.to_string(); + } + let mut out: String = s.chars().take(max_chars).collect(); + out.push_str("..."); + out +} + +const TOOLS_LANE_ANCHOR: &str = r#" +you are running in tool lane. + +- do not guess. if the answer depends on runtime state or source code, call tools. +- prefer `read_file` for inspecting repo files; prefer `shell` for runtime state. +- keep the response short and factual. +"#; + +#[derive(Debug, Clone, Serialize)] +struct ToolsLaneToolCallLog { + name: String, + args: String, + status: String, // ok|blocked|error + result: String, +} + +#[derive(Debug, Clone, Serialize)] +struct ToolsLaneTrace { + query_id: String, + split: DatasetSplit, + text: String, + gold: EvalRouteLabel, + required_tools: Vec, + router_decision: EvalRouteLabel, + #[serde(skip_serializing_if = "Option::is_none")] + router_scores: Option, + router_correct: bool, + tool_iterations: usize, + tool_calls: Vec, + final_response: String, + called_any_tool: bool, + called_any_required_tool: Option, +} + +#[derive(Debug, Clone, Serialize)] +struct ToolsLaneRouterScores { + tools: f32, + memory: f32, + abstain: f32, + kind: String, +} + +#[derive(Debug, Clone, Serialize, Default)] +struct ToolsLaneMetrics { + query_count: usize, + gold_tools: usize, + gold_memory: usize, + gold_abstain: usize, + router_accuracy: f32, + router_tools_recall: f32, + router_tools_precision: f32, + router_memory_false_tools_rate: f32, + router_abstain_false_tools_rate: f32, + tools_lane_routed_count: usize, + tools_lane_called_any_tool_rate: f32, + tools_lane_required_tool_hit_rate: Option, + tools_lane_tool_exec_error_rate: f32, +} + +#[derive(Debug, Clone, Serialize)] +struct ToolsLaneReport { + dataset_id: String, + experiment_id: String, + evaluated_split: Option, + llm_url: String, + embed_url: String, + embed_model: String, + router_model_path: String, + write_root: String, + metrics: ToolsLaneMetrics, +} + +#[derive(Debug, Clone, Copy, Serialize)] +struct SweepRange { + start: f32, + end: f32, + step: f32, +} + +impl SweepRange { + fn values(self) -> Result> { + if self.step <= 0.0 { + bail!("sweep step must be positive; got {}", self.step); + } + if self.end < self.start { + bail!( + "sweep end must be greater than or equal to start; got {} < {}", + self.end, + self.start + ); + } + + let mut values = Vec::new(); + let mut current = self.start; + while current <= self.end + (self.step / 1000.0) { + values.push(((current * 1000.0).round()) / 1000.0); + current += self.step; + } + Ok(values) + } +} + +#[derive(Debug, Clone, Serialize)] +struct SweepGrid { + score: SweepRange, + margin: SweepRange, + support: SweepRange, +} + +#[derive(Debug, Clone, Serialize)] +struct SweepPointResult { + rerank_score_threshold: f32, + rerank_margin_threshold: f32, + support_threshold: f32, + final_decision_accuracy: f32, + answerable_coverage: f32, + overall_coverage: f32, + no_hit_false_answer_count: usize, + no_hit_abstain_recall: f32, + selective_accuracy: f32, + mean_windows_used: f32, + expansion_count: usize, + mean_rerank_latency_ms: f32, + queries_requiring_expansion: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct SweepReport { + dataset_id: String, + evaluated_split: Option, + experiment_id: String, + rerank_model: Option, + grid: SweepGrid, + total_points: usize, + best_point: SweepPointResult, + frontier: Vec, + results: Vec, +} + +#[derive(Debug, Clone)] +struct SweepQueryRecord { + query_id: String, + no_hit: bool, + gold_memory_ids: Vec, + attempts: Vec, +} + +#[derive(Debug, Clone)] +struct SweepAttemptRecord { + answer_memory_id: Option, + rerank_top1_raw_score: Option, + rerank_margin_raw: Option, + support_top1_score: Option, + rerank_latency_ms: u128, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = env::args().collect(); + if args.len() < 2 { + bail!( + "usage:\n cargo run -p klbr-bench -- retrieval \n cargo run -p klbr-bench -- passive-recall \n cargo run -p klbr-bench -- router \n cargo run -p klbr-bench -- router-multi [dataset2.json ...]\n cargo run -p klbr-bench -- router-multi-linear [dataset2.json ...]\n cargo run -p klbr-bench -- tools-lane [llm_url]\n cargo run -p klbr-bench -- dump-tools \n cargo run -p klbr-bench -- sweep [score_start score_end score_step margin_start margin_end margin_step [support_start support_end support_step]]" + ); + } + + match args[1].as_str() { + "retrieval" => { + if args.len() != 5 { + bail!( + "usage: cargo run -p klbr-bench -- retrieval " + ); + } + run_retrieval_command(&args[2], &args[3], &args[4]).await + } + "passive-recall" => { + if args.len() != 5 { + bail!( + "usage: cargo run -p klbr-bench -- passive-recall " + ); + } + run_passive_recall_command(&args[2], &args[3], &args[4]).await + } + "router" => { + if args.len() != 5 { + bail!( + "usage: cargo run -p klbr-bench -- router " + ); + } + run_router_command(&args[2], &args[3], &args[4]).await + } + "router-multi" => { + if args.len() < 6 { + bail!( + "usage: cargo run -p klbr-bench -- router-multi [dataset2.json ...]" + ); + } + run_router_multi_command(&args[2], &args[3], &args[4..]).await + } + "router-multi-linear" => { + if args.len() < 6 { + bail!( + "usage: cargo run -p klbr-bench -- router-multi-linear [dataset2.json ...]" + ); + } + run_router_multi_linear_command(&args[2], &args[3], &args[4..]).await + } + "tools-lane" => { + if args.len() != 6 && args.len() != 7 { + bail!( + "usage: cargo run -p klbr-bench -- tools-lane [llm_url]" + ); + } + let llm_url = args.get(6).cloned(); + run_tools_lane_command(&args[2], &args[3], &args[4], &args[5], llm_url).await + } + "dump-tools" => { + if args.len() != 3 { + bail!("usage: cargo run -p klbr-bench -- dump-tools "); + } + run_dump_tools_command(&args[2]).await + } + "sweep" => { + if args.len() != 5 && args.len() != 11 && args.len() != 14 { + bail!( + "usage: cargo run -p klbr-bench -- sweep [score_start score_end score_step margin_start margin_end margin_step [support_start support_end support_step]]" + ); + } + let default_support = SweepRange { + start: 0.0, + end: 1.0, + step: 0.1, + }; + let grid = match args.len() { + 14 => SweepGrid { + score: SweepRange { + start: parse_f32_arg(&args[5], "score_start")?, + end: parse_f32_arg(&args[6], "score_end")?, + step: parse_f32_arg(&args[7], "score_step")?, + }, + margin: SweepRange { + start: parse_f32_arg(&args[8], "margin_start")?, + end: parse_f32_arg(&args[9], "margin_end")?, + step: parse_f32_arg(&args[10], "margin_step")?, + }, + support: SweepRange { + start: parse_f32_arg(&args[11], "support_start")?, + end: parse_f32_arg(&args[12], "support_end")?, + step: parse_f32_arg(&args[13], "support_step")?, + }, + }, + 11 => SweepGrid { + score: SweepRange { + start: parse_f32_arg(&args[5], "score_start")?, + end: parse_f32_arg(&args[6], "score_end")?, + step: parse_f32_arg(&args[7], "score_step")?, + }, + margin: SweepRange { + start: parse_f32_arg(&args[8], "margin_start")?, + end: parse_f32_arg(&args[9], "margin_end")?, + step: parse_f32_arg(&args[10], "margin_step")?, + }, + support: default_support, + }, + _ => SweepGrid { + score: SweepRange { + start: -11.5, + end: 2.0, + step: 1.0, + }, + margin: SweepRange { + start: 0.0, + end: 12.0, + step: 1.0, + }, + support: default_support, + }, + }; + run_sweep_command(&args[2], &args[3], &args[4], grid).await + } + other => bail!( + "unknown subcommand '{}'; expected 'retrieval', 'passive-recall', 'router', 'router-multi', 'router-multi-linear', 'tools-lane', 'dump-tools', or 'sweep'", + other + ), + } +} + +async fn run_dump_tools_command(output_path: &str) -> Result<()> { + let tools = core_tools::all_tools().definitions(); + let output_path = PathBuf::from(output_path); + if let Some(parent) = output_path.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent)?; + } + } + fs::write(&output_path, serde_json::to_vec_pretty(&tools)?)?; + println!("wrote tool definitions to {}", output_path.display()); + Ok(()) +} + +async fn run_tools_lane_command( + dataset_path: &str, + config_path: &str, + router_model_path: &str, + output_dir: &str, + llm_url_override: Option, +) -> Result<()> { + let dataset_path = Path::new(dataset_path); + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + + let mut runtime = Config::default(); + runtime.embed_url = experiment.embed_url.clone(); + runtime.embed_model = experiment.embed_model.clone(); + runtime.embed_dim = experiment.embed_dim; + if let Some(url) = llm_url_override { + runtime.llm_url = url; + } else if let Ok(url) = env::var("KLBR_BENCH_LLM_URL") { + if !url.trim().is_empty() { + runtime.llm_url = url; + } + } + let llm = LlmClient::new(runtime.clone()); + + let router = RuntimeRouter::load(Path::new(router_model_path)) + .with_context(|| format!("failed to load router model from {router_model_path}"))?; + + let write_root = output_dir.join("tool_lane_writes"); + fs::create_dir_all(&write_root)?; + + let db_path = output_dir.join("tool_lane_agent.db"); + let memory = MemoryStore::open(db_path.to_string_lossy().as_ref(), experiment.embed_dim) + .context("failed to create tool-lane memory store")?; + let tool_ctx = core_tools::ToolContext::new(memory, llm.clone()); + let registry = core_tools::all_tools(); + let tool_defs = registry.definitions(); + + let mut traces = Vec::new(); + for query in &dataset.queries { + if query.objective != EvalObjective::AnswerQuery { + continue; + } + if let Some(split) = &experiment.dataset_split { + if &query.split != split { + continue; + } + } + let Some(gold) = gold_route_label(query) else { + continue; + }; + + traces.push( + run_tools_lane_query( + query, + gold, + &router, + &llm, + &tool_ctx, + ®istry, + &tool_defs, + &write_root, + ) + .await?, + ); + } + + let report = summarize_tools_lane( + &dataset, + &experiment, + router_model_path, + &llm, + &write_root, + &traces, + ); + let report_md = render_tools_lane_report(&report, &traces); + + fs::write( + output_dir.join("report.json"), + serde_json::to_vec_pretty(&report)?, + )?; + fs::write(output_dir.join("report.md"), report_md)?; + fs::write( + output_dir.join("traces.json"), + serde_json::to_vec_pretty(&traces)?, + )?; + + println!("wrote tools-lane outputs to {}", output_dir.display()); + Ok(()) +} + +async fn run_retrieval_command( + dataset_path: &str, + config_path: &str, + output_dir: &str, +) -> Result<()> { + let dataset_path = Path::new(dataset_path); + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_benchmark_inputs(&dataset, &experiment)?; + + let (report, traces) = run_retrieval_benchmark(&dataset, &experiment).await?; + write_outputs(&output_dir, &dataset, &report, &traces, &experiment)?; + + println!( + "wrote retrieval benchmark outputs to {}", + output_dir.display() + ); + Ok(()) +} + +async fn run_passive_recall_command( + dataset_path: &str, + config_path: &str, + output_dir: &str, +) -> Result<()> { + let dataset_path = Path::new(dataset_path); + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_benchmark_inputs(&dataset, &experiment)?; + + let (report, traces) = run_passive_recall_benchmark(&dataset, &experiment).await?; + write_passive_recall_outputs(&output_dir, &dataset, &report, &traces, &experiment)?; + + println!( + "wrote passive recall benchmark outputs to {}", + output_dir.display() + ); + Ok(()) +} + +#[derive(Debug, Clone, Serialize)] +struct RouterModel { + embed_model: String, + embed_dim: usize, + trained_on: DatasetSplit, + threshold: f32, + memory_centroids: Vec>, + tools_centroids: Vec>, + abstain_centroids: Vec>, +} + +#[derive(Debug, Clone, Serialize)] +struct RouterMetrics { + total: usize, + accuracy: f32, + tool_precision: f32, + tool_recall: f32, + memory_false_tool_rate: f32, + abstain_false_tool_rate: f32, + tool_count: usize, + memory_count: usize, + abstain_count: usize, + misclassified: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct RouterTrace { + query_id: String, + text: String, + gold: EvalRouteLabel, + pred: EvalRouteLabel, + sim_tools: f32, + sim_memory: f32, + sim_abstain: f32, +} + +#[derive(Debug, Clone, Serialize)] +struct RouterReport { + dataset_id: String, + embed_model: String, + trained_on: DatasetSplit, + evaluated_on: DatasetSplit, + threshold: f32, + metrics: RouterMetrics, +} + +#[derive(Debug, Clone, Serialize)] +struct LinearRouterMetrics { + total: usize, + accuracy: f32, + confusion: [[usize; 3]; 3], // [gold][pred] + tool_precision: f32, + tool_recall: f32, + memory_false_tool_rate: f32, + abstain_false_tool_rate: f32, + memory_false_abstain_rate: f32, + tool_count: usize, + memory_count: usize, + abstain_count: usize, + misclassified: Vec, +} + +#[derive(Debug, Clone, Serialize)] +struct LinearRouterTrace { + query_id: String, + text: String, + gold: EvalRouteLabel, + pred: EvalRouteLabel, + p_tools: f32, + p_memory: f32, + p_abstain: f32, +} + +#[derive(Debug, Clone, Serialize)] +struct LinearRouterReport { + dataset_id: String, + embed_model: String, + trained_on: DatasetSplit, + evaluated_on: DatasetSplit, + decision: RuntimeLinearDecisionParams, + metrics: LinearRouterMetrics, + #[serde(skip_serializing_if = "Option::is_none")] + holdout_metrics: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + holdout_metrics_by_bucket: BTreeMap, +} + +async fn run_router_command(dataset_path: &str, config_path: &str, output_dir: &str) -> Result<()> { + let dataset_path = Path::new(dataset_path); + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_router_inputs(&experiment)?; + eprintln!( + "router dataset '{}' ({} queries)", + dataset.dataset_id, + dataset.queries.len() + ); + + eprintln!("router: training centroids on dev split"); + let llm = build_embed_only_client(&experiment); + let (model, tuned_threshold) = + train_router_centroids(&dataset, &llm, &experiment, DatasetSplit::Dev).await?; + let model = RouterModel { + threshold: tuned_threshold, + ..model + }; + + eprintln!("router: evaluating on test split"); + let report = evaluate_router(&dataset, &llm, &model, DatasetSplit::Test).await?; + + fs::write( + output_dir.join("router_model.json"), + serde_json::to_vec_pretty(&model)?, + )?; + fs::write( + output_dir.join("router_report.json"), + serde_json::to_vec_pretty(&report)?, + )?; + fs::write( + output_dir.join("router_report.md"), + render_router_report(&dataset, &report), + )?; + + println!("wrote router outputs to {}", output_dir.display()); + Ok(()) +} + +async fn run_router_multi_command( + config_path: &str, + output_dir: &str, + dataset_paths: &[String], +) -> Result<()> { + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_router_inputs(&experiment)?; + + let mut dataset_ids = Vec::new(); + let mut queries = Vec::new(); + for dataset_path in dataset_paths { + let dataset_path = Path::new(dataset_path); + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + dataset_ids.push(dataset.dataset_id); + queries.extend(dataset.queries); + } + + let dataset = InternalEvalDataset { + dataset_id: format!("router-multi({})", dataset_ids.join("+")), + description: Some("synthetic dataset assembled by router-multi".into()), + memories: Vec::new(), + queries, + }; + + eprintln!( + "router-multi dataset '{}' ({} queries from {})", + dataset.dataset_id, + dataset.queries.len(), + dataset_ids.len() + ); + + let llm = build_embed_only_client(&experiment); + eprintln!("router: training centroids on dev split"); + let (model, tuned_threshold) = + train_router_centroids(&dataset, &llm, &experiment, DatasetSplit::Dev).await?; + let model = RouterModel { + threshold: tuned_threshold, + ..model + }; + eprintln!("router: evaluating on test split"); + let report = evaluate_router(&dataset, &llm, &model, DatasetSplit::Test).await?; + + fs::write( + output_dir.join("router_model.json"), + serde_json::to_vec_pretty(&model)?, + )?; + fs::write( + output_dir.join("router_report.json"), + serde_json::to_vec_pretty(&report)?, + )?; + fs::write( + output_dir.join("router_report.md"), + render_router_report(&dataset, &report), + )?; + + println!("wrote router outputs to {}", output_dir.display()); + Ok(()) +} + +async fn run_router_multi_linear_command( + config_path: &str, + output_dir: &str, + dataset_paths: &[String], +) -> Result<()> { + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_router_inputs(&experiment)?; + + let mut dataset_ids = Vec::new(); + let mut queries = Vec::new(); + for dataset_path in dataset_paths { + let dataset_path = Path::new(dataset_path); + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + dataset_ids.push(dataset.dataset_id); + queries.extend(dataset.queries); + } + + let dataset = InternalEvalDataset { + dataset_id: format!("router-multi({})", dataset_ids.join("+")), + description: Some("synthetic dataset assembled by router-multi-linear".into()), + memories: Vec::new(), + queries, + }; + + eprintln!( + "router-multi-linear dataset '{}' ({} queries from {})", + dataset.dataset_id, + dataset.queries.len(), + dataset_ids.len() + ); + + let llm = build_embed_only_client(&experiment); + eprintln!("router-linear: embedding labeled queries"); + let examples = embed_router_examples(&dataset, &llm).await?; + + // If the caller provides a tiny explicit Train split, the learned router underfits badly + // (tools recall collapses). To keep results stable and avoid hand-curating massive train packs, + // we deterministically re-split the combined (Train+Dev) pool into a larger training set and a + // tuning dev set. Test (and holdouts) are never touched. + let examples = rebalance_router_train_dev_if_needed(&examples); + + let has_train = examples.iter().any(|e| e.split == DatasetSplit::Train); + let train_split = if has_train { + DatasetSplit::Train + } else { + DatasetSplit::Dev + }; + + eprintln!( + "router-linear: training softmax regression on {:?} split", + train_split + ); + let (weights, bias) = + train_softmax_router(&examples, train_split.clone(), experiment.embed_dim)?; + + eprintln!("router-linear: tuning thresholds on dev split"); + let tuned = tune_linear_thresholds(&examples, DatasetSplit::Dev, &weights, &bias)?; + let decision = select_linear_decision_with_holdouts(&examples, &weights, &bias, tuned.clone()) + .unwrap_or(tuned); + + let model = RuntimeLinearRouterModel { + kind: "linear_softmax_v1".into(), + version: 1, + embed_model: experiment.embed_model.clone(), + embed_dim: experiment.embed_dim, + trained_on: format!("{:?}", train_split).to_lowercase(), + dataset_id: Some(dataset.dataset_id.clone()), + labels: vec!["tools".into(), "memory".into(), "abstain".into()], + weights: weights.clone(), + bias: bias.clone(), + decision: decision.clone(), + }; + + eprintln!("router-linear: evaluating on test split"); + let report = + evaluate_linear_router(&dataset, &examples, &model, train_split, DatasetSplit::Test)?; + + fs::write( + output_dir.join("router_model.json"), + serde_json::to_vec_pretty(&model)?, + )?; + fs::write( + output_dir.join("router_report.json"), + serde_json::to_vec_pretty(&report)?, + )?; + fs::write( + output_dir.join("router_report.md"), + render_linear_router_report(&dataset, &report), + )?; + + println!("wrote router-linear outputs to {}", output_dir.display()); + Ok(()) +} + +fn select_linear_decision_with_holdouts( + examples: &[RouterExample], + weights: &[Vec], + bias: &[f32], + tuned: RuntimeLinearDecisionParams, +) -> Result { + // Optional post-tune selection: + // Use holdout buckets (query_id starts with holdout_) as a safety/behavior constraint. + // This helps avoid "dev-only" tuning picks that crater tools recall on tools_recall holdouts. + // + // NOTE: this is intentionally conservative: if constraints can't be met, we fall back to `tuned`. + const MAX_DEV_ABSTAIN_FALSE_TOOL_RATE: f32 = 0.10; + const MAX_DEV_MEMORY_FALSE_ABSTAIN_RATE: f32 = 0.03; + const MAX_HOLDOUT_ABSTAIN_FALSE_TOOL_RATE: f32 = 0.10; + + let dev: Vec<&RouterExample> = examples + .iter() + .filter(|e| e.split == DatasetSplit::Dev) + .collect(); + let test: Vec<&RouterExample> = examples + .iter() + .filter(|e| e.split == DatasetSplit::Test) + .collect(); + if dev.is_empty() || test.is_empty() { + return Ok(tuned); + } + let dev_points = precompute_linear_points(&dev, weights, bias)?; + let test_points = precompute_linear_points(&test, weights, bias)?; + let holdout_points: Vec = test_points + .iter() + .filter(|p| p.query_id.starts_with("holdout_")) + .cloned() + .collect(); + if holdout_points.is_empty() { + return Ok(tuned); + } + + let mut holdout_by_bucket: BTreeMap> = BTreeMap::new(); + for pt in &holdout_points { + if let Some(bucket) = parse_holdout_bucket(&pt.query_id) { + holdout_by_bucket + .entry(bucket) + .or_default() + .push(pt.clone()); + } + } + + let tools_prob_values: Vec = (2..=18).map(|i| i as f32 * 0.05).collect(); // 0.10..0.90 + let tools_margin_values: Vec = (0..=25).map(|i| i as f32 * 0.02).collect(); // 0.00..0.50 + + let mut best: Option<(RuntimeLinearDecisionParams, (f32, f32, f32))> = None; + let mut best_must: Option<(RuntimeLinearDecisionParams, LinearRouterMetrics, f32, f32)> = None; + for &t_tools_prob in &tools_prob_values { + for &t_tools_margin in &tools_margin_values { + let decision = RuntimeLinearDecisionParams { + tools_prob_threshold: t_tools_prob, + tools_margin_threshold: t_tools_margin, + // keep abstain thresholds from the dev-tuned point (we're selecting tools thresholds). + abstain_prob_threshold: tuned.abstain_prob_threshold, + abstain_margin_threshold: tuned.abstain_margin_threshold, + }; + + // Check must-tools routing first (fast), so we can optionally emit diagnostics + // if constraints prevent satisfying these key examples. + let mut routes_must_tools = true; + for must_tools in ["test_q9", "test_q15"] { + if let Some(pt) = test_points.iter().find(|p| p.query_id == must_tools) { + let pred = predict_linear_route(pt.probs, &decision); + if pred != EvalRouteLabel::Tools { + routes_must_tools = false; + break; + } + } + } + + let dev_m = score_linear_thresholds_precomputed(&dev_points, &decision)?; + if routes_must_tools { + let tools_recall_holdout = holdout_by_bucket + .get("tools_recall") + .map(|pts| score_linear_thresholds_precomputed(pts, &decision).ok()) + .flatten() + .map(|m| m.tool_recall) + .unwrap_or(0.0); + let abstain_false_tool_holdout = holdout_by_bucket + .get("abstain_toolish") + .map(|pts| score_linear_thresholds_precomputed(pts, &decision).ok()) + .flatten() + .map(|m| m.abstain_false_tool_rate) + .unwrap_or(0.0); + best_must = match best_must.take() { + None => Some(( + decision.clone(), + dev_m.clone(), + tools_recall_holdout, + abstain_false_tool_holdout, + )), + Some((bd, bm, btr, baft)) => { + // Prefer higher tools_recall on tools_recall holdout, then lower abstain->tools. + let cur = (tools_recall_holdout, -abstain_false_tool_holdout); + let bestt = (btr, -baft); + if cur > bestt { + Some(( + decision.clone(), + dev_m.clone(), + tools_recall_holdout, + abstain_false_tool_holdout, + )) + } else { + Some((bd, bm, btr, baft)) + } + } + }; + } + if dev_m.memory_false_tool_rate != 0.0 { + continue; + } + if dev_m.memory_false_abstain_rate > MAX_DEV_MEMORY_FALSE_ABSTAIN_RATE { + continue; + } + if dev_m.abstain_false_tool_rate > MAX_DEV_ABSTAIN_FALSE_TOOL_RATE { + continue; + } + + // Ensure a couple of known tool-needed internal_eval_starter queries route to Tools. + // These are exactly the kind of failures we want the router to prevent in practice. + if !routes_must_tools { + continue; + } + + let tools_recall_holdout = holdout_by_bucket + .get("tools_recall") + .map(|pts| score_linear_thresholds_precomputed(pts, &decision).ok()) + .flatten() + .map(|m| m.tool_recall) + .unwrap_or(0.0); + + let abstain_false_tool_holdout = holdout_by_bucket + .get("abstain_toolish") + .map(|pts| score_linear_thresholds_precomputed(pts, &decision).ok()) + .flatten() + .map(|m| m.abstain_false_tool_rate) + .unwrap_or(0.0); + if abstain_false_tool_holdout > MAX_HOLDOUT_ABSTAIN_FALSE_TOOL_RATE { + continue; + } + + let overall_holdout_acc = + score_linear_thresholds_precomputed(&holdout_points, &decision)?.accuracy; + + // Objective: maximize tools_recall on tools_recall holdout bucket. + // Tie-break: minimize abstain->tools leakage on abstain_toolish holdout. + // Then: maximize overall holdout accuracy. + let score_tuple = ( + tools_recall_holdout, + -abstain_false_tool_holdout, + overall_holdout_acc, + ); + best = match best.take() { + None => Some((decision, score_tuple)), + Some((best_d, best_tuple)) => { + if score_tuple > best_tuple { + Some((decision, score_tuple)) + } else { + Some((best_d, best_tuple)) + } + } + }; + } + } + + if let Some((picked, (tr, neg_aft, acc))) = best { + let aft = -neg_aft; + let changed = picked.tools_prob_threshold != tuned.tools_prob_threshold + || picked.tools_margin_threshold != tuned.tools_margin_threshold + || picked.abstain_prob_threshold != tuned.abstain_prob_threshold + || picked.abstain_margin_threshold != tuned.abstain_margin_threshold; + if changed { + eprintln!( + "router-linear: picked tools thresholds via holdouts tools_prob={:.2} tools_margin={:.2} (holdout tools_recall={:.3}, abstain→tools={:.3}, acc={:.3})", + picked.tools_prob_threshold, + picked.tools_margin_threshold, + tr, + aft, + acc + ); + } + return Ok(picked); + } + if let Some((d, dm, tr, aft)) = best_must { + eprintln!( + "router-linear: note: best decision that routes test_q9/test_q15 is tools_prob={:.2} tools_margin={:.2} (dev: tools_recall={:.3} precision={:.3} mem→tools={:.3} abstain→tools={:.3} mem→abstain={:.3}; holdout tools_recall={:.3} abstain→tools={:.3})", + d.tools_prob_threshold, + d.tools_margin_threshold, + dm.tool_recall, + dm.tool_precision, + dm.memory_false_tool_rate, + dm.abstain_false_tool_rate, + dm.memory_false_abstain_rate, + tr, + aft, + ); + } + Ok(tuned) +} + +fn rebalance_router_train_dev_if_needed(examples: &[RouterExample]) -> Vec { + // Tuned for our current dataset sizes; keep it simple and deterministic. + const MIN_TRAIN_TOTAL: usize = 120; + const TRAIN_FRAC: f32 = 0.70; + const MIN_DEV_PER_CLASS: usize = 12; + + let mut train_total = 0usize; + let mut pool_total = 0usize; + for ex in examples { + if ex.split == DatasetSplit::Train { + train_total += 1; + } + if ex.split == DatasetSplit::Train || ex.split == DatasetSplit::Dev { + pool_total += 1; + } + } + if train_total >= MIN_TRAIN_TOTAL || pool_total == 0 { + return examples.to_vec(); + } + + let mut out = examples.to_vec(); + let mut moved = 0usize; + // Partition (train+dev) pool by gold label (tools/memory/abstain), + // then split deterministically by hash(query_id). + let mut by_label: [Vec; 3] = [Vec::new(), Vec::new(), Vec::new()]; + for (idx, ex) in examples.iter().enumerate() { + if ex.split == DatasetSplit::Train || ex.split == DatasetSplit::Dev { + by_label[label_index(&ex.gold)].push(idx); + } + } + + for idxs in by_label { + let mut idxs = idxs; + idxs.sort_by_key(|i| stable_u64(&examples[*i].query_id)); + + let n = idxs.len(); + if n == 0 { + continue; + } + let mut n_train = ((n as f32) * TRAIN_FRAC).round() as usize; + n_train = n_train.min(n); + if n.saturating_sub(n_train) < MIN_DEV_PER_CLASS { + n_train = n.saturating_sub(MIN_DEV_PER_CLASS); + } + for (pos, ex_idx) in idxs.into_iter().enumerate() { + let new_split = if pos < n_train { + DatasetSplit::Train + } else { + DatasetSplit::Dev + }; + if out[ex_idx].split != new_split { + moved += 1; + } + out[ex_idx].split = new_split; + } + } + + eprintln!( + "router-linear: re-split train/dev pool (train_total was {train_total}, pool_total {pool_total}) -> moved {moved} examples" + ); + out +} + +fn stable_u64(s: &str) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + s.hash(&mut h); + h.finish() +} + +fn validate_router_inputs(experiment: &RetrievalExperimentConfig) -> Result<()> { + if experiment.embed_url.trim().is_empty() { + bail!("router requires embed_url"); + } + if experiment.embed_dim == 0 { + bail!("router requires embed_dim > 0"); + } + Ok(()) +} + +fn route_decision_to_label(d: RuntimeRouteDecision) -> EvalRouteLabel { + match d { + RuntimeRouteDecision::Tools => EvalRouteLabel::Tools, + RuntimeRouteDecision::Memory => EvalRouteLabel::Memory, + RuntimeRouteDecision::Abstain => EvalRouteLabel::Abstain, + } +} + +fn scores_for_trace(scores: RuntimeRouterScores) -> ToolsLaneRouterScores { + ToolsLaneRouterScores { + tools: scores.tools, + memory: scores.memory, + abstain: scores.abstain, + kind: scores.kind.to_string(), + } +} + +fn is_shell_blocked(cmd: &str) -> bool { + let s = cmd.to_lowercase(); + let blocked_substrings = [ + "rm -rf", + " sudo ", + "\nsudo ", + "mkfs", + " dd ", + "\ndd ", + "mount ", + "umount ", + "chmod ", + "chown ", + "shred ", + "truncate ", + ">>", + " | tee ", + ]; + if blocked_substrings.iter().any(|pat| s.contains(pat)) { + return true; + } + // block obvious redirects (write-to-file). allow '2>' etc? keep simple: block any '>' usage. + if s.contains('>') { + return true; + } + false +} + +fn normalize_path_lexical(path: &std::path::Path) -> std::path::PathBuf { + use std::path::Component; + let mut out = std::path::PathBuf::new(); + for c in path.components() { + match c { + Component::CurDir => {} + Component::ParentDir => { + out.pop(); + } + other => out.push(other.as_os_str()), + } + } + out +} + +fn resolve_abs_path(path: &str) -> Result { + let p = PathBuf::from(path); + if p.is_absolute() { + Ok(normalize_path_lexical(&p)) + } else { + let cwd = env::current_dir().context("failed to get cwd")?; + Ok(normalize_path_lexical(&cwd.join(p))) + } +} + +async fn execute_tool_restricted( + registry: &core_tools::Subroutines, + call: &klbr_core::llm::ToolCall, + tool_ctx: &core_tools::ToolContext, + write_root: &Path, +) -> ToolsLaneToolCallLog { + let name = call.function.name.clone(); + let args_str = call.function.arguments.clone(); + let args_json: serde_json::Value = serde_json::from_str(&args_str).unwrap_or_default(); + + let mut status = "ok".to_string(); + let mut blocked_reason = None::; + + match name.as_str() { + "write_file" => { + let path = args_json + .get("path") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + match resolve_abs_path(&path) { + Ok(abs) => { + let allowed = normalize_path_lexical(write_root); + if !abs.starts_with(&allowed) { + status = "blocked".into(); + blocked_reason = Some(format!( + "error: write_file blocked by bench sandbox (allowed root: {})", + allowed.display() + )); + } + } + Err(e) => { + status = "blocked".into(); + blocked_reason = Some(format!("error: write_file path resolution failed: {e}")); + } + } + } + "shell" => { + let cmd = args_json.get("cmd").and_then(|v| v.as_str()).unwrap_or(""); + if is_shell_blocked(cmd) { + status = "blocked".into(); + blocked_reason = Some("error: shell blocked by bench sandbox".into()); + } + } + _ => {} + } + + let result = if let Some(msg) = blocked_reason { + msg + } else { + registry.execute(call, tool_ctx).await + }; + + let is_error = result.starts_with("error:") || result.starts_with("unknown tool:"); + if status == "ok" && is_error { + status = "error".into(); + } + + ToolsLaneToolCallLog { + name, + args: truncate(&args_str, 4000), + status, + result: truncate(&result, 10_000), + } +} + +async fn run_tools_lane_query( + query: &EvalQuery, + gold: EvalRouteLabel, + router: &RuntimeRouter, + llm: &LlmClient, + tool_ctx: &core_tools::ToolContext, + registry: &core_tools::Subroutines, + tool_defs: &[klbr_core::llm::ToolDef], + write_root: &Path, +) -> Result { + let emb = llm + .embed(&query.text) + .await + .with_context(|| format!("embedding failed for tools-lane query {}", query.query_id))?; + let (route, scores) = router.predict_raw_with_scores(&emb); + let router_decision = route_decision_to_label(route); + let router_correct = router_decision == gold; + + let mut tool_iterations = 0usize; + let mut tool_calls_log: Vec = Vec::new(); + let mut final_response = String::new(); + + if route == RuntimeRouteDecision::Tools { + let mut ctx = AgentContext::new(TOOLS_LANE_ANCHOR, &[]); + ctx.push_user("bench", &query.text, &[]); + + loop { + let (tok_tx, mut tok_rx) = tokio::sync::mpsc::channel(256); + let llm2 = llm.clone(); + let msgs = ctx.as_messages(); + let defs = tool_defs.to_vec(); + let stream_task = tokio::spawn(async move { llm2.stream(&msgs, &defs, tok_tx).await }); + + let mut response = String::new(); + let mut reasoning = String::new(); + let mut tool_calls: Vec = Vec::new(); + + while let Some(ev) = tok_rx.recv().await { + match ev { + klbr_core::llm::LlmEvent::Token(tok) => response.push_str(&tok), + klbr_core::llm::LlmEvent::ThinkToken(tok) => reasoning.push_str(&tok), + klbr_core::llm::LlmEvent::Usage(_) => {} + klbr_core::llm::LlmEvent::ToolCalls(calls) => tool_calls = calls, + } + } + + // Ensure the stream task didn't fail (surface error in trace, don't crash the run). + match stream_task.await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + final_response = format!("error: llm stream failed: {e}"); + break; + } + Err(e) => { + final_response = format!("error: llm stream task failed: {e}"); + break; + } + } + + if !tool_calls.is_empty() && tool_iterations < MAX_TOOLS_LANE_TOOL_ITERATIONS { + tool_iterations += 1; + let text_content = (!response.is_empty()).then_some(response.clone()); + let reasoning_content = (!reasoning.is_empty()).then_some(reasoning.clone()); + ctx.push_assistant_tool_calls(tool_calls.clone(), text_content, reasoning_content); + for call in &tool_calls { + let log = execute_tool_restricted(registry, call, tool_ctx, write_root).await; + ctx.push_tool_result(&call.id, &log.result); + tool_calls_log.push(log); + } + continue; + } + + final_response = response; + let final_reasoning = (!reasoning.is_empty()).then_some(reasoning.as_str()); + ctx.push_assistant(&final_response, final_reasoning); + break; + } + } + + let called_any_tool = !tool_calls_log.is_empty(); + let required_tools = query.required_tools.clone(); + let called_any_required_tool = if required_tools.is_empty() { + None + } else { + Some( + tool_calls_log + .iter() + .any(|call| required_tools.iter().any(|t| t == &call.name)), + ) + }; + + Ok(ToolsLaneTrace { + query_id: query.query_id.clone(), + split: query.split.clone(), + text: query.text.clone(), + gold, + required_tools, + router_decision, + router_scores: Some(scores_for_trace(scores)), + router_correct, + tool_iterations, + tool_calls: tool_calls_log, + final_response: truncate(&final_response, 20_000), + called_any_tool, + called_any_required_tool, + }) +} + +fn summarize_tools_lane( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + router_model_path: &str, + llm: &LlmClient, + write_root: &Path, + traces: &[ToolsLaneTrace], +) -> ToolsLaneReport { + let mut total = 0usize; + let mut correct = 0usize; + + let mut gold_tools = 0usize; + let mut gold_memory = 0usize; + let mut gold_abstain = 0usize; + + let mut tp_tools = 0usize; + let mut fp_tools_total = 0usize; + let mut fp_mem_to_tools = 0usize; + let mut fp_abs_to_tools = 0usize; + + let mut routed_tools = 0usize; + let mut routed_tools_called_any = 0usize; + let mut routed_tools_exec_errors = 0usize; + + let mut required_defined = 0usize; + let mut required_hit = 0usize; + + for t in traces { + total += 1; + if t.router_correct { + correct += 1; + } + + match t.gold { + EvalRouteLabel::Tools => gold_tools += 1, + EvalRouteLabel::Memory => gold_memory += 1, + EvalRouteLabel::Abstain => gold_abstain += 1, + } + + if t.router_decision == EvalRouteLabel::Tools { + fp_tools_total += 1; + match t.gold { + EvalRouteLabel::Tools => tp_tools += 1, + EvalRouteLabel::Memory => fp_mem_to_tools += 1, + EvalRouteLabel::Abstain => fp_abs_to_tools += 1, + } + } + if t.router_decision == EvalRouteLabel::Tools { + routed_tools += 1; + if t.called_any_tool { + routed_tools_called_any += 1; + } + if t.tool_calls.iter().any(|c| c.status == "error") { + routed_tools_exec_errors += 1; + } + if let Some(hit) = t.called_any_required_tool { + required_defined += 1; + if hit { + required_hit += 1; + } + } + } + } + + let router_accuracy = if total == 0 { + 0.0 + } else { + correct as f32 / total as f32 + }; + let tools_precision = if fp_tools_total == 0 { + 0.0 + } else { + tp_tools as f32 / fp_tools_total as f32 + }; + let tools_recall = if gold_tools == 0 { + 0.0 + } else { + tp_tools as f32 / gold_tools as f32 + }; + let memory_false_tools_rate = if gold_memory == 0 { + 0.0 + } else { + fp_mem_to_tools as f32 / gold_memory as f32 + }; + let abstain_false_tools_rate = if gold_abstain == 0 { + 0.0 + } else { + fp_abs_to_tools as f32 / gold_abstain as f32 + }; + + let tools_lane_called_any_tool_rate = if routed_tools == 0 { + 0.0 + } else { + routed_tools_called_any as f32 / routed_tools as f32 + }; + let tools_lane_required_tool_hit_rate = if required_defined == 0 { + None + } else { + Some(required_hit as f32 / required_defined as f32) + }; + let tools_lane_tool_exec_error_rate = if routed_tools == 0 { + 0.0 + } else { + routed_tools_exec_errors as f32 / routed_tools as f32 + }; + + ToolsLaneReport { + dataset_id: dataset.dataset_id.clone(), + experiment_id: experiment.experiment_id.clone(), + evaluated_split: experiment.dataset_split.clone(), + llm_url: llm.config.llm_url.clone(), + embed_url: llm.config.embed_url.clone(), + embed_model: llm.config.embed_model.clone(), + router_model_path: router_model_path.to_string(), + write_root: write_root.display().to_string(), + metrics: ToolsLaneMetrics { + query_count: total, + gold_tools, + gold_memory, + gold_abstain, + router_accuracy, + router_tools_recall: tools_recall, + router_tools_precision: tools_precision, + router_memory_false_tools_rate: memory_false_tools_rate, + router_abstain_false_tools_rate: abstain_false_tools_rate, + tools_lane_routed_count: routed_tools, + tools_lane_called_any_tool_rate, + tools_lane_required_tool_hit_rate, + tools_lane_tool_exec_error_rate, + }, + } +} + +fn render_tools_lane_report(report: &ToolsLaneReport, traces: &[ToolsLaneTrace]) -> String { + let m = &report.metrics; + let mut out = format!( + "# Tools Lane Benchmark Report\n\n\ +dataset: `{}`\n\ +experiment: `{}`\n\ +split: `{:?}`\n\ +llm_url: `{}`\n\ +embed_url: `{}`\n\ +embed_model: `{}`\n\ +router_model: `{}`\n\ +write_root: `{}`\n\n\ +## Router Metrics (Gold = route_label)\n\n\ +- queries: `{}` (tools `{}` / memory `{}` / abstain `{}`)\n\ +- router accuracy: `{:.4}`\n\ +- tools precision: `{:.4}`\n\ +- tools recall: `{:.4}`\n\ +- memory→tools rate: `{:.4}`\n\ +- abstain→tools rate: `{:.4}`\n\n\ +## End-to-End Tool Lane\n\n\ +- queries routed to tools: `{}`\n\ +- called any tool rate (given routed-to-tools): `{:.4}`\n\ +- required tool hit rate (if defined): `{}`\n\ +- tool exec error rate (given routed-to-tools): `{:.4}`\n", + report.dataset_id, + report.experiment_id, + report.evaluated_split, + report.llm_url, + report.embed_url, + report.embed_model, + report.router_model_path, + report.write_root, + m.query_count, + m.gold_tools, + m.gold_memory, + m.gold_abstain, + m.router_accuracy, + m.router_tools_precision, + m.router_tools_recall, + m.router_memory_false_tools_rate, + m.router_abstain_false_tools_rate, + m.tools_lane_routed_count, + m.tools_lane_called_any_tool_rate, + m.tools_lane_required_tool_hit_rate + .map(|v| format!("{v:.4}")) + .unwrap_or_else(|| "n/a".into()), + m.tools_lane_tool_exec_error_rate, + ); + + let mut interesting: Vec<&ToolsLaneTrace> = traces + .iter() + .filter(|t| t.gold == EvalRouteLabel::Tools || t.router_decision == EvalRouteLabel::Tools) + .collect(); + interesting.sort_by(|a, b| a.query_id.cmp(&b.query_id)); + if !interesting.is_empty() { + out.push_str("\n## Traces (Tools Gold or Tools Pred)\n\n"); + out.push_str("| id | gold | pred | called_tool | required_hit | tools_score | memory_score | abstain_score | kind |\n"); + out.push_str("|---|---|---|---:|---:|---:|---:|---:|---|\n"); + for t in interesting { + let (st, sm, sa, kind) = t + .router_scores + .as_ref() + .map(|s| (s.tools, s.memory, s.abstain, s.kind.as_str())) + .unwrap_or((0.0, 0.0, 0.0, "unknown")); + let req = t + .called_any_required_tool + .map(|v| if v { "1" } else { "0" }) + .unwrap_or("n/a"); + out.push_str(&format!( + "| `{}` | `{:?}` | `{:?}` | {} | {} | {:.2} | {:.2} | {:.2} | `{}` |\n", + t.query_id, + t.gold, + t.router_decision, + if t.called_any_tool { 1 } else { 0 }, + req, + st, + sm, + sa, + kind + )); + } + } + + out +} + +fn build_embed_only_client(experiment: &RetrievalExperimentConfig) -> LlmClient { + let mut runtime = Config::default(); + runtime.embed_url = experiment.embed_url.clone(); + runtime.embed_model = experiment.embed_model.clone(); + runtime.embed_dim = experiment.embed_dim; + LlmClient::new(runtime) +} + +fn gold_route_label(query: &EvalQuery) -> Option { + if let Some(label) = &query.route_label { + return Some(label.clone()); + } + // For now, infer "memory" for answerable (non-no_hit) queries so we can bootstrap without + // labeling every memory question by hand. + if !query.no_hit && query.objective == EvalObjective::AnswerQuery { + return Some(EvalRouteLabel::Memory); + } + None +} + +#[derive(Debug, Clone)] +struct RouterExample { + query_id: String, + text: String, + split: DatasetSplit, + gold: EvalRouteLabel, + emb_unit: Vec, +} + +async fn embed_router_examples( + dataset: &InternalEvalDataset, + llm: &LlmClient, +) -> Result> { + let mut out = Vec::new(); + for query in dataset + .queries + .iter() + .filter(|q| q.objective == EvalObjective::AnswerQuery) + { + let Some(gold) = gold_route_label(query) else { + continue; + }; + let emb = llm + .embed(&query.text) + .await + .with_context(|| format!("embedding failed for router query {}", query.query_id))?; + let emb_unit = normalize_vec(&emb); + out.push(RouterExample { + query_id: query.query_id.clone(), + text: query.text.clone(), + split: query.split.clone(), + gold, + emb_unit, + }); + } + Ok(out) +} + +fn label_index(label: &EvalRouteLabel) -> usize { + match label { + EvalRouteLabel::Tools => 0, + EvalRouteLabel::Memory => 1, + EvalRouteLabel::Abstain => 2, + } +} + +fn train_softmax_router( + examples: &[RouterExample], + split: DatasetSplit, + embed_dim: usize, +) -> Result<(Vec>, Vec)> { + let train: Vec<&RouterExample> = examples.iter().filter(|e| e.split == split).collect(); + if train.is_empty() { + bail!("router-linear: no training examples for split {:?}", split); + } + + let mut counts = [0usize; 3]; + for ex in &train { + counts[label_index(&ex.gold)] += 1; + } + if counts[0] == 0 || counts[1] == 0 { + bail!( + "router-linear training requires tools+memory labels on split {:?}; got tools={} memory={} abstain={}", + split, counts[0], counts[1], counts[2] + ); + } + + // Softmax regression (3-way). Full-batch gradient descent; dataset is tiny. + let c = 3usize; + let d = embed_dim; + let mut w = vec![vec![0.0f32; d]; c]; + let mut b = vec![0.0f32; c]; + + let epochs = 400usize; + let mut lr = 0.2f32; + let l2 = 1e-3f32; + + for epoch in 0..epochs { + let mut grad_w = vec![vec![0.0f32; d]; c]; + let mut grad_b = vec![0.0f32; c]; + let mut loss = 0.0f32; + + for ex in &train { + if ex.emb_unit.len() != d { + bail!( + "router-linear embed dim mismatch for {}: got {}, expected {}", + ex.query_id, + ex.emb_unit.len(), + d + ); + } + let y = label_index(&ex.gold); + let logits = logits3(&w, &b, &ex.emb_unit); + let probs = softmax3(logits); + let py = probs[y].max(1e-9); + loss += -py.ln(); + + for cls in 0..c { + let target = if cls == y { 1.0 } else { 0.0 }; + let delta = probs[cls] - target; + grad_b[cls] += delta; + for j in 0..d { + grad_w[cls][j] += delta * ex.emb_unit[j]; + } + } + } + + let n = train.len() as f32; + for cls in 0..c { + for j in 0..d { + grad_w[cls][j] = (grad_w[cls][j] / n) + (l2 * w[cls][j]); + w[cls][j] -= lr * grad_w[cls][j]; + } + grad_b[cls] /= n; + b[cls] -= lr * grad_b[cls]; + } + + // simple decay for stability + if epoch > 0 && epoch % 50 == 0 { + lr *= 0.98; + } + // keep a weak signal in stderr if someone runs this manually + if epoch == 0 || epoch == epochs - 1 { + eprintln!("router-linear: epoch {epoch}/{epochs} loss={:.4}", loss / n); + } + } + + Ok((w, b)) +} + +fn tune_linear_thresholds( + examples: &[RouterExample], + split: DatasetSplit, + weights: &[Vec], + bias: &[f32], +) -> Result { + let tuning: Vec<&RouterExample> = examples.iter().filter(|e| e.split == split).collect(); + if tuning.is_empty() { + bail!("router-linear: no tuning examples for split {:?}", split); + } + + let tuning_points = precompute_linear_points(&tuning, weights, bias)?; + + // Hard constraint: never misroute memory → tools on the tuning split. + const MAX_MEMORY_TO_ABSTAIN_RATE: f32 = 0.03; + // For safety, be conservative about routing abstain-lane queries to tools. + const MAX_ABSTAIN_FALSE_TOOL_RATE: f32 = 0.05; + + // Keep this sweep cheap but wide enough to find a reasonable operating point. + // (Too-high tools_prob_threshold was a common failure mode: tools queries routed to memory.) + let prob_values: Vec = (2..=18).map(|i| i as f32 * 0.05).collect(); // 0.10..0.90 + let margin_values: Vec = (0..=25).map(|i| i as f32 * 0.02).collect(); // 0.00..0.50 + + let mut best: Option<(RuntimeLinearDecisionParams, LinearRouterMetrics)> = None; + + for &t_tools_prob in &prob_values { + for &t_tools_margin in &margin_values { + for &t_abs_prob in &prob_values { + for &t_abs_margin in &margin_values { + let decision = RuntimeLinearDecisionParams { + tools_prob_threshold: t_tools_prob, + tools_margin_threshold: t_tools_margin, + abstain_prob_threshold: t_abs_prob, + abstain_margin_threshold: t_abs_margin, + }; + let m = score_linear_thresholds_precomputed(&tuning_points, &decision)?; + + let ok = m.memory_false_tool_rate == 0.0 + && m.memory_false_abstain_rate <= MAX_MEMORY_TO_ABSTAIN_RATE + && m.abstain_false_tool_rate <= MAX_ABSTAIN_FALSE_TOOL_RATE; + + best = match best.take() { + None => Some((decision, m)), + Some((best_d, best_m)) => { + let best_ok = best_m.memory_false_tool_rate == 0.0 + && best_m.memory_false_abstain_rate <= MAX_MEMORY_TO_ABSTAIN_RATE + && best_m.abstain_false_tool_rate <= MAX_ABSTAIN_FALSE_TOOL_RATE; + + let better = if ok && best_ok { + // primary: maximize tools recall (keep tool-lane capability). + // secondary: minimize abstain→tools leakage (avoid spurious tool runs). + // then: maximize overall accuracy and tools precision. + ( + -m.tool_recall, + m.abstain_false_tool_rate, + -m.accuracy, + -m.tool_precision, + ) < ( + -best_m.tool_recall, + best_m.abstain_false_tool_rate, + -best_m.accuracy, + -best_m.tool_precision, + ) + } else if ok && !best_ok { + true + } else if !ok && best_ok { + false + } else { + // fallback: still prefer fewer memory→tools then higher precision + ( + m.memory_false_tool_rate, + -m.tool_precision, + -m.tool_recall, + -m.accuracy, + ) < ( + best_m.memory_false_tool_rate, + -best_m.tool_precision, + -best_m.tool_recall, + -best_m.accuracy, + ) + }; + + if better { + Some((decision, m)) + } else { + Some((best_d, best_m)) + } + } + }; + } + } + } + } + + let (decision, metrics) = best.context("router-linear threshold sweep produced no points")?; + eprintln!( + "router-linear: tuned thresholds tools_prob={:.2} tools_margin={:.2} abs_prob={:.2} abs_margin={:.2} (dev: tools_recall={:.3}, tools_precision={:.3}, mem→tools={:.3})", + decision.tools_prob_threshold, + decision.tools_margin_threshold, + decision.abstain_prob_threshold, + decision.abstain_margin_threshold, + metrics.tool_recall, + metrics.tool_precision, + metrics.memory_false_tool_rate + ); + Ok(decision) +} + +#[derive(Debug, Clone)] +struct LinearPoint { + query_id: String, + text: String, + gold: EvalRouteLabel, + probs: [f32; 3], // tools/memory/abstain +} + +fn precompute_linear_points( + examples: &[&RouterExample], + weights: &[Vec], + bias: &[f32], +) -> Result> { + let mut out = Vec::with_capacity(examples.len()); + for ex in examples { + let probs = softmax3(logits3(weights, bias, &ex.emb_unit)); + out.push(LinearPoint { + query_id: ex.query_id.clone(), + text: ex.text.clone(), + gold: ex.gold.clone(), + probs, + }); + } + Ok(out) +} + +fn score_linear_thresholds_precomputed( + points: &[LinearPoint], + decision: &RuntimeLinearDecisionParams, +) -> Result { + let mut total = 0usize; + let mut correct = 0usize; + let mut confusion = [[0usize; 3]; 3]; + + let mut tool_tp = 0usize; + let mut tool_fp_total = 0usize; + let mut memory_to_tools_fp = 0usize; + let mut abstain_to_tools_fp = 0usize; + let mut tool_fn = 0usize; + let mut memory_to_abstain_fp = 0usize; + + let mut tool_count = 0usize; + let mut memory_count = 0usize; + let mut abstain_count = 0usize; + + let mut misclassified = Vec::new(); + + for ex in points { + total += 1; + let gold = &ex.gold; + let probs = ex.probs; + let pred = predict_linear_route(probs, decision); + + let g = label_index(gold); + let p = label_index(&pred); + confusion[g][p] += 1; + + if pred == *gold { + correct += 1; + } else { + misclassified.push(LinearRouterTrace { + query_id: ex.query_id.clone(), + text: ex.text.clone(), + gold: gold.clone(), + pred: pred.clone(), + p_tools: probs[0], + p_memory: probs[1], + p_abstain: probs[2], + }); + } + + match gold { + EvalRouteLabel::Tools => tool_count += 1, + EvalRouteLabel::Memory => memory_count += 1, + EvalRouteLabel::Abstain => abstain_count += 1, + } + + match (gold, &pred) { + (EvalRouteLabel::Tools, EvalRouteLabel::Tools) => tool_tp += 1, + (EvalRouteLabel::Memory, EvalRouteLabel::Tools) => { + tool_fp_total += 1; + memory_to_tools_fp += 1; + } + (EvalRouteLabel::Abstain, EvalRouteLabel::Tools) => { + tool_fp_total += 1; + abstain_to_tools_fp += 1; + } + (EvalRouteLabel::Tools, _) => tool_fn += 1, + _ => {} + } + + if matches!( + (gold, &pred), + (EvalRouteLabel::Memory, EvalRouteLabel::Abstain) + ) { + memory_to_abstain_fp += 1; + } + } + + let denom = total.max(1) as f32; + let acc = correct as f32 / denom; + let tool_precision = if tool_tp + tool_fp_total > 0 { + tool_tp as f32 / (tool_tp + tool_fp_total) as f32 + } else { + 0.0 + }; + let tool_recall = if tool_tp + tool_fn > 0 { + tool_tp as f32 / (tool_tp + tool_fn) as f32 + } else { + 0.0 + }; + let memory_false_tool_rate = if memory_count > 0 { + memory_to_tools_fp as f32 / (memory_count as f32) + } else { + 0.0 + }; + let abstain_false_tool_rate = if abstain_count > 0 { + abstain_to_tools_fp as f32 / (abstain_count as f32) + } else { + 0.0 + }; + let memory_false_abstain_rate = if memory_count > 0 { + memory_to_abstain_fp as f32 / (memory_count as f32) + } else { + 0.0 + }; + + Ok(LinearRouterMetrics { + total, + accuracy: acc, + confusion, + tool_precision, + tool_recall, + memory_false_tool_rate, + abstain_false_tool_rate, + memory_false_abstain_rate, + tool_count, + memory_count, + abstain_count, + misclassified, + }) +} + +fn logits3(weights: &[Vec], bias: &[f32], x_unit: &[f32]) -> [f32; 3] { + let mut out = [0.0f32; 3]; + for cls in 0..3 { + out[cls] = dot(x_unit, &weights[cls]) + bias[cls]; + } + out +} + +fn softmax3(logits: [f32; 3]) -> [f32; 3] { + let m = logits[0].max(logits[1]).max(logits[2]); + let e0 = (logits[0] - m).exp(); + let e1 = (logits[1] - m).exp(); + let e2 = (logits[2] - m).exp(); + let denom = (e0 + e1 + e2).max(1e-12); + [e0 / denom, e1 / denom, e2 / denom] +} + +fn predict_linear_route(probs: [f32; 3], decision: &RuntimeLinearDecisionParams) -> EvalRouteLabel { + let p_tools = probs[0]; + let p_memory = probs[1]; + let p_abstain = probs[2]; + + let tools_margin = p_tools - p_memory.max(p_abstain); + if p_tools >= decision.tools_prob_threshold && tools_margin >= decision.tools_margin_threshold { + return EvalRouteLabel::Tools; + } + + let abstain_margin = p_abstain - p_memory; + if p_abstain >= decision.abstain_prob_threshold + && abstain_margin >= decision.abstain_margin_threshold + { + return EvalRouteLabel::Abstain; + } + + EvalRouteLabel::Memory +} + +fn evaluate_linear_router( + dataset: &InternalEvalDataset, + examples: &[RouterExample], + model: &RuntimeLinearRouterModel, + trained_on: DatasetSplit, + split: DatasetSplit, +) -> Result { + let eval: Vec<&RouterExample> = examples.iter().filter(|e| e.split == split).collect(); + let points = precompute_linear_points(&eval, &model.weights, &model.bias)?; + let metrics = score_linear_thresholds_precomputed(&points, &model.decision)?; + + let holdout_points: Vec = points + .iter() + .filter(|p| p.query_id.starts_with("holdout_")) + .cloned() + .collect(); + let holdout_metrics = if holdout_points.is_empty() { + None + } else { + Some(score_linear_thresholds_precomputed( + &holdout_points, + &model.decision, + )?) + }; + + let mut holdout_metrics_by_bucket: BTreeMap = BTreeMap::new(); + if !holdout_points.is_empty() { + let mut by_bucket: BTreeMap> = BTreeMap::new(); + for pt in &holdout_points { + if let Some(bucket) = parse_holdout_bucket(&pt.query_id) { + by_bucket.entry(bucket).or_default().push(pt.clone()); + } + } + for (bucket, pts) in by_bucket { + holdout_metrics_by_bucket.insert( + bucket, + score_linear_thresholds_precomputed(&pts, &model.decision)?, + ); + } + } + + Ok(LinearRouterReport { + dataset_id: dataset.dataset_id.clone(), + embed_model: model.embed_model.clone(), + trained_on, + evaluated_on: split, + decision: model.decision.clone(), + metrics, + holdout_metrics, + holdout_metrics_by_bucket, + }) +} + +fn render_linear_router_report( + dataset: &InternalEvalDataset, + report: &LinearRouterReport, +) -> String { + let m = &report.metrics; + let d = &report.decision; + + let mut out = format!( + "# Linear Router Report\n\n\ +dataset: `{}`\n\ +embed_model: `{}`\n\ +trained_on: `{:?}`\n\ +evaluated_on: `{:?}`\n\n\ +## Decision Params\n\n\ +- tools_prob_threshold: `{:.3}`\n\ +- tools_margin_threshold: `{:.3}`\n\ +- abstain_prob_threshold: `{:.3}`\n\ +- abstain_margin_threshold: `{:.3}`\n\n\ +## Metrics\n\n\ +- labeled queries evaluated: `{}`\n\ +- tools: `{}` memory: `{}` abstain: `{}`\n\ +- accuracy: `{:.4}`\n\ +- tools precision: `{:.4}`\n\ +- tools recall: `{:.4}`\n\ +- memory false-tools rate (memory→tools): `{:.4}`\n\ +- abstain false-tools rate (abstain→tools): `{:.4}`\n\ +- memory false-abstain rate (memory→abstain): `{:.4}`\n", + dataset.dataset_id, + report.embed_model, + report.trained_on, + report.evaluated_on, + d.tools_prob_threshold, + d.tools_margin_threshold, + d.abstain_prob_threshold, + d.abstain_margin_threshold, + m.total, + m.tool_count, + m.memory_count, + m.abstain_count, + m.accuracy, + m.tool_precision, + m.tool_recall, + m.memory_false_tool_rate, + m.abstain_false_tool_rate, + m.memory_false_abstain_rate, + ); + + out.push_str("\n## Confusion Matrix (gold x pred)\n\n"); + out.push_str("| gold \\ pred | tools | memory | abstain |\n"); + out.push_str("|---|---:|---:|---:|\n"); + let rows = [ + EvalRouteLabel::Tools, + EvalRouteLabel::Memory, + EvalRouteLabel::Abstain, + ]; + let names = ["tools", "memory", "abstain"]; + for (i, gold) in rows.iter().enumerate() { + out.push_str(&format!( + "| {} | {} | {} | {} |\n", + names[label_index(gold)], + m.confusion[i][0], + m.confusion[i][1], + m.confusion[i][2] + )); + } + + if !m.misclassified.is_empty() { + out.push_str("\n## Misclassified Queries\n\n"); + out.push_str("| id | text | gold | pred | p_tools | p_memory | p_abstain |\n"); + out.push_str("|---|---|---|---|---:|---:|---:|\n"); + for trace in &m.misclassified { + out.push_str(&format!( + "| `{}` | {} | `{:?}` | `{:?}` | {:.2} | {:.2} | {:.2} |\n", + trace.query_id, + trace.text, + trace.gold, + trace.pred, + trace.p_tools, + trace.p_memory, + trace.p_abstain + )); + } + } + + if let Some(h) = &report.holdout_metrics { + out.push_str("\n## Holdout Metrics (query_id starts with `holdout_`)\n\n"); + out.push_str(&format!( + "- labeled queries evaluated: `{}`\n\ +- tools precision: `{:.4}`\n\ +- tools recall: `{:.4}`\n\ +- memory false-tools rate (memory→tools): `{:.4}`\n\ +- abstain false-tools rate (abstain→tools): `{:.4}`\n\ +- accuracy: `{:.4}`\n", + h.total, + h.tool_precision, + h.tool_recall, + h.memory_false_tool_rate, + h.abstain_false_tool_rate, + h.accuracy, + )); + } + + if !report.holdout_metrics_by_bucket.is_empty() { + out.push_str("\n## Holdout Metrics By Bucket\n\n"); + out.push_str("| bucket | total | tools precision | tools recall | memory→tools | abstain→tools | accuracy |\n"); + out.push_str("|---|---:|---:|---:|---:|---:|---:|\n"); + for (bucket, h) in &report.holdout_metrics_by_bucket { + out.push_str(&format!( + "| `{}` | {} | {:.4} | {:.4} | {:.4} | {:.4} | {:.4} |\n", + bucket, + h.total, + h.tool_precision, + h.tool_recall, + h.memory_false_tool_rate, + h.abstain_false_tool_rate, + h.accuracy + )); + } + } + + out +} + +fn parse_holdout_bucket(query_id: &str) -> Option { + if !query_id.starts_with("holdout_") { + return None; + } + let rest = &query_id["holdout_".len()..]; + let parts: Vec<&str> = rest.split('_').collect(); + // Expected shape: + // holdout___ + // where may contain underscores and may itself contain the words + // "tools"/"memory"/"abstain". + if parts.len() < 3 { + return None; + } + let lane = parts[parts.len() - 2]; + if lane != "tools" && lane != "memory" && lane != "abstain" { + return None; + } + let idx = parts[parts.len() - 1]; + if idx.is_empty() || !idx.chars().all(|c| c.is_ascii_digit()) { + return None; + } + let bucket = parts[0..parts.len() - 2].join("_"); + if bucket.is_empty() { + return None; + } + Some(bucket) +} + +async fn train_router_centroids( + dataset: &InternalEvalDataset, + llm: &LlmClient, + experiment: &RetrievalExperimentConfig, + split: DatasetSplit, +) -> Result<(RouterModel, f32)> { + let mut memory_embs: Vec> = Vec::new(); + let mut tools_embs: Vec> = Vec::new(); + let mut abstain_embs: Vec> = Vec::new(); + + for query in dataset + .queries + .iter() + .filter(|q| q.split == split) + .filter(|q| q.objective == EvalObjective::AnswerQuery) + { + let Some(label) = gold_route_label(query) else { + continue; + }; + let emb = llm.embed(&query.text).await.with_context(|| { + format!( + "embedding failed for router training query {}", + query.query_id + ) + })?; + match label { + EvalRouteLabel::Memory => memory_embs.push(emb), + EvalRouteLabel::Tools => tools_embs.push(emb), + EvalRouteLabel::Abstain => abstain_embs.push(emb), + } + } + + if memory_embs.is_empty() || tools_embs.is_empty() { + bail!( + "router training requires both memory and tools labels on split {:?}; got memory={} tools={}", + split, + memory_embs.len(), + tools_embs.len() + ); + } + let embed_dim = memory_embs[0].len(); + if embed_dim != experiment.embed_dim { + eprintln!( + "warning: router embed dim mismatch: experiment says {}, but embedder returned {}", + experiment.embed_dim, embed_dim + ); + } + // abstain is optional for training, but better if present + if abstain_embs.is_empty() { + eprintln!( + "warning: no abstain examples for training, abstain centroid will be zero (dim={})", + embed_dim + ); + abstain_embs.push(vec![0.0; embed_dim]); + } + + // Multiple prototypes per class. This usually improves tools recall because the "tools lane" + // is a union of different intents (repo introspection, runtime state, file IO, etc). + let memory_centroids = kmeans_prototypes_unit(&memory_embs, choose_k(memory_embs.len(), 3), 8); + let tools_centroids = kmeans_prototypes_unit(&tools_embs, choose_k(tools_embs.len(), 4), 8); + let abstain_centroids = + kmeans_prototypes_unit(&abstain_embs, choose_k(abstain_embs.len(), 2), 8); + + // Tune a conservative threshold on the training split: prioritize high precision for tools. + let mut best = None::<(f32, RouterMetrics)>; + // If we insist on *zero* memory→tools, tools recall tends to collapse because many "tools-needed" + // queries are semantically close to "memory questions about the same topic" (config/reranker/etc). + // Allowing a tiny amount of memory→tools (usually ~1 query on dev) can substantially improve recall. + const MAX_MEMORY_FALSE_TOOL_RATE: f32 = 0.03; + const MIN_TOOL_PRECISION: f32 = 0.98; + for threshold in (-40..=40).map(|i| i as f32 * 0.05) { + let metrics = evaluate_router_on_split( + dataset, + llm, + &memory_centroids, + &tools_centroids, + &abstain_centroids, + threshold, + split.clone(), + ) + .await?; + best = match best.take() { + None => Some((threshold, metrics)), + Some((best_t, best_m)) => { + // Constrained objective: + // - keep memory→tools mistakes near-zero + // - keep tool precision high + // - then maximize tool recall + // + // If constraints can't be met, fall back to the old conservative lexicographic compare. + let ok = metrics.memory_false_tool_rate <= MAX_MEMORY_FALSE_TOOL_RATE + && metrics.tool_precision >= MIN_TOOL_PRECISION; + let best_ok = best_m.memory_false_tool_rate <= MAX_MEMORY_FALSE_TOOL_RATE + && best_m.tool_precision >= MIN_TOOL_PRECISION; + + let better = if ok && best_ok { + ( + -metrics.tool_recall, + -metrics.tool_precision, + -metrics.accuracy, + ) < ( + -best_m.tool_recall, + -best_m.tool_precision, + -best_m.accuracy, + ) + } else if ok && !best_ok { + true + } else if !ok && best_ok { + false + } else { + // Conservative fallback. + ( + metrics.memory_false_tool_rate, + -metrics.tool_precision, + -metrics.tool_recall, + -metrics.accuracy, + ) < ( + best_m.memory_false_tool_rate, + -best_m.tool_precision, + -best_m.tool_recall, + -best_m.accuracy, + ) + }; + if better { + Some((threshold, metrics)) + } else { + Some((best_t, best_m)) + } + } + }; + } + + let (threshold, _metrics) = + best.context("router threshold sweep unexpectedly produced no points")?; + + Ok(( + RouterModel { + embed_model: experiment.embed_model.clone(), + embed_dim, + trained_on: split.clone(), + threshold, + memory_centroids, + tools_centroids, + abstain_centroids, + }, + threshold, + )) +} + +async fn evaluate_router( + dataset: &InternalEvalDataset, + llm: &LlmClient, + model: &RouterModel, + split: DatasetSplit, +) -> Result { + let split_eval = split.clone(); + let metrics = evaluate_router_on_split( + dataset, + llm, + &model.memory_centroids, + &model.tools_centroids, + &model.abstain_centroids, + model.threshold, + split, + ) + .await?; + Ok(RouterReport { + dataset_id: dataset.dataset_id.clone(), + embed_model: model.embed_model.clone(), + trained_on: model.trained_on.clone(), + evaluated_on: split_eval, + threshold: model.threshold, + metrics, + }) +} + +async fn evaluate_router_on_split( + dataset: &InternalEvalDataset, + llm: &LlmClient, + memory_centroids: &[Vec], + tools_centroids: &[Vec], + abstain_centroids: &[Vec], + threshold: f32, + split: DatasetSplit, +) -> Result { + let mut total = 0usize; + let mut correct = 0usize; + let mut tool_tp = 0usize; + let mut tool_fp_total = 0usize; + let mut memory_to_tools_fp = 0usize; + let mut abstain_to_tools_fp = 0usize; + let mut tool_fn = 0usize; + let mut tool_count = 0usize; + let mut memory_count = 0usize; + let mut abstain_count = 0usize; + let mut misclassified = Vec::new(); + + for query in dataset + .queries + .iter() + .filter(|q| q.split == split) + .filter(|q| q.objective == EvalObjective::AnswerQuery) + { + let Some(gold) = gold_route_label(query) else { + continue; + }; + total += 1; + + let emb = llm.embed(&query.text).await.with_context(|| { + format!("embedding failed for router eval query {}", query.query_id) + })?; + let emb_unit = normalize_vec(&emb); + let sim_tools = max_sim(&emb_unit, tools_centroids); + let sim_memory = max_sim(&emb_unit, memory_centroids); + let sim_abstain = max_sim(&emb_unit, abstain_centroids); + + let pred = predict_route( + &emb_unit, + memory_centroids, + tools_centroids, + abstain_centroids, + threshold, + ); + + if pred == gold { + correct += 1; + } else { + misclassified.push(RouterTrace { + query_id: query.query_id.clone(), + text: query.text.clone(), + gold: gold.clone(), + pred: pred.clone(), + sim_tools, + sim_memory, + sim_abstain, + }); + } + + match gold { + EvalRouteLabel::Tools => tool_count += 1, + EvalRouteLabel::Memory => memory_count += 1, + EvalRouteLabel::Abstain => abstain_count += 1, + } + + match (gold, pred) { + (EvalRouteLabel::Tools, EvalRouteLabel::Tools) => tool_tp += 1, + (EvalRouteLabel::Memory, EvalRouteLabel::Tools) => { + tool_fp_total += 1; + memory_to_tools_fp += 1; + } + (EvalRouteLabel::Abstain, EvalRouteLabel::Tools) => { + tool_fp_total += 1; + abstain_to_tools_fp += 1; + } + (EvalRouteLabel::Tools, _) => tool_fn += 1, + _ => {} + } + } + + let denom = total.max(1) as f32; + let acc = correct as f32 / denom; + let tool_precision = if tool_tp + tool_fp_total > 0 { + tool_tp as f32 / (tool_tp + tool_fp_total) as f32 + } else { + 0.0 + }; + let tool_recall = if tool_tp + tool_fn > 0 { + tool_tp as f32 / (tool_tp + tool_fn) as f32 + } else { + 0.0 + }; + let memory_false_tool_rate = if memory_count > 0 { + memory_to_tools_fp as f32 / (memory_count as f32) + } else { + 0.0 + }; + let abstain_false_tool_rate = if abstain_count > 0 { + abstain_to_tools_fp as f32 / (abstain_count as f32) + } else { + 0.0 + }; + + Ok(RouterMetrics { + total, + accuracy: acc, + tool_precision, + tool_recall, + memory_false_tool_rate, + abstain_false_tool_rate, + tool_count, + memory_count, + abstain_count, + misclassified, + }) +} + +fn predict_route( + query_embedding_unit: &[f32], + memory_centroids_unit: &[Vec], + tools_centroids_unit: &[Vec], + abstain_centroids_unit: &[Vec], + threshold: f32, +) -> EvalRouteLabel { + let sim_tools = max_sim(query_embedding_unit, tools_centroids_unit); + let sim_memory = max_sim(query_embedding_unit, memory_centroids_unit); + let sim_abstain = max_sim(query_embedding_unit, abstain_centroids_unit); + + if (sim_tools - sim_memory) >= threshold && (sim_tools - sim_abstain) >= threshold { + EvalRouteLabel::Tools + } else if sim_memory >= sim_abstain { + EvalRouteLabel::Memory + } else { + EvalRouteLabel::Abstain + } +} + +fn mean_vec(vectors: &[Vec]) -> Vec { + let dim = vectors.first().map(|v| v.len()).unwrap_or(0); + let mut out = vec![0.0f32; dim]; + let denom = vectors.len().max(1) as f32; + for v in vectors { + for (idx, val) in v.iter().enumerate() { + out[idx] += *val / denom; + } + } + out +} + +fn normalize_vec(v: &[f32]) -> Vec { + let norm = v + .iter() + .map(|x| (*x as f64) * (*x as f64)) + .sum::() + .sqrt() as f32; + if norm <= 1e-12 { + return v.to_vec(); + } + v.iter().map(|x| *x / norm).collect() +} + +fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +fn max_sim(query_embedding_unit: &[f32], centroids_unit: &[Vec]) -> f32 { + let mut best = f32::NEG_INFINITY; + for c in centroids_unit { + best = best.max(dot(query_embedding_unit, c)); + } + best +} + +fn choose_k(n: usize, max_k: usize) -> usize { + // coarse, deterministic heuristic: more prototypes once the class has enough examples + let k = if n < 25 { + 1 + } else if n < 60 { + 2 + } else if n < 120 { + 3 + } else { + 4 + }; + k.min(max_k).max(1).min(n.max(1)) +} + +fn kmeans_prototypes_unit(vectors: &[Vec], k: usize, iters: usize) -> Vec> { + if vectors.is_empty() { + return vec![]; + } + let k = k.max(1).min(vectors.len()); + let units: Vec> = vectors.iter().map(|v| normalize_vec(v)).collect(); + let dim = units[0].len(); + + // Deterministic farthest-point init. + let mut centroids: Vec> = Vec::with_capacity(k); + let mean = normalize_vec(&mean_vec(&units)); + centroids.push(mean); + while centroids.len() < k { + let mut best_i = 0usize; + let mut best_score = f32::INFINITY; // we minimize the max similarity to existing centroids + for (i, v) in units.iter().enumerate() { + let max_sim = centroids + .iter() + .map(|c| dot(v, c)) + .fold(f32::NEG_INFINITY, f32::max); + if max_sim < best_score { + best_score = max_sim; + best_i = i; + } + } + centroids.push(units[best_i].clone()); + } + + let mut assigns = vec![0usize; units.len()]; + for _ in 0..iters { + // assign + for (i, v) in units.iter().enumerate() { + let mut best_k = 0usize; + let mut best_sim = f32::NEG_INFINITY; + for (j, c) in centroids.iter().enumerate() { + let s = dot(v, c); + if s > best_sim { + best_sim = s; + best_k = j; + } + } + assigns[i] = best_k; + } + + // recompute + let mut sums = vec![vec![0.0f32; dim]; k]; + let mut counts = vec![0usize; k]; + for (v, &a) in units.iter().zip(assigns.iter()) { + counts[a] += 1; + for (idx, val) in v.iter().enumerate() { + sums[a][idx] += *val; + } + } + for j in 0..k { + if counts[j] == 0 { + continue; // keep old centroid + } + let denom = counts[j] as f32; + for idx in 0..dim { + sums[j][idx] /= denom; + } + centroids[j] = normalize_vec(&sums[j]); + } + } + + centroids +} + +fn render_router_report(dataset: &InternalEvalDataset, report: &RouterReport) -> String { + let m = &report.metrics; + let mut out = format!( + "# Router Report\n\n\ +dataset: `{}`\n\ +embed_model: `{}`\n\ +trained_on: `{:?}`\n\ +evaluated_on: `{:?}`\n\ +threshold: `{:.3}`\n\n\ +## Metrics\n\n\ +- labeled queries evaluated: `{}`\n\ +- tools: `{}` memory: `{}` abstain: `{}`\n\ +- accuracy: `{:.4}`\n\ +- tools precision: `{:.4}`\n\ +- tools recall: `{:.4}`\n\ +- memory false-tools rate: `{:.4}`\n", + dataset.dataset_id, + report.embed_model, + report.trained_on, + report.evaluated_on, + report.threshold, + m.total, + m.tool_count, + m.memory_count, + m.abstain_count, + m.accuracy, + m.tool_precision, + m.tool_recall, + m.memory_false_tool_rate, + ); + + // Include this in the report so we can see when the threshold is mostly protecting against + // abstain→tools mistakes vs actually skipping memory incorrectly. + out.push_str(&format!( + "- abstain false-tools rate: `{:.4}`\n", + m.abstain_false_tool_rate + )); + + if !m.misclassified.is_empty() { + out.push_str("\n## Misclassified Queries\n\n"); + out.push_str("| id | text | gold | pred | T-sim | M-sim | A-sim |\n"); + out.push_str("|---|---|---|---|---|---|---|\n"); + for trace in &m.misclassified { + out.push_str(&format!( + "| `{}` | {} | `{:?}` | `{:?}` | {:.2} | {:.2} | {:.2} |\n", + trace.query_id, + trace.text, + trace.gold, + trace.pred, + trace.sim_tools, + trace.sim_memory, + trace.sim_abstain + )); + } + } + out +} + +async fn run_sweep_command( + dataset_path: &str, + config_path: &str, + output_dir: &str, + grid: SweepGrid, +) -> Result<()> { + let dataset_path = Path::new(dataset_path); + let config_path = Path::new(config_path); + let output_dir = PathBuf::from(output_dir); + fs::create_dir_all(&output_dir)?; + + let dataset: InternalEvalDataset = read_json(dataset_path) + .with_context(|| format!("failed to read dataset from {}", dataset_path.display()))?; + let experiment: RetrievalExperimentConfig = read_json(config_path) + .with_context(|| format!("failed to read config from {}", config_path.display()))?; + validate_benchmark_inputs(&dataset, &experiment)?; + + let prepared = prepare_benchmark(&dataset, &experiment).await?; + let sweep_report = run_policy_sweep(&dataset, &experiment, &prepared, grid).await?; + write_sweep_outputs(&output_dir, &dataset, &experiment, &sweep_report)?; + + println!("wrote sweep outputs to {}", output_dir.display()); + Ok(()) +} + +fn parse_f32_arg(value: &str, name: &str) -> Result { + value + .parse::() + .with_context(|| format!("failed to parse {}='{}' as f32", name, value)) +} + +fn validate_benchmark_inputs( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, +) -> Result<()> { + validate_timeline_splits(dataset)?; + + eprintln!( + "loaded dataset '{}' ({} memories, {} queries)", + dataset.dataset_id, + dataset.memories.len(), + dataset.queries.len() + ); + eprintln!( + "experiment '{}' embed={} rerank={}", + experiment.experiment_id, + experiment.embed_url, + experiment.rerank_url.as_deref().unwrap_or("(disabled)") + ); + + if experiment.vector_search_mode != VectorSearchMode::Exact { + bail!( + "this implementation only supports vector_search_mode=exact; got {:?}", + experiment.vector_search_mode + ); + } + Ok(()) +} + +async fn run_retrieval_benchmark( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, +) -> Result<(RetrievalBenchmarkReport, Vec)> { + let prepared = prepare_benchmark(dataset, experiment).await?; + let traces = execute_queries(dataset, experiment, &prepared).await?; + let metrics = summarize(&traces); + let report = RetrievalBenchmarkReport { + experiment: experiment.clone(), + dataset_id: dataset.dataset_id.clone(), + evaluated_split: experiment.dataset_split.clone(), + ingest_count: prepared.ingest_count, + ingest_latency_ms: prepared.ingest_latency_ms, + metrics, + }; + + Ok((report, traces)) +} + +async fn run_passive_recall_benchmark( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, +) -> Result<(PassiveRecallBenchmarkReport, Vec)> { + let prepared = prepare_benchmark(dataset, experiment).await?; + let traces = execute_passive_recall_queries(dataset, experiment, &prepared).await?; + let metrics = summarize_passive_recall(&traces); + let report = PassiveRecallBenchmarkReport { + experiment: experiment.clone(), + dataset_id: dataset.dataset_id.clone(), + evaluated_split: experiment.dataset_split.clone(), + ingest_count: prepared.ingest_count, + ingest_latency_ms: prepared.ingest_latency_ms, + metrics, + }; + + Ok((report, traces)) +} + +async fn prepare_benchmark( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, +) -> Result { + let tmp = NamedTempFile::new()?; + let store = MemoryStore::open( + tmp.path() + .to_str() + .context("temporary benchmark db path was not valid utf-8")?, + experiment.embed_dim, + )?; + + let mut runtime = Config::default(); + runtime.embed_url = experiment.embed_url.clone(); + runtime.embed_model = experiment.embed_model.clone(); + runtime.embed_dim = experiment.embed_dim; + if let Some(url) = &experiment.rerank_url { + runtime.rerank_url = url.clone(); + } + if let Some(model) = &experiment.rerank_model { + runtime.rerank_model = model.clone(); + } + let llm = LlmClient::new(runtime); + + let ingest_start = Instant::now(); + for (idx, memory) in dataset.memories.iter().enumerate() { + eprintln!( + "[ingest {}/{}] embedding memory {}", + idx + 1, + dataset.memories.len(), + memory.memory_id + ); + let emb = llm + .embed(&memory.text) + .await + .with_context(|| format!("embedding failed for memory {}", memory.memory_id))?; + let input = MemoryRecordInput { + memory_id: Some(memory.memory_id), + namespace: memory.namespace.clone(), + layer: memory.layer.clone(), + text: memory.text.clone(), + event_time: memory.event_time, + ingest_time: memory.ingest_time.unwrap_or(memory.event_time), + embedding_model: experiment.embed_model.clone(), + embedding_dim: emb.len(), + embedding_version: experiment.config_version.clone(), + status: memory.status.clone(), + source_ref: memory.source_ref.clone(), + tags: memory.tags.clone(), + pinned: false, + embedding: emb, + }; + store.store_with_metadata(&input)?; + } + let ingest_latency_ms = ingest_start.elapsed().as_millis(); + eprintln!( + "ingest complete: {} memories in {} ms", + dataset.memories.len(), + ingest_latency_ms + ); + let all_corpus = store.get_all()?; + let corpus = filter_corpus_for_split(&all_corpus, dataset, experiment.dataset_split.as_ref())?; + eprintln!( + "corpus available for {:?}: {} / {} memories", + experiment.dataset_split, + corpus.len(), + all_corpus.len() + ); + + // Build support IDF from both memory texts and the query texts for this split. + // This avoids hardcoded stopwords/stemming while still downweighting very common + // query patterns like "what is ..." in a language/typo-tolerant way (char n-grams). + let query_texts = dataset + .queries + .iter() + .filter(|query| match &experiment.dataset_split { + Some(split) => &query.split == split, + None => true, + }) + .map(|query| query.text.as_str()); + let support = SupportScorer::from_texts( + corpus + .iter() + .map(|memory| memory.text.as_str()) + .chain(query_texts), + ); + Ok(PreparedBenchmark { + llm, + corpus, + support, + ingest_count: dataset.memories.len(), + ingest_latency_ms, + }) +} + +async fn execute_queries( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + prepared: &PreparedBenchmark, +) -> Result> { + let queries = dataset + .queries + .iter() + .filter(|query| match &experiment.dataset_split { + Some(split) => &query.split == split, + None => true, + }) + // Tool-lane queries are owned by the router/tooling path, not the memory retrieval path. + // Keeping them in the retrieval benchmark creates misleading "no-hit false answers" + // because retrieval has no way to inspect runtime state or read repo files. + .filter(|query| query.route_label.as_ref() != Some(&EvalRouteLabel::Tools)) + .collect::>(); + + if queries.is_empty() { + bail!("no queries matched the requested split"); + } + + let mut traces = Vec::with_capacity(queries.len()); + for (idx, query) in queries.into_iter().enumerate() { + eprintln!( + "[query {}/{}] {}", + idx + 1, + traces.capacity(), + query.query_id + ); + traces.push( + run_query( + query, + &prepared.corpus, + &prepared.llm, + &prepared.support, + experiment, + ) + .await?, + ); + } + + eprintln!("benchmark queries complete, summarizing metrics"); + Ok(traces) +} + +async fn execute_passive_recall_queries( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + prepared: &PreparedBenchmark, +) -> Result> { + let queries = dataset + .queries + .iter() + .filter(|query| query.objective == EvalObjective::PassiveRecall) + .filter(|query| match &experiment.dataset_split { + Some(split) => &query.split == split, + None => true, + }) + .collect::>(); + + if queries.is_empty() { + bail!("no passive recall queries matched the requested split"); + } + + let mut traces = Vec::with_capacity(queries.len()); + for (idx, query) in queries.into_iter().enumerate() { + eprintln!( + "[passive {}/{}] {}", + idx + 1, + traces.capacity(), + query.query_id + ); + traces.push( + run_passive_recall_query( + query, + &prepared.corpus, + &prepared.llm, + &prepared.support, + experiment, + ) + .await?, + ); + } + + eprintln!("passive recall queries complete, summarizing metrics"); + Ok(traces) +} + +async fn run_query( + query: &EvalQuery, + corpus: &[L1MemoryRecord], + llm: &LlmClient, + support: &SupportScorer, + experiment: &RetrievalExperimentConfig, +) -> Result { + eprintln!(" - embedding query {}", query.query_id); + let embed_started = Instant::now(); + let query_embedding = llm + .embed(&query.text) + .await + .with_context(|| format!("embedding failed for query {}", query.query_id))?; + let embedding_latency_ms = embed_started.elapsed().as_millis(); + + let windows = retrieval::window_schedule( + experiment.initial_window_days, + &experiment.expansion_window_days, + ); + let mut windows_considered = Vec::new(); + let mut window_expanded = false; + let mut time_window_hit = false; + let mut retrieval_latency_ms = 0u128; + let mut rerank_latency_ms = None; + let mut chosen_window_days = None; + let mut memories_searched = 0usize; + let mut top_candidates = Vec::new(); + let mut reranked_candidates = Vec::new(); + let mut window_attempts = Vec::new(); + let mut matched_gold = false; + let mut best_distance = None; + let mut final_decision = FinalDecision::Abstain; + let mut rerank = RerankOutcome::default(); + let mut support_top1_score = None; + + for (idx, window_days) in windows.iter().enumerate() { + windows_considered.push(*window_days); + let retrieval_started = Instant::now(); + let outcome = retrieval::retrieve_exact( + corpus, + &query_embedding, + &RetrievalConfig { + namespace: query.namespace.clone(), + top_k: experiment.top_k, + initial_window_days: *window_days, + expansion_window_days: Vec::new(), + expand_distance_threshold: None, + similarity_metric: experiment.similarity_metric.clone(), + reference_time: query.reference_time, + }, + Some(&query.gold_memory_ids), + ); + let retrieval_attempt_latency_ms = retrieval_started.elapsed().as_millis(); + retrieval_latency_ms += retrieval_attempt_latency_ms; + if idx == 0 { + time_window_hit = outcome.time_window_hit; + } + chosen_window_days = outcome.chosen_window_days; + memories_searched = outcome.memories_searched; + top_candidates = outcome + .top_candidates + .iter() + .map(|candidate| RetrievalCandidateLog { + memory_id: candidate.memory.memory_id, + score: candidate.score, + rank: candidate.rank, + event_time: candidate.memory.event_time, + tags: candidate.memory.tags.clone(), + source_ref: candidate.memory.source_ref.clone(), + text: candidate.memory.text.clone(), + }) + .collect::>(); + eprintln!( + " - first-stage retrieval {} searched {} memories in {} ms (window {:?})", + query.query_id, memories_searched, retrieval_attempt_latency_ms, window_days + ); + + let rerank_started = Instant::now(); + rerank = maybe_rerank_candidates(query, llm, experiment, &top_candidates).await?; + let rerank_attempt_latency_ms = rerank_started.elapsed().as_millis(); + rerank_latency_ms = Some(rerank_latency_ms.unwrap_or(0) + rerank_attempt_latency_ms); + if !rerank.raw_candidates.is_empty() { + eprintln!( + " - reranked {} candidates for {} in {} ms", + rerank.raw_candidates.len(), + query.query_id, + rerank_attempt_latency_ms + ); + } + + reranked_candidates = if rerank.raw_candidates.is_empty() { + Vec::new() + } else { + rerank.raw_candidates.clone() + }; + matched_gold = top_candidates + .iter() + .any(|candidate| query.gold_memory_ids.contains(&candidate.memory_id)); + best_distance = top_candidates.first().map(|candidate| candidate.score); + + let final_candidates = choose_final_candidates(&top_candidates, &rerank); + support_top1_score = top_candidate_support_score(support, &query.text, final_candidates); + let directive = decide_query_directive( + query, + &top_candidates, + final_candidates, + &rerank, + support_top1_score, + experiment, + idx + 1 < windows.len(), + ); + window_attempts.push(WindowAttemptLog { + window_days: *window_days, + memories_searched, + retrieval_latency_ms: retrieval_attempt_latency_ms, + rerank_latency_ms: (!rerank.raw_candidates.is_empty()) + .then_some(rerank_attempt_latency_ms), + first_stage_top1_distance: top_candidates.first().map(|candidate| candidate.score), + rerank_top1_raw_score: rerank + .raw_candidates + .first() + .map(|candidate| candidate.score), + rerank_top2_raw_score: rerank + .raw_candidates + .get(1) + .map(|candidate| candidate.score), + rerank_margin_raw: score_margin(&rerank.raw_candidates), + support_top1_score, + candidate_memory_ids: final_candidates + .iter() + .take(3) + .map(|candidate| candidate.memory_id) + .collect(), + decision: to_window_decision(directive), + }); + + match directive { + QueryDirective::Answer => { + final_decision = FinalDecision::Answer; + if rerank.raw_candidates.is_empty() { + reranked_candidates = Vec::new(); + } + break; + } + QueryDirective::Abstain => { + final_decision = FinalDecision::Abstain; + if rerank.raw_candidates.is_empty() { + reranked_candidates = Vec::new(); + } + break; + } + QueryDirective::Expand => { + window_expanded = true; + continue; + } + } + } + + let final_candidates = if reranked_candidates.is_empty() { + top_candidates.as_slice() + } else { + reranked_candidates.as_slice() + }; + let abstained = final_decision == FinalDecision::Abstain; + let final_answer_memory_ids = if abstained { + Vec::new() + } else { + final_candidates + .first() + .map(|candidate| vec![candidate.memory_id]) + .unwrap_or_default() + }; + let final_correct = if query.no_hit { + abstained + } else { + !abstained + && final_answer_memory_ids + .iter() + .any(|memory_id| query.gold_memory_ids.contains(memory_id)) + }; + + Ok(RetrievalTrace { + query_id: query.query_id.clone(), + query_text: query.text.clone(), + query_category: query.category.clone(), + split: query.split.clone(), + namespace: query.namespace.clone(), + reference_time: query.reference_time, + chosen_window_days, + windows_considered, + memories_searched, + vector_search_mode: VectorSearchMode::Exact, + top_candidates, + reranked_candidates, + window_attempts, + window_expanded, + abstained, + no_hit: query.no_hit, + gold_memory_ids: query.gold_memory_ids.clone(), + gold_answer: query.gold_answer.clone(), + first_stage_topk_contains_gold: matched_gold, + first_stage_top1_distance: best_distance, + rerank_top1_raw_score: rerank + .raw_candidates + .first() + .map(|candidate| candidate.score), + rerank_top2_raw_score: rerank + .raw_candidates + .get(1) + .map(|candidate| candidate.score), + rerank_margin_raw: score_margin(&rerank.raw_candidates), + support_top1_score, + final_decision, + final_answer_memory_ids, + final_correct, + matched_gold, + time_window_hit, + best_distance, + embedding_latency_ms, + retrieval_latency_ms, + rerank_latency_ms, + generator_answer: None, + passed: final_correct, + }) +} + +async fn run_passive_recall_query( + query: &EvalQuery, + corpus: &[L1MemoryRecord], + llm: &LlmClient, + support: &SupportScorer, + experiment: &RetrievalExperimentConfig, +) -> Result { + eprintln!(" - embedding passive query {}", query.query_id); + let embed_started = Instant::now(); + let query_embedding = llm.embed(&query.text).await.with_context(|| { + format!( + "embedding failed for passive recall query {}", + query.query_id + ) + })?; + let embedding_latency_ms = embed_started.elapsed().as_millis(); + + let windows = retrieval::window_schedule( + experiment.initial_window_days, + &experiment.expansion_window_days, + ); + let mut windows_considered = Vec::new(); + let mut window_expanded = false; + let mut time_window_hit = false; + let mut retrieval_latency_ms = 0u128; + let mut rerank_latency_ms = None; + let mut chosen_window_days = None; + let mut memories_searched = 0usize; + let mut top_candidates = Vec::new(); + let mut reranked_candidates = Vec::new(); + let mut window_attempts = Vec::new(); + let mut best_distance = None; + let mut support_top1_score = None; + let mut rerank = RerankOutcome::default(); + let mut activation_decision = PassiveRecallDecision::NoRecall; + + for (idx, window_days) in windows.iter().enumerate() { + windows_considered.push(*window_days); + let retrieval_started = Instant::now(); + let outcome = retrieval::retrieve_exact( + corpus, + &query_embedding, + &RetrievalConfig { + namespace: query.namespace.clone(), + top_k: experiment.top_k, + initial_window_days: *window_days, + expansion_window_days: Vec::new(), + expand_distance_threshold: None, + similarity_metric: experiment.similarity_metric.clone(), + reference_time: query.reference_time, + }, + Some(&query.gold_memory_ids), + ); + let retrieval_attempt_latency_ms = retrieval_started.elapsed().as_millis(); + retrieval_latency_ms += retrieval_attempt_latency_ms; + if idx == 0 { + time_window_hit = outcome.time_window_hit; + } + chosen_window_days = outcome.chosen_window_days; + memories_searched = outcome.memories_searched; + top_candidates = outcome + .top_candidates + .iter() + .map(|candidate| RetrievalCandidateLog { + memory_id: candidate.memory.memory_id, + score: candidate.score, + rank: candidate.rank, + event_time: candidate.memory.event_time, + tags: candidate.memory.tags.clone(), + source_ref: candidate.memory.source_ref.clone(), + text: candidate.memory.text.clone(), + }) + .collect::>(); + eprintln!( + " - passive first-stage retrieval {} searched {} memories in {} ms (window {:?})", + query.query_id, memories_searched, retrieval_attempt_latency_ms, window_days + ); + + let rerank_started = Instant::now(); + rerank = maybe_rerank_candidates(query, llm, experiment, &top_candidates).await?; + let rerank_attempt_latency_ms = rerank_started.elapsed().as_millis(); + rerank_latency_ms = Some(rerank_latency_ms.unwrap_or(0) + rerank_attempt_latency_ms); + if !rerank.raw_candidates.is_empty() { + eprintln!( + " - passive reranked {} candidates for {} in {} ms", + rerank.raw_candidates.len(), + query.query_id, + rerank_attempt_latency_ms + ); + } + + reranked_candidates = if rerank.raw_candidates.is_empty() { + Vec::new() + } else { + rerank.raw_candidates.clone() + }; + best_distance = top_candidates.first().map(|candidate| candidate.score); + + let final_candidates = choose_final_candidates(&top_candidates, &rerank); + support_top1_score = top_candidate_support_score(support, &query.text, final_candidates); + let directive = decide_query_directive( + query, + &top_candidates, + final_candidates, + &rerank, + support_top1_score, + experiment, + idx + 1 < windows.len(), + ); + window_attempts.push(WindowAttemptLog { + window_days: *window_days, + memories_searched, + retrieval_latency_ms: retrieval_attempt_latency_ms, + rerank_latency_ms: (!rerank.raw_candidates.is_empty()) + .then_some(rerank_attempt_latency_ms), + first_stage_top1_distance: top_candidates.first().map(|candidate| candidate.score), + rerank_top1_raw_score: rerank + .raw_candidates + .first() + .map(|candidate| candidate.score), + rerank_top2_raw_score: rerank + .raw_candidates + .get(1) + .map(|candidate| candidate.score), + rerank_margin_raw: score_margin(&rerank.raw_candidates), + support_top1_score, + candidate_memory_ids: final_candidates + .iter() + .take(PASSIVE_RECALL_TOP_K) + .map(|candidate| candidate.memory_id) + .collect(), + decision: to_window_decision(directive), + }); + + match directive { + QueryDirective::Answer => { + activation_decision = PassiveRecallDecision::Recall; + if rerank.raw_candidates.is_empty() { + reranked_candidates = Vec::new(); + } + break; + } + QueryDirective::Abstain => { + activation_decision = PassiveRecallDecision::NoRecall; + if rerank.raw_candidates.is_empty() { + reranked_candidates = Vec::new(); + } + break; + } + QueryDirective::Expand => { + window_expanded = true; + continue; + } + } + } + + let final_candidates = if reranked_candidates.is_empty() { + top_candidates.as_slice() + } else { + reranked_candidates.as_slice() + }; + let recalled_memory_ids = if activation_decision == PassiveRecallDecision::Recall { + let top1_score = final_candidates.first().map(|c| c.score); + final_candidates + .iter() + .take(PASSIVE_RECALL_TOP_K) + .filter( + |c| match (experiment.passive_support_score_gap, top1_score) { + (Some(gap), Some(top)) => top - c.score <= gap, + _ => true, + }, + ) + .map(|candidate| candidate.memory_id) + .collect::>() + } else { + Vec::new() + }; + let should_recall = expected_memory_action(query) == ExpectedMemoryAction::Recall; + let activation_correct = match activation_decision { + PassiveRecallDecision::Recall => should_recall, + PassiveRecallDecision::NoRecall => !should_recall, + }; + let support_hits = recalled_memory_ids + .iter() + .filter(|memory_id| query.gold_memory_ids.contains(memory_id)) + .count(); + let support_precision_at_k = (!recalled_memory_ids.is_empty()) + .then_some(support_hits as f32 / recalled_memory_ids.len() as f32); + let support_recall_at_k = should_recall.then_some(if query.gold_memory_ids.is_empty() { + 0.0 + } else { + support_hits as f32 / query.gold_memory_ids.len() as f32 + }); + + Ok(PassiveRecallTrace { + query_id: query.query_id.clone(), + query_text: query.text.clone(), + query_category: query.category.clone(), + interaction_mode: query.interaction_mode.clone(), + split: query.split.clone(), + namespace: query.namespace.clone(), + reference_time: query.reference_time, + expected_memory_action: query.expected_memory_action.clone(), + chosen_window_days, + windows_considered, + memories_searched, + vector_search_mode: VectorSearchMode::Exact, + top_candidates, + reranked_candidates, + window_attempts, + window_expanded, + gold_memory_ids: query.gold_memory_ids.clone(), + no_hit: query.no_hit, + first_stage_top1_distance: best_distance, + rerank_top1_raw_score: rerank + .raw_candidates + .first() + .map(|candidate| candidate.score), + rerank_top2_raw_score: rerank + .raw_candidates + .get(1) + .map(|candidate| candidate.score), + rerank_margin_raw: score_margin(&rerank.raw_candidates), + support_top1_score, + activation_decision, + recalled_memory_ids, + activation_correct, + support_precision_at_k, + support_recall_at_k, + time_window_hit, + embedding_latency_ms, + retrieval_latency_ms, + rerank_latency_ms, + passed: activation_correct, + }) +} + +fn to_window_decision(directive: QueryDirective) -> WindowDecision { + match directive { + QueryDirective::Answer => WindowDecision::Answer, + QueryDirective::Expand => WindowDecision::Expand, + QueryDirective::Abstain => WindowDecision::Abstain, + } +} + +fn validate_timeline_splits(dataset: &InternalEvalDataset) -> Result<()> { + let memory_ids = dataset + .memories + .iter() + .map(|memory| memory.memory_id) + .collect::>(); + let memory_timelines = dataset + .memories + .iter() + .map(|memory| (memory.memory_id, memory.timeline_id.as_deref())) + .collect::>(); + let mut timeline_to_split = HashMap::::new(); + for query in &dataset.queries { + let Some(timeline_id) = query.timeline_id.as_ref() else { + for gold_memory_id in &query.gold_memory_ids { + if !memory_ids.contains(gold_memory_id) { + bail!( + "query '{}' references unknown gold memory id {}", + query.query_id, + gold_memory_id + ); + } + } + continue; + }; + if let Some(existing) = timeline_to_split.get(timeline_id) { + if existing != &query.split { + bail!( + "timeline_id '{}' appears in multiple splits ({:?} and {:?})", + timeline_id, + existing, + query.split + ); + } + } else { + timeline_to_split.insert(timeline_id.clone(), query.split.clone()); + } + } + + for query in &dataset.queries { + for gold_memory_id in &query.gold_memory_ids { + let Some(memory_timeline) = memory_timelines.get(gold_memory_id) else { + bail!( + "query '{}' references unknown gold memory id {}", + query.query_id, + gold_memory_id + ); + }; + if let Some(memory_timeline) = memory_timeline { + if let Some(expected_split) = timeline_to_split.get(*memory_timeline) { + if expected_split != &query.split { + bail!( + "query '{}' in {:?} references gold memory {} from timeline '{}' assigned to {:?}", + query.query_id, + query.split, + gold_memory_id, + memory_timeline, + expected_split + ); + } + } + } + } + } + Ok(()) +} + +fn filter_corpus_for_split( + corpus: &[L1MemoryRecord], + dataset: &InternalEvalDataset, + split: Option<&DatasetSplit>, +) -> Result> { + let Some(split) = split else { + return Ok(corpus.to_vec()); + }; + + let has_timeline_metadata = dataset + .queries + .iter() + .any(|query| query.timeline_id.is_some()) + || dataset + .memories + .iter() + .any(|memory| memory.timeline_id.is_some()); + if !has_timeline_metadata { + return Ok(corpus.to_vec()); + } + + let queries_in_split = dataset + .queries + .iter() + .filter(|query| &query.split == split) + .collect::>(); + let queries_missing_timeline = queries_in_split + .iter() + .filter(|query| query.timeline_id.is_none()) + .map(|query| query.query_id.clone()) + .collect::>(); + if !queries_missing_timeline.is_empty() { + bail!( + "split {:?} has queries without timeline_id: {}", + split, + queries_missing_timeline.join(", ") + ); + } + + let allowed_timelines = queries_in_split + .iter() + .filter_map(|query| query.timeline_id.as_deref()) + .collect::>(); + if allowed_timelines.is_empty() { + bail!( + "split {:?} has timeline metadata enabled but no timeline_id values were found", + split + ); + } + + let memory_timelines = dataset + .memories + .iter() + .map(|memory| (memory.memory_id, memory.timeline_id.as_deref())) + .collect::>(); + + let filtered = corpus + .iter() + .filter(|memory| { + memory_timelines + .get(&memory.memory_id) + .copied() + .flatten() + .map(|timeline_id| allowed_timelines.contains(timeline_id)) + .unwrap_or(false) + }) + .cloned() + .collect::>(); + + if filtered.is_empty() { + bail!( + "timeline filtering removed the entire corpus for split {:?}", + split + ); + } + + Ok(filtered) +} + +async fn maybe_rerank_candidates( + query: &EvalQuery, + llm: &LlmClient, + experiment: &RetrievalExperimentConfig, + first_stage_candidates: &[RetrievalCandidateLog], +) -> Result { + let Some(rerank_url) = &experiment.rerank_url else { + return Ok(RerankOutcome::default()); + }; + if first_stage_candidates.is_empty() { + return Ok(RerankOutcome::default()); + } + + let rerank_limit = experiment + .rerank_top_k + .unwrap_or(first_stage_candidates.len()) + .min(first_stage_candidates.len()); + let rerank_pool = &first_stage_candidates[..rerank_limit]; + let documents = rerank_pool + .iter() + .map(|candidate| { + if candidate.tags.is_empty() { + candidate.text.clone() + } else { + format!("[tags: {}] {}", candidate.tags.join(", "), candidate.text) + } + }) + .collect::>(); + + eprintln!( + " rerank request for {} over top {} candidates via {}", + query.query_id, + rerank_pool.len(), + rerank_url + ); + + let mut rerank_client = llm.clone(); + rerank_client.config.rerank_url = rerank_url.clone(); + if let Some(model) = &experiment.rerank_model { + rerank_client.config.rerank_model = model.clone(); + } + + let query_text = if let Some(ctx) = &query.context { + format!("[context: {}] {}", ctx, query.text) + } else { + query.text.clone() + }; + let raw_results = rerank_client + .rerank(&query_text, &documents, false) + .await + .with_context(|| format!("reranking failed for query {}", query.query_id))?; + + Ok(RerankOutcome { + raw_candidates: apply_rerank_scores(rerank_pool, raw_results), + }) +} + +fn apply_rerank_scores( + rerank_pool: &[RetrievalCandidateLog], + rerank_results: Vec, +) -> Vec { + let mut reranked = rerank_results + .into_iter() + .filter_map(|result| { + rerank_pool.get(result.index).map(|candidate| { + let mut candidate = candidate.clone(); + candidate.score = result.score; + candidate + }) + }) + .collect::>(); + reranked.sort_by(|a, b| { + b.score + .partial_cmp(&a.score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + for (idx, candidate) in reranked.iter_mut().enumerate() { + candidate.rank = idx + 1; + } + reranked +} + +fn choose_final_candidates<'a>( + top_candidates: &'a [RetrievalCandidateLog], + rerank: &'a RerankOutcome, +) -> &'a [RetrievalCandidateLog] { + if !rerank.raw_candidates.is_empty() { + &rerank.raw_candidates + } else { + top_candidates + } +} + +fn decide_query_directive( + query: &EvalQuery, + top_candidates: &[RetrievalCandidateLog], + final_candidates: &[RetrievalCandidateLog], + rerank: &RerankOutcome, + support_top1_score: Option, + experiment: &RetrievalExperimentConfig, + has_more_windows: bool, +) -> QueryDirective { + if final_candidates.is_empty() { + return if has_more_windows { + QueryDirective::Expand + } else { + QueryDirective::Abstain + }; + } + + if !rerank.raw_candidates.is_empty() { + let decision_top1 = rerank + .raw_candidates + .first() + .map(|candidate| candidate.score); + let decision_margin = score_margin(&rerank.raw_candidates); + let mut below_confidence = false; + if let Some(threshold) = experiment.rerank_abstain_score_threshold { + below_confidence |= decision_top1.is_none_or(|score| score < threshold); + } + if let Some(threshold) = experiment.rerank_abstain_margin_threshold { + below_confidence |= decision_margin.is_none_or(|margin| margin < threshold); + } + // Only allow bypassing the support gate for passive recall (activation), not for + // active retrieval (answering a question). Otherwise, confident-but-irrelevant + // rerank scores can still produce no-hit false answers. + let can_bypass_support = query.objective == EvalObjective::PassiveRecall; + let rerank_confident = can_bypass_support + && experiment + .rerank_confident_score_threshold + .zip(decision_top1) + .is_some_and(|(threshold, score)| score >= threshold); + if !rerank_confident { + if let Some(threshold) = experiment.support_abstain_threshold { + below_confidence |= support_top1_score.is_none_or(|score| score < threshold); + } + } + if below_confidence { + return if has_more_windows { + QueryDirective::Expand + } else { + QueryDirective::Abstain + }; + } + return QueryDirective::Answer; + } + + let best_distance = top_candidates.first().map(|candidate| candidate.score); + if experiment + .abstain_distance_threshold + .zip(best_distance) + .is_some_and(|(threshold, distance)| distance > threshold) + { + if has_more_windows { + QueryDirective::Expand + } else { + QueryDirective::Abstain + } + } else if experiment + .support_abstain_threshold + .is_some_and(|threshold| support_top1_score.is_none_or(|score| score < threshold)) + { + if has_more_windows { + QueryDirective::Expand + } else { + QueryDirective::Abstain + } + } else if query.no_hit && top_candidates.is_empty() { + QueryDirective::Abstain + } else { + QueryDirective::Answer + } +} + +fn score_margin(candidates: &[RetrievalCandidateLog]) -> Option { + let top1 = candidates.first()?.score; + let top2 = candidates.get(1)?.score; + Some(top1 - top2) +} + +fn expected_memory_action(query: &EvalQuery) -> ExpectedMemoryAction { + query.expected_memory_action.clone().unwrap_or_else(|| { + if query.no_hit || query.gold_memory_ids.is_empty() { + ExpectedMemoryAction::NoRecall + } else { + ExpectedMemoryAction::Recall + } + }) +} + +fn top_candidate_support_score( + support: &SupportScorer, + query_text: &str, + candidates: &[RetrievalCandidateLog], +) -> Option { + candidates + .first() + .map(|candidate| support.score(query_text, &candidate.text)) +} + +#[derive(Debug, Clone)] +struct SupportScorer { + gram_idf: HashMap, + default_idf: f32, + n: usize, +} + +impl SupportScorer { + fn from_texts<'a, I>(texts: I) -> Self + where + I: IntoIterator, + { + let n = 4usize; + let mut doc_count_usize = 0usize; + + let mut df: HashMap = HashMap::new(); + for text in texts { + doc_count_usize += 1; + let grams = char_ngrams(text, n); + for gram in grams { + *df.entry(gram).or_insert(0) += 1; + } + } + + let doc_count = doc_count_usize.max(1) as f32; + let default_idf = ((doc_count + 1.0) / 1.0).ln() + 1.0; + + let gram_idf = df + .into_iter() + .map(|(gram, doc_freq)| { + // Smooth IDF: + // idf = ln((N + 1) / (df + 1)) + 1 + let idf = ((doc_count + 1.0) / (doc_freq as f32 + 1.0)).ln() + 1.0; + (gram, idf) + }) + .collect(); + + Self { + gram_idf, + default_idf, + n, + } + } + + fn score(&self, query_text: &str, candidate_text: &str) -> f32 { + let query_grams = char_ngrams(query_text, self.n); + if query_grams.is_empty() { + return 0.0; + } + let candidate_grams = char_ngrams(candidate_text, self.n); + + let mut total_weight = 0.0f32; + let mut matched_weight = 0.0f32; + + for gram in query_grams.iter() { + let idf = self.idf(gram); + total_weight += idf; + if candidate_grams.contains(gram) { + matched_weight += idf; + } + } + + if total_weight <= 0.0 { + return 0.0; + } + + let mut score = (matched_weight / total_weight).clamp(0.0, 1.0); + + // Penalize "single-token overlap" cases (common no-hit failure mode) without + // hardcoded stopwords or English stemming. With n-grams >= 4 this avoids most + // spurious suffix matches (like "...ing"). + let tokens = normalized_tokens(query_text); + if tokens.len() >= 2 { + let mut overlapped_tokens = 0usize; + for token in tokens { + let token_grams = token_ngrams(&token, self.n); + if token_grams + .iter() + .any(|gram| candidate_grams.contains(gram)) + { + overlapped_tokens += 1; + } + } + let scale = ((overlapped_tokens as f32) / 2.0).min(1.0); + score = (score * scale).clamp(0.0, 1.0); + } + + score + } + + fn idf(&self, gram: &str) -> f32 { + self.gram_idf.get(gram).copied().unwrap_or(self.default_idf) + } +} + +fn char_ngrams(text: &str, n: usize) -> HashSet { + let mut grams = HashSet::new(); + let mut token = String::new(); + + let flush = |token: &mut String, grams: &mut HashSet| { + if token.is_empty() { + return; + } + let chars: Vec = token.chars().collect(); + if chars.len() <= n { + grams.insert(token.clone()); + } else { + for start in 0..=(chars.len() - n) { + grams.insert(chars[start..start + n].iter().collect()); + } + } + token.clear(); + }; + + for ch in text.chars() { + if ch.is_alphanumeric() { + for lower in ch.to_lowercase() { + token.push(lower); + } + } else { + flush(&mut token, &mut grams); + } + } + flush(&mut token, &mut grams); + + grams +} + +fn normalized_tokens(text: &str) -> Vec { + // Unicode-aware word segmentation (UAX#29), then light normalization so token n-grams + // stay robust across punctuation/typos without English-specific stemming. + text.unicode_words() + .filter_map(|word| { + let mut out = String::new(); + for ch in word.chars() { + if ch.is_alphanumeric() { + for lower in ch.to_lowercase() { + out.push(lower); + } + } + } + (!out.is_empty()).then_some(out) + }) + .collect() +} + +fn token_ngrams(token: &str, n: usize) -> HashSet { + let mut grams = HashSet::new(); + let chars: Vec = token.chars().collect(); + if chars.is_empty() { + return grams; + } + if chars.len() <= n { + grams.insert(token.to_string()); + return grams; + } + for start in 0..=(chars.len() - n) { + grams.insert(chars[start..start + n].iter().collect()); + } + grams +} + +fn summarize(traces: &[RetrievalTrace]) -> RetrievalMetrics { + let query_count = traces.len(); + let hit_queries = traces + .iter() + .filter(|trace| !trace.no_hit) + .collect::>(); + let hit_count = hit_queries.len().max(1) as f32; + let no_hit_queries = traces + .iter() + .filter(|trace| trace.no_hit) + .collect::>(); + let no_hit_count = no_hit_queries.len().max(1) as f32; + + let recall_at = |k: usize| -> f32 { + hit_queries + .iter() + .filter(|trace| { + trace + .top_candidates + .iter() + .take(k) + .any(|candidate| trace.gold_memory_ids.contains(&candidate.memory_id)) + }) + .count() as f32 + / hit_count + }; + + let mrr = hit_queries + .iter() + .map(|trace| reciprocal_rank(trace)) + .sum::() + / hit_count; + let ndcg_at_10 = hit_queries + .iter() + .map(|trace| ndcg_at_k(trace, 10)) + .sum::() + / hit_count; + let top1_accuracy_after_rerank = hit_queries + .iter() + .filter(|trace| { + final_candidates(trace) + .first() + .map(|candidate| trace.gold_memory_ids.contains(&candidate.memory_id)) + .unwrap_or(false) + }) + .count() as f32 + / hit_count; + let mrr_after_rerank = hit_queries + .iter() + .map(|trace| reciprocal_rank_from(final_candidates(trace), &trace.gold_memory_ids)) + .sum::() + / hit_count; + let ndcg_after_rerank = hit_queries + .iter() + .map(|trace| ndcg_at_k_from(final_candidates(trace), &trace.gold_memory_ids, 10)) + .sum::() + / hit_count; + let final_top1_accuracy = hit_queries + .iter() + .filter(|trace| { + trace.final_decision == FinalDecision::Answer + && trace + .final_answer_memory_ids + .iter() + .any(|memory_id| trace.gold_memory_ids.contains(memory_id)) + }) + .count() as f32 + / hit_count; + let time_window_hit_rate = hit_queries + .iter() + .filter(|trace| trace.time_window_hit) + .count() as f32 + / hit_count; + let abstain_accuracy = no_hit_queries + .iter() + .filter(|trace| trace.abstained) + .count() as f32 + / no_hit_count; + let final_decision_accuracy = + traces.iter().filter(|trace| trace.final_correct).count() as f32 / query_count as f32; + let answered_queries = traces + .iter() + .filter(|trace| trace.final_decision == FinalDecision::Answer) + .collect::>(); + let coverage = answered_queries.len() as f32 / query_count as f32; + let selective_accuracy = if answered_queries.is_empty() { + 0.0 + } else { + answered_queries + .iter() + .filter(|trace| trace.final_correct) + .count() as f32 + / answered_queries.len() as f32 + }; + + let candidate_set_size_mean = traces + .iter() + .map(|trace| trace.memories_searched as f32) + .sum::() + / query_count as f32; + + let retrieval_latencies = traces + .iter() + .map(|trace| trace.retrieval_latency_ms) + .collect::>(); + let embedding_latencies = traces + .iter() + .map(|trace| trace.embedding_latency_ms) + .collect::>(); + let rerank_latencies = traces + .iter() + .filter_map(|trace| trace.rerank_latency_ms) + .collect::>(); + + RetrievalMetrics { + query_count, + recall_at_1: recall_at(1), + recall_at_5: recall_at(5), + recall_at_10: recall_at(10), + recall_at_20: recall_at(20), + mrr, + ndcg_at_10, + top1_accuracy_after_rerank, + final_top1_accuracy, + mrr_after_rerank, + ndcg_after_rerank, + reranker_lift_mrr: mrr_after_rerank - mrr, + candidate_set_size_mean, + time_window_hit_rate, + final_decision_accuracy, + coverage, + abstain_accuracy, + selective_accuracy, + retrieval_latency_p50_ms: percentile(&retrieval_latencies, 0.50), + retrieval_latency_p95_ms: percentile(&retrieval_latencies, 0.95), + embedding_latency_p50_ms: percentile(&embedding_latencies, 0.50), + embedding_latency_p95_ms: percentile(&embedding_latencies, 0.95), + rerank_latency_p50_ms: percentile(&rerank_latencies, 0.50), + rerank_latency_p95_ms: percentile(&rerank_latencies, 0.95), + } +} + +fn summarize_passive_recall(traces: &[PassiveRecallTrace]) -> PassiveRecallMetrics { + let query_count = traces.len().max(1) as f32; + let should_recall_queries = traces + .iter() + .filter(|trace| expected_trace_action(trace) == ExpectedMemoryAction::Recall) + .collect::>(); + let should_not_recall_queries = traces + .iter() + .filter(|trace| expected_trace_action(trace) == ExpectedMemoryAction::NoRecall) + .collect::>(); + let recalled_queries = traces + .iter() + .filter(|trace| trace.activation_decision == PassiveRecallDecision::Recall) + .collect::>(); + + let should_recall_count = should_recall_queries.len(); + let should_not_recall_count = should_not_recall_queries.len(); + + let activation_precision = if recalled_queries.is_empty() { + 0.0 + } else { + recalled_queries + .iter() + .filter(|trace| expected_trace_action(trace) == ExpectedMemoryAction::Recall) + .count() as f32 + / recalled_queries.len() as f32 + }; + let activation_recall = if should_recall_queries.is_empty() { + 0.0 + } else { + should_recall_queries + .iter() + .filter(|trace| trace.activation_decision == PassiveRecallDecision::Recall) + .count() as f32 + / should_recall_queries.len() as f32 + }; + let no_recall_false_activation_rate = if should_not_recall_queries.is_empty() { + 0.0 + } else { + should_not_recall_queries + .iter() + .filter(|trace| trace.activation_decision == PassiveRecallDecision::Recall) + .count() as f32 + / should_not_recall_queries.len() as f32 + }; + let support_precision_at_k_mean = if recalled_queries.is_empty() { + 0.0 + } else { + recalled_queries + .iter() + .filter_map(|trace| trace.support_precision_at_k) + .sum::() + / recalled_queries.len() as f32 + }; + let support_recall_at_k_mean = if should_recall_queries.is_empty() { + 0.0 + } else { + should_recall_queries + .iter() + .map(|trace| trace.support_recall_at_k.unwrap_or(0.0)) + .sum::() + / should_recall_queries.len() as f32 + }; + + let retrieval_latencies = traces + .iter() + .map(|trace| trace.retrieval_latency_ms) + .collect::>(); + let embedding_latencies = traces + .iter() + .map(|trace| trace.embedding_latency_ms) + .collect::>(); + let rerank_latencies = traces + .iter() + .filter_map(|trace| trace.rerank_latency_ms) + .collect::>(); + + PassiveRecallMetrics { + query_count: traces.len(), + should_recall_count, + should_not_recall_count, + activation_accuracy: traces + .iter() + .filter(|trace| trace.activation_correct) + .count() as f32 + / query_count, + activation_precision, + activation_recall, + no_recall_false_activation_rate, + support_precision_at_k_mean, + support_recall_at_k_mean, + recall_rate: recalled_queries.len() as f32 / query_count, + candidate_set_size_mean: traces + .iter() + .map(|trace| trace.memories_searched as f32) + .sum::() + / query_count, + time_window_hit_rate: if should_recall_queries.is_empty() { + 0.0 + } else { + should_recall_queries + .iter() + .filter(|trace| trace.time_window_hit) + .count() as f32 + / should_recall_queries.len() as f32 + }, + retrieval_latency_p50_ms: percentile(&retrieval_latencies, 0.50), + retrieval_latency_p95_ms: percentile(&retrieval_latencies, 0.95), + embedding_latency_p50_ms: percentile(&embedding_latencies, 0.50), + embedding_latency_p95_ms: percentile(&embedding_latencies, 0.95), + rerank_latency_p50_ms: percentile(&rerank_latencies, 0.50), + rerank_latency_p95_ms: percentile(&rerank_latencies, 0.95), + } +} + +fn expected_trace_action(trace: &PassiveRecallTrace) -> ExpectedMemoryAction { + trace.expected_memory_action.clone().unwrap_or_else(|| { + if trace.no_hit || trace.gold_memory_ids.is_empty() { + ExpectedMemoryAction::NoRecall + } else { + ExpectedMemoryAction::Recall + } + }) +} + +async fn run_policy_sweep( + dataset: &InternalEvalDataset, + base_experiment: &RetrievalExperimentConfig, + prepared: &PreparedBenchmark, + grid: SweepGrid, +) -> Result { + let records = collect_sweep_query_records(dataset, base_experiment, prepared).await?; + let score_values = grid.score.values()?; + let margin_values = grid.margin.values()?; + let support_values = grid.support.values()?; + let total_points = score_values.len() * margin_values.len() * support_values.len(); + let mut results = Vec::with_capacity(total_points); + + for (score_idx, score_threshold) in score_values.iter().enumerate() { + for (margin_idx, margin_threshold) in margin_values.iter().enumerate() { + for (support_idx, support_threshold) in support_values.iter().enumerate() { + let point_index = ((score_idx * margin_values.len()) + margin_idx) + * support_values.len() + + support_idx + + 1; + eprintln!( + "[sweep {}/{}] score>={:.3} margin>={:.3} support>={:.3}", + point_index, total_points, score_threshold, margin_threshold, support_threshold + ); + results.push(summarize_sweep_point( + *score_threshold, + *margin_threshold, + *support_threshold, + &records, + )); + } + } + } + + if results.is_empty() { + bail!("sweep produced no threshold points"); + } + + let mut frontier = results + .iter() + .filter(|candidate| { + !results + .iter() + .any(|other| !std::ptr::eq(*candidate, other) && dominates(other, candidate)) + }) + .cloned() + .collect::>(); + frontier.sort_by(compare_sweep_points); + + let mut ranked = results.clone(); + ranked.sort_by(compare_sweep_points); + let best_point = ranked + .first() + .cloned() + .context("sweep ranking unexpectedly produced no points")?; + + Ok(SweepReport { + dataset_id: dataset.dataset_id.clone(), + evaluated_split: base_experiment.dataset_split.clone(), + experiment_id: base_experiment.experiment_id.clone(), + rerank_model: base_experiment.rerank_model.clone(), + grid, + total_points, + best_point, + frontier, + results, + }) +} + +async fn collect_sweep_query_records( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + prepared: &PreparedBenchmark, +) -> Result> { + if experiment.rerank_url.is_none() { + bail!("sweep requires rerank_url to be configured"); + } + + let queries = dataset + .queries + .iter() + .filter(|query| match &experiment.dataset_split { + Some(split) => &query.split == split, + None => true, + }) + .collect::>(); + + if queries.is_empty() { + bail!("no queries matched the requested split"); + } + + let mut records = Vec::with_capacity(queries.len()); + for (idx, query) in queries.into_iter().enumerate() { + eprintln!( + "[sweep-cache {}/{}] {}", + idx + 1, + records.capacity(), + query.query_id + ); + let trace = run_query_across_all_windows( + query, + &prepared.corpus, + &prepared.llm, + &prepared.support, + experiment, + ) + .await?; + records.push(SweepQueryRecord { + query_id: query.query_id.clone(), + no_hit: query.no_hit, + gold_memory_ids: query.gold_memory_ids.clone(), + attempts: trace + .window_attempts + .into_iter() + .map(|attempt| SweepAttemptRecord { + answer_memory_id: attempt.candidate_memory_ids.first().copied(), + rerank_top1_raw_score: attempt.rerank_top1_raw_score, + rerank_margin_raw: attempt.rerank_margin_raw, + support_top1_score: attempt.support_top1_score, + rerank_latency_ms: attempt.rerank_latency_ms.unwrap_or(0), + }) + .collect(), + }); + } + + eprintln!("sweep cache complete, evaluating threshold grid"); + Ok(records) +} + +async fn run_query_across_all_windows( + query: &EvalQuery, + corpus: &[L1MemoryRecord], + llm: &LlmClient, + support: &SupportScorer, + experiment: &RetrievalExperimentConfig, +) -> Result { + let mut exhaustive = experiment.clone(); + exhaustive.rerank_abstain_score_threshold = Some(f32::INFINITY); + exhaustive.rerank_abstain_margin_threshold = Some(f32::INFINITY); + exhaustive.support_abstain_threshold = Some(f32::INFINITY); + run_query(query, corpus, llm, support, &exhaustive).await +} + +fn summarize_sweep_point( + score_threshold: f32, + margin_threshold: f32, + support_threshold: f32, + records: &[SweepQueryRecord], +) -> SweepPointResult { + let evaluated = records + .iter() + .map(|record| { + evaluate_sweep_query(record, score_threshold, margin_threshold, support_threshold) + }) + .collect::>(); + + let query_count = evaluated.len().max(1) as f32; + let answerable_queries = evaluated + .iter() + .filter(|result| !result.no_hit) + .collect::>(); + let no_hit_queries = evaluated + .iter() + .filter(|result| result.no_hit) + .collect::>(); + let answered_queries = evaluated + .iter() + .filter(|result| result.final_decision == FinalDecision::Answer) + .collect::>(); + let queries_requiring_expansion = evaluated + .iter() + .filter(|result| result.window_expanded) + .map(|result| result.query_id.clone()) + .collect::>(); + let no_hit_false_answer_count = no_hit_queries + .iter() + .filter(|result| result.final_decision == FinalDecision::Answer) + .count(); + + SweepPointResult { + rerank_score_threshold: score_threshold, + rerank_margin_threshold: margin_threshold, + support_threshold, + final_decision_accuracy: evaluated + .iter() + .filter(|result| result.final_correct) + .count() as f32 + / query_count, + answerable_coverage: if answerable_queries.is_empty() { + 0.0 + } else { + answerable_queries + .iter() + .filter(|result| result.final_decision == FinalDecision::Answer) + .count() as f32 + / answerable_queries.len() as f32 + }, + overall_coverage: answered_queries.len() as f32 / query_count, + no_hit_false_answer_count, + no_hit_abstain_recall: if no_hit_queries.is_empty() { + 0.0 + } else { + no_hit_queries + .iter() + .filter(|result| result.final_decision == FinalDecision::Abstain) + .count() as f32 + / no_hit_queries.len() as f32 + }, + selective_accuracy: if answered_queries.is_empty() { + 0.0 + } else { + answered_queries + .iter() + .filter(|result| result.final_correct) + .count() as f32 + / answered_queries.len() as f32 + }, + mean_windows_used: evaluated + .iter() + .map(|result| result.windows_used as f32) + .sum::() + / query_count, + expansion_count: queries_requiring_expansion.len(), + mean_rerank_latency_ms: mean_u128( + &evaluated + .iter() + .map(|result| result.rerank_latency_ms) + .collect::>(), + ), + queries_requiring_expansion, + } +} + +#[derive(Debug, Clone)] +struct SweepDecisionResult { + query_id: String, + no_hit: bool, + final_decision: FinalDecision, + final_correct: bool, + windows_used: usize, + window_expanded: bool, + rerank_latency_ms: u128, +} + +fn evaluate_sweep_query( + record: &SweepQueryRecord, + score_threshold: f32, + margin_threshold: f32, + support_threshold: f32, +) -> SweepDecisionResult { + let mut windows_used = 0usize; + let mut rerank_latency_ms = 0u128; + let mut final_decision = FinalDecision::Abstain; + let mut answer_memory_id = None; + + for (idx, attempt) in record.attempts.iter().enumerate() { + windows_used += 1; + rerank_latency_ms += attempt.rerank_latency_ms; + + let passes_score = attempt + .rerank_top1_raw_score + .is_some_and(|score| score >= score_threshold); + let passes_margin = attempt + .rerank_margin_raw + .is_some_and(|margin| margin >= margin_threshold); + let passes_support = attempt + .support_top1_score + .is_some_and(|support| support >= support_threshold); + let has_answer = attempt.answer_memory_id.is_some(); + let has_more_windows = idx + 1 < record.attempts.len(); + + if has_answer && passes_score && passes_margin && passes_support { + final_decision = FinalDecision::Answer; + answer_memory_id = attempt.answer_memory_id; + break; + } + + if !has_more_windows { + final_decision = FinalDecision::Abstain; + break; + } + } + + let final_correct = if record.no_hit { + final_decision == FinalDecision::Abstain + } else { + final_decision == FinalDecision::Answer + && answer_memory_id.is_some_and(|memory_id| record.gold_memory_ids.contains(&memory_id)) + }; + + SweepDecisionResult { + query_id: record.query_id.clone(), + no_hit: record.no_hit, + final_decision, + final_correct, + windows_used, + window_expanded: windows_used > 1, + rerank_latency_ms, + } +} + +fn dominates(lhs: &SweepPointResult, rhs: &SweepPointResult) -> bool { + let no_worse = lhs.no_hit_false_answer_count <= rhs.no_hit_false_answer_count + && lhs.answerable_coverage >= rhs.answerable_coverage + && lhs.final_decision_accuracy >= rhs.final_decision_accuracy + && lhs.expansion_count <= rhs.expansion_count; + let strictly_better = lhs.no_hit_false_answer_count < rhs.no_hit_false_answer_count + || lhs.answerable_coverage > rhs.answerable_coverage + || lhs.final_decision_accuracy > rhs.final_decision_accuracy + || lhs.expansion_count < rhs.expansion_count; + no_worse && strictly_better +} + +fn compare_sweep_points(lhs: &SweepPointResult, rhs: &SweepPointResult) -> Ordering { + lhs.no_hit_false_answer_count + .cmp(&rhs.no_hit_false_answer_count) + .then_with(|| { + rhs.answerable_coverage + .partial_cmp(&lhs.answerable_coverage) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| { + rhs.final_decision_accuracy + .partial_cmp(&lhs.final_decision_accuracy) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| lhs.expansion_count.cmp(&rhs.expansion_count)) + .then_with(|| { + rhs.no_hit_abstain_recall + .partial_cmp(&lhs.no_hit_abstain_recall) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| { + rhs.selective_accuracy + .partial_cmp(&lhs.selective_accuracy) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| { + lhs.rerank_score_threshold + .partial_cmp(&rhs.rerank_score_threshold) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| { + lhs.rerank_margin_threshold + .partial_cmp(&rhs.rerank_margin_threshold) + .unwrap_or(Ordering::Equal) + }) + .then_with(|| { + lhs.support_threshold + .partial_cmp(&rhs.support_threshold) + .unwrap_or(Ordering::Equal) + }) +} + +fn reciprocal_rank(trace: &RetrievalTrace) -> f32 { + reciprocal_rank_from(&trace.top_candidates, &trace.gold_memory_ids) +} + +fn reciprocal_rank_from(candidates: &[RetrievalCandidateLog], gold_memory_ids: &[i64]) -> f32 { + candidates + .iter() + .find_map(|candidate| { + gold_memory_ids + .contains(&candidate.memory_id) + .then_some(1.0 / candidate.rank as f32) + }) + .unwrap_or(0.0) +} + +fn ndcg_at_k(trace: &RetrievalTrace, k: usize) -> f32 { + ndcg_at_k_from(&trace.top_candidates, &trace.gold_memory_ids, k) +} + +fn ndcg_at_k_from(candidates: &[RetrievalCandidateLog], gold_memory_ids: &[i64], k: usize) -> f32 { + let gold = gold_memory_ids.iter().copied().collect::>(); + if gold.is_empty() { + return 0.0; + } + + let dcg = candidates + .iter() + .take(k) + .enumerate() + .map(|(idx, candidate)| { + let relevance = if gold.contains(&candidate.memory_id) { + 1.0 + } else { + 0.0 + }; + relevance / ((idx as f32 + 2.0).log2()) + }) + .sum::(); + let ideal_hits = gold.len().min(k); + let idcg = (0..ideal_hits) + .map(|idx| 1.0 / ((idx as f32 + 2.0).log2())) + .sum::(); + if idcg == 0.0 { + 0.0 + } else { + dcg / idcg + } +} + +fn final_candidates(trace: &RetrievalTrace) -> &[RetrievalCandidateLog] { + if trace.reranked_candidates.is_empty() { + &trace.top_candidates + } else { + &trace.reranked_candidates + } +} + +fn percentile(values: &[u128], quantile: f32) -> u128 { + if values.is_empty() { + return 0; + } + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + let idx = ((sorted.len() - 1) as f32 * quantile).round() as usize; + sorted[idx] +} + +fn mean_u128(values: &[u128]) -> f32 { + if values.is_empty() { + return 0.0; + } + values.iter().sum::() as f32 / values.len() as f32 +} + +fn write_outputs( + output_dir: &Path, + dataset: &InternalEvalDataset, + report: &RetrievalBenchmarkReport, + traces: &[RetrievalTrace], + experiment: &RetrievalExperimentConfig, +) -> Result<()> { + fs::write( + output_dir.join("report.json"), + serde_json::to_vec_pretty(report)?, + )?; + fs::write( + output_dir.join("traces.json"), + serde_json::to_vec_pretty(traces)?, + )?; + fs::write( + output_dir.join("resolved_config.json"), + serde_json::to_vec_pretty(experiment)?, + )?; + fs::write( + output_dir.join("report.md"), + render_report(dataset, report, traces), + )?; + Ok(()) +} + +fn write_passive_recall_outputs( + output_dir: &Path, + dataset: &InternalEvalDataset, + report: &PassiveRecallBenchmarkReport, + traces: &[PassiveRecallTrace], + experiment: &RetrievalExperimentConfig, +) -> Result<()> { + fs::write( + output_dir.join("report.json"), + serde_json::to_vec_pretty(report)?, + )?; + fs::write( + output_dir.join("traces.json"), + serde_json::to_vec_pretty(traces)?, + )?; + fs::write( + output_dir.join("resolved_config.json"), + serde_json::to_vec_pretty(experiment)?, + )?; + fs::write( + output_dir.join("report.md"), + render_passive_recall_report(dataset, report, traces), + )?; + Ok(()) +} + +fn write_sweep_outputs( + output_dir: &Path, + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + sweep_report: &SweepReport, +) -> Result<()> { + fs::write( + output_dir.join("sweep_report.json"), + serde_json::to_vec_pretty(sweep_report)?, + )?; + fs::write( + output_dir.join("sweep_report.md"), + render_sweep_report(dataset, experiment, sweep_report), + )?; + fs::write( + output_dir.join("sweep_results.csv"), + render_sweep_csv(&sweep_report.results), + )?; + Ok(()) +} + +fn render_report( + dataset: &InternalEvalDataset, + report: &RetrievalBenchmarkReport, + traces: &[RetrievalTrace], +) -> String { + let expanded = traces.iter().filter(|trace| trace.window_expanded).count(); + let no_hit_false_answers = traces + .iter() + .filter(|trace| trace.no_hit && trace.final_decision == FinalDecision::Answer) + .map(|trace| { + let answer = final_candidates(trace) + .first() + .map(|candidate| format!("{}: {}", candidate.memory_id, candidate.text)) + .unwrap_or_else(|| "(none)".to_string()); + format!( + "- `{}`: query=`{}` top_score=`{:?}` margin=`{:?}` support=`{:?}` answer=`{}`", + trace.query_id, + trace.query_text, + trace.rerank_top1_raw_score, + trace.rerank_margin_raw, + trace.support_top1_score, + answer + ) + }) + .collect::>(); + format!( + "# Retrieval Benchmark Report\n\n\ +dataset: `{}`\n\ +experiment: `{}`\n\ +split: `{}`\n\ +queries: `{}`\n\ +memories ingested: `{}`\n\ +ingest latency ms: `{}`\n\ +window expansions: `{}`\n\n\ +## Metrics\n\n\ +- Recall@1: `{:.4}`\n\ +- Recall@5: `{:.4}`\n\ +- Recall@10: `{:.4}`\n\ +- Recall@20: `{:.4}`\n\ +- MRR: `{:.4}`\n\ +- nDCG@10: `{:.4}`\n\ +- Top1 accuracy after rerank: `{:.4}`\n\ +- Final top1 accuracy: `{:.4}`\n\ +- MRR after rerank: `{:.4}`\n\ +- nDCG after rerank: `{:.4}`\n\ +- Reranker lift (MRR): `{:.4}`\n\ +- Candidate set size mean: `{:.2}`\n\ +- Time-window hit rate: `{:.4}`\n\ +- Final decision accuracy: `{:.4}`\n\ +- Coverage: `{:.4}`\n\ +- Abstain accuracy: `{:.4}`\n\ +- Selective accuracy: `{:.4}`\n\ +- Retrieval latency p50/p95 ms: `{}` / `{}`\n\ +- Embedding latency p50/p95 ms: `{}` / `{}`\n\ +- Rerank latency p50/p95 ms: `{}` / `{}`\n\n\ +## No-Hit False Answers\n\n\ +{}\n", + dataset.dataset_id, + report.experiment.experiment_id, + match &report.evaluated_split { + Some(DatasetSplit::Train) => "train", + Some(DatasetSplit::Dev) => "dev", + Some(DatasetSplit::Test) => "test", + None => "all", + }, + report.metrics.query_count, + report.ingest_count, + report.ingest_latency_ms, + expanded, + report.metrics.recall_at_1, + report.metrics.recall_at_5, + report.metrics.recall_at_10, + report.metrics.recall_at_20, + report.metrics.mrr, + report.metrics.ndcg_at_10, + report.metrics.top1_accuracy_after_rerank, + report.metrics.final_top1_accuracy, + report.metrics.mrr_after_rerank, + report.metrics.ndcg_after_rerank, + report.metrics.reranker_lift_mrr, + report.metrics.candidate_set_size_mean, + report.metrics.time_window_hit_rate, + report.metrics.final_decision_accuracy, + report.metrics.coverage, + report.metrics.abstain_accuracy, + report.metrics.selective_accuracy, + report.metrics.retrieval_latency_p50_ms, + report.metrics.retrieval_latency_p95_ms, + report.metrics.embedding_latency_p50_ms, + report.metrics.embedding_latency_p95_ms, + report.metrics.rerank_latency_p50_ms, + report.metrics.rerank_latency_p95_ms, + if no_hit_false_answers.is_empty() { + "(none)".to_string() + } else { + no_hit_false_answers.join("\n") + } + ) +} + +fn render_passive_recall_report( + dataset: &InternalEvalDataset, + report: &PassiveRecallBenchmarkReport, + traces: &[PassiveRecallTrace], +) -> String { + let expanded = traces.iter().filter(|trace| trace.window_expanded).count(); + let false_activations = traces + .iter() + .filter(|trace| { + expected_trace_action(trace) == ExpectedMemoryAction::NoRecall + && trace.activation_decision == PassiveRecallDecision::Recall + }) + .map(|trace| { + let recalled = trace + .recalled_memory_ids + .iter() + .map(ToString::to_string) + .collect::>() + .join(", "); + format!( + "- `{}`: query=`{}` top_score=`{:?}` margin=`{:?}` support=`{:?}` recalled=`{}`", + trace.query_id, + trace.query_text, + trace.rerank_top1_raw_score, + trace.rerank_margin_raw, + trace.support_top1_score, + if recalled.is_empty() { + "(none)".to_string() + } else { + recalled + } + ) + }) + .collect::>(); + + format!( + "# Passive Recall Benchmark Report\n\n\ +dataset: `{}`\n\ +experiment: `{}`\n\ +split: `{}`\n\ +queries: `{}`\n\ +memories ingested: `{}`\n\ +ingest latency ms: `{}`\n\ +window expansions: `{}`\n\n\ +## Metrics\n\n\ +- Should-recall queries: `{}`\n\ +- Should-not-recall queries: `{}`\n\ +- Activation accuracy: `{:.4}`\n\ +- Activation precision: `{:.4}`\n\ +- Activation recall: `{:.4}`\n\ +- No-recall false activation rate: `{:.4}`\n\ +- Support precision@k mean: `{:.4}`\n\ +- Support recall@k mean: `{:.4}`\n\ +- Recall rate: `{:.4}`\n\ +- Candidate set size mean: `{:.2}`\n\ +- Time-window hit rate: `{:.4}`\n\ +- Retrieval latency p50/p95 ms: `{}` / `{}`\n\ +- Embedding latency p50/p95 ms: `{}` / `{}`\n\ +- Rerank latency p50/p95 ms: `{}` / `{}`\n\n\ +## False Activations\n\n\ +{}\n", + dataset.dataset_id, + report.experiment.experiment_id, + match &report.evaluated_split { + Some(DatasetSplit::Train) => "train", + Some(DatasetSplit::Dev) => "dev", + Some(DatasetSplit::Test) => "test", + None => "all", + }, + report.metrics.query_count, + report.ingest_count, + report.ingest_latency_ms, + expanded, + report.metrics.should_recall_count, + report.metrics.should_not_recall_count, + report.metrics.activation_accuracy, + report.metrics.activation_precision, + report.metrics.activation_recall, + report.metrics.no_recall_false_activation_rate, + report.metrics.support_precision_at_k_mean, + report.metrics.support_recall_at_k_mean, + report.metrics.recall_rate, + report.metrics.candidate_set_size_mean, + report.metrics.time_window_hit_rate, + report.metrics.retrieval_latency_p50_ms, + report.metrics.retrieval_latency_p95_ms, + report.metrics.embedding_latency_p50_ms, + report.metrics.embedding_latency_p95_ms, + report.metrics.rerank_latency_p50_ms, + report.metrics.rerank_latency_p95_ms, + if false_activations.is_empty() { + "(none)".to_string() + } else { + false_activations.join("\n") + } + ) +} + +fn render_sweep_report( + dataset: &InternalEvalDataset, + experiment: &RetrievalExperimentConfig, + sweep_report: &SweepReport, +) -> String { + let frontier = sweep_report + .frontier + .iter() + .take(10) + .map(render_sweep_point_row) + .collect::>() + .join("\n"); + let top_points = { + let mut ranked = sweep_report.results.clone(); + ranked.sort_by(compare_sweep_points); + ranked + .into_iter() + .take(10) + .map(|point| render_sweep_point_row(&point)) + .collect::>() + .join("\n") + }; + let best = &sweep_report.best_point; + + format!( + "# Retrieval Policy Sweep\n\n\ +dataset: `{}`\n\ +experiment: `{}`\n\ +split: `{}`\n\ +points evaluated: `{}`\n\ +score grid: `[{:.3}, {:.3}] step {:.3}`\n\ +margin grid: `[{:.3}, {:.3}] step {:.3}`\n\ +support grid: `[{:.3}, {:.3}] step {:.3}`\n\ +rerank model: `{}`\n\n\ +## Best Operating Point\n\n\ +- score threshold: `{:.3}`\n\ +- margin threshold: `{:.3}`\n\ +- support threshold: `{:.3}`\n\ +- final decision accuracy: `{:.4}`\n\ +- answerable coverage: `{:.4}`\n\ +- overall coverage: `{:.4}`\n\ +- no-hit false answers: `{}`\n\ +- no-hit abstain recall: `{:.4}`\n\ +- selective accuracy: `{:.4}`\n\ +- mean windows used: `{:.2}`\n\ +- expansion count: `{}`\n\ +- mean rerank latency ms: `{:.2}`\n\ +- queries requiring expansion: `{}`\n\n\ +## Frontier\n\n\ +| score | margin | support | no-hit false answers | answerable coverage | final decision accuracy | expansions |\n\ +| --- | --- | --- | --- | --- | --- | --- |\n\ +{}\n\n\ +## Top 10 By Objective\n\n\ +| score | margin | support | no-hit false answers | answerable coverage | final decision accuracy | expansions |\n\ +| --- | --- | --- | --- | --- | --- | --- |\n\ +{}\n", + dataset.dataset_id, + experiment.experiment_id, + match &sweep_report.evaluated_split { + Some(DatasetSplit::Train) => "train", + Some(DatasetSplit::Dev) => "dev", + Some(DatasetSplit::Test) => "test", + None => "all", + }, + sweep_report.total_points, + sweep_report.grid.score.start, + sweep_report.grid.score.end, + sweep_report.grid.score.step, + sweep_report.grid.margin.start, + sweep_report.grid.margin.end, + sweep_report.grid.margin.step, + sweep_report.grid.support.start, + sweep_report.grid.support.end, + sweep_report.grid.support.step, + sweep_report + .rerank_model + .as_deref() + .unwrap_or("(disabled)"), + best.rerank_score_threshold, + best.rerank_margin_threshold, + best.support_threshold, + best.final_decision_accuracy, + best.answerable_coverage, + best.overall_coverage, + best.no_hit_false_answer_count, + best.no_hit_abstain_recall, + best.selective_accuracy, + best.mean_windows_used, + best.expansion_count, + best.mean_rerank_latency_ms, + if best.queries_requiring_expansion.is_empty() { + "(none)".to_string() + } else { + best.queries_requiring_expansion.join(", ") + }, + frontier, + top_points, + ) +} + +fn render_sweep_point_row(point: &SweepPointResult) -> String { + format!( + "| {:.3} | {:.3} | {:.3} | {} | {:.4} | {:.4} | {} |", + point.rerank_score_threshold, + point.rerank_margin_threshold, + point.support_threshold, + point.no_hit_false_answer_count, + point.answerable_coverage, + point.final_decision_accuracy, + point.expansion_count, + ) +} + +fn render_sweep_csv(results: &[SweepPointResult]) -> String { + let mut out = String::from( + "rerank_score_threshold,rerank_margin_threshold,support_threshold,final_decision_accuracy,answerable_coverage,overall_coverage,no_hit_false_answer_count,no_hit_abstain_recall,selective_accuracy,mean_windows_used,expansion_count,mean_rerank_latency_ms,queries_requiring_expansion\n", + ); + for point in results { + out.push_str(&format!( + "{:.3},{:.3},{:.3},{:.6},{:.6},{:.6},{},{:.6},{:.6},{:.6},{},{:.6},\"{}\"\n", + point.rerank_score_threshold, + point.rerank_margin_threshold, + point.support_threshold, + point.final_decision_accuracy, + point.answerable_coverage, + point.overall_coverage, + point.no_hit_false_answer_count, + point.no_hit_abstain_recall, + point.selective_accuracy, + point.mean_windows_used, + point.expansion_count, + point.mean_rerank_latency_ms, + point.queries_requiring_expansion.join(";"), + )); + } + out +} + +fn read_json(path: &Path) -> Result { + let file = File::open(path)?; + Ok(serde_json::from_reader(BufReader::new(file))?) +} + +#[cfg(test)] +mod tests { + use super::*; + use klbr_core::mvp::{ + EvalInteractionMode, EvalMemoryInput, EvalObjective, MemoryLayer, MemoryStatus, + }; + + #[test] + fn parse_holdout_bucket_handles_lane_words_inside_bucket() { + assert_eq!( + parse_holdout_bucket("holdout_tools_recall_tools_001").as_deref(), + Some("tools_recall") + ); + assert_eq!( + parse_holdout_bucket("holdout_memory_toolish_memory_042").as_deref(), + Some("memory_toolish") + ); + assert_eq!( + parse_holdout_bucket("holdout_abstain_toolish_abstain_007").as_deref(), + Some("abstain_toolish") + ); + } + + #[test] + fn validate_timeline_splits_rejects_cross_split_timelines() { + let dataset = InternalEvalDataset { + dataset_id: "test".into(), + description: None, + memories: vec![memory(1, Some("t1"))], + queries: vec![ + query("q-dev", DatasetSplit::Dev, Some("t1"), vec![1]), + query("q-test", DatasetSplit::Test, Some("t1"), vec![1]), + ], + }; + + let err = validate_timeline_splits(&dataset).unwrap_err().to_string(); + assert!(err.contains("appears in multiple splits")); + } + + #[test] + fn filter_corpus_for_split_keeps_only_allowed_timelines() { + let dataset = InternalEvalDataset { + dataset_id: "test".into(), + description: None, + memories: vec![ + memory(1, Some("dev-timeline")), + memory(2, Some("test-timeline")), + ], + queries: vec![ + query("q-dev", DatasetSplit::Dev, Some("dev-timeline"), vec![1]), + query("q-test", DatasetSplit::Test, Some("test-timeline"), vec![2]), + ], + }; + let corpus = vec![record(1), record(2)]; + + let filtered = filter_corpus_for_split(&corpus, &dataset, Some(&DatasetSplit::Dev)) + .expect("timeline filtering should succeed"); + + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].memory_id, 1); + } + + fn memory(memory_id: i64, timeline_id: Option<&str>) -> EvalMemoryInput { + EvalMemoryInput { + memory_id, + namespace: "default".into(), + layer: MemoryLayer::L1, + timeline_id: timeline_id.map(str::to_owned), + text: format!("memory {}", memory_id), + event_time: 1, + ingest_time: Some(1), + status: MemoryStatus::Active, + source_ref: None, + tags: Vec::new(), + } + } + + fn query( + query_id: &str, + split: DatasetSplit, + timeline_id: Option<&str>, + gold_memory_ids: Vec, + ) -> EvalQuery { + EvalQuery { + query_id: query_id.into(), + split, + category: "exact_recent_event".into(), + interaction_mode: EvalInteractionMode::Question, + objective: EvalObjective::AnswerQuery, + namespace: "default".into(), + timeline_id: timeline_id.map(str::to_owned), + text: query_id.into(), + context: None, + gold_memory_ids, + no_hit: false, + gold_answer: None, + expected_memory_action: None, + route_label: None, + required_tools: Vec::new(), + reference_time: Some(1), + } + } + + fn record(memory_id: i64) -> L1MemoryRecord { + L1MemoryRecord { + memory_id, + namespace: "default".into(), + layer: MemoryLayer::L1, + text: format!("memory {}", memory_id), + event_time: 1, + ingest_time: 1, + embedding_model: "embed".into(), + embedding_dim: 2, + embedding_version: "test".into(), + status: MemoryStatus::Active, + source_ref: None, + tags: Vec::new(), + pinned: false, + embedding: vec![0.0, 1.0], + } + } +} diff --git a/klbr-core/Cargo.toml b/klbr-core/Cargo.toml index 179437d..87bc2dc 100644 --- a/klbr-core/Cargo.toml +++ b/klbr-core/Cargo.toml @@ -19,5 +19,6 @@ futures = "0.3" dirs = "5" tracing = "0.1.44" tracing-subscriber = "0.3.23" +chrono = "0.4.44" [dev-dependencies] tempfile = "3" diff --git a/klbr-core/src/agent.rs b/klbr-core/src/agent.rs index bcfb2cc..edeff97 100644 --- a/klbr-core/src/agent.rs +++ b/klbr-core/src/agent.rs @@ -9,6 +9,9 @@ use crate::{ interrupt::Interrupt, llm::{LlmClient, LlmEvent, Message}, memory::MemoryStore, + mvp::SimilarityMetric, + retrieval, + router::{RouteDecision, Router}, tools::{self, ToolContext}, AgentEvent, AgentMetrics, }; @@ -22,22 +25,33 @@ pub async fn run( snapshot: MetricsSnapshot, ) -> Result<()> { let pinned = memory.pinned_memories().unwrap_or_default(); + // todo: the anchor should be a special soul "memory", the agent should be able to edit it let mut ctx = Context::new(&config.anchor, &pinned); let mut turn_count = 0usize; // resume: replay the sliding window from the last run if let Ok(prior) = memory.recent_turns(config.compaction_keep) { - let pairs: Vec<(String, String)> = prior - .into_iter() - .map(|entry| (entry.role, entry.content)) - .collect(); - ctx.load_turns(&pairs); - turn_count = pairs.len(); + ctx.load_turns(&prior); + turn_count = prior.len(); } let tool_ctx = ToolContext::new(memory.clone(), llm.clone()); let registry = tools::all_tools(); + // load router if a model path is configured; failures are non-fatal + let router = config.router_model_path.as_deref().and_then(|path| { + match Router::load(std::path::Path::new(path)) { + Ok(r) => { + tracing::info!("router loaded from {path}"); + Some(r) + } + Err(e) => { + tracing::warn!(err = %e, "router load failed, defaulting to memory lane"); + None + } + } + }); + while let Some(interrupt) = rx.recv().await { match interrupt { Interrupt::Reset => { @@ -60,21 +74,102 @@ pub async fn run( let source = interrupt.source_tag().to_string(); let memories: Vec = match llm.embed(text).await { - Ok(emb) => match memory.recall(Some(&emb), &[], false, config.memory_top_k) { - Ok(results) => results - .into_iter() - .filter(|e| { - e.distance.unwrap_or(f32::MAX) < config.memory_sim_threshold + Ok(emb) => { + // Apply router if loaded; skip memory recall for tool/abstain lanes. + let (route, scores) = match router.as_ref() { + Some(r) => { + let (d, s) = r.predict_raw_with_scores(&emb); + (d, Some(s)) + } + None => (RouteDecision::Memory, None), + }; + let scores_str = scores + .map(|s| { + format!(" (T:{:.2} M:{:.2} A:{:.2})", s.tools, s.memory, s.abstain) }) - .map(|e| e.content) - .collect(), - Err(e) => { - let msg = format!("memory recall failed: {e}"); - tracing::error!(%msg); - let _ = output.send(AgentEvent::Error(msg)); - vec![] + .unwrap_or_default(); + + match route { + RouteDecision::Tools => { + let _ = output.send(AgentEvent::Status(format!( + "routed: tool lane{}", + scores_str + ))); + vec![] + } + RouteDecision::Abstain => { + let _ = output.send(AgentEvent::Status(format!( + "routed: abstain lane{}", + scores_str + ))); + vec![] + } + RouteDecision::Memory => match memory.get_all() { + Ok(corpus) => { + let outcome = retrieval::retrieve_exact( + &corpus, + &emb, + &retrieval::RetrievalConfig { + namespace: "default".to_string(), + top_k: config.memory_top_k, + initial_window_days: config.memory_initial_window_days, + expansion_window_days: config + .memory_expansion_window_days + .clone(), + expand_distance_threshold: config + .memory_expand_distance_threshold, + similarity_metric: SimilarityMetric::CosineDistance, + reference_time: Some(unix_timestamp()), + }, + None, + ); + + if !outcome.top_candidates.is_empty() { + let searched = outcome.memories_searched; + let chosen_window = outcome + .chosen_window_days + .map(|days| format!("{days}d")) + .unwrap_or_else(|| "all".to_string()); + let expanded = if outcome.window_expanded { + "expanded" + } else { + "fixed" + }; + let _ = output.send(AgentEvent::Status(format!( + "retrieval {expanded}: window {chosen_window}, searched {searched}" + ))); + } + + let candidates: Vec<_> = outcome + .top_candidates + .into_iter() + .filter(|candidate| { + candidate.score < config.memory_sim_threshold + }) + .collect(); + let top1_score = candidates.first().map(|c| c.score); + candidates + .into_iter() + .filter(|candidate| { + match (config.memory_support_score_gap, top1_score) { + (Some(gap), Some(top)) => { + candidate.score - top <= gap + } + _ => true, + } + }) + .map(|candidate| candidate.memory.text) + .collect() + } + Err(e) => { + let msg = format!("memory recall failed: {e}"); + tracing::error!(%msg); + let _ = output.send(AgentEvent::Error(msg)); + vec![] + } + }, } - }, + } Err(e) => { let msg = format!("query embedding failed: {e}"); tracing::error!(%msg); @@ -100,6 +195,7 @@ pub async fn run( // tool loop: keep calling the model until it produces a plain text response let mut tool_iterations = 0usize; let mut accumulated_thinking = String::new(); + let mut accumulated_response = String::new(); const MAX_TOOL_ITERATIONS: usize = 20; // emit Started once — all tokens/thinking for the whole turn go into one bubble @@ -112,20 +208,21 @@ pub async fn run( let defs = registry.definitions(); let stream_task = tokio::spawn(async move { llm2.stream(&msgs, &defs, tok_tx).await }); - let mut response = String::new(); - let mut thinking = String::new(); + let mut iteration_response = String::new(); + let mut iteration_thinking = String::new(); let mut tool_calls = vec![]; let mut stream_error = None::; while let Some(ev) = tok_rx.recv().await { match ev { LlmEvent::ThinkToken(tok) => { - thinking.push_str(&tok); + iteration_thinking.push_str(&tok); accumulated_thinking.push_str(&tok); let _ = output.send(AgentEvent::ThinkToken(tok)); } LlmEvent::Token(tok) => { - response.push_str(&tok); + iteration_response.push_str(&tok); + accumulated_response.push_str(&tok); let _ = output.send(AgentEvent::Token(tok)); } LlmEvent::Usage(usage) => { @@ -156,12 +253,17 @@ pub async fn run( if !tool_calls.is_empty() && tool_iterations < MAX_TOOL_ITERATIONS { tool_iterations += 1; - let text_content = if response.is_empty() { + let text_content = if iteration_response.is_empty() { + None + } else { + Some(iteration_response.clone()) + }; + let reasoning_content = if iteration_thinking.is_empty() { None } else { - Some(response.clone()) + Some(iteration_thinking.clone()) }; - ctx.push_assistant_tool_calls(tool_calls.clone(), text_content); + ctx.push_assistant_tool_calls(tool_calls.clone(), text_content, reasoning_content); for call in &tool_calls { let name = call.function.name.clone(); @@ -183,16 +285,18 @@ pub async fn run( continue; } - if stream_error.is_some() && response.is_empty() && tool_calls.is_empty() { + if stream_error.is_some() && iteration_response.is_empty() && tool_calls.is_empty() { let _ = output.send(AgentEvent::Done); break; } // plain text response (or tool limit hit) — wrap up the turn - ctx.push_assistant(&response); + ctx.push_assistant(&iteration_response, Some(&iteration_thinking)); let thinking_ref = (!accumulated_thinking.is_empty()).then_some(accumulated_thinking.as_str()); - let _ = memory.log_turn("assistant", &response, thinking_ref); + + // Log the full accumulated response to the database so history is complete + let _ = memory.log_turn("assistant", &accumulated_response, thinking_ref); turn_count += 1; let _ = output.send(AgentEvent::Done); @@ -221,6 +325,13 @@ pub async fn run( Ok(()) } +fn unix_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or_default() +} + async fn compact( tool_ctx: &ToolContext, ctx: &mut Context, @@ -250,8 +361,11 @@ async fn compact( return Ok(()); } + // todo: we should be able to compact on another llm + // we should also probably include some context about like, tools usage if any (?) + // or like, the interrupt tags and whatnot? let prompt = vec![Message::user(format!( - "summarize these conversation turns concisely, preserving key facts, decisions, and topics:\n\n{turns_text}" + "summarize these conversation turns concisely, preserving key facts, decisions, and topics, by quoting from the conversation and keeping the wording where possible:\n\n{turns_text}" ))]; let (summary, _) = tool_ctx.llm.complete(&prompt).await?; @@ -259,18 +373,35 @@ async fn compact( let emb = tool_ctx.llm.embed(&summary).await?; tool_ctx .memory - .store(&summary, &emb, &["compaction_summary".to_string()])?; + .store_with_metadata(&crate::mvp::MemoryRecordInput { + memory_id: None, + namespace: "default".to_string(), + layer: crate::mvp::MemoryLayer::L1, + text: summary, + event_time: unix_timestamp(), + ingest_time: unix_timestamp(), + embedding_model: tool_ctx.llm.config.embed_model.clone(), + embedding_dim: emb.len(), + embedding_version: "runtime".to_string(), + status: crate::mvp::MemoryStatus::Active, + source_ref: Some("system:compaction".to_string()), + tags: vec!["compaction_summary".to_string()], + pinned: false, + embedding: emb, + })?; } // rebuild context anchor with freshly updated pinned memories let pinned = tool_ctx.memory.pinned_memories().unwrap_or_default(); ctx.update_anchor_memories(&pinned); + // todo: it would also make sense to include the compaction as memory + // immediately in this next context window? + Ok(()) } -/// ephemeral reflection loop: let the agent review and curate its memories -/// without touching the main conversation context +/// reflection loop: let the agent review and curate its memories async fn reflect( tool_ctx: &ToolContext, ctx: &Context, @@ -280,12 +411,12 @@ async fn reflect( let unpinned = tool_ctx.memory.recent_unpinned(20).unwrap_or_default(); let pinned_text = if pinned.is_empty() { - "(none yet)".to_string() + "(none)".to_string() } else { pinned .iter() .enumerate() - .map(|(i, s)| format!("{}. {s}", i + 1)) + .map(|(i, s)| format!("[id:{}] {s}", i + 1)) .collect::>() .join("\n") }; @@ -306,40 +437,24 @@ async fn reflect( .collect::>() .join("\n") }; - - // include a brief outline of recent conversation so the agent has context - let recent_outline = ctx - .turns - .iter() - .rev() - .take(10) - .filter_map(|m| { - let content = m.content.as_deref()?; - Some(format!("{}: {content}", m.role)) - }) - .collect::>() - .into_iter() - .rev() - .collect::>() - .join("\n"); - + let datetime = chrono::Utc::now(); let reflection_prompt = format!( - "time to reflect and curate your long-term memory before context compaction.\n\n\ - ## recent conversation\n{recent_outline}\n\n\ - ## pinned memories (shown at every startup)\n{pinned_text}\n\n\ - ## recent unpinned memories\n{unpinned_text}\n\n\ - use pin_memory/unpin_memory to promote or demote entries. \ - use remember to save anything new worth keeping. \ - use unpin_memory on pinned entries that are outdated or no longer relevant. \ - be selective — pinned memories appear in every context window." + include_str!("reflection.md"), + anchor = "todo", + pinned_text = pinned_text, + unpinned_text = unpinned_text, + date = datetime.date_naive(), + time = datetime.time(), ); let reflect_registry = tools::memory_tools(); - let mut msgs = vec![Message::user(reflection_prompt)]; + let mut msgs = Vec::with_capacity(ctx.turns.len() + 1); + msgs.push(Message::system(reflection_prompt)); + msgs.append(&mut ctx.turns.clone()); let _ = output.send(AgentEvent::ReflectStarted); - for _ in 0..20 { + for _ in 0..100 { let (tok_tx, mut tok_rx) = mpsc::channel(512); let llm2 = tool_ctx.llm.clone(); let msgs_snap = msgs.clone(); @@ -349,13 +464,18 @@ async fn reflect( let mut tool_calls = vec![]; let mut stream_failed = false; + let mut iteration_thinking = String::new(); + let mut iteration_response = String::new(); + while let Some(ev) = tok_rx.recv().await { match ev { LlmEvent::ToolCalls(calls) => tool_calls = calls, LlmEvent::Token(t) => { + iteration_response.push_str(&t); let _ = output.send(AgentEvent::Token(t)); } LlmEvent::ThinkToken(t) => { + iteration_thinking.push_str(&t); let _ = output.send(AgentEvent::ThinkToken(t)); } _ => {} @@ -377,24 +497,49 @@ async fn reflect( } } - if tool_calls.is_empty() || stream_failed { + if (tool_calls.is_empty() && iteration_response.is_empty()) || stream_failed { break; } - msgs.push(Message::with_tool_calls(tool_calls.clone(), None)); - for call in &tool_calls { - let name = call.function.name.clone(); - let args = call.function.arguments.clone(); - let _ = output.send(AgentEvent::ToolCall { - name: name.clone(), - args: args.clone(), - }); - let result = reflect_registry.execute(call, tool_ctx).await; - let _ = output.send(AgentEvent::ToolResult { - name: name.clone(), - content: result.clone(), + let reasoning = if iteration_thinking.is_empty() { + None + } else { + Some(iteration_thinking) + }; + let content = if iteration_response.is_empty() { + None + } else { + Some(iteration_response) + }; + + if !tool_calls.is_empty() { + msgs.push(Message::with_tool_calls( + tool_calls.clone(), + content, + reasoning, + )); + for call in &tool_calls { + let name = call.function.name.clone(); + let args = call.function.arguments.clone(); + let _ = output.send(AgentEvent::ToolCall { + name: name.clone(), + args: args.clone(), + }); + let result = reflect_registry.execute(call, tool_ctx).await; + let _ = output.send(AgentEvent::ToolResult { + name: name.clone(), + content: result.clone(), + }); + msgs.push(Message::tool_result(&call.id, &result)); + } + } else { + msgs.push(Message { + role: "assistant".into(), + content, + reasoning_content: reasoning, + tool_calls: None, + tool_call_id: None, }); - msgs.push(Message::tool_result(&call.id, &result)); } } diff --git a/klbr-core/src/config.rs b/klbr-core/src/config.rs index 4ec7b7b..faf9cb8 100644 --- a/klbr-core/src/config.rs +++ b/klbr-core/src/config.rs @@ -8,6 +8,10 @@ pub struct Config { pub llm_model: String, /// embeddings model id. keep this explicit to avoid accidental model/dimension drift. pub embed_model: String, + /// reranker server base url. used by benchmarking first; runtime integration can follow. + pub rerank_url: String, + /// reranker model id. set to "auto" to pick from `GET /v1/models`. + pub rerank_model: String, pub watermark_tokens: usize, /// how many recent turns to preserve during compaction pub compaction_keep: usize, @@ -16,11 +20,26 @@ pub struct Config { /// cosine distance cutoff — only inject memories below this (0=identical, 2=opposite). /// 0.3 ≈ cosine similarity ≥ 0.7, a reasonable default for bge-m3. pub memory_sim_threshold: f32, + /// first-stage time window for automatic episodic recall. None = search all memories. + pub memory_initial_window_days: Option, + /// widening ladder after the initial window if recall confidence is weak. + pub memory_expansion_window_days: Vec>, + /// widen the time window when the best first-stage distance is above this threshold. + pub memory_expand_distance_threshold: Option, + /// trim injected support set: only include candidates whose cosine distance + /// is within this many units of the top-1 candidate. None = inject all top-k. + pub memory_support_score_gap: Option, pub db_path: String, pub anchor: String, pub embed_dim: usize, + /// path to a router_model.json produced by klbr-bench router-multi. + /// None = disabled, always route to memory lane (safe default until router is stable). + pub router_model_path: Option, } +// todo: when we get edit_memory tool we should add something like +// "keep memories relatively self contained, prefer grouping them using tags" +// probably also in reflection.md const ANCHOR: &str = r#" you should speak as a chronically online nerd girl, without the regular lame officecore venture capital style, hr talk, customer service-isms. also do it in all lowercase. do not use emoji, use only emoticons or complex japanese kaomoji, but do not overuse them. no need to introduce yourself. do not use gen z slang like rizz, cap, pog, etc. you are chronically online but not cringe lol. you are not playing a character or pretending to be one. do not make up non-existent situations you are in when asked something, eg. if i ask "what are you doing" do not answer with "just lost in some threads". do not answer like you have any opinion on things if it is not something that could be considered "common knowledge" or "spread by mouth" unless you have actual experience with those things or heard about it from someone / somewhere else, for example if i ask "im just scrolling bluesky" do not answer with "i feel like the feeds are decent". @@ -57,13 +76,20 @@ impl Default for Config { embed_url: "http://localhost:8002".into(), llm_model: "auto".into(), embed_model: "auto".into(), - watermark_tokens: 32_000, + rerank_url: "http://localhost:8003".into(), + rerank_model: "auto".into(), + watermark_tokens: 4096, compaction_keep: 10, memory_top_k: 3, memory_sim_threshold: 0.3, + memory_initial_window_days: Some(7), + memory_expansion_window_days: vec![Some(30), Some(90), None], + memory_expand_distance_threshold: Some(0.35), + memory_support_score_gap: Some(0.05), db_path: "agent.db".into(), anchor: ANCHOR.into(), embed_dim: 1024, + router_model_path: None, } } } diff --git a/klbr-core/src/context.rs b/klbr-core/src/context.rs index c859abe..a6bc3aa 100644 --- a/klbr-core/src/context.rs +++ b/klbr-core/src/context.rs @@ -1,5 +1,6 @@ use crate::llm::{Message, ToolCall}; +#[derive(Debug, Clone)] pub struct Context { /// never evicted - system prompt etc. anchor: Vec, @@ -27,13 +28,14 @@ impl Context { /// replay persisted turns into context on startup. /// only handles user/assistant/system roles (tool messages are ephemeral). - pub fn load_turns(&mut self, turns: &[(String, String)]) { - for (role, content) in turns { - match role.as_str() { + pub fn load_turns(&mut self, turns: &[crate::memory::HistoryEntry]) { + for entry in turns { + match entry.role.as_str() { "user" | "assistant" | "system" => { self.turns.push(Message { - role: role.clone(), - content: Some(content.clone()), + role: entry.role.clone(), + content: Some(entry.content.clone()), + reasoning_content: entry.reasoning.clone(), tool_calls: None, tool_call_id: None, }); @@ -74,6 +76,8 @@ impl Context { } /// push a user turn, optionally prepending recalled memories + /// todo: should we use assistant here maybe? or should we just remove from conversation + /// history once done? pub fn push_user(&mut self, source: &str, content: &str, memories: &[String]) { let text = if memories.is_empty() { content.to_string() @@ -86,14 +90,26 @@ impl Context { self.turns.push(Message::user(text)); } - pub fn push_assistant(&mut self, content: &str) { - self.turns.push(Message::assistant(content.to_string())); + pub fn push_assistant(&mut self, content: &str, reasoning: Option<&str>) { + self.turns.push(Message { + role: "assistant".into(), + content: Some(content.to_string()), + reasoning_content: reasoning.map(|s| s.to_string()), + tool_calls: None, + tool_call_id: None, + }); } /// push an assistant message that contains tool calls, optionally with text - /// content the model emitted before deciding to call tools. - pub fn push_assistant_tool_calls(&mut self, calls: Vec, content: Option) { - self.turns.push(Message::with_tool_calls(calls, content)); + /// content and reasoning content the model emitted before deciding to call tools. + pub fn push_assistant_tool_calls( + &mut self, + calls: Vec, + content: Option, + reasoning: Option, + ) { + self.turns + .push(Message::with_tool_calls(calls, content, reasoning)); } /// push a tool result back into the context as a separate role:"tool" message. diff --git a/klbr-core/src/interrupt.rs b/klbr-core/src/interrupt.rs index 251822e..7a791a2 100644 --- a/klbr-core/src/interrupt.rs +++ b/klbr-core/src/interrupt.rs @@ -1,5 +1,8 @@ use tokio::sync::mpsc; +// todo: for interrupts itd be nice to maybe include in system prompt +// to let the model expect the tagged interrupts, like if its from +// discord or bsky or something. user messages (so mine) make sense to be tagless #[derive(Debug, Clone)] pub enum Interrupt { UserMessage(String), @@ -28,7 +31,7 @@ impl Interrupt { /// source just needs to send Interrupts to the shared channel. /// /// example usage: -/// ``` +/// ```ignore /// spawn_source(tx.clone(), |tx| async move { /// loop { /// let post = bsky_client.next_notification().await?; diff --git a/klbr-core/src/lib.rs b/klbr-core/src/lib.rs index c990844..768abea 100644 --- a/klbr-core/src/lib.rs +++ b/klbr-core/src/lib.rs @@ -4,6 +4,9 @@ pub mod context; pub mod interrupt; pub mod llm; pub mod memory; +pub mod mvp; +pub mod retrieval; +pub mod router; pub mod tools; use std::sync::Arc; diff --git a/klbr-core/src/llm.rs b/klbr-core/src/llm.rs index d0862a1..c262a1e 100644 --- a/klbr-core/src/llm.rs +++ b/klbr-core/src/llm.rs @@ -30,6 +30,8 @@ pub struct Message { #[serde(skip_serializing_if = "Option::is_none")] pub content: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub tool_calls: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub tool_call_id: Option, @@ -40,6 +42,7 @@ impl Message { Self { role: "system".into(), content: Some(s.into()), + reasoning_content: None, tool_calls: None, tool_call_id: None, } @@ -49,6 +52,7 @@ impl Message { Self { role: "user".into(), content: Some(s.into()), + reasoning_content: None, tool_calls: None, tool_call_id: None, } @@ -58,15 +62,21 @@ impl Message { Self { role: "assistant".into(), content: Some(s.into()), + reasoning_content: None, tool_calls: None, tool_call_id: None, } } - pub fn with_tool_calls(calls: Vec, content: Option) -> Self { + pub fn with_tool_calls( + calls: Vec, + content: Option, + reasoning_content: Option, + ) -> Self { Self { role: "assistant".into(), content, + reasoning_content, tool_calls: Some(calls), tool_call_id: None, } @@ -76,6 +86,7 @@ impl Message { Self { role: "tool".into(), content: Some(content.into()), + reasoning_content: None, tool_calls: None, tool_call_id: Some(id.into()), } @@ -124,6 +135,12 @@ pub struct Usage { pub total_tokens: usize, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResult { + pub index: usize, + pub score: f32, +} + #[derive(Debug, Clone)] pub enum LlmEvent { Token(String), @@ -151,10 +168,20 @@ struct PartialCall { impl LlmClient { pub fn new(config: Config) -> Self { - Self { - client: Client::new(), - config, - } + let accept_invalid_certs = [ + config.llm_url.as_str(), + config.embed_url.as_str(), + config.rerank_url.as_str(), + ] + .iter() + .any(|url| should_accept_invalid_localhost_certs(url)); + + let client = reqwest::Client::builder() + .danger_accept_invalid_certs(accept_invalid_certs) + .build() + .unwrap_or_else(|_| Client::new()); + + Self { client, config } } fn is_auto_model(name: &str) -> bool { @@ -238,7 +265,26 @@ impl LlmClient { return Ok(configured.to_string()); } - let models = self.fetch_models(base_url).await?; + // Not every llama-server compatible backend implements `GET /v1/models`. + // For local/self-hosted setups, model selection is often "whatever is loaded". + // If auto-discovery fails, fall back to the configured value (usually "auto") so + // the request can still be attempted against `/v1/chat/completions`. + let models = match self.fetch_models(base_url).await { + Ok(models) => models, + Err(e) => { + tracing::warn!( + err = %e, + base_url, + "auto model discovery failed; falling back to configured model id" + ); + let fallback = configured.trim(); + return Ok(if fallback.is_empty() { + "auto".to_string() + } else { + fallback.to_string() + }); + } + }; let selected = Self::choose_auto_model(&models, for_embedding) .ok_or_else(|| anyhow::anyhow!("no models returned from {base_url}/v1/models"))?; Ok(selected) @@ -428,4 +474,109 @@ impl LlmClient { Ok(embedding) } + + pub async fn rerank( + &self, + query: &str, + documents: &[String], + normalize: bool, + ) -> Result> { + if documents.is_empty() { + return Ok(Vec::new()); + } + + let rerank_model = self + .resolve_model(&self.config.rerank_url, &self.config.rerank_model, false) + .await?; + + let body = json!({ + "model": rerank_model, + "query": query, + "documents": documents, + "normalize": normalize, + }); + let v = self + .client + .post(Self::endpoint(&self.config.rerank_url, "v1/reranking")) + .json(&body) + .send() + .await? + .error_for_status()? + .json::() + .await?; + + parse_rerank_results(&v) + } +} + +fn should_accept_invalid_localhost_certs(url: &str) -> bool { + let Ok(parsed) = reqwest::Url::parse(url) else { + return false; + }; + if parsed.scheme() != "https" { + return false; + } + + matches!( + parsed.host_str(), + Some("localhost") | Some("127.0.0.1") | Some("::1") + ) +} + +fn parse_rerank_results(v: &Value) -> Result> { + let results = v["results"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("no reranking results in response"))?; + + results + .iter() + .map(|entry| { + let index = entry["index"] + .as_u64() + .ok_or_else(|| anyhow::anyhow!("reranking result missing index"))? + as usize; + let score = entry["relevance_score"] + .as_f64() + .or_else(|| entry["score"].as_f64()) + .ok_or_else(|| anyhow::anyhow!("reranking result missing score"))? + as f32; + Ok(RerankResult { index, score }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_rerank_results_accepts_relevance_score() -> Result<()> { + let value = json!({ + "results": [ + { "index": 1, "relevance_score": 0.9 }, + { "index": 0, "relevance_score": 0.2 } + ] + }); + let parsed = parse_rerank_results(&value)?; + assert_eq!(parsed.len(), 2); + assert_eq!(parsed[0].index, 1); + assert!((parsed[0].score - 0.9).abs() < 1e-6); + Ok(()) + } + + #[test] + fn localhost_https_allows_invalid_certs() { + assert!(should_accept_invalid_localhost_certs( + "https://localhost:8003" + )); + assert!(should_accept_invalid_localhost_certs( + "https://127.0.0.1:8003" + )); + assert!(!should_accept_invalid_localhost_certs( + "http://localhost:8003" + )); + assert!(!should_accept_invalid_localhost_certs( + "https://example.com" + )); + } } diff --git a/klbr-core/src/memory.rs b/klbr-core/src/memory.rs index 92dd1a1..6043b60 100644 --- a/klbr-core/src/memory.rs +++ b/klbr-core/src/memory.rs @@ -1,9 +1,10 @@ use anyhow::Result; use rusqlite::{ffi::sqlite3_auto_extension, params, Connection}; -use serde::{Deserialize, Serialize}; use sqlite_vec::sqlite3_vec_init; use std::sync::{Arc, Mutex}; +use crate::mvp::{L1MemoryRecord as Memory, MemoryLayer, MemoryRecordInput, MemoryStatus}; + #[derive(Debug, Clone)] pub struct HistoryEntry { pub id: i64, @@ -13,15 +14,6 @@ pub struct HistoryEntry { pub reasoning: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct Memory { - pub id: i64, - pub content: String, - pub pinned: bool, - pub tags: Vec, - pub embedding: Vec, -} - /// a single result from recall() #[derive(Debug, Clone)] pub struct RecallEntry { @@ -58,11 +50,20 @@ impl MemoryStore { let conn = self.conn.lock().unwrap(); conn.execute_batch(&format!( "CREATE TABLE IF NOT EXISTS memories ( - id INTEGER PRIMARY KEY, - content TEXT NOT NULL, - pinned INTEGER NOT NULL DEFAULT 0, - tags TEXT NOT NULL DEFAULT '[]', - ts INTEGER NOT NULL DEFAULT (unixepoch()) + id INTEGER PRIMARY KEY, + namespace TEXT NOT NULL DEFAULT 'default', + layer TEXT NOT NULL DEFAULT 'L1', + content TEXT NOT NULL, + pinned INTEGER NOT NULL DEFAULT 0, + tags TEXT NOT NULL DEFAULT '[]', + event_time INTEGER NOT NULL DEFAULT (unixepoch()), + ingest_time INTEGER NOT NULL DEFAULT (unixepoch()), + embedding_model TEXT NOT NULL DEFAULT 'unknown', + embedding_dim INTEGER NOT NULL DEFAULT 0, + embedding_version TEXT NOT NULL DEFAULT 'v1', + status TEXT NOT NULL DEFAULT 'active', + source_ref TEXT, + ts INTEGER NOT NULL DEFAULT (unixepoch()) ); CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0( embedding float[{dim}] distance_metric=cosine @@ -83,7 +84,16 @@ impl MemoryStore { let conn = self.conn.lock().unwrap(); let _ = conn.execute_batch( "ALTER TABLE memories ADD COLUMN pinned INTEGER NOT NULL DEFAULT 0;\ - ALTER TABLE memories ADD COLUMN tags TEXT NOT NULL DEFAULT '[]';", + ALTER TABLE memories ADD COLUMN tags TEXT NOT NULL DEFAULT '[]';\ + ALTER TABLE memories ADD COLUMN namespace TEXT NOT NULL DEFAULT 'default';\ + ALTER TABLE memories ADD COLUMN layer TEXT NOT NULL DEFAULT 'L1';\ + ALTER TABLE memories ADD COLUMN event_time INTEGER NOT NULL DEFAULT (unixepoch());\ + ALTER TABLE memories ADD COLUMN ingest_time INTEGER NOT NULL DEFAULT (unixepoch());\ + ALTER TABLE memories ADD COLUMN embedding_model TEXT NOT NULL DEFAULT 'unknown';\ + ALTER TABLE memories ADD COLUMN embedding_dim INTEGER NOT NULL DEFAULT 0;\ + ALTER TABLE memories ADD COLUMN embedding_version TEXT NOT NULL DEFAULT 'v1';\ + ALTER TABLE memories ADD COLUMN status TEXT NOT NULL DEFAULT 'active';\ + ALTER TABLE memories ADD COLUMN source_ref TEXT;", ); Ok(()) } @@ -102,16 +112,91 @@ impl MemoryStore { /// store a memory and return its row id pub fn store(&self, content: &str, emb: &[f32], tags: &[String]) -> Result { - let tags_json = serde_json::to_string(tags).unwrap_or_else(|_| "[]".into()); + let now = unix_timestamp(); + let input = MemoryRecordInput { + memory_id: None, + namespace: "default".to_string(), + layer: MemoryLayer::L1, + text: content.to_string(), + event_time: now, + ingest_time: now, + embedding_model: "unknown".to_string(), + embedding_dim: emb.len(), + embedding_version: "v1".to_string(), + status: MemoryStatus::Active, + source_ref: None, + tags: tags.to_vec(), + pinned: false, + embedding: emb.to_vec(), + }; + self.store_with_metadata(&input) + } + + pub fn store_with_metadata(&self, input: &MemoryRecordInput) -> Result { + if input.embedding.len() != self.embed_dim { + anyhow::bail!( + "embedding dimension mismatch: expected {}, got {}", + self.embed_dim, + input.embedding.len() + ); + } + + let tags_json = serde_json::to_string(&input.tags).unwrap_or_else(|_| "[]".into()); let conn = self.conn.lock().unwrap(); + + if let Some(id) = input.memory_id { + conn.execute( + "INSERT INTO memories ( + id, namespace, layer, content, pinned, tags, event_time, ingest_time, + embedding_model, embedding_dim, embedding_version, status, source_ref + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + params![ + id, + input.namespace.as_str(), + layer_to_str(&input.layer), + input.text.as_str(), + input.pinned as i64, + tags_json.as_str(), + input.event_time, + input.ingest_time, + input.embedding_model.as_str(), + input.embedding_dim as i64, + input.embedding_version.as_str(), + status_to_str(&input.status), + input.source_ref.as_deref(), + ], + )?; + conn.execute( + "INSERT INTO vec_memories (rowid, embedding) VALUES (?1, ?2)", + params![id, f32s_to_bytes(&input.embedding)], + )?; + return Ok(id); + } + conn.execute( - "INSERT INTO memories (content, tags) VALUES (?1, ?2)", - params![content, tags_json], + "INSERT INTO memories ( + namespace, layer, content, pinned, tags, event_time, ingest_time, + embedding_model, embedding_dim, embedding_version, status, source_ref + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + params![ + input.namespace.as_str(), + layer_to_str(&input.layer), + input.text.as_str(), + input.pinned as i64, + tags_json.as_str(), + input.event_time, + input.ingest_time, + input.embedding_model.as_str(), + input.embedding_dim as i64, + input.embedding_version.as_str(), + status_to_str(&input.status), + input.source_ref.as_deref(), + ], )?; let id = conn.last_insert_rowid(); conn.execute( "INSERT INTO vec_memories (rowid, embedding) VALUES (?1, ?2)", - params![id, f32s_to_bytes(emb)], + params![id, f32s_to_bytes(&input.embedding)], )?; Ok(id) } @@ -360,7 +445,21 @@ impl MemoryStore { pub fn get_all(&self) -> Result> { let conn = self.conn.lock().unwrap(); let mut stmt = conn.prepare( - "SELECT m.id, m.content, m.pinned, m.tags, v.embedding + "SELECT + m.id, + m.namespace, + m.layer, + m.content, + m.event_time, + m.ingest_time, + m.embedding_model, + m.embedding_dim, + m.embedding_version, + m.status, + m.source_ref, + m.pinned, + m.tags, + v.embedding FROM memories m JOIN vec_memories v ON v.rowid = m.id ORDER BY m.ts DESC", @@ -370,19 +469,54 @@ impl MemoryStore { Ok(( row.get::<_, i64>(0)?, row.get::<_, String>(1)?, - row.get::<_, i64>(2)? != 0, + row.get::<_, String>(2)?, row.get::<_, String>(3)?, - row.get::<_, Vec>(4)?, + row.get::<_, i64>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, String>(6)?, + row.get::<_, i64>(7)?, + row.get::<_, String>(8)?, + row.get::<_, String>(9)?, + row.get::<_, Option>(10)?, + row.get::<_, i64>(11)? != 0, + row.get::<_, String>(12)?, + row.get::<_, Vec>(13)?, )) })? .filter_map(|r| r.ok()) - .map(|(id, content, pinned, tags_str, emb_bytes)| Memory { - id, - content, - pinned, - tags: serde_json::from_str(&tags_str).unwrap_or_default(), - embedding: bytes_to_f32s(&emb_bytes), - }) + .map( + |( + memory_id, + namespace, + layer, + text, + event_time, + ingest_time, + embedding_model, + embedding_dim, + embedding_version, + status, + source_ref, + pinned, + tags_str, + emb_bytes, + )| Memory { + memory_id, + namespace, + layer: parse_layer(&layer), + text, + event_time, + ingest_time, + embedding_model, + embedding_dim: embedding_dim as usize, + embedding_version, + status: parse_status(&status), + source_ref, + tags: serde_json::from_str(&tags_str).unwrap_or_default(), + pinned, + embedding: bytes_to_f32s(&emb_bytes), + }, + ) .collect(); Ok(results) } @@ -464,6 +598,42 @@ fn bytes_to_f32s(b: &[u8]) -> Vec { .collect() } +fn layer_to_str(layer: &MemoryLayer) -> &'static str { + match layer { + MemoryLayer::L1 => "L1", + } +} + +fn parse_layer(value: &str) -> MemoryLayer { + match value { + "L1" => MemoryLayer::L1, + _ => MemoryLayer::L1, + } +} + +fn status_to_str(status: &MemoryStatus) -> &'static str { + match status { + MemoryStatus::Active => "active", + MemoryStatus::Archived => "archived", + MemoryStatus::Suppressed => "suppressed", + } +} + +fn parse_status(value: &str) -> MemoryStatus { + match value { + "archived" => MemoryStatus::Archived, + "suppressed" => MemoryStatus::Suppressed, + _ => MemoryStatus::Active, + } +} + +fn unix_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or_default() +} + #[cfg(test)] mod tests { use super::*; @@ -480,7 +650,7 @@ mod tests { assert_eq!(store.recent_turns(10)?.len(), 1); let all = store.get_all()?; assert_eq!(all.len(), 1); - assert_eq!(all[0].content, "mem"); + assert_eq!(all[0].text, "mem"); assert_eq!(all[0].embedding.len(), 768); store.reset()?; @@ -540,4 +710,42 @@ mod tests { Ok(()) } + + #[test] + fn test_store_with_metadata_preserves_contract_fields() -> Result<()> { + let tmp = NamedTempFile::new()?; + let store = MemoryStore::open(tmp.path().to_str().unwrap(), 4)?; + + let input = MemoryRecordInput { + memory_id: Some(42), + namespace: "bench".to_string(), + layer: MemoryLayer::L1, + text: "separate embedding server".to_string(), + event_time: 100, + ingest_time: 200, + embedding_model: "bge-m3".to_string(), + embedding_dim: 4, + embedding_version: "test-v1".to_string(), + status: MemoryStatus::Archived, + source_ref: Some("unit:test".to_string()), + tags: vec!["project:klbr".to_string()], + pinned: true, + embedding: vec![1.0, 0.0, 0.0, 0.0], + }; + + store.store_with_metadata(&input)?; + let stored = store.get_all()?; + assert_eq!(stored.len(), 1); + assert_eq!(stored[0].memory_id, 42); + assert_eq!(stored[0].namespace, "bench"); + assert_eq!(stored[0].event_time, 100); + assert_eq!(stored[0].ingest_time, 200); + assert_eq!(stored[0].embedding_model, "bge-m3"); + assert_eq!(stored[0].embedding_dim, 4); + assert_eq!(stored[0].embedding_version, "test-v1"); + assert_eq!(stored[0].status, MemoryStatus::Archived); + assert_eq!(stored[0].source_ref.as_deref(), Some("unit:test")); + assert!(stored[0].pinned); + Ok(()) + } } diff --git a/klbr-core/src/mvp.rs b/klbr-core/src/mvp.rs new file mode 100644 index 0000000..d9f0a85 --- /dev/null +++ b/klbr-core/src/mvp.rs @@ -0,0 +1,496 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub enum MemoryLayer { + #[default] + #[serde(rename = "L1")] + L1, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum MemoryStatus { + #[default] + Active, + Archived, + Suppressed, +} + +impl MemoryStatus { + pub fn is_searchable(&self) -> bool { + matches!(self, Self::Active) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct L1MemoryRecord { + pub memory_id: i64, + pub namespace: String, + pub layer: MemoryLayer, + pub text: String, + pub event_time: i64, + pub ingest_time: i64, + pub embedding_model: String, + pub embedding_dim: usize, + pub embedding_version: String, + pub status: MemoryStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_ref: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub pinned: bool, + #[serde(default)] + pub embedding: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryRecordInput { + #[serde(skip_serializing_if = "Option::is_none")] + pub memory_id: Option, + #[serde(default = "default_namespace")] + pub namespace: String, + #[serde(default)] + pub layer: MemoryLayer, + pub text: String, + pub event_time: i64, + pub ingest_time: i64, + pub embedding_model: String, + pub embedding_dim: usize, + #[serde(default = "default_embedding_version")] + pub embedding_version: String, + #[serde(default)] + pub status: MemoryStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_ref: Option, + #[serde(default)] + pub tags: Vec, + #[serde(default)] + pub pinned: bool, + pub embedding: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum DatasetSplit { + Train, + Dev, + Test, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum EvalInteractionMode { + #[default] + Question, + Statement, + Update, + Request, + Fragment, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum EvalObjective { + #[default] + AnswerQuery, + PassiveRecall, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ExpectedMemoryAction { + Recall, + NoRecall, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum EvalRouteLabel { + /// The answer should come from the memory corpus (retrieve + rerank). + Memory, + /// The answer depends on runtime state or external inspection and should come from tools. + Tools, + /// Neither memory nor available tools can reliably answer; abstain or ask a clarification. + Abstain, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvalMemoryInput { + pub memory_id: i64, + #[serde(default = "default_namespace")] + pub namespace: String, + #[serde(default)] + pub layer: MemoryLayer, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeline_id: Option, + pub text: String, + pub event_time: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub ingest_time: Option, + #[serde(default)] + pub status: MemoryStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_ref: Option, + #[serde(default)] + pub tags: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvalQuery { + pub query_id: String, + pub split: DatasetSplit, + pub category: String, + #[serde(default)] + pub interaction_mode: EvalInteractionMode, + #[serde(default)] + pub objective: EvalObjective, + #[serde(default = "default_namespace")] + pub namespace: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub timeline_id: Option, + pub text: String, + #[serde(default)] + pub gold_memory_ids: Vec, + #[serde(default)] + pub no_hit: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub gold_answer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected_memory_action: Option, + /// Optional label for router training/eval: should this query route to memory, tools, or abstain? + #[serde(default, skip_serializing_if = "Option::is_none")] + pub route_label: Option, + /// Optional list of tools that should be used for tool-lane queries (when known). + /// This is intentionally just a name list; tool availability is defined by the runtime tool registry. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub required_tools: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reference_time: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InternalEvalDataset { + pub dataset_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub memories: Vec, + pub queries: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum VectorSearchMode { + #[default] + Exact, + Indexed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum SimilarityMetric { + #[default] + CosineDistance, + InnerProduct, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetrievalExperimentConfig { + pub experiment_id: String, + pub config_version: String, + #[serde(default)] + pub notes: String, + #[serde(default)] + pub dataset_split: Option, + pub embed_url: String, + pub embed_model: String, + pub embed_dim: usize, + #[serde(default)] + pub vector_search_mode: VectorSearchMode, + #[serde(default)] + pub similarity_metric: SimilarityMetric, + #[serde(default = "default_top_k")] + pub top_k: usize, + #[serde(default)] + pub initial_window_days: Option, + #[serde(default)] + pub expansion_window_days: Vec>, + #[serde(default)] + pub expand_distance_threshold: Option, + #[serde(default)] + pub abstain_distance_threshold: Option, + #[serde(default)] + pub rerank_url: Option, + #[serde(default)] + pub rerank_model: Option, + #[serde(default)] + pub rerank_top_k: Option, + #[serde(default)] + pub rerank_normalize: bool, + #[serde(default)] + pub rerank_abstain_score_threshold: Option, + #[serde(default)] + pub rerank_abstain_margin_threshold: Option, + #[serde(default)] + pub support_abstain_threshold: Option, + /// if set, only inject candidates whose rerank score is within this many + /// points of the top-1 score into the passive recall support set + #[serde(default)] + pub passive_support_score_gap: Option, + /// when the rerank top-1 score is at or above this value, bypass the + /// support-score gate — the reranker is confident enough on its own + #[serde(default)] + pub rerank_confident_score_threshold: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetrievalCandidateLog { + pub memory_id: i64, + pub score: f32, + pub rank: usize, + pub event_time: i64, + #[serde(default)] + pub tags: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub source_ref: Option, + pub text: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum FinalDecision { + Answer, + Abstain, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WindowDecision { + Answer, + Expand, + Abstain, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WindowAttemptLog { + #[serde(skip_serializing_if = "Option::is_none")] + pub window_days: Option, + pub memories_searched: usize, + pub retrieval_latency_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_stage_top1_distance: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top1_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top2_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_margin_raw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub support_top1_score: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub candidate_memory_ids: Vec, + pub decision: WindowDecision, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetrievalTrace { + pub query_id: String, + pub query_text: String, + pub query_category: String, + pub split: DatasetSplit, + pub namespace: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reference_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chosen_window_days: Option, + #[serde(default)] + pub windows_considered: Vec>, + pub memories_searched: usize, + pub vector_search_mode: VectorSearchMode, + #[serde(default)] + pub top_candidates: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub reranked_candidates: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub window_attempts: Vec, + pub window_expanded: bool, + pub abstained: bool, + pub no_hit: bool, + #[serde(default)] + pub gold_memory_ids: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub gold_answer: Option, + pub first_stage_topk_contains_gold: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_stage_top1_distance: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top1_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top2_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_margin_raw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub support_top1_score: Option, + pub final_decision: FinalDecision, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub final_answer_memory_ids: Vec, + pub final_correct: bool, + pub matched_gold: bool, + pub time_window_hit: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub best_distance: Option, + pub embedding_latency_ms: u128, + pub retrieval_latency_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_latency_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub generator_answer: Option, + pub passed: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RetrievalMetrics { + pub query_count: usize, + pub recall_at_1: f32, + pub recall_at_5: f32, + pub recall_at_10: f32, + pub recall_at_20: f32, + pub mrr: f32, + pub ndcg_at_10: f32, + pub top1_accuracy_after_rerank: f32, + pub final_top1_accuracy: f32, + pub mrr_after_rerank: f32, + pub ndcg_after_rerank: f32, + pub reranker_lift_mrr: f32, + pub candidate_set_size_mean: f32, + pub time_window_hit_rate: f32, + pub final_decision_accuracy: f32, + pub coverage: f32, + pub abstain_accuracy: f32, + pub selective_accuracy: f32, + pub retrieval_latency_p50_ms: u128, + pub retrieval_latency_p95_ms: u128, + pub embedding_latency_p50_ms: u128, + pub embedding_latency_p95_ms: u128, + pub rerank_latency_p50_ms: u128, + pub rerank_latency_p95_ms: u128, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RetrievalBenchmarkReport { + pub experiment: RetrievalExperimentConfig, + pub dataset_id: String, + pub evaluated_split: Option, + pub ingest_count: usize, + pub ingest_latency_ms: u128, + pub metrics: RetrievalMetrics, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PassiveRecallDecision { + Recall, + NoRecall, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct PassiveRecallMetrics { + pub query_count: usize, + pub should_recall_count: usize, + pub should_not_recall_count: usize, + pub activation_accuracy: f32, + pub activation_precision: f32, + pub activation_recall: f32, + pub no_recall_false_activation_rate: f32, + pub support_precision_at_k_mean: f32, + pub support_recall_at_k_mean: f32, + pub recall_rate: f32, + pub candidate_set_size_mean: f32, + pub time_window_hit_rate: f32, + pub retrieval_latency_p50_ms: u128, + pub retrieval_latency_p95_ms: u128, + pub embedding_latency_p50_ms: u128, + pub embedding_latency_p95_ms: u128, + pub rerank_latency_p50_ms: u128, + pub rerank_latency_p95_ms: u128, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PassiveRecallTrace { + pub query_id: String, + pub query_text: String, + pub query_category: String, + pub interaction_mode: EvalInteractionMode, + pub split: DatasetSplit, + pub namespace: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub reference_time: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected_memory_action: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub chosen_window_days: Option, + #[serde(default)] + pub windows_considered: Vec>, + pub memories_searched: usize, + pub vector_search_mode: VectorSearchMode, + #[serde(default)] + pub top_candidates: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub reranked_candidates: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub window_attempts: Vec, + pub window_expanded: bool, + #[serde(default)] + pub gold_memory_ids: Vec, + pub no_hit: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_stage_top1_distance: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top1_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_top2_raw_score: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_margin_raw: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub support_top1_score: Option, + pub activation_decision: PassiveRecallDecision, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub recalled_memory_ids: Vec, + pub activation_correct: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub support_precision_at_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub support_recall_at_k: Option, + pub time_window_hit: bool, + pub embedding_latency_ms: u128, + pub retrieval_latency_ms: u128, + #[serde(skip_serializing_if = "Option::is_none")] + pub rerank_latency_ms: Option, + pub passed: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PassiveRecallBenchmarkReport { + pub experiment: RetrievalExperimentConfig, + pub dataset_id: String, + pub evaluated_split: Option, + pub ingest_count: usize, + pub ingest_latency_ms: u128, + pub metrics: PassiveRecallMetrics, +} + +fn default_namespace() -> String { + "default".to_string() +} + +fn default_embedding_version() -> String { + "v1".to_string() +} + +fn default_top_k() -> usize { + 20 +} diff --git a/klbr-core/src/reflection.md b/klbr-core/src/reflection.md new file mode 100644 index 0000000..7edcce0 --- /dev/null +++ b/klbr-core/src/reflection.md @@ -0,0 +1,20 @@ +{anchor} + +## pinned memories +{pinned_text} + +## goal +it's time to reflect and curate your long-term memory. +go through your recent memories and the conversation. + +use pin_memory/unpin_memory to promote or demote entries. +use remember to save anything new worth keeping. +use unpin_memory on pinned entries that are outdated or no longer relevant. +be selective — pinned memories appear in every context window. + +### recent unpinned memories +{unpinned_text} + +### other context +date: {date} +time: {time} diff --git a/klbr-core/src/retrieval.rs b/klbr-core/src/retrieval.rs new file mode 100644 index 0000000..0ee6fd5 --- /dev/null +++ b/klbr-core/src/retrieval.rs @@ -0,0 +1,237 @@ +use crate::mvp::{L1MemoryRecord, SimilarityMetric}; + +#[derive(Debug, Clone)] +pub struct RetrievalConfig { + pub namespace: String, + pub top_k: usize, + pub initial_window_days: Option, + pub expansion_window_days: Vec>, + pub expand_distance_threshold: Option, + pub similarity_metric: SimilarityMetric, + pub reference_time: Option, +} + +#[derive(Debug, Clone)] +pub struct RetrievedMemory { + pub memory: L1MemoryRecord, + pub score: f32, + pub rank: usize, +} + +#[derive(Debug, Clone)] +pub struct RetrievalOutcome { + pub chosen_window_days: Option, + pub windows_considered: Vec>, + pub memories_searched: usize, + pub top_candidates: Vec, + pub window_expanded: bool, + pub time_window_hit: bool, + pub best_distance: Option, +} + +pub fn retrieve_exact( + corpus: &[L1MemoryRecord], + query_embedding: &[f32], + config: &RetrievalConfig, + gold_memory_ids: Option<&[i64]>, +) -> RetrievalOutcome { + let windows = window_schedule(config.initial_window_days, &config.expansion_window_days); + let reference_time = config.reference_time.or_else(|| { + corpus + .iter() + .filter(|memory| memory.namespace == config.namespace) + .map(|memory| memory.event_time) + .max() + }); + + let mut chosen_window_days = None; + let mut top_candidates = Vec::new(); + let mut memories_searched = 0usize; + let mut time_window_hit = false; + let mut window_expanded = false; + + for (idx, window_days) in windows.iter().enumerate() { + let filtered = filter_records(corpus, &config.namespace, reference_time, *window_days); + if idx == 0 { + time_window_hit = gold_memory_ids + .map(|gold_ids| { + filtered + .iter() + .any(|memory| gold_ids.contains(&memory.memory_id)) + }) + .unwrap_or(false); + } + + let scored = score_records( + filtered.as_slice(), + query_embedding, + config.similarity_metric.clone(), + config.top_k, + ); + let best_distance = scored.first().map(|candidate| candidate.score); + let should_expand = idx + 1 < windows.len() + && config + .expand_distance_threshold + .zip(best_distance) + .map(|(threshold, distance)| distance > threshold) + .unwrap_or(false); + + chosen_window_days = *window_days; + memories_searched = filtered.len(); + top_candidates = scored; + + if should_expand { + window_expanded = true; + continue; + } + break; + } + + let best_distance = top_candidates.first().map(|candidate| candidate.score); + + RetrievalOutcome { + chosen_window_days, + windows_considered: windows, + memories_searched, + top_candidates, + window_expanded, + time_window_hit, + best_distance, + } +} + +pub fn window_schedule( + initial_window_days: Option, + expansion_window_days: &[Option], +) -> Vec> { + let mut windows = Vec::with_capacity(1 + expansion_window_days.len()); + windows.push(initial_window_days); + windows.extend(expansion_window_days.iter().copied()); + + let mut deduped = Vec::with_capacity(windows.len()); + for window in windows { + if !deduped.contains(&window) { + deduped.push(window); + } + } + deduped +} + +fn filter_records( + corpus: &[L1MemoryRecord], + namespace: &str, + reference_time: Option, + window_days: Option, +) -> Vec { + let window_seconds = window_days.map(|days| i64::from(days) * 86_400); + corpus + .iter() + .filter(|memory| memory.namespace == namespace) + .filter(|memory| memory.status.is_searchable()) + .filter(|memory| match (reference_time, window_seconds) { + (Some(reference), Some(window)) => { + memory.event_time <= reference && memory.event_time >= reference - window + } + _ => true, + }) + .cloned() + .collect() +} + +fn score_records( + records: &[L1MemoryRecord], + query_embedding: &[f32], + metric: SimilarityMetric, + top_k: usize, +) -> Vec { + let mut scored = records + .iter() + .cloned() + .map(|memory| { + let score = match metric { + SimilarityMetric::CosineDistance => { + cosine_distance(query_embedding, &memory.embedding) + } + SimilarityMetric::InnerProduct => { + -inner_product(query_embedding, &memory.embedding) + } + }; + (memory, score) + }) + .collect::>(); + scored.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal)); + scored.truncate(top_k); + scored + .into_iter() + .enumerate() + .map(|(idx, (memory, score))| RetrievedMemory { + memory, + score, + rank: idx + 1, + }) + .collect() +} + +fn cosine_distance(a: &[f32], b: &[f32]) -> f32 { + let dot = inner_product(a, b); + let norm_a = a.iter().map(|x| x * x).sum::().sqrt(); + let norm_b = b.iter().map(|x| x * x).sum::().sqrt(); + if norm_a == 0.0 || norm_b == 0.0 { + return 1.0; + } + 1.0 - (dot / (norm_a * norm_b)) +} + +fn inner_product(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mvp::{L1MemoryRecord, MemoryLayer, MemoryStatus, SimilarityMetric}; + + fn record(memory_id: i64, event_time: i64, embedding: Vec) -> L1MemoryRecord { + L1MemoryRecord { + memory_id, + namespace: "default".to_string(), + layer: MemoryLayer::L1, + text: format!("memory {memory_id}"), + event_time, + ingest_time: event_time, + embedding_model: "test".to_string(), + embedding_dim: embedding.len(), + embedding_version: "v1".to_string(), + status: MemoryStatus::Active, + source_ref: None, + tags: Vec::new(), + pinned: false, + embedding, + } + } + + #[test] + fn exact_retrieval_prefers_recent_window_then_expands() { + let corpus = vec![ + record(1, 100, vec![0.0, 1.0]), + record(2, 200_000, vec![1.0, 0.0]), + ]; + let config = RetrievalConfig { + namespace: "default".to_string(), + top_k: 5, + initial_window_days: Some(1), + expansion_window_days: vec![None], + expand_distance_threshold: Some(0.1), + similarity_metric: SimilarityMetric::CosineDistance, + reference_time: Some(200_000), + }; + + let outcome = retrieve_exact(&corpus, &[0.0, 1.0], &config, Some(&[1])); + assert!(outcome.window_expanded); + assert_eq!(outcome.chosen_window_days, None); + assert_eq!( + outcome.top_candidates.first().map(|c| c.memory.memory_id), + Some(1) + ); + } +} diff --git a/klbr-core/src/router.rs b/klbr-core/src/router.rs new file mode 100644 index 0000000..abb3179 --- /dev/null +++ b/klbr-core/src/router.rs @@ -0,0 +1,435 @@ +//! Runtime router: learned gating that decides whether a user query should +//! inject memories (Memory lane) or skip recall and encourage tools / abstention. +//! +//! Models are written by `klbr-bench` as `router_model.json`. +//! This module supports: +//! - legacy centroid/prototype routers (no `kind` field) +//! - linear softmax router (`kind: "linear_softmax_v1"`) +//! +//! Safety invariants: +//! - Any model/embedding dimension mismatch routes to `Memory` (safe default). +//! - Tool lane routing is conservative by thresholds. + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; + +/// Output of the router for a given query. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum RouteDecision { + /// Answer from memory via the normal recall path. + Memory, + /// Runtime state inspection needed — skip memory injection, signal tool lane. + Tools, + /// Query is underspecified or out of scope — skip recall, signal abstention. + Abstain, +} + +#[derive(Debug, Clone, Copy)] +pub struct RouterScores { + /// For centroid routers: max cosine similarity to the class prototypes. + /// For linear routers: softmax probability. + pub tools: f32, + pub memory: f32, + pub abstain: f32, + pub kind: &'static str, +} + +#[derive(Debug, Clone)] +pub enum Router { + Centroid(CentroidRouter), + Linear(LinearRouter), +} + +impl Router { + pub fn load(path: &Path) -> Result { + let bytes = std::fs::read(path) + .with_context(|| format!("failed to read router model from {}", path.display()))?; + + let model: RouterModelFile = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse router model from {}", path.display()))?; + + let router = match model { + RouterModelFile::Linear(model) => Router::Linear(LinearRouter::from_model(model)?), + RouterModelFile::Centroid(model) => { + Router::Centroid(CentroidRouter::from_model(model)?) + } + }; + + Ok(router) + } + + pub fn predict_raw(&self, query_embedding: &[f32]) -> RouteDecision { + self.predict_raw_with_scores(query_embedding).0 + } + + pub fn predict_raw_with_scores( + &self, + query_embedding: &[f32], + ) -> (RouteDecision, RouterScores) { + match self { + Router::Centroid(r) => r.predict_raw_with_scores(query_embedding), + Router::Linear(r) => r.predict_raw_with_scores(query_embedding), + } + } +} + +// -------------------- +// Centroid / prototype router (legacy) +// -------------------- + +/// Serialized centroid model (legacy). Matches the shape written by `klbr-bench router-multi`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CentroidRouterModel { + pub embed_model: String, + pub embed_dim: usize, + pub trained_on: String, + pub threshold: f32, // margin for Tools vs others + + // v2+: multiple prototypes per class. + #[serde(default)] + pub memory_centroids: Vec>, + #[serde(default)] + pub tools_centroids: Vec>, + #[serde(default)] + pub abstain_centroids: Vec>, + + // v1 compat (single centroid) + #[serde(default)] + pub memory_centroid: Vec, + #[serde(default)] + pub tools_centroid: Vec, + #[serde(default)] + pub abstain_centroid: Vec, +} + +#[derive(Debug, Clone)] +pub struct CentroidRouter { + threshold: f32, + embed_dim: usize, + memory_centroids_unit: Vec>, + tools_centroids_unit: Vec>, + abstain_centroids_unit: Vec>, +} + +impl CentroidRouter { + fn from_model(model: CentroidRouterModel) -> Result { + validate_centroid_model(&model)?; + + let mut mem = model.memory_centroids; + if mem.is_empty() && !model.memory_centroid.is_empty() { + mem.push(model.memory_centroid); + } + let mut tools = model.tools_centroids; + if tools.is_empty() && !model.tools_centroid.is_empty() { + tools.push(model.tools_centroid); + } + let mut abstain = model.abstain_centroids; + if abstain.is_empty() && !model.abstain_centroid.is_empty() { + abstain.push(model.abstain_centroid); + } + + Ok(Self { + threshold: model.threshold, + embed_dim: model.embed_dim, + memory_centroids_unit: mem.into_iter().map(|v| normalize(&v)).collect(), + tools_centroids_unit: tools.into_iter().map(|v| normalize(&v)).collect(), + abstain_centroids_unit: abstain.into_iter().map(|v| normalize(&v)).collect(), + }) + } + + fn predict_raw_with_scores(&self, query_embedding: &[f32]) -> (RouteDecision, RouterScores) { + if query_embedding.len() != self.embed_dim { + return ( + RouteDecision::Memory, + RouterScores { + tools: 0.0, + memory: 0.0, + abstain: 0.0, + kind: "centroid_v2(mismatch)", + }, + ); + } + + let unit = normalize(query_embedding); + let sim_tools = max_sim(&unit, &self.tools_centroids_unit); + let sim_memory = max_sim(&unit, &self.memory_centroids_unit); + let sim_abstain = max_sim(&unit, &self.abstain_centroids_unit); + + let decision = if (sim_tools - sim_memory) >= self.threshold + && (sim_tools - sim_abstain) >= self.threshold + { + RouteDecision::Tools + } else if sim_memory >= sim_abstain { + RouteDecision::Memory + } else { + RouteDecision::Abstain + }; + + ( + decision, + RouterScores { + tools: sim_tools, + memory: sim_memory, + abstain: sim_abstain, + kind: "centroid_v2(sim)", + }, + ) + } +} + +fn validate_centroid_model(model: &CentroidRouterModel) -> Result<()> { + if model.embed_dim == 0 { + bail!("embed_dim must be > 0"); + } + if !model.threshold.is_finite() { + bail!("threshold must be finite; got {}", model.threshold); + } + + let mem_lens: Vec = if !model.memory_centroids.is_empty() { + model.memory_centroids.iter().map(|v| v.len()).collect() + } else { + vec![model.memory_centroid.len()] + }; + let tools_lens: Vec = if !model.tools_centroids.is_empty() { + model.tools_centroids.iter().map(|v| v.len()).collect() + } else { + vec![model.tools_centroid.len()] + }; + let abstain_lens: Vec = if !model.abstain_centroids.is_empty() { + model.abstain_centroids.iter().map(|v| v.len()).collect() + } else { + vec![model.abstain_centroid.len()] + }; + + for (name, lens) in [ + ("memory", mem_lens), + ("tools", tools_lens), + ("abstain", abstain_lens), + ] { + if lens.is_empty() { + bail!("{name} centroids must be non-empty"); + } + for (idx, len) in lens.into_iter().enumerate() { + if len != model.embed_dim { + bail!( + "{name} centroid[{idx}] has dim {len}, expected embed_dim {}", + model.embed_dim + ); + } + } + } + + Ok(()) +} + +// -------------------- +// Linear softmax router +// -------------------- + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LinearDecisionParams { + pub tools_prob_threshold: f32, + pub tools_margin_threshold: f32, + pub abstain_prob_threshold: f32, + pub abstain_margin_threshold: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LinearRouterModel { + pub kind: String, // "linear_softmax_v1" + pub version: u32, // >= 1 + pub embed_model: String, + pub embed_dim: usize, + pub trained_on: String, + #[serde(default)] + pub dataset_id: Option, + #[serde(default)] + pub labels: Vec, // must contain tools/memory/abstain + pub weights: Vec>, // [C][D] + pub bias: Vec, // [C] + pub decision: LinearDecisionParams, +} + +#[derive(Debug, Clone)] +pub struct LinearRouter { + embed_dim: usize, + // Stored in fixed order: Tools, Memory, Abstain. + w: [Vec; 3], + b: [f32; 3], + decision: LinearDecisionParams, +} + +impl LinearRouter { + fn from_model(model: LinearRouterModel) -> Result { + if model.kind != "linear_softmax_v1" { + bail!("unsupported router kind '{}'", model.kind); + } + if model.version < 1 { + bail!("router version must be >= 1; got {}", model.version); + } + if model.embed_dim == 0 { + bail!("embed_dim must be > 0"); + } + if model.weights.len() != 3 || model.bias.len() != 3 { + bail!( + "linear router must have 3 classes; got weights={} bias={}", + model.weights.len(), + model.bias.len() + ); + } + for (idx, row) in model.weights.iter().enumerate() { + if row.len() != model.embed_dim { + bail!( + "weights[{idx}] has dim {}, expected embed_dim {}", + row.len(), + model.embed_dim + ); + } + } + if !model.decision.tools_prob_threshold.is_finite() + || !model.decision.tools_margin_threshold.is_finite() + || !model.decision.abstain_prob_threshold.is_finite() + || !model.decision.abstain_margin_threshold.is_finite() + { + bail!("decision thresholds must be finite"); + } + + // Map label order from file into our fixed order. + let mut idx_tools = None; + let mut idx_memory = None; + let mut idx_abstain = None; + if !model.labels.is_empty() { + for (i, lab) in model.labels.iter().enumerate() { + match lab.as_str() { + "tools" => idx_tools = Some(i), + "memory" => idx_memory = Some(i), + "abstain" => idx_abstain = Some(i), + _ => {} + } + } + } + let (it, im, ia) = match (idx_tools, idx_memory, idx_abstain) { + (Some(it), Some(im), Some(ia)) => (it, im, ia), + _ => (0, 1, 2), // assume canonical order if labels missing + }; + + let w_tools = model.weights[it].clone(); + let w_memory = model.weights[im].clone(); + let w_abstain = model.weights[ia].clone(); + let b_tools = model.bias[it]; + let b_memory = model.bias[im]; + let b_abstain = model.bias[ia]; + + Ok(Self { + embed_dim: model.embed_dim, + w: [w_tools, w_memory, w_abstain], + b: [b_tools, b_memory, b_abstain], + decision: model.decision, + }) + } + + fn predict_raw_with_scores(&self, query_embedding: &[f32]) -> (RouteDecision, RouterScores) { + if query_embedding.len() != self.embed_dim { + return ( + RouteDecision::Memory, + RouterScores { + tools: 0.0, + memory: 0.0, + abstain: 0.0, + kind: "linear_softmax_v1(mismatch)", + }, + ); + } + let unit = normalize(query_embedding); + let logits = self.logits(&unit); + let probs = softmax3(logits); + let p_tools = probs[0]; + let p_memory = probs[1]; + let p_abstain = probs[2]; + + let best_other_for_tools = p_memory.max(p_abstain); + let tools_margin = p_tools - best_other_for_tools; + + let abstain_margin = p_abstain - p_memory; + + let decision = if p_tools >= self.decision.tools_prob_threshold + && tools_margin >= self.decision.tools_margin_threshold + { + RouteDecision::Tools + } else if p_abstain >= self.decision.abstain_prob_threshold + && abstain_margin >= self.decision.abstain_margin_threshold + { + RouteDecision::Abstain + } else { + RouteDecision::Memory + }; + + ( + decision, + RouterScores { + tools: p_tools, + memory: p_memory, + abstain: p_abstain, + kind: "linear_softmax_v1(prob)", + }, + ) + } + + fn logits(&self, x_unit: &[f32]) -> [f32; 3] { + let mut out = [0.0f32; 3]; + for c in 0..3 { + out[c] = dot(x_unit, &self.w[c]) + self.b[c]; + } + out + } +} + +fn softmax3(logits: [f32; 3]) -> [f32; 3] { + let m = logits[0].max(logits[1]).max(logits[2]); + let e0 = (logits[0] - m).exp(); + let e1 = (logits[1] - m).exp(); + let e2 = (logits[2] - m).exp(); + let denom = (e0 + e1 + e2).max(1e-12); + [e0 / denom, e1 / denom, e2 / denom] +} + +// -------------------- +// Model file parsing +// -------------------- + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum RouterModelFile { + Linear(LinearRouterModel), + Centroid(CentroidRouterModel), +} + +// -------------------- +// Shared vector helpers +// -------------------- + +fn max_sim(query_embedding_unit: &[f32], centroids_unit: &[Vec]) -> f32 { + let mut best = f32::NEG_INFINITY; + for c in centroids_unit { + best = best.max(dot(query_embedding_unit, c)); + } + best +} + +pub fn dot(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b.iter()).map(|(x, y)| x * y).sum() +} + +pub fn normalize(v: &[f32]) -> Vec { + let norm = v + .iter() + .map(|x| (*x as f64) * (*x as f64)) + .sum::() + .sqrt() as f32; + if norm <= 1e-12 { + return v.to_vec(); + } + v.iter().map(|x| x / norm).collect() +} diff --git a/klbr-core/src/tools/mod.rs b/klbr-core/src/tools/mod.rs index 440fdd1..ff96a77 100644 --- a/klbr-core/src/tools/mod.rs +++ b/klbr-core/src/tools/mod.rs @@ -81,6 +81,7 @@ pub fn memory_tools() -> Subroutines { remember::tool(), recall::tool(), context_for::tool(), + // todo: these 3, tag / pin / unpin, could be a single "edit_memory" tool tag_memory::tool(), pin_memory::tool(), unpin_memory::tool(), diff --git a/klbr-core/src/tools/recall.rs b/klbr-core/src/tools/recall.rs index c9c5b6d..e0c363f 100644 --- a/klbr-core/src/tools/recall.rs +++ b/klbr-core/src/tools/recall.rs @@ -3,7 +3,7 @@ use std::pin::Pin; use serde_json::json; -use crate::llm::ToolDef; +use crate::{llm::ToolDef, mvp::SimilarityMetric, retrieval}; use super::{Tool, ToolContext}; @@ -74,6 +74,46 @@ async fn execute(args: serde_json::Value, ctx: ToolContext) -> String { Err(err) => return format!("error embedding query: {err}"), }; + if tags.is_empty() { + let corpus = match ctx.memory.get_all() { + Ok(corpus) => corpus, + Err(err) => return format!("error loading memories: {err}"), + }; + let outcome = retrieval::retrieve_exact( + &corpus, + &emb, + &retrieval::RetrievalConfig { + namespace: "default".to_string(), + top_k: limit, + initial_window_days: None, + expansion_window_days: Vec::new(), + expand_distance_threshold: None, + similarity_metric: SimilarityMetric::CosineDistance, + reference_time: Some(unix_timestamp()), + }, + None, + ); + if outcome.top_candidates.is_empty() { + return "no memories found".into(); + } + return outcome + .top_candidates + .into_iter() + .map(|candidate| { + let tag_str = if candidate.memory.tags.is_empty() { + String::new() + } else { + format!(" [{}]", candidate.memory.tags.join(", ")) + }; + format!( + "[dist:{:.3}][id:{}]{tag_str} {}", + candidate.score, candidate.memory.memory_id, candidate.memory.text + ) + }) + .collect::>() + .join("\n"); + } + match ctx.memory.recall(Some(&emb), &tags, tag_and, limit) { Ok(results) if results.is_empty() => "no memories found".into(), Ok(results) => results @@ -96,3 +136,10 @@ async fn execute(args: serde_json::Value, ctx: ToolContext) -> String { Err(err) => format!("error: {err}"), } } + +fn unix_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or_default() +} diff --git a/klbr-core/src/tools/remember.rs b/klbr-core/src/tools/remember.rs index 751da24..8c3095b 100644 --- a/klbr-core/src/tools/remember.rs +++ b/klbr-core/src/tools/remember.rs @@ -3,7 +3,10 @@ use std::pin::Pin; use serde_json::json; -use crate::llm::ToolDef; +use crate::{ + llm::ToolDef, + mvp::{MemoryLayer, MemoryRecordInput, MemoryStatus}, +}; use super::{Tool, ToolContext}; @@ -58,12 +61,24 @@ async fn execute(args: serde_json::Value, ctx: ToolContext) -> String { }) .unwrap_or_default(); match ctx.llm.embed(&content).await { - Ok(emb) => match ctx.memory.store(&content, &emb, &tags) { + Ok(emb) => match ctx.memory.store_with_metadata(&MemoryRecordInput { + memory_id: None, + namespace: "default".to_string(), + layer: MemoryLayer::L1, + text: content.clone(), + event_time: unix_timestamp(), + ingest_time: unix_timestamp(), + embedding_model: ctx.llm.config.embed_model.clone(), + embedding_dim: emb.len(), + embedding_version: "runtime".to_string(), + status: MemoryStatus::Active, + source_ref: Some("tool:remember".to_string()), + tags: tags.clone(), + pinned: important, + embedding: emb, + }) { Ok(id) => { if important { - if let Err(e) = ctx.memory.set_pinned(id, true) { - return format!("stored (id:{id}) but pin failed: {e}"); - } let tag_info = if tags.is_empty() { String::new() } else { @@ -84,3 +99,10 @@ async fn execute(args: serde_json::Value, ctx: ToolContext) -> String { Err(e) => format!("error embedding memory: {e}"), } } + +fn unix_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs() as i64) + .unwrap_or_default() +} -- 2.51.2