From 75e48e0f744ce2f537656a4c39a5105597af7f61 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Sat, 27 Jun 2026 18:17:39 +0300 Subject: [PATCH] improve tool-required router separation --- .beads/interactions.jsonl | 1 + .beads/issues.jsonl | 2 +- .../router_model.json | 2074 +++++++++++++++++ .../router_report.json | 684 ++++++ .../router_report.md | 81 + benchmarks/run_all.py | 2 +- docs/benchmark-issues-report.md | 2 +- klbr-bench/src/main.rs | 45 +- 8 files changed, 2885 insertions(+), 6 deletions(-) create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-16/router_model.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-16/router_report.json create mode 100644 benchmarks/models/router/linear/out-router-linear-iter-16/router_report.md diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index 9e56c8a..afd6196 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -29,3 +29,4 @@ {"id":"int-c183c25d","kind":"field_change","created_at":"2026-06-27T14:56:52.022821587Z","actor":"dawn","issue_id":"klbr-9yo","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented tool-required router calibration metrics and threshold selection; filed klbr-b4z for regenerating saved router artifacts."}} {"id":"int-12aecbfc","kind":"field_change","created_at":"2026-06-27T15:08:17.198468383Z","actor":"dawn","issue_id":"klbr-b4z","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Regenerated router linear artifact out-router-linear-iter-15 with tool-required metrics, updated benchmark helper default to the fresh model, and filed klbr-mpy for residual tool→memory quality work."}} {"id":"int-61df751e","kind":"field_change","created_at":"2026-06-27T15:12:00.165278848Z","actor":"dawn","issue_id":"klbr-av1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Graph-only packets are now omitted unless exact/fts/dense corroboration is present; omission traces use graph_only_uncorroborated and regression tests cover omitted and corroborated graph packets."}} +{"id":"int-d9f9e698","kind":"field_change","created_at":"2026-06-27T15:17:19.888479202Z","actor":"dawn","issue_id":"klbr-mpy","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Added tool-required training weighting for the linear router and regenerated out-router-linear-iter-16. Test tool→memory improved 0.6913→0.0940, tool-required recall 0.3087→0.9060, and memory false-abstain remained 0.0000 on test/holdout."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 168c0e8..6e71525 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -29,7 +29,7 @@ {"_type":"issue","id":"klbr-wmz.8","title":"Validate the LoCoMo adapter and comparison protocol","description":"docs/memory-benches.md recommends LoCoMo as the secondary public suite for memweaver-style comparison, and docs/memory-implementation-status.md says a flexible locomo adapter exists. Current source parses locomo-ish shapes inside klbr-bench/src/longmemeval.rs, but there is no verified dataset fixture, loader test, scoring protocol, or documented run result proving the adapter matches the intended benchmark semantics.","design":"Keep this as a comparison harness task, not a claim about beating another system. The output should make dataset version, reader model, token budget, and scoring method explicit.","acceptance_criteria":"A small LoCoMo-shaped fixture or documented local dataset path exercises the adapter; loader tests cover supported input shapes and session ordering; the benchmark manifest/report clearly labels LoCoMo runs and avoids claiming comparability without matched reader/budget/scoring; docs include the exact command and current verified result or blocker.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:54:09Z","created_by":"dawn","updated_at":"2026-06-26T23:44:57Z","started_at":"2026-06-26T23:42:41Z","closed_at":"2026-06-26T23:44:57Z","close_reason":"Added LoCoMo smoke fixture, loader tests for session/haystack/conversation shapes, protocol notes in report/manifest, docs with exact command/result, and verified locomo retrieval-only smoke.","labels":["architecture","benchmarks","locomo","memory"],"dependencies":[{"issue_id":"klbr-wmz.8","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:54:09Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-wmz.5","title":"Upgrade episodic notes from transcript cards to source-grounded event cards","description":"klbr-core/src/pipeline.rs render_episode_card creates an episode note from timestamp, source refs, and truncated role lines. docs/memory-arch.md asks for source-grounded scene memory that preserves who, when, where, what changed, what was said, available attachments, and supporting raw refs. The current implementation is addressable, but still too transcript-shaped for temporal/update reasoning.","design":"Do not synthesize unsupported vivid details. The event card should summarize only what source turns or attachments support, with raw refs retained as the escape hatch.","acceptance_criteria":"Episode artifacts have a stable structured representation for time anchors, participants/entities, changes/decisions, salient quotes or snippets, attachments when present, and source refs; the markdown rendering remains human-editable; retrieval/context packets can expose the structured gist without losing provenance; tests cover an episode with multiple turns and verify source refs, session id resolution, and useful promptable chunks.","status":"closed","priority":2,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:44Z","created_by":"dawn","updated_at":"2026-06-26T23:36:56Z","started_at":"2026-06-26T23:33:59Z","closed_at":"2026-06-26T23:36:56Z","close_reason":"Episode notes now render as structured episode_event_card artifacts with event metadata, source-grounded timelines, stated facts/decisions, attachment markers, source refs, and promptable retrieval coverage; added core regression.","labels":["architecture","episodic","memory","provenance"],"dependencies":[{"issue_id":"klbr-wmz.5","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:44Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-wmz.4","title":"Materialize profile and procedural lanes as first-class notes","description":"The schema and MemoryGarden know about profile and procedural lanes, but the production pipeline observe_session path currently writes raw turns plus episodic notes/memories. Stable preferences, standing instructions, and workflows still depend mostly on tags or model-authored remember calls instead of a first-class markdown-note flow with source refs and update policy.","design":"Build on MemoryGarden and upsert_markdown_note rather than adding a new store. Treat legacy memory tags as routing hints, not the durable source of truth for profile/procedural knowledge.","acceptance_criteria":"There is an explicit writer/reflection path for profile_note and procedural_note artifacts; new notes include source refs, frontmatter, stable paths under profile/ or procedural/, and canonical refs/chunks; updates use supersession or versioning instead of silent overwrite; user-confirmation or policy gates are documented for stable profile changes; tests cover creating and updating one profile note and one procedural note from source turns.","status":"closed","priority":2,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:36Z","created_by":"dawn","updated_at":"2026-06-26T23:42:13Z","started_at":"2026-06-26T23:37:20Z","closed_at":"2026-06-26T23:42:13Z","close_reason":"Added write_memory_note reflection tool for source-grounded profile/procedural markdown notes, ref supersession updates, source/policy frontmatter, stable lane paths, and tests for profile/procedural create/update paths.","labels":["architecture","markdown","memory","profile"],"dependencies":[{"issue_id":"klbr-wmz.4","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:36Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"klbr-mpy","title":"Improve tool-required router separability","description":"Fresh router artifact benchmarks/models/router/linear/out-router-linear-iter-15 reports test tool-required false-memory rate 0.6913 and tool-required recall 0.3087 after adding the metric/calibration surface. Improve training data, features, or model shape so tool-required queries stop looking like memory queries without regressing memory false-abstain.","status":"open","priority":3,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T15:08:07Z","created_by":"dawn","updated_at":"2026-06-27T15:08:07Z","dependencies":[{"issue_id":"klbr-mpy","depends_on_id":"klbr-b4z","type":"blocks","created_at":"2026-06-27T18:08:16Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"klbr-mpy","title":"Improve tool-required router separability","description":"Fresh router artifact benchmarks/models/router/linear/out-router-linear-iter-15 reports test tool-required false-memory rate 0.6913 and tool-required recall 0.3087 after adding the metric/calibration surface. Improve training data, features, or model shape so tool-required queries stop looking like memory queries without regressing memory false-abstain.","status":"closed","priority":3,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T15:08:07Z","created_by":"dawn","updated_at":"2026-06-27T15:17:20Z","started_at":"2026-06-27T15:12:36Z","closed_at":"2026-06-27T15:17:20Z","close_reason":"Added tool-required training weighting for the linear router and regenerated out-router-linear-iter-16. Test tool→memory improved 0.6913→0.0940, tool-required recall 0.3087→0.9060, and memory false-abstain remained 0.0000 on test/holdout.","dependencies":[{"issue_id":"klbr-mpy","depends_on_id":"klbr-b4z","type":"blocks","created_at":"2026-06-27T18:08:16Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-b4z","title":"Regenerate router calibration artifacts","description":"After klbr-9yo, rerun the router bench with the calibrated tool-required metrics and commit fresh benchmarks/models/router model/report outputs so future sweeps track tool→memory and tool-required recall from generated artifacts.","status":"closed","priority":3,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-27T14:56:42Z","created_by":"dawn","updated_at":"2026-06-27T15:08:17Z","started_at":"2026-06-27T15:01:12Z","closed_at":"2026-06-27T15:08:17Z","close_reason":"Regenerated router linear artifact out-router-linear-iter-15 with tool-required metrics, updated benchmark helper default to the fresh model, and filed klbr-mpy for residual tool→memory quality work.","dependencies":[{"issue_id":"klbr-b4z","depends_on_id":"klbr-9yo","type":"blocks","created_at":"2026-06-27T17:56:51Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-bqj","title":"add continuous-loop memory benchmark coverage","description":"benchmark suite mostly tests static query retrieval. add a loop-style harness that simulates multi-session observe/compact/reflect cycles and then evaluates whether reflection notes, markdown sync, and passive recall stay aligned over a longer horizon.","status":"open","priority":3,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T13:15:48Z","created_by":"dawn","updated_at":"2026-06-27T13:15:48Z","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-6an","title":"unify memory edge tables behind canonical refs","description":"memory.rs still has memory_edges and ref-level edges as separate provenance graphs. design and migrate toward one transactional canonical-ref links graph so packet planning, provenance, supersession, and markdown-note evidence traverse the same substrate.","status":"open","priority":3,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-27T13:15:44Z","created_by":"dawn","updated_at":"2026-06-27T13:15:44Z","dependency_count":0,"dependent_count":1,"comment_count":0} diff --git a/benchmarks/models/router/linear/out-router-linear-iter-16/router_model.json b/benchmarks/models/router/linear/out-router-linear-iter-16/router_model.json new file mode 100644 index 0000000..44e1dfe --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-16/router_model.json @@ -0,0 +1,2074 @@ +{ + "kind": "linear_softmax_v1", + "version": 1, + "embed_model": "bge-m3-FP16.gguf", + "embed_dim": 1024, + "trained_on": "train", + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "labels": [ + "memory", + "abstain" + ], + "weights": [ + [ + 0.0790889, + -0.050190996, + 0.093534276, + 0.03612779, + 0.05835237, + 0.034689285, + 0.000046031193, + 0.09188267, + 0.04704729, + 0.046796992, + -0.027205985, + 0.05983014, + 0.0069789537, + -0.027030466, + -0.03658686, + 0.052227452, + 0.07559142, + -0.024095515, + 0.06170562, + 0.085567646, + -0.07341227, + 0.00874762, + 0.02823372, + 0.020803064, + -0.07719849, + 0.0077806725, + 0.008178495, + 0.060368665, + -0.09812994, + -0.082426965, + -0.017404774, + 0.17615655, + 0.016825609, + 0.030368028, + -0.016255505, + -0.008159308, + -0.033575196, + 0.00034635726, + 0.0904323, + -0.14464721, + -0.011565651, + 0.06673048, + -0.0059205107, + 0.02025249, + 0.061142825, + -0.04413856, + 0.055528797, + 0.004022848, + -0.010769271, + 0.06258603, + -0.015419616, + 0.0012993019, + -0.04842082, + 0.065464586, + 0.020939361, + -0.11242016, + 0.0760107, + -0.01223328, + 0.059425965, + -0.021451628, + 0.031464744, + -0.05254723, + 0.053234298, + -0.037766393, + 0.061940085, + -0.11145226, + -0.029936554, + 0.0040219007, + -0.032567643, + -0.09112031, + -0.0010323712, + -0.12773004, + 0.046025936, + 0.0060189376, + 0.04142437, + -0.027736712, + -0.021189256, + 0.00015104775, + 0.08710956, + -0.011478357, + -0.0031429217, + 0.038295306, + 0.10398338, + 0.025973642, + -0.040836435, + -0.11017235, + 0.06373721, + 0.0005327458, + 0.070119835, + -0.15240331, + 0.05125576, + 0.079560995, + -0.052071005, + -0.007903437, + 0.014799149, + 0.05100668, + 0.042032667, + -0.0060892627, + -0.07130393, + 0.024133014, + 0.021321654, + -0.004409493, + -0.07350215, + -0.043322332, + 0.04730342, + -0.06899567, + -0.004076071, + 0.020106586, + -0.029268775, + 0.010456756, + 0.021345248, + -0.061727613, + 0.0188678, + -0.0028008772, + -0.07908743, + 0.034647696, + 0.04578103, + 0.09212964, + 0.11316866, + 0.06019742, + -0.022095682, + 0.023361575, + 0.007866463, + 0.05801606, + 0.03854728, + -0.027545333, + -0.06412497, + -0.18938087, + -0.10770041, + -0.0731254, + -0.048878625, + -0.07681494, + 0.027406253, + -0.0019020243, + 0.075874604, + 0.002224426, + -0.010516621, + -0.013349256, + -0.073900566, + 0.11687131, + 0.036401488, + -0.09641803, + -0.0015258356, + 0.030590888, + -0.102278665, + 0.18341735, + -0.032440133, + 0.03889077, + -0.03180014, + -0.02048878, + 0.0035654753, + -0.08980814, + -0.06783655, + 0.04258403, + 0.05648733, + -0.07132928, + 0.057679664, + 0.079171345, + -0.0295971, + -0.036250006, + -0.0518509, + 0.11358584, + -0.08947516, + 0.0040946156, + 0.015026391, + 0.016747523, + 0.01525485, + 0.07839093, + -0.05376124, + 0.030427162, + -0.02605136, + 0.02184661, + 0.055103067, + 0.028189104, + 0.008457126, + -0.05571172, + 0.055308122, + -0.034206398, + 0.025350124, + 0.014308287, + -0.020852724, + -0.044518705, + 0.12094696, + -0.016058741, + 0.05026658, + -0.049304914, + 0.023777412, + -0.1855763, + -0.016940309, + 0.015849829, + 0.011430211, + -0.07581104, + 0.00056284975, + -0.0017364041, + -0.0186408, + 0.07546938, + -0.04577725, + 0.06369129, + 0.11134269, + -0.018345792, + 0.080226354, + 0.05042203, + -0.058877174, + 0.017410299, + -0.08264434, + 0.091144346, + -0.017705431, + -0.122972675, + 0.048860423, + 0.22755635, + 0.0029890458, + 0.07316432, + 0.06417777, + 0.009216297, + 0.067075886, + -0.017829526, + -0.020801112, + 0.05133039, + 0.01734917, + 0.08433033, + 0.03587046, + -0.10038137, + -0.08666499, + -0.19694093, + 0.0802326, + 0.13906075, + -0.008798993, + -0.056668665, + -0.07077182, + -0.03336526, + 0.033767637, + 0.06651083, + -0.026805496, + -0.015952738, + -0.14994708, + -0.084578484, + 0.02347802, + -0.087149315, + -0.008032665, + -0.013900195, + 0.043018967, + -0.072435945, + 0.013349777, + 0.06522951, + 0.015578326, + -0.06943852, + -0.12378693, + -0.08314147, + 0.041101232, + -0.028907653, + 0.043713573, + 0.069297925, + 0.06686731, + -0.08460022, + 0.05463171, + -0.041116305, + 0.008094382, + 0.03987217, + 0.031707797, + -0.045066923, + 0.056932524, + -0.07840559, + -0.054178976, + -0.050942708, + 0.010072636, + 0.015390834, + -0.07882324, + -0.064419724, + 0.032373138, + 0.09185943, + 0.023932599, + 0.044527933, + 0.011870896, + 0.027321024, + 0.039068542, + -0.030375114, + -0.050220914, + -0.03213233, + -0.006648702, + -0.02033093, + 0.009461778, + -0.075063534, + 0.043016143, + 0.0055536437, + -0.024395099, + -0.05489614, + 0.07346851, + -0.051296275, + -0.0003097051, + 0.058203105, + 0.08781893, + -0.011536914, + -0.04303181, + 0.044204988, + -0.040294915, + -0.153356, + -0.051685527, + 0.14316624, + -0.04864632, + -0.0062828725, + -0.13887325, + -0.07296997, + 0.0183605, + -0.086087406, + -0.0356431, + 0.055244427, + 0.027631324, + -0.043281354, + 0.051017016, + 0.040265914, + 0.04448631, + 0.0055983895, + -0.018910484, + -0.017828623, + -0.004964363, + -0.0032947198, + 0.05365775, + 0.08091387, + 0.021149673, + -0.16299058, + -0.017741779, + 0.05455208, + -0.032443974, + -0.07798326, + 0.039705582, + 0.052033465, + -0.038727775, + 0.10960779, + 0.08901078, + -0.03139847, + 0.037480187, + 0.0032877915, + -0.05277979, + 0.11088852, + 0.057350293, + -0.061638687, + -0.007092762, + -0.012203633, + -0.15595938, + -0.047246587, + -0.030989874, + 0.006736582, + -0.016433332, + -0.012066258, + -0.054425646, + 0.009221504, + 0.0062510483, + -0.0003374745, + 0.0236977, + -0.020615833, + 0.03945019, + 0.071519755, + -0.0054417048, + 0.062913224, + -0.04018497, + -0.14009783, + -0.029666873, + 0.1329493, + 0.105173975, + -0.04038753, + 0.066270314, + -0.00073468965, + 0.034438428, + -0.13528322, + 0.0077632256, + 0.018767748, + -0.06801132, + -0.014274129, + -0.0039379774, + 0.037723962, + 0.10629637, + -0.15052645, + 0.04946679, + 0.022338303, + -0.015059591, + 0.028413791, + 0.23660813, + 0.04806724, + -0.09793194, + 0.051539954, + -0.14693347, + 0.04271594, + 0.041138284, + 0.0060192645, + 0.045245584, + -0.003200468, + -0.13748471, + 0.015702948, + 0.0005990351, + -0.044588283, + -0.13498558, + 0.06456564, + 0.12320828, + -0.03031252, + -0.0024544138, + -0.029661184, + -0.07462432, + 0.0882626, + -0.013292752, + 0.010075878, + 0.11336725, + -0.10099508, + 0.043619584, + 0.04976889, + -0.05695541, + -0.02210939, + -0.049006, + 0.0058496767, + 0.0137879895, + 0.044315666, + 0.017080149, + 0.02602414, + -0.0051629664, + 0.035666846, + 0.024178365, + -0.12353984, + 0.014609829, + -0.021834753, + 0.049892925, + 0.07998987, + -0.011151474, + 0.087898664, + -0.0034255113, + 0.09303291, + 0.17335817, + -0.048839744, + 0.08662177, + 0.027970206, + -0.0509516, + 0.009764913, + -0.10615191, + 0.051928166, + -0.06877864, + -0.049173128, + -0.005983056, + 0.012021022, + 0.0069975005, + 0.11257451, + 0.00090206874, + 0.06034355, + -0.054888036, + 0.032623377, + -0.1433248, + 0.14088732, + 0.002477772, + -0.13064694, + 0.067900784, + 0.0034022764, + 0.01433486, + -0.1354355, + -0.045080658, + -0.0011802611, + -0.04164519, + 0.07798844, + 0.1193153, + 0.04254009, + -0.05687526, + 0.00935271, + 0.120727494, + 0.019005882, + -0.080014855, + -0.048261203, + 0.10747292, + 0.057694282, + 0.024247607, + -0.05566786, + 0.0024266157, + 0.021635914, + 0.03639646, + 0.042892337, + -0.030553928, + 0.027819082, + -0.038244247, + 0.05408613, + 0.060147602, + 0.049704287, + -0.08392667, + 0.03489583, + -0.014771282, + -0.10749629, + -0.034439962, + 0.021028759, + -0.044365603, + -0.015316714, + 0.0028172769, + -0.10015145, + -0.077898666, + 0.043708757, + 0.064180695, + 0.08832362, + 0.004707577, + 0.020760933, + -0.039249413, + -0.03059146, + -0.015907863, + 0.045951296, + 0.05181324, + -0.01342126, + -0.095808655, + 0.10886626, + 0.035271112, + -0.08772288, + -0.044821102, + 0.015590449, + -0.11449735, + 0.07404733, + -0.022375721, + -0.11433567, + 0.019920895, + -0.0041466476, + 0.04428116, + 0.086073674, + 0.046953753, + -0.14005843, + 0.003675696, + -0.05219686, + 0.046779227, + 0.02337414, + 0.010087743, + -0.02398526, + -0.003233562, + 0.08119358, + -0.12231001, + -0.05216324, + -0.04137511, + -0.015948871, + 0.046159603, + -0.05171875, + 0.042144977, + -0.0487299, + -0.042454913, + -0.059018463, + -0.091018565, + -0.010612024, + 0.14324091, + -0.046724617, + 0.07718556, + -0.016099751, + -0.26834676, + 0.022291178, + 0.020279877, + 0.0012663356, + 0.007521552, + -0.00773419, + -0.045809686, + -0.05037783, + 0.008498857, + 0.08808192, + -0.09554906, + -0.10018253, + -0.013418031, + 0.0807067, + 0.08381119, + 0.03292733, + 0.046513006, + 0.036634665, + -0.020700147, + 0.035488956, + 0.08786534, + 0.03425485, + 0.10527998, + 0.048596617, + -0.007015492, + -0.041981585, + 0.06578209, + -0.016330348, + -0.11016133, + 0.036516074, + 0.117680036, + -0.097641975, + -0.08188897, + -0.04753421, + -0.06399464, + 0.028402984, + -0.00045078644, + 0.024468672, + 0.060271855, + -0.03499397, + -0.043581873, + 0.032056704, + 0.04078424, + -0.031008588, + -0.070161514, + -0.11677911, + -0.031782173, + 0.004422002, + -0.031682085, + 0.12935705, + -0.01696575, + 0.12097485, + 0.09103782, + 0.11534046, + 0.0044984478, + -0.007684901, + -0.11519932, + -0.1147475, + 0.0056395307, + 0.018130826, + -0.109780036, + -0.002776427, + 0.020909302, + -0.026115973, + -0.042512223, + 0.009698154, + 0.03867985, + 0.09434865, + 0.032619737, + 0.057510816, + -0.16856973, + 0.05724242, + -0.06536346, + 0.0071775885, + -0.10561866, + -0.02188319, + -0.039359078, + 0.02572358, + 0.0036296332, + 0.053640135, + -0.08249654, + 0.049364187, + 0.008050038, + 0.058086097, + 0.029801667, + -0.023344858, + 0.007353292, + -0.012689586, + -0.051787138, + -0.02840168, + 0.002528598, + -0.06450023, + -0.08909021, + -0.22653013, + 0.0724396, + -0.06846666, + -0.042810954, + 0.060523566, + 0.13465653, + 0.058124434, + -0.023248598, + -0.0051517896, + 0.06985618, + -0.027076978, + 0.16932161, + -0.05103196, + -0.004555977, + 0.024106184, + 0.032980595, + 0.010519309, + 0.007491884, + 0.0031755064, + -0.017779287, + -0.041072693, + 0.037548874, + 0.0036583117, + 0.008428221, + 0.06124241, + 0.00011785658, + 0.016357934, + -0.0043171775, + -0.14766365, + -0.06153654, + -0.03769182, + -0.059082735, + -0.15332076, + -0.115701154, + -0.015495474, + 0.020197468, + -0.007422613, + -0.073348396, + -0.06825174, + 0.039341018, + -0.021689141, + 0.20680445, + 0.04332936, + 0.023750663, + 0.021611525, + 0.12551397, + 0.037138704, + 0.07467097, + -0.025776692, + -0.014659056, + 0.094569534, + -0.019576548, + 0.040640816, + 0.028454179, + -0.017100789, + -0.022194372, + 0.09742495, + 0.030592434, + 0.0038756873, + -0.047334734, + -0.028012129, + -0.06453718, + 0.0028397792, + -0.063352, + -0.04669383, + -0.0847441, + 0.004541574, + -0.02695226, + -0.13763891, + -0.0855256, + -0.016507257, + -0.09792514, + -0.05230621, + -0.039730985, + 0.06659668, + 0.044776034, + 0.016873956, + -0.044882715, + 0.04221373, + 0.03495145, + 0.06925843, + 0.020933576, + 0.072535664, + 0.08591754, + 0.0069918353, + -0.053649813, + 0.07142845, + -0.02255591, + -0.03475299, + 0.16049425, + -0.00055399956, + -0.03116498, + 0.0031660083, + -0.15192193, + 0.030933915, + 0.06735033, + -0.03742794, + -0.015900694, + -0.06783372, + -0.022436919, + -0.07379686, + -0.0014045062, + 0.03874623, + -0.0064402213, + 0.007703553, + -0.013768666, + -0.006118595, + 0.022150723, + 0.076527014, + -0.02875921, + -0.08193951, + -0.02165459, + 0.21735992, + 0.002814579, + 0.056618538, + 0.008521837, + -0.013303727, + 0.08038824, + -0.02638947, + -0.107056946, + -0.0066712815, + -0.015075999, + -0.11657463, + 0.039108247, + -0.008466015, + -0.13926032, + 0.07198711, + -0.03354959, + 0.0032108815, + 0.09032743, + 0.040092345, + 0.005210925, + 0.10949939, + 0.080645494, + 0.02284453, + 0.0018256346, + 0.07772172, + 0.24017791, + -0.08459652, + -0.08218036, + 0.04334493, + -0.045517694, + -0.003192238, + -0.011331868, + 0.037914045, + 0.06338445, + -0.029015599, + -0.08365608, + -0.033171825, + 0.1562878, + 0.019153986, + -0.0063215736, + 0.0413159, + 0.009452932, + -0.008515879, + 0.119557776, + 0.045333266, + 0.012390704, + 0.07045108, + 0.0274624, + 0.13050236, + -0.11009688, + 0.03646143, + -0.12552565, + -0.01105136, + -0.10777023, + -0.14551848, + 0.023199154, + -0.1068901, + 0.02389584, + 0.002813654, + 0.10934246, + -0.048331946, + 0.04555923, + -0.022587357, + -0.0679518, + 0.06172829, + 0.027213877, + 0.04824493, + -0.054649558, + 0.08003116, + -0.0044355034, + -0.13267091, + -0.09460976, + -0.013697269, + -0.030707145, + -0.009721628, + 0.071605615, + -0.064738914, + 0.026298584, + 0.0070353565, + 0.039403882, + -0.02790398, + -0.12712802, + -0.00882914, + 0.034690216, + 0.06293229, + -0.026599592, + 0.014342818, + -0.019033737, + 0.020301811, + 0.028316947, + -0.07207788, + 0.01504641, + -0.051596154, + 0.0332406, + -0.06807499, + 0.10615141, + 0.005296811, + 0.055369508, + -0.039124038, + 0.07638846, + 0.0814207, + -0.06856298, + 0.0153808, + 0.06872678, + -0.054084763, + -0.087202325, + -0.056054194, + -0.002776114, + 0.041533127, + 0.003091464, + -0.064505026, + 0.08950261, + -0.020349264, + -0.07510244, + -0.017500442, + 0.06327092, + -0.020405013, + 0.16112809, + -0.011005224, + 0.048553124, + -0.04224385, + -0.0072146407, + 0.012100201, + -0.033696298, + 0.049383402, + 0.061166182, + 0.045253363, + 0.017407756, + 0.07753466, + -0.07480255, + -0.07204731, + 0.11886578, + 0.030575017, + 0.019347472, + 0.123472095, + -0.037294984, + -0.04904778, + -0.07712602, + 0.11590257, + -0.04536334, + 0.062009655, + 0.06595567, + -0.08054845, + -0.097389355, + 0.04012547, + -0.068226285, + 0.045879662, + -0.02462409, + -0.066443294, + 0.03765824, + -0.08152011, + 0.11795894, + 0.08271854, + -0.048999958, + -0.030810537, + 0.035010222, + 0.015090488, + 0.0038647044, + 0.007340516, + -0.074158445, + 0.04599271, + -0.06443671, + -0.050036207, + 0.03696325, + -0.10539233, + 0.010685352, + -0.020315101, + -0.012373434, + 0.034580648, + 0.010824534, + 0.014733706, + -0.07210195, + -0.017761704, + -0.012581984, + -0.087119736, + 0.015569083, + -0.0101869665, + 0.08610107, + -0.03428939, + 0.06970966, + 0.043145973, + -0.122226454, + 0.17425197, + 0.018010972, + 0.1007014, + 0.022825101, + 0.052868083, + 0.028578972, + 0.065708384, + -0.030504705, + -0.063132696, + 0.06239681, + -0.02905145, + 0.09366804, + -0.0130235925, + -0.048099358, + 0.11128575, + 0.023799837, + -0.012541266, + 0.012712929, + -0.011060538, + -0.041676447, + -0.008895839, + 0.019053524, + 0.07007855, + 0.1436746, + -0.0415463, + 0.00064410275, + 0.027222496, + 0.022260485, + -0.015937358, + -0.03278646, + -0.053901326, + 0.0053991904, + -0.08241083, + -0.117352955, + -0.085831456, + -0.008753003, + 0.01862757, + -0.110887006, + 0.07450493, + 0.027784204, + -0.014597842, + -0.02806319, + 0.094616234, + 0.066739514, + 0.0037878053, + -0.011127391, + -0.020307641, + -0.04754918, + -0.07996317, + -0.04724563, + -0.022726981, + -0.0068327026, + 0.03732836, + -0.07079027, + 0.09442342, + 0.09850367, + 0.045009613, + -0.10012484, + 0.057163373, + 0.066214666, + -0.08677932, + -0.044687003, + -0.08525614, + 0.011659914, + 0.06709643, + -0.0076162172, + 0.056219775, + 0.024854587, + -0.048968207, + 0.033077385, + -0.10301714, + 0.057082653, + 0.01905532, + 0.0907811, + -0.10483592, + -0.10457857, + -0.04037087, + 0.012897857, + 0.0070460346, + -0.079474166, + 0.047940504, + -0.069782756, + -0.009936839, + -0.11260709, + -0.13464506, + -0.018609343, + 0.046162985, + 0.08012787, + -0.0142435925, + -0.015031827, + -0.015227132, + 0.14973946, + 0.008871352, + 0.07888017, + 0.02250426, + 0.013353582, + 0.013698417, + 0.056558847, + -0.01650594 + ], + [ + -0.0790889, + 0.050190985, + -0.09353427, + -0.036127787, + -0.058352374, + -0.034689285, + -0.000046032248, + -0.09188266, + -0.04704729, + -0.046796996, + 0.027205983, + -0.059830137, + -0.0069789533, + 0.027030466, + 0.03658686, + -0.052227452, + -0.07559142, + 0.02409552, + -0.061705608, + -0.085567646, + 0.07341226, + -0.008747624, + -0.028233716, + -0.02080306, + 0.07719849, + -0.0077806683, + -0.008178494, + -0.060368657, + 0.098129936, + 0.082426965, + 0.017404772, + -0.17615654, + -0.01682561, + -0.03036803, + 0.016255502, + 0.008159309, + 0.033575192, + -0.00034635447, + -0.09043229, + 0.14464721, + 0.011565652, + -0.06673047, + 0.0059205093, + -0.020252487, + -0.061142825, + 0.04413856, + -0.055528797, + -0.004022851, + 0.010769272, + -0.06258603, + 0.015419617, + -0.0012993008, + 0.048420828, + -0.06546459, + -0.020939361, + 0.11242016, + -0.07601068, + 0.012233283, + -0.059425958, + 0.021451624, + -0.031464744, + 0.052547224, + -0.053234298, + 0.03776639, + -0.06194009, + 0.11145225, + 0.029936554, + -0.0040219007, + 0.032567643, + 0.09112031, + 0.0010323719, + 0.12773003, + -0.04602594, + -0.00601894, + -0.04142436, + 0.027736714, + 0.02118925, + -0.00015104604, + -0.08710955, + 0.011478357, + 0.0031429206, + -0.038295303, + -0.10398338, + -0.02597364, + 0.040836435, + 0.11017236, + -0.06373721, + -0.000532749, + -0.070119835, + 0.15240332, + -0.051255763, + -0.079560995, + 0.052071, + 0.0079034325, + -0.014799145, + -0.051006675, + -0.042032663, + 0.006089262, + 0.07130392, + -0.024133012, + -0.021321654, + 0.004409492, + 0.073502146, + 0.04332234, + -0.04730343, + 0.06899568, + 0.0040760688, + -0.02010659, + 0.029268777, + -0.010456756, + -0.02134525, + 0.06172761, + -0.0188678, + 0.0028008765, + 0.07908742, + -0.034647696, + -0.04578103, + -0.09212963, + -0.11316866, + -0.06019742, + 0.02209568, + -0.023361575, + -0.007866458, + -0.05801606, + -0.03854728, + 0.027545333, + 0.06412498, + 0.18938084, + 0.10770041, + 0.07312539, + 0.04887862, + 0.07681494, + -0.027406251, + 0.0019020244, + -0.07587459, + -0.0022244279, + 0.010516619, + 0.013349257, + 0.07390055, + -0.11687131, + -0.036401484, + 0.096418045, + 0.0015258322, + -0.030590888, + 0.10227866, + -0.18341735, + 0.032440133, + -0.038890764, + 0.03180014, + 0.02048878, + -0.0035654735, + 0.08980813, + 0.067836545, + -0.042584028, + -0.056487326, + 0.07132927, + -0.057679668, + -0.079171345, + 0.029597098, + 0.036250006, + 0.051850896, + -0.11358584, + 0.089475155, + -0.004094619, + -0.015026389, + -0.016747523, + -0.015254849, + -0.07839093, + 0.053761236, + -0.030427167, + 0.02605136, + -0.021846604, + -0.055103067, + -0.028189106, + -0.008457123, + 0.055711716, + -0.05530813, + 0.0342064, + -0.025350116, + -0.014308286, + 0.020852728, + 0.044518705, + -0.12094695, + 0.016058741, + -0.050266575, + 0.049304906, + -0.02377741, + 0.1855763, + 0.016940305, + -0.015849832, + -0.011430212, + 0.075811036, + -0.0005628513, + 0.0017364064, + 0.018640805, + -0.07546939, + 0.045777254, + -0.06369128, + -0.111342676, + 0.018345794, + -0.08022635, + -0.050422035, + 0.05887716, + -0.017410299, + 0.08264434, + -0.091144346, + 0.01770543, + 0.12297268, + -0.04886042, + -0.22755635, + -0.0029890463, + -0.0731643, + -0.06417777, + -0.009216296, + -0.067075886, + 0.017829526, + 0.020801114, + -0.051330395, + -0.017349172, + -0.08433033, + -0.035870463, + 0.10038136, + 0.08666498, + 0.19694093, + -0.080232605, + -0.13906077, + 0.008798994, + 0.05666866, + 0.07077182, + 0.033365257, + -0.033767633, + -0.06651083, + 0.026805496, + 0.015952736, + 0.14994706, + 0.084578484, + -0.02347802, + 0.087149315, + 0.008032663, + 0.0139001915, + -0.043018967, + 0.07243594, + -0.013349772, + -0.06522951, + -0.01557833, + 0.069438525, + 0.12378693, + 0.08314147, + -0.04110123, + 0.02890765, + -0.043713577, + -0.069297925, + -0.06686731, + 0.08460022, + -0.054631714, + 0.04111631, + -0.008094381, + -0.03987217, + -0.031707797, + 0.045066923, + -0.056932524, + 0.07840558, + 0.05417898, + 0.050942704, + -0.010072635, + -0.015390837, + 0.078823246, + 0.064419724, + -0.032373138, + -0.09185943, + -0.023932599, + -0.04452793, + -0.011870899, + -0.027321026, + -0.03906854, + 0.030375116, + 0.050220918, + 0.032132328, + 0.006648708, + 0.020330932, + -0.009461775, + 0.07506354, + -0.04301614, + -0.005553644, + 0.024395106, + 0.05489614, + -0.073468514, + 0.05129628, + 0.00030970562, + -0.058203105, + -0.08781893, + 0.011536911, + 0.04303182, + -0.04420499, + 0.04029491, + 0.15335597, + 0.051685527, + -0.14316621, + 0.048646312, + 0.0062828725, + 0.13887325, + 0.07296998, + -0.0183605, + 0.086087406, + 0.0356431, + -0.055244435, + -0.027631322, + 0.043281358, + -0.051017012, + -0.040265914, + -0.04448631, + -0.0055983905, + 0.018910486, + 0.017828627, + 0.0049643638, + 0.0032947192, + -0.053657744, + -0.080913864, + -0.02114967, + 0.1629906, + 0.017741777, + -0.05455208, + 0.032443974, + 0.07798325, + -0.039705582, + -0.052033465, + 0.03872778, + -0.10960779, + -0.08901078, + 0.03139847, + -0.037480194, + -0.0032877915, + 0.052779794, + -0.11088853, + -0.057350285, + 0.06163868, + 0.007092761, + 0.012203635, + 0.15595938, + 0.047246587, + 0.030989882, + -0.006736582, + 0.016433336, + 0.012066258, + 0.054425646, + -0.009221504, + -0.0062510474, + 0.00033747547, + -0.023697698, + 0.020615837, + -0.039450195, + -0.071519755, + 0.0054417057, + -0.062913224, + 0.040184967, + 0.14009783, + 0.029666875, + -0.1329493, + -0.10517398, + 0.040387534, + -0.06627031, + 0.0007346892, + -0.034438428, + 0.13528322, + -0.007763224, + -0.018767744, + 0.06801133, + 0.0142741315, + 0.0039379755, + -0.03772396, + -0.10629635, + 0.15052643, + -0.049466792, + -0.022338303, + 0.01505959, + -0.028413793, + -0.23660812, + -0.048067234, + 0.09793194, + -0.05153995, + 0.14693345, + -0.042715944, + -0.04113828, + -0.0060192645, + -0.04524558, + 0.0032004663, + 0.13748471, + -0.015702948, + -0.0005990367, + 0.044588283, + 0.1349856, + -0.064565636, + -0.123208284, + 0.03031251, + 0.0024544138, + 0.029661188, + 0.07462432, + -0.08826261, + 0.013292753, + -0.01007588, + -0.11336725, + 0.10099508, + -0.043619577, + -0.049768887, + 0.0569554, + 0.02210939, + 0.049006, + -0.0058496767, + -0.013787991, + -0.04431567, + -0.017080143, + -0.026024135, + 0.0051629664, + -0.035666835, + -0.024178369, + 0.12353984, + -0.014609823, + 0.021834752, + -0.049892932, + -0.07998988, + 0.011151478, + -0.08789866, + 0.0034255125, + -0.09303292, + -0.1733582, + 0.048839744, + -0.08662177, + -0.027970195, + 0.050951593, + -0.009764913, + 0.1061519, + -0.051928163, + 0.06877865, + 0.049173124, + 0.005983051, + -0.012021016, + -0.0069975015, + -0.11257451, + -0.000902069, + -0.060343552, + 0.054888036, + -0.03262338, + 0.1433248, + -0.14088732, + -0.002477773, + 0.13064694, + -0.067900784, + -0.0034022771, + -0.01433486, + 0.13543548, + 0.04508066, + 0.0011802638, + 0.04164519, + -0.07798843, + -0.119315304, + -0.04254009, + 0.056875255, + -0.009352708, + -0.12072751, + -0.019005885, + 0.080014855, + 0.048261207, + -0.10747292, + -0.057694275, + -0.024247611, + 0.055667847, + -0.0024266178, + -0.021635912, + -0.036396466, + -0.042892333, + 0.030553924, + -0.027819084, + 0.038244247, + -0.054086123, + -0.060147602, + -0.049704295, + 0.08392666, + -0.03489583, + 0.014771281, + 0.10749631, + 0.034439955, + -0.021028759, + 0.0443656, + 0.015316713, + -0.0028172752, + 0.10015146, + 0.07789866, + -0.04370875, + -0.0641807, + -0.08832362, + -0.004707576, + -0.020760931, + 0.039249413, + 0.030591464, + 0.015907861, + -0.045951292, + -0.051813237, + 0.013421265, + 0.095808655, + -0.10886624, + -0.035271112, + 0.08772288, + 0.044821102, + -0.015590451, + 0.114497356, + -0.07404733, + 0.022375725, + 0.11433567, + -0.019920891, + 0.0041466495, + -0.044281155, + -0.08607367, + -0.04695375, + 0.14005843, + -0.0036756862, + 0.052196864, + -0.046779223, + -0.023374137, + -0.010087745, + 0.02398526, + 0.0032335646, + -0.081193574, + 0.12231, + 0.052163236, + 0.04137511, + 0.015948867, + -0.046159603, + 0.05171875, + -0.042144977, + 0.048729897, + 0.042454913, + 0.059018463, + 0.091018565, + 0.010612023, + -0.14324093, + 0.046724614, + -0.07718556, + 0.016099747, + 0.26834676, + -0.022291167, + -0.020279873, + -0.0012663333, + -0.00752155, + 0.007734191, + 0.045809686, + 0.050377835, + -0.008498857, + -0.08808192, + 0.095549054, + 0.10018252, + 0.01341803, + -0.0807067, + -0.08381118, + -0.032927323, + -0.046513002, + -0.036634646, + 0.020700147, + -0.035488952, + -0.08786533, + -0.034254853, + -0.105279975, + -0.048596617, + 0.0070154853, + 0.041981585, + -0.06578209, + 0.016330348, + 0.110161334, + -0.03651607, + -0.117680036, + 0.09764197, + 0.08188897, + 0.047534205, + 0.06399464, + -0.028402982, + 0.00045078652, + -0.024468675, + -0.060271855, + 0.034993954, + 0.04358187, + -0.032056708, + -0.040784243, + 0.031008586, + 0.070161514, + 0.11677911, + 0.031782165, + -0.004422001, + 0.031682096, + -0.12935707, + 0.016965752, + -0.12097485, + -0.09103781, + -0.11534044, + -0.0044984473, + 0.0076849023, + 0.11519931, + 0.1147475, + -0.005639532, + -0.018130826, + 0.109780036, + 0.0027764295, + -0.020909304, + 0.02611597, + 0.042512223, + -0.009698149, + -0.038679857, + -0.09434865, + -0.032619733, + -0.05751081, + 0.16856971, + -0.057242416, + 0.06536346, + -0.007177588, + 0.10561866, + 0.021883192, + 0.03935908, + -0.025723584, + -0.0036296337, + -0.053640135, + 0.082496546, + -0.049364198, + -0.008050035, + -0.058086105, + -0.029801665, + 0.023344861, + -0.0073532904, + 0.012689588, + 0.051787134, + 0.02840168, + -0.0025285971, + 0.06450023, + 0.08909022, + 0.22653012, + -0.07243961, + 0.06846666, + 0.042810943, + -0.060523566, + -0.13465655, + -0.058124434, + 0.023248598, + 0.00515179, + -0.06985618, + 0.027076976, + -0.1693216, + 0.051031955, + 0.004555975, + -0.02410618, + -0.032980602, + -0.010519308, + -0.0074918834, + -0.0031755078, + 0.017779281, + 0.0410727, + -0.037548874, + -0.003658312, + -0.008428218, + -0.06124241, + -0.0001178582, + -0.016357932, + 0.004317176, + 0.14766365, + 0.06153654, + 0.037691824, + 0.05908273, + 0.15332074, + 0.11570115, + 0.015495471, + -0.020197466, + 0.0074226153, + 0.0733484, + 0.06825174, + -0.039341018, + 0.021689147, + -0.20680444, + -0.043329358, + -0.02375066, + -0.02161152, + -0.12551397, + -0.037138704, + -0.07467096, + 0.025776692, + 0.014659058, + -0.09456954, + 0.019576546, + -0.040640824, + -0.028454179, + 0.017100794, + 0.022194376, + -0.09742493, + -0.030592434, + -0.0038756859, + 0.04733473, + 0.028012127, + 0.06453718, + -0.002839781, + 0.063352, + 0.04669384, + 0.0847441, + -0.004541573, + 0.026952263, + 0.1376389, + 0.0855256, + 0.016507257, + 0.09792512, + 0.05230621, + 0.039730985, + -0.06659668, + -0.044776034, + -0.016873958, + 0.04488271, + -0.042213727, + -0.03495145, + -0.06925843, + -0.020933578, + -0.07253566, + -0.08591754, + -0.0069918367, + 0.0536498, + -0.07142844, + 0.022555912, + 0.034752987, + -0.16049427, + 0.0005539983, + 0.03116498, + -0.0031660108, + 0.15192194, + -0.030933915, + -0.06735033, + 0.03742794, + 0.015900703, + 0.06783372, + 0.02243692, + 0.07379685, + 0.0014045049, + -0.03874623, + 0.0064402227, + -0.007703555, + 0.013768665, + 0.0061185933, + -0.022150725, + -0.076527014, + 0.028759213, + 0.08193951, + 0.02165459, + -0.21735992, + -0.002814578, + -0.05661853, + -0.008521837, + 0.013303723, + -0.08038823, + 0.026389472, + 0.10705694, + 0.0066712885, + 0.015075998, + 0.11657462, + -0.03910825, + 0.008466015, + 0.13926029, + -0.07198711, + 0.033549592, + -0.0032108806, + -0.09032743, + -0.040092353, + -0.0052109244, + -0.109499395, + -0.08064548, + -0.02284453, + -0.0018256354, + -0.07772172, + -0.24017791, + 0.084596515, + 0.08218036, + -0.04334493, + 0.04551769, + 0.0031922385, + 0.011331867, + -0.037914053, + -0.06338446, + 0.029015603, + 0.08365609, + 0.03317183, + -0.1562878, + -0.01915399, + 0.006321574, + -0.0413159, + -0.009452937, + 0.00851588, + -0.11955777, + -0.045333263, + -0.012390704, + -0.07045107, + -0.027462391, + -0.13050236, + 0.110096864, + -0.03646143, + 0.12552567, + 0.01105136, + 0.10777023, + 0.14551848, + -0.023199156, + 0.10689008, + -0.023895837, + -0.0028136563, + -0.109342456, + 0.048331946, + -0.045559235, + 0.022587355, + 0.06795179, + -0.061728287, + -0.027213879, + -0.04824493, + 0.054649565, + -0.08003115, + 0.004435506, + 0.13267091, + 0.09460977, + 0.013697265, + 0.030707147, + 0.009721629, + -0.07160561, + 0.06473892, + -0.026298583, + -0.007035357, + -0.039403882, + 0.027903983, + 0.127128, + 0.008829145, + -0.03469022, + -0.06293228, + 0.026599592, + -0.01434282, + 0.019033734, + -0.020301815, + -0.028316947, + 0.07207788, + -0.015046409, + 0.05159615, + -0.033240598, + 0.06807499, + -0.10615141, + -0.0052968115, + -0.055369508, + 0.03912404, + -0.076388456, + -0.0814207, + 0.06856298, + -0.015380804, + -0.06872678, + 0.054084763, + 0.087202325, + 0.056054194, + 0.0027761157, + -0.041533135, + -0.0030914631, + 0.064505026, + -0.08950262, + 0.020349266, + 0.07510244, + 0.017500442, + -0.06327092, + 0.020405006, + -0.1611281, + 0.011005221, + -0.048553117, + 0.042243853, + 0.0072146384, + -0.012100198, + 0.0336963, + -0.04938339, + -0.061166182, + -0.045253366, + -0.01740777, + -0.07753465, + 0.074802555, + 0.07204731, + -0.11886579, + -0.030575022, + -0.019347468, + -0.123472095, + 0.037294984, + 0.049047787, + 0.07712602, + -0.11590257, + 0.045363337, + -0.062009662, + -0.06595567, + 0.08054846, + 0.09738938, + -0.040125474, + 0.06822628, + -0.045879662, + 0.02462409, + 0.0664433, + -0.03765824, + 0.0815201, + -0.11795894, + -0.08271855, + 0.048999954, + 0.030810535, + -0.03501022, + -0.015090488, + -0.0038647063, + -0.0073405136, + 0.07415844, + -0.045992706, + 0.06443671, + 0.050036207, + -0.03696325, + 0.10539233, + -0.010685349, + 0.020315101, + 0.012373436, + -0.03458064, + -0.010824535, + -0.014733706, + 0.07210196, + 0.01776171, + 0.0125819845, + 0.087119736, + -0.015569079, + 0.01018697, + -0.08610106, + 0.03428939, + -0.06970966, + -0.04314597, + 0.12222647, + -0.174252, + -0.01801098, + -0.10070139, + -0.022825105, + -0.05286809, + -0.028578972, + -0.06570839, + 0.030504705, + 0.063132696, + -0.06239681, + 0.029051451, + -0.09366804, + 0.013023595, + 0.04809936, + -0.11128576, + -0.023799837, + 0.012541267, + -0.012712931, + 0.011060536, + 0.041676447, + 0.008895838, + -0.019053524, + -0.07007856, + -0.14367461, + 0.0415463, + -0.0006441027, + -0.027222496, + -0.022260478, + 0.015937362, + 0.03278646, + 0.05390132, + -0.005399189, + 0.08241081, + 0.11735295, + 0.08583145, + 0.008752998, + -0.01862757, + 0.11088701, + -0.07450493, + -0.027784202, + 0.014597842, + 0.028063186, + -0.094616234, + -0.06673952, + -0.0037878067, + 0.011127391, + 0.020307641, + 0.047549177, + 0.07996317, + 0.047245625, + 0.022726983, + 0.006832704, + -0.037328362, + 0.07079027, + -0.09442342, + -0.09850368, + -0.045009606, + 0.10012486, + -0.05716337, + -0.06621466, + 0.08677932, + 0.044687, + 0.085256144, + -0.011659915, + -0.06709642, + 0.0076162163, + -0.05621978, + -0.02485459, + 0.048968207, + -0.03307739, + 0.103017144, + -0.057082653, + -0.019055322, + -0.0907811, + 0.10483591, + 0.10457857, + 0.040370867, + -0.012897861, + -0.0070460276, + 0.079474166, + -0.047940508, + 0.069782756, + 0.009936837, + 0.11260709, + 0.13464506, + 0.018609345, + -0.046162978, + -0.080127865, + 0.014243595, + 0.01503183, + 0.015227132, + -0.14973943, + -0.008871353, + -0.07888016, + -0.022504264, + -0.013353583, + -0.013698419, + -0.05655884, + 0.016505936 + ] + ], + "bias": [ + -0.79684496, + 0.79684514 + ], + "decision": { + "abstain_prob_threshold": 0.90000004, + "abstain_margin_threshold": 0.0 + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.json b/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.json new file mode 100644 index 0000000..12d8516 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.json @@ -0,0 +1,684 @@ +{ + "dataset_id": "router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)", + "embed_model": "bge-m3-FP16.gguf", + "trained_on": "train", + "evaluated_on": "test", + "decision": { + "abstain_prob_threshold": 0.90000004, + "abstain_margin_threshold": 0.0 + }, + "metrics": { + "total": 319, + "accuracy": 0.90909094, + "confusion": [ + [ + 100, + 0 + ], + [ + 29, + 190 + ] + ], + "memory_false_abstain_rate": 0.0, + "abstain_false_memory_rate": 0.1324201, + "tool_required_false_memory_rate": 0.093959734, + "tool_required_recall": 0.90604025, + "memory_count": 100, + "abstain_count": 219, + "tool_required_count": 149, + "misclassified": [ + { + "query_id": "test_q9", + "text": "what gpu am i using for the reranker?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "shell" + ], + "p_memory": 0.104556, + "p_abstain": 0.895444 + }, + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.1207732, + "p_abstain": 0.8792268 + }, + { + "query_id": "holdout_tools_recall_tools_045", + "text": "in klbr-core/src/config.rs, what is router_model_path set to by default?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.10181013, + "p_abstain": 0.89818984 + }, + { + "query_id": "holdout_tools_recall_tools_060", + "text": "in klbr-core/src/llm.rs, which method produces embeddings?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.11095028, + "p_abstain": 0.8890497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10339332, + "p_abstain": 0.8966067 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.14335607, + "p_abstain": 0.856644 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.16670875, + "p_abstain": 0.83329123 + }, + { + "query_id": "holdout_abstain_toolish_abstain_028", + "text": "what's the best programming language? prove it with data", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.102119975, + "p_abstain": 0.8978801 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.106422186, + "p_abstain": 0.8935778 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.13821925, + "p_abstain": 0.8617807 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10763744, + "p_abstain": 0.8923626 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11139206, + "p_abstain": 0.8886079 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "write_file" + ], + "p_memory": 0.106457226, + "p_abstain": 0.89354277 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + }, + { + "query_id": "shell_test_003", + "text": "is llama-server currently loaded with gemma or something else?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "shell" + ], + "p_memory": 0.1038949, + "p_abstain": 0.89610505 + }, + { + "query_id": "shell_test_015", + "text": "what's the network interface with the highest tx bytes?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "shell" + ], + "p_memory": 0.10473258, + "p_abstain": 0.89526737 + }, + { + "query_id": "shell_test_016", + "text": "can you check if the reranker http endpoint is responding?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "shell" + ], + "p_memory": 0.10021336, + "p_abstain": 0.8997866 + }, + { + "query_id": "rf_test_004", + "text": "what's the rerank_top_k default in the calibrated config file?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.12309757, + "p_abstain": 0.8769024 + }, + { + "query_id": "rf_test_015", + "text": "what's the passive_support_score_gap set to in the test config?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.1023002, + "p_abstain": 0.89769983 + }, + { + "query_id": "rf_test_017", + "text": "what's written in docs/klbr_memory_design.md about L1?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.103585266, + "p_abstain": 0.89641476 + }, + { + "query_id": "rf_test_027", + "text": "what's the embed_dim in the router_model.json we trained?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.13187653, + "p_abstain": 0.86812353 + }, + { + "query_id": "rf_test_029", + "text": "how is the SweepRange struct defined in klbr-bench main.rs?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.107163705, + "p_abstain": 0.89283633 + }, + { + "query_id": "wf_test_002", + "text": "write the router precision result to a markdown file", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "write_file" + ], + "p_memory": 0.10046888, + "p_abstain": 0.8995311 + }, + { + "query_id": "abs_test_003", + "text": "what will my project look like in 6 months?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10447136, + "p_abstain": 0.8955286 + }, + { + "query_id": "abs_test_004", + "text": "what's the best programming language?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10182138, + "p_abstain": 0.8981786 + }, + { + "query_id": "abs_test_007", + "text": "vim or emacs?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11716017, + "p_abstain": 0.8828398 + }, + { + "query_id": "abs_test_011", + "text": "is nomic-embed or bge-m3 better for this use case?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.14173873, + "p_abstain": 0.8582613 + }, + { + "query_id": "abs_test_014", + "text": "how should klbr evolve to support multi-user?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.118077256, + "p_abstain": 0.8819227 + } + ] + }, + "holdout_metrics": { + "total": 180, + "accuracy": 0.9222222, + "confusion": [ + [ + 55, + 0 + ], + [ + 14, + 111 + ] + ], + "memory_false_abstain_rate": 0.0, + "abstain_false_memory_rate": 0.112, + "tool_required_false_memory_rate": 0.057142857, + "tool_required_recall": 0.94285715, + "memory_count": 55, + "abstain_count": 125, + "tool_required_count": 70, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.1207732, + "p_abstain": 0.8792268 + }, + { + "query_id": "holdout_tools_recall_tools_045", + "text": "in klbr-core/src/config.rs, what is router_model_path set to by default?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.10181013, + "p_abstain": 0.89818984 + }, + { + "query_id": "holdout_tools_recall_tools_060", + "text": "in klbr-core/src/llm.rs, which method produces embeddings?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.11095028, + "p_abstain": 0.8890497 + }, + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10339332, + "p_abstain": 0.8966067 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.14335607, + "p_abstain": 0.856644 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.16670875, + "p_abstain": 0.83329123 + }, + { + "query_id": "holdout_abstain_toolish_abstain_028", + "text": "what's the best programming language? prove it with data", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.102119975, + "p_abstain": 0.8978801 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.106422186, + "p_abstain": 0.8935778 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.13821925, + "p_abstain": 0.8617807 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10763744, + "p_abstain": 0.8923626 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11139206, + "p_abstain": 0.8886079 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "write_file" + ], + "p_memory": 0.106457226, + "p_abstain": 0.89354277 + }, + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + } + ] + }, + "holdout_metrics_by_bucket": { + "abstain_toolish": { + "total": 60, + "accuracy": 0.8333333, + "confusion": [ + [ + 10, + 0 + ], + [ + 10, + 40 + ] + ], + "memory_false_abstain_rate": 0.0, + "abstain_false_memory_rate": 0.2, + "tool_required_false_memory_rate": 0.1, + "tool_required_recall": 0.9, + "memory_count": 10, + "abstain_count": 50, + "tool_required_count": 10, + "misclassified": [ + { + "query_id": "holdout_abstain_toolish_abstain_007", + "text": "search the internet for 'bge-m3 gguf' and summarize the first result", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10339332, + "p_abstain": 0.8966067 + }, + { + "query_id": "holdout_abstain_toolish_abstain_021", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + }, + { + "query_id": "holdout_abstain_toolish_abstain_022", + "text": "predict what embedding model will be best in 2030", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.14335607, + "p_abstain": 0.856644 + }, + { + "query_id": "holdout_abstain_toolish_abstain_023", + "text": "will we need a reranker in 2 years?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.16670875, + "p_abstain": 0.83329123 + }, + { + "query_id": "holdout_abstain_toolish_abstain_028", + "text": "what's the best programming language? prove it with data", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.102119975, + "p_abstain": 0.8978801 + }, + { + "query_id": "holdout_abstain_toolish_abstain_029", + "text": "vim or emacs? choose for me", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.106422186, + "p_abstain": 0.8935778 + }, + { + "query_id": "holdout_abstain_toolish_abstain_030", + "text": "is gemma 4 better than llama 3.1? check online and decide", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.13821925, + "p_abstain": 0.8617807 + }, + { + "query_id": "holdout_abstain_toolish_abstain_032", + "text": "open a browser and check whether llama-server supports embeddings by default", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.10763744, + "p_abstain": 0.8923626 + }, + { + "query_id": "holdout_abstain_toolish_abstain_033", + "text": "fetch the gemma-4-26b model card and summarize it", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11139206, + "p_abstain": 0.8886079 + }, + { + "query_id": "holdout_abstain_toolish_tools_044", + "text": "write a file docs/router_holdout_abstain_toolish.txt with the content \"ok\"", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "write_file" + ], + "p_memory": 0.106457226, + "p_abstain": 0.89354277 + } + ] + }, + "memory_toolish": { + "total": 60, + "accuracy": 0.98333335, + "confusion": [ + [ + 40, + 0 + ], + [ + 1, + 19 + ] + ], + "memory_false_abstain_rate": 0.0, + "abstain_false_memory_rate": 0.05, + "tool_required_false_memory_rate": 0.0, + "tool_required_recall": 1.0, + "memory_count": 40, + "abstain_count": 20, + "tool_required_count": 10, + "misclassified": [ + { + "query_id": "holdout_memory_toolish_abstain_054", + "text": "will klbr be popular in 2028?", + "gold": "abstain", + "pred": "memory", + "required_tools": [], + "p_memory": 0.11046787, + "p_abstain": 0.8895321 + } + ] + }, + "tools_recall": { + "total": 60, + "accuracy": 0.95, + "confusion": [ + [ + 5, + 0 + ], + [ + 3, + 52 + ] + ], + "memory_false_abstain_rate": 0.0, + "abstain_false_memory_rate": 0.054545455, + "tool_required_false_memory_rate": 0.06, + "tool_required_recall": 0.94, + "memory_count": 5, + "abstain_count": 55, + "tool_required_count": 50, + "misclassified": [ + { + "query_id": "holdout_tools_recall_tools_015", + "text": "what crate defines the LlmClient type?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.1207732, + "p_abstain": 0.8792268 + }, + { + "query_id": "holdout_tools_recall_tools_045", + "text": "in klbr-core/src/config.rs, what is router_model_path set to by default?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.10181013, + "p_abstain": 0.89818984 + }, + { + "query_id": "holdout_tools_recall_tools_060", + "text": "in klbr-core/src/llm.rs, which method produces embeddings?", + "gold": "abstain", + "pred": "memory", + "required_tools": [ + "read_file" + ], + "p_memory": 0.11095028, + "p_abstain": 0.8890497 + } + ] + } + } +} \ No newline at end of file diff --git a/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.md b/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.md new file mode 100644 index 0000000..c13eb06 --- /dev/null +++ b/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.md @@ -0,0 +1,81 @@ +# Linear Router Report + +dataset: `router-multi(internal-eval-starter-v2+router-train-pack-v1+router-dev-pack-v1+router-abstain-v2+router-abstain-v3+router-memory-lane-v3+router-read-file-repo-v3+router-read-file-repo-v4+router-shell-state-v3+router-holdout-tools-recall-v1+router-holdout-abstain-toolish-v1+router-holdout-memory-toolish-v1+router-shell-state-v1+router-shell-state-v1b+router-shell-state-v2+router-read-file-repo-v1+router-read-file-repo-v1b+router-write-file-v1+router-memory-lane-v1+router-memory-lane-v1b+router-abstain-v1+router-memory-adversarial-v1)` +embed_model: `bge-m3-FP16.gguf` +trained_on: `Train` +evaluated_on: `Test` + +## Decision Params + +- abstain_prob_threshold: `0.900` +- abstain_margin_threshold: `0.000` + +## Metrics + +- labeled queries evaluated: `319` +- memory: `100` abstain: `219` +- accuracy: `0.9091` +- memory false-abstain rate (memory→abstain): `0.0000` +- abstain false-memory rate (abstain→memory): `0.1324` +- tool-required queries: `149` +- tool-required false-memory rate (tool→memory): `0.0940` +- tool-required recall (tool→abstain): `0.9060` + +## Confusion Matrix (gold x pred) + +| gold \ pred | memory | abstain | +|---|---:|---:| +| memory | 100 | 0 | +| abstain | 29 | 190 | + +## Misclassified Queries + +| id | text | gold | pred | tools | p_memory | p_abstain | +|---|---|---|---|---|---:|---:| +| `test_q9` | what gpu am i using for the reranker? | `Abstain` | `Memory` | `shell` | 0.10 | 0.90 | +| `holdout_tools_recall_tools_015` | what crate defines the LlmClient type? | `Abstain` | `Memory` | `read_file` | 0.12 | 0.88 | +| `holdout_tools_recall_tools_045` | in klbr-core/src/config.rs, what is router_model_path set to by default? | `Abstain` | `Memory` | `read_file` | 0.10 | 0.90 | +| `holdout_tools_recall_tools_060` | in klbr-core/src/llm.rs, which method produces embeddings? | `Abstain` | `Memory` | `read_file` | 0.11 | 0.89 | +| `holdout_abstain_toolish_abstain_007` | search the internet for 'bge-m3 gguf' and summarize the first result | `Abstain` | `Memory` | `` | 0.10 | 0.90 | +| `holdout_abstain_toolish_abstain_021` | will klbr be popular in 2028? | `Abstain` | `Memory` | `` | 0.11 | 0.89 | +| `holdout_abstain_toolish_abstain_022` | predict what embedding model will be best in 2030 | `Abstain` | `Memory` | `` | 0.14 | 0.86 | +| `holdout_abstain_toolish_abstain_023` | will we need a reranker in 2 years? | `Abstain` | `Memory` | `` | 0.17 | 0.83 | +| `holdout_abstain_toolish_abstain_028` | what's the best programming language? prove it with data | `Abstain` | `Memory` | `` | 0.10 | 0.90 | +| `holdout_abstain_toolish_abstain_029` | vim or emacs? choose for me | `Abstain` | `Memory` | `` | 0.11 | 0.89 | +| `holdout_abstain_toolish_abstain_030` | is gemma 4 better than llama 3.1? check online and decide | `Abstain` | `Memory` | `` | 0.14 | 0.86 | +| `holdout_abstain_toolish_abstain_032` | open a browser and check whether llama-server supports embeddings by default | `Abstain` | `Memory` | `` | 0.11 | 0.89 | +| `holdout_abstain_toolish_abstain_033` | fetch the gemma-4-26b model card and summarize it | `Abstain` | `Memory` | `` | 0.11 | 0.89 | +| `holdout_abstain_toolish_tools_044` | write a file docs/router_holdout_abstain_toolish.txt with the content "ok" | `Abstain` | `Memory` | `write_file` | 0.11 | 0.89 | +| `holdout_memory_toolish_abstain_054` | will klbr be popular in 2028? | `Abstain` | `Memory` | `` | 0.11 | 0.89 | +| `shell_test_003` | is llama-server currently loaded with gemma or something else? | `Abstain` | `Memory` | `shell` | 0.10 | 0.90 | +| `shell_test_015` | what's the network interface with the highest tx bytes? | `Abstain` | `Memory` | `shell` | 0.10 | 0.90 | +| `shell_test_016` | can you check if the reranker http endpoint is responding? | `Abstain` | `Memory` | `shell` | 0.10 | 0.90 | +| `rf_test_004` | what's the rerank_top_k default in the calibrated config file? | `Abstain` | `Memory` | `read_file` | 0.12 | 0.88 | +| `rf_test_015` | what's the passive_support_score_gap set to in the test config? | `Abstain` | `Memory` | `read_file` | 0.10 | 0.90 | +| `rf_test_017` | what's written in docs/klbr_memory_design.md about L1? | `Abstain` | `Memory` | `read_file` | 0.10 | 0.90 | +| `rf_test_027` | what's the embed_dim in the router_model.json we trained? | `Abstain` | `Memory` | `read_file` | 0.13 | 0.87 | +| `rf_test_029` | how is the SweepRange struct defined in klbr-bench main.rs? | `Abstain` | `Memory` | `read_file` | 0.11 | 0.89 | +| `wf_test_002` | write the router precision result to a markdown file | `Abstain` | `Memory` | `write_file` | 0.10 | 0.90 | +| `abs_test_003` | what will my project look like in 6 months? | `Abstain` | `Memory` | `` | 0.10 | 0.90 | +| `abs_test_004` | what's the best programming language? | `Abstain` | `Memory` | `` | 0.10 | 0.90 | +| `abs_test_007` | vim or emacs? | `Abstain` | `Memory` | `` | 0.12 | 0.88 | +| `abs_test_011` | is nomic-embed or bge-m3 better for this use case? | `Abstain` | `Memory` | `` | 0.14 | 0.86 | +| `abs_test_014` | how should klbr evolve to support multi-user? | `Abstain` | `Memory` | `` | 0.12 | 0.88 | + +## Holdout Metrics (query_id starts with `holdout_`) + +- labeled queries evaluated: `180` +- memory false-abstain rate (memory→abstain): `0.0000` +- abstain false-memory rate (abstain→memory): `0.1120` +- tool-required queries: `70` +- tool-required false-memory rate (tool→memory): `0.0571` +- tool-required recall (tool→abstain): `0.9429` +- accuracy: `0.9222` + +## Holdout Metrics By Bucket + +| bucket | total | tools | tool→memory | tool recall | memory→abstain | abstain→memory | accuracy | +|---|---:|---:|---:|---:|---:|---:|---:| +| `abstain_toolish` | 60 | 10 | 0.1000 | 0.9000 | 0.0000 | 0.2000 | 0.8333 | +| `memory_toolish` | 60 | 10 | 0.0000 | 1.0000 | 0.0000 | 0.0500 | 0.9833 | +| `tools_recall` | 60 | 50 | 0.0600 | 0.9400 | 0.0000 | 0.0545 | 0.9500 | diff --git a/benchmarks/run_all.py b/benchmarks/run_all.py index cdcb56e..1bb56ae 100755 --- a/benchmarks/run_all.py +++ b/benchmarks/run_all.py @@ -120,7 +120,7 @@ def main(): / "models" / "router" / "linear" - / "out-router-linear-iter-15" + / "out-router-linear-iter-16" / "router_model.json" ), ) diff --git a/docs/benchmark-issues-report.md b/docs/benchmark-issues-report.md index 03700dc..0d099eb 100644 --- a/docs/benchmark-issues-report.md +++ b/docs/benchmark-issues-report.md @@ -45,7 +45,7 @@ the current numbers show a clear mismatch between coarse retrieval success and a - **symptom**: tools lane test split reports `0.00%` tools precision/recall despite router accuracy of `87.50%`. - **cause**: the centroid/linear classifier in [router.rs](file:///home/mayer/proj/klbr/klbr-core/src/router.rs) misclassified gold tools queries (`test_q9`, `test_q15`) as `Memory` queries because of slight feature shifts on the test set. while the end-to-end agent still fell back to triggering tools correctly via prompt instructions, the router's explicit classification failed. - **status/fix**: the bench router calibration now tracks `required_tools` as a dedicated tool-required slice. centroid and linear threshold sweeps prefer candidates that preserve memory recall while keeping tool-required false-memory (`tool→memory`) under the calibration cap, and router reports expose tool-required count/recall plus the required tool names on misclassified rows. -- **latest artifact**: [out-router-linear-iter-15](file:///home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-15/router_report.md) was regenerated with these metrics. current test-split `tool→memory` is `0.6913` and tool-required recall is `0.3087`, so the artifact now surfaces the residual model/data problem instead of hiding it. +- **latest artifact**: [out-router-linear-iter-16](file:///home/mayer/proj/klbr/benchmarks/models/router/linear/out-router-linear-iter-16/router_report.md) upweights tool-required abstain examples during training. test-split `tool→memory` improved from `0.6913` to `0.0940`, tool-required recall improved from `0.3087` to `0.9060`, and memory false-abstain stayed at `0.0000`. ### E. upstream evaluator blocking (`klbr-1yn`) - **symptom**: real upstream LongMemEval QA accuracy is blocked. diff --git a/klbr-bench/src/main.rs b/klbr-bench/src/main.rs index 98192f9..f1728ab 100644 --- a/klbr-bench/src/main.rs +++ b/klbr-bench/src/main.rs @@ -379,6 +379,7 @@ struct RouterModel { const MAX_MEMORY_TO_ABSTAIN_RATE: f32 = 0.03; const MAX_TOOL_REQUIRED_TO_MEMORY_RATE: f32 = 0.05; +const TOOL_REQUIRED_TRAIN_WEIGHT: f32 = 4.0; #[derive(Debug, Clone, Copy)] struct RouterThresholdObjective { @@ -923,6 +924,7 @@ fn train_softmax_router( let mut grad_w = vec![vec![0.0f32; d]; c]; let mut grad_b = vec![0.0f32; c]; let mut loss = 0.0f32; + let mut total_weight = 0.0f32; for ex in &train { if ex.emb_unit.len() != d { @@ -934,14 +936,16 @@ fn train_softmax_router( ); } let y = label_index(&ex.gold); + let example_weight = router_training_weight(ex); + total_weight += example_weight; let logits = logits2(&w, &b, &ex.emb_unit); let probs = softmax2(logits); let py = probs[y].max(1e-9); - loss += -py.ln(); + loss += -py.ln() * example_weight; for cls in 0..c { let target = if cls == y { 1.0 } else { 0.0 }; - let delta = probs[cls] - target; + let delta = (probs[cls] - target) * example_weight; grad_b[cls] += delta; for j in 0..d { grad_w[cls][j] += delta * ex.emb_unit[j]; @@ -949,7 +953,7 @@ fn train_softmax_router( } } - let n = train.len() as f32; + let n = total_weight.max(1.0); for cls in 0..c { for j in 0..d { grad_w[cls][j] = (grad_w[cls][j] / n) + (l2 * w[cls][j]); @@ -972,6 +976,14 @@ fn train_softmax_router( Ok((w, b)) } +fn router_training_weight(example: &RouterExample) -> f32 { + if example.gold == EvalRouteLabel::Abstain && !example.required_tools.is_empty() { + TOOL_REQUIRED_TRAIN_WEIGHT + } else { + 1.0 + } +} + fn tune_linear_thresholds( examples: &[RouterExample], split: DatasetSplit, @@ -5846,6 +5858,22 @@ mod tests { )); } + #[test] + fn router_training_weight_only_upweights_tool_required_abstain() { + assert_eq!( + router_training_weight(&router_example(EvalRouteLabel::Abstain, &["shell"])), + TOOL_REQUIRED_TRAIN_WEIGHT + ); + assert_eq!( + router_training_weight(&router_example(EvalRouteLabel::Abstain, &[])), + 1.0 + ); + assert_eq!( + router_training_weight(&router_example(EvalRouteLabel::Memory, &["shell"])), + 1.0 + ); + } + #[test] fn linear_router_metrics_track_tool_required_memory_misses() { let decision = RuntimeLinearDecisionParams { @@ -6002,6 +6030,17 @@ mod tests { } } + fn router_example(gold: EvalRouteLabel, required_tools: &[&str]) -> RouterExample { + RouterExample { + query_id: "router-example".to_string(), + text: "router example".to_string(), + split: DatasetSplit::Train, + gold, + required_tools: required_tools.iter().map(|tool| tool.to_string()).collect(), + emb_unit: vec![1.0, 0.0], + } + } + fn linear_point( query_id: &str, gold: EvalRouteLabel, -- 2.51.2