diff --git a/scripts/jev-country-calibrate.py b/scripts/jev-country-calibrate.py index abe57ac..a7b7098 100644 --- a/scripts/jev-country-calibrate.py +++ b/scripts/jev-country-calibrate.py @@ -292,7 +292,9 @@ def parse_answer(resp): return None return {"choice": choice, "probabilities": ans.get("probabilities") or {}, - "confidence": ans.get("confidence")} + "confidence": ans.get("confidence"), + "usage": resp.get("usage") or {}, # {input_tokens, output_tokens} -- the cost record + "model": resp.get("model")} def cmd_judge(args): @@ -440,11 +442,30 @@ def join(gold, answers): return out +def usage_totals(answers): + """Sum the token usage recorded on each answer -- the run's actual cost, per Jev's usage field.""" + total, n = {}, 0 + for a in answers.values(): + u = a.get("usage") or {} + if u: + n += 1 + for k, v in u.items(): + if isinstance(v, (int, float)): + total[k] = total.get(k, 0) + v + return total, n + + def cmd_report(args): - joined = join(load_jsonl(args.gold), load_jsonl(args.answers)) + answers = load_jsonl(args.answers) + joined = join(load_jsonl(args.gold), answers) n_gold = sum(1 for r in joined if r["gold"]) print(f"\n{len(joined)} judged ({n_gold} gold / {len(joined) - n_gold} suspect)\n") + tot, n_use = usage_totals(answers) + if tot: + avg = {k: round(v / n_use, 1) for k, v in tot.items()} + print(f"TOKEN COST total {tot} over {n_use} calls | per-account avg {avg}\n") + print("CALIBRATION (gold location signals only; precision over decided answers)") print(f" {'confidence bin':<16}{'n':>7}{'precision':>11}{'coverage':>10}{'abstain':>9}") for b, d in calibration_table(joined).items(): @@ -542,6 +563,16 @@ def _self_test(): assert crit["nl"].endswith("the Netherlands."), crit["nl"] assert "unknown" in crit and len(crit) == len(opts) + 1 + # usage capture: parse_answer carries the cost record; usage_totals sums it across a run + pa = parse_answer({"model": "jev-1.13.0", "answers": {"country": {"choice": "nl", + "confidence": 0.97, "probabilities": {"nl": 0.97}}}, + "usage": {"input_tokens": 1461, "output_tokens": 339}}) + assert pa["usage"]["input_tokens"] == 1461 and pa["model"] == "jev-1.13.0", pa + tot, n = usage_totals({"a": {"usage": {"input_tokens": 10, "output_tokens": 3}}, + "b": {"usage": {"input_tokens": 20, "output_tokens": 7}}, + "c": {}}) # no usage -> not counted + assert tot == {"input_tokens": 30, "output_tokens": 10} and n == 2, (tot, n) + print("jev-country-calibrate self-test: OK")