Something went wrong. Try again.
watches turns and automatically creates and resolves tasks that get pinned into main agent context
Something went wrong. Try again.
5.9 kB · 158 lines
Python
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159
import json, pathlib, reimport pandas as pdimport numpy as np
ROOT = pathlib.Path('/Users/dawn/proj/prime-agent/.prime/agent/session-artifacts/01a06166-382f-703f-8f17-bc939488de93/observer-bench')
def score_multi_intent(): p = ROOT / "results-multi-intent.jsonl" if not p.exists(): return None rows = [json.loads(l) for l in p.read_text().splitlines()] if not rows: return None records = [] for r in rows: gold = r["gold"] res = r["result"] gold_k = gold["intent_count"] if "error" in res: records.append({"id": r["itemId"], "gold_k": gold_k, "pred_k": 0, "ok": False, "exact": False, "decomposed": False}) continue try: out = json.loads(res["output"]) adds = out.get("additions", []) pred_k = len(adds) exact = (pred_k == gold_k) decomposed = (pred_k > 1) if gold_k > 1 else (pred_k == 1) records.append({ "id": r["itemId"], "gold_k": gold_k, "pred_k": pred_k, "ok": True, "exact": exact, "decomposed": decomposed, "ms": res.get("ms", 0) }) except Exception: records.append({"id": r["itemId"], "gold_k": gold_k, "pred_k": 0, "ok": False, "exact": False, "decomposed": False}) df = pd.DataFrame(records) summary = { "total": len(df), "exact_count_acc": round(df.exact.mean(), 3), "decomposition_rate": round(df.decomposed.mean(), 3), "by_gold_k": df.groupby("gold_k").agg(exact=("exact", "mean"), count=("exact", "count")).to_dict() } return summary, df
def score_supersession(): p = ROOT / "results-supersession.jsonl" if not p.exists(): return None rows = [json.loads(l) for l in p.read_text().splitlines()] if not rows: return None records = [] for r in rows: gold = r["gold"] res = r["result"] gold_exist = gold.get("goldExisting", {}) gold_superseded_ids = {tid for tid, stat in gold_exist.items() if stat == "superseded"} if "error" in res: records.append({"id": r["itemId"], "ok": False, "tp": 0, "fp": 0, "fn": len(gold_superseded_ids)}) continue try: out = json.loads(res["output"]) exist = out.get("existing", []) pred_superseded_ids = {t["id"] for t in exist if t.get("status") == "superseded"} tp = len(pred_superseded_ids & gold_superseded_ids) fp = len(pred_superseded_ids - gold_superseded_ids) fn = len(gold_superseded_ids - pred_superseded_ids) records.append({ "id": r["itemId"], "ok": True, "gold_superseded": len(gold_superseded_ids), "pred_superseded": len(pred_superseded_ids), "tp": tp, "fp": fp, "fn": fn, "hit": (pred_superseded_ids == gold_superseded_ids), "ms": res.get("ms", 0) }) except Exception: records.append({"id": r["itemId"], "ok": False, "tp": 0, "fp": 0, "fn": len(gold_superseded_ids)}) df = pd.DataFrame(records) total_gold = df.tp.sum() + df.fn.sum() recall = round(df.tp.sum() / max(1, total_gold), 3) precision = round(df.tp.sum() / max(1, df.tp.sum() + df.fp.sum()), 3) summary = { "total": len(df), "gold_supersessions": int(total_gold), "superseded_recall": recall, "superseded_precision": precision, "exact_match_rate": round(df.hit.mean(), 3) } return summary, df
def score_real_candidates(): p = ROOT / "results-real-candidates.jsonl" if not p.exists(): return None rows = [json.loads(l) for l in p.read_text().splitlines()] if not rows: return None records = [] for r in rows: res = r["result"] if "error" in res: records.append({"id": r["itemId"], "ok": False}) continue try: out = json.loads(res["output"]) adds = out.get("additions", []) exist = out.get("existing", []) # check role-restricted enum: 'done' must never appear all_statuses = [t.get("status") for t in (adds + exist)] has_done = "done" in all_statuses # check multi-intent decomposition n_threads = len(adds) + len(exist) # check evidence action citations has_citations = any(len(t.get("evidenceActionIds", [])) > 0 for t in (adds + exist)) records.append({ "id": r["itemId"], "ok": True, "has_done": has_done, "n_threads": n_threads, "has_citations": has_citations, "ms": res.get("ms", 0) }) except Exception: records.append({"id": r["itemId"], "ok": False}) df = pd.DataFrame(records) summary = { "total": len(df), "valid_json_rate": round(df.ok.mean(), 3), "no_done_guarantee": bool((~df.has_done).all()), "avg_threads": round(df.n_threads.mean(), 2), "action_citation_rate": round(df.has_citations.mean(), 3) } return summary, df
if __name__ == "__main__": print("=== Multi-Intent Segmentation ===") res_mi = score_multi_intent() if res_mi: print(json.dumps(res_mi[0], indent=2)) print("\n=== Supersession Detection ===") res_sup = score_supersession() if res_sup: print(json.dumps(res_sup[0], indent=2)) print("\n=== Real Coding Candidates ===") res_real = score_real_candidates() if res_real: print(json.dumps(res_real[0], indent=2))