Something went wrong. Try again.
zig langref cli nate.tngl.io/zigman
Something went wrong. Try again.
3.7 kB · 91 lines
Python
at main
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192"""command line: `zigman-eval selftest` and `zigman-eval run`."""
from __future__ import annotations
import argparseimport osimport sysimport urllib.errorimport urllib.requestfrom pathlib import Path
from .config import Config, find_skill_dir, find_zigmanfrom .loop import ablate, render, result_line, savefrom .tasks import validate_oracles
def _selftest(zig: str) -> int: rows = validate_oracles(zig=zig) fails = 0 for task, ok, detail in rows: print(f" {'PASS' if ok else 'FAIL'} {task.id:<14} {task.note}") if not ok: print(f" oracle rejected reference: {detail}") fails += 1 print(f"\n{len(rows) - fails}/{len(rows)} oracles valid") return 1 if fails else 0
def _server_up(base_url: str) -> bool: """reachable if the endpoint responds at all — a 401/403 from /models still means a server is there (matters for hosted providers that require auth).""" try: urllib.request.urlopen(base_url.rstrip("/") + "/models", timeout=5) return True except urllib.error.HTTPError: return True # server answered (e.g. 401 unauthorized) — it's up except Exception: # noqa: BLE001 return False
def _run(args) -> int: if not _server_up(args.base_url): sys.exit( f"model server not reachable at {args.base_url}. start one, e.g.:\n" f" mlx_vlm.server --model {args.model} --port 1234\n" f"(or point --base-url at LM Studio on the same port)" ) skill_dir = Path(args.skill) if args.skill else find_skill_dir() if args.arms != "bare" and skill_dir is None: sys.exit("could not find skills/zigman; pass --skill <dir> (or run from the repo)") cfg = Config( model=args.model, provider=args.provider, base_url=args.base_url, skill_dir=skill_dir, zigman_bin=find_zigman(), zig=args.zig, timeout=args.timeout, out_dir=Path(args.out), ) arms = ("bare", "zigman") if args.arms == "both" else (args.arms,) report = ablate(cfg, task_filter=args.k, arms=arms, on_result=lambda r: print(result_line(r))) print(render(report)) print(f"evidence: {save(report, cfg.out_dir)}") return 0
def main() -> int: p = argparse.ArgumentParser(prog="zigman-eval", description=__doc__) sub = p.add_subparsers(dest="cmd", required=True)
st = sub.add_parser("selftest", help="validate oracles against reference solutions (no model)") st.add_argument("--zig", default="zig")
rn = sub.add_parser("run", help="ablate the corpus: bare vs. zigman, scored by the compiler") rn.add_argument("--model", default=os.environ.get("ZIGMAN_EVAL_MODEL"), help="provider model id (or set ZIGMAN_EVAL_MODEL)") rn.add_argument("--provider", default="local", help="Pi provider name (from ~/.pi/agent/models.json)") rn.add_argument("--base-url", default="http://localhost:1234/v1") rn.add_argument("--skill", default="", help="path to skills/zigman (default: discover by walking up)") rn.add_argument("--arms", choices=["both", "bare", "zigman"], default="both") rn.add_argument("--timeout", type=int, default=300, help="hard per-run timeout (s)") rn.add_argument("--out", default="reports", help="directory for the json report") rn.add_argument("--zig", default="zig") rn.add_argument("-k", default="", help="substring filter on task id")
args = p.parse_args() if args.cmd == "selftest": return _selftest(args.zig) if not args.model: sys.exit("no model: pass --model <id> or set ZIGMAN_EVAL_MODEL") return _run(args)
if __name__ == "__main__": raise SystemExit(main())