#!/usr/bin/env python3
"""Job records for submit.sh and poll-and-land.sh (specs/job-submit-land.md): one JSON file
per job,
/.json, in a directory outside the repository (JOBS_STATE_DIR, default
$HOME/.local/state/nev-jobs), so no host or address can reach a commit.
records.py get prints the field ("" when absent)
records.py set =... creates or updates the record
records.py open " " for every record whose
outcome is empty and that has a run id
A record holds:
name the job's name
resends how many times submit.sh enqueued it after the first (a number; resends
after a `run` step exit 3, and resends of a needs-person record, are not
counted)
run_id the last Dagu run id, which `bench/dagu-lane.sh watch` needs
outcome empty while the job is in Dagu's hands, then landed, refused, conflict,
failed, stopped, parked or needs-person (the job's failed rework rounds are
used up and it waits for a person's call, specs/job-review-rework.md)
message the last message
spec_sha the spec's blob at base that the last run was sent with (submit.sh); once
outcome is landed, the blob that landed. submit.sh refuses a landed job only
while the spec's blob at base is still this one
base_sha the full commit the last run was sent with as BASE_SHA (submit.sh). A resend
whose spec blob is still spec_sha goes at this commit, since the lane resumes
a job only at the base it started with
Nothing about a run's progress is kept (queued, running or ended comes from Dagu each
time), and neither is the lane (bench/dagu-lane.sh resolves it by index).
"""
import json
import os
import re
import sys
import tempfile
FIELDS = {"name", "resends", "run_id", "outcome", "message", "spec_sha", "base_sha"}
OUTCOMES = {"", "landed", "refused", "conflict", "failed", "stopped", "parked", "needs-person"}
NAME = re.compile(r"^[A-Za-z0-9._-]+$")
def path(d, name):
if not NAME.match(name) or ".." in name:
raise SystemExit(f"records: unusable job name: {name}")
return os.path.join(d, name + ".json")
def load(d, name):
try:
with open(path(d, name)) as f:
return json.load(f)
except FileNotFoundError:
return None
def save(d, name, rec):
os.makedirs(d, mode=0o700, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=d, prefix=".rec-")
with os.fdopen(fd, "w") as f:
json.dump(rec, f, indent=2, sort_keys=True)
f.write("\n")
os.replace(tmp, path(d, name))
def records(d):
if not os.path.isdir(d):
return []
out = []
for entry in sorted(os.listdir(d)):
if entry.endswith(".json") and not entry.startswith("."):
try:
with open(os.path.join(d, entry)) as f:
out.append(json.load(f))
except (OSError, ValueError):
continue
return out
def set_fields(d, name, pairs):
rec = load(d, name) or {"name": name, "resends": 0, "run_id": "", "outcome": "", "message": "", "spec_sha": ""}
for pair in pairs:
field, eq, value = pair.partition("=")
if not eq or field not in FIELDS or field == "name":
raise SystemExit(f"records: not a record field: {pair}")
if field == "resends":
if not value.isdigit():
raise SystemExit(f"records: resends is not a count: {value}")
rec[field] = int(value)
elif field == "outcome" and value not in OUTCOMES:
raise SystemExit(f"records: not an outcome: {value}")
else:
rec[field] = value
save(d, name, rec)
def main(argv):
if len(argv) < 2:
print(__doc__, file=sys.stderr)
return 2
d, cmd, args = argv[0], argv[1], argv[2:]
if cmd == "get" and len(args) == 2:
rec = load(d, args[0]) or {}
value = rec.get(args[1], "")
print("" if value is None else value)
return 0
if cmd == "set" and len(args) >= 2:
set_fields(d, args[0], args[1:])
return 0
if cmd == "open" and not args:
for r in records(d):
if not r.get("outcome") and r.get("run_id") and r.get("name"):
print(r["name"], r["run_id"])
return 0
print(__doc__, file=sys.stderr)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))