Something went wrong. Try again.
a tiny pet with a GPT-2 brain, living in your browser
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657"""Mix an older dataset into a new one, keeping only rows the new format accepts.
Usage: mix_data.py OLD.jsonl NEW.jsonl OUT.jsonl
tokenling-v2 changed care from direct events ("* Sam fed Mochi a cookie *")to offers the pet answers with [yes]/[no]. Old diaries and old chats withoutcare events are still valid v2 data, so they are reused. Every row, old andnew, is re-checked with gen_data.recheck (the current line rules), and rowsthat fail are dropped.Prints what was kept and why rows were dropped."""import collectionsimport jsonimport sys
import gen_data as G
def row_ok(row): try: G.recheck(row) return "ok" except ValueError as e: return str(e).split(":")[0]
def main(old, new, out): why = collections.Counter() rows = [] for line in open(old): row = json.loads(line) r = row_ok(row) # v1 had no [walk]; a v1 teacher that wrote one was off-spec. if r == "ok" and row["kind"] == "chat" and "[walk:" in row["text"]: r = "new tag in old row" why[f"old {row['kind']}: {r}"] += 1 if r == "ok": rows.append(row) for line in open(new): row = json.loads(line) r = row_ok(row) why[f"new {row['kind']}: {r}"] += 1 if r == "ok": rows.append(row) with open(out, "w") as f: for row in rows: f.write(json.dumps(row) + "\n") for k, v in sorted(why.items()): print(f"{v:6} {k}") print(f"{len(rows):6} rows written to {out}")
if __name__ == "__main__": if len(sys.argv) != 4: sys.exit(__doc__) main(*sys.argv[1:])