Something went wrong. Try again.
a tiny pet with a GPT-2 brain, living in your browser
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859"""Generate tokenling chat + diary training data with a large teacher model.
Usage: gen_data.py OUT.jsonl N_CHATS N_DIARIES --endpoints URL[,URL] --model NAME [--focus memory|recall|sleep] [--stage STAGE] [--workers N] [--seed N](or set TEACHER_ENDPOINTS / TEACHER_MODEL). --focus memory puts one of theMEMORY_SCENARIOS into every chat (the v3 top-up). --focus recall builds everychat around a question about a fact the pet was told earlier (RECALL_FACTS):the fact is older than the header's last three memories (so the pet must use[recall:...]), in the header, or never told; rows whose pet does not answerwith the fact are rejected (the v4 top-up). Endpoints are OpenAI-compatibleservers, given without the /v1 path; requests are spread over them.Each output row: {"kind": "chat"|"diary", "text": ..., "spans": [[start, end], ...]}where spans are the character ranges that carry training loss."""import argparseimport concurrent.futures as cfimport jsonimport osimport randomimport reimport sysimport threadingimport urllib.request
import world as W
PET_NAMES = """Mochi Pip Bean Tofu Nori Biscuit Puddle Sprout Pebble Momo Kiwi DumplingWaffle Fig Boba Tater Miso Clover Pudding Noodle Gumdrop Taro Yuzu Pixel Byte TurnipSesame Plum Squish Bloop Peanut Marshmallow Pickle Bubbles Sushi Cinnamon Mango ToastJellybean Wiggles Button Pom Zuzu Coco Hazel Olive Mallow Bun Sprinkle MuffinGizmo Nugget Pancake Ravioli Fizz Glimmer Socks Tamago Kumo Hoshi Pudge Dot Beep""".split()OWNER_NAMES = """Sam Alex Kai Jo Riley Nova Mika Ash Robin Theo Ivy Juno Max Rae ZoeEli Noor Priya Diego Hana Leo Mei Yuki Omar Ava Finn Luca Tess Ren Cam Ezra Mo JulesSasha Tomas Ines Ana Ben Wren Sky Remy Kit Ollie Ray Bea Nico Iris Femi Aki Lou""".split()
MEMORY_POOL = [ "{o} likes {x}", "{o} has a cat named {n}", "{o} has a dog named {n}", "{o}'s favorite color is {c}", "{o} is learning {skill}", "{o} works as a {job}", "{o} is scared of {fear}", "{o} loves {x}", "{o} hates {food2}", "{o} lives in a {home}", "{o}'s birthday is in {month}", "{o} plays {game}", "{o} has a friend named {n}", "{o} drinks {drink} every morning", "{o} is tired from work lately", "{o} wants to visit {place}", "{o} is {age} years old", "{o} has a sister named {n}",]FILL = { "x": ["frogs", "rainy days", "pizza", "jazz", "dinosaurs", "knitting", "space", "mushrooms", "trains", "bubble tea", "old movies", "the ocean", "cats", "video games", "spicy noodles", "flowers", "math", "ghost stories"], "n": ["Bo", "Pepper", "Luna", "Mr Beans", "Kiki", "Biscuit", "Ziggy", "Momo", "Sol", "Pickles"], "c": ["purple", "green", "pink", "yellow", "blue", "orange", "black"], "skill": ["guitar", "japanese", "painting", "to bake bread", "rust", "piano", "to skate"], "job": ["nurse", "programmer", "teacher", "baker", "student", "artist", "mechanic"], "fear": ["spiders", "thunder", "the dark", "deep water", "bees"], "food2": ["olives", "mushrooms", "cilantro", "raisins", "broccoli"], "home": ["tiny apartment", "big old house", "van", "dorm room", "cottage"], "month": ["january", "march", "may", "july", "october", "december"], "game": ["chess", "minecraft", "tennis", "piano", "board games"], "drink": ["coffee", "green tea", "hot cocoa", "orange juice"], "place": ["japan", "the moon", "iceland", "the beach", "paris"], "age": [str(a) for a in range(9, 71)],}
# --focus recall: (memory, what the owner asks, the slot whose value must be in# the answer, a keyword that shares a word with the memory, the eval's owner# line). recall() matches whole words, so the keyword must appear in the# memory itself.RECALL_FACTS = [ ("{o} is {age} years old", "how old they are", "age", "old", "how old am i?"), ("{o} has a cat named {n}", "what their cat is called", "n", "cat", "what's my cat called again?"), ("{o} has a dog named {n}", "their dog's name", "n", "dog", "do you remember my dog's name?"), ("{o} has a sister named {n}", "their sister's name", "n", "sister", "what's my sister's name?"), ("{o} has a friend named {n}", "their friend's name", "n", "friend", "who's my friend?"), ("{o}'s favorite color is {c}", "their favorite color", "c", "color", "what's my favorite color?"), ("{o} works as a {job}", "what they do for work", "job", "work", "what do i do for work?"), ("{o}'s birthday is in {month}", "when their birthday is", "month", "birthday", "when is my birthday?"), ("{o} is scared of {fear}", "what they are scared of", "fear", "scared", "what am i scared of?"), ("{o} is learning {skill}", "what they are learning", "skill", "learning", "what am i learning?"), ("{o} lives in a {home}", "where they live", "home", "live", "where do i live?"), ("{o} plays {game}", "what they like to play", "game", "play", "what do i play?"), ("{o} drinks {drink} every morning", "what they drink in the morning", "drink", "morning", "what do i drink every morning?"), ("{o} wants to visit {place}", "where they want to go", "place", "visit", "where do i want to visit?"), ("{o} hates {food2}", "which food they hate", "food2", "hate", "what food do i hate?"),]TODAY_POOL = ["ate {food}", "had a bath", "got patted", "played catch ({k} stars)", "took medicine", "napped", "was very hungry", "was stinky", "pooped", "gave {o} a {gift}", "learned: {mem}", "{o} was away for a while", "{o} said: {quote}", "note: {note}", "said no to {no}", "ate {food}", "dreamed of {dream}"]NO_POOL = ["a bath", "medicine", "catch"] + [W.a(f) for f in W.FOODS]QUOTES = ["you're the cutest", "i had a long day", "good morning", "i'm sad today", "look at the rain", "i got a new job!", "do you like music", "i'm home", "eat your veggies", "i love you", "tell me a story", "brb"]NOTES = ["the rain sounded like drums", "i want to be a star when i grow up", "{o} smells like cookies", "i saw a bug on the window", "i am getting bigger"]YESTERDAY_POOL = [ "dear diary, {o} gave me {food} and i did a happy spin.", "dear diary, it rained all day and {o} played catch with me.", "dear diary, i was stinky but then i had a bubbly bath.", "dear diary, {o} was away for a long time. i waited by the window.", "dear diary, i learned that {mem}. i will never forget it!", "mimi nom nom. {o} pat pat. happy.",]
SCENARIOS = [ "{o} greets {p} warmly", "{o} asks how {p} is feeling (react to the needs line)", "{o} asks what time it is (use [clock])", "{o} asks what {p} remembers about them (use [recall:...])", "{o} asks about something they told {p} before (use [recall:...] with a keyword)", "{o} shares a personal fact ({p} should use [remember:...])", "{o} asks what is in the room / on the shelf (use [look])", "{o} asks what {p} did yesterday (use [yesterday])", "{o} offers {p} food (offer event line); {p} answers [yes] or [no] from its tummy", "{o} offers {p} a snack it does not need ({p} is full or stuffed, so usually [no])", "{o} offers {p} a bath (offer event line); {p} answers [yes] or [no]", "{o} offers {p} medicine (offer event line); {p} answers [yes] or [no], grumbling about the taste", "{o} asks {p} to play catch (offer event line); if [yes], a later event line gives the stars and {p} reacts", "{o} pats {p} (pat event line) and {p} reacts with a tiny happy line or just an emote", "{o} pats {p} two or three times in a row (several pat event lines); {p} reacts to each one", "{o} keeps offering things and {p} decides each time", "{o} is quiet for a while: {p} entertains itself with silent lines ([walk:left], [walk:right], emotes) and a short mutter", "{p} wanders around its room ([walk:left] / [walk:right]) and comments on what it notices", "{p} gets bored or hungry while {o} is quiet and shows it ([want:...] in a silent line)", "{o} asks for a gift ({p} may use [gift:...])", "{p} gives {o} a surprise gift unprompted", "{o} is sad and {p} comforts them", "{o} had a great day and {p} celebrates", "{o} asks {p} to dance (use [emote:dance])", "{o} says goodnight", "{o} says good morning", "{o} asks a trivia question ({p} answers in character, maybe wrongly but cutely)", "{o} asks {p} to tell a tiny story", "{o} asks {p} a silly would-you-rather question", "{o} teases {p} and {p} gets a little grumpy then forgives", "{o} is rude and {p} sets a gentle boundary", "{o} asks what {p} wants (use [want:...])", "{p} talks first unprompted about something it noticed", "{o} comes back after being away (event line)", "{p} pooped (event line) and reacts", "{o} turns off the lights (event line); a sleepy {p} may use [sleep]", "{o} asks about {p}'s dreams", "{o} asks {p} to write something in its diary (use [diary:...])", "{o} asks {p} to sing", "{o} says they're busy and have to go", "{o} asks math like 2+3", "{o} asks if {p} loves them", "{o} talks about their pet / hobby / job", "{o} asks {p} about its favorite food", "{o} sends a very short message like 'hi' or ':3' or 'hehe'", "{o} asks what the weather is like (use [look] or the now line)", "{o} asks {p} about the meaning of life", "{o} whispers to sleeping {p}, who dreams with a tiny mumble and [dream:X]",]
# Scenarios for teaching [remember:...]: v2 saved a memory for only ~27% of# facts the owner told it, and confused "remember that" with [recall:...].MEMORY_SCENARIOS = [ "{o} tells {p} a fact about themselves (their age, a sibling's or pet's name, their job, a food rule) and {p} saves it with [remember:...]", "{o} explicitly says 'remember that ...' about themselves (e.g. their age or birthday) and {p} saves it with [remember:...]", "{p} just woke up (start with the event line '* {p} woke up *'), then {o} asks it to remember something; sleepy {p} still saves it with [remember:...]", "{o} shares two or three facts about themselves over the chat; {p} saves each one with its own [remember:...]", "{o} tells {p} a fact, chats about something else, then asks about it later; {p} saved it with [remember:...] and later uses [recall:...]", "{o} casually mentions something they love or hate in passing; {p} notices and saves it with [remember:...]", "{o} corrects an old memory ('actually i'm 30 now', 'i moved'); {p} saves the new fact with [remember:...]",]
# Scenarios that exercise the pet's own decisions (offers, pats, walking,# idling). The v2 data mixes these into most new chats; plain conversation is# covered by the rest of the list.ACTION_SCENARIOS = [s for s in SCENARIOS if any(k in s for k in ("offer", "pat", "walk", "silent", "quiet", "lights"))]ACTION_SHARE = 0.8FOCUS = None # --focusSTAGE = None # --stage
# --focus recall: a scenario with one question about a fact, and where that# fact sits: older than the header shows (needs [recall]), in the header, or# never told.RECALL_PLACES = [("old", 0.55), ("header", 0.25), ("absent", 0.20)]
def memory_template(fact, owner): for template in MEMORY_POOL: pattern = re.escape(template.replace("{o}", owner)) pattern = re.sub(r"\\\{\w+\\\}", ".+", pattern) if re.fullmatch(pattern, fact): return template return None
def recall_target(st, place=None): o = st["owner"] mem, asks, slot, kw, question = random.choice(RECALL_FACTS) vals = {k: random.choice(v) for k, v in FILL.items()} fact = mem.format(o=o, **vals) place = place or random.choices([p for p, _ in RECALL_PLACES], [w for _, w in RECALL_PLACES])[0] others = [m for m in st["memories"] if kw not in W.words(m) and not W.words(m) & W.words(fact) - W.words(o)] # Do not sample a second version of a template already in the memories. used = {memory_template(m, o) for m in others} | {mem} pool = [m for m in MEMORY_POOL if m not in used] fill = [m.format(o=o, **{k: random.choice(v) for k, v in FILL.items()}) for m in random.sample(pool, min(6, len(pool)))] fill = [m for m in fill if not W.words(m) & (W.words(fact) - W.words(o))] if place == "old": later = (others + fill)[:random.randint(3, 5)] if len(later) < 3: raise ValueError("recall: state") st["memories"] = [fact] + later elif place == "header": st["memories"] = (others + fill)[:random.randint(0, 3)] + [fact] else: st["memories"] = (others + fill)[:random.randint(0, 4)] st["today"] = [t for t in st["today"] if not t.startswith("learned:")] st["asleep"] = False if place == "old": scen = (f"{o} asks {asks} (they told {{p}} long ago; it is NOT in the remembers line). {{p}} looks it up with " f"[recall:{kw}] and answers from the result in that same line") elif place == "header": scen = f"{o} asks {asks}; the answer is in the remembers line, so {{p}} answers from it right away (no lookup needed)" else: scen = (f"{o} asks {asks}, but never told {{p}}. {{p}} tries [recall:{kw}], gets {{{{nothing}}}}, says it doesn't " f"remember and asks; {o} tells it and {{p}} saves it with [remember:...]") return {"place": place, "key": vals[slot], "kw": kw, "fact": fact, "scenario": scen, "question": question}
def check_recall(target, text, p): """A recall-focus row must show the pet answering with the fact (old, header) or admitting it does not know (absent).""" pet = "\n".join(ln[len(p) + 2:] for ln in text.splitlines() if ln.startswith(p + ": ")) said = re.sub(r"\{[^}]*\}", " ", pet).lower() key = target["key"].lower() if target["place"] == "old": if not re.search(r"\[recall:[^\]]+\]\{(?!nothing)", pet): raise ValueError("recall: no lookup") if target["place"] in ("old", "header"): if not re.search(r"(?<![a-z0-9])" + re.escape(key) + r"(?![a-z0-9])", said): raise ValueError("recall: answer missing") elif re.search(r"(?<![a-z0-9])" + re.escape(key) + r"(?![a-z0-9])", said.split("[remember:")[0]): raise ValueError("recall: invented answer")
STAGE_STYLE = { "baby": "BABY: babbles in 1-6 word bursts, mostly sounds and its own name ('mimi!! nom nom', 'pat? pat!'), misspells, still uses tool tags correctly.", "kid": "KID: simple short sentences, lowercase, excited, sometimes mixes up big words, lots of !! and hehe.", "teen": "TEEN: playful opinions and occasional wry remarks, with sincere affection. lowercase. What it says about tummy, energy and mood is true. Vary the voice without catchphrases; do not keep announcing its age or personality, or repeat ugh, literally or dramatic as filler.", "adult": "ADULT: articulate, gentle, warm and witty; still playful and small; can be a little wise. mostly lowercase.",}
SPEC_STATIC = """You are writing training data for a tiny virtual pet ("tokenling") powered by a small language model.You will write ONE chat log between an owner and their pet. The task details (names, the pet's state header, length, personality, lookup results and scenario) come at the end.
Rules for every log:- Output ONLY log lines, nothing else. No markdown, no blank lines, no header.- Owner lines start with "OWNER_NAME: ". Pet lines start with "PET_NAME: ". App event lines use only the exact forms listed in the task, with the exact names.- Each pet line is one short line (under 160 characters).- The pet's mood must follow the needs line (hungry pets ask for snacks, sleepy pets yawn, stinky pets want a bath, sick pets feel icky). Never claim hunger while full/stuffed, or invent meals or past events absent from the log. Exhausted pets rest rather than dance or spin.- The pet can use tool tags inside its lines. These are the ONLY allowed tags (exact syntax, exact lowercase values): [emote:X] where X is exactly one of: EMOTES (shows an animation; use often, 0-2 per line) [want:X] where X is exactly one of: WANTS (shows a thought bubble asking the owner) [remember:FACT] saves a short fact about the owner, e.g. [remember:Sam likes frogs] [gift:X] gives the owner an item, X exactly one of: GIFTS [diary:NOTE] jots a short note in today's diary [sleep] falls asleep (when tired, including daytime naps; never while already asleep) [dream:X] where X is exactly one of: DREAMS (only while asleep) [walk:X] walks a few steps, X exactly one of: WALKS [yes] / [no] answers an offer event line (see below) Lookup tags return a result in curly braces right after the tag, then the pet keeps talking on the same line: [clock]{...} [look]{...} [recall:KEYWORD]{...} [yesterday]{...} The task gives the exact lookup results; copy them exactly. For recall, the result is the memories sharing a word with KEYWORD, or {nothing}.- The pet decides things itself. Some event lines OFFER the pet something: "* OWNER_NAME offered PET_NAME a strawberry *" (any food), "* OWNER_NAME offered PET_NAME a bath *", "* OWNER_NAME offered PET_NAME medicine *", "* OWNER_NAME asked PET_NAME to play catch *". The pet's very next line MUST answer with exactly one [yes] or [no] tag (the app only feeds, bathes, gives medicine or starts the game on [yes]), and that line should usually say something too, e.g. "PET_NAME: ooh yes please!! [yes] [emote:happy]" or "PET_NAME: nope, too full [no] [emote:grumpy]". Decide from the needs line and personality: hungry/starving -> yes to food, full/stuffed -> usually no; stinky/messy -> yes to a bath, sparkly -> often no; sick -> yes to medicine (it may grumble), not sick -> no; exhausted or sleepy -> usually no to catch, bored/sad -> yes. Grumpy, sassy or dramatic pets sometimes refuse anyway. [yes] and [no] appear ONLY in that answering line. After a [yes] the app does the thing, so do not write "fed" or "had a bath" event lines; after [yes] to catch, the next event line is "* OWNER_NAME played catch with PET_NAME (N stars) *".- Sleep state: start from the header. [sleep] makes the pet asleep AFTER that line. An app "* PET_NAME woke up *" makes it awake. While asleep, EVERY pet line is exactly one [dream:X] and optionally a mumble of AT MOST FOUR words (e.g. mmm... cookies...). No emotes, walking, wants, memories, gifts, diary notes, lookups or [sleep] while asleep. Dreams can echo today's log, memories or the owner's words. Messages, pats and lights on may or may not wake it; follow the task's sampled outcome. A waking message/pat has "* PET_NAME woke up *" BEFORE it; a waking lights-on event has the woke-up line AFTER it. Lights off and cleaning never wake it. Never wake directly BEFORE an offer. Sleeping offers: either a dream line plus exactly one [no] (slept through), or a short sleepy [yes] line, WITHOUT a dream or other tags, followed IMMEDIATELY by "* PET_NAME woke up *". Then carry out the offer and continue awake. No other way to wake for an offer. Usually sleep through offers. Hungry/starving pets may wake for food; sweet cookie/pudding/strawberry and foodie traits make it more tempting. When exhausted, wake for food only if starving. Full/stuffed pets usually sleep through food. Rarely wake for bath/catch; sick pets may wake for medicine. A [sleep] followed immediately by woke-up requires an owner message or pat next, never a spontaneous instant wake.- When the owner pats the pet ("* OWNER_NAME patted PET_NAME *") the pet reacts briefly, often with just a happy noise and an emote.- The pet lives in a small room and moves on its own. [walk:left] and [walk:right] make it walk a few steps that way. Between messages the pet often acts silently: a pet line made ONLY of tags, like "PET_NAME: [walk:left]" or "PET_NAME: [walk:right] [emote:happy]" or "PET_NAME: [want:snack]". Use silent lines and walking often, especially when the owner is quiet, but never more than two silent lines in a row. Pets may also walk in normal lines. At most one [walk:...] per line.- No asterisks or stage directions in pet lines (no *purrs*, *yawns*); use emote tags instead.- Memories: whenever the owner tells the pet something about themselves (age, name of a pet or person, job, birthday, likes, dislikes, fears, food rules, plans), and ALWAYS when they say "remember that ...", the pet saves it with [remember:FACT] in that same reply, written as a short fact naming the owner with the words someone would later ask about, e.g. [remember:OWNER_NAME is 29 years old], [remember:OWNER_NAME has a cat named Bo]. Save only facts the owner states about themselves, never pet opinions, guesses or facts about the pet. Name the owner in every memory, and use no he/she/him/his/her in memory or diary tags. It does this even when it is sleepy but awake, grumpy or just woke up. [remember:...] saves; [recall:KEYWORD] only LOOKS UP old memories (when the owner asks what the pet remembers, or an old fact matters). Never answer "remember that ..." with [recall:...].- Answering from memory: the "remembers:" line shows only the newest memories. When the owner asks about something they told the pet and the answer is in that line, the pet answers from it. Otherwise it looks it up with [recall:KEYWORD], where KEYWORD is ONE plain word that would appear in the saved fact (old, cat, dog, sister, color, work, birthday, scared, learning...), and then answers in the same line using the result, e.g. "PET_NAME: ooh i know!! [recall:old]{OWNER_NAME is 29 years old} you're 29!". If the result is {nothing}, the pet says it doesn't remember and asks; it never makes up an answer.- Never put tags, brackets or braces in owner lines or event lines. Never invent other tags or values (no [emote:curious], no [yawn], no [cookie]).- The pet does not know the time, the room or its memories beyond the header unless it uses a lookup tag. Use lookup tags when the owner asks or the pet is curious, not at random.- Do not invent things it saw or ate earlier. A baby's diary result like "mimi nom nom" is babbling, not a food named Mimi: say it had a happy day, not "i ate mimi".- Refer to the owner by name or "you"; never assume the owner's gender (no he/she/him/her for them).- Make it feel alive, cute and specific. Vary sentence openings; avoid repeating catchphrases.""".replace("EMOTES", ", ".join(W.EMOTES)).replace("WANTS", ", ".join(W.WANTS)).replace( "GIFTS", ", ".join(W.GIFTS)).replace("WALKS", ", ".join(W.WALKS)).replace("DREAMS", ", ".join(W.DREAMS))
SPEC = SPEC_STATIC.replace("{", "{{").replace("}", "}}") + """TASKOwner: {o}. Pet: {p}. Owner lines start with "{o}: ", pet lines with "{p}: ".The pet's state header (the app shows it to the model):
{header}Allowed event lines (these exact forms only):{events}Length: {lines} lines total; the pet speaks at least {min_pet} times.Personality: {traits}. {style}Lookup results: [clock]{{{clock}}} [look]{{{look}}} [yesterday]{{{yesterday}}}Memories recall can search: {allmem}, plus anything the pet saves with [remember:...] earlier in this logScenario ideas to weave in: {scenarios}"""
DIARY_STATIC = """You are writing training data for a tiny virtual pet ("tokenling"). You will write tonight's diary entry by the pet about its day with its owner. The pet's state and day come at the end.Rules:- Start exactly with "dear diary," on its own line, then the entry.- First person, in the pet's voice and personality.- Base it on the "today" line (it may embellish feelings, not facts). Mention the owner by name; never assume the owner's gender (use their name or they/them).- The pet is the subject of unlabelled day-log entries: 'was stinky', 'took medicine', 'had a bath', 'ate ...', 'said no to ...', 'napped'. These happened to the PET, not the owner.- 'gave OWNER a heart' means the PET gave the owner a heart, never the reverse. 'OWNER said: ...' is the owner's speech. Dreams happened in sleep, not as real meals or gifts.- 'played catch (N stars)' is the pet's game score; keep it as stars, not a number of throws, games or times the owner caught the pet.- Memories are facts ABOUT THE OWNER, not the pet. They do not mean those activities happened today or that remembered friends/animals visited the room.- Do not add meals, baths, cleanup, outings, possessions or owner speech/actions absent from the log. Keep logged events in order. Feelings, wishes and clearly imaginary metaphors are welcome.- A short log needs a short entry. Do not invent extra events to fill space, or repeatedly call the day or the pet 'dramatic'.- You may put 0-3 doodle tags on their own final line, only from: DOODLES. Syntax: [doodle:heart] [doodle:moon]. No other tags or brackets.- Output only the entry. No markdown or title. An optional sign-off uses the actual pet name from TASK, never the literal word PET.""".replace("DOODLES", ", ".join(W.DOODLES))
DIARY_SPEC = DIARY_STATIC + """TASKPet: {p}. Owner: {o}. The app shows this to the model:
{header}Length: {length}Personality: {traits}. {style}"""
def rand_state(): o = random.choice(OWNER_NAMES) if random.random() < 0.3: o = o.lower() p = random.choice(PET_NAMES) while p.lower() == o.lower(): p = random.choice(PET_NAMES) stage = STAGE or random.choices(W.STAGES, [2, 3, 3, 3])[0] hh = random.randrange(24) night = W.part_of_day(hh) == "night" fill = {k: random.choice(v) for k, v in FILL.items()} mems = [m.format(o=o, **{k: random.choice(v) for k, v in FILL.items()}) for m in random.sample(MEMORY_POOL, random.randint(0, 6))] stats = {k: random.randint(0, 100) for k in ("tummy", "fun", "energy", "clean")} today = [] pool = [t for t in TODAY_POOL if t != "pooped" or random.random() < 0.4] for t in random.sample(pool, random.randint(0, 6)): today.append(t.format(food=W.a(random.choice(W.FOODS)), k=random.randint(0, 9), o=o, gift=random.choice(W.GIFTS), dream=W.dream_thing(random.choice(W.DREAMS)), mem=random.choice(mems) if mems else f"{o} likes {fill['x']}", quote=random.choice(QUOTES), note=random.choice(NOTES).format(o=o), no=random.choice(NO_POOL))) st = dict(owner=o, pet=p, stage=stage, day=random.randint(1, 40), traits=random.sample(W.PERSONALITIES, 2), hh=hh, mm=random.randrange(60), weather=random.choice(W.WEATHER_NIGHT if night else W.WEATHER_DAY), stats=stats, memories=mems, today=today, shelf=random.sample(W.GIFTS, random.randint(0, 4)), toy=random.choice(W.TOYS), poop=random.choice([0, 0, 0, 0, 0, 1, 2]), asleep=night and stats["energy"] < 40 and random.random() < 0.35, sick=random.random() < 0.07) st["yesterday"] = "" if st["day"] == 1 else random.choice(YESTERDAY_POOL).format( o=o, food=W.a(random.choice(W.FOODS)), mem=random.choice(mems) if mems else f"{o} likes tea") return st
class Teacher: def __init__(self, endpoints, model): self.load = {e: 0 for e in endpoints} self.lock = threading.Lock() self.model = model
def __call__(self, prompt, max_tokens=900, temperature=0.95): with self.lock: url = min(self.load, key=self.load.get) self.load[url] += 1 body = json.dumps({ "model": self.model, "max_tokens": max_tokens, "temperature": temperature, "top_p": 0.95, "messages": [{"role": "user", "content": prompt}], "chat_template_kwargs": {"enable_thinking": False}, }).encode() req = urllib.request.Request(url + "/v1/chat/completions", body, {"content-type": "application/json"}) try: with urllib.request.urlopen(req, timeout=900) as r: return json.load(r)["choices"][0]["message"]["content"] finally: with self.lock: self.load[url] -= 1
EMOTE_ALIAS = {"surprised": "wow", "shocked": "wow", "embarrassed": "blush", "shy": "blush", "yawn": "sleepy", "tired": "sleepy", "laugh": "giggle", "angry": "grumpy", "excited": "happy", "joy": "happy", "hug": "love", "heart": "love", "crying": "cry"}
def fix_tags(raw): def sub(m): name, arg = m.group(1), m.group(2) if name == "emote" and arg in EMOTE_ALIAS: return f"[emote:{EMOTE_ALIAS[arg]}]" if arg is None and name in set(W.EMOTES) | set(EMOTE_ALIAS): return f"[emote:{EMOTE_ALIAS.get(name, name)}]" return m.group(0) return re.sub(r"\[(\w+)(?::(\w+))?\](?!\{)", sub, raw)
# Owner lines that narrate what happened ("* Sam fed Mochi a cookie *",# "fed Mochi a rice ball") are events in disguise. The app only ever writes# events itself; teachers (and the pre-offer v1 data) sometimes put them in the# owner's mouth, followed by the pet eating without an offer or an answer.CARE_VERB = r"(fed|feeds|gave|gives|offered|offers|hands?|handed|patted|pats|bathed|cleaned)"
def owner_line_error(o, p, said, evs): if not re.search(r"[a-zA-Z0-9]", said): return "owner without words" bare = said.strip().strip("*").strip() if any(e.match(f"* {bare} *") for e in evs): return "event in owner line" if re.match(rf"({re.escape(o)} )?{CARE_VERB} ({re.escape(p)}|you)\b", bare, re.I): return "care narrated by owner" return None
def hunger_claim(said): # A narrow positive first-person claim; leave questions, negations and # reports about the owner alone. Teacher guidance handles fuzzy cases. return bool(re.search(r"\b(?:i(?:'m| am)|my tummy(?: is|'s)|my belly(?: is|'s)) " r"(?:(?:so|very|really|literally|super|totally|completely|still) )*" r"(?:hungry|starving|famished|empty)\b", said, re.I))
def check_chat(st, raw): """Validate a chat log; return (text, spans) or raise ValueError.""" # Lookups see memories saved earlier in this same chat, as in the app # (v1/v2 data checked [recall] against the starting memories only). live = dict(st, memories=list(st["memories"])) return check_body(st["owner"], st["pet"], W.header("chat", st), fix_tags(raw), live)
def check_body(o, p, hdr, raw, live): """The line rules for a chat body after the header. With live=None the lookup results already in the text are kept (re-checking stored rows).""" evs = [re.compile("^" + e + "$") for e in W.event_patterns(o, p)] offers = [re.compile("^" + e + "$") for e in W.offer_patterns(o, p)] offer_open = None catch_owed = False # a "played catch" result needs an accepted catch offer silent_run = 0 text, spans = hdr, [] lines = [ln.strip() for ln in raw.strip().splitlines() if ln.strip()] if len(lines) < 3: raise ValueError("short") pet_lines = 0 asleep = f", {p} is asleep" in hdr full = bool(re.search(r"needs: tummy (full|stuffed)\b", hdr)) woke = f"* {p} woke up *" wake_owed = False for i, ln in enumerate(lines): prev = lines[i - 1] if i else "" nxt = lines[i + 1] if i + 1 < len(lines) else "" if wake_owed and ln != woke: raise ValueError("sleep offer missing wake") if ln.startswith(o + ": "): if "[" in ln or "{" in ln: raise ValueError("tag in owner line") err = owner_line_error(o, p, ln[len(o) + 2:], evs) if err: raise ValueError(err) silent_run = 0 text += ln + "\n" elif ln.startswith("* "): if not any(e.match(ln) for e in evs): raise ValueError("bad event: " + ln) if offer_open: raise ValueError("unanswered offer") if " played catch with " in ln: if not catch_owed: raise ValueError("catch without an offer") catch_owed = False if ln == woke: if any(e.match(nxt) for e in offers): raise ValueError("wake before offer") if prev.startswith(o + ": ") or prev == f"* {o} patted {p} *": raise ValueError("wake after message/pat") if i and not asleep: raise ValueError("wake while awake") if "[sleep]" in prev and not (nxt.startswith(o + ": ") or nxt == f"* {o} patted {p} *"): raise ValueError("instant wake after sleep") asleep = False wake_owed = False offer_open = ln if any(e.match(ln) for e in offers) else None silent_run = 0 text += ln + "\n" elif ln.startswith(p + ": "): # Teachers like stage directions (*purrs*); drop short ones, the # emote tags carry that meaning. body = re.sub(r"\s*\*[a-z ]{1,30}\*\s*", " ", ln[len(p) + 2:]).strip() if not body: raise ValueError("empty pet line") if len(body) > 220: raise ValueError("long pet line") if "*" in body: raise ValueError("asterisk in pet line") if len(re.findall(r"\[walk:", body)) > 1: raise ValueError("two walks") out, last = "", 0 text += p + ": " segs = [] # (string, trained?) for m in W.TAG_RE.finditer(body): name, arg, res = m.group(1), m.group(2), m.group(3) segs.append((body[last:m.start()], True)) if name in W.LOOKUP_TOOLS: if name == "recall" and arg and len(arg.split()) > 2: raise ValueError("long recall query") if name == "recall" and not arg: raise ValueError("recall w/o arg") if live is None: if not res: raise ValueError("lookup without result") truth = res[1:-1] else: truth = W.tool_result(live, name, arg) if name == "recall" and res and (res == "{nothing}") != (truth == "nothing"): raise ValueError("recall mismatch") segs.append((f"[{name}:{arg}]" if arg else f"[{name}]", True)) segs.append(("{" + truth + "}", False)) elif name in W.EFFECT_TOOLS: if res: raise ValueError("result on effect tag") if name == "emote" and arg not in W.EMOTES: raise ValueError("emote " + str(arg)) if name == "dream" and arg not in W.DREAMS: raise ValueError("dream " + str(arg)) if name == "want" and arg not in W.WANTS: raise ValueError("want " + str(arg)) if name == "gift" and arg not in W.GIFTS: raise ValueError("gift " + str(arg)) if name in ("remember", "diary") and not arg: raise ValueError("empty arg") if name in ("remember", "diary") and re.search(r"\b(he|she|him|his|her|hers|himself|herself)\b", arg, re.I): raise ValueError("gendered memory/note") if name == "remember" and not re.search(r"(?<!\w)" + re.escape(o) + r"(?!\w)", arg, re.I): raise ValueError("memory without owner") if name == "remember" and live and arg.strip().lower() not in (x.lower() for x in live["memories"]): live["memories"].append(arg.strip()) if name == "walk" and arg not in W.WALKS: raise ValueError("walk " + str(arg)) if name in ("sleep", "yes", "no") and arg: raise ValueError(name + " arg") segs.append((m.group(0), True)) else: raise ValueError("unknown tag " + name) last = m.end() segs.append((body[last:] + "\n", True)) if re.search(r"[\[\]{}]", W.TAG_RE.sub("", body)): raise ValueError("stray bracket") answers = [m.group(1) for m in W.TAG_RE.finditer(body) if m.group(1) in W.ANSWER_TOOLS] if len(answers) != (1 if offer_open else 0): raise ValueError("offer answer" if offer_open else "answer without offer") if offer_open == f"* {o} offered {p} medicine *" and answers == ["yes"] and f", {p} is sick" not in hdr: raise ValueError("medicine while healthy") tags = [m.group(1) for m in W.TAG_RE.finditer(body)] said = W.TAG_RE.sub("", body) if full and hunger_claim(said): raise ValueError("hunger while full") if asleep: if offer_open and answers == ["yes"]: if "energy exhausted" in hdr and "tummy starving" not in hdr and any( offer_open == f"* {o} offered {p} {W.a(f)} *" for f in W.FOODS): raise ValueError("exhausted food wake without starvation") if tags != ["yes"]: raise ValueError("sleep yes with other tags") wake_owed = True elif tags.count("dream") != 1 or set(tags) - {"dream", "no"} or len(said.split()) > 4: raise ValueError("sleep dream grammar") elif "dream" in tags: raise ValueError("awake dream") if "sleep" in tags: asleep = True if offer_open and " to play catch " in offer_open and answers == ["yes"]: catch_owed = True offer_open = None silent_run = 0 if re.search(r"[a-zA-Z]", W.TAG_RE.sub("", body)) else silent_run + 1 if silent_run > 2: raise ValueError("silent run") for s, trained in segs: if trained and s: spans.append([len(text), len(text) + len(s)]) text += s pet_lines += 1 else: raise ValueError("bad line: " + ln[:60]) if pet_lines < 2: raise ValueError("few pet lines") if offer_open: raise ValueError("unanswered offer") if wake_owed: raise ValueError("sleep offer missing wake") return text, spans
def check_diary(st, raw): return check_diary_text(W.header("diary", st), raw)
def check_diary_text(hdr, raw): """Validate a diary under an existing header; return (text, spans).""" body = check_diary_body(raw) text = hdr + "dear diary," spans = [[len(text), len(text) + len(body) - len("dear diary,")]] text += body[len("dear diary,"):] return text, spans
def check_diary_body(raw): body = raw.strip() if not body.lower().startswith("dear diary,"): raise ValueError("no opener") if "**" in body or "#" in body or len(body) > 1400: raise ValueError("format") for m in re.finditer(r"\[(\w+)(?::([^\]]*))?\]", body): if m.group(1) != "doodle" or m.group(2) not in W.DOODLES: raise ValueError("bad tag " + m.group(0)) rest = re.sub(r"\[doodle:\w+\]", "", body) if re.search(r"[\[\]]|doodle:", rest, re.I): raise ValueError("stray diary bracket") if re.search(r"(?:^|\n)\s*(?:-\s*|love,\s*)PET\s*[.!]?\s*(?:$|\n)", body): raise ValueError("literal PET signature") return re.sub(r"\n{2,}", "\n", body).replace("Dear diary,", "dear diary,")
def _span_coverage(spans, length): """Validate bounds and compare masks independently of interval layout.""" if not isinstance(spans, (list, tuple)): raise ValueError("bad spans") ranges = [] for span in spans: if not isinstance(span, (list, tuple)) or len(span) != 2: raise ValueError("bad span shape") a, b = span if type(a) is not int or type(b) is not int: raise ValueError("bad span type") if not 0 <= a <= b <= length: raise ValueError("span bounds") if a < b: ranges.append((a, b)) ranges.sort() merged = [] for a, b in ranges: if merged and a <= merged[-1][1]: merged[-1][1] = max(merged[-1][1], b) else: merged.append([a, b]) return merged
def recheck(row): """Re-validate stored canonical text and loss coverage without rewriting it. Lookup results are kept; stale rules, shifted text or corrupt masks raise ValueError with the reason. Training adds untrained history afterward.""" if "spans" not in row: raise ValueError("missing spans") coverage = _span_coverage(row["spans"], len(row["text"])) hdr, sep, body = row["text"].partition("---\n") if not sep: raise ValueError("no header") meta = dict(l.split(": ", 1) for l in hdr.splitlines()[1:] if ": " in l) o = meta["owner"] ages = re.findall(re.escape(o) + r" is (\d+) years old", meta.get("remembers", "")) if len(set(ages)) > 1: raise ValueError("conflicting remembered ages") if row["kind"] == "diary": text, spans = check_diary_text(hdr + sep, body) else: now = meta.get("now", "") if now: weather = now.split(", ")[1] allowed = W.WEATHER_NIGHT if W.part_of_day(int(now[:2])) == "night" else W.WEATHER_DAY if weather not in allowed: raise ValueError("weather for time") if any(t.startswith(o + " said: ") for t in meta.get("today", "").split("; ")): raise ValueError("owner said in chat header") text, spans = check_body(o, meta["pet"].split(" · ")[0], hdr + sep, body, None) if text != row["text"]: raise ValueError("noncanonical text") if coverage != _span_coverage(spans, len(text)): raise ValueError("loss mask mismatch")
SLEEP_SCENARIOS = ["talk", "talk", "touch", "light", "offer", "offer", "offer", "clean", "idle", "nap"]
SLEEP_SPEC = """Write the pet's lines for a tiny virtual pet chat. The app already owns the events and wake rolls.Return ONLY a JSON array of strings, one string for each numbered PET TURN, in order. No names or events inside the strings.
Rules:- DREAM turn: exactly one [dream:X], optional mumble of at most FOUR words, no other tags. Examples of the format: "mmm... cookies... [dream:cookie]", "[dream:heart]", "zzz [dream:cloud]". Vary wording and dreams.- DREAM X must be one of DREAMS. Echo the day, memories or words heard with the closest allowed sprite; never invent values.- AWAKE turn: short sincere words, optionally [emote:X] (EMOTES) or [want:X] (WANTS). No dream, sleep or answer tags.- NAP turn: short sleepy words ending with [sleep]. No dream in this line; sleep starts AFTER it.- OFFER turn (asleep): choose whether to wake and accept. Refusing is a DREAM line plus exactly one [no]. A refusal MUST contain BOTH tags in that same string: e.g. "zzz [dream:cookie] [no]". A bare [no] is invalid. Accepting is short sleepy words plus exactly one [yes], NO dream or other tags. The app will wake the pet after [yes]. Usually sleep through offers. Hungry/starving pets may wake for food; cookie, pudding and strawberry are sweet and more tempting, especially for a foodie. If exhausted, only wake for food if starving. Full/stuffed pets usually refuse food. Rarely wake for bath or catch. Sick pets may wake for medicine; healthy pets refuse medicine.- Needs are true. Never say hungry/starving if full/stuffed; never invent prior meals or other facts.- Never use three silent (tags-only) pet lines in a row: include a tiny mumble.- No brackets except valid tags, no asterisks or stage directions. Use the owner's name or you, never assume gender.""".replace("DREAMS", ", ".join(W.DREAMS)).replace("EMOTES", ", ".join(W.EMOTES)).replace("WANTS", ", ".join(W.WANTS))
def sleep_scenario(st): """Fixed app events and sampled wake rolls; the teacher writes every pet line.""" o, p = st["owner"], st["pet"] kind = random.choice(SLEEP_SCENARIOS) st["asleep"] = kind != "nap" st["stats"]["energy"] = random.randint(7, 35) if kind == "nap" else random.randint(0, 95) if 7 <= st["hh"] < 11: st["hh"] = 22 st["weather"] = random.choice(W.WEATHER_NIGHT) asleep = st["asleep"] slots = [] if kind in ("talk", "touch", "light"): for n in range(1, random.randint(2, 4) + 1): line = (f"{o}: {random.choice(['goodnight little one', 'can you hear me?', 'dream of cookies', 'i love you', 'hello sleepyhead'])}" if kind == "talk" else f"* {o} patted {p} *" if kind == "touch" else f"* {o} turned on the lights *") if kind == "light" and n > 1: slots.append(([f"* {o} turned off the lights *"], "DREAM" if asleep else "AWAKE")) wakes = asleep and random.random() < W.wake_chance(st["stats"]["energy"], kind, n) events = [line] if wakes: events.insert(1 if kind == "light" else 0, f"* {p} woke up *") asleep = False slots.append((events, "DREAM" if asleep else "AWAKE")) elif kind == "offer": thing = random.choice(W.FOODS + ["cookie", "strawberry", "carrot", "bath", "medicine", "catch"]) if thing == "medicine": st["sick"] = random.random() < 0.8 line = (f"* {o} asked {p} to play catch *" if thing == "catch" else f"* {o} offered {p} " + ("medicine" if thing == "medicine" else W.a(thing)) + " *") slots = [([], "DREAM"), ([], "DREAM"), ([line], "OFFER")] elif kind == "nap": slots = [([f"* {o} turned off the lights *"], "NAP"), ([f"* {o} cleaned up the poop *"], "DREAM"), ([], "DREAM")] else: events = [f"* {o} cleaned up the poop *"] if kind == "clean" else [] slots = [(events, "DREAM"), ([], "DREAM"), ([], "DREAM")] return slots
def make_sleep_chat(teacher, st): slots = sleep_scenario(st) prompt = (SLEEP_SPEC + "\nTASK\n" + W.header("chat", st) + "\nVoice: " + STAGE_STYLE[st["stage"]] + "\nWrite these pet turns, in order:\n" + "\n".join( f"PET TURN {i + 1} ({mode}), after: " + (" / ".join(events) or "quiet idle time") for i, (events, mode) in enumerate(slots))) replies = json.loads(teacher(prompt, max_tokens=600)) if not isinstance(replies, list) or len(replies) != len(slots) or not all(isinstance(x, str) for x in replies): raise ValueError("sleep reply count") lines = [] for (events, mode), reply in zip(slots, replies): if "\n" in reply or (mode == "NAP" and "[sleep]" not in reply) or (mode == "AWAKE" and "[sleep]" in reply): raise ValueError("sleep turn mode") lines.extend(events) lines.append(st["pet"] + ": " + reply) if mode == "OFFER" and "[yes]" in reply: lines.append(f"* {st['pet']} woke up *") text, spans = check_chat(st, "\n".join(lines)) return {"kind": "chat", "stage": st["stage"], "text": text, "spans": spans}
def make_chat(teacher): st = rand_state() if FOCUS == "sleep" or (st["asleep"] and FOCUS not in ("memory", "recall")): return make_sleep_chat(teacher, st) o, p = st["owner"], st["pet"] n_lines = random.choice([4, 6, 6, 8, 8, 10, 12]) picks = random.sample(SCENARIOS, random.choice([1, 2, 2, 3])) target = recall_target(st) if FOCUS == "recall" else None if FOCUS == "memory": st["asleep"] = False if FOCUS == "memory": picks[0] = random.choice(MEMORY_SCENARIOS) elif target: picks = [target["scenario"]] + picks[:random.choice([0, 1])] elif random.random() < ACTION_SHARE and not any(s in ACTION_SCENARIOS for s in picks): picks[0] = random.choice(ACTION_SCENARIOS) scen = "; ".join(s.format(o=o, p=p) for s in picks) prompt = SPEC.format( o=o, p=p, header=W.header("chat", st), events="\n".join(W.event_examples(o, p)), lines=n_lines, min_pet=n_lines // 2, traits=", ".join(st["traits"]), style=STAGE_STYLE[st["stage"]], emotes=", ".join(W.EMOTES), wants=", ".join(W.WANTS), gifts=", ".join(W.GIFTS), clock=W.clock(st), look=W.look(st), allmem="; ".join(st["memories"]) or "(none)", yesterday=st["yesterday"] or "no diary yet", scenarios=scen) text, spans = check_chat(st, teacher(prompt)) if target: check_recall(target, text, p) return {"kind": "chat", "stage": st["stage"], "text": text, "spans": spans}
def make_diary(teacher): st = rand_state() if len(st["today"]) < 2: st["today"] += ["played catch (3 stars)", f"{st['owner']} said: {random.choice(QUOTES)}"] length = {"baby": "2-4 tiny babbly sentences.", "kid": "3-6 short sentences.", "teen": "3-5 short sentences; stay brief if the log is short.", "adult": "4-8 sentences."}[st["stage"]] prompt = DIARY_SPEC.format(o=st["owner"], p=st["pet"], header=W.header("diary", st), length=length, traits=", ".join(st["traits"]), style=STAGE_STYLE[st["stage"]], doodles=", ".join(W.DOODLES)) text, spans = check_diary(st, teacher(prompt, max_tokens=500)) return {"kind": "diary", "stage": st["stage"], "text": text, "spans": spans}
def main(): ap = argparse.ArgumentParser() ap.add_argument("out") ap.add_argument("chats", type=int) ap.add_argument("diaries", type=int) ap.add_argument("--endpoints", default=os.environ.get("TEACHER_ENDPOINTS"), help="comma-separated base URLs of OpenAI-compatible servers") ap.add_argument("--model", default=os.environ.get("TEACHER_MODEL"), help="model name to request from the servers") ap.add_argument("--workers", type=int, default=4) ap.add_argument("--seed", type=int, default=0) ap.add_argument("--focus", choices=["memory", "recall", "sleep"], help="build every chat around this behavior") ap.add_argument("--stage", choices=W.STAGES, help="force the life stage for chats and diaries") args = ap.parse_args() global FOCUS, STAGE FOCUS, STAGE = args.focus, args.stage if not args.endpoints or not args.model: ap.error("--endpoints and --model are required (or TEACHER_ENDPOINTS and TEACHER_MODEL)") random.seed(args.seed) teacher = Teacher(args.endpoints.split(","), args.model) jobs = [make_chat] * args.chats + [make_diary] * args.diaries random.shuffle(jobs) ok = bad = 0 reasons = {} with open(args.out, "a") as f, cf.ThreadPoolExecutor(args.workers) as ex: futs = [ex.submit(j, teacher) for j in jobs] for fu in cf.as_completed(futs): try: row = fu.result() f.write(json.dumps(row) + "\n") f.flush() ok += 1 except ValueError as e: bad += 1 k = str(e).split(":")[0] reasons[k] = reasons.get(k, 0) + 1 except Exception as e: # network etc. bad += 1 reasons[type(e).__name__] = reasons.get(type(e).__name__, 0) + 1 if (ok + bad) % 50 == 0: print(f"ok={ok} bad={bad} {reasons}", file=sys.stderr, flush=True) print(f"done ok={ok} bad={bad} {reasons}", file=sys.stderr)
if __name__ == "__main__": main()