Something went wrong. Try again.
a tiny pet with a GPT-2 brain, living in your browser
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128"""Migrate v5 rows to v6, replacing old teen rows with separately generated data.
Usage: python3 train/migrate_v6.py OLD.jsonl OUT.jsonl [NEW.jsonl ...] [--exclude FILE]No environment variables. Drops old teen rows, filters chat-header ownerquotes, inserts only app-possible wake events, rebuilds loss spans and checksevery row. Prints source counts and rejection reasons. Inputs are untouched.Optional FILE is a JSON object mapping SHA-256 of row text to a confirmedmanual-review rejection reason. Hashes match before or after migration."""import argparseimport collectionsimport hashlibimport jsonimport re
import gen_data as Gimport world as W
def migrate(row): hdr, sep, body = row['text'].partition('---\n') if not sep: raise ValueError('no header') if row['kind'] == 'diary': text, spans = G.check_diary_text(hdr + sep, body) else: meta = dict(l.split(': ', 1) for l in hdr.splitlines()[1:] if ': ' in l) o, p = meta['owner'], meta['pet'].split(' · ')[0] lines = [] for ln in hdr.splitlines(): if ln.startswith('today: '): today = [t for t in ln[7:].split('; ') if not t.startswith(o + ' said: ')] if not today: continue ln = 'today: ' + '; '.join(today) lines.append(ln) hdr = '\n'.join(lines) + '\n---\n' try: # A current-valid sleeping transcript already records no-wake rolls. text, spans = G.check_body(o, p, hdr, body, None) except ValueError as e: if str(e) not in ('sleep dream grammar', 'sleep offer missing wake'): raise asleep = f', {p} is asleep' in hdr woke = f'* {p} woke up *' offer = False pending_wake = None fixed = [] body_lines = [ln.strip() for ln in body.splitlines() if ln.strip()] offers = [re.compile(e) for e in W.offer_patterns(o, p)] for i, ln in enumerate(body_lines): nxt = body_lines[i + 1] if i + 1 < len(body_lines) else '' # Defer the possible roll until the reply proves a wake is needed. # A dream consumes a no-wake outcome, including in mixed old logs. if asleep and pending_wake is None: if ln.startswith(o + ': ') or ln == f'* {o} patted {p} *': pending_wake = len(fixed) elif ln == f'* {o} turned on the lights *': pending_wake = len(fixed) + 1 fixed.append(ln) if ln == woke: asleep = False pending_wake = None offer = False elif ln.startswith('* '): offer = any(e.fullmatch(ln) for e in offers) elif ln.startswith(p + ': '): tags = [m.group(1) for m in W.TAG_RE.finditer(ln)] if asleep and offer and 'yes' in tags: if nxt != woke: fixed.append(woke) asleep = False elif asleep and 'dream' not in tags and pending_wake is not None: fixed.insert(pending_wake, woke) asleep = False if 'sleep' in tags: asleep = True pending_wake = None offer = False text, spans = G.check_body(o, p, hdr, '\n'.join(fixed), None) # Migration is the rewrite path; recheck itself never repairs stored masks. out = {**row, 'text': text, 'spans': spans} G.recheck(out) return out
def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument('old') ap.add_argument('out') ap.add_argument('new', nargs='*') ap.add_argument('--exclude', help='JSON object: SHA-256 of row text -> confirmed review reason') args = ap.parse_args() old, out, new = args.old, args.out, args.new excluded = json.load(open(args.exclude)) if args.exclude else {} def reviewed(row): reason = excluded.get(hashlib.sha256(row['text'].encode()).hexdigest()) if reason: raise ValueError('review: ' + reason) counts = collections.Counter() rows = [] for source in [old] + new: for line in open(source): row = json.loads(line) try: reviewed(row) if source == old: if row.get('stage') == 'teen' or ' · teen · ' in row['text'].split('---\n')[0]: raise ValueError('old teen replaced') row = migrate(row) reviewed(row) G.recheck(row) rows.append(row) reason = 'kept' except ValueError as e: reason = str(e).split(':')[0] counts[(source, row['kind'], reason)] += 1 with open(out, 'w') as f: for row in rows: f.write(json.dumps(row) + '\n') for (source, kind, why), n in sorted(counts.items()): print(f'{n:6} {source} {kind}: {why}') print(f'{len(rows)} rows written')
if __name__ == '__main__': main()