From 6b0c80e7b33d23d98b3559ddf1238c42a94c0cc6 Mon Sep 17 00:00:00 2001 From: ilo Niki Date: Wed, 7 Oct 2026 16:34:53 -0700 Subject: [PATCH] Fix browser lifecycle, training masks, and deployment safety Prevent duplicate catch-up simulation and recover cleanly from blocked storage or synchronous worker startup failures. Validate stored supervision masks and preserve valid sleeping transcripts while rebuilding migration masks. Stage site builds and model/runtime publication before commit, refresh preview runtimes, and add regression coverage and updated documentation. --- AGENTS.md | 4 + README.md | 1 + docs/architecture.md | 3 +- docs/deployment.md | 15 ++- docs/export-and-runtime.md | 14 ++- docs/gotchas.md | 29 ++++- docs/pet-simulation.md | 9 +- docs/testing.md | 39 +++++- docs/training.md | 24 +++- site/build.sh | 24 ++-- test/data_rules.py | 190 ++++++++++++++++++++++++++++- test/dev-site.sh | 12 +- test/site_tools.py | 239 +++++++++++++++++++++++++++++++++++++ train/gen_data.py | 69 ++++++++--- train/migrate_v6.py | 101 +++++++++------- train/publish.sh | 33 +++-- web/pet.js | 23 ++-- 17 files changed, 721 insertions(+), 108 deletions(-) create mode 100644 test/site_tools.py diff --git a/AGENTS.md b/AGENTS.md index 2cf0861..2ca23a7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,10 @@ tolerates invented tags; fix the data instead. Every row is re-checked against the current rules (`gen_data.recheck`) before training: `train.py` refuses data with stale rows unless given `--drop-stale`. +Stored text must be canonical and loss spans must match the parser's trained +character coverage, excluding owner/event lines, headers and lookup results. +`recheck` refuses corrupt masks; explicit migration rebuilds them. Training-only +untrained history is added after re-checking. When the app's behavior or a rule changes, add the rule to `check_body` in `train/gen_data.py` so rows written for the old behavior are caught. Never switch the check off to get a run through. diff --git a/README.md b/README.md index 4dfff43..d0c98a5 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Mochi: [clock]{21:40 night} it's so late!! [emote:sleepy] [want:snack] | `train/check_onnx.py` | greedy-decode check of an export against torch | | `test/parity.mjs`, `test/parity_states.py` | JS/Python prompt parity | | `test/data_rules.py` | training-contract and migration regression cases, including a stand-in sleep teacher | +| `test/site_tools.py` | stdlib regressions for safe build/publish failures, immutable artifacts and preview runtime refresh | | `test/contract.mjs` | checks the app, `site/index.html` and `docs/host-page.md` agree on element ids and config keys | | `test/dev-site.sh`, `test/serve.py` | local preview: builds the site, stages the runtime and models and serves them with COOP/COEP headers | | `test/bench.html` | brain load time, speed and memory in a real browser (included as unlinked `bench.html` in every site build) | diff --git a/docs/architecture.md b/docs/architecture.md index 7c2ebb7..4b290eb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -41,7 +41,8 @@ browser | `web/scene.js` | Draws the LCD: window with weather, shelf of gifts, poop, the pet (per-stage size, decorations, face, animations), the egg, effects and the minigame. | | `web/sprites.js` | 8x8 pixel icons as strings, canvas and `` helpers, and a 3x5 pixel font. | | `train/migrate_v6.py` | Rewrites old chat headers and sleep events, rebuilds loss spans, removes old teen rows and validates the combined dataset. | -| `test/data_rules.py` | Regression cases for sleep, offers, validation and migration; includes a stand-in sleep teacher. | +| `test/data_rules.py` | Regression cases for sleep, offers, validation, loss masks and migration; includes a stand-in sleep teacher. | +| `test/site_tools.py` | Stdlib-only regressions that exercise the real build, publish and preview scripts with disposable model/runtime stand-ins. | | `site/sw.js` | Service worker beside the page: keeps what the page lists for offline use (cache first; the page itself network first), shows due alerts, opens the app from one. | | `site/build.sh` | Fills `site/index.html` and `test/bench.html` from `site/release.json`, copies the app under its hash, the assets and the worker, and writes `_headers`. | diff --git a/docs/deployment.md b/docs/deployment.md index 3dad300..f0110bb 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -25,9 +25,18 @@ and how to ship and roll back a model. For the markup the app expects, see | Runtime | `train/vendor.sh OUT` | pinned transformers.js and its matching ONNX Runtime build | | Install model and runtime | `STATIC_ROOT=… train/publish.sh MODEL_DIR MODEL_ID RUNTIME_DIR RUNTIME_NAME` | verifies checksums first; `BASE` (default `tokenling`), `FILE_MODE` (default 0644) and `DIR_MODE` (default 0755) are optional | -`publish.sh` stages into `.partial`, renames when complete, and refuses to -replace a model directory that already exists. To build into the same tree, -point `site/build.sh` at `$STATIC_ROOT/$BASE`. +`site/build.sh` copies and renders the new static files in a temporary sibling +directory before replacing output entries. A copy, release-validation or +render failure during staging leaves the previous output untouched; `models/` +and `runtime/` are preserved on successful builds too. The final multi-entry +installation is not an atomic directory swap. + +`publish.sh` finishes and checks the model partial and any needed runtime +partial before publishing either. A new runtime is renamed first, then the +model; a runtime-copy failure removes the partials without consuming the model +id, so the same command can be retried. Existing runtime directories are left +alone, and an existing model directory is never replaced. To build into the +same tree, point `site/build.sh` at `$STATIC_ROOT/$BASE`. ### `site/release.json` diff --git a/docs/export-and-runtime.md b/docs/export-and-runtime.md index cabf2d2..b652bd9 100644 --- a/docs/export-and-runtime.md +++ b/docs/export-and-runtime.md @@ -157,12 +157,14 @@ half-finished upgrade can resume). "Delete brain" in settings removes them all. A 90-second watchdog, re-armed by each progress message, retries a stalled load once with `numThreads = 1` (no `SharedArrayBuffer` needed), then reports "the model runtime did not start". A load error is reported directly. Not -exercised against a real stall. Both reported load errors and worker crashes -terminate the worker and cancel the watchdog, so a manual retry starts a fresh -worker with fresh progress. The egg shows a retry button; settings also retain -the download button. Retrying keeps saved pet/egg state and completed cached -files. Stream and ONNX protobuf-parse errors evict only the failing brain's -cached ONNX, so retry can replace an incomplete/corrupt download. Unrelated +exercised against a real stall. Reported load errors, worker crashes and +synchronous failures constructing the worker or sending its initial load +message all cancel the watchdog and release any created worker, so a manual +retry starts a fresh worker with fresh progress. The egg shows a retry button; +settings also retain the download button. Retrying keeps saved pet/egg state +and completed cached files. Stream and ONNX protobuf-parse errors evict only +the failing brain's cached ONNX, so retry can replace an incomplete/corrupt +download. Unrelated runtime failures retain it. A failed stream starts that file over; downloads do not resume by range. diff --git a/docs/gotchas.md b/docs/gotchas.md index 7a1da84..cae53fd 100644 --- a/docs/gotchas.md +++ b/docs/gotchas.md @@ -1,7 +1,7 @@ # Gotchas -Each of these cost real time while building `tokenling-v1` to `-v5`. Format: what you -see, why, and what to do. +Each of these cost real time while building or maintaining tokenling. Format: +what you see, why, and what to do. ## Model and prompting @@ -93,6 +93,18 @@ rows at the next run. Anything that consumes `random` before the shuffle, or a changed row count, changes the split. +**A valid transcript can still have a corrupt loss mask.** Checking a +canonicalized body without checking the stored text and `spans` accepts empty, +shifted or owner/header/result-trained masks. `recheck` now compares canonical +text and semantic span coverage and refuses these rows; explicit migration +rebuilds the masks instead. Training-only donor history is added afterward +and stays untrained. + +**Sleep migration drops a valid dream after a message or pat.** A disturbance +can fail to wake the pet. Preserve already-valid no-wake transcripts, and in +legacy mixed logs insert a wake only where an awake reply needs it. Forcing a +wake at every disturbance turns later dream tags into invalid awake replies. + ## GPUs **A shared model server is too small for bulk generation** (a handful of @@ -197,6 +209,12 @@ must wait for `brain.state === "ready"` after "get an egg". **You cannot catch an animation with separate CLI calls.** Trigger and screenshot inside a single `run-code`. +**Stats jump twice when returning to a tab.** Catch-up replay and the live +animation-frame tick have separate clocks. After replaying an absence, +advance the live baseline too; otherwise its next frame applies that same +elapsed time again and can bypass the seven-day cap. Short gaps not replayed +by catch-up must still reach the live tick once. + **A storage change made in a test vanishes.** The running app saves the whole pet every second, so a `localStorage.setItem` on the pet page is overwritten before a reload reads it. Write from another same-origin page that does not run @@ -222,6 +240,13 @@ chips or an ellipsis. Inspect `window.tokenling.state().chat` for the raw text. `immutable`; a changed file under the same URL is ignored by every browser that has it. Publish a new directory name. +**A rejected build deletes the previous page, or a failed publish cannot be +retried.** Do not replace live output before staging and validating a build. +Likewise, finish the runtime copy before committing an immutable model id; +otherwise a runtime failure strands the model and the existing-id guard +blocks retry. The builder stages first, and the publisher completes both +partials before publishing the runtime and then the model. + **Static files unreadable by the web server.** `train/publish.sh` writes files `0644` and directories `0755` by default. If your server runs as a different user in a shared group, set `FILE_MODE` and `DIR_MODE` (for example `0640` and diff --git a/docs/pet-simulation.md b/docs/pet-simulation.md index 50d623f..af97523 100644 --- a/docs/pet-simulation.md +++ b/docs/pet-simulation.md @@ -47,7 +47,10 @@ While sick, `fun` falls twice as fast. The tick runs once a second. After the tab was hidden or closed, `catchUp()` replays up to seven days in 10-minute steps and writes what happened into the day log ("was very hungry", "was stinky", "pooped", "napped", " was away for a while"). Away for 30 -minutes or more adds `* OWNER came back after … *` and a reaction. +minutes or more adds `* OWNER came back after … *` and a reaction. Catch-up +also advances the live tick's clock, so the next animation frame does not +apply the same elapsed time again; gaps of two minutes or less are still +handled once by the live tick. - **Poop:** every 2.5–5 awake hours, up to four. Each costs 15 cleanliness. - **Sickness:** three or more poops, or `tummy` or `clean` below 5, for an hour @@ -218,7 +221,9 @@ for a Web Lock named after the save key (`ifAvailable`). A tab that does not get it shows "Your pet is open in another tab" and never loads, ticks or saves the pet; "use it here" reloads that tab and steals the lock. The tab that lost the lock stops at once (no more saves, brain stopped) and shows the same -notice. Browsers without Web Locks run every tab as before. +notice. Browsers without Web Locks run every tab as before. Blocked +`sessionStorage` does not stop the app from booting: it still checks the lock +normally, but cannot retain the "use it here" takeover marker across reload. Export, import and start over are on the "settings" tab. Start over is a hard reset: it removes the `tokenling.v1` key synchronously, blocks further saves, diff --git a/docs/testing.md b/docs/testing.md index b556e61..fbf3f9c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5,9 +5,10 @@ | Prompt parity | `node test/parity.mjs` | JS/Python drift in headers, stat words, word lists, look/clock/recall, event lines and offers | | Model behavior | `train/eval_pet.py CKPT 40` | invalid tags, unresolved lookups, unanswered offers (including after an opener that skipped the answer), decisions that ignore the tummy, chatty idle turns, diaries that do not end | | Quantization | `train/quant_eval.py CKPT EXPORT data.jsonl` | 4-bit quality loss | -| Data rules | `python3 test/data_rules.py` | sleep transitions, offers, header filtering, memory and diary rules, migration masks and a stand-in sleep teacher | +| Data rules | `python3 test/data_rules.py` | sleep/offer rules, corrupt or shifted supervision, equivalent span coverage, diary masks, no-wake/mixed migration and a stand-in sleep teacher | | Page contract | `node test/contract.mjs` | element ids or config keys the app uses but `site/index.html` or `host-page.md` lacks | | Site build | `site/build.sh /tmp/site-out` | unfilled placeholders, a config block that is not valid JSON | +| Script lifecycle | `python3 test/site_tools.py` | failed-build preservation, publication retry/immutability and preview runtime refresh | | Brain speed and memory | `/bench.html` in every site build | load time, decode tokens/s, prefill time, page memory | | Browser flow | `playwright-cli` against `test/dev-site.sh` or the live URL | everything the visitor sees | @@ -15,6 +16,13 @@ There is no unit-test runner for the browser code; the app is checked by driving a real browser with `playwright-cli` (run `playwright-cli --help` or see the skill for the full command list). +`test/site_tools.py` uses Python's standard library and the actual shell +scripts with disposable model/runtime stand-ins. It needs no downloaded +weights, teacher, GPU or browser. It exercises failed release/config/render +builds without destroying old output, runtime-publication failure followed by +same-id retry, immutable artifacts and permission overrides, and repeat +previews over HTTP, including obsolete runtime removal and `--rebuild`. + ## Preview server ```sh @@ -27,6 +35,9 @@ The script writes a release description for the preview and runs `site/build.sh` (the same build as a real deploy, including `test/bench.html`), symlinks the exported model, copies the runtime, and serves everything with the COOP/COEP headers, so `crossOriginIsolated` is true and WASM threads work. +A full run refreshes the local runtime from `RUNTIME`, removing obsolete files +only after the new copy succeeds. `--rebuild` changes page/app files only and +leaves the staged runtime and models alone. Options are environment variables, documented in the script header: `PORT`, `BASE` (the URL base path, default `pet`), `MODEL_ID`, `OLDER` (export directories staged as older brains), `NEWS`, `FEATURES`, and `TRAINED` and @@ -86,6 +97,32 @@ release the worker so retry can start. Test the retry row at 360 px. An interrupted response may instead report `protobuf parsing failed`; check that the bad ONNX entry is evicted and retry downloads a fresh copy. +### Catch-up and restricted browser APIs + +Use a disposable profile for controlled-clock checks. Freeze `Date.now()`, +let an animation frame establish the live tick baseline, then advance it and +dispatch `visibilitychange` while the document is visible. Set an awake pet's +energy to 80 with other needs comfortable and no poop due: after a three-hour +return energy should be 65 and stay there on the next frame, not decay twice. +A one-minute gap should still be applied once by the live tick. With lights +off, a sleeping pet at energy 10 reaches 21 after half an hour, without a +second recovery on the next frame. Also check that a ten-day absence consumes +only the capped seven-day replay and is not reapplied by the live tick. Reload +and discard the test save after replacing the clock. + +Before submitting "get an egg", inject a throwing `Worker` constructor, then +repeat with a real worker whose initial `postMessage` throws. Both should +reach `brain.state === "error"` immediately, release the worker and expose +`#setup-retry`. Restore the native API and retry: a new worker must start with +the same egg and edited pet name. This exercises startup recovery without +requiring model weights; successful download/inference needs the regular +model preview below. + +For blocked storage, use `page.addInitScript` to make the `sessionStorage` +getter throw `SecurityError` before loading the page. The app must still boot +without an uncaught error or horizontal overflow at 360 px. Check normal +two-tab ownership and "use it here" separately with session storage available. + ### Regular flows Use a **named session with a persistent profile** whenever the brain is diff --git a/docs/training.md b/docs/training.md index 666ddfb..670a527 100644 --- a/docs/training.md +++ b/docs/training.md @@ -116,6 +116,12 @@ an offer, then owner lines with tags and invented emote values. **Re-checking stored rows (`recheck`).** `gen_data.recheck(row)` applies the same line rules to a stored row, keeping the lookup results already in it. +It also requires canonical text and a valid loss mask: span endpoints must be +integers within the text, and their combined coverage must match the pet +words/tags or diary body rebuilt by the parser. Equivalent merged, split, +reordered or overlapping intervals are accepted; missing, shifted or incorrect +coverage is rejected with `ValueError`, not silently repaired. Owner/event +lines, headers, pet-name prefixes and lookup results remain untrained. `train.py` re-checks every row before training and refuses to start if any row fails, listing the reasons; `--drop-stale` drops those rows instead. `mix_data.py` and `quant_eval.py` apply the same check, so data written for an @@ -124,10 +130,15 @@ the app's behavior changes, add the rule to `check_body` and the next run reports every older row it catches. **v6 migration.** `python3 train/migrate_v6.py OLD.jsonl OUT.jsonl NEW.jsonl …` -removes all old teen rows, filters visible chat-header quotes, inserts possible -wake outcomes before old messages/pats or after lights on, and wakes after an -asleep `[yes]`. It then rebuilds text and loss spans with the current parser -and rejects remaining stale rows. It does not fabricate replacement dreams. +removes all old teen rows and filters visible chat-header quotes. It first +rebuilds valid chat bodies without inventing events, preserving failed wake +rolls and their dream replies. For legacy bodies needing a wake, it inserts a +possible wake before a message/pat or after lights on only when an awake reply +requires one; earlier no-wake dreams in mixed transcripts are retained. It +also wakes after an asleep `[yes]`. Chat and diary text and loss spans are +rebuilt with the current parser before re-checking, repairing their masks only +in this explicit migration path and rejecting remaining stale rows. It does +not fabricate replacement dreams. Old headers store only six log entries, so earlier filtered-out history cannot be recovered. Every new row is also checked; counts are printed by source, kind and first rejection reason. Optional `--exclude FILE` reads a JSON @@ -141,8 +152,9 @@ Donor text remains untrained. This prevents stale donor naps from contradicting the trained awake continuation. **Spans.** Each row stores `text` plus `spans`, the character ranges that carry -loss: the pet's words and tags, and the diary body. Owner lines, event lines, -headers and `{results}` are excluded. +loss: the pet's words and tags, and the diary body after the fixed +`dear diary,` prefix. Owner lines, event lines, headers, pet-name prefixes and +`{results}` are excluded. ### Running the teacher diff --git a/site/build.sh b/site/build.sh index 41dc4ad..c955663 100755 --- a/site/build.sh +++ b/site/build.sh @@ -19,6 +19,8 @@ # characters); overrides release analyticsToken. Empty disables it. # MODEL_ID, SIZE override the release's model (dev-site.sh uses these) # OUT_DIR is replaced except for models/ and runtime/, which are left alone. +# Copying, rendering and validation finish in a sibling staging directory before +# existing output is changed; installing the staged entries is not atomic. set -euo pipefail [ $# -eq 1 ] || { sed -n '2,/^set -e/p' "$0" | sed '$d;s/^# \{0,1\}//' >&2; exit 2; } HERE=$(cd "$(dirname "$0")" && pwd); REPO=$(dirname "$HERE") @@ -29,14 +31,16 @@ case "$BASE" in /*/|/) ;; *) echo "BASE must start and end with /" >&2; exit 2;; # App modules go under a content-hashed path so a browser never mixes versions. VER=$(cat "$REPO"/web/*.js | sha256sum | cut -c1-12) -mkdir -p "$OUT" -find "$OUT" -mindepth 1 -maxdepth 1 ! -name models ! -name runtime -exec rm -rf {} + -mkdir -p "$OUT/app/$VER" "$OUT/assets" -cp "$REPO"/web/*.js "$OUT/app/$VER/" -cp "$HERE"/assets/* "$OUT/assets/" -cp "$HERE/sw.js" "$OUT/sw.js" +mkdir -p "$(dirname "$OUT")" +STAGE=$(mktemp -d "$(dirname "$OUT")/.$(basename "$OUT").partial.XXXXXX") +trap 'rm -rf "$STAGE"' EXIT +trap 'exit 1' HUP INT TERM +mkdir -p "$STAGE/app/$VER" "$STAGE/assets" +cp "$REPO"/web/*.js "$STAGE/app/$VER/" +cp "$HERE"/assets/* "$STAGE/assets/" +cp "$HERE/sw.js" "$STAGE/sw.js" -python3 - "$HERE/index.html" "$OUT" "$RELEASE" "$BASE" "$VER" "$REPO/test/bench.html" <<'PY' +python3 - "$HERE/index.html" "$STAGE" "$RELEASE" "$BASE" "$VER" "$REPO/test/bench.html" <<'PY' import json, os, re, sys page, out, release, base, ver, bench = sys.argv[1:] r = json.load(open(release)) @@ -93,4 +97,10 @@ open(f"{out}/_headers", "w").write(f"""{base}* Cache-Control: no-cache """) PY + +# Keep published model/runtime directories, replacing the other entries only +# after every staged file has been copied and both pages have rendered. +mkdir -p "$OUT" +find "$OUT" -mindepth 1 -maxdepth 1 ! -name models ! -name runtime -exec rm -rf {} + +mv "$STAGE"/* "$OUT/" echo "built $OUT (app $VER)" diff --git a/test/data_rules.py b/test/data_rules.py index 4ead525..640b57d 100644 --- a/test/data_rules.py +++ b/test/data_rules.py @@ -16,9 +16,11 @@ from unittest.mock import patch class Rules(unittest.TestCase): def setUp(self): - self.st = G.rand_state() - self.st.update(owner='Sam', pet='Mochi', asleep=True, memories=[], today=[]) - self.st['stats'].update(tummy=95, energy=10) + self.st = dict(owner='Sam', pet='Mochi', stage='kid', day=4, + traits=['sleepy', 'curious'], hh=10, mm=0, weather='sunny', + stats=dict(tummy=95, fun=50, energy=10, clean=80), + memories=[], today=[], shelf=[], toy='ball', poop=0, + asleep=True, sick=False, yesterday='') def check(self, raw, error=None): if error: @@ -92,6 +94,119 @@ class Rules(unittest.TestCase): with self.assertRaisesRegex(ValueError, 'weather for time'): G.recheck(dict(kind='chat', text=text, spans=spans)) + def stored_rows(self): + st = copy.deepcopy(self.st) + st['asleep'] = False + rows = [] + for kind, raw in [('chat', 'Sam: hello\n* Sam patted Mochi *\n' + 'Mochi: [clock] hello [emote:happy]\nMochi: hi [emote:wave]'), + ('diary', 'dear diary,\ni dreamed of cookies.\n- Mochi\n[doodle:star]')]: + text, spans = (G.check_chat if kind == 'chat' else G.check_diary)(st, raw) + rows.append(dict(kind=kind, text=text, spans=spans)) + return rows + + def test_recheck_masks(self): + for row in self.stored_rows(): + with self.subTest(kind=row['kind'], mask='canonical'): + original = copy.deepcopy(row) + G.recheck(row) + self.assertEqual(row, original) + missing = {k: v for k, v in row.items() if k != 'spans'} + shortened = copy.deepcopy(row['spans']) + shortened[0][0] += 1 + cases = [('missing', missing), + ('empty', {**row, 'spans': []}), + ('all text', {**row, 'spans': [[0, len(row['text'])]]}), + ('missing trained character', {**row, 'spans': shortened})] + for label, bad in cases: + with self.subTest(kind=row['kind'], mask=label): + original = copy.deepcopy(bad) + with self.assertRaises(ValueError): + G.recheck(bad) + self.assertEqual(bad, original) + + def test_recheck_untrained_characters(self): + for row in self.stored_rows(): + text = row['text'] + positions = [('header', 0), ('separator', text.index('---\n'))] + if row['kind'] == 'chat': + positions += [('owner', text.index('Sam: hello')), + ('event', text.index('* Sam patted Mochi *')), + ('pet prefix', text.index('Mochi: [clock]')), + ('lookup brace', text.index('{10:00 morning}')), + ('lookup result', text.index('10:00 morning}', text.index('---\n')))] + else: + positions.append(('diary opener', text.index('dear diary,'))) + for label, at in positions: + with self.subTest(kind=row['kind'], character=label): + bad = {**row, 'spans': row['spans'] + [[at, at + 1]]} + with self.assertRaisesRegex(ValueError, 'loss mask'): + G.recheck(bad) + + def test_recheck_bad_spans(self): + for row in self.stored_rows(): + n = len(row['text']) + cases = [('null', None), ('number', 1), ('boolean', True), + ('string', '0,1'), ('object', {}), ('null span', [None]), + ('short span', [[0]]), ('long span', [[0, 1, 2]]), + ('string bound', [['0', 1]]), ('float bound', [[0, 1.5]]), + ('boolean start', [[False, 1]]), ('boolean end', [[0, True]]), + ('negative bound', [[-1, 1]]), ('reversed bounds', [[1, 0]]), + ('past end', [[n, n + 1]]), ('past end empty', [[n + 1, n + 1]])] + for label, spans in cases: + with self.subTest(kind=row['kind'], spans=label): + with self.assertRaises(ValueError): + G.recheck({**row, 'spans': spans}) + + def test_recheck_equivalent_masks(self): + for row in self.stored_rows(): + merged = [] + for a, b in row['spans']: + if merged and a == merged[-1][1]: + merged[-1][1] = b + else: + merged.append([a, b]) + split = [] + for a, b in merged: + split.extend([[a, a + 1], [a + 1, b]] if b - a > 1 else [[a, b]]) + for label, spans in [('merged', merged), ('split reversed', list(reversed(split))), + ('overlapping', split + [split[0]])]: + with self.subTest(kind=row['kind'], spans=label): + equivalent = {**row, 'spans': spans} + original = copy.deepcopy(equivalent) + G.recheck(equivalent) + self.assertEqual(equivalent, original) + + def test_recheck_shifted_text(self): + chat, diary = self.stored_rows() + at = chat['text'].index('Mochi: [clock]') + indented = chat['text'][:at] + ' ' + chat['text'][at:] + shifted = [[a + 2, b + 2] for a, b in chat['spans']] + cases = [('indented, old spans', {**chat, 'text': indented}), + ('indented, shifted spans', {**chat, 'text': indented, 'spans': shifted}), + ('chat blank line', {**chat, 'text': chat['text'].replace('Sam: hello\n', 'Sam: hello\n\n')}), + ('stage directions', {**chat, 'text': chat['text'].replace('Mochi: [clock]', 'Mochi: *purrs* [clock]')}), + ('chat trailing spaces', {**chat, 'text': chat['text'].replace('Sam: hello\n', 'Sam: hello \n')}), + ('diary opener case', {**diary, 'text': diary['text'].replace('dear diary,', 'Dear diary,')}), + ('diary blank line', {**diary, 'text': diary['text'].replace('dear diary,\n', 'dear diary,\n\n')}), + ('diary trailing newline', {**diary, 'text': diary['text'] + '\n'})] + for label, bad in cases: + with self.subTest(text=label): + original = copy.deepcopy(bad) + with self.assertRaisesRegex(ValueError, 'noncanonical text'): + G.recheck(bad) + self.assertEqual(bad, original) + + def test_recheck_keeps_lookup_results(self): + self.st['asleep'] = False + text, spans = G.check_body('Sam', 'Mochi', W.header('chat', self.st), + 'Sam: hello\nMochi: [clock]{23:59 night} late!\nMochi: hello', None) + row = dict(kind='chat', text=text, spans=spans) + original = copy.deepcopy(row) + G.recheck(row) + self.assertEqual(row, original) + self.assertIn('[clock]{23:59 night}', row['text']) + def test_recall_templates(self): for _ in range(500): st=G.rand_state() @@ -110,6 +225,75 @@ class Rules(unittest.TestCase): wake = row['text'].rindex('* Mochi woke up *') self.assertTrue(all(b <= wake for a, b in row['spans'])) + def test_migration_diary_masks(self): + for raw in ['dear diary,\ni dreamed of cookies.\n- Mochi\n[doodle:star]', + ' Dear diary,\n\ni dreamed of cookies.\n- Mochi\n[doodle:star]\n ']: + text, spans = G.check_diary(self.st, raw) + legacy_text = W.header('diary', self.st) + raw + cases = [('empty', dict(spans=[])), + ('header trained', dict(spans=[[0, len(legacy_text)]])), + ('missing', {})] + for label, mask in cases: + with self.subTest(body=raw, mask=label): + old = dict(kind='diary', text=legacy_text, **mask) + original = copy.deepcopy(old) + migrated = M.migrate(old) + self.assertEqual(migrated['text'], text) + self.assertEqual(migrated['spans'], spans) + self.assertEqual(old, original) + G.recheck(migrated) + opener_end = text.index('dear diary,') + len('dear diary,') + self.assertEqual(spans, [[opener_end, len(text)]]) + + def test_migration_no_wake(self): + for action in ['Sam: hello', '* Sam patted Mochi *', '* Sam turned on the lights *']: + with self.subTest(action=action): + raw = action + '\nMochi: zzz [dream:moon]\nMochi: [dream:star]' + text, spans = G.check_chat(self.st, raw) + old = dict(kind='chat', text=text, spans=[]) + migrated = M.migrate(old) + self.assertEqual(migrated['text'], text) + self.assertEqual(migrated['spans'], spans) + self.assertNotIn('* Mochi woke up *', migrated['text']) + G.recheck(migrated) + + def test_migration_mixed_wakes(self): + actions = ['Sam: hello', '* Sam patted Mochi *', '* Sam turned on the lights *'] + for sleeping_action in actions + ['\n'.join(actions)]: + for awake_action in actions: + with self.subTest(sleeping=sleeping_action, awake=awake_action): + raw = sleeping_action + '\nMochi: zzz [dream:moon]\n' + awake_action + '\n' + raw += 'Mochi: hello Sam\nMochi: [emote:wave]' + woke_action = (awake_action + '\n* Mochi woke up *' if 'lights' in awake_action + else '* Mochi woke up *\n' + awake_action) + expected = sleeping_action + '\nMochi: zzz [dream:moon]\n' + woke_action + '\n' + expected += 'Mochi: hello Sam\nMochi: [emote:wave]' + text, spans = G.check_chat(self.st, expected) + old = dict(kind='chat', text=W.header('chat', self.st) + raw, spans=[]) + migrated = M.migrate(old) + self.assertEqual(migrated['text'], text) + self.assertEqual(migrated['spans'], spans) + self.assertEqual(migrated['text'].count('* Mochi woke up *'), 1) + self.assertLess(migrated['text'].index('[dream:moon]'), migrated['text'].index('* Mochi woke up *')) + G.recheck(migrated) + + def test_migration_sleep_yes(self): + self.st['stats']['tummy'] = 10 + offer = '* Sam offered Mochi a cookie *\nMochi: cookie? [yes]\n' + for prefix in ['', 'Sam: hello\nMochi: zzz [dream:moon]\n']: + with self.subTest(prefix=prefix): + raw = prefix + offer + 'Mochi: yummy [emote:happy]' + expected = prefix + offer + '* Mochi woke up *\nMochi: yummy [emote:happy]' + text, spans = G.check_chat(self.st, expected) + old = dict(kind='chat', text=W.header('chat', self.st) + raw, spans=[]) + migrated = M.migrate(old) + self.assertEqual(migrated['text'], text) + self.assertEqual(migrated['spans'], spans) + self.assertIn('[yes]\n* Mochi woke up *\n', migrated['text']) + G.recheck(migrated) + # Already-current rows must not acquire a second wake either. + self.assertEqual(M.migrate({**migrated, 'spans': []}), migrated) + def test_migration_standin(self): hdr = W.header('chat', self.st).replace('---\n', 'today: Sam said: hello; ate a cookie\n---\n') row = M.migrate(dict(kind='chat', text=hdr + 'Sam: hello\nMochi: [recall:cat]{nothing} hello\nMochi: hi\n', spans=[])) diff --git a/test/dev-site.sh b/test/dev-site.sh index c62823b..fd74357 100755 --- a/test/dev-site.sh +++ b/test/dev-site.sh @@ -46,8 +46,16 @@ PY RELEASE=$DIR/release.json BASE=/$BASE/ "$REPO/site/build.sh" "$ROOT" [ "${2:-}" = "--rebuild" ] && exit 0 -mkdir -p "$ROOT/runtime/runtime" -cp -n "$RUNTIME"/* "$ROOT/runtime/runtime/" +# Full previews refresh the local runtime exactly; published runtimes remain +# immutable. Finish a fresh copy before removing the previous preview runtime. +mkdir -p "$ROOT/runtime" +RUNTIME_STAGE=$(mktemp -d "$ROOT/runtime/.runtime.partial.XXXXXX") +trap 'rm -rf "$RUNTIME_STAGE"' EXIT +trap 'exit 1' HUP INT TERM +cp -a "$RUNTIME/." "$RUNTIME_STAGE/" +rm -rf "$ROOT/runtime/runtime" +mv "$RUNTIME_STAGE" "$ROOT/runtime/runtime" +trap - EXIT HUP INT TERM for m in "$MODEL" ${OLDER:-}; do id=$(basename "$(realpath "$m")"); [ "$m" = "$MODEL" ] && id=$MODEL_ID mkdir -p "$ROOT/models/$id/onnx" diff --git a/test/site_tools.py b/test/site_tools.py new file mode 100644 index 0000000..a47ced9 --- /dev/null +++ b/test/site_tools.py @@ -0,0 +1,239 @@ +"""Stdlib regressions for build, publish and preview. Run: python3 test/site_tools.py.""" +import contextlib +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import tempfile +import time +import unittest +from urllib.error import HTTPError, URLError +from urllib.request import ProxyHandler, build_opener + +HTTP = build_opener(ProxyHandler({})) +REPO = Path(__file__).resolve().parents[1] +SCRIPT_ENV = """RELEASE BASE MODELS_URL RUNTIME_URL SOURCE CF_ANALYTICS_TOKEN MODEL_ID SIZE +STATIC_ROOT FILE_MODE DIR_MODE MODEL RUNTIME OLDER PORT NEWS FEATURES TRAINED +ISOLATE ISOLATE_PREFIX ONNX_BYTES_PER_SECOND BASH_ENV ENV""".split() + + +class SiteTools(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.env = os.environ.copy() + for key in SCRIPT_ENV: + self.env.pop(key, None) + + def model(self, name, content=b"fixture model\n"): + path = self.root / name + (path / "onnx").mkdir(parents=True) + (path / "config.json").write_text('{"model_type": "fixture"}\n') + (path / "onnx/model_q4.onnx").write_bytes(content) + (path / "SHA256SUMS").write_text("".join( + f"{hashlib.sha256((path / file).read_bytes()).hexdigest()} {file}\n" + for file in ("config.json", "onnx/model_q4.onnx"))) + return path + + def runtime(self, name, label): + path = self.root / name + path.mkdir(parents=True) + (path / "transformers.min.js").write_text(f"export const fixture = '{label}';\n") + (path / "ort.wasm").write_bytes(label.encode()) + return path + + def tree(self, path): + result = {} + for entry in path.rglob("*"): + name = entry.relative_to(path).as_posix() + result[name] = (("link", os.readlink(entry)) if entry.is_symlink() else + ("dir",) if entry.is_dir() else ("file", entry.read_bytes())) + return result + + def run_script(self, name, *args, success=True, **env): + result = subprocess.run([str(REPO / name), *map(str, args)], cwd=REPO, + env={**self.env, **env}, capture_output=True, text=True, timeout=20) + message = result.stdout + result.stderr + if success: + self.assertEqual(result.returncode, 0, message) + else: + self.assertNotEqual(result.returncode, 0, message) + return result + + def config(self, page): + match = re.search(r'id="pet-config">(.*?)', page, re.S) + self.assertIsNotNone(match) + return json.loads(match.group(1)) + + def assert_manifest(self, path): + for line in (path / "SHA256SUMS").read_text().splitlines(): + digest, name = line.split(" ", 1) + self.assertEqual(hashlib.sha256((path / name).read_bytes()).hexdigest(), digest) + + @staticmethod + def fetch(url): + with HTTP.open(url, timeout=1) as response: + return response.read(), response.headers + + @contextlib.contextmanager + def preview(self, output, model, runtime): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + url = f"http://127.0.0.1:{port}" + with tempfile.TemporaryFile(mode="w+") as log: + process = subprocess.Popen([str(REPO / "test/dev-site.sh"), str(output)], cwd=REPO, + env={**self.env, "MODEL": str(model), "RUNTIME": str(runtime), + "BASE": "pet", "MODEL_ID": "demo", "PORT": str(port)}, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if process.poll() is not None: + log.seek(0) + self.fail(f"preview exited with {process.returncode}: {log.read()}") + try: + self.fetch(url + "/pet/") + break + except (URLError, TimeoutError): + time.sleep(0.05) + else: + log.seek(0) + self.fail(f"preview did not start: {log.read()}") + yield url + finally: + if process.poll() is None: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=5) + + def test_failed_build_preserves_rendered_site_and_published_directories(self): + output = self.root / "site" + self.model("site/models/demo") + self.runtime("site/runtime/fixture-runtime", "preserved") + published = {name: self.tree(output / name) for name in ("models", "runtime")} + release = {"modelId": "demo", "size": 14, "runtime": "fixture-runtime", + "trained": "fixture build", "news": "fixture release", "features": [], "older": []} + release_file = self.root / "release.json" + release_file.write_text(json.dumps(release)) + options = {"RELEASE": str(release_file), "BASE": "/pet/"} + self.run_script("site/build.sh", output, **options) + for name in published: + self.assertEqual(self.tree(output / name), published[name]) + config = self.config((output / "index.html").read_text()) + self.assertEqual(config["modelId"], "demo") + (output / "old-entry.txt").write_text("old output must survive failures\n") + before = self.tree(output) + siblings = set(self.root.iterdir()) + failures = [ + ("invalid analytics", json.dumps(release), {"CF_ANALYTICS_TOKEN": "invalid"}), + ("malformed release", "{", {}), + ("unfilled render", json.dumps({**release, "trained": "@UNFILLED@"}), {}), + ("invalid rendered config", json.dumps(release), {"MODEL_ID": 'bad"id'}), + ] + for name, content, overrides in failures: + with self.subTest(name=name): + release_file.write_text(content) + self.run_script("site/build.sh", output, success=False, **options, **overrides) + self.assertEqual(self.tree(output), before) + self.assertEqual(self.config((output / "index.html").read_text()), config) + self.assertEqual(set(self.root.iterdir()), siblings) + release_file.write_text(json.dumps({**release, "news": "rebuilt release"})) + self.run_script("site/build.sh", output, **options) + self.assertFalse((output / "old-entry.txt").exists()) + self.assertEqual(self.config((output / "index.html").read_text())["news"], "rebuilt release") + for name in published: + self.assertEqual(self.tree(output / name), published[name]) + + def test_publish_runtime_failure_allows_same_id_retry(self): + model = self.model("model") + runtime = self.runtime("runtime", "A") + root = self.root / "published" + options = {"STATIC_ROOT": str(root), "BASE": "pet", "FILE_MODE": "0640", "DIR_MODE": "0750"} + # Exercise both a missing runtime and a copy that fails after copying files. + broken = self.runtime("broken-runtime", "broken") + (broken / "z-directory").mkdir() + for source in (self.root / "missing-runtime", broken): + with self.subTest(runtime=source.name): + self.run_script("train/publish.sh", model, "demo", source, "fixture-runtime", + success=False, **options) + self.assertEqual(list((root / "pet/models").iterdir()), []) + self.assertEqual(list((root / "pet/runtime").iterdir()), []) + self.run_script("train/publish.sh", model, "demo", runtime, "fixture-runtime", **options) + published_model = root / "pet/models/demo" + published_runtime = root / "pet/runtime/fixture-runtime" + self.assertEqual(self.tree(published_model), self.tree(model)) + payload = self.tree(published_runtime) + payload.pop("SHA256SUMS") + self.assertEqual(payload, self.tree(runtime)) + for path in (published_model, published_runtime): + self.assert_manifest(path) + for entry in (path, *path.rglob("*")): + self.assertEqual(entry.stat().st_mode & 0o777, 0o750 if entry.is_dir() else 0o640) + + def test_published_model_and_existing_runtime_are_immutable(self): + model_a = self.model("model-a", b"model A\n") + model_b = self.model("model-b", b"model B\n") + runtime_a = self.runtime("runtime-a", "A") + runtime_b = self.runtime("runtime-b", "B") + root = self.root / "published" + options = {"STATIC_ROOT": str(root), "BASE": "pet"} + self.run_script("train/publish.sh", model_a, "demo", runtime_a, "fixture-runtime", **options) + before = self.tree(root) + self.run_script("train/publish.sh", model_b, "demo", runtime_b, "other-runtime", + success=False, **options) + self.assertEqual(self.tree(root), before) + self.run_script("train/publish.sh", model_b, "next", runtime_b, "fixture-runtime", **options) + self.assertEqual(self.tree(root / "pet/models/demo"), self.tree(model_a)) + self.assertEqual(self.tree(root / "pet/models/next"), self.tree(model_b)) + original_runtime = {key.removeprefix("pet/runtime/fixture-runtime/"): value + for key, value in before.items() if key.startswith("pet/runtime/fixture-runtime/")} + self.assertEqual(self.tree(root / "pet/runtime/fixture-runtime"), original_runtime) + self.assert_manifest(root / "pet/runtime/fixture-runtime") + + def test_full_preview_refreshes_runtime_but_rebuild_does_not(self): + model = self.model("model") + runtime_a = self.runtime("runtime-a", "A") + (runtime_a / "obsolete.wasm").write_bytes(b"obsolete") + runtime_b = self.runtime("runtime-b", "B") + (runtime_b / ".fixture").write_text("hidden runtime metadata\n") + (runtime_b / "nested").mkdir() + (runtime_b / "nested/fixture.txt").write_text("nested runtime metadata\n") + output = self.root / "preview" + selected = output / "pet/runtime/runtime" + with self.preview(output, model, runtime_a) as url: + body, headers = self.fetch(url + "/pet/runtime/runtime/transformers.min.js") + self.assertEqual(body, (runtime_a / "transformers.min.js").read_bytes()) + self.assertEqual(headers["Cross-Origin-Opener-Policy"], "same-origin") + self.assertEqual(headers["Cross-Origin-Embedder-Policy"], "require-corp") + self.run_script("test/dev-site.sh", output, "--rebuild", MODEL=str(model), RUNTIME=str(runtime_b), + BASE="pet", MODEL_ID="demo") + self.assertEqual(self.tree(selected), self.tree(runtime_a)) + with self.preview(output, model, runtime_b) as url: + body, _ = self.fetch(url + "/pet/runtime/runtime/transformers.min.js") + self.assertEqual(body, (runtime_b / "transformers.min.js").read_bytes()) + self.assertEqual(self.tree(selected), self.tree(runtime_b)) + with self.assertRaises(HTTPError) as error: + self.fetch(url + "/pet/runtime/runtime/obsolete.wasm") + with error.exception: + self.assertEqual(error.exception.code, 404) + previous = self.tree(selected) + siblings = set(selected.parent.iterdir()) + self.run_script("test/dev-site.sh", output, success=False, MODEL=str(model), + RUNTIME=str(self.root / "missing-runtime"), BASE="pet", MODEL_ID="demo") + self.assertEqual(self.tree(selected), previous) + self.assertEqual(set(selected.parent.iterdir()), siblings) + + +if __name__ == "__main__": + unittest.main() diff --git a/train/gen_data.py b/train/gen_data.py index 11d5c9a..591351a 100644 --- a/train/gen_data.py +++ b/train/gen_data.py @@ -596,8 +596,12 @@ def check_body(o, p, hdr, raw, live): def check_diary(st, raw): + return check_diary_text(W.header("diary", st), raw) + + +def check_diary_text(hdr, raw): + """Validate a diary under an existing header; return (text, spans).""" body = check_diary_body(raw) - hdr = W.header("diary", st) text = hdr + "dear diary," spans = [[len(text), len(text) + len(body) - len("dear diary,")]] text += body[len("dear diary,"):] @@ -621,29 +625,62 @@ def check_diary_body(raw): return re.sub(r"\n{2,}", "\n", body).replace("Dear diary,", "dear diary,") +def _span_coverage(spans, length): + """Validate bounds and compare masks independently of interval layout.""" + if not isinstance(spans, (list, tuple)): + raise ValueError("bad spans") + ranges = [] + for span in spans: + if not isinstance(span, (list, tuple)) or len(span) != 2: + raise ValueError("bad span shape") + a, b = span + if type(a) is not int or type(b) is not int: + raise ValueError("bad span type") + if not 0 <= a <= b <= length: + raise ValueError("span bounds") + if a < b: + ranges.append((a, b)) + ranges.sort() + merged = [] + for a, b in ranges: + if merged and a <= merged[-1][1]: + merged[-1][1] = max(merged[-1][1], b) + else: + merged.append([a, b]) + return merged + + def recheck(row): - """Re-validate a stored row (from any older dataset) against the current - rules, so data written for an older app cannot teach stale behavior. - Raises ValueError with the reason.""" + """Re-validate stored canonical text and loss coverage without rewriting it. + Lookup results are kept; stale rules, shifted text or corrupt masks raise + ValueError with the reason. Training adds untrained history afterward.""" + if "spans" not in row: + raise ValueError("missing spans") + coverage = _span_coverage(row["spans"], len(row["text"])) hdr, sep, body = row["text"].partition("---\n") if not sep: raise ValueError("no header") meta = dict(l.split(": ", 1) for l in hdr.splitlines()[1:] if ": " in l) - ages = re.findall(re.escape(meta["owner"]) + r" is (\d+) years old", meta.get("remembers", "")) + o = meta["owner"] + ages = re.findall(re.escape(o) + r" is (\d+) years old", meta.get("remembers", "")) if len(set(ages)) > 1: raise ValueError("conflicting remembered ages") if row["kind"] == "diary": - check_diary_body(body) - return - now = meta.get("now", "") - if now: - weather = now.split(", ")[1] - allowed = W.WEATHER_NIGHT if W.part_of_day(int(now[:2])) == "night" else W.WEATHER_DAY - if weather not in allowed: - raise ValueError("weather for time") - if any(t.startswith(meta["owner"] + " said: ") for t in meta.get("today", "").split("; ")): - raise ValueError("owner said in chat header") - check_body(meta["owner"], meta["pet"].split(" · ")[0], hdr + sep, body, None) + text, spans = check_diary_text(hdr + sep, body) + else: + now = meta.get("now", "") + if now: + weather = now.split(", ")[1] + allowed = W.WEATHER_NIGHT if W.part_of_day(int(now[:2])) == "night" else W.WEATHER_DAY + if weather not in allowed: + raise ValueError("weather for time") + if any(t.startswith(o + " said: ") for t in meta.get("today", "").split("; ")): + raise ValueError("owner said in chat header") + text, spans = check_body(o, meta["pet"].split(" · ")[0], hdr + sep, body, None) + if text != row["text"]: + raise ValueError("noncanonical text") + if coverage != _span_coverage(spans, len(text)): + raise ValueError("loss mask mismatch") SLEEP_SCENARIOS = ["talk", "talk", "touch", "light", "offer", "offer", "offer", "clean", "idle", "nap"] diff --git a/train/migrate_v6.py b/train/migrate_v6.py index cf378c3..46563f8 100644 --- a/train/migrate_v6.py +++ b/train/migrate_v6.py @@ -18,50 +18,67 @@ import world as W def migrate(row): + hdr, sep, body = row['text'].partition('---\n') + if not sep: + raise ValueError('no header') if row['kind'] == 'diary': - G.recheck(row) - return row - hdr, _, body = row['text'].partition('---\n') - meta = dict(l.split(': ', 1) for l in hdr.splitlines()[1:] if ': ' in l) - o, p = meta['owner'], meta['pet'].split(' · ')[0] - lines = [] - for ln in hdr.splitlines(): - if ln.startswith('today: '): - today = [t for t in ln[7:].split('; ') if not t.startswith(o + ' said: ')] - if not today: - continue - ln = 'today: ' + '; '.join(today) - lines.append(ln) - hdr = '\n'.join(lines) + '\n---\n' - asleep = f', {p} is asleep' in hdr - woke = f'* {p} woke up *' - offer = False - fixed = [] - body_lines = body.splitlines() - offers = [re.compile(e) for e in W.offer_patterns(o, p)] - for i, ln in enumerate(body_lines): - nxt = body_lines[i + 1] if i + 1 < len(body_lines) else '' - if asleep and (ln.startswith(o + ': ') or ln == f'* {o} patted {p} *'): - fixed.append(woke) # one possible body-roll outcome - asleep = False - fixed.append(ln) - if ln == woke: - asleep = False - elif asleep and ln == f'* {o} turned on the lights *' and nxt != woke: - fixed.append(woke) - asleep = False - elif ln.startswith('* '): - offer = any(e.fullmatch(ln) for e in offers) - elif ln.startswith(p + ': '): - if asleep and offer and '[yes]' in ln and nxt != woke: - fixed.append(woke) - asleep = False - if '[sleep]' in ln: - asleep = True + text, spans = G.check_diary_text(hdr + sep, body) + else: + meta = dict(l.split(': ', 1) for l in hdr.splitlines()[1:] if ': ' in l) + o, p = meta['owner'], meta['pet'].split(' · ')[0] + lines = [] + for ln in hdr.splitlines(): + if ln.startswith('today: '): + today = [t for t in ln[7:].split('; ') if not t.startswith(o + ' said: ')] + if not today: + continue + ln = 'today: ' + '; '.join(today) + lines.append(ln) + hdr = '\n'.join(lines) + '\n---\n' + try: + # A current-valid sleeping transcript already records no-wake rolls. + text, spans = G.check_body(o, p, hdr, body, None) + except ValueError as e: + if str(e) not in ('sleep dream grammar', 'sleep offer missing wake'): + raise + asleep = f', {p} is asleep' in hdr + woke = f'* {p} woke up *' offer = False - # Rebuilding from the canonical parser preserves lookup-result masks and - # shifts all trained spans across changed headers and inserted events. - text, spans = G.check_body(o, p, hdr, '\n'.join(fixed), None) + pending_wake = None + fixed = [] + body_lines = [ln.strip() for ln in body.splitlines() if ln.strip()] + offers = [re.compile(e) for e in W.offer_patterns(o, p)] + for i, ln in enumerate(body_lines): + nxt = body_lines[i + 1] if i + 1 < len(body_lines) else '' + # Defer the possible roll until the reply proves a wake is needed. + # A dream consumes a no-wake outcome, including in mixed old logs. + if asleep and pending_wake is None: + if ln.startswith(o + ': ') or ln == f'* {o} patted {p} *': + pending_wake = len(fixed) + elif ln == f'* {o} turned on the lights *': + pending_wake = len(fixed) + 1 + fixed.append(ln) + if ln == woke: + asleep = False + pending_wake = None + offer = False + elif ln.startswith('* '): + offer = any(e.fullmatch(ln) for e in offers) + elif ln.startswith(p + ': '): + tags = [m.group(1) for m in W.TAG_RE.finditer(ln)] + if asleep and offer and 'yes' in tags: + if nxt != woke: + fixed.append(woke) + asleep = False + elif asleep and 'dream' not in tags and pending_wake is not None: + fixed.insert(pending_wake, woke) + asleep = False + if 'sleep' in tags: + asleep = True + pending_wake = None + offer = False + text, spans = G.check_body(o, p, hdr, '\n'.join(fixed), None) + # Migration is the rewrite path; recheck itself never repairs stored masks. out = {**row, 'text': text, 'spans': spans} G.recheck(out) return out diff --git a/train/publish.sh b/train/publish.sh index 50d8304..ea75bdf 100755 --- a/train/publish.sh +++ b/train/publish.sh @@ -19,16 +19,29 @@ FMODE=${FILE_MODE:-0644} DMODE=${DIR_MODE:-0755} (cd "$SRC" && sha256sum -c --quiet SHA256SUMS) install -d -m "$DMODE" "$ROOT" "$ROOT/models" "$ROOT/runtime" if [ -e "$ROOT/models/$ID" ]; then echo "$ROOT/models/$ID exists; refusing to replace a published model" >&2; exit 1; fi -rm -rf "$ROOT/models/$ID.partial" -install -d -m "$DMODE" "$ROOT/models/$ID.partial" "$ROOT/models/$ID.partial/onnx" -for f in "$SRC"/*.json "$SRC"/SHA256SUMS; do install -m "$FMODE" "$f" "$ROOT/models/$ID.partial/"; done -for f in "$SRC"/onnx/*.onnx; do install -m "$FMODE" "$f" "$ROOT/models/$ID.partial/onnx/"; done -mv "$ROOT/models/$ID.partial" "$ROOT/models/$ID" +MODEL_PARTIAL=$ROOT/models/$ID.partial +RUNTIME_PARTIAL= +trap 'rm -rf "$MODEL_PARTIAL"; [ -z "$RUNTIME_PARTIAL" ] || rm -rf "$RUNTIME_PARTIAL"' EXIT +trap 'exit 1' HUP INT TERM +rm -rf "$MODEL_PARTIAL" +install -d -m "$DMODE" "$MODEL_PARTIAL" "$MODEL_PARTIAL/onnx" +for f in "$SRC"/*.json "$SRC"/SHA256SUMS; do install -m "$FMODE" "$f" "$MODEL_PARTIAL/"; done +for f in "$SRC"/onnx/*.onnx; do install -m "$FMODE" "$f" "$MODEL_PARTIAL/onnx/"; done +(cd "$MODEL_PARTIAL" && sha256sum -c --quiet SHA256SUMS) if [ ! -e "$ROOT/runtime/$RTNAME" ]; then - rm -rf "$ROOT/runtime/$RTNAME.partial" - install -d -m "$DMODE" "$ROOT/runtime/$RTNAME.partial" - for f in "$RT"/*; do install -m "$FMODE" "$f" "$ROOT/runtime/$RTNAME.partial/"; done - (cd "$ROOT/runtime/$RTNAME.partial" && sha256sum -- * > SHA256SUMS && chmod "$FMODE" SHA256SUMS) - mv "$ROOT/runtime/$RTNAME.partial" "$ROOT/runtime/$RTNAME" + RUNTIME_PARTIAL=$ROOT/runtime/$RTNAME.partial + rm -rf "$RUNTIME_PARTIAL" + install -d -m "$DMODE" "$RUNTIME_PARTIAL" + for f in "$RT"/*; do install -m "$FMODE" "$f" "$RUNTIME_PARTIAL/"; done + # Generate the manifest from the copied payload, not from an old manifest. + rm -f "$RUNTIME_PARTIAL/SHA256SUMS" + (cd "$RUNTIME_PARTIAL" && sha256sum -- * > SHA256SUMS && chmod "$FMODE" SHA256SUMS && sha256sum -c --quiet SHA256SUMS) fi +# Both copies are complete. Publish a newly needed runtime first, so a failed +# runtime copy never consumes the immutable model id and a retry can reuse it. +if [ -n "$RUNTIME_PARTIAL" ]; then + mv -T -n "$RUNTIME_PARTIAL" "$ROOT/runtime/$RTNAME" +fi +mv -T -n "$MODEL_PARTIAL" "$ROOT/models/$ID" +[ ! -e "$MODEL_PARTIAL" ] || { echo "$ROOT/models/$ID exists; refusing to replace a published model" >&2; exit 1; } du -sh "$ROOT/models/$ID" "$ROOT/runtime/$RTNAME" diff --git a/web/pet.js b/web/pet.js index 1b6682b..9c9925f 100644 --- a/web/pet.js +++ b/web/pet.js @@ -172,6 +172,7 @@ function catchUp() { const away = now - S.lastTick; if (S.pet && away > 2 * MIN) { const evs = simulate(S.lastTick, Math.min(now, S.lastTick + 7 * DAY), true); + lastSecond = now; // the next frame must not replay time already caught up if (evs.includes("pooped")) logToday("pooped"); if (evs.includes("hungry")) logToday("was very hungry"); if (evs.includes("stinky")) logToday("was stinky"); @@ -506,11 +507,10 @@ async function loadBrain(id = CFG.modelId) { try { navigator.storage?.persist?.().catch(() => {}); } catch { /* optional */ } Object.assign(brain, { state: "downloading", id, progress: null, error: "" }); renderAll(); - const w = new Worker(new URL("./brain.js", import.meta.url), { type: "module" }); - brain.worker = w; + let w = null; const fail = async (message) => { clearTimeout(watchdog); - w.terminate(); + w?.terminate(); for (const p of pending.values()) p.reject(new Error(message)); pending.clear(); // An interrupted body can leave a padded, invalid ONNX in the runtime's @@ -523,6 +523,10 @@ async function loadBrain(id = CFG.modelId) { Object.assign(brain, { worker: null, busy: false, state: "error", error: message }); renderAll(); }; + try { + w = new Worker(new URL("./brain.js", import.meta.url), { type: "module" }); + brain.worker = w; + } catch (e) { await fail(e.message || "worker failed"); return; } w.onmessage = ({ data }) => { if (brain.worker !== w) return; if (data.type === "progress") { @@ -550,8 +554,10 @@ async function loadBrain(id = CFG.modelId) { } }; w.onerror = (e) => { if (brain.worker === w) fail(e.message || "worker failed"); }; - w.postMessage({ type: "load", runtime: CFG.runtime, models: CFG.models, modelId: id, dtype: "q4", threads: brain.threads || undefined }); - armWatchdog(); + try { + w.postMessage({ type: "load", runtime: CFG.runtime, models: CFG.models, modelId: id, dtype: "q4", threads: brain.threads || undefined }); + armWatchdog(); + } catch (e) { await fail(e.message || "worker failed"); } } function gen(opts, onToken) { @@ -1648,8 +1654,11 @@ function renderTip() { // ---------- boot ---------- // Only the tab that holds the pet runs it; "use it here" reloads and takes over. -const steal = sessionStorage.getItem("tokenling.steal") === "1"; -try { sessionStorage.removeItem("tokenling.steal"); } catch { /* optional */ } +let steal = false; +try { + steal = sessionStorage.getItem("tokenling.steal") === "1"; + sessionStorage.removeItem("tokenling.steal"); +} catch { /* optional */ } $("#elsewhere-here").addEventListener("click", () => { try { sessionStorage.setItem("tokenling.steal", "1"); } catch { /* optional */ } location.reload(); -- 2.54.0