Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471#!/usr/bin/env fish# lith Deployment Script# Deploys the AC monolith (frontend + API) to DigitalOcean droplet
set RED '\033[0;31m'set GREEN '\033[0;32m'set YELLOW '\033[1;33m'set NC '\033[0m'
set SCRIPT_DIR (dirname (status --current-filename))set REPO_ROOT (realpath "$SCRIPT_DIR/..")set VAULT_DIR "$SCRIPT_DIR/../aesthetic-computer-vault"set SSH_KEY "$VAULT_DIR/home/.ssh/id_rsa"set SERVICE_ENV "$VAULT_DIR/lith/.env"set LITH_USER "root"set REMOTE_DIR "/opt/ac"set DEFAULT_LITH_HOST "lith.aesthetic.computer"set DEFAULT_LITH_DROPLET_NAME "ac-lith"set TARGET_HOST $DEFAULT_LITH_HOSTset TARGET_DROPLET_NAME $DEFAULT_LITH_DROPLET_NAMEset LOCAL_BRANCH (git -C $REPO_ROOT branch --show-current 2>/dev/null)set TARGET_BRANCH $LOCAL_BRANCH
if set -q LITH_HOST set TARGET_HOST $LITH_HOSTend
if set -q LITH_DROPLET_NAME set TARGET_DROPLET_NAME $LITH_DROPLET_NAMEend
if set -q DEPLOY_BRANCH set TARGET_BRANCH $DEPLOY_BRANCHend
if test -z "$TARGET_BRANCH" set TARGET_BRANCH mainend
function ssh_ok --argument host ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o ConnectTimeout=10 $LITH_USER@$host "echo ok" &>/dev/nullend
function get_do_token if set -q DIGITALOCEAN_ACCESS_TOKEN echo $DIGITALOCEAN_ACCESS_TOKEN return 0 end
if set -q DO_TOKEN echo $DO_TOKEN return 0 end
for token_file in \ "$VAULT_DIR/help/deploy.env" \ "$VAULT_DIR/judge/deploy.env" \ "$VAULT_DIR/oven/deploy.env" \ "$VAULT_DIR/at/deploy.env" if not test -f $token_file continue end
set token_line (rg -m1 '^DO_TOKEN=' $token_file) if test -n "$token_line" string replace -r '^DO_TOKEN=' '' -- $token_line return 0 end end
return 1end
function get_lith_host_from_do if not command -sq doctl return 1 end
set do_token (get_do_token) if test -z "$do_token" return 1 end
set row (env DIGITALOCEAN_ACCESS_TOKEN="$do_token" \ doctl compute droplet list --format Name,PublicIPv4 --no-header 2>/dev/null | \ rg "^$TARGET_DROPLET_NAME\\s")
if test -z "$row" return 1 end
set compact_row (string replace -ra '\s+' ' ' -- (string trim -- $row)) set fields (string split ' ' -- $compact_row)
if test (count $fields) -lt 2 return 1 end
echo $fields[2]end
# Check for required files. If the plaintext key is missing but the GPG-armored# vault copy exists, ask the slab menubar daemon for the passphrase and# decrypt to a tempfile we use for this run only.set DECRYPTED_KEY ""function cleanup_decrypted_key --on-event fish_exit if test -n "$DECRYPTED_KEY"; and test -f $DECRYPTED_KEY rm -f $DECRYPTED_KEY endend
if not test -f $SSH_KEY set GPG_KEY "$SSH_KEY.gpg" if not test -f $GPG_KEY echo -e "$RED x SSH key not found: $SSH_KEY (and no $GPG_KEY to decrypt)$NC" exit 1 end
set HELPER "$REPO_ROOT/slab/bin/ac-passphrase" if not test -x $HELPER echo -e "$RED x Vault is encrypted but $HELPER is missing/not executable.$NC" exit 1 end
echo -e "$GREEN-> Requesting vault passphrase via slab daemon...$NC" set passphrase ($HELPER vault 600) if test -z "$passphrase" echo -e "$RED x No passphrase provided; aborting.$NC" exit 1 end
set DECRYPTED_KEY (mktemp -t ac-lith-key) chmod 600 $DECRYPTED_KEY if not echo -n "$passphrase" | gpg --batch --pinentry-mode loopback \ --passphrase-fd 0 --decrypt $GPG_KEY >$DECRYPTED_KEY 2>/dev/null rm -f $DECRYPTED_KEY set DECRYPTED_KEY "" echo -e "$RED x Failed to decrypt $GPG_KEY (wrong passphrase?).$NC" exit 1 end set SSH_KEY $DECRYPTED_KEYend
# Env upload is optional: if the vault has a lith/.env we upload it, otherwise# we trust the env already present on the server at /opt/ac/system/.env.set UPLOAD_ENV trueif not test -f $SERVICE_ENV echo -e "$YELLOW Service env not found: $SERVICE_ENV$NC" echo -e "$YELLOW Skipping env upload; preserving existing /opt/ac/system/.env on the server.$NC" set UPLOAD_ENV falseelse if not rg -q '^DEPLOY_SECRET=' $SERVICE_ENV echo -e "$RED x DEPLOY_SECRET missing from $SERVICE_ENV$NC" echo -e "$YELLOW Remove the file or add DEPLOY_SECRET. lith reads this file via /opt/ac/system/.env on the server.$NC" exit 1end
# Test SSH connectionecho -e "$GREEN-> Testing SSH connection to $TARGET_HOST...$NC"if not ssh_ok $TARGET_HOST set fallback_host (get_lith_host_from_do)
if test -n "$fallback_host"; and test "$fallback_host" != "$TARGET_HOST" echo -e "$YELLOW Falling back to DigitalOcean droplet $TARGET_DROPLET_NAME at $fallback_host.$NC" set TARGET_HOST $fallback_host end
if not ssh_ok $TARGET_HOST echo -e "$RED x Cannot connect to $TARGET_HOST$NC" exit 1 endend
echo -e "$GREEN-> Connected to $TARGET_HOST.$NC"
# Deploy from pushed git state only. This avoids production drift from local rsync overlays.echo -e "$GREEN-> Verifying origin/$TARGET_BRANCH...$NC"git -C $REPO_ROOT fetch origin $TARGET_BRANCH --quietset ORIGIN_HEAD (git -C $REPO_ROOT rev-parse origin/$TARGET_BRANCH)
# A feature branch must carry the current production routes and assets. An old# branch deployment previously removed mime.ac and served the AC prompt there.if test "$TARGET_BRANCH" != "main" if not git -C $REPO_ROOT fetch origin main --quiet echo -e "$RED x Could not verify current main; leaving production unchanged.$NC" exit 1 end if not git -C $REPO_ROOT merge-base --is-ancestor origin/main $ORIGIN_HEAD echo -e "$RED x $TARGET_BRANCH is missing current main; deployment would roll back live changes.$NC" echo -e "$YELLOW Rebase or merge origin/main into the branch, then push and deploy again.$NC" exit 1 endend
if test "$LOCAL_BRANCH" = "$TARGET_BRANCH" set LOCAL_HEAD (git -C $REPO_ROOT rev-parse HEAD) if test "$LOCAL_HEAD" != "$ORIGIN_HEAD" echo -e "$RED x Local $TARGET_BRANCH is ahead of origin/$TARGET_BRANCH.$NC" echo -e "$YELLOW Push first. This deploy script no longer rsyncs uncommitted or unpushed code into production.$NC" exit 1 endend
echo -e "$GREEN-> Deploying branch $TARGET_BRANCH at $ORIGIN_HEAD...$NC"set PREVIOUS_HEAD (ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR && git rev-parse HEAD")if test -z "$PREVIOUS_HEAD" echo -e "$RED x Could not resolve the currently deployed commit.$NC" exit 1end
if not ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "\cd $REMOTE_DIR && \git fetch origin $TARGET_BRANCH --quiet && \if git show-ref --verify --quiet refs/heads/$TARGET_BRANCH; then \ git checkout $TARGET_BRANCH --quiet; \else \ git checkout -B $TARGET_BRANCH origin/$TARGET_BRANCH --quiet; \fi && \git reset --hard origin/$TARGET_BRANCH --quiet && \git rev-parse HEAD > system/public/.commit-ref && \sh xbox/tools/precompress-live.sh" echo -e "$RED x Failed to check out origin/$TARGET_BRANCH on $TARGET_HOST.$NC" exit 1end
# Advisory, not a gate: oskiewar ships on its own clock (npm run# oskiewar:deploy reburns the preview before it deploys), so unreleased# oskiewar commits on main must not hold every other lith change hostage.echo -e "$GREEN-> Verifying oskiewar social preview freshness...$NC"if not ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR && node xbox/live/render-social-preview.mjs --check" echo -e "$RED ! oskiewar social preview is stale; deploying anyway. Run npm run oskiewar:deploy to release oskiewar.$NC"end
echo -e "$GREEN-> Verifying disk worker bundle freshness...$NC"if not ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR/system && node scripts/verify-disk-worker.mjs" echo -e "$RED x Disk worker bundle is missing or stale; restoring $PREVIOUS_HEAD.$NC" ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "\cd $REMOTE_DIR && \git reset --hard $PREVIOUS_HEAD --quiet && \git rev-parse HEAD > system/public/.commit-ref && \sh xbox/tools/precompress-live.sh" exit 1end
# Upload env (only if the vault has one — otherwise keep the remote's existing env)# Note: lith.service reads EnvironmentFile=/opt/ac/system/.env, so the canonical# vault source lives at aesthetic-computer-vault/lith/.env and is uploaded into# system/.env on the remote host.if test $UPLOAD_ENV = true # The env is the one thing a deploy ships without ever trying it first, and # on 2026-08-09 a stale MongoDB password rode one up and took every # database-backed endpoint down for fourteen hours. Try the credential from # here; a password that will not open the database on this machine will not # open it on the server either. echo -e "$GREEN-> Verifying environment credentials...$NC" if not node $REPO_ROOT/lith/verify-env.mjs $SERVICE_ENV echo -e "$RED x Refusing to upload an environment that cannot reach its database.$NC" echo -e "$YELLOW $SERVICE_ENV is stale. Production is untouched — the running$NC" echo -e "$YELLOW env on $TARGET_HOST is still whatever last worked.$NC" exit 1 end echo -e "$GREEN-> Uploading environment...$NC" scp -i $SSH_KEY $SERVICE_ENV $LITH_USER@$TARGET_HOST:$REMOTE_DIR/system/.envelse echo -e "$GREEN-> Using existing remote environment (no local vault env to upload).$NC"end
# Install depsecho -e "$GREEN-> Installing dependencies...$NC"ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR/lith && npm install --omit=dev && cd $REMOTE_DIR/system && npm install --omit=dev && cd $REMOTE_DIR/oven && PUPPETEER_SKIP_DOWNLOAD=1 npm install --omit=dev"
# notepat.com amxd build stream.# Modeled after `ac-os upload`'s OTA flow: only rebuild + re-upload# when an amxd input actually changed since the last successful build# (via --if-stale), then push the versioned artifact + latest.json to# DO Spaces (--sync-spaces) so each release has a durable CDN URL# outside lith.## DO Spaces credentials live in aesthetic-computer-vault/spaces/.env# (canonical: spaces/.env.gpg). We decrypt locally, ship to /tmp on# lith for the build's lifetime, then remove — avoids storing S3 keys# permanently in /opt/ac/system/.env. If the vault file is missing or# GPG can't decrypt it, the build still runs — `--sync-spaces` just# gracefully skips the upload with a warning.## The remote sources this file with `.`, so it must contain assignments and# nothing else. A plaintext vault .env written by a `gpg -d > .env` that# forgot to redirect stderr carries gpg's "encrypted with … key" preamble on# its first lines, and each of those becomes a `command not found` on lith.# Keep only well-formed KEY=VALUE lines — the whole line is preserved, so# values containing `=` or spaces survive intact.set SPACES_ENV_SRC "$VAULT_DIR/spaces/.env"set SPACES_ENV_GPG "$VAULT_DIR/spaces/.env.gpg"set TMP_SPACES (mktemp)set SPACES_READY falseif test -f $SPACES_ENV_SRC grep -E '^[A-Za-z_][A-Za-z0-9_]*=' $SPACES_ENV_SRC >$TMP_SPACES set SPACES_READY trueelse if test -f $SPACES_ENV_GPG gpg --batch --pinentry-mode loopback -d $SPACES_ENV_GPG 2>/dev/null \ | grep -E '^[A-Za-z_][A-Za-z0-9_]*=' >$TMP_SPACES if test -s $TMP_SPACES set SPACES_READY true endend
# The tarball aesel.sh (and easel.sh) downloads is built from aesel/, not# committed, so it is packed on the box after the pull. Building it here rather than locally means# the installer can never point at a version older than the source that shipped# with it.echo -e "$GREEN-> Packing the Aesel installer tarball...$NC"ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR && node aesel/bin/pack.mjs" 2>&1 | tail -2
echo -e "$GREEN-> Refreshing notepat.com.amxd build stream...$NC"if test $SPACES_READY = true scp -i $SSH_KEY -q $TMP_SPACES $LITH_USER@$TARGET_HOST:/tmp/notepat-spaces.env ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR && set -a && . /tmp/notepat-spaces.env && set +a && node ac-m4l/build-notepat.mjs --if-stale --sync-spaces; rc=\$?; rm -f /tmp/notepat-spaces.env; exit \$rc"else echo -e "$YELLOW spaces creds unavailable — building without S3 sync.$NC" ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR && node ac-m4l/build-notepat.mjs --if-stale"endrm -f $TMP_SPACES
# Install service file + Caddy config from the deployed checkoutecho -e "$GREEN-> Updating service + Caddy config...$NC"ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "\cp $REMOTE_DIR/lith/lith.service /etc/systemd/system/lith.service && \cp $REMOTE_DIR/lith/lith-mail.service /etc/systemd/system/lith-mail.service && \id -u lith-mail >/dev/null 2>&1 || useradd --system --shell /usr/sbin/nologin --home-dir /nonexistent lith-mail && \install -m 755 $REMOTE_DIR/lith/lith-mail-renew.sh /etc/letsencrypt/renewal-hooks/deploy/lith-mail.sh 2>/dev/null; \sh $REMOTE_DIR/lith/lith-mail-renew.sh && \systemctl enable -q lith-mail && \cp $REMOTE_DIR/lith/Caddyfile /etc/caddy/Caddyfile && \mkdir -p /var/lib/aesthetic-computer/gym.anthonyzollo.com && \if [ ! -f /var/lib/aesthetic-computer/gym.anthonyzollo.com/index.html ]; then \ cp $REMOTE_DIR/lith/gym/index.html /var/lib/aesthetic-computer/gym.anthonyzollo.com/index.html; \fi && \systemctl daemon-reload && \systemctl reload caddy"
# Restart lith service, and the Amail SMTP door beside itecho -e "$GREEN-> Restarting lith + lith-mail...$NC"ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "systemctl restart lith lith-mail"
# Stop automatic transcript deletion immediately, even before the next API call.echo -e "$GREEN-> Migrating Aesel transcript expiration to a soft marker...$NC"if not ssh -i $SSH_KEY $LITH_USER@$TARGET_HOST "cd $REMOTE_DIR/system && node --env-file=.env --input-type=module -e 'const {connect,closePool}=await import(\"./backend/database.mjs\"); const {ensureTranscriptIndexes}=await import(\"./backend/easel-transcripts.mjs\"); try { const {db}=await connect(); await ensureTranscriptIndexes(db); } finally { await closePool(); }'" echo -e "$RED x Transcript retention migration failed.$NC" exit 1end# Purge the Cloudflare cache so new code is served immediately. Runtime .mjs# (kidlisp, disk, graph, …) are STATIC sub-imports without the ?v= cache-bust# boot.mjs puts on top-level modules, so the edge would otherwise serve stale# code until the TTL. Creds come from the process env or a vault env file; if# absent we skip (the short Caddy TTL still bounds staleness).## Auth priority: the Global API Key (email + key) is preferred because the# scoped CLOUDFLARE_API_TOKEN has gone stale in several env files; Bearer token# is the fallback when that's all we have.set -l CF_ENV_FILES $SERVICE_ENV \ "$VAULT_DIR/.devcontainer/envs/devcontainer.env" \ "$VAULT_DIR/oven/deploy.env" "$VAULT_DIR/nanos/conductor.env" \ "$VAULT_DIR/help/deploy.env" "$VAULT_DIR/at/deploy.env"
# Read the first value found for each key across the candidate env files# (an already-exported process env var wins).function cf_lookup --argument-names key for f in $cf_env_files_g test -f $f; or continue set -l line (rg -m1 "^(export )?$key=" $f) if test -n "$line" string replace -r "^(export )?$key=" '' -- $line | string trim --chars '"\'' return 0 end end return 1endset -g cf_env_files_g $CF_ENV_FILES
set -l CF_EMAIL $CLOUDFLARE_EMAILset -l CF_KEY $CLOUDFLARE_API_KEYset -l CF_TOKEN $CLOUDFLARE_API_TOKENtest -z "$CF_EMAIL"; and set CF_EMAIL (cf_lookup CLOUDFLARE_EMAIL)test -z "$CF_KEY"; and set CF_KEY (cf_lookup CLOUDFLARE_API_KEY)test -z "$CF_TOKEN"; and set CF_TOKEN (cf_lookup CLOUDFLARE_API_TOKEN)set -e cf_env_files_g
# Build curl auth args: Global API Key first, else Bearer token.set -l CF_AUTHif test -n "$CF_EMAIL"; and test -n "$CF_KEY" set CF_AUTH -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_KEY"else if test -n "$CF_TOKEN" set CF_AUTH -H "Authorization: Bearer $CF_TOKEN"end
echo -e "$GREEN-> Purging Cloudflare cache...$NC"if test -z "$CF_AUTH" echo -e "$YELLOW No Cloudflare credentials found — skipping purge (Caddy short TTL still applies).$NC"else # Every zone this deploy serves, not just the apex. prompt.ac is a separate # Cloudflare zone, so purging aesthetic.computer left its edge holding # whatever it had — including a cached 404 for a file that had since been # built, which is exactly how prompt.ac/easel.tar.gz stayed missing after a # deploy that produced it. for CF_HOST in aesthetic.computer prompt.ac set CF_ZONE (curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$CF_HOST" \ $CF_AUTH -H "content-type: application/json" \ | python3 -c "import json,sys; r=json.load(sys.stdin).get('result') or []; print(r[0]['id'] if r else '')" 2>/dev/null) if test -z "$CF_ZONE" echo -e "$YELLOW $CF_HOST: could not resolve zone id — skipping.$NC" else set CF_RESULT (curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$CF_ZONE/purge_cache" \ $CF_AUTH -H "content-type: application/json" \ --data '{"purge_everything":true}' \ | python3 -c "import json,sys; d=json.load(sys.stdin); print('ok' if d.get('success') else 'failed: '+str(d.get('errors')))" 2>/dev/null) echo -e "$GREEN purge $CF_HOST: $CF_RESULT$NC" end endend
echo -e "$GREEN-> Done. lith deployed to $TARGET_HOST$NC"
# Mirror slab/menuband/ to its standalone GitHub repo. Runs after a# successful site deploy so the mirror's release pace matches what's# actually live on aesthetic.computer/menuband. Failure here doesn't# void the deploy — the mirror is a courtesy surface for external# contributors, not a critical path.set MIRROR_SYNC "$REPO_ROOT/slab/menuband/bin/mirror-sync.sh"if test -x "$MIRROR_SYNC" echo -e "$GREEN-> Syncing menuband mirror...$NC" bash "$MIRROR_SYNC" 2>&1 | tail -3 or echo -e "$YELLOW menuband mirror sync failed (non-fatal)$NC"end
# Last, and deliberately last: ask a database-backed endpoint for a real answer.## Everything above can pass while every query behind it fails — that is exactly# what happened on 2026-08-09, when a stale credential rode up with a perfectly# good build and nothing noticed for fourteen hours. So the deploy is not# allowed to report success without one live query having worked.## It runs after the purge and the mirror rather than before, because a failure# here is a report, not a gate: the code is already out, and skipping the cache# purge on the way past would only add a second problem. Nor does it roll back# — restoring a commit does not fix a password, and pretending it might would# hide the actual fault.## The public host, not $TARGET_HOST: lith.aesthetic.computer is the SSH target# and serves no vhost, so probing it only ever measures curl's disappointment.set DB_PROBE_HOST "aesthetic.computer"if set -q LITH_PROBE_HOST set DB_PROBE_HOST $LITH_PROBE_HOSTendecho -e "$GREEN-> Verifying the database is answering...$NC"set DB_CODE "000"for attempt in 1 2 3 4 5 6 7 8 9 10 set DB_CODE (curl -s -o /dev/null -w '%{http_code}' --max-time 15 \ "https://$DB_PROBE_HOST/api/oskiewar-replays?limit=1") if test "$DB_CODE" = "200" break end sleep 3endif test "$DB_CODE" = "200" echo -e "$GREEN database: $DB_PROBE_HOST is answering queries$NC"else echo -e "$RED x Deployed, but the database is not answering (HTTP $DB_CODE).$NC" echo -e "$YELLOW The code is live; queries are failing. Check the credentials in$NC" echo -e "$YELLOW $SERVICE_ENV, then: ssh $LITH_USER@$TARGET_HOST journalctl -u lith -n 50$NC" exit 1end