Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
Shell
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377#!/bin/bash# Fast oven deploy with verbose output# Usage: ./deploy.sh [--no-restart]
set -e
OVEN_HOST="137.184.237.166"SSH_KEY="${SSH_KEY:-$(dirname "$0")/../aesthetic-computer-vault/oven/ssh/oven-deploy-key}"REMOTE_DIR="/opt/oven"NATIVE_GIT_FETCH_URL="${NATIVE_GIT_FETCH_URL:-https://tangled.org/aesthetic.computer/core.git}"SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"AC_SOURCE="$SCRIPT_DIR/../system/public/aesthetic.computer"FEDAC_SOURCE="$SCRIPT_DIR/../fedac"VAULT_OS_KEY="$SCRIPT_DIR/../aesthetic-computer-vault/oven/os-build-admin-key.txt"VAULT_ENV="$SCRIPT_DIR/../aesthetic-computer-vault/oven/.env"
# Millisecond epoch — portable (macOS `date +%N` doesn't exist, GNU does).ms() { python3 -c 'import time; print(int(time.time()*1000))'; }
echo "🚀 Deploying oven..."echo " Host: $OVEN_HOST"echo " Key: $SSH_KEY"
# Get current git version for OVEN_VERSION env varGIT_VERSION=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")echo " Version: $GIT_VERSION"
# Time the rsyncSTART_TIME=$(ms)
echo ""echo "📦 Syncing oven files..."rsync -avz --progress --delete \ --exclude='node_modules' \ --exclude='.git' \ --exclude='.env' \ --exclude='*.log' \ --exclude='ac-source' \ --exclude='native-git' \ --exclude='secrets' \ --exclude='state' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$SCRIPT_DIR/" \ "root@$OVEN_HOST:$REMOTE_DIR/"
END_SYNC=$(ms)SYNC_TIME=$((END_SYNC - START_TIME))echo ""echo "✅ Oven sync complete in ${SYNC_TIME}ms"
# The source tree intentionally does not contain .env. Keep rsync --delete from# removing production credentials, then refresh them from the local vault when# it is available. Runtime-managed values are appended later in this script.if [ -f "$VAULT_ENV" ]; then echo "🔐 Syncing Oven environment from vault..." rsync -az \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$VAULT_ENV" \ "root@$OVEN_HOST:$REMOTE_DIR/.env" ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "chmod 600 $REMOTE_DIR/.envif id -u oven >/dev/null 2>&1; then chown oven:oven $REMOTE_DIR/.envfi" echo "✅ Oven environment synced"else echo "⚠️ No vault environment at $VAULT_ENV; preserving remote .env"fi
# Sync aesthetic.computer source files needed for bundle generationecho ""echo "📦 Syncing ac-source files for bundler..."rsync -avz --progress --delete \ --include='*/' \ --include='*.mjs' \ --include='*.js' \ --include='*.json' \ --include='*.lisp' \ --exclude='*' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$AC_SOURCE/" \ "root@$OVEN_HOST:$REMOTE_DIR/ac-source/"
END_AC_SYNC=$(ms)AC_SYNC_TIME=$((END_AC_SYNC - END_SYNC))echo ""echo "✅ ac-source sync complete in ${AC_SYNC_TIME}ms"
# Sync fedac scripts/overlays used by background OS base-image build jobsecho ""echo "📦 Syncing fedac OS build pipeline..."rsync -avz --progress --delete \ --exclude='.git' \ --exclude='*.img' \ --exclude='*.iso' \ --exclude='*.qcow2' \ --exclude='*.log' \ --exclude='native/build/' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$FEDAC_SOURCE/" \ "root@$OVEN_HOST:$REMOTE_DIR/fedac/"
END_FEDAC_SYNC=$(ms)FEDAC_SYNC_TIME=$((END_FEDAC_SYNC - END_AC_SYNC))echo ""echo "✅ fedac sync complete in ${FEDAC_SYNC_TIME}ms"
# Install kernel build tools needed for native OTA builds (idempotent)echo ""echo "🔧 Installing native kernel build tools..."ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" \ "apt-get install -y -q gcc make flex bison libelf-dev libssl-dev bc cpio lz4 musl-tools python3 pahole libdrm-dev libasound2-dev flite1-dev pkg-config dosfstools mtools util-linux 2>&1 | tail -5 || true"echo "✅ Kernel build tools ready"
# Install Nix package manager for NixOS-based native builds (idempotent)echo ""echo "❄️ Installing Nix package manager..."ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" bash -s <<'NIX_EOF' if command -v nix >/dev/null 2>&1; then echo "Nix already installed: $(nix --version)" else curl -sSf -L https://install.determinate.systems/nix | sh -s -- install --no-confirm 2>&1 | tail -10 fi NIX_BIN="" for candidate in \ /nix/var/nix/profiles/default/bin/nix \ /root/.nix-profile/bin/nix \ /home/oven/.nix-profile/bin/nix; do if [ -x "$candidate" ]; then NIX_BIN="$candidate" break fi done if [ -n "$NIX_BIN" ]; then ln -sf "$NIX_BIN" /usr/local/bin/nix NIX_GC_BIN="$(dirname "$NIX_BIN")/nix-collect-garbage" if [ -x "$NIX_GC_BIN" ]; then ln -sf "$NIX_GC_BIN" /usr/local/bin/nix-collect-garbage fi echo "Nix binary: $NIX_BIN" else echo "WARNING: nix binary not found after install" fi if id -u oven >/dev/null 2>&1; then mkdir -p /home/oven/.cache/nix chown -R oven:oven /home/oven/.cache fi # Enable flakes mkdir -p /etc/nix grep -q 'experimental-features' /etc/nix/nix.conf 2>/dev/null || \ echo 'experimental-features = nix-command flakes' >> /etc/nix/nix.conf # Garbage collection timer (weekly, keep 7 days) if ! systemctl is-enabled nix-gc.timer >/dev/null 2>&1; then cat > /etc/systemd/system/nix-gc.service <<'SVC'[Unit]Description=Nix store garbage collection[Service]Type=oneshotExecStart=/nix/var/nix/profiles/default/bin/nix-collect-garbage --delete-older-than 7dSVC cat > /etc/systemd/system/nix-gc.timer <<'TMR'[Unit]Description=Weekly Nix garbage collection[Timer]OnCalendar=weeklyPersistent=true[Install]WantedBy=timers.targetTMR systemctl daemon-reload systemctl enable --now nix-gc.timer fiNIX_EOFecho "✅ Nix ready"
# Install TeX Live for papers PDF builds (idempotent)echo ""echo "📄 Installing TeX Live for papers builds..."ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" \ "apt-get install -y -q texlive-xetex texlive-fonts-extra texlive-latex-extra texlive-bibtex-extra texlive-lang-chinese texlive-lang-cjk fonts-droid-fallback 2>&1 | tail -5 || true"echo "✅ TeX Live ready"
# The mill imports one package that is not a Node builtin: jszip, used by# papers/source-bundle.mjs to build the ZIP each paper ends with. It arrived on# 2026-09-05 and the oven's checkout has never had a node_modules, so every# papers build since has died at module resolution before reaching xelatex —# invisibly, because the poller that would have run them was paused in May.## --no-save --no-package-lock matters: package.json and package-lock.json are# tracked files in that checkout, and a build that mutates tracked files is the# exact bug this deploy is fixing elsewhere.echo ""echo "📄 Ensuring papers build dependencies..."# Only when it is actually missing: `npm install <pkg>` in a directory with a# package.json resolves that whole tree (1105 packages here), which is a minute# nobody should pay on every deploy. --engine-strict=false because the oven runs# Node 20 and the repo asks for 24; the mill itself is fine on 20, and this is# not the place to relitigate that.ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" \ "cd /opt/oven/native-git && if [ -d node_modules/jszip ]; then echo ' jszip present'; else npm install --no-save --no-package-lock --no-audit --no-fund --engine-strict=false jszip >/dev/null 2>&1 && chown -R oven:oven node_modules && echo ' jszip installed'; fi" || \ echo " ⚠️ could not install jszip — papers builds will fail at source-bundle.mjs"
# Optional vault-managed admin key for /os-base-build endpointsecho ""if [ -f "$VAULT_OS_KEY" ]; then echo "🔐 Syncing OS build admin key from vault..." ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "mkdir -p $REMOTE_DIR/secretschmod 700 $REMOTE_DIR/secretsif id -u oven >/dev/null 2>&1; then chown oven:oven $REMOTE_DIR/secretsfi" rsync -avz --progress \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$VAULT_OS_KEY" \ "root@$OVEN_HOST:$REMOTE_DIR/secrets/os-build-admin-key.txt" ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "chmod 600 $REMOTE_DIR/secrets/os-build-admin-key.txtif id -u oven >/dev/null 2>&1; then chown oven:oven $REMOTE_DIR/secrets/os-build-admin-key.txtfiif grep -q '^OS_BUILD_ADMIN_KEY_FILE=' $REMOTE_DIR/.env 2>/dev/null; then sed -i 's|^OS_BUILD_ADMIN_KEY_FILE=.*|OS_BUILD_ADMIN_KEY_FILE=$REMOTE_DIR/secrets/os-build-admin-key.txt|' $REMOTE_DIR/.envelse echo 'OS_BUILD_ADMIN_KEY_FILE=$REMOTE_DIR/secrets/os-build-admin-key.txt' >> $REMOTE_DIR/.envfi" echo "✅ OS build admin key synced"else echo "⚠️ No vault key at $VAULT_OS_KEY (skipping OS_BUILD_ADMIN_KEY_FILE provisioning)"fi
END_SECRET_SYNC=$(ms)SECRET_SYNC_TIME=$((END_SECRET_SYNC - END_FEDAC_SYNC))echo "✅ Secret sync stage complete in ${SECRET_SYNC_TIME}ms"
# Sync BDF font files + glyph caches for bundle font embeddingecho ""echo "📦 Syncing font assets (BDF + glyph caches)..."# These live in the assets bucket, not in git, so a checkout that has not run# `npm run assets:sync:down` simply does not have them. Under `set -e` that# aborted the whole deploy at this line — a hundred lines before the unit is# installed — so a machine missing an optional font cache could not deploy the# oven at all, and said so only as an rsync status 23 buried in the log.if [ ! -d "$SCRIPT_DIR/../system/public/assets/type" ]; then echo "⏭️ no local system/public/assets/type — skipping (run 'npm run assets:sync:down' to include fonts)"elsersync -avz --progress \ --include='*/' \ --include='*.bdf' \ --include='*.bdf.gz' \ --include='*.json' \ --exclude='*' \ -e "ssh -i $SSH_KEY -o StrictHostKeyChecking=no" \ "$SCRIPT_DIR/../system/public/assets/type/" \ "root@$OVEN_HOST:$REMOTE_DIR/assets-type/"fi
END_FONT_SYNC=$(ms)FONT_SYNC_TIME=$((END_FONT_SYNC - END_SECRET_SYNC))echo ""echo "✅ Font glyph sync complete in ${FONT_SYNC_TIME}ms"
# Ensure OS cache path is writable by the oven service user.echo ""echo "🧹 Ensuring OS cache directory + permissions..."ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "mkdir -p $REMOTE_DIR/cacheif id -u oven >/dev/null 2>&1; then chown -R oven:oven $REMOTE_DIRfiif grep -q '^OS_CACHE_DIR=' $REMOTE_DIR/.env 2>/dev/null; then sed -i 's|^OS_CACHE_DIR=.*|OS_CACHE_DIR=$REMOTE_DIR/cache|' $REMOTE_DIR/.envelse echo 'OS_CACHE_DIR=$REMOTE_DIR/cache' >> $REMOTE_DIR/.envfi"echo "✅ OS cache path ready: $REMOTE_DIR/cache"
# Set up native git repo for auto-polling OTA buildsecho ""echo "📦 Setting up native git repo for OTA auto-builds..."echo " Fetch remote: $NATIVE_GIT_FETCH_URL"ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "set -euo pipefailNATIVE_GIT_DIR=/opt/oven/native-gitNATIVE_GIT_FETCH_URL='$NATIVE_GIT_FETCH_URL'if [ ! -d \$NATIVE_GIT_DIR/.git ]; then echo ' Cloning repo (first time)...' git clone --branch main --single-branch \$NATIVE_GIT_FETCH_URL \$NATIVE_GIT_DIRelse echo ' Git repo exists, fetching latest...' cd \$NATIVE_GIT_DIR git remote set-url origin \$NATIVE_GIT_FETCH_URL git fetch origin main --quiet || true git branch --set-upstream-to=origin/main main >/dev/null 2>&1 || true if git rev-parse --verify origin/main >/dev/null 2>&1 && git merge-base --is-ancestor HEAD origin/main; then git merge origin/main --ff-only --quiet || true else echo ' Repo has local commits or divergence; leaving checkout as-is (native build preflight will hard-sync to origin/main).' fifiif id -u oven >/dev/null 2>&1; then chown -R oven:oven \$NATIVE_GIT_DIR su - oven -s /bin/bash -c 'git config --global --add safe.directory /opt/oven/native-git' 2>/dev/null || truefiecho ' Done.'"echo "✅ Native git repo ready"
# Restart unless --no-restart flagif [ "$1" != "--no-restart" ]; then echo "" echo "🔄 Restarting oven service..."
# Update OVEN_VERSION in .env and rewrite the managed systemd override, then restart ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" "set -euo pipefailcd $REMOTE_DIR# Update version in .envif grep -q '^OVEN_VERSION=' .env 2>/dev/null; then sed -i 's/^OVEN_VERSION=.*/OVEN_VERSION=$GIT_VERSION/' .envelse echo 'OVEN_VERSION=$GIT_VERSION' >> .envfiinstall -m 0644 $REMOTE_DIR/infra/oven.service /etc/systemd/system/oven.service# Rewrite systemd override from scratch so stale directives do not survive deploys.mkdir -p /etc/systemd/system/oven.service.d# Emergency pauses are not configuration. This one — PAPERS_POLLER_DISABLED=1,# added by hand on 2026-05-13 to stop a rebuild loop — outlived its cause by# four months, because rewriting override.conf 'from scratch' never touched the# drop-ins beside it. Every deploy now clears it: the fix for a loop is a# deploy, so if the loop is really back, re-add it and write down why.rm -f /etc/systemd/system/oven.service.d/pause-papers-poller.confcat > /etc/systemd/system/oven.service.d/override.conf <<EOF[Service]Environment=OVEN_VERSION=$GIT_VERSIONEnvironment=PATH=/usr/local/bin:/nix/var/nix/profiles/default/bin:/home/oven/.nix-profile/bin:/root/.nix-profile/bin:/usr/local/sbin:/usr/sbin:/usr/bin:/sbin:/binLimitNOFILE=65536EOFsystemctl daemon-reloadsystemctl restart ovensleep 2systemctl is-active --quiet oven.servicesystemctl status oven.service --no-pager | sed -n '1,5p'"
END_RESTART=$(ms) RESTART_TIME=$((END_RESTART - END_FONT_SYNC))
echo "" echo "✅ Restart complete in ${RESTART_TIME}ms"
# Prewarm the bundle cache after restart echo "" echo "🔥 Prewarming bundle cache..." PREWARM_RESULT=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no "root@$OVEN_HOST" \ "curl -s -X POST http://localhost:3002/bundle-prewarm --max-time 120" 2>/dev/null || echo '{"error":"prewarm timeout"}') echo " $PREWARM_RESULT"
END_PREWARM=$(ms) PREWARM_TIME=$((END_PREWARM - END_RESTART)) TOTAL_TIME=$((END_PREWARM - START_TIME))
echo "" echo "✅ Prewarm complete in ${PREWARM_TIME}ms" echo "🏁 Total deploy time: ${TOTAL_TIME}ms"else echo "" echo "⏭️ Skipped restart (--no-restart)"fi
echo ""echo "🔥 Done! https://oven.aesthetic.computer"