From 21ce11c7738e38cd9111d6868ee8b9fdada77db8 Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Mon, 3 Aug 2026 20:38:19 +0200 Subject: [PATCH] ci: build the image in CI and pull it on the droplet --- .tangled/workflows/deploy-microvm.yml | 39 +++-- .tangled/workflows/deploy.yml | 77 --------- Dockerfile | 10 ++ deploy/AUTOPUSH.md | 222 +++++++++----------------- deploy/at-record-pull.service | 12 ++ deploy/at-record-pull.timer | 10 ++ deploy/deploy-on-push.sh | 47 ------ deploy/docker-compose.yml | 36 +---- deploy/publish.sh | 48 ++++-- deploy/setup-autopull.sh | 37 +++++ 10 files changed, 199 insertions(+), 339 deletions(-) delete mode 100644 .tangled/workflows/deploy.yml create mode 100644 deploy/at-record-pull.service create mode 100644 deploy/at-record-pull.timer delete mode 100755 deploy/deploy-on-push.sh create mode 100755 deploy/setup-autopull.sh diff --git a/.tangled/workflows/deploy-microvm.yml b/.tangled/workflows/deploy-microvm.yml index 5e784ac..057fb36 100644 --- a/.tangled/workflows/deploy-microvm.yml +++ b/.tangled/workflows/deploy-microvm.yml @@ -1,25 +1,16 @@ -# ASPIRATIONAL: enable only if the test-postgres.yml canary proves the -# hosted spindle's microVMs are x86_64+KVM. If it comes back arm64 (or -# emulated), delete or ignore this file and keep using deploy.yml instead: -# an amd64 `docker build` in here would hit the same Erlang/OTP#10355 crash -# under QEMU that native-builds-on-the-droplet exists to avoid. +# THE DEPLOY PATH. Builds the amd64 image inside the microVM +# (virtualisation.docker gives it a real docker-in-VM, not +# docker-in-a-container) and pushes it to ghcr; the droplet's +# at-record-pull.timer notices the new digest and rolls onto it. Nothing SSHes +# into production. deploy/publish.sh is the manual equivalent. # -# Disabled by default: `when` below only fires on a manual trigger, never on -# push, so committing this file cannot start pushing images on its own. To -# actually enable it once the canary is green, change `when` to: -# when: -# - event: ["push"] -# branch: ["main"] -# and set up Watchtower on the droplet (deploy/AUTOPUSH.md, -# deploy/docker-compose.yml's commented-out watchtower block) so the new -# `:latest` image actually gets pulled and deployed. -# -# Unlike deploy.yml, this path builds the amd64 image natively inside the -# microVM (virtualisation.docker gives it a real docker-in-VM, not -# docker-in-a-container), so no droplet SSH round-trip is needed at all: -# CI pushes the image, Watchtower on the droplet does the rollout. +# The droplet runs linux/amd64, so an arm64 runner would have to emulate. That +# is survivable now (the Dockerfile's build stage sets +JMsingle, see the +# comment there) but slow, so the probe step below prints the arch: if it says +# aarch64, the builds are emulated and worth moving somewhere native. when: - - event: ["manual"] + - event: ["push"] + branch: ["main"] engine: microvm image: nixos @@ -42,9 +33,15 @@ dependencies: - git steps: + - name: report the runner architecture + command: | + uname -m + nproc + test -e /dev/kvm && echo "kvm: present" || echo "kvm: absent" + # Tests gate the push below: if any of these fail, the pipeline stops # here and no image is ever built or pushed, so a red build can never - # reach ghcr.io/latest and Watchtower never rolls it out. + # reach ghcr.io/latest and the droplet never rolls onto it. - name: generate codecs + check formatting command: | make gen diff --git a/.tangled/workflows/deploy.yml b/.tangled/workflows/deploy.yml deleted file mode 100644 index baadd0e..0000000 --- a/.tangled/workflows/deploy.yml +++ /dev/null @@ -1,77 +0,0 @@ -# PRIMARY autopush path: arch-independent, works regardless of the hosted -# spindle's CPU architecture. This workflow never builds anything itself; it -# only opens an SSH connection to the droplet and streams the source over. -# The droplet decides what to do with it (forced-command posture): the -# deploy key's authorized_keys entry pins `command=".../deploy-on-push.sh"`, -# so sshd ignores whatever this workflow asks for and always runs that -# script. See deploy/deploy-on-push.sh and deploy/AUTOPUSH.md. -# -# This exists because an emulated amd64 build of the Erlang toolchain -# crashes under QEMU (OTP#10355, see deploy/publish.sh), so the actual -# `docker build` has to happen natively on the droplet, same as it does -# today for the manual deploy. Superseded by deploy-microvm.yml only if the -# test-postgres.yml canary proves the hosted spindle's microVMs are -# x86_64+KVM (native amd64 builds, no emulation). -# -# Manual until the canary decides the autopush path. To enable this -# SSH-trigger path, set the secrets in deploy/AUTOPUSH.md and change `when` -# to: -# when: -# - event: ["push"] -# branch: ["main"] -when: - - event: ["manual"] - -engine: nixery - -# openssh for the ssh client; git for `git archive`. -dependencies: - nixpkgs/nixpkgs-unstable: - - openssh - - git - -steps: - - name: ship HEAD to the droplet over ssh - command: | - set -euo pipefail - - : "${DEPLOY_SSH_KEY:?set as a repo secret, see deploy/AUTOPUSH.md}" - : "${DEPLOY_HOST:?set as a repo secret, see deploy/AUTOPUSH.md}" - - key_file="$(pwd)/.deploy-ssh-key" - known_hosts_file="$(pwd)/.deploy-known-hosts" - trap 'rm -f "$key_file" "$known_hosts_file"' EXIT - - printf '%s\n' "$DEPLOY_SSH_KEY" >"$key_file" - chmod 600 "$key_file" - - # Pin the host key when DEPLOY_KNOWN_HOSTS is set (recommended: run - # `ssh-keyscan -t ed25519 ` once from a trusted vantage - # point and store the output as that secret). Without it we fall back - # to trust-on-first-connect, which is weaker but still bounded: the - # deploy key is dedicated, forced-command-locked, and revocable, so a - # spoofed host can at worst see a source tarball, not gain a shell. - if [ -n "${DEPLOY_KNOWN_HOSTS:-}" ]; then - printf '%s\n' "$DEPLOY_KNOWN_HOSTS" >"$known_hosts_file" - strict_flag="-o StrictHostKeyChecking=yes" - else - echo "DEPLOY_KNOWN_HOSTS not set, falling back to TOFU (deploy/AUTOPUSH.md)" >&2 - : >"$known_hosts_file" - strict_flag="-o StrictHostKeyChecking=accept-new" - fi - - short_sha="${TANGLED_SHA:0:7}" - git archive --format=tar.gz -o /tmp/at-record-src.tgz HEAD - - # The command string below is never executed remotely: the forced - # command in authorized_keys overrides it. It only arrives on the - # other end as $SSH_ORIGINAL_COMMAND, which deploy-on-push.sh reads as - # plain data to learn the tag to build, never as something to eval. - ssh -F /dev/null \ - -i "$key_file" \ - -o UserKnownHostsFile="$known_hosts_file" \ - -o BatchMode=yes \ - -o ConnectTimeout=20 \ - $strict_flag \ - "deploy@${DEPLOY_HOST}" "tag=${short_sha}" \ - <"/tmp/at-record-src.tgz" diff --git a/Dockerfile b/Dockerfile index 4d8f0ac..82c8abe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,16 @@ FROM ghcr.io/gleam-lang/gleam:v1.17.0-erlang AS build WORKDIR /build +# Makes an amd64 build work on an arm64 host. BeamAsm dual-maps its JIT code +# (executable + writable aliases of the same pages) and user-mode emulators +# cannot track writes through the alias, so the BEAM crashes before it boots +# (QEMU#1034; erlang/otp#10355 is the same thing reported against OTP 28 and +# closed as unsupported, with this flag as the sanctioned answer). QEMU 8.1 +# fixed the aarch64-guest direction only; amd64-on-arm64 stays broken, and +# Rosetta has the same hole. Single-mapped RWX is a build-stage-only tradeoff; +# the runtime stage keeps normal W^X. No effect on a native build. +ENV ERL_FLAGS="+JMsingle true" + RUN apt-get update \ && apt-get install -y --no-install-recommends make \ && rm -rf /var/lib/apt/lists/* diff --git a/deploy/AUTOPUSH.md b/deploy/AUTOPUSH.md index 74b92d3..aad1597 100644 --- a/deploy/AUTOPUSH.md +++ b/deploy/AUTOPUSH.md @@ -1,152 +1,86 @@ -# Autopush: turning the manual deploy into CI-triggered deploys - -`deploy/publish.sh` stays as the break-glass manual path no matter which -option below gets enabled. Nothing here is live yet: the workflow files it -describes are either not wired to run automatically (`deploy-microvm.yml` -fires on manual trigger only) or wired but missing the secrets they need -(`deploy.yml` fails loudly on missing `DEPLOY_SSH_KEY`/`DEPLOY_HOST`), and no -droplet-side setup has been done. Enabling either path is Niels-only work: -setting repo secrets and touching the droplet. - -## The one unknown, and the canary that answers it - -Everything below hinges on whether the hosted Tangled spindle's microVMs run -on x86_64 or arm64 hardware. If x86_64: a `docker build` inside a microVM -produces a real native amd64 image, no emulation, so CI can build and push -images directly. If arm64: an amd64 build inside that microVM would emulate -under QEMU and hit the same Erlang/OTP crash (OTP#10355) that already rules -out cross-building from Niels's arm Mac. - -`.tangled/workflows/test-postgres.yml` (already committed, engine: microvm) -is the canary. It doesn't test architecture on purpose, but it's the first -real workload on that engine, and its outcome tells us what we're dealing -with. - -**Next action**: push to main (or open a PR) and watch `test-postgres.yml` -run in Tangled's pipeline UI. - -- **Green, and postgres-gated tests behave normally** → the microVM is a - real, working Linux environment. Check the runner's arch specifically - (e.g. add a throwaway step `uname -m` to any workflow, or check spindle - logs/docs) before fully trusting it's x86_64: a green test run proves the - microVM engine works, not which architecture it's on. If confirmed - x86_64+KVM: use **Path B** (`deploy-microvm.yml` + Watchtower). -- **Fails, hangs, or `uname -m` says `aarch64`** → the shared spindle is - arm64 (or the microVM engine has other problems on the hosted instance - regardless of arch). Use **Path A** (`deploy.yml`, SSH-triggered, engine - nixery, arch-independent since the actual build stays on the droplet). - -Path A works either way, so it's the safe default if the canary result is -ambiguous or Niels wants deploys working before investigating further. - -## Path A: SSH-triggered deploy (`deploy.yml`, arch-independent) - -CI never builds anything. It archives `HEAD` and streams it over SSH to a -dedicated, forced-command-locked key on the droplet; the droplet rebuilds -the image natively and rolls it out, same as `publish.sh` does today. - -### Repo secrets to set (Settings → Secrets) - -| Secret | Holds | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `DEPLOY_SSH_KEY` | Private half of a **dedicated** ed25519 keypair generated just for this (`ssh-keygen -t ed25519 -f deploy_ci -N ""`, no passphrase since it runs unattended). Never Niels's personal key. | -| `DEPLOY_HOST` | The droplet's IP or hostname, e.g. `206.189.15.37`. | -| `DEPLOY_KNOWN_HOSTS` | Optional but recommended. Output of `ssh-keyscan -t ed25519 `, run once from a vantage point that already trusts the droplet (e.g. Niels's machine, which already has it in `~/.ssh/known_hosts` from using `publish.sh`). Pins the host key instead of trust-on-first-connect. | - -### One-time droplet setup +# Autopush: how a push to main reaches production + +A push to `main` runs `.tangled/workflows/deploy-microvm.yml`, which runs the +test suite, builds the amd64 image inside the microVM, and pushes it to +`ghcr.io/nmokkenstorm/at-record` as both `:` and `:latest`. On the +droplet, `at-record-pull.timer` polls once a minute; when the digest behind +`:latest` changes it pulls and recreates the `app` container. Tests gate the +push, so a red build never becomes an image. + +Nothing SSHes into production to deploy, and production holds no CI +credentials. `deploy/publish.sh` does the same two steps by hand for +break-glass: build and push from your machine, then start the timer's unit +immediately instead of waiting for the next poll. + +## Repo secrets (Settings → Secrets) + +| Secret | Holds | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REGISTRY_USER` | GitHub username/org for ghcr.io, e.g. `nmokkenstorm`. | +| `REGISTRY_TOKEN` | GitHub PAT with `write:packages`, plus `read:packages` if the package stays private. Scope a fine-grained token to this one package. The hosted spindle is shared, so scope it as narrowly as it allows. | + +## One-time droplet setup ```sh -# on the droplet, as root (or via publish.sh's existing root access) -useradd -r -m -d /home/deploy -s /usr/sbin/nologin deploy -usermod -aG docker deploy -chown -R deploy:deploy /opt/at-record # so the deploy user can rebuild/redeploy - -mkdir -p /home/deploy/.ssh -chmod 700 /home/deploy/.ssh -cat >> /home/deploy/.ssh/authorized_keys <<'EOF' -command="/opt/at-record/deploy/deploy-on-push.sh",no-agent-forwarding,no-X11-forwarding,no-port-forwarding,no-pty ssh-ed25519 AAAA...replace-with-deploy_ci.pub... deploy-ci -EOF -chmod 600 /home/deploy/.ssh/authorized_keys -chown -R deploy:deploy /home/deploy/.ssh +deploy/setup-autopull.sh ``` -`deploy/deploy-on-push.sh` already lives in the repo (ships with every -deploy, same as `publish.sh`); no separate copy step needed beyond it being -present at `/opt/at-record/deploy/deploy-on-push.sh` on whatever revision is -currently checked out there. `-s /usr/sbin/nologin` and the `command=` -forced option are redundant with each other, both are kept: nologin blocks -any other login path to the account, and the forced command blocks this one -specific key from doing anything but run the deploy script even for an -interactive `ssh -t`. - -### Security note - -The hosted spindle is shared: its operator's runtime handles `DEPLOY_SSH_KEY` -in plaintext at execution time, same as it would handle anyone else's -secrets on that runner. That's exactly why the key must be: - -- **Dedicated**: only ever used for this, never reused for Niels's own - droplet access. -- **Revocable**: deleting one `authorized_keys` line kills it, no rotation - of anything else. -- **Forced-command-locked**: `command="..."` in `authorized_keys` means - even a fully leaked key only ever runs `deploy-on-push.sh`, never a shell. - Combined with `no-pty`/`no-port-forwarding`/`no-agent-forwarding`/ - `no-X11-forwarding`, a leaked key cannot pivot into anything else on the - droplet. - -## Path B: ghcr autopush (`deploy-microvm.yml` + Watchtower), aspirational - -Enable only after the canary confirms x86_64+KVM. CI builds and pushes the -image natively in a microVM (tests gate the push, so a red build never -reaches `:latest`); Watchtower on the droplet notices the new digest and -recreates the `app` container on its own. No SSH round-trip from CI at all. - -### To enable - -1. In `.tangled/workflows/deploy-microvm.yml`, change: - ```yaml - when: - - event: ["manual"] - ``` - to: - ```yaml - when: - - event: ["push"] - branch: ["main"] - ``` -2. Uncomment the `watchtower` service and the `app` service's - `labels: watchtower.enable=true` in `deploy/docker-compose.yml`. -3. Do the droplet setup below, then `docker compose up -d` on the droplet - once to start Watchtower (a one-time manual step, not part of the - autopush loop itself). - -### Repo secrets to set (Settings → Secrets) - -| Secret | Holds | -| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `REGISTRY_USER` | GitHub username/org for ghcr.io, e.g. `nmokkenstorm`. | -| `REGISTRY_TOKEN` | A GitHub PAT (classic or fine-grained) with `write:packages` (CI push) and, if the package stays private, `read:packages` too (droplet pull). Scope it to this package only if using a fine-grained token. | - -### One-time droplet setup - -Only needed if `ghcr.io/nmokkenstorm/at-record` stays a **private** package. -If it's made public instead, skip straight to `docker compose up -d`, since -an anonymous `docker pull` needs no credentials at all and Watchtower -inherits that. +Installs `at-record-pull.{service,timer}` into `/etc/systemd/system`, enables +the timer, points `.env` at the `latest` tag, and prints the schedule. +Idempotent. + +If the ghcr package is private (GitHub creates new packages private, and the +first CI push is what creates this one), the droplet needs read credentials +first or every pull 403s: ```sh -# on the droplet, as whichever user runs docker compose (root today) -echo "$REGISTRY_TOKEN" | docker login ghcr.io -u "$REGISTRY_USER" --password-stdin -# writes ~/.docker/config.json, which deploy/docker-compose.yml's -# watchtower service mounts read-only +ssh root@ 'docker login ghcr.io -u nmokkenstorm' ``` -### Security note - -Same shared-runner caveat as Path A applies to `REGISTRY_TOKEN`: scope it as -narrowly as GitHub allows (a fine-grained PAT limited to this one package, -`write:packages` only, no repo/admin scopes), and revoke/rotate it if this -path is ever abandoned. Unlike the SSH key, this token can't be -forced-command-locked, since ghcr.io doesn't support that; narrow scoping is -the only mitigation available. +Making the package public removes that step and the stored credential. The +image holds compiled application code; the secrets it runs on come from `.env` +at runtime and are not baked in. + +## Rolling back + +`AT_RECORD_TAG` in the droplet's `.env` is what the timer re-resolves on each +poll, so it must be `latest` for autopush to move at all. Setting it to a sha +and running `docker compose up -d` does double duty: it pins that image _and_ +freezes the timer on it, since an immutable tag never resolves to anything new. +Set it back to `latest` to resume. + +## The emulation constraint, and why it no longer shapes any of this + +The droplet is x86_64 and runs a `linux/amd64` image, so that is what has to be +produced. Until 2026-08-03 the belief was that only an amd64 machine could +produce it, which is why deploys used to ship a source tarball to the droplet +and build there. + +The real constraint is narrower. BeamAsm dual-maps its JIT code, one executable +and one writable alias of the same pages, and user-mode emulators cannot track +writes through the alias, so the BEAM crashes before it boots +([QEMU#1034](https://gitlab.com/qemu-project/qemu/-/issues/1034)). +[erlang/otp#10355](https://github.com/erlang/otp/issues/10355) is that same +failure reported against OTP 28 and closed as an unsupported configuration, the +BeamAsm author supplying `+JMsingle true` as the answer. QEMU 8.1 fixed the +aarch64-guest direction only; amd64-on-arm64 is the unfixed one and the +maintainers have said it stays that way. Rosetta has the same hole, so Docker +Desktop and OrbStack do not rescue it either. + +The Dockerfile's build stage therefore sets `ERL_FLAGS="+JMsingle true"`, and +cross-building amd64 from an arm Mac works. Verified 2026-08-03: without it, +`docker run --platform linux/amd64 erlang:29-slim erl` dies with +`failed_to_start_child,user,nouser`; with it, it boots. + +Two consequences worth keeping in mind: + +- CI is expected to be amd64 anyway (Tangled's hosted Nixery serves + `architecture: amd64`, and the microVM engine uses KVM so guest arch must + match host), in which case nothing emulates and the flag costs nothing. + That is inference, not documentation, which is why the workflow's first step + prints `uname -m`, `nproc` and whether `/dev/kvm` exists. +- If CI ever cannot produce an amd64 image, the fallback is `publish.sh` from + any machine, not a rebuild on the droplet. The old SSH-triggered path + (a `deploy` service account, a forced-command key, and a source tarball + streamed into production) existed only because local cross-builds were + believed impossible, and was deleted once they weren't. diff --git a/deploy/at-record-pull.service b/deploy/at-record-pull.service new file mode 100644 index 0000000..44759ff --- /dev/null +++ b/deploy/at-record-pull.service @@ -0,0 +1,12 @@ +[Unit] +Description=Roll at-record forward to the newest published image +Requires=docker.service +After=docker.service + +[Service] +Type=oneshot +WorkingDirectory=/opt/at-record +ExecStart=/usr/bin/docker compose pull --quiet app +ExecStart=/usr/bin/docker compose up -d app +# Each pull leaves the previous :latest dangling; nothing else prunes here. +ExecStartPost=-/usr/bin/docker image prune -f diff --git a/deploy/at-record-pull.timer b/deploy/at-record-pull.timer new file mode 100644 index 0000000..3778f69 --- /dev/null +++ b/deploy/at-record-pull.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Poll ghcr.io for a new at-record image + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +AccuracySec=15s + +[Install] +WantedBy=timers.target diff --git a/deploy/deploy-on-push.sh b/deploy/deploy-on-push.sh deleted file mode 100755 index 1ee1349..0000000 --- a/deploy/deploy-on-push.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -# Forced-command target for the CI deploy key (.tangled/workflows/deploy.yml). -# Lives on the droplet at /opt/at-record/deploy/deploy-on-push.sh and is -# never invoked directly: the deploy key's authorized_keys entry pins -# `command=/opt/at-record/deploy/deploy-on-push.sh`, so sshd runs this -# unconditionally no matter what the client asked for. Whatever the client -# did ask for survives as $SSH_ORIGINAL_COMMAND; the only thing we trust -# from it is the short git sha to tag, and only after validating its shape. -# -# How this differs from deploy/publish.sh: -# - publish.sh runs from Niels's machine: scp a tarball, then a second ssh -# session runs a heredoc of commands (build, sed the tag into .env, -# compose up). It stays as the break-glass manual path. -# - deploy-on-push.sh runs as the forced command on a single ssh session -# opened by CI: the source tarball arrives on stdin (the same channel, -# no separate scp), and the tag arrives via $SSH_ORIGINAL_COMMAND -# instead of being interpolated into a remote command string, because -# the forced command posture means CI never gets to choose the remote -# command at all. -# Both converge on the same remote steps (rebuild the image natively, point -# .env at the new tag, `docker compose up -d`, prune) since both exist to -# work around the same amd64-under-QEMU crash (OTP#10355). -set -euo pipefail - -img="ghcr.io/nmokkenstorm/at-record" -app_dir="/opt/at-record" - -tag="$(printf '%s' "${SSH_ORIGINAL_COMMAND:-}" | sed -n 's/^tag=\([0-9a-f]\{7,40\}\)$/\1/p')" -if [ -z "$tag" ]; then - echo "deploy-on-push: missing or malformed tag in \$SSH_ORIGINAL_COMMAND, refusing to deploy" >&2 - exit 1 -fi - -cd "$app_dir" -rm -rf src && mkdir src -tar xzf - -C src - -cd src -docker build -t "$img:$tag" -t "$img:latest" . - -cd "$app_dir" -sed -i "s/^AT_RECORD_TAG=.*/AT_RECORD_TAG=$tag/" .env -docker compose up -d -docker image prune -f >/dev/null 2>&1 || true -docker compose ps - -echo "deploy-on-push: deployed $tag" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 3ff99cc..6331ff0 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -22,11 +22,8 @@ services: timeout: 3s retries: 3 start_period: 15s - # Uncomment together with the watchtower service below (ghcr autopush - # path only, see deploy/AUTOPUSH.md): scopes Watchtower to this - # container so it never touches db/caddy. - # labels: - # - "com.centurylinklabs.watchtower.enable=true" + # Autopush needs AT_RECORD_TAG=latest: the pull timer re-resolves whatever + # tag this is started with, so a sha pins it forever (which is the rollback). db: image: postgres:18-alpine @@ -60,35 +57,6 @@ services: app: condition: service_healthy - # Disabled by default: only relevant for the ghcr autopush path - # (.tangled/workflows/deploy-microvm.yml), which pushes a new ":latest" - # from CI instead of an SSH-triggered rebuild. Watchtower polls ghcr.io - # and recreates any container labeled watchtower.enable=true (the "app" - # label above) when a newer digest shows up, so nothing else needs to - # SSH-trigger the rollout for that path. Not needed and not safe to - # enable for the deploy.yml SSH path: that path already recreates the - # container itself via `docker compose up -d`, so a second automatic - # recreate loop would just race it. - # - # ghcr.io/nmokkenstorm/at-record is currently a private package, so - # Watchtower needs read credentials too: run `docker login ghcr.io` once - # on the droplet as the same user Watchtower runs as (REGISTRY_USER + - # a read:packages token, see deploy/AUTOPUSH.md), which writes - # ~/.docker/config.json; mount that in read-only below. Making the ghcr - # package public instead removes this credential entirely and is the - # simpler option if there's no reason to keep it private. - # watchtower: - # image: containrrr/watchtower - # restart: unless-stopped - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # - ${HOME}/.docker/config.json:/config.json:ro - # environment: - # - WATCHTOWER_LABEL_ENABLE=true - # - WATCHTOWER_POLL_INTERVAL=60 - # - WATCHTOWER_CLEANUP=true - # command: --label-enable - volumes: caddy-data: caddy-config: diff --git a/deploy/publish.sh b/deploy/publish.sh index f173990..c95c97f 100755 --- a/deploy/publish.sh +++ b/deploy/publish.sh @@ -1,28 +1,44 @@ #!/usr/bin/env bash -# Publish the current HEAD to the crate.mokkenstorm.dev droplet. The image is -# built ON the droplet (native amd64) because an emulated amd64 build of the -# Erlang toolchain crashes under QEMU (OTP#10355), so a cross-build from an arm -# Mac can't produce a working image. Run from the repo root. +# Break-glass manual deploy. The normal path is a push to main +# (.tangled/workflows/deploy-microvm.yml); this does the same two things by +# hand: build the amd64 image and push it to ghcr, then kick the droplet's pull +# timer rather than waiting up to a minute for its next poll. Nothing is built +# on the droplet and no source is shipped there. Run from the repo root. +# +# Cross-building amd64 from an arm Mac works because the Dockerfile's build +# stage sets +JMsingle; see the comment there. set -euo pipefail droplet="${AT_RECORD_DROPLET:-root@206.189.15.37}" -tag="$(git rev-parse --short HEAD)" img="ghcr.io/nmokkenstorm/at-record" +builder="at-record" +tag="$(git rev-parse --short HEAD)" + +# buildx builds the working tree, not HEAD, so a dirty tree would ship changes +# under a tag naming a commit that does not contain them. +if [ -n "$(git status --porcelain --untracked-files=no)" ] && [ -z "${AT_RECORD_ALLOW_DIRTY:-}" ]; then + echo "refusing: working tree is dirty, but the image would be tagged $tag" >&2 + echo "commit first, or set AT_RECORD_ALLOW_DIRTY=1 to override" >&2 + exit 1 +fi + +# The default "docker" driver has no registry exporter, so --push needs a +# docker-container builder. Created once, reused after that. +docker buildx inspect "$builder" >/dev/null 2>&1 \ + || docker buildx create --name "$builder" --driver docker-container >/dev/null -echo "publishing $tag to $droplet" -git archive --format=tar.gz -o /tmp/at-record-src.tgz HEAD -scp -o ConnectTimeout=15 /tmp/at-record-src.tgz "$droplet":/opt/at-record/src.tgz +echo "building $img:$tag for linux/amd64" +docker buildx build \ + --builder "$builder" \ + --platform linux/amd64 \ + -t "$img:$tag" -t "$img:latest" \ + --push . +echo "rolling the droplet onto $tag" ssh -o ConnectTimeout=20 "$droplet" " set -e - cd /opt/at-record - rm -rf src && mkdir src && tar xzf src.tgz -C src - cd src && docker build -t $img:$tag -t $img:latest . - cd /opt/at-record - sed -i 's/^AT_RECORD_TAG=.*/AT_RECORD_TAG=$tag/' .env - docker compose up -d - docker image prune -f >/dev/null 2>&1 || true - docker compose ps + systemctl start at-record-pull.service + cd /opt/at-record && docker compose ps " echo "published $tag; verify:" diff --git a/deploy/setup-autopull.sh b/deploy/setup-autopull.sh new file mode 100755 index 0000000..792c2f8 --- /dev/null +++ b/deploy/setup-autopull.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# One-time droplet setup for the ghcr autopush path +# (.tangled/workflows/deploy-microvm.yml): installs the timer that polls for a +# newly published image and rolls the app container onto it. Idempotent. Run +# from the repo root: +# +# deploy/setup-autopull.sh +# +# If ghcr.io/nmokkenstorm/at-record is a private package, log in on the droplet +# first, or every pull fails with a 403: +# ssh root@ 'docker login ghcr.io -u ' +set -euo pipefail + +droplet="${AT_RECORD_DROPLET:-root@206.189.15.37}" + +echo "installing the pull timer on $droplet" +tar cf - -C deploy at-record-pull.service at-record-pull.timer \ + | ssh -o ConnectTimeout=20 "$droplet" " + set -e + tar xf - -C /etc/systemd/system + chown root:root /etc/systemd/system/at-record-pull.{service,timer} + chmod 644 /etc/systemd/system/at-record-pull.{service,timer} + systemctl daemon-reload + systemctl enable --now at-record-pull.timer +" + +# The timer re-resolves whatever tag the container runs; a pinned sha would +# never move. Setting it back to a sha is how a rollback freezes. +echo "pointing the app at the moving tag" +ssh -o ConnectTimeout=20 "$droplet" " + set -e + cd /opt/at-record + sed -i 's/^AT_RECORD_TAG=.*/AT_RECORD_TAG=latest/' .env + grep -q '^AT_RECORD_TAG=latest\$' .env || { echo 'no AT_RECORD_TAG line in .env' >&2; exit 1; } + docker compose up -d app + systemctl list-timers at-record-pull.timer --no-pager +" -- 2.51.2