diff --git a/Dockerfile b/Dockerfile index c734942..1544471 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* COPY --from=build /build/server/build/erlang-shipment ./ -COPY deploy/Caddyfile /app/deploy/Caddyfile +COPY deploy/Caddyfile deploy/docker-compose.yml /app/deploy/ RUN useradd --system --no-create-home --uid 10001 app \ && chown -R app /app diff --git a/deploy/AUTOPUSH.md b/deploy/AUTOPUSH.md index f02627b..62fb98b 100644 --- a/deploy/AUTOPUSH.md +++ b/deploy/AUTOPUSH.md @@ -8,10 +8,15 @@ droplet, `crate-pull.timer` polls once a minute; when the digest behind push, so a red build never becomes an image. Nothing SSHes into production to deploy, and production holds no CI -credentials. The pull unit's `ExecStartPre` runs `deploy/sync-caddy.sh` -(installed on the droplet at `/opt/crate/deploy/sync-caddy.sh`), which -extracts the versioned Caddyfile from the pulled app image and recreates -Caddy only when that file changed, before replacing the app container. It's +credentials. The pull unit's `ExecStartPre` runs `deploy/sync-config.sh` +(installed on the droplet at `/opt/crate/deploy/sync-config.sh`), which +extracts the versioned Caddyfile and compose file from the pulled app image +and installs each one that changed, recreating Caddy in that case, before +replacing the app container. Both files travel in the image because config +that does not travel never lands: `docker-compose.yml` was edited in the repo +for a healthcheck change that the droplet's own copy never saw. Anything else +the droplet runs on belongs in the image and in this script for the same +reason. It's a real script file rather than an inline `sh -c '...'` on the unit's `Exec` line because systemd expands `$VAR` on those lines itself before the shell sees them, which would silently break the script's own `$(...)` and @@ -33,9 +38,11 @@ deploy/setup-autopull.sh ``` Installs `crate-pull.{service,timer}` into `/etc/systemd/system` and -`sync-caddy.sh` into `/opt/crate/deploy/`, enables the timer, installs the -initial Caddyfile, points `.env` at the `latest` tag, and prints the -schedule. Subsequent image pulls extract the versioned Caddyfile, validate +`sync-config.sh` into `/opt/crate/deploy/`, enables the timer, installs the +initial Caddyfile and compose file, points `.env` at the `latest` tag, and +prints the schedule. It overwrites `/opt/crate/docker-compose.yml`: the repo's +copy is the only one, and a hand-edit on the droplet is what the next pull +undoes anyway. Subsequent image pulls extract the versioned Caddyfile, validate it in the Caddy image, and force-recreate Caddy only when it changes. Invalid configuration leaves the running proxy untouched. Idempotent. diff --git a/deploy/crate-pull.service b/deploy/crate-pull.service index 58e4287..b668971 100644 --- a/deploy/crate-pull.service +++ b/deploy/crate-pull.service @@ -7,7 +7,7 @@ After=docker.service Type=oneshot WorkingDirectory=/opt/crate ExecStartPre=/usr/bin/docker compose pull --quiet app -ExecStartPre=/opt/crate/deploy/sync-caddy.sh +ExecStartPre=/opt/crate/deploy/sync-config.sh ExecStart=/usr/bin/docker compose up -d app # Each pull leaves the previous :latest dangling; nothing else prunes here. ExecStartPost=-/usr/bin/docker image prune -f diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index b713a73..3f2711b 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -16,12 +16,8 @@ services: depends_on: db: condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8080/readyz"] - interval: 30s - timeout: 3s - retries: 3 - start_period: 15s + # No healthcheck block: the image's own HEALTHCHECK is the one definition + # of what "healthy" means, and it ships with every pull. # Autopush needs CRATE_TAG=latest: the pull timer re-resolves whatever # tag this is started with, so a sha pins it forever (which is the rollback). diff --git a/deploy/setup-autopull.sh b/deploy/setup-autopull.sh index ff19879..4aeee8f 100755 --- a/deploy/setup-autopull.sh +++ b/deploy/setup-autopull.sh @@ -14,7 +14,9 @@ set -euo pipefail droplet="${CRATE_DROPLET:-root@206.189.15.37}" echo "installing the pull timer on $droplet" -tar cf - -C deploy crate-pull.service crate-pull.timer Caddyfile sync-caddy.sh \ +# Every file sync-config.sh keeps in step has to be here too, or the droplet +# starts from a copy the next pull then overwrites. +tar cf - -C deploy crate-pull.service crate-pull.timer Caddyfile docker-compose.yml sync-config.sh \ | ssh -o ConnectTimeout=20 "$droplet" " set -e tar xf - -C /etc/systemd/system @@ -22,8 +24,10 @@ tar cf - -C deploy crate-pull.service crate-pull.timer Caddyfile sync-caddy.sh \ install -m 644 /etc/systemd/system/Caddyfile /tmp/crate.Caddyfile docker run --rm -v /tmp/crate.Caddyfile:/etc/caddy/Caddyfile:ro caddy:2 caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile install -m 644 /tmp/crate.Caddyfile /opt/crate/Caddyfile - install -m 755 /etc/systemd/system/sync-caddy.sh /opt/crate/deploy/sync-caddy.sh - rm /etc/systemd/system/Caddyfile /etc/systemd/system/sync-caddy.sh + install -m 644 /etc/systemd/system/docker-compose.yml /opt/crate/docker-compose.yml + install -m 755 /etc/systemd/system/sync-config.sh /opt/crate/deploy/sync-config.sh + rm /etc/systemd/system/Caddyfile /etc/systemd/system/docker-compose.yml /etc/systemd/system/sync-config.sh + cd /opt/crate && docker compose config -q chown root:root /etc/systemd/system/crate-pull.{service,timer} chmod 644 /etc/systemd/system/crate-pull.{service,timer} systemctl daemon-reload diff --git a/deploy/sync-caddy.sh b/deploy/sync-caddy.sh deleted file mode 100755 index da1cdfc..0000000 --- a/deploy/sync-caddy.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -# ExecStartPre for crate-pull.service, run from /opt/crate on the droplet -# after the app image is pulled. Extracts the versioned Caddyfile baked into -# that image and, only if it differs from what's installed, validates it -# inside the caddy image before installing it and force-recreating caddy. -# Invalid config always leaves the running proxy untouched. -# -# Split out of the unit file rather than left as an inline `sh -c '...'`: -# systemd expands `$VAR` on Exec lines itself before the shell ever sees -# them, which silently breaks the `$(...)` and "$image" here -- an empty -# $image would make this whole step a no-op that nobody notices until a -# Caddyfile change fails to deploy. -set -euo pipefail - -image="$(docker compose config --images app)" -docker run --rm --entrypoint cat "$image" /app/deploy/Caddyfile \ - > /tmp/crate.Caddyfile - -if cmp -s /tmp/crate.Caddyfile /opt/crate/Caddyfile; then - exit 0 -fi - -docker run --rm -v /tmp/crate.Caddyfile:/etc/caddy/Caddyfile:ro caddy:2 \ - caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile - -install -m 644 /tmp/crate.Caddyfile /opt/crate/Caddyfile -docker compose up -d --force-recreate caddy diff --git a/deploy/sync-config.sh b/deploy/sync-config.sh new file mode 100755 index 0000000..7583a5c --- /dev/null +++ b/deploy/sync-config.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# ExecStartPre for crate-pull.service, run from /opt/crate on the droplet +# after the app image is pulled. Extracts the versioned Caddyfile and compose +# file baked into that image and, for each one that differs from what is +# installed, validates it before installing it. Invalid config always leaves +# the running proxy and the installed files untouched. +# +# Everything the droplet runs on has to travel in the image, or it silently +# never lands: the compose file joined the Caddyfile here after a healthcheck +# change shipped in the repo and stayed on the droplet's old copy for good. +# +# Split out of the unit file rather than left as an inline `sh -c '...'`: +# systemd expands `$VAR` on Exec lines itself before the shell ever sees them, +# which silently breaks the `$(...)` and "$image" below. An empty $image makes +# this whole step a no-op that nobody notices until a config change fails to +# deploy. +set -euo pipefail + +image="$(docker compose config --images app)" +changed=0 + +extract() { + docker run --rm --entrypoint cat "$image" "/app/deploy/$1" +} + +extract docker-compose.yml > /tmp/crate.compose.yml +extract Caddyfile > /tmp/crate.Caddyfile + +if ! cmp -s /tmp/crate.compose.yml /opt/crate/docker-compose.yml; then + docker compose --project-directory /opt/crate \ + -f /tmp/crate.compose.yml config -q + install -m 644 /tmp/crate.compose.yml /opt/crate/docker-compose.yml + changed=1 +fi + +if ! cmp -s /tmp/crate.Caddyfile /opt/crate/Caddyfile; then + docker run --rm -v /tmp/crate.Caddyfile:/etc/caddy/Caddyfile:ro caddy:2 \ + caddy validate --config /etc/caddy/Caddyfile --adapter caddyfile + install -m 644 /tmp/crate.Caddyfile /opt/crate/Caddyfile + changed=1 +fi + +# The unit's ExecStart recreates app; caddy only moves when its own config or +# service definition did. +if [ "$changed" -eq 1 ]; then + docker compose up -d --force-recreate caddy +fi diff --git a/docker-compose.yml b/docker-compose.yml index 2b6bcc7..4bb9292 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,13 +24,9 @@ services: depends_on: db: condition: service_healthy + # The image's own HEALTHCHECK governs; overriding it here is a second + # definition that drifts. restart: unless-stopped - healthcheck: - test: ["CMD", "curl", "-fsS", "http://localhost:8080/readyz"] - interval: 30s - timeout: 3s - retries: 3 - start_period: 10s db: image: postgres:16-alpine