From 6afc73956fc5ed0ba621ef949b76cd2a3f027e1e Mon Sep 17 00:00:00 2001 From: Natalie Rose Date: Thu, 10 Sep 2026 13:10:21 +1000 Subject: [PATCH] Add deploy hook --- .env.example | 1 + CLAUDE.md | 43 +++++++++++++--- Dockerfile.deploy-trigger | 5 ++ deploy-trigger/hooks.json | 23 +++++++++ deploy-trigger/receive-deploy.sh | 33 +++++++++++++ docker-compose.yml | 44 +++++++++++++++++ systemd/README.md | 19 +++++++ systemd/deploy-trigger.path | 9 ++++ systemd/deploy-trigger.service | 7 +++ systemd/run-queued-deploy.sh | 52 ++++++++++++++++++++ tailscale/deploy-trigger/deploy-trigger.json | 19 +++++++ 11 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 Dockerfile.deploy-trigger create mode 100644 deploy-trigger/hooks.json create mode 100755 deploy-trigger/receive-deploy.sh create mode 100644 systemd/README.md create mode 100644 systemd/deploy-trigger.path create mode 100644 systemd/deploy-trigger.service create mode 100755 systemd/run-queued-deploy.sh create mode 100644 tailscale/deploy-trigger/deploy-trigger.json diff --git a/.env.example b/.env.example index 50de537..5f0b83c 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,4 @@ CADDY_DOMAIN= TAILSCALE_AUTHKEY= PORTAINER_AGENT_EDGE_ID= PORTAINER_AGENT_EDGE_KEY= +DEPLOY_WEBHOOK_SECRET= diff --git a/CLAUDE.md b/CLAUDE.md index 5b6e313..1eb938a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -74,7 +74,7 @@ if one exists and otherwise reverse-proxies to `server:5000`. **Data + infra.** None of these publish host ports (only `caddy` does). `mysql` 5.5 (root password from `LACUNA_MYSQL_ROOT_PASSWORD`; compose healthcheck via `mysqladmin ping`), with `mysql-dump.sh` bind-mounted in and nightly logical dumps written to `./data/backups` (7-day retention, driven by -the Ofelia `mysql-dump` job). `phpmyadmin` (`phpmyadmin:5.2-apache`). `memcached` for sessions/cache +the Ofelia `mysql-dump` job). `memcached` for sessions/cache — currently ephemeral; the file-persistence `command`/`volumes` in `docker-compose.yml` are commented out. `beanstalk` job queue (schickling/beanstalkd) with `beanstalk-console`. @@ -91,12 +91,36 @@ commented out. `beanstalk` job queue (schickling/beanstalkd) with `beanstalk-con `logrotate` once a day against the shared `./log` tree using the policy in `lacuna.logrotate` (`copytruncate`, since producers in other containers hold log handles open with no reopen signal). +**Management.** `phpmyadmin`, `portainer-agent`, and `deploy-trigger` are each reachable **only** +over Tailscale, never through `caddy` or the public internet (no Caddyfile route exists for any of +them). Each follows the same two-container pattern: a `-tailscale` sidecar +(`tailscale/tailscale:latest`) owns the network namespace — its own `TS_AUTHKEY`, tun device, +`net_admin`/`sys_module` caps, tsnet state under `./data/tailscale-`, and a Tailscale Serve +config at `./tailscale//.json` (`AllowFunnel: false` — tailnet-only) — while the real +service container joins it via `network_mode: service:-tailscale` with no ports/caps of its +own. `phpmyadmin` proxies to its own apache on `:80`; `portainer-agent` (`EDGE`-mode, `/var/run/docker.sock` ++ `/:/host` mounted) proxies to `:9001`. +`deploy-trigger` is an HMAC-gated webhook (`adnanh/webhook`, built via `Dockerfile.deploy-trigger`, +config in `./tailscale/deploy-trigger/hooks.json`) that CI POSTs to in order to trigger a deploy. It +deliberately does **not** run `deploy.sh` or touch docker itself — `deploy.sh` needs root, and giving +a network-reachable container that power (whether via `sudo` or a mounted docker socket) would just +relocate the same risk. Instead `hooks.json` runs `./tailscale/deploy-trigger/receive-deploy.sh`, +which records the request (with CI-supplied `actor`/`ref`/`commit`/`ci_run_url`) into +`./data/deploy-requests/` and appends a line to `./log/deploy/deploy-audit.log`. A host-level systemd +`.path`/`.service` pair (`./systemd/deploy-trigger.path`, `./systemd/deploy-trigger.service`, +`./systemd/run-queued-deploy.sh` — see `./systemd/README.md` for the one-time install) is the only +thing outside Docker Compose in this repo: it watches that directory and, running as root, actually +executes `deploy.sh`, logging full output to `./log/deploy/deploy-.log` and appending the +outcome (`succeeded`/`failed`, exit code, duration) to the audit log. + ## Configuration Gitignored, copy from the `*.example` sibling and fill in: - `.env` (from `.env.example`) — `CONTAINER_REGISTRY_URL`, `LACUNA_MYSQL_ROOT_PASSWORD`, - `CADDY_DOMAIN` (front-proxy public hostname). + `CADDY_DOMAIN` (front-proxy public hostname), `TAILSCALE_AUTHKEY`, `PORTAINER_AGENT_EDGE_ID` / + `PORTAINER_AGENT_EDGE_KEY`, `DEPLOY_WEBHOOK_SECRET` (HMAC secret CI signs deploy-trigger requests + with). - `etc/lacuna.conf` (from `etc/lacuna.example.conf`) — the main game config: JSON-with-`#`-comments. Covers db DSN, beanstalk/memcached endpoints, captcha fonts, map size and neutral/starter zones, payment gateways, `server_url`/`assets_url`. @@ -104,9 +128,12 @@ Gitignored, copy from the `*.example` sibling and fill in: Tracked config and scripts: `etc/log4perl.conf` (Perl logging — INFO to stderr + `log/server/lacuna.log`), `caddy/Caddyfile`, `schedule.ini` (Ofelia cron), `setup.sql` (db + user bootstrap), `mysql-dump.sh` (nightly db backup, bind-mounted into `mysql`), `Dockerfile.logrotate` + -`entrypoint-logrotate.sh` + `lacuna.logrotate` (the logrotate sidecar), and `deploy.sh` (pull → -`up -d --remove-orphans` → `image prune`). Repo files are Prettier-formatted: `npm run format:check` -/ `npm run format:fix` (config in `.prettierrc`). +`entrypoint-logrotate.sh` + `lacuna.logrotate` (the logrotate sidecar), `deploy.sh` (pull → +`up -d --remove-orphans` → `image prune`), `./tailscale//` (Serve configs for the Management +services, see Architecture), `Dockerfile.deploy-trigger` + `./tailscale/deploy-trigger/{hooks.json, +receive-deploy.sh}`, and `./systemd/` (the host-installed deploy-trigger units — the one thing here +Compose doesn't manage). Repo files are Prettier-formatted: `npm run format:check` / +`npm run format:fix` (config in `.prettierrc`). ## Host-mounted trees @@ -125,4 +152,8 @@ Created on the deploy host, mostly not in git: per-SPA `app-` / `alliance-starmap-` / `super-ui-` `caddy-data` + `caddy-config` dirs; and `public-captcha` / `public-network19` / `public-missioncommand` / `public-server` — written by `server`, read by `caddy` for the static routes. (`data/memcached` is used only if memcached - persistence is re-enabled.) + persistence is re-enabled.) `tailscale-phpmyadmin` / `tailscale-portainer-agent` / + `tailscale-deploy-trigger` hold each Management sidecar's tsnet state. `deploy-requests` is not + persistent state but a transient work queue: `deploy-trigger` writes one JSON file per triggered + deploy there, and the host `deploy-trigger.service` (see Architecture → Management) deletes it once + `deploy.sh` has run. diff --git a/Dockerfile.deploy-trigger b/Dockerfile.deploy-trigger new file mode 100644 index 0000000..62f9715 --- /dev/null +++ b/Dockerfile.deploy-trigger @@ -0,0 +1,5 @@ +FROM adnanh/webhook:latest + +RUN apk add --no-cache jq + +ENTRYPOINT ["/usr/local/bin/webhook"] diff --git a/deploy-trigger/hooks.json b/deploy-trigger/hooks.json new file mode 100644 index 0000000..1564ded --- /dev/null +++ b/deploy-trigger/hooks.json @@ -0,0 +1,23 @@ +[ + { + "id": "deploy", + "execute-command": "/scripts/receive-deploy.sh", + "command-working-directory": "/scripts", + "pass-arguments-to-command": [], + "pass-environment-to-command": [ + { "source": "entire-payload", "envname": "HOOK_PAYLOAD" }, + { "source": "header", "name": "X-Forwarded-For", "envname": "HOOK_SOURCE_IP" } + ], + "response-message": "deploy request accepted", + "trigger-rule": { + "match": { + "type": "payload-hmac-sha256", + "secret": "{{ getenv \"DEPLOY_WEBHOOK_SECRET\" }}", + "parameter": { + "source": "header", + "name": "X-Hub-Signature-256" + } + } + } + } +] diff --git a/deploy-trigger/receive-deploy.sh b/deploy-trigger/receive-deploy.sh new file mode 100755 index 0000000..1a68f76 --- /dev/null +++ b/deploy-trigger/receive-deploy.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +# +# Runs inside the deploy-trigger webhook container after the request's HMAC +# signature has already been verified by webhook itself (see hooks.json). +# +# This script never touches docker or sudo. It only records the request - a +# host-side systemd unit (../../systemd/deploy-trigger.{path,service}) watches +# /deploy-requests and performs the actual deploy as root. Keeping privileged +# execution entirely out of this container means a compromised webhook can at +# worst write a request file, never pull images or restart containers itself. +# + +set -eu + +REQUEST_ID=$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' \n') +TS="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + +ACTOR=$(echo "$HOOK_PAYLOAD" | jq -r '.actor // "unknown"') +REF=$(echo "$HOOK_PAYLOAD" | jq -r '.ref // "unknown"') +COMMIT=$(echo "$HOOK_PAYLOAD" | jq -r '.commit // "unknown"') +CI_RUN_URL=$(echo "$HOOK_PAYLOAD" | jq -r '.ci_run_url // "unknown"') +SOURCE_IP="${HOOK_SOURCE_IP:-unknown}" + +mkdir -p /deploy-requests + +cat > "/deploy-requests/${REQUEST_ID}.json" <> /deploy-log/deploy-audit.log + +echo "queued deploy request ${REQUEST_ID}" diff --git a/docker-compose.yml b/docker-compose.yml index 446fbe3..4680d04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -306,6 +306,50 @@ services: portainer-agent-tailscale: condition: service_healthy + deploy-trigger-tailscale: + image: tailscale/tailscale:latest + container_name: deploy-trigger-tailscale + hostname: tle-meanjin-one-deploy-trigger + environment: + - TS_AUTHKEY=${TAILSCALE_AUTHKEY:?error} + - TS_EXTRA_ARGS=--advertise-tags=tag:container + - TS_SERVE_CONFIG=/config/deploy-trigger.json + - TS_STATE_DIR=/var/lib/tailscale + - TS_USERSPACE=false + - TS_ENABLE_HEALTH_CHECK=true + healthcheck: + test: ['CMD', 'wget', '--spider', '-q', 'http://127.0.0.1:9002/healthz'] + interval: 10s + timeout: 3s + volumes: + - ./data/tailscale-deploy-trigger:/var/lib/tailscale + - ./tailscale/deploy-trigger:/config + devices: + - /dev/net/tun:/dev/net/tun + cap_add: + - net_admin + - sys_module + restart: unless-stopped + + deploy-trigger: + build: + context: . + dockerfile: Dockerfile.deploy-trigger + container_name: deploy-trigger + restart: unless-stopped + network_mode: service:deploy-trigger-tailscale + environment: + DEPLOY_WEBHOOK_SECRET: ${DEPLOY_WEBHOOK_SECRET:?error} + volumes: + - ./deploy-trigger/hooks.json:/etc/webhook/hooks.json:ro + - ./deploy-trigger/receive-deploy.sh:/scripts/receive-deploy.sh:ro + - ./data/deploy-requests:/deploy-requests + - ./log/deploy:/deploy-log + command: ['-hooks=/etc/webhook/hooks.json', '-template', '-verbose'] + depends_on: + deploy-trigger-tailscale: + condition: service_healthy + networks: default: name: lacuna diff --git a/systemd/README.md b/systemd/README.md new file mode 100644 index 0000000..07c22bf --- /dev/null +++ b/systemd/README.md @@ -0,0 +1,19 @@ +# deploy-trigger host units + +`deploy.sh` needs root (it calls `sudo docker compose ...`). Rather than give the Tailscale-gated +`deploy-trigger` container that power directly, it only writes a request file into +`../data/deploy-requests/`. These two units are the only thing on the host, outside Docker Compose, +that turns a queued request into an actual deploy - and they do it as root, so no sudoers changes or +container privileges are needed anywhere else. + +Install once per deploy host: + +```bash +sudo ln -s "$(pwd)/deploy-trigger.path" /etc/systemd/system/ +sudo ln -s "$(pwd)/deploy-trigger.service" /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now deploy-trigger.path +``` + +Check it's watching: `systemctl status deploy-trigger.path`. Check the last run: +`journalctl -u deploy-trigger.service` and `tail -f ../log/deploy/deploy-audit.log`. diff --git a/systemd/deploy-trigger.path b/systemd/deploy-trigger.path new file mode 100644 index 0000000..0ac04f3 --- /dev/null +++ b/systemd/deploy-trigger.path @@ -0,0 +1,9 @@ +[Unit] +Description=Watch for TLE deploy trigger requests + +[Path] +PathExistsGlob=/home/nat/lacuna-deployment/data/deploy-requests/*.json +Unit=deploy-trigger.service + +[Install] +WantedBy=multi-user.target diff --git a/systemd/deploy-trigger.service b/systemd/deploy-trigger.service new file mode 100644 index 0000000..71912c4 --- /dev/null +++ b/systemd/deploy-trigger.service @@ -0,0 +1,7 @@ +[Unit] +Description=Run TLE deploy.sh in response to a queued deploy request + +[Service] +Type=oneshot +WorkingDirectory=/home/nat/lacuna-deployment +ExecStart=/home/nat/lacuna-deployment/systemd/run-queued-deploy.sh diff --git a/systemd/run-queued-deploy.sh b/systemd/run-queued-deploy.sh new file mode 100755 index 0000000..4c8d0f1 --- /dev/null +++ b/systemd/run-queued-deploy.sh @@ -0,0 +1,52 @@ +#!/usr/bin/bash + +# +# Runs as root via systemd (deploy-trigger.path -> deploy-trigger.service). +# This is the only place a remotely-triggered deploy actually gains root: it +# is invoked by systemd, never by a container, and reads its work from +# ./data/deploy-requests/*.json - files the deploy-trigger webhook container +# can write but not execute. +# +# Drains the request queue one file at a time, runs deploy.sh for each, and +# appends a start/finish line to the shared audit log. +# + +set -euo pipefail + +cd "$(dirname "$0")/.." + +REQ_DIR=./data/deploy-requests +AUDIT_LOG=./log/deploy/deploy-audit.log + +for req in "$REQ_DIR"/*.json; do + [ -e "$req" ] || continue + + REQUEST_ID=$(jq -r '.request_id' "$req") + ACTOR=$(jq -r '.actor' "$req") + REF=$(jq -r '.ref' "$req") + COMMIT=$(jq -r '.commit' "$req") + CI_RUN_URL=$(jq -r '.ci_run_url' "$req") + SOURCE_IP=$(jq -r '.source_ip' "$req") + + STAMP=$(date -u '+%Y%m%d-%H%M%S') + LOG_FILE="deploy-${STAMP}.log" + START_TS="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + START_EPOCH=$(date +%s) + + echo "{\"ts\":\"${START_TS}\",\"request_id\":\"${REQUEST_ID}\",\"source_ip\":\"${SOURCE_IP}\",\"triggered_by\":\"ci\",\"actor\":\"${ACTOR}\",\"ref\":\"${REF}\",\"commit\":\"${COMMIT}\",\"ci_run_url\":\"${CI_RUN_URL}\",\"status\":\"started\",\"log_file\":\"${LOG_FILE}\"}" >> "$AUDIT_LOG" + + set +e + ./deploy.sh > "./log/deploy/${LOG_FILE}" 2>&1 + EXIT_CODE=$? + set -e + + END_EPOCH=$(date +%s) + END_TS="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + DURATION=$((END_EPOCH - START_EPOCH)) + STATUS="succeeded" + [ "$EXIT_CODE" -ne 0 ] && STATUS="failed" + + echo "{\"ts\":\"${END_TS}\",\"request_id\":\"${REQUEST_ID}\",\"status\":\"${STATUS}\",\"exit_code\":${EXIT_CODE},\"duration_s\":${DURATION},\"log_file\":\"${LOG_FILE}\"}" >> "$AUDIT_LOG" + + rm -f "$req" +done diff --git a/tailscale/deploy-trigger/deploy-trigger.json b/tailscale/deploy-trigger/deploy-trigger.json new file mode 100644 index 0000000..cdee8c1 --- /dev/null +++ b/tailscale/deploy-trigger/deploy-trigger.json @@ -0,0 +1,19 @@ +{ + "TCP": { + "443": { + "HTTPS": true + } + }, + "Web": { + "${TS_CERT_DOMAIN}:443": { + "Handlers": { + "/": { + "Proxy": "http://127.0.0.1:9000" + } + } + } + }, + "AllowFunnel": { + "${TS_CERT_DOMAIN}:443": false + } +} -- 2.51.2