diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml index ead0332..8d27e8f 100644 --- a/.github/workflows/publish-docker.yaml +++ b/.github/workflows/publish-docker.yaml @@ -6,6 +6,7 @@ on: paths: - 'server/**' - 'client/**' + - 'db/**' - 'Dockerfile' - 'client/Dockerfile' - '.github/workflows/publish-docker.yaml' @@ -25,6 +26,7 @@ jobs: outputs: server: ${{ steps.filter.outputs.server }} client: ${{ steps.filter.outputs.client }} + db: ${{ steps.filter.outputs.db }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 @@ -38,6 +40,8 @@ jobs: - 'Dockerfile' client: - 'client/**' + db: + - 'db/**' # On release/dispatch, paths-filter is skipped so outputs are empty. # The downstream jobs use != 'false', so empty triggers a build. @@ -138,3 +142,49 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64,linux/arm64 + + # Bundles the db/ migration files + configs into an image meant to be run as a + # one-shot task on deploy. + # On push to main it publishes only when something under db/ changed (e.g. a + # new migration script). On a release it always publishes a versioned tag. + build-migrations: + needs: changes + if: needs.changes.outputs.db != 'false' + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + + - uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0 + + - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0 + id: meta + with: + images: ${{ env.REGISTRY }}/${{ github.repository_owner }}/coop-migrations + tags: | + type=sha,prefix= + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=raw,value=latest,enable={{is_default_branch}} + + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: db + file: db/Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 diff --git a/db/Dockerfile b/db/Dockerfile index 32a3e11..15ffa9d 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -17,16 +17,25 @@ FROM node:24.14.1-bullseye-slim AS base WORKDIR /app -COPY --from=builder /app/build ./ +# Keep output under ./build (not flattened) so the `db:*` scripts resolve +# `build/index.js` both here and locally after `npm run build`. +COPY --from=builder /app/build ./build +COPY --from=builder /app/package.json ./package.json COPY --from=builder /app/node_modules ./node_modules # I create a separate stage for these steps because they are not needed for # development and chown can take a long time to run FROM base -RUN groupadd -r coop && useradd -r -g coop -u 1001 coop +# -m creates a home dir for `coop` so `npm run` has a writable HOME for its +# cache/logs (the container runs `npm run db:*` as this non-root user). +RUN groupadd -r coop && useradd -r -g coop -u 1001 -m coop RUN chown -R coop:coop /app # kubernetes requires us to use a numeric id for the user USER 1001 + +# One-shot migrations task; override the command at runtime, e.g. +# npm run db:update -- --db api-server-pg --env prod +CMD ["node", "build/index.js"] diff --git a/db/package.json b/db/package.json index 44150ac..6b1ab13 100644 --- a/db/package.json +++ b/db/package.json @@ -6,7 +6,12 @@ "type": "module", "scripts": { "build": "tsc && cp -R src/scripts package.json build/", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "db:add": "node build/index.js add", + "db:clean": "node build/index.js clean", + "db:update": "node build/index.js apply", + "db:create": "node build/index.js create", + "db:drop": "node build/index.js drop" }, "author": "Roostorg", "license": "ISC", diff --git a/docker-compose.images.yaml b/docker-compose.images.yaml index b49d8c3..e33d05a 100644 --- a/docker-compose.images.yaml +++ b/docker-compose.images.yaml @@ -29,10 +29,16 @@ services: volumes: - scylla_data:/var/lib/scylla healthcheck: - test: ['CMD-SHELL', 'nodetool status || exit 1'] + # Real CQL query, not `nodetool status`, so 9042 accepts connections + # before migrations start. + test: + [ + 'CMD-SHELL', + 'cqlsh -e "SELECT release_version FROM system.local" || exit 1', + ] interval: 5s - timeout: 5s - retries: 28 + timeout: 10s + retries: 30 start_period: 35s clickhouse: @@ -51,14 +57,10 @@ services: retries: 5 migrations: - image: node:24.14.1-bullseye-slim - command: bash -c 'set -e - && npm i + image: ghcr.io/roostorg/coop-migrations:latest + command: sh -c 'set -e && for db in api-server-pg scylla clickhouse; do npm run db:create -- --db "$$db" --env staging; npm run db:update -- --db "$$db" --env staging; done' - working_dir: /src env_file: ./.env.docker - volumes: - - .:/src depends_on: postgres: condition: service_healthy diff --git a/docs/development/docker.md b/docs/development/docker.md index e910ae4..2595185 100644 --- a/docs/development/docker.md +++ b/docs/development/docker.md @@ -3,12 +3,16 @@ Pre-built images are published to the GitHub Container Registry on every push to `main`: ``` -ghcr.io/roostorg/coop-server # API server -ghcr.io/roostorg/coop-worker # Background worker -ghcr.io/roostorg/coop-client # Frontend (nginx) +ghcr.io/roostorg/coop-server # API server +ghcr.io/roostorg/coop-worker # Background worker +ghcr.io/roostorg/coop-client # Frontend (nginx) +ghcr.io/roostorg/coop-migrations # One-shot database migrations runner ``` -Images are tagged with `latest`, the git SHA, and semver tags on release. +Images are tagged with `latest`, the git SHA, and semver tags on release. The +`coop-migrations` image is only rebuilt when something under `db/` changes (e.g. +a new migration script) or on a release, so its tags track the schema rather +than every server/client change. ## Quick start @@ -75,10 +79,47 @@ docker compose -f docker-compose.images.yaml down -v ## Image details -| Image | Dockerfile | Build target | Base | -| ------------- | ------------------- | --------------------- | --------------------------------- | -| `coop-server` | `Dockerfile` | `build_server` | node:24-bullseye-slim + dumb-init | -| `coop-worker` | `Dockerfile` | `build_worker_runner` | node:24-bullseye-slim + dumb-init | -| `coop-client` | `client/Dockerfile` | `serve` | nginx:1.27-bookworm | +| Image | Dockerfile | Build target | Base | +| ----------------- | ------------------- | --------------------- | --------------------------------- | +| `coop-server` | `Dockerfile` | `build_server` | node:24-bullseye-slim + dumb-init | +| `coop-worker` | `Dockerfile` | `build_worker_runner` | node:24-bullseye-slim + dumb-init | +| `coop-client` | `client/Dockerfile` | `serve` | nginx:1.27-bookworm | +| `coop-migrations` | `db/Dockerfile` | _(final stage)_ | node:24-bullseye-slim | The client image serves the Vite-built SPA via nginx and proxies `/api/` requests (including `/api/v1/graphql`) to a backend service named `server` on port 8080. + +## Running migrations + +The `coop-migrations` image bundles the migration scripts under `db/` together +with the migrator engine, so it can run as a one-shot task (an ECS `RunTask`, a +Kubernetes `Job`, or the `migrations` service in `docker-compose.images.yaml`). +It exposes the same `npm run db:*` commands used in local development, so the +invocation matches what you'd run from the repo root: + +```bash +# Apply all pending migrations to an existing prod database +docker run --rm --env-file .env.docker ghcr.io/roostorg/coop-migrations:latest \ + npm run db:update -- --db api-server-pg --env prod +``` + +Supported `--db` values are `api-server-pg`, `scylla`, and `clickhouse`. +Connection settings come from environment variables (see `db/.env.example`), +which must be present for any command since the database configs are read at +startup. + +### What `--env` controls + +`--env` (`staging` or `prod`) only affects **seed scripts**, not migrations: + +- Migration scripts run in every environment. +- A seed named `*.seed..sql` runs **only** when `--env` matches it. Seed + files are timestamp-prefixed; the repo currently ships + `db/src/scripts/api-server-pg/2025.12.01T00.00.01.initial-test-data.seed.staging.sql`, + which creates a sample org with default-password users — so **use `--env prod` + in production** to skip it. Running `--env staging` against a prod database + would seed that test data. +- `db:create` ignores `--env` functionally and is allowed in prod, so you can + provision schemas for self-hosted Scylla/ClickHouse (which have no managed + "create the database" step). +- `db:clean` and `db:drop` are destructive and **reject `--env prod`** as a + safety guard. diff --git a/migrator/cli/index.ts b/migrator/cli/index.ts index f3f5179..a5dc739 100644 --- a/migrator/cli/index.ts +++ b/migrator/cli/index.ts @@ -1,5 +1,5 @@ -import path from 'path'; import { glob } from 'node:fs/promises'; +import path from 'path'; import '@total-typescript/ts-reset/array-includes'; @@ -330,18 +330,13 @@ export function makeCli(dbs: { [k: string]: DatabaseConfig }) { command: 'create', describe: 'Creates the databases specified in the ENV vars.', builder: (yargs) => { - return yargs - .option('db', dbOpt) - .option('env', { - ...envOpt, - demand: - 'Must provide an environment (even though it has no effect; the ' + - 'connection-related env vars determine which db(s) are cleaned) ' + - 'to help prevent accidentally deleting prod!', - }) - .check((opts) => { - return opts.env !== 'prod'; - }); + return yargs.option('db', dbOpt).option('env', { + ...envOpt, + demand: + 'Must provide an environment (even though it has no functional ' + + 'effect; the connection-related env vars determine which db(s) ' + + 'are created).', + }); }, handler: async ({ db: optDbs }) => { await Promise.all(