From 25d85c294e20684369ff4f466d6eebcceb7113dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tao=20Bojl=C3=A9n?= Date: Tue, 23 Jun 2026 16:44:19 +0100 Subject: [PATCH] ci: make image pulls resilient to Docker Hub flakiness (#824) * ci: make image pulls resilient to Docker Hub flakiness Intermittent CI failures came from transient Docker Hub registry/auth timeouts (auth.docker.io / registry-1.docker.io) during image pulls at `docker compose run` time. These are timeouts, not rate limits, so authenticating wouldn't help (and can't on fork PRs anyway). For each compose job in apply_pr_checks.yaml: - Configure a docker.io pull-through mirror (mirror.gcr.io) via /etc/docker/daemon.json, bypassing the flaky Docker Hub auth/token path. No secrets, so it still works on fork PRs. - Pre-pull/build the images each job needs in a 3-attempt retry loop before the real steps. Images are then cached locally, so the existing `docker compose run` steps don't hit the registry -- isolating a flaky pull to the retried prep step without masking real lint/build/test failures. Co-Authored-By: Claude Opus 4.8 (1M context) * ci: extract image-prep into a composite action The mirror-config + retry-pull logic was duplicated across all three compose jobs. Extract it into a local composite action (.github/actions/prepare-docker-images) parameterized by the services to pull/build, so each job calls it in one step. Inputs are passed via env rather than interpolated into the run script, matching the repo's zizmor template-injection guard. Co-Authored-By: Claude Opus 4.8 (1M context) * ci: run dockerised checks when CI plumbing changes The check_api_server / check_generated_graphql / frontend jobs are gated by paths-filter on server/** and client/**, so a PR that only touches CI or docker plumbing (this workflow, the compose file, the Dockerfile, the prepare-docker-images action) skipped all of them -- meaning changes to the image-pull setup were never actually exercised in CI. Add those plumbing paths to the server and client filters via a YAML anchor so the relevant jobs run when they change. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../actions/prepare-docker-images/action.yml | 51 +++++++++++++++++++ .github/workflows/apply_pr_checks.yaml | 26 ++++++++++ 2 files changed, 77 insertions(+) create mode 100644 .github/actions/prepare-docker-images/action.yml diff --git a/.github/actions/prepare-docker-images/action.yml b/.github/actions/prepare-docker-images/action.yml new file mode 100644 index 0000000..3b408ad --- /dev/null +++ b/.github/actions/prepare-docker-images/action.yml @@ -0,0 +1,51 @@ +name: Prepare Docker images +description: > + Pull (and optionally build) images from a docker-compose + file, with special handling to avoid flakiness (since Docker Hub + sometimes returns errors for no good reason). + +inputs: + pull: + description: Space-separated compose services whose images should be pulled. + required: false + default: '' + build: + description: Space-separated compose services that should be built. + required: false + default: '' + +runs: + using: composite + steps: + - name: Configure Docker Hub mirror + # Route docker.io pulls through Google's pull-through cache to avoid + # transient Docker Hub registry/auth timeouts. + shell: bash + run: | + sudo mkdir -p /etc/docker + echo '{"registry-mirrors":["https://mirror.gcr.io"]}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + + - name: Pre-pull images (with retry) + shell: bash + env: + PULL_SERVICES: ${{ inputs.pull }} + BUILD_SERVICES: ${{ inputs.build }} + run: | + # In addition to using the registry mirror above, we also + # retry Docker image pulls a few times to handle flakiness. + for attempt in 1 2 3; do + ok=true + if [ -n "$PULL_SERVICES" ]; then + docker compose pull --quiet $PULL_SERVICES || ok=false + fi + if [ "$ok" = true ] && [ -n "$BUILD_SERVICES" ]; then + docker compose build --quiet $BUILD_SERVICES || ok=false + fi + if [ "$ok" = true ]; then exit 0; fi + if [ "$attempt" -lt 3 ]; then + echo "::warning::image prep attempt $attempt failed; retrying in $((attempt * 15))s" + sleep $((attempt * 15)) + fi + done + echo "::error::image prep failed after 3 attempts"; exit 1 diff --git a/.github/workflows/apply_pr_checks.yaml b/.github/workflows/apply_pr_checks.yaml index 96b6fe2..cbed22c 100644 --- a/.github/workflows/apply_pr_checks.yaml +++ b/.github/workflows/apply_pr_checks.yaml @@ -31,10 +31,21 @@ jobs: id: filter with: filters: | + # Changes to the dockerised CI plumbing re-run the jobs that + # depend on it, so this workflow, the compose file, and the + # Dockerfile are exercised even when no app code changed. + docker: &docker + - 'docker-compose.yaml' + - 'Dockerfile' + - '.env.githubci' + - '.github/workflows/apply_pr_checks.yaml' + - '.github/actions/prepare-docker-images/**' server: - 'server/**' + - *docker client: - 'client/**' + - *docker db: - 'db/**' migrator: @@ -70,6 +81,10 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false + - name: Prepare Docker images + uses: ./.github/actions/prepare-docker-images + with: + build: codegen-check - name: Check generated GraphQL is up to date run: docker compose run --rm --quiet-pull codegen-check @@ -172,6 +187,12 @@ jobs: with: persist-credentials: false + - name: Prepare Docker images + uses: ./.github/actions/prepare-docker-images + with: + pull: postgres scylla clickhouse redis + build: backend test + - name: Lint run: docker compose run --rm --quiet-pull backend npm run lint @@ -210,6 +231,11 @@ jobs: with: persist-credentials: false + - name: Prepare Docker images + uses: ./.github/actions/prepare-docker-images + with: + build: client + - name: Lint client run: docker compose run --rm --quiet-pull client npm run lint -- 2.51.2