Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
core docker-compose.mill.yml
5.6 kB · 157 lines
YAML
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158# turns the primary spindle into a mill host and runs a three-executor fleet# against it. the executors differ on two axes so placement is testable:## executor labels images can run# --------- ------------- ----------- ------------------------# executor-a linux, fast full image/alpine, image/nixos# executor-b linux, slow full image/alpine, image/nixos# executor-c linux, gpu alpine only image/alpine
x-mill-executor: &mill-executor profiles: ["linux"] build: context: . dockerfile: localinfra/spindle.Dockerfile restart: unless-stopped environment: &mill-executor-env SPINDLE_ROLE: executor SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe SPINDLE_SERVER_DEV: "true" SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload # dials the mill container directly using ws SPINDLE_MILL_URL: ws://spindle:6555/mill devices: - /dev/vsock:/dev/vsock - /dev/kvm:/dev/kvm - /dev/vhost-vsock:/dev/vhost-vsock - /dev/net/tun:/dev/net/tun cap_add: - NET_ADMIN - SYS_ADMIN security_opt: - label=disable - seccomp=unconfined healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] interval: 2s timeout: 2s retries: 30 start_period: 5s depends_on: &mill-executor-deps plc: condition: service_started jetstream: condition: service_started init-accounts: condition: service_completed_successfully ncps: condition: service_started spindle: condition: service_healthy mill-tokens: condition: service_completed_successfully networks: [tngl]
services: # configures the primary spindle container as a mill host spindle: environment: SPINDLE_ROLE: mill SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk volumes: - spindle-artifacts:/var/lib/spindle/artifacts
mill-tokens: profiles: ["linux"] build: context: . dockerfile: localinfra/spindle.Dockerfile restart: "no" entrypoint: ["/bin/sh", "-c"] environment: SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db command: - | set -eu seed() { spindle mill executor revoke "$$1" >/dev/null 2>&1 || true args="" for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token" } seed executor-a linux,fast seed executor-b linux,slow seed executor-c linux,gpu echo "seeded executor tokens" volumes: - spindle-data:/var/lib/spindle - init-state:/shared depends_on: spindle: condition: service_healthy networks: [tngl]
spindle-executor-a: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,fast SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" volumes: - spindle-executor-a-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
spindle-executor-b: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,slow SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" volumes: - spindle-executor-b-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
spindle-executor-c: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,gpu SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" volumes: - spindle-executor-c-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
volumes: spindle-executor-a-data: spindle-executor-b-data: spindle-executor-c-data: spindle-artifacts: