Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
core docker-compose.mill.yml
7.3 kB · 199 lines
YAML
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200# turns primary spindle into a mill and also adds three executors with differing# labels for testing and with different images.## executor labels images can run# --------- ------------- ----------- ------------------------# executor-a linux, fast full image/alpine, image/nixos# executor-b linux, slow full image/alpine, image/nixos# executor-c linux, gpu alpine only image/alpine
x-mill-executor: &mill-executor profiles: ["linux"] build: context: . dockerfile: localinfra/spindle.Dockerfile restart: unless-stopped environment: &mill-executor-env SPINDLE_ROLE: executor SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 SPINDLE_SERVER_METRICS_LISTEN_ADDR: 0.0.0.0:9091 SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe SPINDLE_SERVER_DEV: "true" SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED: "true" SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR: 0.0.0.0:2223 SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST: spindle-executor SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST: debug@spindle.tngl.boltless.dev:2224 SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD: 10m SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH: /var/lib/spindle/debug_ssh_host_key # dials the mill container directly using ws SPINDLE_MILL_URL: ws://spindle:6555/mill SPINDLE_TRACING_ENDPOINT: "${SPINDLE_TRACING_ENDPOINT:-}" SPINDLE_TRACING_INSECURE: "${SPINDLE_TRACING_INSECURE:-false}" SPINDLE_LOG_FORMAT: json SPINDLE_LOGGING_ENDPOINT: "${SPINDLE_LOGGING_ENDPOINT:-}" SPINDLE_LOGGING_INSECURE: "${SPINDLE_LOGGING_INSECURE:-false}" SPINDLE_LOGGING_SERVICE_NAME: spindle devices: - /dev/vsock:/dev/vsock - /dev/kvm:/dev/kvm - /dev/vhost-vsock:/dev/vhost-vsock - /dev/net/tun:/dev/net/tun cap_add: - NET_ADMIN - SYS_ADMIN security_opt: - label=disable - seccomp=unconfined healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] interval: 2s timeout: 2s retries: 30 start_period: 5s depends_on: &mill-executor-deps plc: condition: service_started jetstream: condition: service_started init-accounts: condition: service_completed_successfully ncps: condition: service_started spindle: condition: service_healthy mill-tokens: condition: service_completed_successfully networks: [tngl]
services: # configures the primary spindle container as a mill host spindle: environment: SPINDLE_ROLE: mill SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk SPINDLE_MILL_JUMP_LISTEN_ADDR: 0.0.0.0:2224 SPINDLE_MILL_JUMP_HOST_KEY_PATH: /var/lib/spindle/debug_jump_host_key SPINDLE_MILL_DEBUG_EXECUTOR_PORT: "2223" SPINDLE_MILL_MAX_JUMP_CONNECTIONS: "128" volumes: - spindle-artifacts:/var/lib/spindle/artifacts ports: !override - "127.0.0.1:2224:2224"
prometheus: command: - --config.file=/etc/prometheus/prometheus.mill.yml - --storage.tsdb.path=/prometheus - --enable-feature=exemplar-storage
mill-tokens: profiles: ["linux"] build: context: . dockerfile: localinfra/spindle.Dockerfile restart: "no" entrypoint: ["/bin/sh", "-c"] environment: SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db command: - | set -eu umask 077 seed() { token_file="/shared/$$1.mill-token" if [ ! -s "$$token_file" ]; then spindle mill executor token generate > "$$token_file.tmp" mv "$$token_file.tmp" "$$token_file" fi args="" for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done spindle mill executor add "$$1" --token-file "$$token_file" $$args } seed executor-a linux,fast seed executor-b linux,slow seed executor-c linux,gpu echo "seeded executor tokens" volumes: - spindle-data:/var/lib/spindle - init-state:/shared depends_on: spindle: condition: service_healthy networks: [tngl]
spindle-executor-a: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,fast SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST: executor-a volumes: - spindle-executor-a-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro networks: tngl: aliases: [executor-a]
spindle-executor-b: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,slow SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST: executor-b volumes: - spindle-executor-b-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro networks: tngl: aliases: [executor-b]
spindle-executor-c: <<: *mill-executor environment: <<: *mill-executor-env SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev SPINDLE_MILL_LABELS: linux,gpu SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST: executor-c volumes: - spindle-executor-c-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro networks: tngl: aliases: [executor-c]
volumes: spindle-executor-a-data: spindle-executor-b-data: spindle-executor-c-data: spindle-artifacts: