Something went wrong. Try again.
Monorepo for Tangled tangled.org
Something went wrong. Try again.
Nix
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083{ config, lib, pkgs, ...}: let cfg = config.services.tangled.spindle;in with lib; { options = let processOptions = { package = mkOption { type = types.package; description = "Package to use for the spindle"; };
server = { listenAddr = mkOption { type = types.str; default = "0.0.0.0:6555"; description = "Address to listen on"; };
metricsListenAddr = mkOption { type = types.str; default = "127.0.0.1:9091"; description = "Address for the metrics server to listen on"; };
dbPath = mkOption { type = types.path; default = "/var/lib/spindle/spindle.db"; description = "Path to the database file"; };
repoDir = mkOption { type = types.path; default = "/var/lib/spindle/repos"; description = "Path where synced git repositories live"; };
hostname = mkOption { type = types.str; example = "my.spindle.com"; description = "Hostname for the server (required)"; };
plcUrl = mkOption { type = types.str; default = "https://plc.directory"; description = "atproto PLC directory"; };
jetstreamEndpoint = mkOption { type = types.str; default = "wss://jetstream1.us-west.bsky.network/subscribe"; description = "Jetstream endpoint to subscribe to"; };
dev = mkOption { type = types.bool; default = false; description = "Enable development mode (disables signature verification)"; };
owner = mkOption { type = types.str; example = "did:plc:qfpnj4og54vl56wngdriaxug"; description = "DID of owner (required)"; };
maxJobCount = mkOption { type = types.int; default = 2; example = 5; description = "Maximum number of concurrent jobs to run"; };
queueSize = mkOption { type = types.int; default = 100; example = 100; description = "Maximum number of jobs queue up"; };
secrets = { provider = mkOption { type = types.str; default = "sqlite"; description = "Backend to use for secret management, valid options are 'sqlite', and 'openbao'."; };
openbao = { proxyAddr = mkOption { type = types.str; default = "http://127.0.0.1:8200"; description = "Address of the OpenBAO proxy server"; }; mount = mkOption { type = types.str; default = "spindle"; description = "Mount path in OpenBAO to read secrets from"; }; }; };
tap = { embed = mkOption { type = types.bool; default = true; description = "Run an embedded tap inside the spindle process"; };
url = mkOption { type = types.str; default = "http://[::1]:2480"; description = "URL the spindle's tap client dials"; };
bind = mkOption { type = types.str; default = "[::1]:2480"; description = "Loopback address the embedded tap server listens on"; };
dbPath = mkOption { type = types.path; default = "/var/lib/spindle/tap.db"; description = "Path to the embedded tap sqlite database"; };
relayUrl = mkOption { type = types.str; default = "https://bsky.network"; description = "Relay used by the embedded tap firehose"; }; }; };
artifactStores = { disk.dir = mkOption { type = types.path; default = "/var/log/spindle"; description = "Root directory for disk artifacts"; };
s3.bucket = mkOption { type = types.str; default = "tangled-logs"; description = "S3 bucket for artifacts"; };
s3.region = mkOption { type = types.str; default = "us-east-1"; description = "AWS region for the artifact bucket"; }; };
pipelines = { workflowTimeout = mkOption { type = types.str; default = "5m"; description = "Timeout for a whole workflow, covering the wait for a concurrency slot, setup, and every step in it"; };
nixery = { nixery = mkOption { type = types.str; default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet description = "Nixery instance to use"; };
maxJobMemoryMb = mkOption { type = types.int; default = 6144; description = "Memory limit per nixery workflow container in MiB (default 6 GiB)"; }; maxConcurrentWorkflows = mkOption { type = types.int; default = 8; description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit."; }; };
microvm = { enableKVM = mkOption { type = types.bool; default = true; description = "Enable KVM hardware acceleration"; };
imageDir = mkOption { type = types.str; default = "/var/lib/spindle/images"; description = "Directory containing microVM image spec JSONs or image spec directories"; }; overlayDir = mkOption { type = types.str; default = "/tmp"; description = "Directory to store microVM temporary overlay files"; }; defaultImage = mkOption { type = types.str; default = "nixos"; description = "Default microVM image spec to use if none is specified in workflow"; }; agentPort = mkOption { type = types.port; default = 10240; description = "Host vsock port the microVM agent connects back to"; };
limits = { total = { memoryMiB = mkOption { type = types.int; default = 0; description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit."; }; vcpus = mkOption { type = types.int; default = 0; description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit."; }; diskMiB = mkOption { type = types.int; default = 0; description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit."; }; };
workflow = { memoryMiB = mkOption { type = types.int; default = 0; description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit."; }; vcpus = mkOption { type = types.int; default = 0; description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit."; }; diskMiB = mkOption { type = types.int; default = 0; description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit."; }; }; };
cgroup = { enable = mkOption { type = types.bool; default = false; description = "Enable cgroup v2 containment for microVM processes."; }; parent = mkOption { type = types.str; default = "self"; description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup."; }; pidsMax = mkOption { type = types.int; default = 4096; description = "Maximum number of processes allowed in each microVM workflow cgroup."; }; swapMaxMiB = mkOption { type = types.int; default = 0; description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap."; }; supervisorMinMiB = mkOption { type = types.int; default = 512; description = '' Amount of memory in MiB that will be protected by the cgroup for the spindle (allowing it to not get OOMed first.) ''; }; };
debugSsh = { enable = mkOption { type = types.bool; default = false; description = '' Enable the debug ssh server that lets authorized users ssh into a failed microVM to debug it. ''; }; listenAddr = mkOption { type = types.str; default = "0.0.0.0:2222"; example = "0.0.0.0:2225"; description = "Address for the debug ssh server to listen on."; }; host = mkOption { type = types.str; default = ""; example = "127.0.0.1"; description = "Host reached from the SSH jump host."; }; jumpHost = mkOption { type = types.str; default = ""; example = "spindle.example.com"; description = "SSH jump host used in the printed debug command."; }; hostKeyPath = mkOption { type = with types; nullOr path; default = null; example = "/var/lib/spindle/debug_ssh_host_key"; description = '' Path to the ssh host key for the debug server. If null, one is generated once and persisted next to the spindle db. ''; }; gracePeriod = mkOption { type = types.str; default = "5m"; description = '' How long a failed workflow's microVM is kept alive for the user to ssh in. ''; }; }; };
nixCache = { readUrls = mkOption { type = types.listOf types.str; default = []; example = ["http://ncps.internal:8501" "ssh-ng://user@my-awesome-cache"]; description = "Nix binary cache URLs the Spindle guest should read from."; };
trustedPublicKeys = mkOption { type = types.listOf types.str; default = []; example = ["internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]; description = "Public keys trusted for the configured Nix binary caches."; };
uploadUrl = mkOption { type = types.str; default = ""; example = "local"; description = "Optional cache upload URL used by live cache import paths."; };
requireSignedUploads = mkOption { type = types.bool; default = false; description = "Require uploaded narinfos to have a signature from a configured trusted public key."; }; }; };
tracing = { endpoint = mkOption { type = types.str; default = ""; description = "OpenTelemetry collector endpoint (empty disables tracing)"; };
serviceName = mkOption { type = types.str; default = "spindle"; description = "OpenTelemetry service name"; };
sampleRatio = mkOption { type = types.number; default = 1.0; description = "Fraction of traces to sample"; };
insecure = mkOption { type = types.bool; default = false; description = "Use plaintext OTLP/HTTP"; }; };
logging = { format = mkOption { type = types.enum ["text" "json"]; default = "text"; description = "Local structured log format"; };
endpoint = mkOption { type = types.str; default = ""; description = "OpenTelemetry log collector endpoint (empty disables export)"; };
serviceName = mkOption { type = types.str; default = "spindle"; description = "OpenTelemetry log service name"; };
insecure = mkOption { type = types.bool; default = false; description = "Use plaintext OTLP/HTTP for logs"; }; };
quota = { user = { cacheStorageMiB = mkOption { type = types.int; default = 0; description = "Default cache storage limit for users in MiB. Zero is unlimited."; }; workflows = mkOption { type = types.int; default = 0; description = "Default concurrent workflows limit for users. Zero is unlimited."; }; vCPUs = mkOption { type = types.int; default = 0; description = "Default vCPU limit for users. Zero is unlimited."; }; memoryMiB = mkOption { type = types.int; default = 0; description = "Default memory limit for users in MiB. Zero is unlimited."; }; diskMiB = mkOption { type = types.int; default = 0; description = "Default disk limit for users in MiB. Zero is unlimited."; }; }; repo = { cacheStorageMiB = mkOption { type = types.int; default = 0; description = "Default cache storage limit for repositories in MiB. Zero is unlimited."; }; workflows = mkOption { type = types.int; default = 0; description = "Default concurrent workflows limit for repositories. Zero is unlimited."; }; vCPUs = mkOption { type = types.int; default = 0; description = "Default vCPU limit for repositories. Zero is unlimited."; }; memoryMiB = mkOption { type = types.int; default = 0; description = "Default memory limit for repositories in MiB. Zero is unlimited."; }; diskMiB = mkOption { type = types.int; default = 0; description = "Default disk limit for repositories in MiB. Zero is unlimited."; }; }; };
environmentFile = mkOption { type = with types; nullOr path; default = null; example = "/etc/spindle.env"; description = '' Additional environment file as defined in {manpage}`systemd.exec(5)`.
Sensitive secrets such as {env}`AWS_SECRET_ACCESS_KEY`, {env}`AWS_ACCESS_KEY_ID`, {env}`AWS_REGION` may be passed to the service without making them world readable in the nix store. ''; };
oidc = { issuer = mkOption { type = types.str; default = ""; example = "https://tokens.spindle.tangled.sh"; description = "OIDC issuer origin; the iss claim of CI ID tokens minted for workflows that declare an audience. empty disables minting"; };
keyPath = mkOption { type = types.str; default = ""; example = "/var/lib/secrets/spindle-oidc.pem"; description = "PEM file holding the EC P-256 signing key. required when issuer is set"; };
keyId = mkOption { type = types.str; default = ""; description = "kid of minted tokens. defaults to the RFC 7638 thumbprint of the key"; }; }; };
localServiceType = name: types.submodule ({config, ...}: { options = processOptions // { generateToken = mkOption { type = types.bool; default = false; description = "Generate the token file once at a persistent path."; };
millUrl = mkOption { type = types.str; default = "ws://127.0.0.1:6555/mill"; description = "Mill URL used by this executor."; };
seats = mkOption { type = types.ints.positive; default = 4; description = "Jobs this executor may run at once."; };
stateDirectory = mkOption { type = types.str; default = "spindle-executor-${name}"; description = "State directory used by this executor."; }; };
config = mkMerge [ { package = mkDefault cfg.package; server = mkDefault cfg.server; artifactStores = mkDefault cfg.artifactStores; pipelines = mkDefault cfg.pipelines; environmentFile = mkDefault cfg.environmentFile; tracing = mkDefault cfg.tracing; logging = mkDefault cfg.logging; quota = mkDefault cfg.quota; } { server.listenAddr = mkOverride 900 "127.0.0.1:0"; server.dbPath = mkOverride 900 "/var/lib/${config.stateDirectory}/spindle.db"; server.repoDir = mkOverride 900 "/var/lib/${config.stateDirectory}/repos"; server.hostname = mkOverride 900 name; server.owner = mkOverride 900 ""; # a colocated executor cannot share the mill's metrics port; opt in per executor server.metricsListenAddr = mkOverride 900 ""; pipelines.microvm.debugSsh.listenAddr = mkOverride 900 "127.0.0.1:${toString cfg.mill.debugExecutorPort}"; } ]; });
executorType = types.submodule ({name, ...}: { options = { tokenFile = mkOption { type = types.str; description = "File containing this executor's token."; };
labels = mkOption { type = types.listOf types.str; default = []; description = "Labels this executor may use."; };
localService = mkOption { type = with types; nullOr (localServiceType name); default = null; description = "Service running this executor on the mill host."; }; }; });
dedicatedExecutorType = types.submodule { options = { name = mkOption { type = types.str; description = "Name registered for this executor."; };
millUrl = mkOption { type = types.str; description = "Mill URL used by this executor."; };
tokenFile = mkOption { type = types.str; description = "File containing this executor's token."; };
labels = mkOption { type = types.listOf types.str; default = []; description = "Labels this executor may use."; };
seats = mkOption { type = types.ints.positive; default = 4; description = "Jobs this executor may run at once."; }; }; }; in { services.tangled.spindle = processOptions // { enable = mkOption { type = types.bool; default = false; description = "Enable a tangled spindle"; };
role = mkOption { type = types.enum ["standalone" "mill" "executor"]; default = "standalone"; description = "How the main spindle runs."; };
executor = mkOption { type = with types; nullOr dedicatedExecutorType; default = null; description = "Settings used when the main spindle runs as an executor."; };
mill = { artifactStore = mkOption { type = types.enum ["disk" "s3"]; default = "s3"; description = "Artifact store shared by the mill and its executors."; };
drainTimeout = mkOption { type = types.ints.positive; default = 1200; description = "Seconds an executor waits for running jobs to finish before stopping."; };
executors = mkOption { type = types.attrsOf executorType; default = {}; description = "Executors allowed to connect to this mill."; };
jumpListenAddr = mkOption { type = types.str; default = ""; example = "0.0.0.0:22"; description = "Address for the mill's restricted debug SSH jump server."; };
jumpHostKeyPath = mkOption { type = with types; nullOr path; default = null; example = "/var/lib/spindle/debug_jump_host_key"; description = "Path to the debug SSH jump server host key."; };
debugExecutorPort = mkOption { type = types.port; default = 2223; description = "Private debug SSH port shared by executors."; };
maxJumpConnections = mkOption { type = types.ints.positive; default = 128; description = "Maximum concurrent connections to the mill's debug SSH jump server."; }; }; }; }; config = let deps = [ pkgs.git pkgs.qemu pkgs.e2fsprogs pkgs.slirp4netns pkgs.iproute2 pkgs.util-linux config.nix.package ]; localExecutors = filterAttrs (_: executor: executor.localService != null) cfg.mill.executors; localServices = mapAttrs (_: executor: executor.localService) localExecutors; localAgentPorts = mapAttrsToList (_: service: service.pipelines.microvm.agentPort) localServices; localStateDirectories = mapAttrsToList (_: service: service.stateDirectory) localServices; localDbPaths = mapAttrsToList (_: service: toString service.server.dbPath) localServices; localRepoDirs = mapAttrsToList (_: service: toString service.server.repoDir) localServices; metricsServices = filterAttrs (_: service: service.server.metricsListenAddr != "") localServices; localMetricsListenAddrs = mapAttrsToList (_: service: service.server.metricsListenAddr) metricsServices; debugServices = filterAttrs (_: service: service.pipelines.microvm.debugSsh.enable) localServices; localDebugSshListenAddrs = mapAttrsToList (_: service: service.pipelines.microvm.debugSsh.listenAddr) debugServices; hasRegistrations = cfg.role == "mill" && cfg.mill.executors != {}; isDedicatedExecutor = cfg.role == "executor" && cfg.executor != null; executorServiceName = name: "spindle-executor-${name}"; executorExecStart = name: package: pkgs.writeShellScript name '' export SPINDLE_MILL_SHARED_SECRET="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/mill-token")" exec ${pkgs.coreutils}/bin/env \ SPINDLE_ROLE=executor \ SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} \ SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s \ SPINDLE_MILL_JUMP_LISTEN_ADDR= \ SPINDLE_MILL_JUMP_HOST_KEY_PATH= \ ${package}/bin/spindle '';
processEnvironment = instance: [ "SPINDLE_SERVER_LISTEN_ADDR=${instance.server.listenAddr}" "SPINDLE_SERVER_METRICS_LISTEN_ADDR=${instance.server.metricsListenAddr}" "SPINDLE_SERVER_DB_PATH=${instance.server.dbPath}" "SPINDLE_SERVER_REPO_DIR=${instance.server.repoDir}" "SPINDLE_SERVER_HOSTNAME=${instance.server.hostname}" "SPINDLE_SERVER_PLC_URL=${instance.server.plcUrl}" "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${instance.server.jetstreamEndpoint}" "SPINDLE_SERVER_DEV=${lib.boolToString instance.server.dev}" "SPINDLE_SERVER_OWNER=${instance.server.owner}" "SPINDLE_SERVER_MAX_JOB_COUNT=${toString instance.server.maxJobCount}" "SPINDLE_SERVER_QUEUE_SIZE=${toString instance.server.queueSize}" "SPINDLE_SERVER_SECRETS_PROVIDER=${instance.server.secrets.provider}" "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${instance.server.secrets.openbao.proxyAddr}" "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${instance.server.secrets.openbao.mount}" "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString instance.server.tap.embed}" "SPINDLE_SERVER_TAP_URL=${instance.server.tap.url}" "SPINDLE_SERVER_TAP_BIND=${instance.server.tap.bind}" "SPINDLE_SERVER_TAP_DB_PATH=${instance.server.tap.dbPath}" "SPINDLE_SERVER_TAP_RELAY_URL=${instance.server.tap.relayUrl}" "SPINDLE_NIXERY_PIPELINES_NIXERY=${instance.pipelines.nixery.nixery}" "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}" "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString instance.pipelines.nixery.maxJobMemoryMb}" "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString instance.pipelines.nixery.maxConcurrentWorkflows}" "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${instance.pipelines.microvm.imageDir}" "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${instance.pipelines.microvm.overlayDir}" "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${instance.pipelines.microvm.defaultImage}" "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString instance.pipelines.microvm.agentPort}" "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString instance.pipelines.microvm.enableKVM}" "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}" "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString instance.pipelines.microvm.limits.total.memoryMiB}" "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString instance.pipelines.microvm.limits.total.vcpus}" "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString instance.pipelines.microvm.limits.total.diskMiB}" "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString instance.pipelines.microvm.limits.workflow.memoryMiB}" "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString instance.pipelines.microvm.limits.workflow.vcpus}" "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString instance.pipelines.microvm.limits.workflow.diskMiB}" "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString instance.pipelines.microvm.cgroup.enable}" "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${instance.pipelines.microvm.cgroup.parent}" "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString instance.pipelines.microvm.cgroup.pidsMax}" "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString instance.pipelines.microvm.cgroup.swapMaxMiB}" "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString instance.pipelines.microvm.cgroup.supervisorMinMiB}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED=${lib.boolToString instance.pipelines.microvm.debugSsh.enable}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR=${instance.pipelines.microvm.debugSsh.listenAddr}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST=${instance.pipelines.microvm.debugSsh.host}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST=${instance.pipelines.microvm.debugSsh.jumpHost}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH=${optionalString (instance.pipelines.microvm.debugSsh.hostKeyPath != null) (toString instance.pipelines.microvm.debugSsh.hostKeyPath)}" "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD=${instance.pipelines.microvm.debugSsh.gracePeriod}" "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," instance.pipelines.nixCache.readUrls}" "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," instance.pipelines.nixCache.trustedPublicKeys}" "SPINDLE_NIX_CACHE_UPLOAD_URL=${instance.pipelines.nixCache.uploadUrl}" "SPINDLE_NIX_CACHE_REQUIRE_SIGNED_UPLOADS=${lib.boolToString instance.pipelines.nixCache.requireSignedUploads}" "SPINDLE_QUOTA_USER_CACHE_STORAGE_MIB=${toString instance.quota.user.cacheStorageMiB}" "SPINDLE_QUOTA_USER_WORKFLOWS=${toString instance.quota.user.workflows}" "SPINDLE_QUOTA_USER_VCPUS=${toString instance.quota.user.vCPUs}" "SPINDLE_QUOTA_USER_MEMORY_MIB=${toString instance.quota.user.memoryMiB}" "SPINDLE_QUOTA_USER_DISK_MIB=${toString instance.quota.user.diskMiB}" "SPINDLE_QUOTA_REPO_CACHE_STORAGE_MIB=${toString instance.quota.repo.cacheStorageMiB}" "SPINDLE_QUOTA_REPO_WORKFLOWS=${toString instance.quota.repo.workflows}" "SPINDLE_QUOTA_REPO_VCPUS=${toString instance.quota.repo.vCPUs}" "SPINDLE_QUOTA_REPO_MEMORY_MIB=${toString instance.quota.repo.memoryMiB}" "SPINDLE_QUOTA_REPO_DISK_MIB=${toString instance.quota.repo.diskMiB}" "SPINDLE_ARTIFACT_STORES_DISK_DIR=${instance.artifactStores.disk.dir}" "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${instance.artifactStores.s3.bucket}" "SPINDLE_ARTIFACT_STORES_S3_REGION=${instance.artifactStores.s3.region}" "SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore}" "SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s" "SPINDLE_TRACING_ENDPOINT=${instance.tracing.endpoint}" "SPINDLE_TRACING_SERVICE_NAME=${instance.tracing.serviceName}" "SPINDLE_TRACING_SAMPLE_RATIO=${toString instance.tracing.sampleRatio}" "SPINDLE_TRACING_INSECURE=${lib.boolToString instance.tracing.insecure}" "SPINDLE_LOG_FORMAT=${instance.logging.format}" "SPINDLE_LOGGING_ENDPOINT=${instance.logging.endpoint}" "SPINDLE_LOGGING_SERVICE_NAME=${instance.logging.serviceName}" "SPINDLE_LOGGING_INSECURE=${lib.boolToString instance.logging.insecure}" ];
connectionEnvironment = executor: [ "SPINDLE_MILL_URL=${executor.millUrl}" "SPINDLE_MILL_SEATS=${toString executor.seats}" "SPINDLE_MILL_LABELS=${concatStringsSep "," executor.labels}" ];
mainEnvironment = processEnvironment cfg ++ [ "SPINDLE_ROLE=${cfg.role}" "SPINDLE_MILL_JUMP_LISTEN_ADDR=${cfg.mill.jumpListenAddr}" "SPINDLE_MILL_JUMP_HOST_KEY_PATH=${optionalString (cfg.mill.jumpHostKeyPath != null) (toString cfg.mill.jumpHostKeyPath)}" "SPINDLE_MILL_DEBUG_EXECUTOR_PORT=${toString cfg.mill.debugExecutorPort}" "SPINDLE_MILL_MAX_JUMP_CONNECTIONS=${toString cfg.mill.maxJumpConnections}" ] # only the mill mints tokens; the key must not reach executor environments ++ optionals (cfg.role == "mill") [ "SPINDLE_OIDC_ISSUER=${cfg.oidc.issuer}" "SPINDLE_OIDC_KEY_PATH=${cfg.oidc.keyPath}" "SPINDLE_OIDC_KEY_ID=${cfg.oidc.keyId}" ] ++ optionals isDedicatedExecutor (connectionEnvironment cfg.executor);
localExecutorEnvironment = executor: processEnvironment executor.localService ++ [ "SPINDLE_ROLE=executor" "SPINDLE_SERVER_LOG_DIR=/var/log/${executor.localService.stateDirectory}" ] ++ connectionEnvironment { inherit (executor) labels; inherit (executor.localService) millUrl seats; };
registrationCommands = concatStringsSep "\n" (mapAttrsToList ( name: executor: let tokenFile = escapeShellArg executor.tokenFile; tokenDir = escapeShellArg (builtins.dirOf executor.tokenFile); tokenTemplate = escapeShellArg "${executor.tokenFile}.tmp.XXXXXX"; generateCommand = escapeShellArgs [ "${cfg.package}/bin/spindle" "mill" "executor" "token" "generate" "--db" (toString cfg.server.dbPath) ]; registerCommand = escapeShellArgs ( [ "${cfg.package}/bin/spindle" "mill" "executor" "add" "--db" (toString cfg.server.dbPath) "--token-file" executor.tokenFile ] ++ concatMap (label: ["--label" label]) executor.labels ++ [name] ); emptyGeneratedToken = escapeShellArg "generated token for ${name} is empty"; missingToken = escapeShellArg "token file for ${name} is missing or empty: ${executor.tokenFile}"; in '' ${optionalString (executor.localService != null && executor.localService.generateToken) '' if [ ! -e ${tokenFile} ]; then if [ ! -d ${tokenDir} ]; then ${pkgs.coreutils}/bin/install -d -m 0700 ${tokenDir} fi umask 0077 tmp="$(${pkgs.coreutils}/bin/mktemp ${tokenTemplate})" trap '${pkgs.coreutils}/bin/rm -f "$tmp"' EXIT ${generateCommand} > "$tmp" if [ ! -s "$tmp" ]; then printf '%s\n' ${emptyGeneratedToken} >&2 exit 1 fi ${pkgs.coreutils}/bin/chmod 0600 "$tmp" ${pkgs.coreutils}/bin/mv "$tmp" ${tokenFile} trap - EXIT fi ''} if [ ! -f ${tokenFile} ] || [ ! -s ${tokenFile} ]; then printf '%s\n' ${missingToken} >&2 exit 1 fi ${registerCommand} '' ) cfg.mill.executors);
executorServices = mapAttrs' ( name: executor: let service = executor.localService; in nameValuePair (executorServiceName name) { description = "spindle mill executor ${name}"; after = [ "network.target" "docker.service" "spindle.service" "spindle-mill-executors.service" ]; requires = [ "docker.service" "spindle-mill-executors.service" ]; partOf = ["spindle.service"]; wantedBy = ["multi-user.target"]; path = deps; serviceConfig = { LogsDirectory = service.stateDirectory; StateDirectory = service.stateDirectory; Delegate = service.pipelines.microvm.cgroup.enable; EnvironmentFile = mkIf (service.environmentFile != null) service.environmentFile; Environment = localExecutorEnvironment executor; LoadCredential = "mill-token:${executor.tokenFile}"; ExecStart = executorExecStart (executorServiceName name) service.package; TimeoutStopSec = cfg.mill.drainTimeout + 510; Restart = "always"; }; } ) localExecutors; in mkIf cfg.enable { assertions = mapAttrsToList (name: executor: { assertion = !hasPrefix "/nix/store/" executor.tokenFile; message = "services.tangled.spindle.mill.executors.${name}.tokenFile must not point into the Nix store"; }) cfg.mill.executors ++ mapAttrsToList (name: executor: { assertion = hasPrefix "/" executor.tokenFile; message = "services.tangled.spindle.mill.executors.${name}.tokenFile must be an absolute path"; }) cfg.mill.executors ++ mapAttrsToList (name: executor: { assertion = executor.localService == null || !executor.localService.generateToken || (!hasPrefix "/run/" executor.tokenFile && !hasPrefix "/tmp/" executor.tokenFile); message = "services.tangled.spindle.mill.executors.${name}.tokenFile must persist when localService.generateToken is enabled"; }) cfg.mill.executors ++ [ { assertion = (cfg.role == "executor") == (cfg.executor != null); message = "services.tangled.spindle.executor must be set exactly when role is executor"; } { assertion = cfg.mill.executors == {} || cfg.role == "mill"; message = "services.tangled.spindle.mill.executors requires role mill"; } { assertion = cfg.executor == null || !hasPrefix "/nix/store/" cfg.executor.tokenFile; message = "services.tangled.spindle.executor.tokenFile must not point into the Nix store"; } { assertion = cfg.executor == null || hasPrefix "/" cfg.executor.tokenFile; message = "services.tangled.spindle.executor.tokenFile must be an absolute path"; } { assertion = length localAgentPorts == length (unique localAgentPorts); message = "services.tangled.spindle.mill.executors must use different localService.pipelines.microvm.agentPort values"; } { assertion = all (port: port != cfg.pipelines.microvm.agentPort) localAgentPorts; message = "services.tangled.spindle.mill.executors must not use the main spindle's pipelines.microvm.agentPort"; } { assertion = length localStateDirectories == length (unique localStateDirectories); message = "services.tangled.spindle.mill.executors must use different localService state directories"; } { assertion = !elem "spindle" localStateDirectories; message = "services.tangled.spindle.mill.executors must not use the main spindle's state directory"; } { assertion = length localDbPaths == length (unique localDbPaths); message = "services.tangled.spindle.mill.executors must use different localService.server.dbPath values"; } { assertion = !elem (toString cfg.server.dbPath) localDbPaths; message = "services.tangled.spindle.mill.executors must not use the main spindle's server.dbPath"; } { assertion = length localRepoDirs == length (unique localRepoDirs); message = "services.tangled.spindle.mill.executors must use different localService.server.repoDir values"; } { assertion = !elem (toString cfg.server.repoDir) localRepoDirs; message = "services.tangled.spindle.mill.executors must not use the main spindle's server.repoDir"; } { assertion = length localDebugSshListenAddrs == length (unique localDebugSshListenAddrs); message = "services.tangled.spindle.mill.executors must use different localService.pipelines.microvm.debugSsh.listenAddr values"; } { assertion = length localMetricsListenAddrs == length (unique localMetricsListenAddrs); message = "services.tangled.spindle.mill.executors must use different localService.server.metricsListenAddr values"; } { assertion = !elem cfg.server.metricsListenAddr localMetricsListenAddrs; message = "services.tangled.spindle.mill.executors must not use the main spindle's server.metricsListenAddr"; } { assertion = if cfg.mill.artifactStore == "disk" then all (service: toString service.artifactStores.disk.dir == toString cfg.artifactStores.disk.dir) (attrValues localServices) else all ( service: service.artifactStores.s3.bucket == cfg.artifactStores.s3.bucket && service.artifactStores.s3.region == cfg.artifactStores.s3.region ) (attrValues localServices); message = "services.tangled.spindle.mill.executors must use the main spindle's configured artifact store"; } ];
services.tangled.spindle.server = mkIf isDedicatedExecutor { listenAddr = mkDefault "127.0.0.1:0"; hostname = mkForce cfg.executor.name; owner = mkForce ""; };
environment.systemPackages = [ (pkgs.writeShellScriptBin "spindle" '' export PATH="${lib.makeBinPath deps}:$PATH" ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"} ${lib.concatMapStringsSep "\n" (entry: "export ${entry}") mainEnvironment} ${lib.optionalString isDedicatedExecutor "export SPINDLE_MILL_SHARED_SECRET=\"$(${pkgs.coreutils}/bin/cat ${escapeShellArg cfg.executor.tokenFile})\""} exec ${cfg.package}/bin/spindle "$@" '') ];
virtualisation.docker.enable = true;
boot.kernelModules = ["vhost" "vsock" "vhost_vsock"];
systemd.services = { spindle-mill-executors = mkIf hasRegistrations { description = "register spindle mill executors"; before = ["spindle.service"] ++ mapAttrsToList (name: _: "${executorServiceName name}.service") localExecutors; wantedBy = ["multi-user.target"]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; StateDirectory = "spindle"; }; script = registrationCommands; };
spindle = { description = "spindle service"; after = [ "network.target" "docker.service" ] ++ optional hasRegistrations "spindle-mill-executors.service"; requires = optional hasRegistrations "spindle-mill-executors.service"; wantedBy = ["multi-user.target"]; path = deps; serviceConfig = { LogsDirectory = "spindle"; StateDirectory = "spindle"; Delegate = cfg.pipelines.microvm.cgroup.enable; EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; Environment = mainEnvironment; LoadCredential = mkIf isDedicatedExecutor "mill-token:${cfg.executor.tokenFile}"; ExecStart = if isDedicatedExecutor then executorExecStart "spindle-executor" cfg.package else "${pkgs.coreutils}/bin/env SPINDLE_ROLE=${cfg.role} SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s ${cfg.package}/bin/spindle"; TimeoutStopSec = cfg.mill.drainTimeout + 510; Restart = "always"; RestartMode = mkIf (localExecutors != {}) "direct"; }; }; } // executorServices; }; }