diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix index 117de611..e4bf62b9 100644 --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -7,13 +7,8 @@ cfg = config.services.tangled.spindle; in with lib; { - options = { - services.tangled.spindle = { - enable = mkOption { - type = types.bool; - default = false; - description = "Enable a tangled spindle"; - }; + options = let + processOptions = { package = mkOption { type = types.package; description = "Package to use for the spindle"; @@ -348,39 +343,6 @@ in }; }; }; - mill = { - drainTimeout = mkOption { - type = types.ints.positive; - default = 1200; - description = "Seconds an executor waits for running jobs to finish before stopping."; - }; - - jumpListenAddr = mkOption { - type = types.str; - default = ""; - example = "0.0.0.0:22"; - description = "Address for the mill's restricted debug SSH jump server."; - }; - - jumpHostKeyPath = mkOption { - type = with types; nullOr path; - default = null; - example = "/var/lib/spindle/debug_jump_host_key"; - description = "Path to the debug SSH jump server host key."; - }; - - debugExecutorPort = mkOption { - type = types.port; - default = 2223; - description = "Private debug SSH port shared by executors."; - }; - - maxJumpConnections = mkOption { - type = types.ints.positive; - default = 128; - description = "Maximum concurrent connections to the mill's debug SSH jump server."; - }; - }; environmentFile = mkOption { type = with types; nullOr path; @@ -396,8 +358,131 @@ in ''; }; }; - }; + executorType = types.submodule ({ + name, + config, + ... + }: { + options = + processOptions + // { + enable = mkOption { + type = types.bool; + default = false; + description = "Run this executor on the same host."; + }; + + tokenFile = mkOption { + type = types.str; + description = "File containing this executor's token."; + }; + + labels = mkOption { + type = types.listOf types.str; + default = []; + description = "Labels this executor may use."; + }; + + url = mkOption { + type = types.str; + default = "ws://127.0.0.1:6555/mill"; + description = "Mill URL used by this executor."; + }; + + seats = mkOption { + type = types.ints.positive; + default = 4; + description = "Jobs this executor may run at once."; + }; + + stateDirectory = mkOption { + type = types.str; + default = "spindle-executor-${name}"; + description = "State directory used by this executor."; + }; + }; + + config = mkMerge [ + { + package = mkDefault cfg.package; + server = mkDefault cfg.server; + artifactStores = mkDefault cfg.artifactStores; + pipelines = mkDefault cfg.pipelines; + environmentFile = mkDefault cfg.environmentFile; + } + { + server.listenAddr = mkOverride 900 "127.0.0.1:0"; + server.dbPath = mkOverride 900 "/var/lib/${config.stateDirectory}/spindle.db"; + server.repoDir = mkOverride 900 "/var/lib/${config.stateDirectory}/repos"; + server.hostname = mkOverride 900 name; + pipelines.microvm.debugSsh.listenAddr = mkOverride 900 "127.0.0.1:${toString cfg.mill.debugExecutorPort}"; + } + ]; + }); + in { + services.tangled.spindle = + processOptions + // { + enable = mkOption { + type = types.bool; + default = false; + description = "Enable a tangled spindle"; + }; + + role = mkOption { + type = types.enum ["standalone" "mill"]; + default = "standalone"; + description = "How the main spindle runs."; + }; + + mill = { + artifactStore = mkOption { + type = types.enum ["disk" "s3"]; + default = "s3"; + description = "Artifact store shared by the mill and its executors."; + }; + + drainTimeout = mkOption { + type = types.ints.positive; + default = 1200; + description = "Seconds an executor waits for running jobs to finish before stopping."; + }; + + executors = mkOption { + type = types.attrsOf executorType; + default = {}; + description = "Executors allowed to connect to this mill."; + }; + + jumpListenAddr = mkOption { + type = types.str; + default = ""; + example = "0.0.0.0:22"; + description = "Address for the mill's restricted debug SSH jump server."; + }; + + jumpHostKeyPath = mkOption { + type = with types; nullOr path; + default = null; + example = "/var/lib/spindle/debug_jump_host_key"; + description = "Path to the debug SSH jump server host key."; + }; + + debugExecutorPort = mkOption { + type = types.port; + default = 2223; + description = "Private debug SSH port shared by executors."; + }; + + maxJumpConnections = mkOption { + type = types.ints.positive; + default = 128; + description = "Maximum concurrent connections to the mill's debug SSH jump server."; + }; + }; + }; + }; config = let deps = [ pkgs.git @@ -408,99 +493,264 @@ in pkgs.util-linux config.nix.package ]; + enabledExecutors = filterAttrs (_: executor: executor.enable) cfg.mill.executors; + enabledAgentPorts = mapAttrsToList (_: executor: executor.pipelines.microvm.agentPort) enabledExecutors; + enabledStateDirectories = mapAttrsToList (_: executor: executor.stateDirectory) enabledExecutors; + enabledDbPaths = mapAttrsToList (_: executor: toString executor.server.dbPath) enabledExecutors; + enabledRepoDirs = mapAttrsToList (_: executor: toString executor.server.repoDir) enabledExecutors; + debugExecutors = filterAttrs (_: executor: executor.pipelines.microvm.debugSsh.enable) enabledExecutors; + enabledDebugSshListenAddrs = + mapAttrsToList (_: executor: executor.pipelines.microvm.debugSsh.listenAddr) debugExecutors; + hasExecutors = cfg.mill.executors != {}; + executorServiceName = name: "spindle-executor-${name}"; + + processEnvironment = instance: [ + "SPINDLE_SERVER_LISTEN_ADDR=${instance.server.listenAddr}" + "SPINDLE_SERVER_DB_PATH=${instance.server.dbPath}" + "SPINDLE_SERVER_REPO_DIR=${instance.server.repoDir}" + "SPINDLE_SERVER_HOSTNAME=${instance.server.hostname}" + "SPINDLE_SERVER_PLC_URL=${instance.server.plcUrl}" + "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${instance.server.jetstreamEndpoint}" + "SPINDLE_SERVER_DEV=${lib.boolToString instance.server.dev}" + "SPINDLE_SERVER_OWNER=${instance.server.owner}" + "SPINDLE_SERVER_MAX_JOB_COUNT=${toString instance.server.maxJobCount}" + "SPINDLE_SERVER_QUEUE_SIZE=${toString instance.server.queueSize}" + "SPINDLE_SERVER_SECRETS_PROVIDER=${instance.server.secrets.provider}" + "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${instance.server.secrets.openbao.proxyAddr}" + "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${instance.server.secrets.openbao.mount}" + "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString instance.server.tap.embed}" + "SPINDLE_SERVER_TAP_URL=${instance.server.tap.url}" + "SPINDLE_SERVER_TAP_BIND=${instance.server.tap.bind}" + "SPINDLE_SERVER_TAP_DB_PATH=${instance.server.tap.dbPath}" + "SPINDLE_SERVER_TAP_RELAY_URL=${instance.server.tap.relayUrl}" + "SPINDLE_NIXERY_PIPELINES_NIXERY=${instance.pipelines.nixery.nixery}" + "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}" + "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString instance.pipelines.nixery.maxJobMemoryMb}" + "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString instance.pipelines.nixery.maxConcurrentWorkflows}" + "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${instance.pipelines.microvm.imageDir}" + "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${instance.pipelines.microvm.overlayDir}" + "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${instance.pipelines.microvm.defaultImage}" + "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString instance.pipelines.microvm.agentPort}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString instance.pipelines.microvm.enableKVM}" + "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${instance.pipelines.workflowTimeout}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString instance.pipelines.microvm.limits.total.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString instance.pipelines.microvm.limits.total.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString instance.pipelines.microvm.limits.total.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString instance.pipelines.microvm.limits.workflow.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString instance.pipelines.microvm.limits.workflow.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString instance.pipelines.microvm.limits.workflow.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString instance.pipelines.microvm.cgroup.enable}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${instance.pipelines.microvm.cgroup.parent}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString instance.pipelines.microvm.cgroup.pidsMax}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString instance.pipelines.microvm.cgroup.swapMaxMiB}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString instance.pipelines.microvm.cgroup.supervisorMinMiB}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED=${lib.boolToString instance.pipelines.microvm.debugSsh.enable}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR=${instance.pipelines.microvm.debugSsh.listenAddr}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST=${instance.pipelines.microvm.debugSsh.host}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST=${instance.pipelines.microvm.debugSsh.jumpHost}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH=${optionalString (instance.pipelines.microvm.debugSsh.hostKeyPath != null) (toString instance.pipelines.microvm.debugSsh.hostKeyPath)}" + "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD=${instance.pipelines.microvm.debugSsh.gracePeriod}" + "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," instance.pipelines.nixCache.readUrls}" + "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," instance.pipelines.nixCache.trustedPublicKeys}" + "SPINDLE_NIX_CACHE_UPLOAD_URL=${instance.pipelines.nixCache.uploadUrl}" + "SPINDLE_ARTIFACT_STORES_DISK_DIR=${instance.artifactStores.disk.dir}" + "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${instance.artifactStores.s3.bucket}" + "SPINDLE_ARTIFACT_STORES_S3_REGION=${instance.artifactStores.s3.region}" + "SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore}" + "SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s" + ]; + + mainEnvironment = + processEnvironment cfg + ++ [ + "SPINDLE_ROLE=${cfg.role}" + "SPINDLE_MILL_JUMP_LISTEN_ADDR=${cfg.mill.jumpListenAddr}" + "SPINDLE_MILL_JUMP_HOST_KEY_PATH=${optionalString (cfg.mill.jumpHostKeyPath != null) (toString cfg.mill.jumpHostKeyPath)}" + "SPINDLE_MILL_DEBUG_EXECUTOR_PORT=${toString cfg.mill.debugExecutorPort}" + "SPINDLE_MILL_MAX_JUMP_CONNECTIONS=${toString cfg.mill.maxJumpConnections}" + ]; + + executorEnvironment = executor: + processEnvironment executor + ++ [ + "SPINDLE_ROLE=executor" + "SPINDLE_MILL_URL=${executor.url}" + "SPINDLE_MILL_SEATS=${toString executor.seats}" + "SPINDLE_MILL_LABELS=${concatStringsSep "," executor.labels}" + ]; + + registrationCommands = concatStringsSep "\n" (mapAttrsToList ( + name: executor: + escapeShellArgs ( + [ + "${cfg.package}/bin/spindle" + "mill" + "executor" + "add" + "--db" + (toString cfg.server.dbPath) + "--token-file" + executor.tokenFile + ] + ++ concatMap (label: ["--label" label]) executor.labels + ++ [name] + ) + ) + cfg.mill.executors); + + executorServices = + mapAttrs' ( + name: executor: + nameValuePair (executorServiceName name) { + description = "spindle mill executor ${name}"; + after = [ + "network.target" + "docker.service" + "spindle.service" + "spindle-mill-executors.service" + ]; + requires = [ + "docker.service" + "spindle-mill-executors.service" + ]; + wantedBy = ["multi-user.target"]; + path = deps; + serviceConfig = { + LogsDirectory = "spindle"; + StateDirectory = executor.stateDirectory; + Delegate = executor.pipelines.microvm.cgroup.enable; + EnvironmentFile = mkIf (executor.environmentFile != null) executor.environmentFile; + Environment = executorEnvironment executor; + LoadCredential = "mill-token:${executor.tokenFile}"; + ExecStart = pkgs.writeShellScript (executorServiceName name) '' + export SPINDLE_MILL_SHARED_SECRET="$(${pkgs.coreutils}/bin/cat "$CREDENTIALS_DIRECTORY/mill-token")" + exec ${pkgs.coreutils}/bin/env \ + SPINDLE_ROLE=executor \ + SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} \ + SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s \ + SPINDLE_MILL_JUMP_LISTEN_ADDR= \ + SPINDLE_MILL_JUMP_HOST_KEY_PATH= \ + ${executor.package}/bin/spindle + ''; + TimeoutStopSec = cfg.mill.drainTimeout + 510; + Restart = "always"; + }; + } + ) + enabledExecutors; in mkIf cfg.enable { + assertions = + mapAttrsToList (name: executor: { + assertion = !hasPrefix "/nix/store/" executor.tokenFile; + message = "services.tangled.spindle.mill.executors.${name}.tokenFile must not point into the Nix store"; + }) + cfg.mill.executors + ++ mapAttrsToList (name: executor: { + assertion = hasPrefix "/" executor.tokenFile; + message = "services.tangled.spindle.mill.executors.${name}.tokenFile must be an absolute path"; + }) + cfg.mill.executors + ++ [ + { + assertion = length enabledAgentPorts == length (unique enabledAgentPorts); + message = "services.tangled.spindle.mill.executors must use different pipelines.microvm.agentPort values"; + } + { + assertion = all (port: port != cfg.pipelines.microvm.agentPort) enabledAgentPorts; + message = "services.tangled.spindle.mill.executors must not use the main spindle's pipelines.microvm.agentPort"; + } + { + assertion = length enabledStateDirectories == length (unique enabledStateDirectories); + message = "services.tangled.spindle.mill.executors must use different state directories"; + } + { + assertion = !elem "spindle" enabledStateDirectories; + message = "services.tangled.spindle.mill.executors must not use the main spindle's state directory"; + } + { + assertion = length enabledDbPaths == length (unique enabledDbPaths); + message = "services.tangled.spindle.mill.executors must use different server.dbPath values"; + } + { + assertion = !elem (toString cfg.server.dbPath) enabledDbPaths; + message = "services.tangled.spindle.mill.executors must not use the main spindle's server.dbPath"; + } + { + assertion = length enabledRepoDirs == length (unique enabledRepoDirs); + message = "services.tangled.spindle.mill.executors must use different server.repoDir values"; + } + { + assertion = !elem (toString cfg.server.repoDir) enabledRepoDirs; + message = "services.tangled.spindle.mill.executors must not use the main spindle's server.repoDir"; + } + { + assertion = length enabledDebugSshListenAddrs == length (unique enabledDebugSshListenAddrs); + message = "services.tangled.spindle.mill.executors must use different pipelines.microvm.debugSsh.listenAddr values"; + } + { + assertion = + if cfg.mill.artifactStore == "disk" + then all (executor: toString executor.artifactStores.disk.dir == toString cfg.artifactStores.disk.dir) (attrValues enabledExecutors) + else + all ( + executor: + executor.artifactStores.s3.bucket + == cfg.artifactStores.s3.bucket + && executor.artifactStores.s3.region == cfg.artifactStores.s3.region + ) + (attrValues enabledExecutors); + message = "services.tangled.spindle.mill.executors must use the main spindle's configured artifact store"; + } + ]; + environment.systemPackages = [ (pkgs.writeShellScriptBin "spindle" '' export PATH="${lib.makeBinPath deps}:$PATH" ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"} - ${lib.concatMapStringsSep "\n" ( - e: "export ${e}" - ) - config.systemd.services.spindle.serviceConfig.Environment} + ${lib.concatMapStringsSep "\n" (entry: "export ${entry}") mainEnvironment} exec ${cfg.package}/bin/spindle "$@" '') ]; virtualisation.docker.enable = true; - systemd.services.spindle = { - description = "spindle service"; - after = [ - "network.target" - "docker.service" - ]; - wantedBy = ["multi-user.target"]; - path = deps; - serviceConfig = { - LogsDirectory = "spindle"; - StateDirectory = "spindle"; - Delegate = cfg.pipelines.microvm.cgroup.enable; - EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; - - Environment = [ - "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}" - "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}" - "SPINDLE_SERVER_REPO_DIR=${cfg.server.repoDir}" - "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}" - "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}" - "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}" - "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" - "SPINDLE_SERVER_OWNER=${cfg.server.owner}" - "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}" - "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}" - "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" - "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" - "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" - "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}" - "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}" - "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}" - "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}" - "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}" - "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}" - "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" - "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}" - "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}" - "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}" - "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}" - "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}" - "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}" - "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}" - "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" - "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}" - "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}" - "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}" - "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}" - "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}" - "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}" - "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}" - "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}" - "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}" - "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}" - "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_ENABLED=${lib.boolToString cfg.pipelines.microvm.debugSsh.enable}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_LISTEN_ADDR=${cfg.pipelines.microvm.debugSsh.listenAddr}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST=${cfg.pipelines.microvm.debugSsh.host}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_JUMP_HOST=${cfg.pipelines.microvm.debugSsh.jumpHost}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_HOST_KEY_PATH=${optionalString (cfg.pipelines.microvm.debugSsh.hostKeyPath != null) (toString cfg.pipelines.microvm.debugSsh.hostKeyPath)}" - "SPINDLE_MICROVM_PIPELINES_DEBUG_SSH_GRACE_PERIOD=${cfg.pipelines.microvm.debugSsh.gracePeriod}" - "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.pipelines.nixCache.readUrls}" - "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.pipelines.nixCache.trustedPublicKeys}" - "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.pipelines.nixCache.uploadUrl}" - "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}" - "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}" - "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}" - "SPINDLE_MILL_ARTIFACT_STORE=s3" - "SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s" - "SPINDLE_MILL_JUMP_LISTEN_ADDR=${cfg.mill.jumpListenAddr}" - "SPINDLE_MILL_JUMP_HOST_KEY_PATH=${optionalString (cfg.mill.jumpHostKeyPath != null) (toString cfg.mill.jumpHostKeyPath)}" - "SPINDLE_MILL_DEBUG_EXECUTOR_PORT=${toString cfg.mill.debugExecutorPort}" - "SPINDLE_MILL_MAX_JUMP_CONNECTIONS=${toString cfg.mill.maxJumpConnections}" - ]; - ExecStart = "${pkgs.coreutils}/bin/env SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s ${cfg.package}/bin/spindle"; - TimeoutStopSec = cfg.mill.drainTimeout + 510; - Restart = "always"; - }; - }; + systemd.services = + { + spindle-mill-executors = mkIf hasExecutors { + description = "register spindle mill executors"; + before = + ["spindle.service"] + ++ mapAttrsToList (name: _: "${executorServiceName name}.service") enabledExecutors; + wantedBy = ["multi-user.target"]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + StateDirectory = "spindle"; + }; + script = registrationCommands; + }; + + spindle = { + description = "spindle service"; + after = [ + "network.target" + "docker.service" + ]; + wantedBy = ["multi-user.target"]; + path = deps; + serviceConfig = { + LogsDirectory = "spindle"; + StateDirectory = "spindle"; + Delegate = cfg.pipelines.microvm.cgroup.enable; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + Environment = mainEnvironment; + ExecStart = "${pkgs.coreutils}/bin/env SPINDLE_ROLE=${cfg.role} SPINDLE_MILL_ARTIFACT_STORE=${cfg.mill.artifactStore} SPINDLE_MILL_DRAIN_TIMEOUT=${toString cfg.mill.drainTimeout}s ${cfg.package}/bin/spindle"; + TimeoutStopSec = cfg.mill.drainTimeout + 510; + Restart = "always"; + }; + }; + } + // executorServices; }; }