From e752854621560225e7dba8b20d7066c059a62034 Mon Sep 17 00:00:00 2001 From: Aly Raffauf Date: Sun, 16 Aug 2026 09:39:07 -0400 Subject: [PATCH] perf: avoid repeated deployment evaluation --- .tangled/workflows/build.yml | 2 +- docs/flake-integration.md | 5 +- flake/blzrd.nix | 3 +- src/main.rs | 7 +- src/models.rs | 9 ++ src/nix/deploy.rs | 4 +- src/nix/evaluate.rs | 189 +++++++++++++++++++---------------- 7 files changed, 125 insertions(+), 94 deletions(-) diff --git a/.tangled/workflows/build.yml b/.tangled/workflows/build.yml index ad64fbb..29da05e 100644 --- a/.tangled/workflows/build.yml +++ b/.tangled/workflows/build.yml @@ -10,4 +10,4 @@ dependencies: steps: - name: Build - command: nix build --print-build-logs \ No newline at end of file + command: nix build --print-build-logs diff --git a/docs/flake-integration.md b/docs/flake-integration.md index 49d2030..5c97871 100644 --- a/docs/flake-integration.md +++ b/docs/flake-integration.md @@ -34,7 +34,10 @@ configuration: ``` The module exposes `blzrd.nodes` as a flake output and adds -`checks..blzrd-nodes`. The check is enabled by default. To disable it: +`checks..blzrd-nodes`. The public node output stays lazy, so listing +nodes or reading deployment metadata does not evaluate their system +derivations. The check is enabled by default and performs full validation. To +disable it: ```nix blzrd.checks.enable = false; diff --git a/flake/blzrd.nix b/flake/blzrd.nix index 7871bba..5eb2007 100644 --- a/flake/blzrd.nix +++ b/flake/blzrd.nix @@ -5,7 +5,6 @@ }: let cfg = config.blzrd; blzrdLib = import ./lib.nix {inherit lib;}; - validatedNodes = blzrdLib.assertValidNodes cfg.nodes; in { options.blzrd = { nodes = lib.mkOption { @@ -48,7 +47,7 @@ in { }; config = { - flake.blzrd.nodes = validatedNodes; + flake.blzrd.nodes = cfg.nodes; perSystem = {pkgs, ...}: lib.mkIf cfg.checks.enable { diff --git a/src/main.rs b/src/main.rs index 11ca45d..5d683b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -19,9 +19,13 @@ async fn main() -> anyhow::Result<()> { .init(); let ui = ui::Ui::new(); + let (op, common) = args.command.into_deploy(); ui.start_section("Evaluating"); - let eval_result = nix::eval_deployments(&args.flake, |name| ui.finish_job_success(&name)).await; + let eval_result = nix::eval_deployments(&args.flake, &common.nodes, |name| { + ui.finish_job_success(&name) + }) + .await; let jobs = match eval_result { Ok(result) => { @@ -35,7 +39,6 @@ async fn main() -> anyhow::Result<()> { } }; - let (op, common) = args.command.into_deploy(); match workflow::run_deploy(op, common, jobs, &ui).await { Ok(()) => ui.finalize_success(), Err(error) => { diff --git a/src/models.rs b/src/models.rs index 7b8a0f3..8045ab5 100644 --- a/src/models.rs +++ b/src/models.rs @@ -56,6 +56,15 @@ pub struct NixEvalJobsResult { pub attr_path: Option>, pub drv_path: Option, pub error: Option, + pub extra_value: Option, pub outputs: Option>, pub system: Option, } + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct NodeMetadata { + pub hostname: Option, + pub user: Option, + #[serde(rename = "type")] + pub type_name: Option, +} diff --git a/src/nix/deploy.rs b/src/nix/deploy.rs index 3bb31ca..4be8c52 100644 --- a/src/nix/deploy.rs +++ b/src/nix/deploy.rs @@ -1,13 +1,13 @@ use std::time::Duration; use anyhow::{Context, Result}; -use tokio::time::{sleep, Instant}; +use tokio::time::{Instant, sleep}; use uuid::Uuid; use crate::models::{BuildResult, JobSpec, SystemType}; use crate::op::Operation; use crate::process::{run_json, run_json_with_env, run_with_env}; -use crate::ssh::{run as run_ssh, HostKeyPolicy}; +use crate::ssh::{HostKeyPolicy, run as run_ssh}; /// Build a job's derivation and return the `out` store path. pub async fn build_closure( diff --git a/src/nix/evaluate.rs b/src/nix/evaluate.rs index 49a979a..f5d159a 100644 --- a/src/nix/evaluate.rs +++ b/src/nix/evaluate.rs @@ -2,31 +2,30 @@ use std::collections::HashMap; use std::path::PathBuf; use anyhow::{Context, Result}; -use serde::Deserialize; -use crate::models::{JobSpec, NixEvalJobsResult, SystemType}; -use crate::process::{run_json, run_json_lines}; +use crate::models::{JobSpec, NixEvalJobsResult, NodeMetadata, SystemType}; +use crate::process::run_json_lines; -const NODE_METADATA_PROJECTION: &str = - "node: { hostname = node.hostname or \"\"; user = node.user or \"\"; type = node.type or \"\"; }"; +const SELECT_METADATA: &str = "output: output.blzrdMetadata"; /// Evaluate the flake's `blzrd.nodes` and return enriched `JobSpec`s. pub async fn eval_deployments( cfg: &str, + selected_nodes: &[String], mut on_job_completed: OnJobCompleted, ) -> Result> where OnJobCompleted: FnMut(String), { - let results = run_eval_jobs(cfg).await?; - let basics = collect_basics(&results)?; - build_job_specs(cfg, &results, &basics, &mut on_job_completed).await + let results = run_eval_jobs(cfg, selected_nodes).await?; + collect_job_specs(&results, &mut on_job_completed) } /// Run `nix-eval-jobs` on the flake's `blzrd.nodes`. -async fn run_eval_jobs(cfg: &str) -> Result> { +async fn run_eval_jobs(cfg: &str, selected_nodes: &[String]) -> Result> { let flake_reference = format!("{cfg}#blzrd.nodes"); let gc_roots_dir = gc_roots_dir()?; + let deployment_selector = deployment_selector(selected_nodes); run_json_lines( "nix-eval-jobs", @@ -36,12 +35,49 @@ async fn run_eval_jobs(cfg: &str) -> Result> { "--force-recurse", "--flake", &flake_reference, + "--select", + &deployment_selector, + "--apply", + SELECT_METADATA, ], |_: &NixEvalJobsResult| {}, ) .await } +fn deployment_selector(selected_nodes: &[String]) -> String { + let serialized_node_names = selected_nodes + .iter() + .map(|name| serde_json::to_string(name).expect("node names must serialize as JSON")) + .collect::>() + .join(" "); + + format!( + r#" +nodes: +let + selectedNames = [ {serialized_node_names} ]; + requestedNodes = builtins.listToAttrs ( + map (name: {{ inherit name; value = null; }}) selectedNames + ); + nodesToDeploy = + if selectedNames == [ ] + then nodes + else builtins.intersectAttrs requestedNodes nodes; +in +builtins.mapAttrs (_: node: + node.output // {{ + blzrdMetadata = {{ + hostname = node.hostname or null; + type = node.type or ""; + user = node.user or ""; + }}; + }} +) nodesToDeploy +"# + ) +} + fn gc_roots_dir() -> Result { let cache_home = std::env::var("XDG_CACHE_HOME") .ok() @@ -59,10 +95,15 @@ fn gc_roots_dir() -> Result { Ok(gc_roots_dir) } -/// First pass: extract the basic `(output, drv_path)` pair for each job. -fn collect_basics(results: &[NixEvalJobsResult]) -> Result> { - let mut basics = HashMap::with_capacity(results.len()); - for result in results { +fn collect_job_specs( + results: &[NixEvalJobsResult], + on_job_completed: &mut impl FnMut(String), +) -> Result> { + let mut sorted_results: Vec<_> = results.iter().collect(); + sorted_results.sort_by(|left, right| left.attr.cmp(&right.attr)); + + let mut jobs = HashMap::with_capacity(results.len()); + for result in sorted_results { if let Some(error) = &result.error { anyhow::bail!("job {}: {error}", result.attr); } @@ -71,51 +112,25 @@ fn collect_basics(results: &[NixEvalJobsResult]) -> Result, - on_job_completed: &mut impl FnMut(String), -) -> Result> { - let mut jobs = HashMap::with_capacity(basics.len()); - let mut names: Vec<_> = basics.keys().cloned().collect(); - names.sort(); - - for name in names { - let (output, drv_path) = basics.get(&name).cloned().unwrap_or_default(); - let metadata = node_metadata(cfg, &name).await; + let metadata = result.extra_value.clone().unwrap_or_default(); let hostname = metadata.hostname_or_name(&name); let user = metadata.user(); - let system = metadata.system_type(&name, results)?; + let system = metadata.system_type(&name, result.system.as_deref())?; + let drv_path = result.drv_path.clone().unwrap_or_default(); - if output.is_empty() { - anyhow::bail!("job {name}: missing output path"); - } if user.is_empty() { anyhow::bail!("job {name}: missing user"); } @@ -135,35 +150,8 @@ async fn build_job_specs( Ok(jobs) } -async fn node_metadata(cfg: &str, name: &str) -> NodeMetadata { - let node_reference = format!("{cfg}#blzrd.nodes.{name}"); - run_json( - "nix", - &[ - "eval", - "--json", - "--apply", - NODE_METADATA_PROJECTION, - &node_reference, - ], - ) - .await - .unwrap_or_default() -} - -fn infer_system_type(name: &str, results: &[NixEvalJobsResult]) -> Result { - let system = results - .iter() - .find(|result| { - result - .attr_path - .as_deref() - .and_then(|path| path.first()) - .is_some_and(|job_name| job_name == name) - }) - .and_then(|result| result.system.as_deref()) - .unwrap_or_default(); - +fn infer_system_type(name: &str, system: Option<&str>) -> Result { + let system = system.unwrap_or_default(); if system.contains("darwin") { Ok("darwin".to_string()) } else if system.contains("linux") { @@ -173,14 +161,6 @@ fn infer_system_type(name: &str, results: &[NixEvalJobsResult]) -> Result, - user: Option, - #[serde(rename = "type")] - type_name: Option, -} - impl NodeMetadata { fn hostname_or_name(&self, name: &str) -> String { self.hostname @@ -194,7 +174,7 @@ impl NodeMetadata { self.user.clone().unwrap_or_default() } - fn system_type(&self, name: &str, results: &[NixEvalJobsResult]) -> Result { + fn system_type(&self, name: &str, system: Option<&str>) -> Result { let configured_type = self .type_name .as_deref() @@ -202,7 +182,7 @@ impl NodeMetadata { .map(str::to_owned); let type_name = match configured_type { Some(type_name) => type_name, - None => infer_system_type(name, results)?, + None => infer_system_type(name, system)?, }; SystemType::parse(&type_name).map_err(|error| anyhow::anyhow!("job {name}: {error}")) @@ -213,6 +193,43 @@ impl NodeMetadata { mod tests { use super::*; + #[test] + fn deployment_selector_quotes_selected_node_names() { + let selector = deployment_selector(&["mauville".to_string(), "quoted\"node".to_string()]); + + assert!(selector.contains(r#"selectedNames = [ "mauville" "quoted\"node" ];"#)); + } + + #[test] + fn collect_job_specs_uses_embedded_metadata() { + let results = [NixEvalJobsResult { + attr: "mauville".to_string(), + attr_path: Some(vec!["mauville".to_string()]), + drv_path: Some("/nix/store/mauville.drv".to_string()), + error: None, + extra_value: Some(NodeMetadata { + hostname: None, + type_name: Some("nixos".to_string()), + user: Some("root".to_string()), + }), + outputs: Some(HashMap::from([( + "out".to_string(), + "/nix/store/mauville".to_string(), + )])), + system: Some("x86_64-linux".to_string()), + }]; + let mut completed = Vec::new(); + + let jobs = collect_job_specs(&results, &mut |name| completed.push(name)).unwrap(); + let mauville = &jobs["mauville"]; + + assert_eq!(completed, ["mauville"]); + assert_eq!(mauville.hostname, "mauville"); + assert_eq!(mauville.system, SystemType::Nixos); + assert_eq!(mauville.user, "root"); + assert_eq!(mauville.drv_path, "/nix/store/mauville.drv"); + } + #[test] fn node_metadata_accepts_a_null_hostname() { let metadata: NodeMetadata = -- 2.51.2