From 27e86c4375d9925a96d68ff9482c3880ee328fb0 Mon Sep 17 00:00:00 2001 From: marshmallow Date: Mon, 22 Dec 2025 16:00:25 +1100 Subject: [PATCH] add `wire build` command (#383) --- CHANGELOG.md | 7 + crates/cli/src/apply.rs | 250 ++++++++++++++++++++----- crates/cli/src/cli.rs | 104 +++++++++- crates/cli/src/main.rs | 48 ++++- crates/core/src/commands/common.rs | 12 +- crates/core/src/hive/mod.rs | 1 - crates/core/src/hive/node.rs | 133 +++++++++---- crates/core/src/hive/steps/activate.rs | 29 ++- crates/core/src/hive/steps/build.rs | 33 +++- crates/core/src/hive/steps/cleanup.rs | 4 +- crates/core/src/hive/steps/evaluate.rs | 7 +- crates/core/src/hive/steps/keys.rs | 30 ++- crates/core/src/hive/steps/ping.rs | 8 +- crates/core/src/hive/steps/push.rs | 20 +- doc/.vitepress/config.ts | 1 + doc/guides/build-in-ci.md | 36 ++++ doc/snippets/guides/example-action.yml | 40 ++++ 17 files changed, 632 insertions(+), 131 deletions(-) create mode 100644 doc/guides/build-in-ci.md create mode 100644 doc/snippets/guides/example-action.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 8be951c..0ae130d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a `--substitute-on-destination` argument. - Add the `meta.nodeSpecialArgs` meta option. +- Add `wire build`, a new command to build nodes offline. + It is distinct from `wire apply build`, as it will not ping + or push the result, making it useful for CI. + +### Changed + +- Build store paths will be output to std out ### Fixed diff --git a/crates/cli/src/apply.rs b/crates/cli/src/apply.rs index 1214d27..b83abbf 100644 --- a/crates/cli/src/apply.rs +++ b/crates/cli/src/apply.rs @@ -4,18 +4,19 @@ use futures::{FutureExt, StreamExt}; use itertools::{Either, Itertools}; use miette::{Diagnostic, IntoDiagnostic, Result}; +use std::any::Any; use std::collections::HashSet; use std::io::{Read, stderr}; use std::sync::Arc; use std::sync::atomic::AtomicBool; use thiserror::Error; -use tracing::{Span, error, info}; -use wire_core::hive::node::{Context, GoalExecutor, Name, StepState, should_apply_locally}; +use tracing::{error, info}; +use wire_core::hive::node::{Context, GoalExecutor, Name, Node, Objective, StepState}; use wire_core::hive::{Hive, HiveLocation}; use wire_core::status::STATUS; use wire_core::{SubCommandModifiers, errors::HiveLibError}; -use crate::cli::{ApplyArgs, ApplyTarget}; +use crate::cli::{ApplyTarget, CommonVerbArgs, Partitions}; #[derive(Debug, Error, Diagnostic)] #[error("node {} failed to apply", .0)] @@ -49,23 +50,11 @@ fn read_apply_targets_from_stdin() -> Result<(Vec, Vec)> { })) } -// #[instrument(skip_all, fields(goal = %args.goal, on = %args.on.iter().join(", ")))] -pub async fn apply( - hive: &mut Hive, - should_shutdown: Arc, - location: HiveLocation, - args: ApplyArgs, - mut modifiers: SubCommandModifiers, -) -> Result<()> { - let header_span = Span::current(); - let location = Arc::new(location); - - // Respect user's --always-build-local arg - hive.force_always_local(args.always_build_local)?; - - let header_span_enter = header_span.enter(); - - let (tags, names) = args.on.iter().fold( +fn resolve_targets( + on: &[ApplyTarget], + modifiers: &mut SubCommandModifiers, +) -> (HashSet, HashSet) { + on.iter().fold( (HashSet::new(), HashSet::new()), |(mut tags, mut names), target| { match target { @@ -86,45 +75,85 @@ pub async fn apply( } (tags, names) }, - ); + ) +} + +fn partition_arr(arr: Vec, partition: &Partitions) -> Vec +where + T: Any + Clone, +{ + if arr.is_empty() { + return arr; + } + + let items_per_chunk = arr.len().div_ceil(partition.maximum); - let selected_nodes: Vec<_> = hive + arr.chunks(items_per_chunk) + .nth(partition.current - 1) + .unwrap_or(&[]) + .to_vec() +} + +pub async fn apply( + hive: &mut Hive, + should_shutdown: Arc, + location: HiveLocation, + args: CommonVerbArgs, + partition: Partitions, + make_objective: F, + mut modifiers: SubCommandModifiers, +) -> Result<()> +where + F: Fn(&Name, &Node) -> Objective, +{ + let location = Arc::new(location); + + let (tags, names) = resolve_targets(&args.on, &mut modifiers); + + let selected_names: Vec<_> = hive .nodes - .iter_mut() + .iter() .filter(|(name, node)| { args.on.is_empty() || names.contains(name) || node.tags.iter().any(|tag| tags.contains(tag)) }) + .sorted_by_key(|(name, _)| *name) + .map(|(name, _)| name.clone()) .collect(); - STATUS.lock().add_many( - &selected_nodes - .iter() - .map(|(name, _)| *name) - .collect::>(), - ); + let num_selected = selected_names.len(); - let mut set = selected_nodes - .into_iter() + let partitioned_names = partition_arr(selected_names, &partition); + + if num_selected != partitioned_names.len() { + info!( + "Partitioning reduced selected number of nodes from {num_selected} to {}", + partitioned_names.len() + ); + } + + STATUS + .lock() + .add_many(&partitioned_names.iter().collect::>()); + + let mut set = hive + .nodes + .iter_mut() + .filter(|(name, _)| partitioned_names.contains(name)) .map(|(name, node)| { info!("Resolved {:?} to include {}", args.on, name); - let should_apply_locally = should_apply_locally(node.allow_local_deployment, &name.0); + let objective = make_objective(name, node); let context = Context { node, name, - goal: args.goal.clone().try_into().unwrap(), + objective, state: StepState::default(), - no_keys: args.no_keys, hive_location: location.clone(), modifiers, - reboot: args.reboot, - substitute_on_destination: args.substitute_on_destination, - should_apply_locally, - handle_unreachable: args.handle_unreachable.clone().into(), - should_shutdown: should_shutdown.clone(), + should_quit: should_shutdown.clone(), }; GoalExecutor::new(context) @@ -155,9 +184,6 @@ pub async fn apply( ); } - std::mem::drop(header_span_enter); - std::mem::drop(header_span); - if !errors.is_empty() { // clear the status bar if we are about to print error messages STATUS.lock().clear(&mut stderr()); @@ -173,3 +199,143 @@ pub async fn apply( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(clippy::too_many_lines)] + fn test_partitioning() { + let arr = (1..=10).collect::>(); + assert_eq!(arr, partition_arr(arr.clone(), &Partitions::default())); + + assert_eq!( + vec![1, 2, 3, 4, 5], + partition_arr( + arr.clone(), + &Partitions { + current: 1, + maximum: 2 + } + ) + ); + assert_eq!( + vec![6, 7, 8, 9, 10], + partition_arr( + arr, + &Partitions { + current: 2, + maximum: 2 + } + ) + ); + + // test odd number + let arr = (1..10).collect::>(); + assert_eq!( + arr.clone(), + partition_arr(arr.clone(), &Partitions::default()) + ); + + assert_eq!( + vec![1, 2, 3, 4, 5], + partition_arr( + arr.clone(), + &Partitions { + current: 1, + maximum: 2 + } + ) + ); + assert_eq!( + vec![6, 7, 8, 9], + partition_arr( + arr.clone(), + &Partitions { + current: 2, + maximum: 2 + } + ) + ); + + // test large number of partitions + let arr = (1..=10).collect::>(); + assert_eq!( + arr.clone(), + partition_arr(arr.clone(), &Partitions::default()) + ); + + for i in 1..=10 { + assert_eq!( + vec![i], + partition_arr( + arr.clone(), + &Partitions { + current: i, + maximum: 10 + } + ) + ); + + assert_eq!( + vec![i], + partition_arr( + arr.clone(), + &Partitions { + current: i, + maximum: 15 + } + ) + ); + } + + // stretching thin with higher partitions will start to leave higher ones empty + assert_eq!( + Vec::::new(), + partition_arr( + arr, + &Partitions { + current: 11, + maximum: 15 + } + ) + ); + + // test the above holds for a lot of numbers + for i in 1..1000 { + let arr: Vec = (0..i).collect(); + let total = arr.len(); + + assert_eq!( + arr.clone(), + partition_arr(arr.clone(), &Partitions::default()), + ); + + let buckets = 2; + let chunk_size = total.div_ceil(buckets); + let split_index = std::cmp::min(chunk_size, total); + + assert_eq!( + &arr.clone()[..split_index], + partition_arr( + arr.clone(), + &Partitions { + current: 1, + maximum: 2 + } + ), + ); + assert_eq!( + &arr.clone()[split_index..], + partition_arr( + arr.clone(), + &Partitions { + current: 2, + maximum: 2 + } + ), + ); + } + } +} diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index 8e7b9b4..2fadf0e 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -95,6 +95,28 @@ fn more_than_zero(s: &str) -> Result { number_range(s, 1, usize::MAX) } +fn parse_partitions(s: &str) -> Result { + let parts: [&str; 2] = s + .split('/') + .collect::>() + .try_into() + .map_err(|_| "partition must contain exactly one '/'")?; + + let (current, maximum) = + std::array::from_fn(|i| parts[i].parse::().map_err(|x| x.to_string())).into(); + let (current, maximum) = (current?, maximum?); + + if current > maximum { + return Err("current is more than total".to_string()); + } + + if current == 0 || maximum == 0 { + return Err("partition segments cannot be 0.".to_string()); + } + + Ok(Partitions { current, maximum }) +} + #[derive(Clone)] pub enum HandleUnreachableArg { Ignore, @@ -132,12 +154,8 @@ impl From for HandleUnreachable { } } -#[allow(clippy::struct_excessive_bools)] #[derive(Args)] -pub struct ApplyArgs { - #[arg(value_enum, default_value_t)] - pub goal: Goal, - +pub struct CommonVerbArgs { /// List of literal node names, a literal `-`, or `@` prefixed tags. /// /// `-` will read additional values from stdin, separated by whitespace. @@ -147,6 +165,16 @@ pub struct ApplyArgs { #[arg(short, long, default_value_t = 10, value_parser=more_than_zero)] pub parallel: usize, +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Args)] +pub struct ApplyArgs { + #[command(flatten)] + pub common: CommonVerbArgs, + + #[arg(value_enum, default_value_t)] + pub goal: Goal, /// Skip key uploads. noop when [GOAL] = Keys #[arg(short, long, default_value_t = false)] @@ -179,10 +207,44 @@ pub struct ApplyArgs { pub ssh_accept_host: bool, } +#[derive(Clone, Debug)] +pub struct Partitions { + pub current: usize, + pub maximum: usize, +} + +impl Default for Partitions { + fn default() -> Self { + Self { + current: 1, + maximum: 1, + } + } +} + +#[derive(Args)] +pub struct BuildArgs { + #[command(flatten)] + pub common: CommonVerbArgs, + + /// Partition builds into buckets. + /// + /// In the format of `current/total`, where 1 <= current <= total. + #[arg(short = 'P', default_value="1/1", long, value_parser=parse_partitions)] + pub partition: Option, +} + #[derive(Subcommand)] pub enum Commands { /// Deploy nodes Apply(ApplyArgs), + /// Build nodes offline + /// + /// This is distinct from `wire apply build`, as it will not ping or push + /// the result, making it useful for CI. + /// + /// Additionally, you may partition the build jobs into buckets. + Build(BuildArgs), /// Inspect hive #[clap(visible_alias = "show")] Inspect { @@ -209,13 +271,13 @@ pub enum Goal { /// Make the configuration the boot default and activate now #[default] Switch, - /// Build the configuration but do nothing with it + /// Build the configuration & push the results Build, - /// Copy system derivation to remote hosts + /// Copy the system derivation to the remote hosts Push, - /// Push deployment keys to remote hosts + /// Push deployment keys to the remote hosts Keys, - /// Activate system profile on next boot + /// Activate the system profile on next boot Boot, /// Activate the configuration, but don't make it the boot default Test, @@ -310,3 +372,27 @@ fn node_names_completer(current: &std::ffi::OsStr) -> Vec { completions }) } + +#[cfg(test)] +mod tests { + use std::assert_matches::assert_matches; + + use crate::cli::{Partitions, parse_partitions}; + + #[test] + fn test_partition_parsing() { + assert_matches!(parse_partitions(""), Err(..)); + assert_matches!(parse_partitions("/"), Err(..)); + assert_matches!(parse_partitions(" / "), Err(..)); + assert_matches!(parse_partitions("abc/"), Err(..)); + assert_matches!(parse_partitions("abc"), Err(..)); + assert_matches!(parse_partitions("1/1"), Ok(Partitions { + current, + maximum + }) if current == 1 && maximum == 1); + assert_matches!(parse_partitions("0/1"), Err(..)); + assert_matches!(parse_partitions("-11/1"), Err(..)); + assert_matches!(parse_partitions("100/99"), Err(..)); + assert_matches!(parse_partitions("5/10"), Ok(Partitions { current, maximum }) if current == 5 && maximum == 10); + } +} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 156c499..7ad74e6 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,14 +1,17 @@ // SPDX-License-Identifier: AGPL-3.0-or-later // Copyright 2024-2025 wire Contributors +#![deny(clippy::pedantic)] #![feature(sync_nonpoison)] #![feature(nonpoison_mutex)] +#![feature(assert_matches)] use std::process::Command; use std::sync::Arc; use std::sync::atomic::AtomicBool; use crate::cli::Cli; +use crate::cli::Partitions; use crate::cli::ToSubCommandModifiers; use crate::sigint::handle_signals; use crate::tracing_setup::setup_logging; @@ -25,6 +28,9 @@ use wire_core::cache::InspectionCache; use wire_core::commands::common::get_hive_node_names; use wire_core::hive::Hive; use wire_core::hive::get_hive_location; +use wire_core::hive::node::ApplyObjective; +use wire_core::hive::node::Objective; +use wire_core::hive::node::should_apply_locally; #[macro_use] extern crate enum_display_derive; @@ -74,7 +80,47 @@ async fn main() -> Result<()> { match args.command { cli::Commands::Apply(apply_args) => { let mut hive = Hive::new_from_path(&location, cache.clone(), modifiers).await?; - apply::apply(&mut hive, should_shutdown, location, apply_args, modifiers).await?; + let goal: wire_core::hive::node::Goal = apply_args.goal.clone().try_into().unwrap(); + + // Respect user's --always-build-local arg + hive.force_always_local(apply_args.always_build_local)?; + + apply::apply( + &mut hive, + should_shutdown, + location, + apply_args.common, + Partitions::default(), + |name, node| { + Objective::Apply(ApplyObjective { + goal, + no_keys: apply_args.no_keys, + reboot: apply_args.reboot, + substitute_on_destination: apply_args.substitute_on_destination, + should_apply_locally: should_apply_locally( + node.allow_local_deployment, + &name.0, + ), + handle_unreachable: apply_args.handle_unreachable.clone().into(), + }) + }, + modifiers, + ) + .await?; + } + cli::Commands::Build(build_args) => { + let mut hive = Hive::new_from_path(&location, cache.clone(), modifiers).await?; + + apply::apply( + &mut hive, + should_shutdown, + location, + build_args.common, + build_args.partition.unwrap_or_default(), + |_name, _node| Objective::BuildLocally, + modifiers, + ) + .await?; } cli::Commands::Inspect { json, selection } => println!("{}", { match selection { diff --git a/crates/core/src/commands/common.rs b/crates/core/src/commands/common.rs index dcb4cf7..ab31e49 100644 --- a/crates/core/src/commands/common.rs +++ b/crates/core/src/commands/common.rs @@ -14,7 +14,7 @@ use crate::{ errors::{CommandError, HiveInitialisationError, HiveLibError}, hive::{ HiveLocation, - node::{Context, Push}, + node::{Context, Objective, Push}, }, }; @@ -32,10 +32,12 @@ pub async fn push(context: &Context<'_>, push: Push<'_>) -> Result<(), HiveLibEr let mut command_string = CommandStringBuilder::nix(); command_string.args(&["--extra-experimental-features", "nix-command", "copy"]); - command_string.opt_arg( - context.substitute_on_destination, - "--substitute-on-destination", - ); + if let Objective::Apply(apply_objective) = context.objective { + command_string.opt_arg( + apply_objective.substitute_on_destination, + "--substitute-on-destination", + ); + } command_string.arg("--to"); command_string.args(&[ format!( diff --git a/crates/core/src/hive/mod.rs b/crates/core/src/hive/mod.rs index e2364da..0f0fd81 100644 --- a/crates/core/src/hive/mod.rs +++ b/crates/core/src/hive/mod.rs @@ -209,7 +209,6 @@ impl HiveLocation { ) -> Result { let mut command_string = CommandStringBuilder::nix(); command_string.args(&[ - "nix", "flake", "prefetch", "--extra-experimental-features", diff --git a/crates/core/src/hive/node.rs b/crates/core/src/hive/node.rs index 570053a..63d9c9a 100644 --- a/crates/core/src/hive/node.rs +++ b/crates/core/src/hive/node.rs @@ -29,7 +29,9 @@ use crate::{EvalGoal, StrictHostKeyChecking, SubCommandModifiers}; use super::HiveLibError; use super::steps::activate::SwitchToConfiguration; -#[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq, derive_more::Display)] +#[derive( + Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq, PartialOrd, Ord, derive_more::Display, +)] pub struct Name(pub Arc); #[derive(Serialize, Deserialize, Clone, Debug, Hash, Eq, PartialEq)] @@ -111,14 +113,16 @@ impl<'a> Context<'a> { node, hive_location: Arc::new(hive_location), modifiers: SubCommandModifiers::default(), - no_keys: false, + objective: Objective::Apply(ApplyObjective { + goal: Goal::SwitchToConfiguration(SwitchToConfigurationGoal::Switch), + no_keys: false, + reboot: false, + should_apply_locally: false, + substitute_on_destination: false, + handle_unreachable: HandleUnreachable::default(), + }), state: StepState::default(), - goal: Goal::SwitchToConfiguration(SwitchToConfigurationGoal::Switch), - reboot: false, - should_apply_locally: false, - substitute_on_destination: false, - handle_unreachable: HandleUnreachable::default(), - should_shutdown: Arc::new(AtomicBool::new(false)), + should_quit: Arc::new(AtomicBool::new(false)), } } } @@ -273,6 +277,24 @@ pub enum Goal { Keys, } +// TODO: Get rid of this allow and resolve it +#[allow(clippy::struct_excessive_bools)] +#[derive(Clone, Copy)] +pub struct ApplyObjective { + pub goal: Goal, + pub no_keys: bool, + pub reboot: bool, + pub should_apply_locally: bool, + pub substitute_on_destination: bool, + pub handle_unreachable: HandleUnreachable, +} + +#[derive(Clone, Copy)] +pub enum Objective { + Apply(ApplyObjective), + BuildLocally, +} + #[enum_dispatch] pub(crate) trait ExecuteStep: Send + Sync + Display + std::fmt::Debug { async fn execute(&self, ctx: &mut Context<'_>) -> Result<(), HiveLibError>; @@ -282,7 +304,7 @@ pub(crate) trait ExecuteStep: Send + Sync + Display + std::fmt::Debug { // may include other options such as FailAll in the future #[non_exhaustive] -#[derive(Clone, Default)] +#[derive(Clone, Copy, Default)] pub enum HandleUnreachable { Ignore, #[default] @@ -297,21 +319,14 @@ pub struct StepState { pub key_agent_directory: Option, } -// TODO: Get rid of this allow and resolve it -#[allow(clippy::struct_excessive_bools)] pub struct Context<'a> { pub name: &'a Name, pub node: &'a mut Node, pub hive_location: Arc, pub modifiers: SubCommandModifiers, - pub no_keys: bool, pub state: StepState, - pub goal: Goal, - pub reboot: bool, - pub should_apply_locally: bool, - pub substitute_on_destination: bool, - pub handle_unreachable: HandleUnreachable, - pub should_shutdown: Arc, + pub should_quit: Arc, + pub objective: Objective, } #[enum_dispatch(ExecuteStep)] @@ -352,7 +367,7 @@ pub struct GoalExecutor<'a> { /// returns Err if the application should shut down. fn app_shutdown_guard(context: &Context) -> Result<(), HiveLibError> { if context - .should_shutdown + .should_quit .load(std::sync::atomic::Ordering::Relaxed) { return Err(HiveLibError::Sigint); @@ -382,7 +397,6 @@ impl<'a> GoalExecutor<'a> { Step::Keys(Keys { filter: UploadKeyAt::PostActivation, }), - Step::CleanUp(CleanUp), ], context, } @@ -427,7 +441,12 @@ impl<'a> GoalExecutor<'a> { .is_some() ); - if !matches!(self.context.goal, Goal::Keys) { + let spawn_evaluator = match self.context.objective { + Objective::Apply(apply_objective) => !matches!(apply_objective.goal, Goal::Keys), + Objective::BuildLocally => true, + }; + + if spawn_evaluator { tokio::spawn( GoalExecutor::evaluate_task( tx, @@ -468,8 +487,12 @@ impl<'a> GoalExecutor<'a> { // discard error from cleanup let _ = CleanUp.execute(&mut self.context).await; - if matches!(step, Step::Ping(..)) - && matches!(self.context.handle_unreachable, HandleUnreachable::Ignore) + if let Objective::Apply(apply_objective) = self.context.objective + && matches!(step, Step::Ping(..)) + && matches!( + apply_objective.handle_unreachable, + HandleUnreachable::Ignore, + ) { return Ok(()); } @@ -564,7 +587,6 @@ mod tests { filter: UploadKeyAt::PostActivation } .into(), - CleanUp.into() ] ); } @@ -576,7 +598,11 @@ mod tests { let name = &Name(function_name!().into()); let mut context = Context::create_test_context(location, name, &mut node); - context.goal = Goal::Keys; + let Objective::Apply(ref mut apply_objective) = context.objective else { + unreachable!() + }; + + apply_objective.goal = Goal::Keys; let executor = GoalExecutor::new(context); let steps = get_steps(executor); @@ -590,19 +616,21 @@ mod tests { filter: UploadKeyAt::NoFilter } .into(), - CleanUp.into() ] ); } #[tokio::test] - async fn order_build_only() { + async fn order_build() { let location = location!(get_test_path!()); let mut node = Node::default(); let name = &Name(function_name!().into()); let mut context = Context::create_test_context(location, name, &mut node); - context.goal = Goal::Build; + let Objective::Apply(ref mut apply_objective) = context.objective else { + unreachable!() + }; + apply_objective.goal = Goal::Build; let executor = GoalExecutor::new(context); let steps = get_steps(executor); @@ -614,7 +642,6 @@ mod tests { crate::hive::steps::evaluate::Evaluate.into(), crate::hive::steps::build::Build.into(), crate::hive::steps::push::PushBuildOutput.into(), - CleanUp.into() ] ); } @@ -626,7 +653,10 @@ mod tests { let name = &Name(function_name!().into()); let mut context = Context::create_test_context(location, name, &mut node); - context.goal = Goal::Push; + let Objective::Apply(ref mut apply_objective) = context.objective else { + unreachable!() + }; + apply_objective.goal = Goal::Push; let executor = GoalExecutor::new(context); let steps = get_steps(executor); @@ -637,7 +667,6 @@ mod tests { Ping.into(), crate::hive::steps::evaluate::Evaluate.into(), crate::hive::steps::push::PushEvaluatedOutput.into(), - CleanUp.into() ] ); } @@ -671,7 +700,6 @@ mod tests { filter: UploadKeyAt::PostActivation } .into(), - CleanUp.into() ] ); } @@ -683,7 +711,12 @@ mod tests { let name = &Name(function_name!().into()); let mut context = Context::create_test_context(location, name, &mut node); - context.no_keys = true; + + let Objective::Apply(ref mut apply_objective) = context.objective else { + unreachable!() + }; + apply_objective.no_keys = true; + let executor = GoalExecutor::new(context); let steps = get_steps(executor); @@ -695,7 +728,6 @@ mod tests { crate::hive::steps::build::Build.into(), crate::hive::steps::push::PushBuildOutput.into(), SwitchToConfiguration.into(), - CleanUp.into() ] ); } @@ -707,8 +739,13 @@ mod tests { let name = &Name(function_name!().into()); let mut context = Context::create_test_context(location, name, &mut node); - context.no_keys = true; - context.should_apply_locally = true; + + let Objective::Apply(ref mut apply_objective) = context.objective else { + unreachable!() + }; + apply_objective.no_keys = true; + apply_objective.should_apply_locally = true; + let executor = GoalExecutor::new(context); let steps = get_steps(executor); @@ -722,6 +759,28 @@ mod tests { ); } + #[tokio::test] + async fn order_build_only() { + let location = location!(get_test_path!()); + let mut node = Node::default(); + + let name = &Name(function_name!().into()); + let mut context = Context::create_test_context(location, name, &mut node); + + context.objective = Objective::BuildLocally; + + let executor = GoalExecutor::new(context); + let steps = get_steps(executor); + + assert_eq!( + steps, + vec![ + crate::hive::steps::evaluate::Evaluate.into(), + crate::hive::steps::build::Build.into() + ] + ); + } + #[test] fn target_fails_increments() { let mut target = Target::from_host("localhost"); @@ -870,7 +929,7 @@ mod tests { let name = &Name(function_name!().into()); let context = Context::create_test_context(location, name, &mut node); context - .should_shutdown + .should_quit .store(true, std::sync::atomic::Ordering::Relaxed); let executor = GoalExecutor::new(context); let status = executor.execute().await; diff --git a/crates/core/src/hive/steps/activate.rs b/crates/core/src/hive/steps/activate.rs index ce9c71f..df7c94d 100644 --- a/crates/core/src/hive/steps/activate.rs +++ b/crates/core/src/hive/steps/activate.rs @@ -9,7 +9,7 @@ use crate::{ HiveLibError, commands::{CommandArguments, WireCommandChip, builder::CommandStringBuilder, run_command}, errors::{ActivationError, NetworkError}, - hive::node::{Context, ExecuteStep, Goal, SwitchToConfigurationGoal}, + hive::node::{Context, ExecuteStep, Goal, Objective, SwitchToConfigurationGoal}, }; #[derive(Debug, PartialEq)] @@ -51,10 +51,14 @@ async fn set_profile( command_string.args(&["-p", "/nix/var/nix/profiles/system", "--set"]); command_string.arg(built_path); + let Objective::Apply(apply_objective) = ctx.objective else { + unreachable!() + }; + let child = run_command( &CommandArguments::new(command_string, ctx.modifiers) .mode(crate::commands::ChildOutputMode::Nix) - .on_target(if ctx.should_apply_locally { + .on_target(if apply_objective.should_apply_locally { None } else { Some(&ctx.node.target) @@ -75,14 +79,23 @@ async fn set_profile( impl ExecuteStep for SwitchToConfiguration { fn should_execute(&self, ctx: &Context) -> bool { - matches!(ctx.goal, Goal::SwitchToConfiguration(..)) + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + matches!(apply_objective.goal, Goal::SwitchToConfiguration(..)) } + #[allow(clippy::too_many_lines)] #[instrument(skip_all, name = "activate")] async fn execute(&self, ctx: &mut Context<'_>) -> Result<(), HiveLibError> { let built_path = ctx.state.build.as_ref().unwrap(); - let Goal::SwitchToConfiguration(goal) = &ctx.goal else { + let Objective::Apply(apply_objective) = ctx.objective else { + unreachable!() + }; + + let Goal::SwitchToConfiguration(goal) = &apply_objective.goal else { unreachable!("Cannot reach as guarded by should_execute") }; @@ -108,7 +121,7 @@ impl ExecuteStep for SwitchToConfiguration { let child = run_command( &CommandArguments::new(command_string, ctx.modifiers) - .on_target(if ctx.should_apply_locally { + .on_target(if apply_objective.should_apply_locally { None } else { Some(&ctx.node.target) @@ -122,11 +135,11 @@ impl ExecuteStep for SwitchToConfiguration { match result { Ok(_) => { - if !ctx.reboot { + if !apply_objective.reboot { return Ok(()); } - if ctx.should_apply_locally { + if apply_objective.should_apply_locally { error!("Refusing to reboot local machine!"); return Ok(()); @@ -176,7 +189,7 @@ impl ExecuteStep for SwitchToConfiguration { // Bail if the command couldn't of broken the system // and don't try to regain connection to localhost if matches!(goal, SwitchToConfigurationGoal::DryActivate) - || ctx.should_apply_locally + || apply_objective.should_apply_locally { return Err(HiveLibError::ActivationError( ActivationError::SwitchToConfigurationError(*goal, ctx.name.clone(), error), diff --git a/crates/core/src/hive/steps/build.rs b/crates/core/src/hive/steps/build.rs index eade933..6ce9f50 100644 --- a/crates/core/src/hive/steps/build.rs +++ b/crates/core/src/hive/steps/build.rs @@ -11,7 +11,7 @@ use crate::{ CommandArguments, Either, WireCommandChip, builder::CommandStringBuilder, run_command_with_env, }, - hive::node::{Context, ExecuteStep, Goal}, + hive::node::{Context, ExecuteStep, Goal, Objective}, }; #[derive(Debug, PartialEq)] @@ -25,7 +25,12 @@ impl Display for Build { impl ExecuteStep for Build { fn should_execute(&self, ctx: &Context) -> bool { - !matches!(ctx.goal, Goal::Keys | Goal::Push) + match ctx.objective { + Objective::Apply(apply_objective) => { + !matches!(apply_objective.goal, Goal::Keys | Goal::Push) + } + Objective::BuildLocally => true, + } } #[instrument(skip_all, name = "build")] @@ -46,13 +51,19 @@ impl ExecuteStep for Build { let status = run_command_with_env( &CommandArguments::new(command_string, ctx.modifiers) // build remotely if asked for AND we arent applying locally - // building remotely but applying locally does not logically - // make any sense - .on_target(if ctx.node.build_remotely && !ctx.should_apply_locally { - Some(&ctx.node.target) - } else { - None - }) + // + // (building remotely but applying locally does not logically + // make any sense) + .on_target( + if ctx.node.build_remotely + && let Objective::Apply(apply_objective) = ctx.objective + && apply_objective.should_apply_locally + { + Some(&ctx.node.target) + } else { + None + }, + ) .mode(crate::commands::ChildOutputMode::Nix) .log_stdout(), std::collections::HashMap::new(), @@ -70,6 +81,10 @@ impl ExecuteStep for Build { }; info!("Built output: {stdout:?}"); + + // print built path to stdout + println!("{stdout}"); + ctx.state.build = Some(stdout); Ok(()) diff --git a/crates/core/src/hive/steps/cleanup.rs b/crates/core/src/hive/steps/cleanup.rs index b7fb0fd..f8964f0 100644 --- a/crates/core/src/hive/steps/cleanup.rs +++ b/crates/core/src/hive/steps/cleanup.rs @@ -18,8 +18,8 @@ impl Display for CleanUp { } impl ExecuteStep for CleanUp { - fn should_execute(&self, ctx: &Context) -> bool { - !ctx.should_apply_locally + fn should_execute(&self, _ctx: &Context) -> bool { + false } async fn execute(&self, _ctx: &mut Context<'_>) -> Result<(), HiveLibError> { diff --git a/crates/core/src/hive/steps/evaluate.rs b/crates/core/src/hive/steps/evaluate.rs index ee148bd..72b4764 100644 --- a/crates/core/src/hive/steps/evaluate.rs +++ b/crates/core/src/hive/steps/evaluate.rs @@ -7,7 +7,7 @@ use tracing::instrument; use crate::{ HiveLibError, - hive::node::{Context, ExecuteStep, Goal}, + hive::node::{Context, ExecuteStep, Goal, Objective}, }; #[derive(Debug, PartialEq)] @@ -21,7 +21,10 @@ impl Display for Evaluate { impl ExecuteStep for Evaluate { fn should_execute(&self, ctx: &Context) -> bool { - !matches!(ctx.goal, Goal::Keys) + match ctx.objective { + Objective::Apply(apply_objective) => !matches!(apply_objective.goal, Goal::Keys), + Objective::BuildLocally => true, + } } #[instrument(skip_all, name = "eval")] diff --git a/crates/core/src/hive/steps/keys.rs b/crates/core/src/hive/steps/keys.rs index 8fd20a7..6a76a5c 100644 --- a/crates/core/src/hive/steps/keys.rs +++ b/crates/core/src/hive/steps/keys.rs @@ -31,7 +31,7 @@ use crate::commands::builder::CommandStringBuilder; use crate::commands::common::push; use crate::commands::{CommandArguments, WireCommandChip, run_command}; use crate::errors::KeyError; -use crate::hive::node::{Context, ExecuteStep, Goal, Push, SwitchToConfigurationGoal}; +use crate::hive::node::{Context, ExecuteStep, Goal, Objective, Push, SwitchToConfigurationGoal}; #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] #[serde(tag = "t", content = "c")] @@ -226,14 +226,18 @@ where impl ExecuteStep for Keys { fn should_execute(&self, ctx: &Context) -> bool { - if ctx.no_keys { + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + if apply_objective.no_keys { return false; } // should execute if no filter, and the goal is keys. // otherwise, only execute if the goal is switch and non-nofilter matches!( - (&self.filter, &ctx.goal), + (&self.filter, &apply_objective.goal), (UploadKeyAt::NoFilter, Goal::Keys) | ( UploadKeyAt::PreActivation | UploadKeyAt::PostActivation, @@ -256,9 +260,13 @@ impl ExecuteStep for Keys { let command_string = CommandStringBuilder::new(format!("{agent_directory}/bin/wire-key-agent")); + let Objective::Apply(apply_objective) = ctx.objective else { + unreachable!() + }; + let mut child = run_command( &CommandArguments::new(command_string, ctx.modifiers) - .on_target(if ctx.should_apply_locally { + .on_target(if apply_objective.should_apply_locally { None } else { Some(&ctx.node.target) @@ -321,12 +329,16 @@ impl Keys { impl ExecuteStep for PushKeyAgent { fn should_execute(&self, ctx: &Context) -> bool { - if ctx.no_keys { + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + if apply_objective.no_keys { return false; } matches!( - &ctx.goal, + &apply_objective.goal, Goal::Keys | Goal::SwitchToConfiguration(SwitchToConfigurationGoal::Switch) ) } @@ -347,7 +359,11 @@ impl ExecuteStep for PushKeyAgent { ), }; - if !ctx.should_apply_locally { + let Objective::Apply(apply_objective) = ctx.objective else { + unreachable!() + }; + + if !apply_objective.should_apply_locally { push(ctx, Push::Path(&agent_directory)).await?; } diff --git a/crates/core/src/hive/steps/ping.rs b/crates/core/src/hive/steps/ping.rs index dffeba1..fcf31f6 100644 --- a/crates/core/src/hive/steps/ping.rs +++ b/crates/core/src/hive/steps/ping.rs @@ -7,7 +7,7 @@ use tracing::{Level, event, instrument}; use crate::{ HiveLibError, - hive::node::{Context, ExecuteStep}, + hive::node::{Context, ExecuteStep, Objective}, }; #[derive(Debug, PartialEq)] @@ -21,7 +21,11 @@ impl Display for Ping { impl ExecuteStep for Ping { fn should_execute(&self, ctx: &Context) -> bool { - !ctx.should_apply_locally + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + !apply_objective.should_apply_locally } #[instrument(skip_all, name = "ping")] diff --git a/crates/core/src/hive/steps/push.rs b/crates/core/src/hive/steps/push.rs index a7fc4f3..06cfc0f 100644 --- a/crates/core/src/hive/steps/push.rs +++ b/crates/core/src/hive/steps/push.rs @@ -8,7 +8,7 @@ use tracing::instrument; use crate::{ HiveLibError, commands::common::push, - hive::node::{Context, ExecuteStep, Goal}, + hive::node::{Context, ExecuteStep, Goal, Objective}, }; #[derive(Debug, PartialEq)] @@ -30,9 +30,13 @@ impl Display for PushBuildOutput { impl ExecuteStep for PushEvaluatedOutput { fn should_execute(&self, ctx: &Context) -> bool { - !matches!(ctx.goal, Goal::Keys) - && !ctx.should_apply_locally - && (ctx.node.build_remotely | matches!(ctx.goal, Goal::Push)) + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + !matches!(apply_objective.goal, Goal::Keys) + && !apply_objective.should_apply_locally + && (ctx.node.build_remotely | matches!(apply_objective.goal, Goal::Push)) } #[instrument(skip_all, name = "push_eval")] @@ -47,7 +51,11 @@ impl ExecuteStep for PushEvaluatedOutput { impl ExecuteStep for PushBuildOutput { fn should_execute(&self, ctx: &Context) -> bool { - if matches!(ctx.goal, Goal::Keys | Goal::Push) { + let Objective::Apply(apply_objective) = ctx.objective else { + return false; + }; + + if matches!(apply_objective.goal, Goal::Keys | Goal::Push) { // skip if we are not building return false; } @@ -57,7 +65,7 @@ impl ExecuteStep for PushBuildOutput { return false; } - if ctx.should_apply_locally { + if apply_objective.should_apply_locally { // skip step if we are applying locally return false; } diff --git a/doc/.vitepress/config.ts b/doc/.vitepress/config.ts index 51fc57a..47b3a84 100644 --- a/doc/.vitepress/config.ts +++ b/doc/.vitepress/config.ts @@ -119,6 +119,7 @@ export default defineConfig({ }, { text: "Apply your Config", link: "/guides/apply" }, { text: "Target Nodes", link: "/guides/targeting" }, + { text: "Build in CI", link: "/guides/build-in-ci" }, { text: "Features", items: [ diff --git a/doc/guides/build-in-ci.md b/doc/guides/build-in-ci.md new file mode 100644 index 0000000..19080ff --- /dev/null +++ b/doc/guides/build-in-ci.md @@ -0,0 +1,36 @@ +--- +comment: true +title: Build in CI +--- + +# Build in CI + +## The `wire build` command + +`wire build` builds nodes locally. It is distinct from +`wire apply build`, as it will not ping or push the result, +making it useful for CI. + +It accepts the same `--on` argument as `wire apply` does. + +## Partitioning builds + +`wire build` accepts a `--partition` option inspired by +[cargo-nextest](https://nexte.st/docs/ci-features/partitioning/), which splits +selected nodes into buckets to be built separately. + +It accepts values in the format `--partition current/total`, where 1 ≤ current ≤ total. + +For example, these two commands will build the entire hive in two invocations: + +```sh +wire build --partition 1/2 + +# later or synchronously: + +wire build --partition 2/2 +``` + +## Example: Build in Github Actions + +<<< @/snippets/guides/example-action.yml [.github/workflows/build.yml] diff --git a/doc/snippets/guides/example-action.yml b/doc/snippets/guides/example-action.yml new file mode 100644 index 0000000..0e1e507 --- /dev/null +++ b/doc/snippets/guides/example-action.yml @@ -0,0 +1,40 @@ +name: Build + +on: + push: + branches: [main] + +jobs: + build-partitioned: + name: Build Partitioned + runs-on: ubuntu-latest + permissions: {} + strategy: + matrix: + # Break into 4 partitions + partition: [1, 2, 3, 4] + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + # This will likely be required if you have multiple architectures + # in your hive. + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 + - uses: cachix/install-nix-action@4e002c8ec80594ecd40e759629461e26c8abed15 + with: + nix_path: nixpkgs=channel:nixos-unstable + extra_nix_config: | + # Install binary cache as described in the install wire guide + trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g= + substituters = https://cache.nixos.org/ https://cache.garnix.io + + # Again, include additional architectures if you have multiple + # architectures in your hive + extra-platforms = aarch64-linux i686-linux + # Uses wire from your shell (as described in the install wire guide). + - name: Build partition ${{ matrix.partition }} + run: nix develop -Lvc wire \ + build \ + --parallel 1 \ + --partition ${{ matrix.partition }}/4 -- 2.51.2