diff --git a/runtime/module/config.nix b/runtime/module/config.nix index fa9d11c..75df994 100644 --- a/runtime/module/config.nix +++ b/runtime/module/config.nix @@ -22,7 +22,74 @@ } ) config.deployment.keys; - services = lib.mapAttrs' ( + system.activationScripts.setup-wire-rollback.text = '' + mkdir -p /var/lib/wire-rollback + chmod 700 /var/lib/wire-rollback + ''; + + services = { + wire-rollback = { + enable = config.deployment.rollback; + description = "Rolls back the NixOS profile if `/var/lib/wire-rollback/heartbeat` is not created in 30 + seconds after this service starts."; + documentation = [ + "https://wire.althaea.zone/guides/rollback" + ]; + path = [ + pkgs.coreutils + ]; + wantedBy = [ "multi-user.target" ]; + script = '' + set -euo pipefail + + goal=$(<"/var/lib/wire-rollback/goal") + + case $goal in + "check" | "switch" | "boot" | "test" | "dry-activate") + echo "<5>using goal $goal" + ;; + *) + echo "<3>'$goal' is not a valid goal." + exit 1 + ;; + esac + + sleep 30 + + if [ -f "/var/lib/wire-rollback/heartbeat" ]; then + exit 0 + fi + + echo "<1>/var/lib/wire-rollback/heartbeat does not exist, rolling back system" + + # set current system + nix-env --rollback --profile /nix/var/nix/profiles/system + # get the path to the system we are now rolling back to + system=$(readlink -f /nix/var/nix/profiles/system) + + echo "<5>rolling back to $system" + + # switch to the system using goal + "$system/bin/switch-to-configuration $goal" + ''; + unitConfig = { + ConditionPathExists = [ + "/var/lib/wire-rollback/goal" + "!/var/lib/wire-rollback/heartbeat" + ]; + }; + serviceConfig = { + Type = "oneshot"; + Restart = "no"; + StateDirectory = "wire-rollback"; + NotifyAccess = "all"; + RemainAfterExit = "yes"; + + ExecStopPost = "${pkgs.coreutils}/bin/rm -f /var/lib/wire-rollback/goal"; + }; + }; + } + // (lib.mapAttrs' ( _name: value: lib.nameValuePair "${value.name}-key" { description = "Service that requires ${value.path}"; @@ -55,7 +122,7 @@ RemainAfterExit = "yes"; }; } - ) config.deployment.keys; + ) config.deployment.keys); }; deployment = { diff --git a/runtime/module/options.nix b/runtime/module/options.nix index ff4fca6..6eee73c 100644 --- a/runtime/module/options.nix +++ b/runtime/module/options.nix @@ -50,6 +50,12 @@ in default = { }; }; + rollback = lib.mkOption { + type = types.bool; + default = true; + description = "Attempt to rollback this node if it cannot be contacted after activation."; + }; + buildOnTarget = lib.mkOption { type = types.bool; default = false; diff --git a/wire/lib/src/errors.rs b/wire/lib/src/errors.rs index 4c116d4..0830c56 100644 --- a/wire/lib/src/errors.rs +++ b/wire/lib/src/errors.rs @@ -87,6 +87,20 @@ pub enum ActivationError { )] #[error("failed to run switch-to-configuration {0} on node {1}")] SwitchToConfigurationError(SwitchToConfigurationGoal, Name, #[source] CommandError), + + #[diagnostic( + code(wire::activation::Heartbeat), + url("{DOCS_URL}#{}", self.code().unwrap()) + )] + #[error("failed to touch /var/lib/wire-rollback/heartbeat on node {name}")] + FailedHeartbeatError { + name: Name, + #[source] + activation_failure: CommandError, + + #[related] + related_errors: Vec, + }, } #[derive(Debug, Diagnostic, Error)] diff --git a/wire/lib/src/hive/node.rs b/wire/lib/src/hive/node.rs index 22b3be7..4518847 100644 --- a/wire/lib/src/hive/node.rs +++ b/wire/lib/src/hive/node.rs @@ -154,11 +154,19 @@ impl Display for Target { } } +const fn rollback_default() -> bool { + true +} + #[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] pub struct Node { #[serde(rename = "target")] pub target: Target, + /// default value as this is a new attribute + #[serde(rename = "rollback", default = "rollback_default")] + pub rollback: bool, + #[serde(rename = "buildOnTarget")] pub build_remotely: bool, @@ -192,6 +200,7 @@ impl Default for Node { allow_local_deployment: true, build_remotely: false, host_platform: "x86_64-linux".into(), + rollback: rollback_default(), } } } @@ -294,6 +303,7 @@ pub struct StepState { pub agent_directory: Option, } +#[allow(clippy::struct_excessive_bools)] pub struct Context<'a> { pub name: &'a Name, pub node: &'a mut Node, diff --git a/wire/lib/src/hive/steps/activate.rs b/wire/lib/src/hive/steps/activate.rs index 2e2b4fb..c7b7f3d 100644 --- a/wire/lib/src/hive/steps/activate.rs +++ b/wire/lib/src/hive/steps/activate.rs @@ -116,7 +116,34 @@ async fn reboot(ctx: &Context<'_>) -> Result<(), HiveLibError> { )); } -async fn reconnect( +async fn rollback(ctx: &Context<'_>, goal: &SwitchToConfigurationGoal, original_error: CommandError) -> Result<(), HiveLibError> { + let command_string = "touch /var/lib/wire-rollback/heartbeat".to_string(); + + let child = run_command( + &CommandArguments::new(command_string, ctx.modifiers) + .on_target(Some(&ctx.node.target)) + .elevated(ctx.node) + .log_stdout(), + ) + .await?; + + let result = child.wait_till_success().await; + + match result { + Ok(_) => Err(HiveLibError::ActivationError( + ActivationError::SwitchToConfigurationError(*goal, ctx.name.clone(), original_error), + )), + Err(err) => Err(HiveLibError::ActivationError( + ActivationError::FailedHeartbeatError { + name: ctx.name.clone(), + activation_failure: original_error, + related_errors: vec![err] + }, + )), + } +} + +async fn reconnect_or_rollback( ctx: &Context<'_>, goal: &SwitchToConfigurationGoal, error: CommandError, @@ -135,6 +162,10 @@ async fn reconnect( } if wait_for_ping(ctx).await.is_ok() { + if ctx.node.rollback { + return rollback(ctx, goal, error).await; + } + return Err(HiveLibError::ActivationError( ActivationError::SwitchToConfigurationError(*goal, ctx.name.clone(), error), )); @@ -175,13 +206,19 @@ impl ExecuteStep for SwitchToConfiguration { info!("Running switch-to-configuration {goal}"); + let goal_str = match goal { + SwitchToConfigurationGoal::Switch => "switch", + SwitchToConfigurationGoal::Boot => "boot", + SwitchToConfigurationGoal::Test => "test", + SwitchToConfigurationGoal::DryActivate => "dry-activate", + }; + let command_string = format!( - "{built_path}/bin/switch-to-configuration {}", - match goal { - SwitchToConfigurationGoal::Switch => "switch", - SwitchToConfigurationGoal::Boot => "boot", - SwitchToConfigurationGoal::Test => "test", - SwitchToConfigurationGoal::DryActivate => "dry-activate", + "{rollback}{built_path}/bin/switch-to-configuration {goal_str}", + rollback = if ctx.node.rollback { + format!("echo \"{goal_str}\" > /var/lib/wire-rollback/goal && ") + } else { + String::new() } ); @@ -201,7 +238,7 @@ impl ExecuteStep for SwitchToConfiguration { match result { Ok(_) => reboot(ctx).await, - Err(error) => reconnect(ctx, goal, error).await, + Err(error) => reconnect_or_rollback(ctx, goal, error).await, } } }