#!/usr/bin/env bash
# lib/drill-common.sh — shared drill machinery (§8). Source, do not execute.
#
# Drills run FROM THE LAPTOP, never from a server timer: provisioning needs the
# Hetzner API token, and putting that on the server would contradict §5.1 and
# add a sixth §7.3 item with more blast radius than the other five — it can
# create and destroy infrastructure.
#
# Dead-man coverage is preserved by inverting who is watched: the healthchecks
# check still exists, but the ping comes from here. Miss a quarter and it goes
# DOWN. The switch is on the operator, not the server.

set -uo pipefail

DRILL_INSTANCE=""
DRILL_RC=1

# --------------------------------------------------------- step zero (§8)
# The 90-day token expiry and the ~91-day drill cadence collide by
# construction. Without this check the drill fails at the credential-injection
# step, minutes and one provisioned CX53 into the run, and reports as a drill
# failure — which reads like a broken restore path rather than routine expiry.
drill_precondition() {
  # Both drills call this first, so a --print-only guard here covers both
  # without either script needing its own. Nothing below this point runs:
  # no credential check, no vault touch, matching --print-only's promise in
  # bootstrap.sh's usage() to mutate nothing, anywhere.
  if [[ $PRINT_ONLY -eq 1 ]]; then
    info "plan: provision a CX53, run a full bootstrap against it,"
    info "      restore from restic, then destroy the instance"
    finish 0
  fi

  log "step 0: preconditions"
  # restic itself never runs on the laptop — every invocation goes through
  # drill_ssh and executes on the server. Requiring it here would check the
  # wrong machine (§13.2, dimension "binary").
  need_cmd hcloud; need_cmd jq; need_cmd pass-cli

  # Both drills run under 'pass-cli run --env-file .env', and both provision
  # an instance a few lines below this — checked here, before that, so a
  # vault mismatch fails before a CX53 is billed rather than after.
  vault_check_env

  vault_session_ok || die "pass-cli token invalid or expired.
Regenerate it in Proton Pass before drilling — this drill would otherwise fail
after provisioning a CX53, and report as a broken restore path rather than
routine credential expiry (§8)."

  for v in HCLOUD_TOKEN TS_AUTHKEY_EPHEMERAL HC_PING_KEY; do
    [[ -n "${!v:-}" ]] || die "missing vault value: $v"
  done

  # RESTIC_REPOSITORY and RESTIC_PASSWORD do not exist at 'pass-cli run' time —
  # phase 9 creates them — so, unlike the values above, .env cannot name them.
  # Checked straight against the vault instead (same reason as
  # storagebox-user/-host in run_server_lib()); each caller re-reads its own
  # copy at the point of use rather than relying on what this check discards.
  [[ -n "$(vault_get "$VAULT_ITEM_RESTIC_REPOSITORY")" ]] || die "missing vault value: $VAULT_ITEM_RESTIC_REPOSITORY"
  [[ -n "$(vault_get "$VAULT_ITEM_RESTIC_PASSWORD")" ]] || die "missing vault value: $VAULT_ITEM_RESTIC_PASSWORD"

  [[ -n "${TS_AUTHKEY_EPHEMERAL:-}" ]] || die "drills require the EPHEMERAL Tailscale key.
Reusing the persistent key leaves drill nodes lingering in the tailnet after
they are destroyed, and can leave a second machine answering to 'dev' (§8)."
  info "preconditions ok"
}

# ------------------------------------------------- instance lifecycle (§8)
# Destroy on ALL paths including failure — a drill that dies must not leave a
# billing instance behind. The trap is installed before creation so even a
# failure mid-create is cleaned up.
drill_cleanup() {
  local rc=$?
  if [[ -n "$DRILL_INSTANCE" ]]; then
    # Deregister from the tailnet before the machine goes away. An ephemeral
    # auth key is meant to make this unnecessary, but ephemerality is a
    # property of the key as generated in the Tailscale console — nothing in
    # this repository can assert it, and nothing here can tell an ephemeral key
    # from a persistent one by looking at it. A key generated without that box
    # ticked leaves one dead node per drill, which is how eighteen of them
    # accumulated before anyone looked.
    #
    # Best-effort and never fatal: the instance is being deleted either way, a
    # drill that has already reported its result must not fail here, and this
    # runs on the failure paths too, where the box may never have answered.
    #
    # A nonzero status here does NOT mean the logout failed. This ssh runs over
    # the tailnet, and `tailscale logout` tears down the very path carrying it,
    # so the connection drops before a status can come back — success and
    # failure both arrive as nonzero and this command cannot tell them apart.
    # The check that can is the tailnet device list: laptop/prune-tailnet.sh
    # --print-only, or the Tailscale console.
    if [[ -n "${DRILL_IP:-}" ]]; then
      drill_ssh 'sudo tailscale logout' >/dev/null 2>&1 \
        || warn "no status back from the tailnet logout on $DRILL_INSTANCE — expected when the ssh path IS the tailnet, and not evidence either way. Check with laptop/prune-tailnet.sh --print-only."
    fi
    log "destroying $DRILL_INSTANCE"
    hcloud server delete "$DRILL_INSTANCE" >/dev/null 2>&1 \
      || warn "could not delete $DRILL_INSTANCE — CHECK YOUR HETZNER CONSOLE"
  fi
  drill_report "$DRILL_RC"
  exit "$rc"
}

drill_create() {
  local kind=$1 image_arg=$2
  # Same default as bootstrap.sh's phase_1_provision() — not a second,
  # competing one: neither file may edit the other, and lib/common.sh
  # documents the derivation without exporting it.
  local keyname="${HCLOUD_SSH_KEY_NAME:-$FLIT_NAME-operator}"
  # Before creating anything: ensure_operator_key() (lib/common.sh) is the
  # same call phase_1_provision() makes, and it must run here too — a drill
  # against an already-provisioned box, or any bootstrap.sh run that starts
  # later than phase 1, never takes the path that used to register this key,
  # so `--ssh-key` below could not assume the name existed. Idempotent: a
  # matching registration is a no-op. It also resolves OPERATOR_PUBKEY itself
  # if a caller has not already done so, which is why it is safe to call
  # before the cloud-init render below — that render dies on an unset
  # OPERATOR_PUBKEY otherwise.
  ensure_operator_key "$keyname"
  DRILL_INSTANCE="drill-${kind}-$(date +%s)"
  trap drill_cleanup EXIT INT TERM
  log "creating $DRILL_INSTANCE (cx53, identical to production)"

  # kind=rebuild (drill-rebuild.sh) boots a stock ubuntu-24.04 image, which
  # has only 'root' until cloud-init runs — the same cloud-init that
  # phase_1_provision() (bootstrap.sh) passes when it creates the production
  # server, and the only place FLIT_USER and its authorized_keys get written.
  # Without it, the wait loop below probes an account that will never exist
  # and burns all 40 attempts before failing.
  #
  # kind=restore (drill-restore.sh) boots from a golden snapshot or an
  # automatic backup — both images of a production host that already ran
  # server/lib/40-firewall.sh, so public SSH is closed and the only inbound
  # path is the tailscale0 interface. The same disk carries production's
  # tailscale identity, which server/lib/30-tailscale.sh's clone guard
  # (server/bin/reset-tailscale-clone.sh) strips before tailscaled starts,
  # because this instance's DMI serial never matches the one recorded when
  # production joined. So the instance comes up with FLIT_USER's account
  # intact but no way onto the tailnet and no public SSH either — user-data
  # is the only path back in, and server/files/restore-cloud-init.sh is what
  # takes it: authenticate a fresh node with the EPHEMERAL key under a
  # hostname distinct from production's, so the node self-removes when the
  # drill destroys the instance instead of lingering, or contending with
  # 'dev' for the same MagicDNS name (§8).
  local cloud_init_file="" user_data_args=()
  if [[ "$kind" == "rebuild" ]]; then
    cloud_init_file=$(mktemp)
    # The console password exists only for the Hetzner browser-console
    # break-glass path (§4.2) and is generated fresh here rather than
    # escrowed: escrowing it under the same vault item phase_1_provision()
    # uses would overwrite the production server's real console password,
    # and drill_cleanup destroys this instance regardless of outcome, so
    # nothing needs the password to outlive the run.
    server/files/cloud-init.sh "$(openssl rand -base64 24)" >"$cloud_init_file" \
      || { rm -f "$cloud_init_file"; die "could not generate cloud-init user data"; }
    user_data_args=(--user-data-from-file "$cloud_init_file")
  elif [[ "$kind" == "restore" ]]; then
    cloud_init_file=$(mktemp)
    server/files/restore-cloud-init.sh "$DRILL_INSTANCE" "$TS_AUTHKEY_EPHEMERAL" >"$cloud_init_file" \
      || { rm -f "$cloud_init_file"; die "could not generate cloud-init user data"; }
    user_data_args=(--user-data-from-file "$cloud_init_file")
  fi

  # shellcheck disable=SC2086
  hcloud server create --name "$DRILL_INSTANCE" --type cx53 \
    --location "${HCLOUD_LOCATION:-fsn1}" \
    $image_arg \
    --ssh-key "$keyname" \
    "${user_data_args[@]}" \
    >/dev/null || { rm -f "$cloud_init_file"; die "could not create $DRILL_INSTANCE.
'resource_unavailable' is Hetzner capacity for cx53 in that location, not a
fault here — set HCLOUD_LOCATION to one that has it and re-run."; }
  rm -f "$cloud_init_file"
  local public_ip
  public_ip=$(hcloud server ip "$DRILL_INSTANCE")
  # A server created seconds ago has a host key nothing can have seen before,
  # so any known_hosts entry for this address belongs to a machine that no
  # longer exists. Hetzner hands the same address to the next server in the
  # pool, which makes this the normal case after a drill rather than an
  # exceptional one — same reasoning as bootstrap.sh's phase_1_provision(),
  # applied here because this path also just created the instance.
  ssh-keygen -R "$public_ip" >/dev/null 2>&1
  if [[ "$kind" == "restore" ]]; then
    # $public_ip is unreachable by design (see the comment above) — nothing
    # in the rest of this drill needs it. DRILL_IP holds a MagicDNS hostname
    # here, not an IP; every caller reaches it through drill_ssh(), which
    # only ever interpolates the variable, never parses it as an address.
    DRILL_IP="$DRILL_INSTANCE"
    info "public IP $public_ip is firewalled by the golden image (§8); waiting for $DRILL_IP on the tailnet instead"
  else
    DRILL_IP="$public_ip"
    info "waiting for ssh on $DRILL_IP"
  fi
  local attempt err="" ts_ip=""
  for attempt in $(seq 1 40); do
    # Keep the last failure rather than discarding it, same reasoning as
    # bootstrap.sh's wait_for_ssh(): "unreachable" alone does not say whether
    # cloud-init is still running, the account never got created, or Hetzner
    # is just slow, and those read identically after 400 silent seconds.
    if err=$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 \
        "$FLIT_USER@$DRILL_IP" true 2>&1); then
      info "reachable after ${attempt} attempt(s)"
      return 0
    fi
    # A name that will not resolve is not a node that is down. On the restore
    # path DRILL_IP holds a MagicDNS name, so a laptop that cannot complete
    # that name — macOS on a phone hotspot ends up with no search domain for
    # the tailnet suffix — spends all 40 attempts and then dies blaming the
    # ephemeral Tailscale key or MagicDNS propagation, neither of which is
    # wrong. tailscale knows the address without going through DNS, so it is
    # asked before an attempt is written off. The name is still tried first:
    # when it works it proves the resolution path the drill is meant to prove.
    if [[ "$DRILL_IP" == "$DRILL_INSTANCE" ]] \
       && command -v tailscale >/dev/null 2>&1 \
       && ts_ip=$(tailscale ip -4 "$DRILL_INSTANCE" 2>/dev/null) && [[ -n "$ts_ip" ]] \
       && err=$(ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 \
           "$FLIT_USER@$ts_ip" true 2>&1); then
      warn "'$DRILL_INSTANCE' does not resolve on this network; reaching it at $ts_ip instead.
The drill continues against that address. A name that will not resolve here is
not a node that is down, and every later step goes through drill_ssh(), which
uses whatever this resolved to."
      DRILL_IP="$ts_ip"
      info "reachable after ${attempt} attempt(s)"
      return 0
    fi
    # Same fast-fail as bootstrap.sh's wait_for_ssh(): waiting cannot fix a
    # host-key mismatch, so do not spend 400s discovering one 10s at a time.
    if grep -q 'REMOTE HOST IDENTIFICATION HAS CHANGED' <<<"$err"; then
      die "the host key for $DRILL_IP does not match ~/.ssh/known_hosts.
Hetzner reuses addresses, so a freshly created instance can answer on an
address whose old key you still trust. Drop the stale entry and re-run:
  ssh-keygen -R $DRILL_IP"
    fi
    sleep 10
  done
  die "$DRILL_INSTANCE never became reachable as $FLIT_USER@$DRILL_IP after ~400s. Last error:
$err
Likely cause on the stock-image path (kind=rebuild): cloud-init is still
writing '$FLIT_USER' and its authorized_keys, or failed to. Likely cause on
the snapshot/backup path (kind=restore): the node never joined the tailnet —
check that the ephemeral Tailscale key is still valid and Reusable, and that
MagicDNS has propagated the '$DRILL_INSTANCE' hostname. Check the Hetzner
console for cloud-init output; $DRILL_INSTANCE is destroyed automatically
once this die propagates."
}

drill_ssh() { ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new "$FLIT_USER@$DRILL_IP" "$@"; }

# Same PATH verify.sh exports, and for the same reason: an ssh command runs a
# NON-interactive shell, Ubuntu's ~/.bashrc returns on its first line in one,
# and the `mise activate` line that puts node, pnpm, cargo and claude within
# reach lives below that return. `bash -lc` does not help — ~/.profile sources
# the same ~/.bashrc. Only the assertions that invoke a managed toolchain need
# this; plain commands like apt-mark and test keep using drill_ssh directly.
drill_ssh_toolchain() {
  drill_ssh "export PATH=\"\$HOME/.local/bin:\$HOME/.local/share/mise/shims:\$HOME/.cargo/bin:\$HOME/.local/share/pnpm/bin:\$PATH\"; $*"
}

drill_report() {
  local rc=$1
  [[ -z "${DRILL_SLUG:-}" || -z "${HC_PING_KEY:-}" ]] && return 0
  curl -fsS -m 10 --retry 5 --data-binary "@${DRILL_LOG:-/dev/null}" \
    "https://hc-ping.com/${HC_PING_KEY}/${DRILL_SLUG}/${rc}" >/dev/null || true
}

# ------------------------------------------------------- assertions (§8)
# Build assertions run against fixtures/, never ~/workspace. At build time the
# workspace is empty, so workspace-based build checks would pass vacuously; and
# once it holds real repos they become non-deterministic — a drill would start
# failing because YOUR PROJECT broke, not because the restore did.
drill_assert_fixtures() {
  log "asserting fixtures build"
  drill_ssh_toolchain "cd ~/$FLIT_NAME/fixtures/node-fixture && pnpm install --frozen-lockfile" \
    || die "fixture: pnpm install failed"
  drill_ssh_toolchain "cd ~/$FLIT_NAME/fixtures/rust-fixture && cargo build --quiet" \
    || die "fixture: cargo build failed"
  # pass-cli takes a token only as `--pat` and reads no environment variable
  # for it, so the token phase 9 installed in /etc/$FLIT_NAME/secrets.env
  # establishes nothing until it is redeemed; vault_session_ok() in the
  # delivered lib/common.sh is the one place that does that, and probes for an
  # existing session first. Reading the box's OWN token rather than forwarding
  # the laptop's is what makes this an assertion about the rebuilt host: it
  # proves the credential phase 9 placed there resolves a secret.
  drill_ssh_toolchain "PASS_CLI_TOKEN=\$(sudo sed -n 's/^PASS_CLI_TOKEN=//p' /etc/$FLIT_NAME/secrets.env); \
                       export PASS_CLI_TOKEN; \
                       . ~/$FLIT_NAME/lib/common.sh; \
                       vault_session_ok || { echo 'no vault session on the rebuilt host' >&2; exit 1; }; \
                       pass-cli run --env-file ~/$FLIT_NAME/fixtures/.env -- printenv FIXTURE_SECRET" \
    | grep -q . || die "fixture: pass-cli did not resolve a secret"
  drill_ssh_toolchain 'claude --version' >/dev/null || die "claude --version did not exit zero"
  info "fixtures ok"
}

# Restored workspace is checked for PRESENCE, not buildability. That is what
# proves the restore worked; buildability is the fixtures' job.
drill_assert_restored_workspace() {
  log "asserting restored workspace content"
  drill_ssh "test -f ~/workspace/.$FLIT_NAME-sentinel" \
    || die "restored workspace has no sentinel — the backup does not contain what it should"
  local age
  age=$(drill_ssh "cat ~/workspace/.$FLIT_NAME-sentinel")
  info "sentinel present, written $age"
}