From c7c025cad18950be1ac863e1fd7ce080a8fa2111 Mon Sep 17 00:00:00 2001 From: dawn Date: Sat, 27 Jun 2026 11:21:56 +0000 Subject: [PATCH] nix,shuttle: setup cgroups, xdg runtime dir, subord id ranges in alpine image Signed-off-by: dawn --- nix/pkgs/spindle-alpine-image.nix | 25 +++++++++++++++++++++---- shuttle/src/exec.rs | 36 +++++++++++++++++++++++++++++++++++- spindle/engines/microvm/test-spindle-microvm.sh | 34 ++++++++++++++++++++++++++++++++++ 3 file(s) changed, 90 insertion(s)(+), 5 deletion(s)(-) diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix --- a/nix/pkgs/spindle-alpine-image.nix +++ b/nix/pkgs/spindle-alpine-image.nix @@ -40,15 +40,26 @@ mountpoint -q /sys || mount -t sysfs sys /sys mountpoint -q /dev || mount -t devtmpfs dev /dev mountpoint -q /dev/pts || { - mkdir -p /dev/pts + install -d /dev/pts mount -t devpts devpts /dev/pts } mountpoint -q /dev/shm || { - mkdir -p /dev/shm + install -d /dev/shm mount -t tmpfs -o mode=1777 shm /dev/shm } mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp + + # setup xdg runtime dir, podman eg. needs it + install -d -m 0700 -o spindle-workflow -g spindle-workflow /run/user/970 + + # cgroup2 setup, normally we would do this with rc-service + # but minirootfs does not ship with those so we set it up ourselves. + mountpoint -q /sys/fs/cgroup || { + install -d /sys/fs/cgroup + mount -t cgroup2 -o nsdelegate cgroup2 /sys/fs/cgroup + chown -R spindle-workflow:spindle-workflow /sys/fs/cgroup 2>/dev/null || true + } # the initramfs mdev leaves these 0660, which breaks non-root workflows chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null @@ -61,8 +72,7 @@ # /dev/vda is the squashfs root; the first spindle volume backs /workspace if [ -b /dev/vdb ]; then mount -t ext4 /dev/vdb /workspace - mkdir -p /workspace/repo - chown spindle-workflow:spindle-workflow /workspace /workspace/repo + install -d -o spindle-workflow -g spindle-workflow /workspace /workspace/repo fi ip link set lo up @@ -207,6 +217,13 @@ echo "spindle-workflow:x:970:" >> rootfs/etc/group echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow mkdir -p rootfs/workspace + + # subordinate id ranges so the workflow user can run rootless containers + # (podman/buildah): without these, user-namespace id mapping falls back to a + # single 970->0 map and any layer that chowns to another uid fails. the range + # is well clear of 970 and the 30000-block nixbld users. + echo "spindle-workflow:100000:65536" >> rootfs/etc/subuid + echo "spindle-workflow:100000:65536" >> rootfs/etc/subgid # setup nix build users for the daemon members="" diff --git a/shuttle/src/exec.rs b/shuttle/src/exec.rs --- a/shuttle/src/exec.rs +++ b/shuttle/src/exec.rs @@ -2,9 +2,10 @@ use crate::protocol::{self, Message, v1}; use nix::unistd::{Group, User}; use std::ffi::OsString; +use std::path::PathBuf; use std::time::Duration; use tokio::sync::mpsc::Sender; -use tracing::info; +use tracing::{info, warn}; const DEFAULT_USER: &str = "spindle-workflow"; @@ -40,8 +41,20 @@ } }; + let mut env = run_as.login_env(); + let runtime_dir = run_as.runtime_dir(); + match runtime_dir.try_exists() { + Ok(true) => env.push(( + OsString::from("XDG_RUNTIME_DIR"), + runtime_dir.into_os_string(), + )), + Ok(false) => {} + Err(err) => warn!(error = %err, "could not stat XDG_RUNTIME_DIR for workflow user"), + } + let mut spec = Spec::new(req.argv[0].clone()) .args(req.argv[1..].iter().cloned()) + .envs(env) .envs(parse_env(&req.env)) .run_as(run_as.uid, run_as.gid); if !req.cwd.is_empty() { @@ -106,6 +119,23 @@ name: String, uid: u32, gid: u32, + home: OsString, + shell: OsString, +} + +impl ResolvedUser { + fn login_env(&self) -> Vec<(OsString, OsString)> { + vec![ + (OsString::from("USER"), OsString::from(&self.name)), + (OsString::from("LOGNAME"), OsString::from(&self.name)), + (OsString::from("HOME"), self.home.clone()), + (OsString::from("SHELL"), self.shell.clone()), + ] + } + + fn runtime_dir(&self) -> PathBuf { + PathBuf::from(format!("/run/user/{}", self.uid)) + } } fn resolve_user(spec: &str) -> Result { @@ -137,6 +167,8 @@ name: name.to_owned(), uid: user.uid.as_raw(), gid: user.gid.as_raw(), + home: user.dir.into_os_string(), + shell: user.shell.into_os_string(), }), Ok(None) => { let uid = name @@ -146,6 +178,8 @@ name: name.to_owned(), uid, gid: uid, + home: OsString::from("/"), + shell: OsString::from("/bin/sh"), }) } Err(error) => Err(format!("lookup workflow user {name:?}: {error}")), diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh --- a/spindle/engines/microvm/test-spindle-microvm.sh +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -814,6 +814,39 @@ echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned + installed over the network, and substituted+ran a package from cache.nixos.org over HTTPS" } +test_alpine_podman() { + # install podman via apk and run a real container as the workflow user. + # rootless podman lives entirely in the writable workspace (storage + runroot + # under XDG dirs there), uses podman's default storage driver, and pulls over + # the guest network like the other alpine tests. + local out + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-podman" --timeout "300s" --no-cache -- /bin/sh -lc ' +set -eu +# no env setup here on purpose: shuttle seeds USER/LOGNAME/HOME/SHELL from the +# workflow users passwd entry and provisions XDG_RUNTIME_DIR, so rootless podman +# works out of the box. asserting those below doubles as a check on that. + +# shadow-uidmap ships newuidmap/newgidmap which rootless podman uses to apply the +# /etc/subuid + /etc/subgid ranges baked into the image. +apk add podman shadow-uidmap +echo "user=$(id -un) USER=$USER HOME=$HOME XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR" +echo "newuidmap=$(command -v newuidmap)" +echo "podman_version=$(podman --version)" + +podman info >/dev/null +echo "storage_driver=$(podman info --format "{{.Store.GraphDriverName}}")" + +podman run --rm --network=host docker.io/library/alpine cat /etc/alpine-release | sed "s/^/container_release=/" +podman run --rm --network=host docker.io/library/alpine echo container-ran-ok +' sh) || return 1 + + check_needles "$out" \ + "user=spindle-workflow USER=spindle-workflow HOME=/workspace XDG_RUNTIME_DIR=/run/user/970" \ + "newuidmap=/" "podman_version=" \ + "storage_driver=" "container_release=[0-9]+\." "container-ran-ok" || return 1 + echo "success: alpine guest installed podman via apk and pulled + ran a rootless container" +} + # asserts a store path's narinfo shows up in the local cache, retrying briefly # since the post-build-hook enqueues uploads asynchronously. cache_has_path() { @@ -914,6 +947,7 @@ TESTS=( test_alpine test_alpine_nix + test_alpine_podman test_realize test_build_upload test_ssh_store_upload -- tangled.sh