diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go --- a/cmd/spindle-microvm-run/main_linux.go +++ b/cmd/spindle-microvm-run/main_linux.go @@ -10,6 +10,7 @@ "log/slog" "net" "os" + "path/filepath" "time" "github.com/mdlayher/vsock" @@ -74,9 +75,10 @@ Usage: "timeout for the guest command", }, &cli.DurationFlag{ - Name: "cache-drain-timeout", - Value: 5 * time.Minute, - Usage: "how long to wait for queued cache uploads after the guest command exits", + Name: "cache-upload-wait-timeout", + Aliases: []string{"cache-drain-timeout"}, + Value: 5 * time.Minute, + Usage: "how long to wait for guest cache uploads to finish after the command exits", }, &cli.DurationFlag{ Name: "shutdown-timeout", @@ -177,6 +179,9 @@ } jobID := "spindle-microvm-run" execID := "dev-1" + var pendingConfigKey string + var pendingConfigToplevel string + var configCacheDB *db.DB fmt.Fprintf(os.Stderr, "listening for agent on %s\n", ln.Addr()) conn, err := acceptExpectedVsockConn(ln, vm.CID(), logger) @@ -203,7 +208,7 @@ var uploadCache *microvm.UploadCacheProxy if cmd.String("cache-upload-url") != "" { var err error - uploadCache, err = microvm.StartUploadCacheProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, logger) + uploadCache, err = microvm.StartUploadCacheProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, filepath.Join(vm.WorkDir(), "upload-cache"), logger) if err != nil { return fmt.Errorf("start upload cache proxy: %w", err) } @@ -243,22 +248,21 @@ return fmt.Errorf("calculate base config hash: %w", err) } - var d *db.DB var configKey string var cachedToplevel string if cmd.String("db") != "" { - d, err = db.Make(ctx, cmd.String("db")) + configCacheDB, err = db.Make(ctx, cmd.String("db")) if err != nil { return fmt.Errorf("failed to open database: %w", err) } - defer d.Close() + defer configCacheDB.Close() configKey, err = microvm.BuildConfigKey(imageSpec, cmd.String("activate-config")) if err != nil { return fmt.Errorf("calculate config key: %w", err) } - record, err := d.GetNixOSToplevelCacheRecord(configKey) + record, err := configCacheDB.GetNixOSToplevelCacheRecord(configKey) if err != nil { if !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("lookup config cache: %w", err) @@ -280,10 +284,12 @@ } fmt.Fprintf(os.Stderr, "activated config toplevel: %s\n", result.Toplevel) - if d != nil && cachedToplevel == "" && result.Toplevel != "" && configKey != "" { - err = d.SaveNixOSToplevelCacheRecord(configKey, result.Toplevel) - if err != nil { - return fmt.Errorf("save config cache: %w", err) + if configCacheDB != nil && cachedToplevel == "" && result.Toplevel != "" && configKey != "" { + if uploadCache == nil { + fmt.Fprintln(os.Stderr, "skipping config cache metadata commit: no cache upload url configured") + } else { + pendingConfigKey = configKey + pendingConfigToplevel = result.Toplevel } } } @@ -302,17 +308,22 @@ } if uploadCache != nil { - drainCtx := ctx - if cmd.Duration("cache-drain-timeout") > 0 { + uploadWaitCtx := ctx + if cmd.Duration("cache-upload-wait-timeout") > 0 { var cancel context.CancelFunc - drainCtx, cancel = context.WithTimeout(ctx, cmd.Duration("cache-drain-timeout")) + uploadWaitCtx, cancel = context.WithTimeout(ctx, cmd.Duration("cache-upload-wait-timeout")) defer cancel() } - uploaded, err := session.Drain(drainCtx) + uploaded, err := session.Drain(uploadWaitCtx) if err != nil { return err } fmt.Printf("cache uploaded: %d\n", uploaded) + if configCacheDB != nil && pendingConfigKey != "" && pendingConfigToplevel != "" { + if err := configCacheDB.SaveNixOSToplevelCacheRecord(pendingConfigKey, pendingConfigToplevel); err != nil { + return fmt.Errorf("save config cache: %w", err) + } + } } // mirror the engine shutdown order: ask the agent to power off first, diff --git a/spindle/engines/microvm/README.md b/spindle/engines/microvm/README.md --- a/spindle/engines/microvm/README.md +++ b/spindle/engines/microvm/README.md @@ -188,7 +188,10 @@ Teardown is same whether the workflow succeeded, failed or timed out: drain the guest's pending Nix cache uploads, ask the agent to power off and wait for QEMU to exit (falling back to QMP `system_powerdown` and finally a kill if it -doesn't), then close the proxies and remove the work directory. +doesn't), then close the proxies and remove the work directory. For non-HTTP +upload targets the host-side import already happened synchronously when the +guest committed each narinfo, so there is no second host-side cache drain step +at teardown. ### Nix cache @@ -204,5 +207,29 @@ The upload proxy goes the other way: paths built inside the guest are pushed to spindle's configured upload cache (if any) so the next workflow that needs them doesn't rebuild. Paths already present on any configured read cache are skipped. -The agent queues built paths and they're uploaded eagerly as they appear; any -still in flight at teardown block the drain step until they finish. + +For `http://` and `https://` upload targets the proxy just reverse-proxies the +guest's binary-cache upload traffic to the configured remote cache, while still +answering narinfo existence checks across the upload target plus the read +caches. + +For `ssh://`, `ssh-ng://`, `daemon`, and `local` targets spindle implements the +small HTTP binary-cache upload surface itself. It stages uploaded `nar/` objects +and narinfos under the workflow workdir, validates the narinfo, then treats the +narinfo upload as the commit point: once `.narinfo` is written spindle +runs: + +```bash +nix copy \ + --from file:// \ + --to \ + --no-check-sigs \ + --substitute-on-destination \ + +``` + +That copy is synchronous. If it fails, spindle removes the staged narinfo again +so future `GET`/`HEAD .narinfo` requests do not falsely dedupe a path that +never made it to the destination store. The guest still only ever sees the same +HTTP binary-cache upload protocol over vsock; it never gets direct access to +SSH credentials or the destination store itself. diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -8,6 +8,7 @@ "io" "log/slog" "os" + "path/filepath" "slices" "sync" "sync/atomic" @@ -231,7 +232,8 @@ return err } state.ReadCache = readCache - uploadCache, err := StartUploadCacheProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, l) + stagingDir := filepath.Join(workDir, "upload-cache") + uploadCache, err := StartUploadCacheProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, stagingDir, l) if err != nil { return err } @@ -451,9 +453,7 @@ return nil } - drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) - defer cancel() - if _, err := state.Agent.Drain(drainCtx); err != nil { + if err := e.drainNixCache(ctx, state); err != nil { return fmt.Errorf("drain config cache uploads before metadata commit: %w", err) } if err := state.NixOSToplevelCache.Commit(configKey, result.Toplevel); err != nil { diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh --- a/spindle/engines/microvm/test-spindle-microvm.sh +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -15,6 +15,7 @@ declare -a TEST_NAMES=() declare -a TEST_STATUSES=() declare -a TEST_TIMES=() +SKIP_TEST_RC=200 get_time_ms() { local t="${EPOCHREALTIME:-}" @@ -46,6 +47,7 @@ log "test summary" echo "=========================================" local passed_count=0 + local skipped_count=0 local failed_count=0 local total_time=0 for i in "${!TEST_NAMES[@]}"; do @@ -59,6 +61,9 @@ if [ "$status" = "Failed" ]; then status_color="\033[0;31m" failed_count=$((failed_count + 1)) + elif [ "$status" = "Skipped" ]; then + status_color="\033[0;33m" + skipped_count=$((skipped_count + 1)) else passed_count=$((passed_count + 1)) fi @@ -70,9 +75,19 @@ local total_tests="${#TEST_NAMES[@]}" local total_time_str total_time_str=$(format_duration "$total_time") - printf " total: %d tests, %d passed, %d failed\n" "$total_tests" "$passed_count" "$failed_count" + printf " total: %d tests, %d passed, %d skipped, %d failed\n" "$total_tests" "$passed_count" "$skipped_count" "$failed_count" printf " total execution time: %s\n" "$total_time_str" echo "=========================================" +} + +skip_test() { + echo "skipped: $*" + return "$SKIP_TEST_RC" +} + +host_is_nixos() { + [ -e /etc/NIXOS ] && return 0 + [ -r /etc/os-release ] && grep -q '^ID=nixos$' /etc/os-release } JOBS="${JOBS:-4}" @@ -124,6 +139,11 @@ log "setup local cache & temp environment" TEMP_DIR=$(mktemp -d -t test-spindle-microvm-XXXXXX) + +if [ ! -e /dev/vsock ]; then + echo "error: /dev/vsock is missing; run sudo modprobe vhost_vsock" >&2 + exit 1 +fi log "build spindle & microvm image tarball" nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run @@ -201,6 +221,7 @@ local name="" local timeout="60s" local upload=0 + local upload_url="" local activate="" local no_cache=0 local db="" @@ -223,6 +244,11 @@ --upload) upload=1 shift + ;; + --upload-url) + upload=1 + upload_url="$2" + shift 2 ;; --activate) activate="$2" @@ -266,8 +292,11 @@ fi if [ "$upload" -eq 1 ]; then + if [ -z "$upload_url" ]; then + upload_url="$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" + fi args+=( - --cache-upload-url "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" + --cache-upload-url "$upload_url" ) fi @@ -305,8 +334,15 @@ log "[$name] start (vsock port $port)" local status="Passed" - if ! "$func" > "$logfile" 2>&1; then - status="Failed" + if "$func" > "$logfile" 2>&1; then + status="Passed" + else + local rc=$? + if [ "$rc" -eq "$SKIP_TEST_RC" ]; then + status="Skipped" + else + status="Failed" + fi fi local duration_ms=$(($(get_time_ms) - start)) @@ -316,6 +352,9 @@ duration_str=$(format_duration "$duration_ms") if [ "$status" = "Failed" ]; then printf "\n\033[0;31m>>> [%s] FAILED (%s)\033[0m\n" "$name" "$duration_str" + strip_ansi "$logfile" || true + elif [ "$status" = "Skipped" ]; then + printf "\n\033[0;33m>>> [%s] skipped (%s)\033[0m\n" "$name" "$duration_str" strip_ansi "$logfile" || true else printf "\n\033[0;32m>>> [%s] passed (%s)\033[0m\n" "$name" "$duration_str" @@ -435,6 +474,42 @@ return 1 fi echo "success: store path uploaded to cache" +} + +test_ssh_store_upload() { + if ! host_is_nixos; then + skip_test "ssh store upload smoke only runs on nixos hosts" + fi + + run_ssh_store_upload_case() { + local target="$1" + local label="$2" + local name="uploaded-test-file-${label}" + local content="hello from vm upload via ${label}" + local out + out=$(run_vm --name "$label" --timeout "180s" --upload-url "$target" -- /run/current-system/sw/bin/bash -l -c "nix-build -E 'with import {}; writeText \"$name\" \"$content\"' --no-out-link") || return 1 + + local built_path + built_path=$(echo "$out" | strip_ansi | grep -v '\.drv' | grep -o "/nix/store/[a-z0-9]*-${name}" | head -n 1 || true) + if [ -z "$built_path" ]; then + echo "error: could not find built store path in vm output for $label" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + if [ ! -e "$built_path" ]; then + echo "error: uploaded path missing from host store for $label: $built_path" >&2 + return 1 + fi + if [ "$(cat "$built_path")" != "$content" ]; then + echo "error: uploaded path content mismatch for $label" >&2 + echo "path=$built_path" >&2 + return 1 + fi + echo "success: store path uploaded to $target via spindle" + } + + run_ssh_store_upload_case "ssh-ng://localhost" "ssh-ng-upload" + run_ssh_store_upload_case "ssh://localhost" "ssh-upload" } test_networking() { @@ -831,6 +906,7 @@ test_alpine_nix test_realize test_build_upload + test_ssh_store_upload test_networking test_substitution_and_no_upload test_activation_services diff --git a/spindle/engines/microvm/upload_cache_http.go b/spindle/engines/microvm/upload_cache_http.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_http.go @@ -0,0 +1,101 @@ +package microvm + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "strings" +) + +// httpUploadBackend reverse-proxies guest binary-cache upload traffic to an +// http(s) upload cache such as ncps. +type httpUploadBackend struct { + handler http.Handler +} + +func newHTTPUploadProxyBackend(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) *httpUploadBackend { + return &httpUploadBackend{handler: uploadProxyHandler(target, readUpstreams, logger)} +} + +func (b *httpUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { + b.handler.ServeHTTP(w, r) +} + +func (b *httpUploadBackend) Close() error { return nil } + +func uploadProxyHandler(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) http.Handler { + rp := httputil.NewSingleHostReverseProxy(target) + rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) + + origDirector := rp.Director + rp.Director = func(req *http.Request) { + origDirector(req) + // ensure host matches target + req.Host = target.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.User; user != nil { + password, _ := user.Password() + req.SetBasicAuth(user.Username(), password) + } + } + + // before uploading, nix copy asks the destination whether it already has each + // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer + // that check across the upload target *and* the read caches: if any of them + // already serves the path there is no point uploading it (the guest would + // just substitute it from there anyway). + narinfoUpstreams := append([]CacheUpstream{{url: target}}, readUpstreams...) + exists := newNarinfoExistenceTransport(narinfoUpstreams, logger) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isNarinfoExistenceCheck(r) { + serveNarinfoExistence(w, r, exists, logger) + return + } + rp.ServeHTTP(w, r) + }) +} + +func newNarinfoExistenceTransport(upstreams []CacheUpstream, logger *slog.Logger) http.RoundTripper { + return ¶llelRacingTransport{ + upstreams: upstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + } +} + +func isNarinfoExistenceCheck(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + return strings.HasSuffix(r.URL.Path, ".narinfo") +} + +func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { + probe := r.Clone(r.Context()) + probe.RequestURI = "" + + resp, err := exists.RoundTrip(probe) + if err != nil { + logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) + w.WriteHeader(http.StatusNotFound) + return + } + defer resp.Body.Close() + + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { + logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) + } +} diff --git a/spindle/engines/microvm/upload_cache_narinfo.go b/spindle/engines/microvm/upload_cache_narinfo.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_narinfo.go @@ -0,0 +1,124 @@ +package microvm + +import ( + "bufio" + "fmt" + "io" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +type narinfo struct { + StorePath string + URL string + NarHash string + NarSize int64 +} + +const ( + maxNarinfoSize = 1 << 20 // 1 MiB + storePrefix = "/nix/store/" + maxNarinfoLineLen = maxNarinfoSize +) + +var nixStorePathBaseRe = regexp.MustCompile(`^[0-9abcdfghijklmnpqrsvwxyz]{32}-[^/]+$`) + +// parseNarinfo parses and validates a narinfo body. +// - required fields must be present +// - StorePath must be under /nix/store/ +// - URL must be a relative, traversal-safe path referencing a NAR in the +// same staging cache +// - NarSize must be a non-negative integer +func parseNarinfo(r io.Reader) (*narinfo, error) { + lr := io.LimitReader(r, maxNarinfoSize+1) + scanner := bufio.NewScanner(lr) + scanner.Buffer(make([]byte, 4096), maxNarinfoLineLen) + + var info narinfo + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + return nil, fmt.Errorf("invalid narinfo line %q", line) + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + + switch key { + case "StorePath": + info.StorePath = value + case "URL": + info.URL = value + case "NarHash": + info.NarHash = value + case "NarSize": + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid NarSize %q: %w", value, err) + } + info.NarSize = n + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read narinfo: %w", err) + } + + if err := validateNarinfo(&info); err != nil { + return nil, err + } + return &info, nil +} + +func validateNarinfo(info *narinfo) error { + if info.StorePath == "" { + return fmt.Errorf("narinfo missing StorePath") + } + if _, _, err := parseStorePath(info.StorePath); err != nil { + return fmt.Errorf("invalid StorePath: %w", err) + } + if info.URL == "" { + return fmt.Errorf("narinfo missing URL") + } + if strings.HasPrefix(info.URL, "/") || strings.Contains(info.URL, "..") { + return fmt.Errorf("narinfo URL %q is not a safe relative path", info.URL) + } + if !strings.HasPrefix(info.URL, "nar/") { + return fmt.Errorf("narinfo URL %q must reference a staged nar/ object", info.URL) + } + name := strings.TrimPrefix(info.URL, "nar/") + if name == "" || name == "." || name != filepath.Base(name) || strings.Contains(name, "/") { + return fmt.Errorf("narinfo URL %q is not a safe nar object path", info.URL) + } + if info.NarHash == "" { + return fmt.Errorf("narinfo missing NarHash") + } + if info.NarSize < 0 { + return fmt.Errorf("narinfo NarSize must be non-negative") + } + return nil +} + +func parseStorePath(path string) (hash string, name string, err error) { + if !strings.HasPrefix(path, storePrefix) { + return "", "", fmt.Errorf("store path %q does not start with %q", path, storePrefix) + } + + base := strings.TrimPrefix(path, storePrefix) + if base == "" || strings.Contains(base, "/") { + return "", "", fmt.Errorf("store path %q has invalid base name", path) + } + if !nixStorePathBaseRe.MatchString(base) { + return "", "", fmt.Errorf("store path %q is not a valid nix store path", path) + } + + hash, name, ok := strings.Cut(base, "-") + if !ok || hash == "" || name == "" { + return "", "", fmt.Errorf("store path %q is missing hash or name", path) + } + return hash, name, nil +} diff --git a/spindle/engines/microvm/upload_cache_nix_store.go b/spindle/engines/microvm/upload_cache_nix_store.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_nix_store.go @@ -0,0 +1,436 @@ +package microvm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// we have an interface for running commands so we can swap it in tests +type CommandRunner interface { + Run(ctx context.Context, name string, args ...string) error +} + +type execRunner struct{} + +func (execRunner) Run(ctx context.Context, name string, args ...string) error { + // nosemgrep: go.lang.security.audit.dangerous-exec-command.dangerous-exec-command + cmd := exec.CommandContext(ctx, name, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, string(out)) + } + return nil +} + +const ( + nixStoreCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 50\n" + maxNarUploadSize = 5 << 30 // 5gib +) + +type NixStoreUploadBackend struct { + stagingDir string + targetStore string + readUpstreams []CacheUpstream + logger *slog.Logger + runner CommandRunner + maxNarUploadSize int64 +} + +func newNixStoreUploadBackend(targetStore, stagingDir string, readUpstreams []CacheUpstream, logger *slog.Logger, runner CommandRunner) (*NixStoreUploadBackend, error) { + absStaging, err := filepath.Abs(stagingDir) + if err != nil { + return nil, fmt.Errorf("resolve staging dir %q: %w", stagingDir, err) + } + if logger == nil { + logger = slog.Default() + } + + if err := os.MkdirAll(filepath.Join(absStaging, "nar"), 0o755); err != nil { + return nil, fmt.Errorf("create staging cache directories: %w", err) + } + infoPath := filepath.Join(absStaging, "nix-cache-info") + if _, err := os.Stat(infoPath); errors.Is(err, os.ErrNotExist) { + if err := os.WriteFile(infoPath, []byte(nixStoreCacheInfo), 0o644); err != nil { + return nil, fmt.Errorf("write nix-cache-info: %w", err) + } + } + + if runner == nil { + runner = execRunner{} + } + + return &NixStoreUploadBackend{ + stagingDir: absStaging, + targetStore: targetStore, + readUpstreams: readUpstreams, + logger: logger, + runner: runner, + maxNarUploadSize: maxNarUploadSize, + }, nil +} + +func (b *NixStoreUploadBackend) Close() error { return nil } + +func (b *NixStoreUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { + relPath, err := normalizeUploadCachePath(r.URL.Path) + if err != nil { + b.logger.Warn("refusing upload cache request with unsafe path", "path", r.URL.Path, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + switch r.Method { + case http.MethodGet, http.MethodHead: + switch { + case relPath == "nix-cache-info": + b.serveCacheInfo(w, r) + return + case isNarinfoObjectPath(relPath): + b.serveNarinfo(w, r, relPath) + return + } + + case http.MethodPut: + switch { + case relPath == "nix-cache-info": + b.putCacheInfo(w, r) + return + case isNarObjectPath(relPath): + b.putNar(w, r, relPath) + return + case isNarinfoObjectPath(relPath): + b.putNarinfo(w, r, relPath) + return + } + } + + http.Error(w, "not found", http.StatusNotFound) +} + +func (b *NixStoreUploadBackend) serveCacheInfo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/x-nix-cache-info") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(nixStoreCacheInfo))) + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusOK) + return + } + _, _ = w.Write([]byte(nixStoreCacheInfo)) +} + +func (b *NixStoreUploadBackend) putCacheInfo(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, io.LimitReader(r.Body, int64(len(nixStoreCacheInfo))+1)) + w.WriteHeader(http.StatusOK) +} + +func (b *NixStoreUploadBackend) serveNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { + localPath, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing narinfo request with unsafe path", "path", relPath, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + fi, err := os.Stat(localPath) + if err == nil && !fi.IsDir() { + if _, err := readNarinfoFile(localPath); err != nil { + b.logger.Warn("staged narinfo is invalid", "path", relPath, "error", err) + http.Error(w, "invalid staged narinfo", http.StatusInternalServerError) + return + } + b.serveLocalFile(w, r, localPath, fi) + return + } + if !errors.Is(err, os.ErrNotExist) { + b.logger.Warn("stat staged narinfo failed", "path", relPath, "error", err) + } + + if len(b.readUpstreams) > 0 { + probe := r.Clone(r.Context()) + probe.URL.Path = "/" + relPath + serveNarinfoExistence(w, probe, newNarinfoExistenceTransport(b.readUpstreams, b.logger), b.logger) + return + } + + http.Error(w, "not found", http.StatusNotFound) +} + +func (b *NixStoreUploadBackend) serveLocalFile(w http.ResponseWriter, r *http.Request, localPath string, fi os.FileInfo) { + w.Header().Set("Content-Type", "text/x-nix-narinfo") + w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size())) + w.Header().Set("Last-Modified", fi.ModTime().UTC().Format(http.TimeFormat)) + + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusOK) + return + } + + f, err := os.Open(localPath) + if err != nil { + b.logger.Warn("open staged narinfo failed", "path", localPath, "error", err) + http.Error(w, "not found", http.StatusNotFound) + return + } + defer f.Close() + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, f); err != nil && !errors.Is(err, context.Canceled) { + b.logger.Warn("copy staged narinfo failed", "path", localPath, "error", err) + } +} + +func (b *NixStoreUploadBackend) putNar(w http.ResponseWriter, r *http.Request, relPath string) { + name := strings.TrimPrefix(relPath, "nar/") + dst, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing nar upload with unsafe path", "name", name, "error", err) + http.Error(w, "invalid nar path", http.StatusBadRequest) + return + } + r.Body = http.MaxBytesReader(w, r.Body, b.maxNarUploadSize) + + var copyErr error + written, err := writeFileAtomic(dst, ".tmp-nar", func(f *os.File) (int64, error) { + n, err := io.Copy(f, r.Body) + copyErr = err + return n, err + }) + if err != nil { + b.logger.Warn("stage nar upload failed", "name", name, "error", err) + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + http.Error(w, "nar too large", http.StatusRequestEntityTooLarge) + return + } + if copyErr != nil { + http.Error(w, "upload failed", http.StatusBadRequest) + return + } + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + b.logger.Debug("staged nar", "name", name, "bytes", written) + w.WriteHeader(http.StatusOK) +} + +func (b *NixStoreUploadBackend) putNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { + body, err := io.ReadAll(io.LimitReader(r.Body, maxNarinfoSize+1)) + if err != nil { + b.logger.Warn("read narinfo body failed", "path", relPath, "error", err) + http.Error(w, "upload failed", http.StatusBadRequest) + return + } + if len(body) > maxNarinfoSize { + b.logger.Warn("narinfo body exceeds maximum size", "path", relPath, "bytes", len(body)) + http.Error(w, "narinfo too large", http.StatusBadRequest) + return + } + + info, err := parseNarinfo(bytes.NewReader(body)) + if err != nil { + b.logger.Warn("refusing narinfo upload with invalid body", "path", relPath, "error", err) + http.Error(w, "invalid narinfo: "+err.Error(), http.StatusBadRequest) + return + } + storePathHash, _, err := parseStorePath(info.StorePath) + if err != nil { + b.logger.Warn("refusing narinfo upload with invalid store path", "path", relPath, "storePath", info.StorePath, "error", err) + http.Error(w, "invalid StorePath", http.StatusBadRequest) + return + } + fileHash := strings.TrimSuffix(filepath.Base(relPath), ".narinfo") + if fileHash != storePathHash { + b.logger.Warn("refusing narinfo upload with mismatched filename hash", "path", relPath, "storePath", info.StorePath) + http.Error(w, "narinfo filename does not match StorePath hash", http.StatusBadRequest) + return + } + if !isNarObjectPath(info.URL) { + b.logger.Warn("narinfo references invalid nar URL", "path", relPath, "url", info.URL) + http.Error(w, "invalid nar URL", http.StatusBadRequest) + return + } + + narPath, err := b.stagingObjectPath(info.URL) + if err != nil { + b.logger.Warn("narinfo references unsafe nar URL", "path", relPath, "url", info.URL, "error", err) + http.Error(w, "invalid nar URL", http.StatusBadRequest) + return + } + if _, err := os.Stat(narPath); err != nil { + b.logger.Warn("narinfo references missing nar", "path", relPath, "url", info.URL, "error", err) + http.Error(w, "referenced nar does not exist", http.StatusBadRequest) + return + } + + dst, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing narinfo upload with unsafe path", "path", relPath, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + if _, err := writeFileAtomic(dst, ".tmp-narinfo", func(f *os.File) (int64, error) { + n, err := f.Write(body) + return int64(n), err + }); err != nil { + b.logger.Warn("stage narinfo upload failed", "path", relPath, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if err := b.importStorePath(r.Context(), info.StorePath); err != nil { + b.logger.Warn("import staged narinfo failed", "path", relPath, "storePath", info.StorePath, "error", err) + if cleanupErr := removeFileAndSyncDir(dst); cleanupErr != nil { + b.logger.Error("remove staged narinfo after failed import", "path", relPath, "error", cleanupErr) + } + http.Error(w, "import failed", http.StatusBadGateway) + return + } + + b.logger.Debug("staged narinfo", "path", relPath, "storePath", info.StorePath) + w.WriteHeader(http.StatusOK) +} + +func normalizeUploadCachePath(path string) (string, error) { + if path == "" || path == "/" { + return "", fmt.Errorf("empty path") + } + if !strings.HasPrefix(path, "/") { + return "", fmt.Errorf("path must start with /") + } + if strings.Contains(path, "..") { + return "", fmt.Errorf("path traversal") + } + + return strings.TrimPrefix(path, "/"), nil +} + +func isNarinfoObjectPath(relPath string) bool { + if !strings.HasSuffix(relPath, ".narinfo") { + return false + } + if relPath != filepath.Base(relPath) { + return false + } + base := filepath.Base(relPath) + return base != "" && base != "." && base != ".narinfo" +} + +func isNarObjectPath(relPath string) bool { + if !strings.HasPrefix(relPath, "nar/") { + return false + } + name := strings.TrimPrefix(relPath, "nar/") + return name != "" && name != "." && name == filepath.Base(name) && !strings.Contains(name, "/") +} + +func (b *NixStoreUploadBackend) stagingObjectPath(relPath string) (string, error) { + if !isNarObjectPath(relPath) && !isNarinfoObjectPath(relPath) { + return "", fmt.Errorf("invalid cache object path %q", relPath) + } + + local, err := filepath.Localize(relPath) + if err != nil { + return "", fmt.Errorf("unsafe cache object path %q: %w", relPath, err) + } + + return filepath.Join(b.stagingDir, local), nil +} + +// todo(dawn): ideally we don't use `nix copy` here but instead have our own +// `nix copy` impl so we don't need nix on host. but that's a far stretch goal :p +func (b *NixStoreUploadBackend) importStorePath(ctx context.Context, storePath string) error { + fromURL := url.URL{Scheme: "file", Path: b.stagingDir} + args := []string{ + "copy", + "--from", fromURL.String(), + "--to", b.targetStore, + // todo(dawn): ideally we support signing in spindle itself. + // but for now harmonia can sign things on serve so this is ok. + "--no-check-sigs", + "--substitute-on-destination", + storePath, + } + + b.logger.Info("importing staged cache path", "target", b.targetStore, "storePath", storePath) + if err := b.runner.Run(ctx, "nix", args...); err != nil { + return fmt.Errorf("nix copy to %s: %w", b.targetStore, err) + } + return nil +} + +func readNarinfoFile(path string) (*narinfo, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return parseNarinfo(f) +} + +func writeFileAtomic(dst, tempPrefix string, write func(*os.File) (int64, error)) (written int64, err error) { + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0o755); err != nil { + return 0, fmt.Errorf("create directory %q: %w", dir, err) + } + + tmp, err := os.CreateTemp(dir, tempPrefix+"-*-"+filepath.Base(dst)) + if err != nil { + return 0, fmt.Errorf("create temporary file in %q: %w", dir, err) + } + tmpName := tmp.Name() + defer func() { + if err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + } + }() + + written, err = write(tmp) + if err != nil { + return 0, err + } + if err := tmp.Sync(); err != nil { + return 0, fmt.Errorf("fsync temporary file %q: %w", tmpName, err) + } + if err := tmp.Close(); err != nil { + return 0, fmt.Errorf("close temporary file %q: %w", tmpName, err) + } + if err := os.Rename(tmpName, dst); err != nil { + return 0, fmt.Errorf("rename %q to %q: %w", tmpName, dst, err) + } + + if err := syncDir(dir); err != nil { + return 0, err + } + + return written, nil +} + +func removeFileAndSyncDir(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove %q: %w", path, err) + } + return syncDir(filepath.Dir(path)) +} + +func syncDir(dir string) error { + dirFile, err := os.Open(dir) + if err != nil { + return fmt.Errorf("open directory %q: %w", dir, err) + } + defer dirFile.Close() + if err := dirFile.Sync(); err != nil { + return fmt.Errorf("sync directory %q: %w", dir, err) + } + return nil +} diff --git a/spindle/engines/microvm/upload_cache_nix_store_test.go b/spindle/engines/microvm/upload_cache_nix_store_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_nix_store_test.go @@ -0,0 +1,438 @@ +package microvm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" +) + +const ( + testStoreHash = "0123456789abcdfghijklmnpqrsvwxyz" + testStorePath = "/nix/store/" + testStoreHash + "-abc-output" +) + +func TestUploadCacheBackendSchemeDispatch(t *testing.T) { + staging := t.TempDir() + logger := slog.Default() + + cases := []struct { + uploadURL string + wantErr bool + wantType string + }{ + {"https://cache.example/upload", false, "*microvm.httpUploadBackend"}, + {"http://cache.example/upload", false, "*microvm.httpUploadBackend"}, + {"ssh://cache-host", false, "*microvm.NixStoreUploadBackend"}, + {"ssh-ng://cache-host", false, "*microvm.NixStoreUploadBackend"}, + {"daemon", false, "*microvm.NixStoreUploadBackend"}, + {"local", false, "*microvm.NixStoreUploadBackend"}, + {"ftp://cache.example", true, ""}, + {"/some/path", true, ""}, + } + + for _, tc := range cases { + t.Run(tc.uploadURL, func(t *testing.T) { + backend, err := newUploadCacheBackend(tc.uploadURL, nil, staging, logger) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q", tc.uploadURL) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := fmt.Sprintf("%T", backend) + if got != tc.wantType { + t.Fatalf("backend type: got %s, want %s", got, tc.wantType) + } + }) + } +} + +func TestUploadCacheBackendEmptyURL(t *testing.T) { + backend, err := newUploadCacheBackend("", nil, t.TempDir(), slog.Default()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if backend != nil { + t.Fatalf("expected nil backend for empty URL, got %T", backend) + } +} + +func newTestNixStoreBackend(t *testing.T, target string, runner CommandRunner) (*NixStoreUploadBackend, string) { + t.Helper() + staging := t.TempDir() + if target == "" { + target = "ssh-ng://cache-host" + } + b, err := newNixStoreUploadBackend(target, staging, nil, slog.Default(), runner) + if err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + return b, staging +} + +func mustUploadNar(t *testing.T, b *NixStoreUploadBackend, name, body string) { + t.Helper() + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/"+name, strings.NewReader(body))) + if rec.Code != http.StatusOK { + t.Fatalf("upload nar %q: got %d, want 200; body=%q", name, rec.Code, rec.Body.String()) + } +} + +func TestNixStoreBackendNixCacheInfo(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/nix-cache-info", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET status: got %d, want 200", rec.Code) + } + if !strings.Contains(rec.Body.String(), "StoreDir: /nix/store") { + t.Fatalf("cache info missing StoreDir: %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodHead, "/nix-cache-info", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("HEAD status: got %d, want 200", rec.Code) + } + if rec.Body.Len() != 0 { + t.Fatalf("HEAD body should be empty, got %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nix-cache-info", strings.NewReader("ignored"))) + if rec.Code != http.StatusOK { + t.Fatalf("PUT /nix-cache-info status: got %d, want 200", rec.Code) + } +} + +func TestNixStoreBackendRejectsTraversalNar(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/../../evil", strings.NewReader("bad"))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("traversal nar status: got %d, want 400", rec.Code) + } + + if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "evil")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("traversal nar escaped staging dir: %v", err) + } +} + +func TestNixStoreBackendRejectsOversizedNarUpload(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + b.maxNarUploadSize = 3 + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/foo.nar", strings.NewReader("four"))) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized nar status: got %d, want 413; body=%q", rec.Code, rec.Body.String()) + } + + if _, err := os.Stat(filepath.Join(staging, "nar", "foo.nar")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("oversized nar should not have been staged: %v", err) + } +} + +func TestNixStoreBackendNarinfoRequiresExistingNar(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/abc.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("narinfo before nar status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } +} + +type fakeRunner struct { + mu sync.Mutex + calls [][]string + nextErr error +} + +func (f *fakeRunner) Run(ctx context.Context, name string, args ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + call := append([]string{name}, args...) + f.calls = append(f.calls, call) + return f.nextErr +} + +func (f *fakeRunner) Calls() [][]string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.calls) +} + +func TestNixStoreBackendImportsNarinfoImmediately(t *testing.T) { + runner := &fakeRunner{} + b, staging := newTestNixStoreBackend(t, "ssh-ng://spindle-upload@cache-host", runner) + + mustUploadNar(t, b, "foo.nar.zst", "nar-body") + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusOK { + t.Fatalf("PUT narinfo status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + + calls := runner.Calls() + if len(calls) != 1 { + t.Fatalf("expected 1 nix copy call, got %d", len(calls)) + } + call := calls[0] + wantFrom := (&url.URL{Scheme: "file", Path: staging}).String() + want := []string{ + "nix", + "copy", + "--from", wantFrom, + "--to", "ssh-ng://spindle-upload@cache-host", + "--no-check-sigs", + "--substitute-on-destination", + testStorePath, + } + if !slices.Equal(call, want) { + t.Fatalf("nix copy args:\n got: %v\nwant: %v", call, want) + } + + data, err := os.ReadFile(filepath.Join(staging, testStoreHash+".narinfo")) + if err != nil { + t.Fatalf("staged narinfo missing: %v", err) + } + if string(data) != narinfo { + t.Fatalf("staged narinfo contents: got %q, want %q", string(data), narinfo) + } +} + +func TestNixStoreBackendRemovesNarinfoOnImportFailure(t *testing.T) { + runner := &fakeRunner{nextErr: errors.New("nix copy failed")} + b, staging := newTestNixStoreBackend(t, "ssh://cache-host", runner) + + mustUploadNar(t, b, "foo.nar.zst", "nar-body") + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusBadGateway { + t.Fatalf("failed import status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) + } + + if _, err := os.Stat(filepath.Join(staging, testStoreHash+".narinfo")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("narinfo should be removed after failed import: %v", err) + } +} + +func TestNixStoreBackendNarinfoReadUpstream(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/miss.narinfo" { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = io.WriteString(w, "StorePath: /nix/store/upstream\nURL: nar/upstream.nar\nNarHash: sha256:up\nNarSize: 1\n") + })) + defer upstream.Close() + + upURL, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + + staging := t.TempDir() + b, err := newNixStoreUploadBackend("ssh://cache-host", staging, []CacheUpstream{{url: upURL}}, slog.Default(), nil) + if err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/present.narinfo", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET upstream-present narinfo status: got %d, want 200", rec.Code) + } + if !strings.Contains(rec.Body.String(), "/nix/store/upstream") { + t.Fatalf("unexpected upstream narinfo body: %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/miss.narinfo", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("GET upstream-missing narinfo status: got %d, want 404", rec.Code) + } + +} + +func TestNixStoreBackendRejectsInvalidLocalNarinfo(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + if err := os.WriteFile(filepath.Join(staging, testStoreHash+".narinfo"), []byte("not-a-narinfo\n"), 0o644); err != nil { + t.Fatalf("write invalid staged narinfo: %v", err) + } + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/"+testStoreHash+".narinfo", nil)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("invalid local narinfo status: got %d, want 500; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestNixStoreBackendNarinfoValidation(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + mustUploadNar(t, b, "x.nar", "x") + + cases := []struct { + name string + body string + wantErr string + }{ + { + name: "missing StorePath", + body: "URL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "StorePath", + }, + { + name: "bad StorePath", + body: "StorePath: /tmp/evil\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "invalid StorePath", + }, + { + name: "malformed StorePath", + body: "StorePath: /nix/store/not-a-real-store-path\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "invalid StorePath", + }, + { + name: "missing URL", + body: "StorePath: " + testStorePath + "\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "absolute URL", + body: "StorePath: " + testStorePath + "\nURL: /etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "traversal URL", + body: "StorePath: " + testStorePath + "\nURL: nar/../../etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "non nar URL", + body: "StorePath: " + testStorePath + "\nURL: nix-cache-info\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "nar/", + }, + { + name: "nested nar URL", + body: "StorePath: " + testStorePath + "\nURL: nar/dir/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "safe nar object path", + }, + { + name: "missing NarHash", + body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarSize: 1\n", + wantErr: "NarHash", + }, + { + name: "bad NarSize", + body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: huge\n", + wantErr: "NarSize", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(tc.body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), tc.wantErr) { + t.Fatalf("body %q should mention %q", rec.Body.String(), tc.wantErr) + } + }) + } +} + +func TestNixStoreBackendUploadCacheInfoFileExists(t *testing.T) { + staging := t.TempDir() + if _, err := newNixStoreUploadBackend("ssh://host", staging, nil, slog.Default(), nil); err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + data, err := os.ReadFile(filepath.Join(staging, "nix-cache-info")) + if err != nil { + t.Fatalf("nix-cache-info missing: %v", err) + } + if !bytes.Contains(data, []byte("StoreDir: /nix/store")) { + t.Fatalf("unexpected nix-cache-info: %q", string(data)) + } +} + +func TestNixStoreBackendRejectsTraversalNarinfoPath(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/../etc/passwd.narinfo", strings.NewReader(body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("traversal narinfo status: got %d, want 400", rec.Code) + } + + if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "etc", "passwd.narinfo")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("traversal narinfo escaped staging dir: %v", err) + } +} + +func TestNixStoreBackendRejectsNarinfoFilenameHashMismatch(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + mustUploadNar(t, b, "x.nar", "x") + + body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/11111111111111111111111111111111.narinfo", strings.NewReader(body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("mismatched narinfo status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "filename does not match") { + t.Fatalf("unexpected body: %q", rec.Body.String()) + } +} + +func TestParseNarinfoAcceptsLargeReferencesLine(t *testing.T) { + var refs []string + for range 12000 { + refs = append(refs, "0123456789abcdfghijklmnpqrsvwxy-ref") + } + + body := strings.Join([]string{ + "StorePath: " + testStorePath, + "URL: nar/x.nar", + "NarHash: sha256:abc", + "NarSize: 1", + "References: " + strings.Join(refs, " "), + "", + }, "\n") + + info, err := parseNarinfo(strings.NewReader(body)) + if err != nil { + t.Fatalf("parseNarinfo failed for large references line: %v", err) + } + if info.StorePath != testStorePath { + t.Fatalf("StorePath: got %q, want %q", info.StorePath, testStorePath) + } +} diff --git a/spindle/engines/microvm/upload_cache_proxy.go b/spindle/engines/microvm/upload_cache_proxy.go --- a/spindle/engines/microvm/upload_cache_proxy.go +++ b/spindle/engines/microvm/upload_cache_proxy.go @@ -4,11 +4,9 @@ "context" "errors" "fmt" - "io" "log/slog" "net" "net/http" - "net/http/httputil" "net/url" "strings" "time" @@ -16,14 +14,20 @@ "github.com/mdlayher/vsock" ) +type UploadCacheBackend interface { + http.Handler + Close() error +} + type UploadCacheProxy struct { port uint32 - ln *vsock.Listener - server *http.Server + ln *vsock.Listener + server *http.Server + backend UploadCacheBackend } -func StartUploadCacheProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []CacheUpstream, logger *slog.Logger) (*UploadCacheProxy, error) { +func StartUploadCacheProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []CacheUpstream, stagingDir string, logger *slog.Logger) (*UploadCacheProxy, error) { if strings.TrimSpace(uploadURL) == "" { return nil, nil } @@ -33,15 +37,9 @@ } logger = logger.With("where", "upload_cache_proxy", "cid", cid, "uploadURL", uploadURL) - target, err := url.Parse(uploadURL) + backend, err := newUploadCacheBackend(uploadURL, readUpstreams, stagingDir, logger) if err != nil { - return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) - } - if target.Scheme != "http" && target.Scheme != "https" { - return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q (must be http or https)", uploadURL, target.Scheme) - } - if target.Host == "" { - return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) + return nil, err } ln, port, err := listenRandomVsockUploadPort(ctx) @@ -50,11 +48,12 @@ } proxy := &UploadCacheProxy{ - port: port, - ln: ln, + port: port, + ln: ln, + backend: backend, } proxy.server = &http.Server{ - Handler: uploadProxyHandler(target, readUpstreams, logger), + Handler: backend, Protocols: cacheProxyProtocols(), ReadHeaderTimeout: 30 * time.Second, } @@ -72,6 +71,39 @@ logger.Info("started upload cache proxy", "port", port, "target", uploadURL, "readUpstreams", len(readUpstreams)) return proxy, nil +} + +func newUploadCacheBackend(uploadURL string, readUpstreams []CacheUpstream, stagingDir string, logger *slog.Logger) (UploadCacheBackend, error) { + if strings.TrimSpace(uploadURL) == "" { + return nil, nil + } + + target, err := url.Parse(uploadURL) + if err != nil { + return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) + } + + switch target.Scheme { + case "http", "https": + if target.Host == "" { + return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) + } + return newHTTPUploadProxyBackend(target, readUpstreams, logger), nil + + case "ssh", "ssh-ng": + return newNixStoreUploadBackend(target.String(), stagingDir, readUpstreams, logger, nil) + + case "": + switch uploadURL { + case "daemon", "local": + return newNixStoreUploadBackend(uploadURL, stagingDir, readUpstreams, logger, nil) + default: + return nil, fmt.Errorf("unsupported upload URL %q", uploadURL) + } + + default: + return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q", uploadURL, target.Scheme) + } } func (p *UploadCacheProxy) Port() uint32 { @@ -97,76 +129,10 @@ closeErr = errors.Join(closeErr, p.ln.Close()) p.ln = nil } + if p.backend != nil { + closeErr = errors.Join(closeErr, p.backend.Close()) + } return closeErr -} - -func uploadProxyHandler(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) http.Handler { - rp := httputil.NewSingleHostReverseProxy(target) - rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) - - origDirector := rp.Director - rp.Director = func(req *http.Request) { - origDirector(req) - // ensure host matches target - req.Host = target.Host - // the transport doesn't turn URL userinfo into basic auth, only - // http.Client does, so do it ourselves - if user := target.User; user != nil { - password, _ := user.Password() - req.SetBasicAuth(user.Username(), password) - } - } - - // before uploading, nix copy asks the destination whether it already has each - // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer - // that check across the upload target *and* the read caches: if any of them - // already serves the path there is no point uploading it (the guest would - // just substitute it from there anyway). - narinfoUpstreams := append([]CacheUpstream{{url: target}}, readUpstreams...) - exists := ¶llelRacingTransport{ - upstreams: narinfoUpstreams, - underlying: proxyTransport, - guardedUnderlying: guardedProxyTransport, - logger: logger, - } - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if isNarinfoExistenceCheck(r) { - serveNarinfoExistence(w, r, exists, logger) - return - } - rp.ServeHTTP(w, r) - }) -} - -func isNarinfoExistenceCheck(r *http.Request) bool { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - return false - } - return strings.HasSuffix(r.URL.Path, ".narinfo") -} - -func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { - probe := r.Clone(r.Context()) - probe.RequestURI = "" - - resp, err := exists.RoundTrip(probe) - if err != nil { - logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) - w.WriteHeader(http.StatusNotFound) - return - } - defer resp.Body.Close() - - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) - } - } - w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { - logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) - } } func listenRandomVsockUploadPort(ctx context.Context) (*vsock.Listener, uint32, error) { diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go --- a/spindle/engines/microvm/vm.go +++ b/spindle/engines/microvm/vm.go @@ -182,15 +182,17 @@ } func (e *Engine) drainNixCache(ctx context.Context, state *workflowState) error { - if state.Agent == nil || e.cfg.NixCache.UploadURL == "" { + if e.cfg.NixCache.UploadURL == "" { return nil } drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) defer cancel() - if _, err := state.Agent.Drain(drainCtx); err != nil { - return fmt.Errorf("drain nix cache: %w", err) + if state.Agent != nil { + if _, err := state.Agent.Drain(drainCtx); err != nil { + return fmt.Errorf("drain guest nix cache uploads: %w", err) + } } return nil }