From fd3ff70851f8e8cc5a2707b77b0d904afafcea2a Mon Sep 17 00:00:00 2001 From: dawn Date: Fri, 17 Jul 2026 11:10:54 +0000 Subject: [PATCH] spindle/engines/microvm: rename nix cache -> substituters Signed-off-by: dawn --- docs/DOCS.md | 6 +++--- cmd/spindle-microvm-run/main_linux.go | 10 +++++----- spindle/engine/manifest.go | 22 ++++++++++++++++------ spindle/engines/microvm/README.md | 2 +- spindle/engines/microvm/engine.go | 82 +++++++++++++++++++++++++++++++++++++++++----------------------------------------- spindle/engines/microvm/engine_test.go | 23 +++++++++++++++++++++++ spindle/engines/microvm/models.go | 12 ++++++------ spindle/engines/microvm/models_test.go | 10 +++++----- spindle/engines/microvm/narinfo.go | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/nixos_toplevel.go | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/nixos_toplevel_cache.go | 105 --------------------------------------------------------------------------------------------------------- spindle/engines/microvm/read_cache_proxy.go | 412 ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/read_cache_proxy_test.go | 181 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/start-test-cache.sh | 86 -------------------------------------------------------------------------------------- spindle/engines/microvm/start-test-substituter.sh | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_proxy.go | 413 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_proxy_test.go | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_upload_http.go | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_upload_nix_store.go | 541 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_upload_nix_store_test.go | 496 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_upload_proxy.go | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/substituter_upload_proxy_test.go | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/test-spindle-microvm.sh | 2 +- spindle/engines/microvm/upload_cache_http.go | 103 ------------------------------------------------------------------------------------------------------- spindle/engines/microvm/upload_cache_narinfo.go | 130 ---------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/upload_cache_nix_store.go | 541 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/upload_cache_nix_store_test.go | 496 ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/upload_cache_proxy.go | 160 ---------------------------------------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/upload_cache_proxy_test.go | 130 ---------------------------------------------------------------------------------------------------------------------------------- spindle/engines/microvm/vm.go | 34 +++++++++++++++++----------------- 30 file(s) changed, 2463 insertion(s)(+), 2429 deletion(s)(-) diff --git a/docs/DOCS.md b/docs/DOCS.md --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -1163,15 +1163,15 @@ myflake: github:me/x ``` -#### Caches +#### Substituters -The `caches` field is a map of Nix binary cache URL to its +The `substituters` field is a map of Nix binary cache URL to its trusted public key. These are fed into the spindle's read proxy, so the guest can substitute prebuilt paths from them instead of building everything from scratch. ```yaml -caches: +substituters: https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" ``` diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go --- a/cmd/spindle-microvm-run/main_linux.go +++ b/cmd/spindle-microvm-run/main_linux.go @@ -190,25 +190,25 @@ } defer conn.Close() - upstreams, err := microvm.BuildCacheUpstreams(cmd.StringSlice("cache-read-url"), nil) + upstreams, err := microvm.BuildSubstituterUpstreams(cmd.StringSlice("cache-read-url"), nil) if err != nil { return fmt.Errorf("build cache upstreams: %w", err) } - var readCache *microvm.ReadCacheProxy + var readCache *microvm.SubstituterProxy if len(cmd.StringSlice("cache-read-url")) > 0 { var err error - readCache, err = microvm.StartReadCacheProxy(ctx, vm.CID(), upstreams, logger) + readCache, err = microvm.StartSubstituterProxy(ctx, vm.CID(), upstreams, logger) if err != nil { return fmt.Errorf("start read cache proxy: %w", err) } defer readCache.Close() } - var uploadCache *microvm.UploadCacheProxy + var uploadCache *microvm.SubstituterUploadProxy if cmd.String("cache-upload-url") != "" { var err error - uploadCache, err = microvm.StartUploadCacheProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, filepath.Join(vm.WorkDir(), "upload-cache"), logger) + uploadCache, err = microvm.StartSubstituterUploadProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, filepath.Join(vm.WorkDir(), "upload-cache"), logger) if err != nil { return fmt.Errorf("start upload cache proxy: %w", err) } diff --git a/spindle/engine/manifest.go b/spindle/engine/manifest.go --- a/spindle/engine/manifest.go +++ b/spindle/engine/manifest.go @@ -78,7 +78,7 @@ // at the field that's actually mis-shaped. // // returns nil when nothing is structurally wrong. -func DescribeManifestError(raw string, schema any) error { +func DescribeManifestError(raw string, schema any, renamed ...map[string]string) error { var doc yaml.Node if err := yaml.Unmarshal([]byte(raw), &doc); err != nil { return nil @@ -86,7 +86,11 @@ if len(doc.Content) == 0 { return nil } - err := checkNode(doc.Content[0], reflect.TypeOf(schema), "", genericWorkflowKeys) + var renames map[string]string + if len(renamed) > 0 { + renames = renamed[0] + } + err := checkNode(doc.Content[0], reflect.TypeOf(schema), "", genericWorkflowKeys, renames) var me *manifestError if !errors.As(err, &me) { return err // nil @@ -100,7 +104,8 @@ // checkNode walks a yaml node against the type it's expected to decode into, // recursing through structs, maps and slices. allowExtra names keys that are // valid at this level despite not being in the struct (only the root uses it). -func checkNode(node *yaml.Node, t reflect.Type, path string, allowExtra map[string]bool) error { +// renamed maps removed root-level keys to their new names, for better errors. +func checkNode(node *yaml.Node, t reflect.Type, path string, allowExtra map[string]bool, renamed map[string]string) error { if node.Kind == yaml.AliasNode && node.Alias != nil { node = node.Alias } @@ -141,24 +146,29 @@ if allowExtra[key.Value] { continue } + if newName, wasRenamed := renamed[key.Value]; wasRenamed && path == "" { + return &manifestError{line: key.Line, msg: fmt.Sprintf( + "field %s was renamed to %s (line %d)", + describePath(joinKey(path, key.Value)), newName, key.Line)} + } return &manifestError{line: key.Line, msg: fmt.Sprintf( "unknown field %s (line %d)", describePath(joinKey(path, key.Value)), key.Line)} } - if err := checkNode(val, ft, joinKey(path, key.Value), nil); err != nil { + if err := checkNode(val, ft, joinKey(path, key.Value), nil, nil); err != nil { return err } } case reflect.Map: for i := 0; i+1 < len(node.Content); i += 2 { key, val := node.Content[i], node.Content[i+1] - if err := checkNode(val, t.Elem(), joinKey(path, key.Value), nil); err != nil { + if err := checkNode(val, t.Elem(), joinKey(path, key.Value), nil, nil); err != nil { return err } } case reflect.Slice, reflect.Array: for idx, val := range node.Content { - if err := checkNode(val, t.Elem(), fmt.Sprintf("%s[%d]", path, idx), nil); err != nil { + if err := checkNode(val, t.Elem(), fmt.Sprintf("%s[%d]", path, idx), nil, nil); err != nil { return err } } diff --git a/spindle/engines/microvm/README.md b/spindle/engines/microvm/README.md --- a/spindle/engines/microvm/README.md +++ b/spindle/engines/microvm/README.md @@ -14,7 +14,7 @@ Currently two kinds of images are supported: - NixOS images: these allow configuration such as `dependencies`, `services`, - `virtualisation`, `registry`, `caches` in the workflow file itself. The guest + `virtualisation`, `registry`, `substituters` in the workflow file itself. The guest agent will build (or if it's cached, spindle will send the store path for realization) and activate it before any workflow steps are ran. - Non-NixOS: this is just Alpine & AlmaLinux for now, but can be anything else. diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -32,15 +32,15 @@ ) const ( - guestWorkDir = "/workspace/repo" - guestBasePATH = "/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - guestDevShellEnvPath = "/run/spindle/devshell-env.sh" - activationStepAction = "activate-config" - agentAcceptTimeout = 2 * time.Minute - agentHandshakeTimeout = 30 * time.Second - cacheDrainTimeout = 5 * time.Minute - vmShutdownTimeout = 10 * time.Second - guestTimeoutGrace = 5 * time.Second + guestWorkDir = "/workspace/repo" + guestBasePATH = "/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + guestDevShellEnvPath = "/run/spindle/devshell-env.sh" + activationStepAction = "activate-config" + agentAcceptTimeout = 2 * time.Minute + agentHandshakeTimeout = 30 * time.Second + substituterDrainTimeout = 5 * time.Minute + vmShutdownTimeout = 10 * time.Second + guestTimeoutGrace = 5 * time.Second ) type cleanupFunc func(context.Context) error @@ -124,7 +124,7 @@ swf := &models.Workflow{} var dwf manifestWorkflow - if err := engine.DescribeManifestError(twf.Raw, manifestWorkflow{}); err != nil { + if err := engine.DescribeManifestError(twf.Raw, manifestWorkflow{}, map[string]string{"caches": "substituters"}); err != nil { return nil, err } if err := yaml.Unmarshal([]byte(twf.Raw), &dwf); err != nil { @@ -196,20 +196,20 @@ swf.Steps[insertAt] = activationStep } - cacheURLs, cacheKeys, err := workflowCaches(dwf.Caches) + substituterURLs, substituterKeys, err := workflowSubstituters(dwf.Substituters) if err != nil { return nil, err } swf.Data = &workflowState{ - ImageSpec: imageSpec, - ImageSpecPath: imageSpecPath, - Config: config, - ConfigKey: configKey, - Image: imageName, - CacheReadURLs: cacheURLs, - CacheTrustedPublicKeys: cacheKeys, - NixOSToplevelCache: newNixOSToplevelCacheStore(e.db), + ImageSpec: imageSpec, + ImageSpecPath: imageSpecPath, + Config: config, + ConfigKey: configKey, + Image: imageName, + SubstituterReadURLs: substituterURLs, + SubstituterTrustedPublicKeys: substituterKeys, + NixOSToplevels: newNixOSToplevelStore(e.db), } return swf, nil } @@ -270,21 +270,21 @@ } }() - upstreams, err := BuildCacheUpstreams(e.cfg.NixCache.ReadURLs, state.CacheReadURLs) + upstreams, err := BuildSubstituterUpstreams(e.cfg.NixCache.ReadURLs, state.SubstituterReadURLs) if err != nil { return err } - readCache, err := StartReadCacheProxy(ctx, cid, upstreams, l) + substituter, err := StartSubstituterProxy(ctx, cid, upstreams, l) if err != nil { return err } - state.ReadCache = readCache - stagingDir := filepath.Join(workDir, "upload-cache") - uploadCache, err := StartUploadCacheProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, stagingDir, l) + state.Substituter = substituter + stagingDir := filepath.Join(workDir, "substituter-upload") + substituterUpload, err := StartSubstituterUploadProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, stagingDir, l) if err != nil { return err } - state.UploadCache = uploadCache + state.SubstituterUpload = substituterUpload dnsProxy, err := StartDNSProxy(ctx, cid, l) if err != nil { return err @@ -328,9 +328,9 @@ defer cancelInit() if err := agentSession.Init(initCtx, &agentv1.Init{ JobId: wid.String(), - CacheTrustedPublicKeys: append(slices.Clone(e.cfg.NixCache.TrustedPublicKeys), state.CacheTrustedPublicKeys...), - CacheReadProxyPort: readCache.Port(), - CacheUploadProxyPort: uploadCache.Port(), + CacheTrustedPublicKeys: append(slices.Clone(e.cfg.NixCache.TrustedPublicKeys), state.SubstituterTrustedPublicKeys...), + CacheReadProxyPort: substituter.Port(), + CacheUploadProxyPort: substituterUpload.Port(), DnsProxyPort: dnsProxy.Port(), }); err != nil { _ = agentSession.Close() @@ -486,15 +486,15 @@ var cachedToplevel string if configKey != "" { - if record, ok, err := state.NixOSToplevelCache.Lookup(configKey); err != nil { + if record, ok, err := state.NixOSToplevels.Lookup(configKey); err != nil { return err } else if ok { // todo(dawn): we should probably use gc roots to eliminate TOCTOU // the spindle will have to manage the gc roots, and for remote we have to // ssh in to the host and add / remove gc root. - // we need to have this check anyway since the only check http caches can + // we need to have this check anyway since the only check http substituters can // use is this one, since we cant manage gc roots there... - if e.anyCacheHasPath(ctx, state, record.Toplevel) { + if e.anySubstituterHasPath(ctx, state, record.Toplevel) { cachedToplevel = record.Toplevel fmt.Fprintf(out, "realizing cached NixOS config %s\n", cachedToplevel) } @@ -524,7 +524,7 @@ return nil } if e.cfg.NixCache.UploadURL == "" { - e.l.Warn("not committing config cache metadata: no upload URL configured", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel) + e.l.Warn("not committing toplevel metadata: no upload URL configured", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel) return nil } @@ -532,20 +532,20 @@ // a partial upload would leave the cache unable to realize this toplevel, // so skip the metadata commit rather than poison it with an un-realizable // key. the config still activated fine, so don't fail the workflow. - e.l.Warn("cache drain failed; skipping config cache metadata commit", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel, "error", err) + e.l.Warn("substituter upload drain failed; skipping toplevel metadata commit", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel, "error", err) return nil } - if err := state.NixOSToplevelCache.Commit(configKey, result.Toplevel); err != nil { + if err := state.NixOSToplevels.Commit(configKey, result.Toplevel); err != nil { return err } - fmt.Fprintf(out, "committed config cache metadata %s -> %s\n", configKey, result.Toplevel) + fmt.Fprintf(out, "committed toplevel metadata %s -> %s\n", configKey, result.Toplevel) return nil } -func (e *Engine) anyCacheHasPath(ctx context.Context, state *workflowState, storePath string) bool { - upstreams, err := BuildCacheUpstreams(e.cfg.NixCache.ReadURLs, state.CacheReadURLs) +func (e *Engine) anySubstituterHasPath(ctx context.Context, state *workflowState, storePath string) bool { + upstreams, err := BuildSubstituterUpstreams(e.cfg.NixCache.ReadURLs, state.SubstituterReadURLs) if err != nil { - e.l.Warn("config cache check: build upstreams failed; treating as absent", "path", storePath, "error", err) + e.l.Warn("toplevel check: build upstreams failed; treating as absent", "path", storePath, "error", err) return false } if len(upstreams) == 0 { @@ -553,17 +553,17 @@ } hash, _, err := parseStorePath(storePath) if err != nil { - e.l.Warn("config cache check: invalid toplevel path; treating as absent", "path", storePath, "error", err) + e.l.Warn("toplevel check: invalid toplevel path; treating as absent", "path", storePath, "error", err) return false } req, err := http.NewRequestWithContext(ctx, http.MethodHead, "http://upstream/"+hash+".narinfo", nil) if err != nil { - e.l.Warn("config cache check: build request failed; treating as absent", "path", storePath, "error", err) + e.l.Warn("toplevel check: build request failed; treating as absent", "path", storePath, "error", err) return false } resp, err := newNarinfoExistenceTransport(upstreams, e.l).RoundTrip(req) if err != nil { - e.l.Warn("config cache check: narinfo probe failed; treating as absent", "path", storePath, "error", err) + e.l.Warn("toplevel check: narinfo probe failed; treating as absent", "path", storePath, "error", err) return false } defer resp.Body.Close() diff --git a/spindle/engines/microvm/engine_test.go b/spindle/engines/microvm/engine_test.go --- a/spindle/engines/microvm/engine_test.go +++ b/spindle/engines/microvm/engine_test.go @@ -127,3 +127,26 @@ t.Fatalf("first step should be the activation step, got %+v", wf.Steps[0]) } } + +func TestInitWorkflowCachesRenameHint(t *testing.T) { + dir := t.TempDir() + writeTestImageSpec(t, dir, "alpine", validImageSpec()) + + e := testEngine(t, dir) + _, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: alpine +caches: + https://cache.nixos.org: "" +steps: + - name: hello + command: hello +`, + }, tangled.Pipeline{}) + if err == nil { + t.Fatal("expected an error for the old caches key") + } + if !strings.Contains(err.Error(), "renamed to substituters") { + t.Fatalf("error should point at substituters, got: %v", err) + } +} diff --git a/spindle/engines/microvm/models.go b/spindle/engines/microvm/models.go --- a/spindle/engines/microvm/models.go +++ b/spindle/engines/microvm/models.go @@ -14,7 +14,7 @@ Dependencies []string `yaml:"dependencies"` Registry map[string]any `yaml:"registry"` Environment map[string]string `yaml:"environment"` - Caches map[string]string `yaml:"caches"` + Substituters map[string]string `yaml:"substituters"` Steps []struct { Name string `yaml:"name"` Command string `yaml:"command"` @@ -22,16 +22,16 @@ } `yaml:"steps"` } -// flattens the caches map into sorted substituter URLs and trusted public keys -func workflowCaches(caches map[string]string) (urls []string, keys []string, err error) { - for cacheURL, key := range caches { +// sorted so the guest env is deterministic +func workflowSubstituters(substituters map[string]string) (urls []string, keys []string, err error) { + for cacheURL, key := range substituters { urls = append(urls, cacheURL) if key != "" { keys = append(keys, key) } } - if _, err := parseCacheUpstreams(urls); err != nil { - return nil, nil, fmt.Errorf("caches: %w", err) + if _, err := parseSubstituterUpstreams(urls); err != nil { + return nil, nil, fmt.Errorf("substituters: %w", err) } slices.Sort(urls) slices.Sort(keys) diff --git a/spindle/engines/microvm/models_test.go b/spindle/engines/microvm/models_test.go --- a/spindle/engines/microvm/models_test.go +++ b/spindle/engines/microvm/models_test.go @@ -7,8 +7,8 @@ "testing" ) -func TestWorkflowCaches(t *testing.T) { - urls, keys, err := workflowCaches(map[string]string{ +func TestWorkflowSubstituters(t *testing.T) { + urls, keys, err := workflowSubstituters(map[string]string{ "https://hydra.nixos.org/": "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=", "https://cache.garnix.io/": "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=", "https://unsigned.example/": "", @@ -34,10 +34,10 @@ } } -func TestWorkflowCachesRejectsBadURLs(t *testing.T) { +func TestWorkflowSubstitutersRejectsBadURLs(t *testing.T) { for _, bad := range []string{"ftp://cache.example/", "not a url"} { - if _, _, err := workflowCaches(map[string]string{bad: ""}); err == nil { - t.Errorf("workflowCaches(%q): expected error, got nil", bad) + if _, _, err := workflowSubstituters(map[string]string{bad: ""}); err == nil { + t.Errorf("workflowSubstituters(%q): expected error, got nil", bad) } } } diff --git a/spindle/engines/microvm/narinfo.go b/spindle/engines/microvm/narinfo.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/narinfo.go @@ -0,0 +1,130 @@ +//go:build linux + +package microvm + +import ( + "bufio" + "fmt" + "io" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +type narinfo struct { + StorePath string + URL string + NarHash string + NarSize int64 + // paths this path depends on + References []string +} + +const ( + maxNarinfoSize = 1 << 20 // 1 MiB + storePrefix = "/nix/store/" + maxNarinfoLineLen = maxNarinfoSize +) + +var nixStorePathBaseRe = regexp.MustCompile(`^[0-9abcdfghijklmnpqrsvwxyz]{32}-[^/]+$`) + +// parseNarinfo parses and validates a narinfo body. +// - required fields must be present +// - StorePath must be under /nix/store/ +// - URL must be a relative, traversal-safe path referencing a NAR in the +// same staging cache +// - NarSize must be a non-negative integer +func parseNarinfo(r io.Reader) (*narinfo, error) { + lr := io.LimitReader(r, maxNarinfoSize+1) + scanner := bufio.NewScanner(lr) + scanner.Buffer(make([]byte, 4096), maxNarinfoLineLen) + + var info narinfo + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + return nil, fmt.Errorf("invalid narinfo line %q", line) + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + + switch key { + case "StorePath": + info.StorePath = value + case "URL": + info.URL = value + case "NarHash": + info.NarHash = value + case "NarSize": + n, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid NarSize %q: %w", value, err) + } + info.NarSize = n + case "References": + info.References = strings.Fields(value) + } + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("read narinfo: %w", err) + } + + if err := validateNarinfo(&info); err != nil { + return nil, err + } + return &info, nil +} + +func validateNarinfo(info *narinfo) error { + if info.StorePath == "" { + return fmt.Errorf("narinfo missing StorePath") + } + if _, _, err := parseStorePath(info.StorePath); err != nil { + return fmt.Errorf("invalid StorePath: %w", err) + } + if info.URL == "" { + return fmt.Errorf("narinfo missing URL") + } + if strings.HasPrefix(info.URL, "/") || strings.Contains(info.URL, "..") { + return fmt.Errorf("narinfo URL %q is not a safe relative path", info.URL) + } + if !strings.HasPrefix(info.URL, "nar/") { + return fmt.Errorf("narinfo URL %q must reference a staged nar/ object", info.URL) + } + name := strings.TrimPrefix(info.URL, "nar/") + if name == "" || name == "." || name != filepath.Base(name) || strings.Contains(name, "/") { + return fmt.Errorf("narinfo URL %q is not a safe nar object path", info.URL) + } + if info.NarHash == "" { + return fmt.Errorf("narinfo missing NarHash") + } + if info.NarSize < 0 { + return fmt.Errorf("narinfo NarSize must be non-negative") + } + return nil +} + +func parseStorePath(path string) (hash string, name string, err error) { + if !strings.HasPrefix(path, storePrefix) { + return "", "", fmt.Errorf("store path %q does not start with %q", path, storePrefix) + } + + base := strings.TrimPrefix(path, storePrefix) + if base == "" || strings.Contains(base, "/") { + return "", "", fmt.Errorf("store path %q has invalid base name", path) + } + if !nixStorePathBaseRe.MatchString(base) { + return "", "", fmt.Errorf("store path %q is not a valid nix store path", path) + } + + hash, name, ok := strings.Cut(base, "-") + if !ok || hash == "" || name == "" { + return "", "", fmt.Errorf("store path %q is missing hash or name", path) + } + return hash, name, nil +} diff --git a/spindle/engines/microvm/nixos_toplevel.go b/spindle/engines/microvm/nixos_toplevel.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/nixos_toplevel.go @@ -0,0 +1,105 @@ +//go:build linux + +package microvm + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "tangled.org/core/spindle/db" +) + +// v2 keys are scoped by repo did in v1 keys were global +const nixosToplevelSchemaVersion = 2 + +type nixosToplevelRecord struct { + ConfigKey string `json:"config_key"` + Toplevel string `json:"toplevel"` + UpdatedAt time.Time `json:"updated_at"` +} + +type nixosToplevelStore struct { + db *db.DB +} + +func newNixOSToplevelStore(d *db.DB) nixosToplevelStore { + return nixosToplevelStore{db: d} +} + +func (s nixosToplevelStore) Lookup(configKey string) (nixosToplevelRecord, bool, error) { + if s.db == nil { + return nixosToplevelRecord{}, false, nil + } + r, err := s.db.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nixosToplevelRecord{}, false, nil + } + return nixosToplevelRecord{}, false, err + } + return nixosToplevelRecord{ + ConfigKey: r.ConfigKey, + Toplevel: r.Toplevel, + UpdatedAt: r.UpdatedAt, + }, true, nil +} + +func (s nixosToplevelStore) Commit(configKey, toplevel string) error { + if configKey == "" { + return fmt.Errorf("config key is empty") + } + if toplevel == "" { + return fmt.Errorf("config toplevel is empty") + } + if s.db == nil { + return nil + } + return s.db.SaveNixOSToplevelCacheRecord(configKey, toplevel) +} + +func BaseConfigHash(imageSpec ImageSpec) (string, error) { + if imageSpec.BaseConfigHash == "" { + return "", fmt.Errorf("microvm image spec missing baseConfigHash") + } + return imageSpec.BaseConfigHash, nil +} + +func userConfigHash(cfg manifestConfig) string { + data, _ := json.Marshal(cfg) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func buildConfigKey(imageSpec ImageSpec, cfg manifestConfig, repoDid string) (string, error) { + baseHash, err := BaseConfigHash(imageSpec) + if err != nil { + return "", err + } + payload := struct { + Schema int `json:"schema"` + BaseConfig string `json:"base_config"` + UserConfig string `json:"user_config"` + RepoDid string `json:"repo_did"` + }{ + Schema: nixosToplevelSchemaVersion, + BaseConfig: baseHash, + UserConfig: userConfigHash(cfg), + RepoDid: repoDid, + } + data, _ := json.Marshal(payload) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func BuildConfigKey(imageSpec ImageSpec, userConfigJSON string, repoDid string) (string, error) { + var cfg manifestConfig + if err := json.Unmarshal([]byte(userConfigJSON), &cfg); err != nil { + return "", err + } + return buildConfigKey(imageSpec, cfg, repoDid) +} diff --git a/spindle/engines/microvm/nixos_toplevel_cache.go b/spindle/engines/microvm/nixos_toplevel_cache.go deleted file mode 100644 --- a/spindle/engines/microvm/nixos_toplevel_cache.go +++ /dev/null @@ -1,105 +0,0 @@ -//go:build linux - -package microvm - -import ( - "crypto/sha256" - "database/sql" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "time" - - "tangled.org/core/spindle/db" -) - -// v2 keys are scoped by repo did in v1 keys were global -const nixosToplevelCacheSchemaVersion = 2 - -type nixosToplevelCacheRecord struct { - ConfigKey string `json:"config_key"` - Toplevel string `json:"toplevel"` - UpdatedAt time.Time `json:"updated_at"` -} - -type nixosToplevelCacheStore struct { - db *db.DB -} - -func newNixOSToplevelCacheStore(d *db.DB) nixosToplevelCacheStore { - return nixosToplevelCacheStore{db: d} -} - -func (s nixosToplevelCacheStore) Lookup(configKey string) (nixosToplevelCacheRecord, bool, error) { - if s.db == nil { - return nixosToplevelCacheRecord{}, false, nil - } - r, err := s.db.GetNixOSToplevelCacheRecord(configKey) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nixosToplevelCacheRecord{}, false, nil - } - return nixosToplevelCacheRecord{}, false, err - } - return nixosToplevelCacheRecord{ - ConfigKey: r.ConfigKey, - Toplevel: r.Toplevel, - UpdatedAt: r.UpdatedAt, - }, true, nil -} - -func (s nixosToplevelCacheStore) Commit(configKey, toplevel string) error { - if configKey == "" { - return fmt.Errorf("config key is empty") - } - if toplevel == "" { - return fmt.Errorf("config toplevel is empty") - } - if s.db == nil { - return nil - } - return s.db.SaveNixOSToplevelCacheRecord(configKey, toplevel) -} - -func BaseConfigHash(imageSpec ImageSpec) (string, error) { - if imageSpec.BaseConfigHash == "" { - return "", fmt.Errorf("microvm image spec missing baseConfigHash") - } - return imageSpec.BaseConfigHash, nil -} - -func userConfigHash(cfg manifestConfig) string { - data, _ := json.Marshal(cfg) - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]) -} - -func buildConfigKey(imageSpec ImageSpec, cfg manifestConfig, repoDid string) (string, error) { - baseHash, err := BaseConfigHash(imageSpec) - if err != nil { - return "", err - } - payload := struct { - Schema int `json:"schema"` - BaseConfig string `json:"base_config"` - UserConfig string `json:"user_config"` - RepoDid string `json:"repo_did"` - }{ - Schema: nixosToplevelCacheSchemaVersion, - BaseConfig: baseHash, - UserConfig: userConfigHash(cfg), - RepoDid: repoDid, - } - data, _ := json.Marshal(payload) - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]), nil -} - -func BuildConfigKey(imageSpec ImageSpec, userConfigJSON string, repoDid string) (string, error) { - var cfg manifestConfig - if err := json.Unmarshal([]byte(userConfigJSON), &cfg); err != nil { - return "", err - } - return buildConfigKey(imageSpec, cfg, repoDid) -} diff --git a/spindle/engines/microvm/read_cache_proxy.go b/spindle/engines/microvm/read_cache_proxy.go deleted file mode 100644 --- a/spindle/engines/microvm/read_cache_proxy.go +++ /dev/null @@ -1,412 +0,0 @@ -//go:build linux - -package microvm - -import ( - "context" - "crypto/rand" - "encoding/binary" - "errors" - "fmt" - "io" - "log/slog" - "net" - "net/http" - "net/http/httputil" - "net/url" - "strings" - "sync" - "time" - - "github.com/mdlayher/vsock" - - "tangled.org/core/spindle/netguard" -) - -const ( - readCacheProxyPortMin = 20000 - readCacheProxyPortMax = 60000 -) - -type ReadCacheProxy struct { - port uint32 - - ln *vsock.Listener - server *http.Server -} - -func StartReadCacheProxy(ctx context.Context, cid uint32, upstreams []CacheUpstream, logger *slog.Logger) (*ReadCacheProxy, error) { - if logger == nil { - logger = slog.Default() - } - logger = logger.With("where", "read_cache", "cid", cid) - - if len(upstreams) == 0 { - return nil, nil - } - - ln, port, err := listenRandomVsockPort(ctx) - if err != nil { - return nil, err - } - - proxy := &ReadCacheProxy{ - port: port, - ln: ln, - } - proxy.server = &http.Server{ - Handler: cacheProxyHandler(upstreams, logger), - Protocols: cacheProxyProtocols(), - ReadHeaderTimeout: 10 * time.Second, - } - - filtered := &cidFilteredVsockListener{ - Listener: ln, - cid: cid, - logger: logger, - } - go func() { - if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { - logger.Warn("proxy stopped", "cid", cid, "port", port, "error", err) - } - }() - - logger.Info("started proxy", "cid", cid, "port", port, "upstreams", len(upstreams)) - return proxy, nil -} - -func (p *ReadCacheProxy) Port() uint32 { - if p == nil { - return 0 - } - return p.port -} - -func (p *ReadCacheProxy) Close() error { - if p == nil { - return nil - } - - var closeErr error - if p.server != nil { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) - cancel() - p.server = nil - } - if p.ln != nil { - closeErr = errors.Join(closeErr, p.ln.Close()) - p.ln = nil - } - return closeErr -} - -type cidFilteredVsockListener struct { - *vsock.Listener - cid uint32 - logger *slog.Logger -} - -func (l *cidFilteredVsockListener) Accept() (net.Conn, error) { - for { - conn, err := l.Listener.Accept() - if err != nil { - return nil, err - } - - addr, ok := conn.RemoteAddr().(*vsock.Addr) - if ok && addr.ContextID == l.cid { - return conn, nil - } - - l.logger.Warn("dropping proxy connection from unexpected cid", "remote", conn.RemoteAddr(), "expectedCID", l.cid) - _ = conn.Close() - } -} - -func parseCacheUpstreams(raw []string) ([]*url.URL, error) { - upstreams := make([]*url.URL, 0, len(raw)) - seen := make(map[string]struct{}, len(raw)) - for _, value := range raw { - value = strings.TrimSpace(value) - if value == "" { - continue - } - if _, ok := seen[value]; ok { - continue - } - seen[value] = struct{}{} - - parsed, err := url.Parse(value) - if err != nil { - return nil, fmt.Errorf("parse URL %q: %w", value, err) - } - if parsed.Scheme != "http" && parsed.Scheme != "https" { - return nil, fmt.Errorf("URL %q uses unsupported scheme %q", value, parsed.Scheme) - } - if parsed.Host == "" { - return nil, fmt.Errorf("URL %q is missing host", value) - } - upstreams = append(upstreams, parsed) - } - return upstreams, nil -} - -type CacheUpstream struct { - url *url.URL - // guarded upstreams come from the workflow file - // requests to them are refused for special-purpose address ranges - guarded bool -} - -func BuildCacheUpstreams(rawTrusted, rawGuarded []string) ([]CacheUpstream, error) { - trusted, err := parseCacheUpstreams(rawTrusted) - if err != nil { - return nil, err - } - guarded, err := parseCacheUpstreams(rawGuarded) - if err != nil { - return nil, err - } - return mergeCacheUpstreams(trusted, guarded), nil -} - -func mergeCacheUpstreams(trusted, guarded []*url.URL) []CacheUpstream { - merged := make([]CacheUpstream, 0, len(trusted)+len(guarded)) - seen := make(map[string]struct{}, len(trusted)+len(guarded)) - for _, u := range trusted { - if _, ok := seen[u.String()]; ok { - continue - } - seen[u.String()] = struct{}{} - merged = append(merged, CacheUpstream{url: u}) - } - for _, u := range guarded { - if _, ok := seen[u.String()]; ok { - continue - } - seen[u.String()] = struct{}{} - merged = append(merged, CacheUpstream{url: u, guarded: true}) - } - return merged -} - -func listenRandomVsockPort(ctx context.Context) (*vsock.Listener, uint32, error) { - var lastErr error - for range 32 { - port, err := randomVsockPort() - if err != nil { - return nil, 0, err - } - ln, err := vsock.ListenContextID(vsock.Host, port, nil) - if err == nil { - return ln, port, nil - } - lastErr = err - - select { - case <-ctx.Done(): - return nil, 0, ctx.Err() - default: - } - } - return nil, 0, fmt.Errorf("listen on random vsock port: %w", lastErr) -} - -func randomVsockPort() (uint32, error) { - var data [4]byte - if _, err := rand.Read(data[:]); err != nil { - return 0, fmt.Errorf("allocate read vsock port: %w", err) - } - span := uint32(readCacheProxyPortMax - readCacheProxyPortMin) - return readCacheProxyPortMin + binary.BigEndian.Uint32(data[:])%span, nil -} - -var proxyTransport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, -} - -// for guarded upstreams, this will refuse requests made to blocked addresses -var guardedProxyTransport = &http.Transport{ - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - Control: netguard.RefuseSpecialPurposeAddrs, - }).DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, -} - -// the proxy is the cache as far as the guest is concerned, so we answer -// /nix-cache-info ourselves instead of racing the upstreams for it. merging -// those also doesn't make any sense (none of the options make sense for -// merging) -const nixCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n" - -func cacheProxyHandler(upstreams []CacheUpstream, logger *slog.Logger) http.Handler { - proxy := &httputil.ReverseProxy{ - // nothing to do here: the racing transport builds the full URL per - // upstream, it just needs the guest's path/query left intact - Rewrite: func(*httputil.ProxyRequest) {}, - ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), - Transport: ¶llelRacingTransport{ - upstreams: upstreams, - underlying: proxyTransport, - guardedUnderlying: guardedProxyTransport, - logger: logger, - }, - } - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/nix-cache-info" { - w.Header().Set("Content-Type", "text/x-nix-cache-info") - _, _ = io.WriteString(w, nixCacheInfo) - return - } - proxy.ServeHTTP(w, r) - }) -} - -func cacheProxyProtocols() *http.Protocols { - protocols := new(http.Protocols) - protocols.SetHTTP1(true) - protocols.SetUnencryptedHTTP2(true) - return protocols -} - -func mergeQuery(base, extra string) string { - switch { - case base == "": - return extra - case extra == "": - return base - default: - return base + "&" + extra - } -} - -type parallelRacingTransport struct { - upstreams []CacheUpstream - underlying http.RoundTripper - guardedUnderlying http.RoundTripper - logger *slog.Logger -} - -func (t *parallelRacingTransport) RoundTrip(req *http.Request) (*http.Response, error) { - type result struct { - resp *http.Response - err error - is404 bool - idx int - } - - resCh := make(chan result, len(t.upstreams)) - cancels := make([]context.CancelFunc, len(t.upstreams)) - var wg sync.WaitGroup - - for i, upstream := range t.upstreams { - wg.Add(1) - ctx, cancel := context.WithCancel(req.Context()) - cancels[i] = cancel - - go func(idx int, target CacheUpstream, uCtx context.Context) { - defer wg.Done() - - raceReq := req.Clone(uCtx) - // rewrite to the target, joining the upstream's base path/query - // with what the guest asked for - raceReq.URL.Scheme = target.url.Scheme - raceReq.URL.Host = target.url.Host - raceReq.URL.Path = strings.TrimSuffix(target.url.Path, "/") + req.URL.Path - raceReq.URL.RawQuery = mergeQuery(target.url.RawQuery, req.URL.RawQuery) - // Host wins over URL.Host for the outgoing Host header, and the - // reverse proxy preserves the guest's (127.0.0.1:), which - // host-routed upstreams like fastly reject with a 421 - raceReq.Host = target.url.Host - // the transport doesn't turn URL userinfo into basic auth, only - // http.Client does, so do it ourselves - if user := target.url.User; user != nil { - password, _ := user.Password() - raceReq.SetBasicAuth(user.Username(), password) - } - - rt := t.underlying - if target.guarded { - rt = t.guardedUnderlying - } - resp, err := rt.RoundTrip(raceReq) - if err != nil { - resCh <- result{err: err, idx: idx} - return - } - if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() // don't care about the body of a 404 - resCh <- result{is404: true, idx: idx} - return - } - if resp.StatusCode >= 400 { - // an erroring upstream must not win over a healthy one - _ = resp.Body.Close() - resCh <- result{err: fmt.Errorf("upstream returned status %d", resp.StatusCode), idx: idx} - return - } - // yay, ok - resCh <- result{resp: resp, idx: idx} - }(i, upstream, ctx) - } - - go func() { - wg.Wait() - close(resCh) - }() - - var total404s int - for res := range resCh { - if res.is404 { - total404s++ - if total404s == len(t.upstreams) { - for _, cancel := range cancels { - cancel() - } - return &http.Response{ - StatusCode: http.StatusNotFound, - Body: io.NopCloser(strings.NewReader("404 nix path not found")), - Header: make(http.Header), - Request: req, - }, nil - } - continue - } - - if res.err != nil { - if !errors.Is(res.err, context.Canceled) { - t.logger.Warn("upstream failed", - "path", req.URL.Path, - "error", res.err, - ) - } - continue - } - - // cancel other requests - for i, cancel := range cancels { - if i != res.idx { - cancel() - } - } - return res.resp, nil - } - - for _, cancel := range cancels { - cancel() - } - return nil, errors.New("all upstreams failed or timed out") -} diff --git a/spindle/engines/microvm/read_cache_proxy_test.go b/spindle/engines/microvm/read_cache_proxy_test.go deleted file mode 100644 --- a/spindle/engines/microvm/read_cache_proxy_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build linux - -package microvm - -import ( - "io" - "log/slog" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "tangled.org/core/spindle/netguard" -) - -func TestCacheProxyFallsBackOnNotFound(t *testing.T) { - first := httptest.NewServer(http.NotFoundHandler()) - defer first.Close() - second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/abc.narinfo" { - t.Fatalf("path: got %q, want /abc.narinfo", req.URL.Path) - } - _, _ = io.WriteString(w, "ok") - })) - defer second.Close() - - upstreams, err := parseCacheUpstreams([]string{first.URL, second.URL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - if got := rec.Body.String(); got != "ok" { - t.Fatalf("body: got %q, want ok", got) - } -} - -func TestCacheProxyServesNixCacheInfoItself(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - t.Errorf("upstream should not be hit, got request for %q", req.URL.Path) - })) - defer upstream.Close() - - upstreams, err := parseCacheUpstreams([]string{upstream.URL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://guest/nix-cache-info", nil) - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - if got := rec.Body.String(); got != nixCacheInfo { - t.Fatalf("body: got %q, want %q", got, nixCacheInfo) - } -} - -func TestCacheProxyErrorStatusDoesNotWinRace(t *testing.T) { - erroring := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - http.Error(w, "misdirected", http.StatusMisdirectedRequest) - })) - defer erroring.Close() - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - time.Sleep(50 * time.Millisecond) // lose the race to the erroring upstream - _, _ = io.WriteString(w, "ok") - })) - defer healthy.Close() - - upstreams, err := parseCacheUpstreams([]string{erroring.URL, healthy.URL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - if got := rec.Body.String(); got != "ok" { - t.Fatalf("body: got %q, want ok", got) - } -} - -func TestCacheProxyJoinsSubpathQueryAndAuth(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/sub/cache/abc.narinfo" { - t.Errorf("path: got %q, want /sub/cache/abc.narinfo", req.URL.Path) - } - if got := req.URL.Query().Get("token"); got != "s3cret" { - t.Errorf("token: got %q, want s3cret", got) - } - if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { - t.Errorf("basic auth: got %q/%q/%v, want dawn/woof/true", user, pass, ok) - } - _, _ = io.WriteString(w, "ok") - })) - defer upstream.Close() - - upstreamURL := "http://dawn:woof@" + strings.TrimPrefix(upstream.URL, "http://") + "/sub/cache/?token=s3cret" - upstreams, err := parseCacheUpstreams([]string{upstreamURL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - if got := rec.Body.String(); got != "ok" { - t.Fatalf("body: got %q, want ok", got) - } -} - -func TestCacheProxyGuardAllowsPublicIPv4(t *testing.T) { - if err := netguard.RefuseSpecialPurposeAddrs("tcp", "104.26.13.82:443", nil); err != nil { - t.Fatalf("public IPv4 address was blocked: %v", err) - } -} - -func TestCacheProxyGuardedUpstreamCannotReachBlockedRanges(t *testing.T) { - // httptest listens on 127.0.0.1, which is in the blocked ranges; reaching - // it would mean a workflow-defined cache can hit the host's loopback - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - t.Errorf("guarded upstream on loopback should not be reachable, got request for %q", req.URL.Path) - })) - defer upstream.Close() - - upstreams, err := parseCacheUpstreams([]string{upstream.URL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(nil, upstreams), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusBadGateway { - t.Fatalf("status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) - } -} - -func TestCacheProxyRewritesHostHeader(t *testing.T) { - var upstreamHost string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Host != upstreamHost { - t.Errorf("host: got %q, want %q", req.Host, upstreamHost) - } - _, _ = io.WriteString(w, "ok") - })) - defer upstream.Close() - upstreamHost = strings.TrimPrefix(upstream.URL, "http://") - - upstreams, err := parseCacheUpstreams([]string{upstream.URL}) - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10500/abc.narinfo", nil) - req.Host = "127.0.0.1:10500" - rec := httptest.NewRecorder() - cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } -} diff --git a/spindle/engines/microvm/start-test-cache.sh b/spindle/engines/microvm/start-test-cache.sh deleted file mode 100644 --- a/spindle/engines/microvm/start-test-cache.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# start a local ncps binary cache -# usage: ./start-test-cache.sh [port] - -if [ "$#" -lt 1 ]; then - echo "Usage: $0 [ncps-port]" - exit 1 -fi - -TEST_DIR="$(mkdir -p "$1" && cd "$1" && pwd)" -PORT="${2:-8501}" - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -SECRET_KEY_PATH="$TEST_DIR/test-cache-key.secret" -PUBLIC_KEY_PATH="$TEST_DIR/test-cache-key.pub" -DB_PATH="$TEST_DIR/ncps.sqlite" -CONFIG_PATH="$TEST_DIR/ncps-config.yaml" -STORAGE_DIR="$TEST_DIR/storage" -ENV_PATH="$TEST_DIR/env.sh" -PID_PATH="$TEST_DIR/ncps.pid" - -mkdir -p "$STORAGE_DIR" - -echo "generating binary cache keys.." -nix-store --generate-binary-cache-key test-cache-key "$SECRET_KEY_PATH" "$PUBLIC_KEY_PATH" -PUBKEY_VAL=$(cat "$PUBLIC_KEY_PATH") - -echo "initializing ncps db..." -nix shell nixpkgs#dbmate --command dbmate \ - --migrations-dir "$(nix build --no-link --print-out-paths nixpkgs#ncps)/share/ncps/db/migrations/sqlite" \ - -u "sqlite:$DB_PATH" \ - up - -echo "writing ncps configuration..." -cat < "$CONFIG_PATH" -cache: - allow-delete-verb: true - allow-put-verb: true - hostname: "cache.local" - database-url: "sqlite:$DB_PATH" - secret-key-path: "$SECRET_KEY_PATH" - sign-narinfo: true - storage: - local: "$STORAGE_DIR" - upstream: - urls: - - https://cache.nixos.org - public-keys: - - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= -server: - addr: "127.0.0.1:$PORT" -EOF - -echo "starting ncps on port $PORT..." -export CACHE_ALLOW_PUT_VERB=true -nix shell nixpkgs#ncps --command ncps serve --config "$CONFIG_PATH" & -NCPS_PID=$! -echo "$NCPS_PID" > "$PID_PATH" - -# wait for connection -for i in {1..30}; do - if curl -s "http://127.0.0.1:$PORT/nix-cache-info" > /dev/null; then - echo "ncps is healthy." - break - fi - sleep 0.5 - if ! kill -0 "$NCPS_PID" 2>/dev/null; then - echo "ncps exited unexpectedly during startup." - exit 1 - fi -done - -cat < "$ENV_PATH" -export CACHE_PUBKEY="$PUBKEY_VAL" -export CACHE_PORT="$PORT" -export CACHE_URL="http://127.0.0.1:$PORT" -export CACHE_UPLOAD_URL="http://127.0.0.1:$PORT/upload" -export CACHE_SECRET_KEY_PATH="$SECRET_KEY_PATH" -export NCPS_PID="$NCPS_PID" -export TEST_DIR="$TEST_DIR" -EOF - -echo "cache server started successfully. source $ENV_PATH to use, and kill PID $NCPS_PID or check $PID_PATH to stop it." diff --git a/spindle/engines/microvm/start-test-substituter.sh b/spindle/engines/microvm/start-test-substituter.sh new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/start-test-substituter.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# start a local ncps binary cache +# usage: ./start-test-cache.sh [port] + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [ncps-port]" + exit 1 +fi + +TEST_DIR="$(mkdir -p "$1" && cd "$1" && pwd)" +PORT="${2:-8501}" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +SECRET_KEY_PATH="$TEST_DIR/test-cache-key.secret" +PUBLIC_KEY_PATH="$TEST_DIR/test-cache-key.pub" +DB_PATH="$TEST_DIR/ncps.sqlite" +CONFIG_PATH="$TEST_DIR/ncps-config.yaml" +STORAGE_DIR="$TEST_DIR/storage" +ENV_PATH="$TEST_DIR/env.sh" +PID_PATH="$TEST_DIR/ncps.pid" + +mkdir -p "$STORAGE_DIR" + +echo "generating binary cache keys.." +nix-store --generate-binary-cache-key test-cache-key "$SECRET_KEY_PATH" "$PUBLIC_KEY_PATH" +PUBKEY_VAL=$(cat "$PUBLIC_KEY_PATH") + +echo "initializing ncps db..." +nix shell nixpkgs#dbmate --command dbmate \ + --migrations-dir "$(nix build --no-link --print-out-paths nixpkgs#ncps)/share/ncps/db/migrations/sqlite" \ + -u "sqlite:$DB_PATH" \ + up + +echo "writing ncps configuration..." +cat < "$CONFIG_PATH" +cache: + allow-delete-verb: true + allow-put-verb: true + hostname: "cache.local" + database-url: "sqlite:$DB_PATH" + secret-key-path: "$SECRET_KEY_PATH" + sign-narinfo: true + storage: + local: "$STORAGE_DIR" + upstream: + urls: + - https://cache.nixos.org + public-keys: + - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +server: + addr: "127.0.0.1:$PORT" +EOF + +echo "starting ncps on port $PORT..." +export CACHE_ALLOW_PUT_VERB=true +nix shell nixpkgs#ncps --command ncps serve --config "$CONFIG_PATH" & +NCPS_PID=$! +echo "$NCPS_PID" > "$PID_PATH" + +# wait for connection +for i in {1..30}; do + if curl -s "http://127.0.0.1:$PORT/nix-cache-info" > /dev/null; then + echo "ncps is healthy." + break + fi + sleep 0.5 + if ! kill -0 "$NCPS_PID" 2>/dev/null; then + echo "ncps exited unexpectedly during startup." + exit 1 + fi +done + +cat < "$ENV_PATH" +export CACHE_PUBKEY="$PUBKEY_VAL" +export CACHE_PORT="$PORT" +export CACHE_URL="http://127.0.0.1:$PORT" +export CACHE_UPLOAD_URL="http://127.0.0.1:$PORT/upload" +export CACHE_SECRET_KEY_PATH="$SECRET_KEY_PATH" +export NCPS_PID="$NCPS_PID" +export TEST_DIR="$TEST_DIR" +EOF + +echo "cache server started successfully. source $ENV_PATH to use, and kill PID $NCPS_PID or check $PID_PATH to stop it." diff --git a/spindle/engines/microvm/substituter_proxy.go b/spindle/engines/microvm/substituter_proxy.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_proxy.go @@ -0,0 +1,413 @@ +//go:build linux + +package microvm + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" + "time" + + "github.com/mdlayher/vsock" + + "tangled.org/core/spindle/netguard" +) + +const ( + substituterProxyPortMin = 20000 + substituterProxyPortMax = 60000 +) + +type SubstituterProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server +} + +func StartSubstituterProxy(ctx context.Context, cid uint32, upstreams []SubstituterUpstream, logger *slog.Logger) (*SubstituterProxy, error) { + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "read_cache", "cid", cid) + + if len(upstreams) == 0 { + return nil, nil + } + + ln, port, err := listenRandomVsockPort(ctx) + if err != nil { + return nil, err + } + + proxy := &SubstituterProxy{ + port: port, + ln: ln, + } + proxy.server = &http.Server{ + Handler: cacheProxyHandler(upstreams, logger), + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 10 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("proxy stopped", "cid", cid, "port", port, "error", err) + } + }() + + logger.Info("started proxy", "cid", cid, "port", port, "upstreams", len(upstreams)) + return proxy, nil +} + +func (p *SubstituterProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *SubstituterProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + return closeErr +} + +type cidFilteredVsockListener struct { + *vsock.Listener + cid uint32 + logger *slog.Logger +} + +func (l *cidFilteredVsockListener) Accept() (net.Conn, error) { + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == l.cid { + return conn, nil + } + + l.logger.Warn("dropping proxy connection from unexpected cid", "remote", conn.RemoteAddr(), "expectedCID", l.cid) + _ = conn.Close() + } +} + +func parseSubstituterUpstreams(raw []string) ([]*url.URL, error) { + upstreams := make([]*url.URL, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, value := range raw { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + + parsed, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("parse URL %q: %w", value, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("URL %q uses unsupported scheme %q", value, parsed.Scheme) + } + if parsed.Host == "" { + return nil, fmt.Errorf("URL %q is missing host", value) + } + upstreams = append(upstreams, parsed) + } + return upstreams, nil +} + +type SubstituterUpstream struct { + url *url.URL + // guarded upstreams come from the workflow file + // requests to them are refused for special-purpose address ranges + guarded bool +} + +func BuildSubstituterUpstreams(rawTrusted, rawGuarded []string) ([]SubstituterUpstream, error) { + trusted, err := parseSubstituterUpstreams(rawTrusted) + if err != nil { + return nil, err + } + guarded, err := parseSubstituterUpstreams(rawGuarded) + if err != nil { + return nil, err + } + return mergeSubstituterUpstreams(trusted, guarded), nil +} + +func mergeSubstituterUpstreams(trusted, guarded []*url.URL) []SubstituterUpstream { + merged := make([]SubstituterUpstream, 0, len(trusted)+len(guarded)) + seen := make(map[string]struct{}, len(trusted)+len(guarded)) + for _, u := range trusted { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, SubstituterUpstream{url: u}) + } + for _, u := range guarded { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, SubstituterUpstream{url: u, guarded: true}) + } + return merged +} + +func listenRandomVsockPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.ListenContextID(vsock.Host, port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock port: %w", lastErr) +} + +func randomVsockPort() (uint32, error) { + var data [4]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, fmt.Errorf("allocate read vsock port: %w", err) + } + span := uint32(substituterProxyPortMax - substituterProxyPortMin) + return substituterProxyPortMin + binary.BigEndian.Uint32(data[:])%span, nil +} + +var proxyTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// for guarded upstreams, this will refuse requests made to blocked addresses +var guardedProxyTransport = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: netguard.RefuseSpecialPurposeAddrs, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + + +// the proxy is the cache as far as the guest is concerned, so we answer +// /nix-cache-info ourselves instead of racing the upstreams for it. merging +// those also doesn't make any sense (none of the options make sense for +// merging) +const nixCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n" + +func cacheProxyHandler(upstreams []SubstituterUpstream, logger *slog.Logger) http.Handler { + proxy := &httputil.ReverseProxy{ + // nothing to do here: the racing transport builds the full URL per + // upstream, it just needs the guest's path/query left intact + Rewrite: func(*httputil.ProxyRequest) {}, + ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), + Transport: ¶llelRacingTransport{ + upstreams: upstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + }, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/nix-cache-info" { + w.Header().Set("Content-Type", "text/x-nix-cache-info") + _, _ = io.WriteString(w, nixCacheInfo) + return + } + proxy.ServeHTTP(w, r) + }) +} + +func cacheProxyProtocols() *http.Protocols { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetUnencryptedHTTP2(true) + return protocols +} + +func mergeQuery(base, extra string) string { + switch { + case base == "": + return extra + case extra == "": + return base + default: + return base + "&" + extra + } +} + +type parallelRacingTransport struct { + upstreams []SubstituterUpstream + underlying http.RoundTripper + guardedUnderlying http.RoundTripper + logger *slog.Logger +} + +func (t *parallelRacingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + type result struct { + resp *http.Response + err error + is404 bool + idx int + } + + resCh := make(chan result, len(t.upstreams)) + cancels := make([]context.CancelFunc, len(t.upstreams)) + var wg sync.WaitGroup + + for i, upstream := range t.upstreams { + wg.Add(1) + ctx, cancel := context.WithCancel(req.Context()) + cancels[i] = cancel + + go func(idx int, target SubstituterUpstream, uCtx context.Context) { + defer wg.Done() + + raceReq := req.Clone(uCtx) + // rewrite to the target, joining the upstream's base path/query + // with what the guest asked for + raceReq.URL.Scheme = target.url.Scheme + raceReq.URL.Host = target.url.Host + raceReq.URL.Path = strings.TrimSuffix(target.url.Path, "/") + req.URL.Path + raceReq.URL.RawQuery = mergeQuery(target.url.RawQuery, req.URL.RawQuery) + // Host wins over URL.Host for the outgoing Host header, and the + // reverse proxy preserves the guest's (127.0.0.1:), which + // host-routed upstreams like fastly reject with a 421 + raceReq.Host = target.url.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.url.User; user != nil { + password, _ := user.Password() + raceReq.SetBasicAuth(user.Username(), password) + } + + rt := t.underlying + if target.guarded { + rt = t.guardedUnderlying + } + resp, err := rt.RoundTrip(raceReq) + if err != nil { + resCh <- result{err: err, idx: idx} + return + } + if resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() // don't care about the body of a 404 + resCh <- result{is404: true, idx: idx} + return + } + if resp.StatusCode >= 400 { + // an erroring upstream must not win over a healthy one + _ = resp.Body.Close() + resCh <- result{err: fmt.Errorf("upstream returned status %d", resp.StatusCode), idx: idx} + return + } + // yay, ok + resCh <- result{resp: resp, idx: idx} + }(i, upstream, ctx) + } + + go func() { + wg.Wait() + close(resCh) + }() + + var total404s int + for res := range resCh { + if res.is404 { + total404s++ + if total404s == len(t.upstreams) { + for _, cancel := range cancels { + cancel() + } + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader("404 nix path not found")), + Header: make(http.Header), + Request: req, + }, nil + } + continue + } + + if res.err != nil { + if !errors.Is(res.err, context.Canceled) { + t.logger.Warn("upstream failed", + "path", req.URL.Path, + "error", res.err, + ) + } + continue + } + + // cancel other requests + for i, cancel := range cancels { + if i != res.idx { + cancel() + } + } + return res.resp, nil + } + + for _, cancel := range cancels { + cancel() + } + return nil, errors.New("all upstreams failed or timed out") +} diff --git a/spindle/engines/microvm/substituter_proxy_test.go b/spindle/engines/microvm/substituter_proxy_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_proxy_test.go @@ -0,0 +1,181 @@ +//go:build linux + +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "tangled.org/core/spindle/netguard" +) + +func TestCacheProxyFallsBackOnNotFound(t *testing.T) { + first := httptest.NewServer(http.NotFoundHandler()) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Fatalf("path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "ok") + })) + defer second.Close() + + upstreams, err := parseSubstituterUpstreams([]string{first.URL, second.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyServesNixCacheInfoItself(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("upstream should not be hit, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseSubstituterUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/nix-cache-info", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != nixCacheInfo { + t.Fatalf("body: got %q, want %q", got, nixCacheInfo) + } +} + +func TestCacheProxyErrorStatusDoesNotWinRace(t *testing.T) { + erroring := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Error(w, "misdirected", http.StatusMisdirectedRequest) + })) + defer erroring.Close() + healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + time.Sleep(50 * time.Millisecond) // lose the race to the erroring upstream + _, _ = io.WriteString(w, "ok") + })) + defer healthy.Close() + + upstreams, err := parseSubstituterUpstreams([]string{erroring.URL, healthy.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyJoinsSubpathQueryAndAuth(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/sub/cache/abc.narinfo" { + t.Errorf("path: got %q, want /sub/cache/abc.narinfo", req.URL.Path) + } + if got := req.URL.Query().Get("token"); got != "s3cret" { + t.Errorf("token: got %q, want s3cret", got) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/woof/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + + upstreamURL := "http://dawn:woof@" + strings.TrimPrefix(upstream.URL, "http://") + "/sub/cache/?token=s3cret" + upstreams, err := parseSubstituterUpstreams([]string{upstreamURL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyGuardedUpstreamCannotReachBlockedRanges(t *testing.T) { + // httptest listens on 127.0.0.1, which is in the blocked ranges; reaching + // it would mean a workflow-defined cache can hit the host's loopback + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("guarded upstream on loopback should not be reachable, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseSubstituterUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(nil, upstreams), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestCacheProxyRewritesHostHeader(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + upstreams, err := parseSubstituterUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10500/abc.narinfo", nil) + req.Host = "127.0.0.1:10500" + rec := httptest.NewRecorder() + cacheProxyHandler(mergeSubstituterUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestCacheProxyGuardAllowsPublicIPv4(t *testing.T) { + if err := netguard.RefuseSpecialPurposeAddrs("tcp", "104.26.13.82:443", nil); err != nil { + t.Fatalf("public IPv4 address was blocked: %v", err) + } +} diff --git a/spindle/engines/microvm/substituter_upload_http.go b/spindle/engines/microvm/substituter_upload_http.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_upload_http.go @@ -0,0 +1,103 @@ +//go:build linux + +package microvm + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httputil" + "net/url" + "strings" +) + +// httpUploadBackend reverse-proxies guest binary-cache upload traffic to an +// http(s) upload cache such as ncps. +type httpUploadBackend struct { + handler http.Handler +} + +func newHTTPUploadProxyBackend(target *url.URL, readUpstreams []SubstituterUpstream, logger *slog.Logger) *httpUploadBackend { + return &httpUploadBackend{handler: uploadProxyHandler(target, readUpstreams, logger)} +} + +func (b *httpUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { + b.handler.ServeHTTP(w, r) +} + +func (b *httpUploadBackend) Close() error { return nil } + +func uploadProxyHandler(target *url.URL, readUpstreams []SubstituterUpstream, logger *slog.Logger) http.Handler { + rp := httputil.NewSingleHostReverseProxy(target) + rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) + + origDirector := rp.Director + rp.Director = func(req *http.Request) { + origDirector(req) + // ensure host matches target + req.Host = target.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.User; user != nil { + password, _ := user.Password() + req.SetBasicAuth(user.Username(), password) + } + } + + // before uploading, nix copy asks the destination whether it already has each + // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer + // that check across the upload target *and* the read substituters: if any of them + // already serves the path there is no point uploading it (the guest would + // just substitute it from there anyway). + narinfoUpstreams := append([]SubstituterUpstream{{url: target}}, readUpstreams...) + exists := newNarinfoExistenceTransport(narinfoUpstreams, logger) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isNarinfoExistenceCheck(r) { + serveNarinfoExistence(w, r, exists, logger) + return + } + rp.ServeHTTP(w, r) + }) +} + +func newNarinfoExistenceTransport(upstreams []SubstituterUpstream, logger *slog.Logger) http.RoundTripper { + return ¶llelRacingTransport{ + upstreams: upstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + } +} + +func isNarinfoExistenceCheck(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + return strings.HasSuffix(r.URL.Path, ".narinfo") +} + +func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { + probe := r.Clone(r.Context()) + probe.RequestURI = "" + + resp, err := exists.RoundTrip(probe) + if err != nil { + logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) + w.WriteHeader(http.StatusNotFound) + return + } + defer resp.Body.Close() + + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { + logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) + } +} diff --git a/spindle/engines/microvm/substituter_upload_nix_store.go b/spindle/engines/microvm/substituter_upload_nix_store.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_upload_nix_store.go @@ -0,0 +1,541 @@ +//go:build linux + +package microvm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// we have an interface for running commands so we can swap it in tests +type CommandRunner interface { + Run(ctx context.Context, name string, args ...string) error +} + +type execRunner struct{} + +func (execRunner) Run(ctx context.Context, name string, args ...string) error { + // nosemgrep: go.lang.security.audit.dangerous-exec-command.dangerous-exec-command + cmd := exec.CommandContext(ctx, name, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, string(out)) + } + return nil +} + +const ( + nixStoreCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 50\n" + maxNarUploadSize = 5 << 30 // 5gib +) + +type NixStoreUploadBackend struct { + stagingDir string + targetStore string + readUpstreams []SubstituterUpstream + logger *slog.Logger + runner CommandRunner + maxNarUploadSize int64 +} + +func newNixStoreUploadBackend(targetStore, stagingDir string, readUpstreams []SubstituterUpstream, logger *slog.Logger, runner CommandRunner) (*NixStoreUploadBackend, error) { + absStaging, err := filepath.Abs(stagingDir) + if err != nil { + return nil, fmt.Errorf("resolve staging dir %q: %w", stagingDir, err) + } + if logger == nil { + logger = slog.Default() + } + + if err := os.MkdirAll(filepath.Join(absStaging, "nar"), 0o755); err != nil { + return nil, fmt.Errorf("create staging cache directories: %w", err) + } + infoPath := filepath.Join(absStaging, "nix-cache-info") + if _, err := os.Stat(infoPath); errors.Is(err, os.ErrNotExist) { + if err := os.WriteFile(infoPath, []byte(nixStoreCacheInfo), 0o644); err != nil { + return nil, fmt.Errorf("write nix-cache-info: %w", err) + } + } + + if runner == nil { + runner = execRunner{} + } + + return &NixStoreUploadBackend{ + stagingDir: absStaging, + targetStore: targetStore, + readUpstreams: readUpstreams, + logger: logger, + runner: runner, + maxNarUploadSize: maxNarUploadSize, + }, nil +} + +func (b *NixStoreUploadBackend) Close() error { return nil } + +func (b *NixStoreUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { + relPath, err := normalizeUploadCachePath(r.URL.Path) + if err != nil { + b.logger.Warn("refusing upload cache request with unsafe path", "path", r.URL.Path, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + switch r.Method { + case http.MethodGet, http.MethodHead: + switch { + case relPath == "nix-cache-info": + b.serveCacheInfo(w, r) + return + case isNarinfoObjectPath(relPath): + b.serveNarinfo(w, r, relPath) + return + } + + case http.MethodPut: + switch { + case relPath == "nix-cache-info": + b.putCacheInfo(w, r) + return + case isNarObjectPath(relPath): + b.putNar(w, r, relPath) + return + case isNarinfoObjectPath(relPath): + b.putNarinfo(w, r, relPath) + return + } + } + + http.Error(w, "not found", http.StatusNotFound) +} + +func (b *NixStoreUploadBackend) serveCacheInfo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/x-nix-cache-info") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(nixStoreCacheInfo))) + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusOK) + return + } + _, _ = w.Write([]byte(nixStoreCacheInfo)) +} + +func (b *NixStoreUploadBackend) putCacheInfo(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, io.LimitReader(r.Body, int64(len(nixStoreCacheInfo))+1)) + w.WriteHeader(http.StatusOK) +} + +func (b *NixStoreUploadBackend) serveNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { + localPath, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing narinfo request with unsafe path", "path", relPath, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + fi, err := os.Stat(localPath) + if err == nil && !fi.IsDir() { + if _, err := readNarinfoFile(localPath); err != nil { + b.logger.Warn("staged narinfo is invalid", "path", relPath, "error", err) + http.Error(w, "invalid staged narinfo", http.StatusInternalServerError) + return + } + b.serveLocalFile(w, r, localPath, fi) + return + } + if !errors.Is(err, os.ErrNotExist) { + b.logger.Warn("stat staged narinfo failed", "path", relPath, "error", err) + } + + if len(b.readUpstreams) > 0 { + probe := r.Clone(r.Context()) + probe.URL.Path = "/" + relPath + serveNarinfoExistence(w, probe, newNarinfoExistenceTransport(b.readUpstreams, b.logger), b.logger) + return + } + + http.Error(w, "not found", http.StatusNotFound) +} + +func (b *NixStoreUploadBackend) serveLocalFile(w http.ResponseWriter, r *http.Request, localPath string, fi os.FileInfo) { + w.Header().Set("Content-Type", "text/x-nix-narinfo") + w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size())) + w.Header().Set("Last-Modified", fi.ModTime().UTC().Format(http.TimeFormat)) + + if r.Method == http.MethodHead { + w.WriteHeader(http.StatusOK) + return + } + + f, err := os.Open(localPath) + if err != nil { + b.logger.Warn("open staged narinfo failed", "path", localPath, "error", err) + http.Error(w, "not found", http.StatusNotFound) + return + } + defer f.Close() + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, f); err != nil && !errors.Is(err, context.Canceled) { + b.logger.Warn("copy staged narinfo failed", "path", localPath, "error", err) + } +} + +func (b *NixStoreUploadBackend) putNar(w http.ResponseWriter, r *http.Request, relPath string) { + name := strings.TrimPrefix(relPath, "nar/") + dst, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing nar upload with unsafe path", "name", name, "error", err) + http.Error(w, "invalid nar path", http.StatusBadRequest) + return + } + r.Body = http.MaxBytesReader(w, r.Body, b.maxNarUploadSize) + + var copyErr error + written, err := writeFileAtomic(dst, ".tmp-nar", func(f *os.File) (int64, error) { + n, err := io.Copy(f, r.Body) + copyErr = err + return n, err + }) + if err != nil { + b.logger.Warn("stage nar upload failed", "name", name, "error", err) + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + http.Error(w, "nar too large", http.StatusRequestEntityTooLarge) + return + } + if copyErr != nil { + http.Error(w, "upload failed", http.StatusBadRequest) + return + } + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + b.logger.Debug("staged nar", "name", name, "bytes", written) + w.WriteHeader(http.StatusOK) +} + +func (b *NixStoreUploadBackend) putNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { + body, err := io.ReadAll(io.LimitReader(r.Body, maxNarinfoSize+1)) + if err != nil { + b.logger.Warn("read narinfo body failed", "path", relPath, "error", err) + http.Error(w, "upload failed", http.StatusBadRequest) + return + } + if len(body) > maxNarinfoSize { + b.logger.Warn("narinfo body exceeds maximum size", "path", relPath, "bytes", len(body)) + http.Error(w, "narinfo too large", http.StatusBadRequest) + return + } + + info, err := parseNarinfo(bytes.NewReader(body)) + if err != nil { + b.logger.Warn("refusing narinfo upload with invalid body", "path", relPath, "error", err) + http.Error(w, "invalid narinfo: "+err.Error(), http.StatusBadRequest) + return + } + storePathHash, _, err := parseStorePath(info.StorePath) + if err != nil { + b.logger.Warn("refusing narinfo upload with invalid store path", "path", relPath, "storePath", info.StorePath, "error", err) + http.Error(w, "invalid StorePath", http.StatusBadRequest) + return + } + fileHash := strings.TrimSuffix(filepath.Base(relPath), ".narinfo") + if fileHash != storePathHash { + b.logger.Warn("refusing narinfo upload with mismatched filename hash", "path", relPath, "storePath", info.StorePath) + http.Error(w, "narinfo filename does not match StorePath hash", http.StatusBadRequest) + return + } + if !isNarObjectPath(info.URL) { + b.logger.Warn("narinfo references invalid nar URL", "path", relPath, "url", info.URL) + http.Error(w, "invalid nar URL", http.StatusBadRequest) + return + } + + narPath, err := b.stagingObjectPath(info.URL) + if err != nil { + b.logger.Warn("narinfo references unsafe nar URL", "path", relPath, "url", info.URL, "error", err) + http.Error(w, "invalid nar URL", http.StatusBadRequest) + return + } + if _, err := os.Stat(narPath); err != nil { + b.logger.Warn("narinfo references missing nar", "path", relPath, "url", info.URL, "error", err) + http.Error(w, "referenced nar does not exist", http.StatusBadRequest) + return + } + + dst, err := b.stagingObjectPath(relPath) + if err != nil { + b.logger.Warn("refusing narinfo upload with unsafe path", "path", relPath, "error", err) + http.Error(w, "invalid path", http.StatusBadRequest) + return + } + + if _, err := writeNarinfoFile(dst, body); err != nil { + b.logger.Warn("stage narinfo upload failed", "path", relPath, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + if err := b.importStorePath(r.Context(), info.StorePath); err != nil { + b.logger.Warn("import staged narinfo failed", "path", relPath, "storePath", info.StorePath, "error", err) + if cleanupErr := removeFileAndSyncDir(dst); cleanupErr != nil { + b.logger.Error("remove staged narinfo after failed import", "path", relPath, "error", cleanupErr) + } + http.Error(w, "import failed", http.StatusBadGateway) + return + } + + b.logger.Debug("staged narinfo", "path", relPath, "storePath", info.StorePath) + w.WriteHeader(http.StatusOK) +} + +func normalizeUploadCachePath(path string) (string, error) { + if path == "" || path == "/" { + return "", fmt.Errorf("empty path") + } + if !strings.HasPrefix(path, "/") { + return "", fmt.Errorf("path must start with /") + } + if strings.Contains(path, "..") { + return "", fmt.Errorf("path traversal") + } + + return strings.TrimPrefix(path, "/"), nil +} + +func isNarinfoObjectPath(relPath string) bool { + if !strings.HasSuffix(relPath, ".narinfo") { + return false + } + if relPath != filepath.Base(relPath) { + return false + } + base := filepath.Base(relPath) + return base != "" && base != "." && base != ".narinfo" +} + +func isNarObjectPath(relPath string) bool { + if !strings.HasPrefix(relPath, "nar/") { + return false + } + name := strings.TrimPrefix(relPath, "nar/") + return name != "" && name != "." && name == filepath.Base(name) && !strings.Contains(name, "/") +} + +func (b *NixStoreUploadBackend) stagingObjectPath(relPath string) (string, error) { + if !isNarObjectPath(relPath) && !isNarinfoObjectPath(relPath) { + return "", fmt.Errorf("invalid cache object path %q", relPath) + } + + local, err := filepath.Localize(relPath) + if err != nil { + return "", fmt.Errorf("unsafe cache object path %q: %w", relPath, err) + } + + return filepath.Join(b.stagingDir, local), nil +} + +// makes the full reference graph of rootStorePath resolvable in the staging +// cache. `nix copy` computes the closure from the --from store, so every +// referenced narinfo must be present there or the walk fails with "path ... is +// not valid". newly-built deps are already staged by the guest, but deps that +// live only in a read cache were skipped during upload, so we backfill their +// narinfos here. only the narinfos (the reference graph) are needed: the +// destination supplies the NAR data via --substitute-on-destination. +func (b *NixStoreUploadBackend) ensureClosureStaged(ctx context.Context, rootStorePath string) error { + visited := map[string]bool{} + queue := []string{rootStorePath} + for len(queue) > 0 { + storePath := queue[0] + queue = queue[1:] + if visited[storePath] { + continue + } + visited[storePath] = true + + info, err := b.resolveStagedNarinfo(ctx, storePath) + if err != nil { + // the root must resolve (the guest just staged it); a dep we can't + // find anywhere is left for `nix copy` to surface with its own error. + if storePath == rootStorePath { + return fmt.Errorf("resolve narinfo for %s: %w", storePath, err) + } + b.logger.Warn("closure dep narinfo unresolved; leaving to nix copy", "storePath", storePath, "error", err) + continue + } + + for _, ref := range info.References { + refPath := storePrefix + ref + if refPath == storePath { + continue // self-reference + } + if !visited[refPath] { + queue = append(queue, refPath) + } + } + } + return nil +} + +// returns parsed narinfo for store path, backfilling from readUpstreams if not found +func (b *NixStoreUploadBackend) resolveStagedNarinfo(ctx context.Context, storePath string) (*narinfo, error) { + hash, _, err := parseStorePath(storePath) + if err != nil { + return nil, err + } + localPath := filepath.Join(b.stagingDir, hash+".narinfo") + info, err := readNarinfoFile(localPath) + if err == nil { + return info, nil + } + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + // dep missing from staging because it was skipped during upload + // (lives on a read cache) so we backfill it from readUpstreams. + body, err := b.fetchUpstreamNarinfo(ctx, hash) + if err != nil { + return nil, err + } + written, err := writeNarinfoFile(localPath, body) + if err != nil { + return nil, err + } + b.logger.Debug("backfilled narinfo", "hash", hash, "bytes", written) + + return parseNarinfo(bytes.NewReader(body)) +} + +// fetches narinfo from readUpstreams +func (b *NixStoreUploadBackend) fetchUpstreamNarinfo(ctx context.Context, hash string) ([]byte, error) { + if len(b.readUpstreams) == 0 { + return nil, os.ErrNotExist + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://upstream/"+hash+".narinfo", nil) + if err != nil { + return nil, err + } + resp, err := newNarinfoExistenceTransport(b.readUpstreams, b.logger).RoundTrip(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound { + return nil, os.ErrNotExist + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("upstream narinfo %s: status %d", hash, resp.StatusCode) + } + return io.ReadAll(io.LimitReader(resp.Body, maxNarinfoSize+1)) +} + +// todo(dawn): ideally we don't use `nix copy` here but instead have our own +// `nix copy` impl so we don't need nix on host. but that's a far stretch goal :p +func (b *NixStoreUploadBackend) importStorePath(ctx context.Context, storePath string) error { + if err := b.ensureClosureStaged(ctx, storePath); err != nil { + return fmt.Errorf("stage closure for %s: %w", storePath, err) + } + + fromURL := url.URL{Scheme: "file", Path: b.stagingDir} + args := []string{ + "copy", + "--from", fromURL.String(), + "--to", b.targetStore, + // todo(dawn): ideally we support signing in spindle itself. + // but for now harmonia can sign things on serve so this is ok. + "--no-check-sigs", + "--substitute-on-destination", + storePath, + } + + b.logger.Info("importing staged cache path", "target", b.targetStore, "storePath", storePath) + if err := b.runner.Run(ctx, "nix", args...); err != nil { + return fmt.Errorf("nix copy to %s: %w", b.targetStore, err) + } + return nil +} + +func readNarinfoFile(path string) (*narinfo, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return parseNarinfo(f) +} + +func writeNarinfoFile(path string, body []byte) (int64, error) { + return writeFileAtomic(path, ".tmp-narinfo", func(f *os.File) (int64, error) { + n, err := f.Write(body) + return int64(n), err + }) +} + +func writeFileAtomic(dst, tempPrefix string, write func(*os.File) (int64, error)) (written int64, err error) { + dir := filepath.Dir(dst) + if err := os.MkdirAll(dir, 0o755); err != nil { + return 0, fmt.Errorf("create directory %q: %w", dir, err) + } + + tmp, err := os.CreateTemp(dir, tempPrefix+"-*-"+filepath.Base(dst)) + if err != nil { + return 0, fmt.Errorf("create temporary file in %q: %w", dir, err) + } + tmpName := tmp.Name() + defer func() { + if err != nil { + _ = tmp.Close() + _ = os.Remove(tmpName) + } + }() + + written, err = write(tmp) + if err != nil { + return 0, err + } + if err := tmp.Sync(); err != nil { + return 0, fmt.Errorf("fsync temporary file %q: %w", tmpName, err) + } + if err := tmp.Close(); err != nil { + return 0, fmt.Errorf("close temporary file %q: %w", tmpName, err) + } + if err := os.Rename(tmpName, dst); err != nil { + return 0, fmt.Errorf("rename %q to %q: %w", tmpName, dst, err) + } + + if err := syncDir(dir); err != nil { + return 0, err + } + + return written, nil +} + +func removeFileAndSyncDir(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove %q: %w", path, err) + } + return syncDir(filepath.Dir(path)) +} + +func syncDir(dir string) error { + dirFile, err := os.Open(dir) + if err != nil { + return fmt.Errorf("open directory %q: %w", dir, err) + } + defer dirFile.Close() + if err := dirFile.Sync(); err != nil { + return fmt.Errorf("sync directory %q: %w", dir, err) + } + return nil +} diff --git a/spindle/engines/microvm/substituter_upload_nix_store_test.go b/spindle/engines/microvm/substituter_upload_nix_store_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_upload_nix_store_test.go @@ -0,0 +1,496 @@ +//go:build linux + +package microvm + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" +) + +const ( + testStoreHash = "0123456789abcdfghijklmnpqrsvwxyz" + testStorePath = "/nix/store/" + testStoreHash + "-abc-output" +) + +func TestSubstituterUploadBackendSchemeDispatch(t *testing.T) { + staging := t.TempDir() + logger := slog.Default() + + cases := []struct { + uploadURL string + wantErr bool + wantType string + }{ + {"https://cache.example/upload", false, "*microvm.httpUploadBackend"}, + {"http://cache.example/upload", false, "*microvm.httpUploadBackend"}, + {"ssh://cache-host", false, "*microvm.NixStoreUploadBackend"}, + {"ssh-ng://cache-host", false, "*microvm.NixStoreUploadBackend"}, + {"daemon", false, "*microvm.NixStoreUploadBackend"}, + {"local", false, "*microvm.NixStoreUploadBackend"}, + {"ftp://cache.example", true, ""}, + {"/some/path", true, ""}, + } + + for _, tc := range cases { + t.Run(tc.uploadURL, func(t *testing.T) { + backend, err := newSubstituterUploadBackend(tc.uploadURL, nil, staging, logger) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q", tc.uploadURL) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := fmt.Sprintf("%T", backend) + if got != tc.wantType { + t.Fatalf("backend type: got %s, want %s", got, tc.wantType) + } + }) + } +} + +func TestSubstituterUploadBackendEmptyURL(t *testing.T) { + backend, err := newSubstituterUploadBackend("", nil, t.TempDir(), slog.Default()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if backend != nil { + t.Fatalf("expected nil backend for empty URL, got %T", backend) + } +} + +func newTestNixStoreBackend(t *testing.T, target string, runner CommandRunner) (*NixStoreUploadBackend, string) { + t.Helper() + staging := t.TempDir() + if target == "" { + target = "ssh-ng://cache-host" + } + b, err := newNixStoreUploadBackend(target, staging, nil, slog.Default(), runner) + if err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + return b, staging +} + +func mustUploadNar(t *testing.T, b *NixStoreUploadBackend, name, body string) { + t.Helper() + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/"+name, strings.NewReader(body))) + if rec.Code != http.StatusOK { + t.Fatalf("upload nar %q: got %d, want 200; body=%q", name, rec.Code, rec.Body.String()) + } +} + +func TestNixStoreBackendNixCacheInfo(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/nix-cache-info", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET status: got %d, want 200", rec.Code) + } + if !strings.Contains(rec.Body.String(), "StoreDir: /nix/store") { + t.Fatalf("cache info missing StoreDir: %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodHead, "/nix-cache-info", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("HEAD status: got %d, want 200", rec.Code) + } + if rec.Body.Len() != 0 { + t.Fatalf("HEAD body should be empty, got %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nix-cache-info", strings.NewReader("ignored"))) + if rec.Code != http.StatusOK { + t.Fatalf("PUT /nix-cache-info status: got %d, want 200", rec.Code) + } +} + +func TestNixStoreBackendRejectsTraversalNar(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/../../evil", strings.NewReader("bad"))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("traversal nar status: got %d, want 400", rec.Code) + } + + if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "evil")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("traversal nar escaped staging dir: %v", err) + } +} + +func TestNixStoreBackendRejectsOversizedNarUpload(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + b.maxNarUploadSize = 3 + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/foo.nar", strings.NewReader("four"))) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversized nar status: got %d, want 413; body=%q", rec.Code, rec.Body.String()) + } + + if _, err := os.Stat(filepath.Join(staging, "nar", "foo.nar")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("oversized nar should not have been staged: %v", err) + } +} + +func TestNixStoreBackendNarinfoRequiresExistingNar(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/abc.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("narinfo before nar status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } +} + +type fakeRunner struct { + mu sync.Mutex + calls [][]string + nextErr error +} + +func (f *fakeRunner) Run(ctx context.Context, name string, args ...string) error { + f.mu.Lock() + defer f.mu.Unlock() + call := append([]string{name}, args...) + f.calls = append(f.calls, call) + return f.nextErr +} + +func (f *fakeRunner) Calls() [][]string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.calls) +} + +func TestNixStoreBackendImportsNarinfoImmediately(t *testing.T) { + runner := &fakeRunner{} + b, staging := newTestNixStoreBackend(t, "ssh-ng://spindle-upload@cache-host", runner) + + mustUploadNar(t, b, "foo.nar.zst", "nar-body") + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusOK { + t.Fatalf("PUT narinfo status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + + calls := runner.Calls() + if len(calls) != 1 { + t.Fatalf("expected 1 nix copy call, got %d", len(calls)) + } + call := calls[0] + wantFrom := (&url.URL{Scheme: "file", Path: staging}).String() + want := []string{ + "nix", + "copy", + "--from", wantFrom, + "--to", "ssh-ng://spindle-upload@cache-host", + "--no-check-sigs", + "--substitute-on-destination", + testStorePath, + } + if !slices.Equal(call, want) { + t.Fatalf("nix copy args:\n got: %v\nwant: %v", call, want) + } + + data, err := os.ReadFile(filepath.Join(staging, testStoreHash+".narinfo")) + if err != nil { + t.Fatalf("staged narinfo missing: %v", err) + } + if string(data) != narinfo { + t.Fatalf("staged narinfo contents: got %q, want %q", string(data), narinfo) + } +} + +func TestNixStoreBackendBackfillsClosureDepNarinfo(t *testing.T) { + const depHash = "abcdfghijklmnpqrsvwxyz0123456789" + depNarinfo := "StorePath: /nix/store/" + depHash + "-dep\nURL: nar/dep.nar.zst\nNarHash: sha256:dep\nNarSize: 1\n" + + var depRequests int + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/"+depHash+".narinfo" { + depRequests++ + _, _ = io.WriteString(w, depNarinfo) + return + } + w.WriteHeader(http.StatusNotFound) + })) + defer upstream.Close() + + upURL, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + + runner := &fakeRunner{} + staging := t.TempDir() + b, err := newNixStoreUploadBackend("ssh-ng://cache-host", staging, []SubstituterUpstream{{url: upURL}}, slog.Default(), runner) + if err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + + mustUploadNar(t, b, "foo.nar.zst", "nar-body") + + // the top path references the dep, which is absent from staging. + narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\nReferences: " + depHash + "-dep\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusOK { + t.Fatalf("PUT narinfo status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + + // the dep narinfo must have been fetched from the upstream and staged... + if depRequests == 0 { + t.Fatalf("expected the dep narinfo to be fetched from the upstream") + } + staged, err := os.ReadFile(filepath.Join(staging, depHash+".narinfo")) + if err != nil { + t.Fatalf("dep narinfo not backfilled into staging: %v", err) + } + if string(staged) != depNarinfo { + t.Fatalf("backfilled dep narinfo contents: got %q, want %q", string(staged), depNarinfo) + } + + // ...and the import still copies just the requested top path. + calls := runner.Calls() + if len(calls) != 1 || calls[0][len(calls[0])-1] != testStorePath { + t.Fatalf("expected a single nix copy for %s, got %v", testStorePath, calls) + } +} + +func TestNixStoreBackendRemovesNarinfoOnImportFailure(t *testing.T) { + runner := &fakeRunner{nextErr: errors.New("nix copy failed")} + b, staging := newTestNixStoreBackend(t, "ssh://cache-host", runner) + + mustUploadNar(t, b, "foo.nar.zst", "nar-body") + + narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) + if rec.Code != http.StatusBadGateway { + t.Fatalf("failed import status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) + } + + if _, err := os.Stat(filepath.Join(staging, testStoreHash+".narinfo")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("narinfo should be removed after failed import: %v", err) + } +} + +func TestNixStoreBackendNarinfoReadUpstream(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/miss.narinfo" { + w.WriteHeader(http.StatusNotFound) + return + } + _, _ = io.WriteString(w, "StorePath: /nix/store/upstream\nURL: nar/upstream.nar\nNarHash: sha256:up\nNarSize: 1\n") + })) + defer upstream.Close() + + upURL, err := url.Parse(upstream.URL) + if err != nil { + t.Fatal(err) + } + + staging := t.TempDir() + b, err := newNixStoreUploadBackend("ssh://cache-host", staging, []SubstituterUpstream{{url: upURL}}, slog.Default(), nil) + if err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/present.narinfo", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("GET upstream-present narinfo status: got %d, want 200", rec.Code) + } + if !strings.Contains(rec.Body.String(), "/nix/store/upstream") { + t.Fatalf("unexpected upstream narinfo body: %q", rec.Body.String()) + } + + rec = httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/miss.narinfo", nil)) + if rec.Code != http.StatusNotFound { + t.Fatalf("GET upstream-missing narinfo status: got %d, want 404", rec.Code) + } + +} + +func TestNixStoreBackendRejectsInvalidLocalNarinfo(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + if err := os.WriteFile(filepath.Join(staging, testStoreHash+".narinfo"), []byte("not-a-narinfo\n"), 0o644); err != nil { + t.Fatalf("write invalid staged narinfo: %v", err) + } + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/"+testStoreHash+".narinfo", nil)) + if rec.Code != http.StatusInternalServerError { + t.Fatalf("invalid local narinfo status: got %d, want 500; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestNixStoreBackendNarinfoValidation(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + + mustUploadNar(t, b, "x.nar", "x") + + cases := []struct { + name string + body string + wantErr string + }{ + { + name: "missing StorePath", + body: "URL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "StorePath", + }, + { + name: "bad StorePath", + body: "StorePath: /tmp/evil\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "invalid StorePath", + }, + { + name: "malformed StorePath", + body: "StorePath: /nix/store/not-a-real-store-path\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "invalid StorePath", + }, + { + name: "missing URL", + body: "StorePath: " + testStorePath + "\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "absolute URL", + body: "StorePath: " + testStorePath + "\nURL: /etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "traversal URL", + body: "StorePath: " + testStorePath + "\nURL: nar/../../etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "URL", + }, + { + name: "non nar URL", + body: "StorePath: " + testStorePath + "\nURL: nix-cache-info\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "nar/", + }, + { + name: "nested nar URL", + body: "StorePath: " + testStorePath + "\nURL: nar/dir/x.nar\nNarHash: sha256:x\nNarSize: 1\n", + wantErr: "safe nar object path", + }, + { + name: "missing NarHash", + body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarSize: 1\n", + wantErr: "NarHash", + }, + { + name: "bad NarSize", + body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: huge\n", + wantErr: "NarSize", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(tc.body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), tc.wantErr) { + t.Fatalf("body %q should mention %q", rec.Body.String(), tc.wantErr) + } + }) + } +} + +func TestNixStoreBackendUploadCacheInfoFileExists(t *testing.T) { + staging := t.TempDir() + if _, err := newNixStoreUploadBackend("ssh://host", staging, nil, slog.Default(), nil); err != nil { + t.Fatalf("newNixStoreUploadBackend: %v", err) + } + data, err := os.ReadFile(filepath.Join(staging, "nix-cache-info")) + if err != nil { + t.Fatalf("nix-cache-info missing: %v", err) + } + if !bytes.Contains(data, []byte("StoreDir: /nix/store")) { + t.Fatalf("unexpected nix-cache-info: %q", string(data)) + } +} + +func TestNixStoreBackendRejectsTraversalNarinfoPath(t *testing.T) { + b, staging := newTestNixStoreBackend(t, "", nil) + + body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/../etc/passwd.narinfo", strings.NewReader(body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("traversal narinfo status: got %d, want 400", rec.Code) + } + + if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "etc", "passwd.narinfo")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("traversal narinfo escaped staging dir: %v", err) + } +} + +func TestNixStoreBackendRejectsNarinfoFilenameHashMismatch(t *testing.T) { + b, _ := newTestNixStoreBackend(t, "", nil) + mustUploadNar(t, b, "x.nar", "x") + + body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" + rec := httptest.NewRecorder() + b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/11111111111111111111111111111111.narinfo", strings.NewReader(body))) + if rec.Code != http.StatusBadRequest { + t.Fatalf("mismatched narinfo status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), "filename does not match") { + t.Fatalf("unexpected body: %q", rec.Body.String()) + } +} + +func TestParseNarinfoAcceptsLargeReferencesLine(t *testing.T) { + var refs []string + for range 12000 { + refs = append(refs, "0123456789abcdfghijklmnpqrsvwxy-ref") + } + + body := strings.Join([]string{ + "StorePath: " + testStorePath, + "URL: nar/x.nar", + "NarHash: sha256:abc", + "NarSize: 1", + "References: " + strings.Join(refs, " "), + "", + }, "\n") + + info, err := parseNarinfo(strings.NewReader(body)) + if err != nil { + t.Fatalf("parseNarinfo failed for large references line: %v", err) + } + if info.StorePath != testStorePath { + t.Fatalf("StorePath: got %q, want %q", info.StorePath, testStorePath) + } +} diff --git a/spindle/engines/microvm/substituter_upload_proxy.go b/spindle/engines/microvm/substituter_upload_proxy.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_upload_proxy.go @@ -0,0 +1,160 @@ +//go:build linux + +package microvm + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/mdlayher/vsock" +) + +type SubstituterUploadBackend interface { + http.Handler + Close() error +} + +type SubstituterUploadProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server + backend SubstituterUploadBackend +} + +func StartSubstituterUploadProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []SubstituterUpstream, stagingDir string, logger *slog.Logger) (*SubstituterUploadProxy, error) { + if strings.TrimSpace(uploadURL) == "" { + return nil, nil + } + + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "upload_cache_proxy", "cid", cid, "uploadURL", uploadURL) + + backend, err := newSubstituterUploadBackend(uploadURL, readUpstreams, stagingDir, logger) + if err != nil { + return nil, err + } + + ln, port, err := listenRandomVsockUploadPort(ctx) + if err != nil { + return nil, fmt.Errorf("listen for cache upload proxy: %w", err) + } + + proxy := &SubstituterUploadProxy{ + port: port, + ln: ln, + backend: backend, + } + proxy.server = &http.Server{ + Handler: backend, + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 30 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("substituter upload proxy stopped", "port", port, "error", err) + } + }() + + logger.Info("started substituter upload proxy", "port", port, "target", uploadURL, "readUpstreams", len(readUpstreams)) + return proxy, nil +} + +func newSubstituterUploadBackend(uploadURL string, readUpstreams []SubstituterUpstream, stagingDir string, logger *slog.Logger) (SubstituterUploadBackend, error) { + if strings.TrimSpace(uploadURL) == "" { + return nil, nil + } + + target, err := url.Parse(uploadURL) + if err != nil { + return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) + } + + switch target.Scheme { + case "http", "https": + if target.Host == "" { + return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) + } + return newHTTPUploadProxyBackend(target, readUpstreams, logger), nil + + case "ssh", "ssh-ng": + return newNixStoreUploadBackend(target.String(), stagingDir, readUpstreams, logger, nil) + + case "": + switch uploadURL { + case "daemon", "local": + return newNixStoreUploadBackend(uploadURL, stagingDir, readUpstreams, logger, nil) + default: + return nil, fmt.Errorf("unsupported upload URL %q", uploadURL) + } + + default: + return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q", uploadURL, target.Scheme) + } +} + +func (p *SubstituterUploadProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *SubstituterUploadProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + if p.backend != nil { + closeErr = errors.Join(closeErr, p.backend.Close()) + } + return closeErr +} + +func listenRandomVsockUploadPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.ListenContextID(vsock.Host, port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock upload port: %w", lastErr) +} diff --git a/spindle/engines/microvm/substituter_upload_proxy_test.go b/spindle/engines/microvm/substituter_upload_proxy_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/substituter_upload_proxy_test.go @@ -0,0 +1,130 @@ +//go:build linux + +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestUploadProxyRewritesHostAndAuth(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + if req.URL.Path != "/sub/abc.narinfo" { + t.Errorf("path: got %q, want /sub/abc.narinfo", req.URL.Path) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/hunter2/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + target, err := url.Parse("http://dawn:woof@" + upstreamHost + "/sub/") + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodPut, "http://127.0.0.1:10501/abc.narinfo", strings.NewReader("narinfo")) + req.Host = "127.0.0.1:10501" + rec := httptest.NewRecorder() + uploadProxyHandler(target, nil, slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +func TestUploadProxySkipsNarinfoAvailableUpstream(t *testing.T) { + var uploadHits int + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + uploadHits++ + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Errorf("upstream path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []SubstituterUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200 (so nix treats the path as present and skips upload)", rec.Code) + } + if !strings.Contains(rec.Body.String(), "StorePath: /nix/store/abc") { + t.Fatalf("body: got %q, want the upstream narinfo body", rec.Body.String()) + } +} + +func TestUploadProxyUploadsNarinfoNobodyHas(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []SubstituterUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status: got %d, want 404 (so nix uploads the path)", rec.Code) + } +} + +func TestUploadProxySkipsNarinfoAlreadyOnTarget(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer target.Close() + + handler := uploadProxyHandler(mustParseURL(t, target.URL), nil, slog.Default()) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200", rec.Code) + } +} diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh --- a/spindle/engines/microvm/test-spindle-microvm.sh +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -277,7 +277,7 @@ trap 'exit 143' TERM CACHE_PORT=$(pick_free_port) -./spindle/engines/microvm/start-test-cache.sh "$TEMP_DIR" "$CACHE_PORT" +./spindle/engines/microvm/start-test-substituter.sh "$TEMP_DIR" "$CACHE_PORT" source "$TEMP_DIR/env.sh" run_vm() { diff --git a/spindle/engines/microvm/upload_cache_http.go b/spindle/engines/microvm/upload_cache_http.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_http.go +++ /dev/null @@ -1,103 +0,0 @@ -//go:build linux - -package microvm - -import ( - "context" - "errors" - "io" - "log/slog" - "net/http" - "net/http/httputil" - "net/url" - "strings" -) - -// httpUploadBackend reverse-proxies guest binary-cache upload traffic to an -// http(s) upload cache such as ncps. -type httpUploadBackend struct { - handler http.Handler -} - -func newHTTPUploadProxyBackend(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) *httpUploadBackend { - return &httpUploadBackend{handler: uploadProxyHandler(target, readUpstreams, logger)} -} - -func (b *httpUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { - b.handler.ServeHTTP(w, r) -} - -func (b *httpUploadBackend) Close() error { return nil } - -func uploadProxyHandler(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) http.Handler { - rp := httputil.NewSingleHostReverseProxy(target) - rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) - - origDirector := rp.Director - rp.Director = func(req *http.Request) { - origDirector(req) - // ensure host matches target - req.Host = target.Host - // the transport doesn't turn URL userinfo into basic auth, only - // http.Client does, so do it ourselves - if user := target.User; user != nil { - password, _ := user.Password() - req.SetBasicAuth(user.Username(), password) - } - } - - // before uploading, nix copy asks the destination whether it already has each - // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer - // that check across the upload target *and* the read caches: if any of them - // already serves the path there is no point uploading it (the guest would - // just substitute it from there anyway). - narinfoUpstreams := append([]CacheUpstream{{url: target}}, readUpstreams...) - exists := newNarinfoExistenceTransport(narinfoUpstreams, logger) - - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if isNarinfoExistenceCheck(r) { - serveNarinfoExistence(w, r, exists, logger) - return - } - rp.ServeHTTP(w, r) - }) -} - -func newNarinfoExistenceTransport(upstreams []CacheUpstream, logger *slog.Logger) http.RoundTripper { - return ¶llelRacingTransport{ - upstreams: upstreams, - underlying: proxyTransport, - guardedUnderlying: guardedProxyTransport, - logger: logger, - } -} - -func isNarinfoExistenceCheck(r *http.Request) bool { - if r.Method != http.MethodGet && r.Method != http.MethodHead { - return false - } - return strings.HasSuffix(r.URL.Path, ".narinfo") -} - -func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { - probe := r.Clone(r.Context()) - probe.RequestURI = "" - - resp, err := exists.RoundTrip(probe) - if err != nil { - logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) - w.WriteHeader(http.StatusNotFound) - return - } - defer resp.Body.Close() - - for key, values := range resp.Header { - for _, value := range values { - w.Header().Add(key, value) - } - } - w.WriteHeader(resp.StatusCode) - if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { - logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) - } -} diff --git a/spindle/engines/microvm/upload_cache_narinfo.go b/spindle/engines/microvm/upload_cache_narinfo.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_narinfo.go +++ /dev/null @@ -1,130 +0,0 @@ -//go:build linux - -package microvm - -import ( - "bufio" - "fmt" - "io" - "path/filepath" - "regexp" - "strconv" - "strings" -) - -type narinfo struct { - StorePath string - URL string - NarHash string - NarSize int64 - // paths this path depends on - References []string -} - -const ( - maxNarinfoSize = 1 << 20 // 1 MiB - storePrefix = "/nix/store/" - maxNarinfoLineLen = maxNarinfoSize -) - -var nixStorePathBaseRe = regexp.MustCompile(`^[0-9abcdfghijklmnpqrsvwxyz]{32}-[^/]+$`) - -// parseNarinfo parses and validates a narinfo body. -// - required fields must be present -// - StorePath must be under /nix/store/ -// - URL must be a relative, traversal-safe path referencing a NAR in the -// same staging cache -// - NarSize must be a non-negative integer -func parseNarinfo(r io.Reader) (*narinfo, error) { - lr := io.LimitReader(r, maxNarinfoSize+1) - scanner := bufio.NewScanner(lr) - scanner.Buffer(make([]byte, 4096), maxNarinfoLineLen) - - var info narinfo - for scanner.Scan() { - line := scanner.Text() - if line == "" { - continue - } - key, value, ok := strings.Cut(line, ":") - if !ok { - return nil, fmt.Errorf("invalid narinfo line %q", line) - } - key = strings.TrimSpace(key) - value = strings.TrimSpace(value) - - switch key { - case "StorePath": - info.StorePath = value - case "URL": - info.URL = value - case "NarHash": - info.NarHash = value - case "NarSize": - n, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return nil, fmt.Errorf("invalid NarSize %q: %w", value, err) - } - info.NarSize = n - case "References": - info.References = strings.Fields(value) - } - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("read narinfo: %w", err) - } - - if err := validateNarinfo(&info); err != nil { - return nil, err - } - return &info, nil -} - -func validateNarinfo(info *narinfo) error { - if info.StorePath == "" { - return fmt.Errorf("narinfo missing StorePath") - } - if _, _, err := parseStorePath(info.StorePath); err != nil { - return fmt.Errorf("invalid StorePath: %w", err) - } - if info.URL == "" { - return fmt.Errorf("narinfo missing URL") - } - if strings.HasPrefix(info.URL, "/") || strings.Contains(info.URL, "..") { - return fmt.Errorf("narinfo URL %q is not a safe relative path", info.URL) - } - if !strings.HasPrefix(info.URL, "nar/") { - return fmt.Errorf("narinfo URL %q must reference a staged nar/ object", info.URL) - } - name := strings.TrimPrefix(info.URL, "nar/") - if name == "" || name == "." || name != filepath.Base(name) || strings.Contains(name, "/") { - return fmt.Errorf("narinfo URL %q is not a safe nar object path", info.URL) - } - if info.NarHash == "" { - return fmt.Errorf("narinfo missing NarHash") - } - if info.NarSize < 0 { - return fmt.Errorf("narinfo NarSize must be non-negative") - } - return nil -} - -func parseStorePath(path string) (hash string, name string, err error) { - if !strings.HasPrefix(path, storePrefix) { - return "", "", fmt.Errorf("store path %q does not start with %q", path, storePrefix) - } - - base := strings.TrimPrefix(path, storePrefix) - if base == "" || strings.Contains(base, "/") { - return "", "", fmt.Errorf("store path %q has invalid base name", path) - } - if !nixStorePathBaseRe.MatchString(base) { - return "", "", fmt.Errorf("store path %q is not a valid nix store path", path) - } - - hash, name, ok := strings.Cut(base, "-") - if !ok || hash == "" || name == "" { - return "", "", fmt.Errorf("store path %q is missing hash or name", path) - } - return hash, name, nil -} diff --git a/spindle/engines/microvm/upload_cache_nix_store.go b/spindle/engines/microvm/upload_cache_nix_store.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_nix_store.go +++ /dev/null @@ -1,541 +0,0 @@ -//go:build linux - -package microvm - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" -) - -// we have an interface for running commands so we can swap it in tests -type CommandRunner interface { - Run(ctx context.Context, name string, args ...string) error -} - -type execRunner struct{} - -func (execRunner) Run(ctx context.Context, name string, args ...string) error { - // nosemgrep: go.lang.security.audit.dangerous-exec-command.dangerous-exec-command - cmd := exec.CommandContext(ctx, name, args...) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, string(out)) - } - return nil -} - -const ( - nixStoreCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 50\n" - maxNarUploadSize = 5 << 30 // 5gib -) - -type NixStoreUploadBackend struct { - stagingDir string - targetStore string - readUpstreams []CacheUpstream - logger *slog.Logger - runner CommandRunner - maxNarUploadSize int64 -} - -func newNixStoreUploadBackend(targetStore, stagingDir string, readUpstreams []CacheUpstream, logger *slog.Logger, runner CommandRunner) (*NixStoreUploadBackend, error) { - absStaging, err := filepath.Abs(stagingDir) - if err != nil { - return nil, fmt.Errorf("resolve staging dir %q: %w", stagingDir, err) - } - if logger == nil { - logger = slog.Default() - } - - if err := os.MkdirAll(filepath.Join(absStaging, "nar"), 0o755); err != nil { - return nil, fmt.Errorf("create staging cache directories: %w", err) - } - infoPath := filepath.Join(absStaging, "nix-cache-info") - if _, err := os.Stat(infoPath); errors.Is(err, os.ErrNotExist) { - if err := os.WriteFile(infoPath, []byte(nixStoreCacheInfo), 0o644); err != nil { - return nil, fmt.Errorf("write nix-cache-info: %w", err) - } - } - - if runner == nil { - runner = execRunner{} - } - - return &NixStoreUploadBackend{ - stagingDir: absStaging, - targetStore: targetStore, - readUpstreams: readUpstreams, - logger: logger, - runner: runner, - maxNarUploadSize: maxNarUploadSize, - }, nil -} - -func (b *NixStoreUploadBackend) Close() error { return nil } - -func (b *NixStoreUploadBackend) ServeHTTP(w http.ResponseWriter, r *http.Request) { - relPath, err := normalizeUploadCachePath(r.URL.Path) - if err != nil { - b.logger.Warn("refusing upload cache request with unsafe path", "path", r.URL.Path, "error", err) - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - switch r.Method { - case http.MethodGet, http.MethodHead: - switch { - case relPath == "nix-cache-info": - b.serveCacheInfo(w, r) - return - case isNarinfoObjectPath(relPath): - b.serveNarinfo(w, r, relPath) - return - } - - case http.MethodPut: - switch { - case relPath == "nix-cache-info": - b.putCacheInfo(w, r) - return - case isNarObjectPath(relPath): - b.putNar(w, r, relPath) - return - case isNarinfoObjectPath(relPath): - b.putNarinfo(w, r, relPath) - return - } - } - - http.Error(w, "not found", http.StatusNotFound) -} - -func (b *NixStoreUploadBackend) serveCacheInfo(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/x-nix-cache-info") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(nixStoreCacheInfo))) - if r.Method == http.MethodHead { - w.WriteHeader(http.StatusOK) - return - } - _, _ = w.Write([]byte(nixStoreCacheInfo)) -} - -func (b *NixStoreUploadBackend) putCacheInfo(w http.ResponseWriter, r *http.Request) { - _, _ = io.Copy(io.Discard, io.LimitReader(r.Body, int64(len(nixStoreCacheInfo))+1)) - w.WriteHeader(http.StatusOK) -} - -func (b *NixStoreUploadBackend) serveNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { - localPath, err := b.stagingObjectPath(relPath) - if err != nil { - b.logger.Warn("refusing narinfo request with unsafe path", "path", relPath, "error", err) - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - fi, err := os.Stat(localPath) - if err == nil && !fi.IsDir() { - if _, err := readNarinfoFile(localPath); err != nil { - b.logger.Warn("staged narinfo is invalid", "path", relPath, "error", err) - http.Error(w, "invalid staged narinfo", http.StatusInternalServerError) - return - } - b.serveLocalFile(w, r, localPath, fi) - return - } - if !errors.Is(err, os.ErrNotExist) { - b.logger.Warn("stat staged narinfo failed", "path", relPath, "error", err) - } - - if len(b.readUpstreams) > 0 { - probe := r.Clone(r.Context()) - probe.URL.Path = "/" + relPath - serveNarinfoExistence(w, probe, newNarinfoExistenceTransport(b.readUpstreams, b.logger), b.logger) - return - } - - http.Error(w, "not found", http.StatusNotFound) -} - -func (b *NixStoreUploadBackend) serveLocalFile(w http.ResponseWriter, r *http.Request, localPath string, fi os.FileInfo) { - w.Header().Set("Content-Type", "text/x-nix-narinfo") - w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.Size())) - w.Header().Set("Last-Modified", fi.ModTime().UTC().Format(http.TimeFormat)) - - if r.Method == http.MethodHead { - w.WriteHeader(http.StatusOK) - return - } - - f, err := os.Open(localPath) - if err != nil { - b.logger.Warn("open staged narinfo failed", "path", localPath, "error", err) - http.Error(w, "not found", http.StatusNotFound) - return - } - defer f.Close() - w.WriteHeader(http.StatusOK) - if _, err := io.Copy(w, f); err != nil && !errors.Is(err, context.Canceled) { - b.logger.Warn("copy staged narinfo failed", "path", localPath, "error", err) - } -} - -func (b *NixStoreUploadBackend) putNar(w http.ResponseWriter, r *http.Request, relPath string) { - name := strings.TrimPrefix(relPath, "nar/") - dst, err := b.stagingObjectPath(relPath) - if err != nil { - b.logger.Warn("refusing nar upload with unsafe path", "name", name, "error", err) - http.Error(w, "invalid nar path", http.StatusBadRequest) - return - } - r.Body = http.MaxBytesReader(w, r.Body, b.maxNarUploadSize) - - var copyErr error - written, err := writeFileAtomic(dst, ".tmp-nar", func(f *os.File) (int64, error) { - n, err := io.Copy(f, r.Body) - copyErr = err - return n, err - }) - if err != nil { - b.logger.Warn("stage nar upload failed", "name", name, "error", err) - var maxErr *http.MaxBytesError - if errors.As(err, &maxErr) { - http.Error(w, "nar too large", http.StatusRequestEntityTooLarge) - return - } - if copyErr != nil { - http.Error(w, "upload failed", http.StatusBadRequest) - return - } - http.Error(w, "internal error", http.StatusInternalServerError) - return - } - - b.logger.Debug("staged nar", "name", name, "bytes", written) - w.WriteHeader(http.StatusOK) -} - -func (b *NixStoreUploadBackend) putNarinfo(w http.ResponseWriter, r *http.Request, relPath string) { - body, err := io.ReadAll(io.LimitReader(r.Body, maxNarinfoSize+1)) - if err != nil { - b.logger.Warn("read narinfo body failed", "path", relPath, "error", err) - http.Error(w, "upload failed", http.StatusBadRequest) - return - } - if len(body) > maxNarinfoSize { - b.logger.Warn("narinfo body exceeds maximum size", "path", relPath, "bytes", len(body)) - http.Error(w, "narinfo too large", http.StatusBadRequest) - return - } - - info, err := parseNarinfo(bytes.NewReader(body)) - if err != nil { - b.logger.Warn("refusing narinfo upload with invalid body", "path", relPath, "error", err) - http.Error(w, "invalid narinfo: "+err.Error(), http.StatusBadRequest) - return - } - storePathHash, _, err := parseStorePath(info.StorePath) - if err != nil { - b.logger.Warn("refusing narinfo upload with invalid store path", "path", relPath, "storePath", info.StorePath, "error", err) - http.Error(w, "invalid StorePath", http.StatusBadRequest) - return - } - fileHash := strings.TrimSuffix(filepath.Base(relPath), ".narinfo") - if fileHash != storePathHash { - b.logger.Warn("refusing narinfo upload with mismatched filename hash", "path", relPath, "storePath", info.StorePath) - http.Error(w, "narinfo filename does not match StorePath hash", http.StatusBadRequest) - return - } - if !isNarObjectPath(info.URL) { - b.logger.Warn("narinfo references invalid nar URL", "path", relPath, "url", info.URL) - http.Error(w, "invalid nar URL", http.StatusBadRequest) - return - } - - narPath, err := b.stagingObjectPath(info.URL) - if err != nil { - b.logger.Warn("narinfo references unsafe nar URL", "path", relPath, "url", info.URL, "error", err) - http.Error(w, "invalid nar URL", http.StatusBadRequest) - return - } - if _, err := os.Stat(narPath); err != nil { - b.logger.Warn("narinfo references missing nar", "path", relPath, "url", info.URL, "error", err) - http.Error(w, "referenced nar does not exist", http.StatusBadRequest) - return - } - - dst, err := b.stagingObjectPath(relPath) - if err != nil { - b.logger.Warn("refusing narinfo upload with unsafe path", "path", relPath, "error", err) - http.Error(w, "invalid path", http.StatusBadRequest) - return - } - - if _, err := writeNarinfoFile(dst, body); err != nil { - b.logger.Warn("stage narinfo upload failed", "path", relPath, "error", err) - http.Error(w, "internal error", http.StatusInternalServerError) - return - } - - if err := b.importStorePath(r.Context(), info.StorePath); err != nil { - b.logger.Warn("import staged narinfo failed", "path", relPath, "storePath", info.StorePath, "error", err) - if cleanupErr := removeFileAndSyncDir(dst); cleanupErr != nil { - b.logger.Error("remove staged narinfo after failed import", "path", relPath, "error", cleanupErr) - } - http.Error(w, "import failed", http.StatusBadGateway) - return - } - - b.logger.Debug("staged narinfo", "path", relPath, "storePath", info.StorePath) - w.WriteHeader(http.StatusOK) -} - -func normalizeUploadCachePath(path string) (string, error) { - if path == "" || path == "/" { - return "", fmt.Errorf("empty path") - } - if !strings.HasPrefix(path, "/") { - return "", fmt.Errorf("path must start with /") - } - if strings.Contains(path, "..") { - return "", fmt.Errorf("path traversal") - } - - return strings.TrimPrefix(path, "/"), nil -} - -func isNarinfoObjectPath(relPath string) bool { - if !strings.HasSuffix(relPath, ".narinfo") { - return false - } - if relPath != filepath.Base(relPath) { - return false - } - base := filepath.Base(relPath) - return base != "" && base != "." && base != ".narinfo" -} - -func isNarObjectPath(relPath string) bool { - if !strings.HasPrefix(relPath, "nar/") { - return false - } - name := strings.TrimPrefix(relPath, "nar/") - return name != "" && name != "." && name == filepath.Base(name) && !strings.Contains(name, "/") -} - -func (b *NixStoreUploadBackend) stagingObjectPath(relPath string) (string, error) { - if !isNarObjectPath(relPath) && !isNarinfoObjectPath(relPath) { - return "", fmt.Errorf("invalid cache object path %q", relPath) - } - - local, err := filepath.Localize(relPath) - if err != nil { - return "", fmt.Errorf("unsafe cache object path %q: %w", relPath, err) - } - - return filepath.Join(b.stagingDir, local), nil -} - -// makes the full reference graph of rootStorePath resolvable in the staging -// cache. `nix copy` computes the closure from the --from store, so every -// referenced narinfo must be present there or the walk fails with "path ... is -// not valid". newly-built deps are already staged by the guest, but deps that -// live only in a read cache were skipped during upload, so we backfill their -// narinfos here. only the narinfos (the reference graph) are needed: the -// destination supplies the NAR data via --substitute-on-destination. -func (b *NixStoreUploadBackend) ensureClosureStaged(ctx context.Context, rootStorePath string) error { - visited := map[string]bool{} - queue := []string{rootStorePath} - for len(queue) > 0 { - storePath := queue[0] - queue = queue[1:] - if visited[storePath] { - continue - } - visited[storePath] = true - - info, err := b.resolveStagedNarinfo(ctx, storePath) - if err != nil { - // the root must resolve (the guest just staged it); a dep we can't - // find anywhere is left for `nix copy` to surface with its own error. - if storePath == rootStorePath { - return fmt.Errorf("resolve narinfo for %s: %w", storePath, err) - } - b.logger.Warn("closure dep narinfo unresolved; leaving to nix copy", "storePath", storePath, "error", err) - continue - } - - for _, ref := range info.References { - refPath := storePrefix + ref - if refPath == storePath { - continue // self-reference - } - if !visited[refPath] { - queue = append(queue, refPath) - } - } - } - return nil -} - -// returns parsed narinfo for store path, backfilling from readUpstreams if not found -func (b *NixStoreUploadBackend) resolveStagedNarinfo(ctx context.Context, storePath string) (*narinfo, error) { - hash, _, err := parseStorePath(storePath) - if err != nil { - return nil, err - } - localPath := filepath.Join(b.stagingDir, hash+".narinfo") - info, err := readNarinfoFile(localPath) - if err == nil { - return info, nil - } - if !errors.Is(err, os.ErrNotExist) { - return nil, err - } - - // dep missing from staging because it was skipped during upload - // (lives on a read cache) so we backfill it from readUpstreams. - body, err := b.fetchUpstreamNarinfo(ctx, hash) - if err != nil { - return nil, err - } - written, err := writeNarinfoFile(localPath, body) - if err != nil { - return nil, err - } - b.logger.Debug("backfilled narinfo", "hash", hash, "bytes", written) - - return parseNarinfo(bytes.NewReader(body)) -} - -// fetches narinfo from readUpstreams -func (b *NixStoreUploadBackend) fetchUpstreamNarinfo(ctx context.Context, hash string) ([]byte, error) { - if len(b.readUpstreams) == 0 { - return nil, os.ErrNotExist - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://upstream/"+hash+".narinfo", nil) - if err != nil { - return nil, err - } - resp, err := newNarinfoExistenceTransport(b.readUpstreams, b.logger).RoundTrip(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode == http.StatusNotFound { - return nil, os.ErrNotExist - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("upstream narinfo %s: status %d", hash, resp.StatusCode) - } - return io.ReadAll(io.LimitReader(resp.Body, maxNarinfoSize+1)) -} - -// todo(dawn): ideally we don't use `nix copy` here but instead have our own -// `nix copy` impl so we don't need nix on host. but that's a far stretch goal :p -func (b *NixStoreUploadBackend) importStorePath(ctx context.Context, storePath string) error { - if err := b.ensureClosureStaged(ctx, storePath); err != nil { - return fmt.Errorf("stage closure for %s: %w", storePath, err) - } - - fromURL := url.URL{Scheme: "file", Path: b.stagingDir} - args := []string{ - "copy", - "--from", fromURL.String(), - "--to", b.targetStore, - // todo(dawn): ideally we support signing in spindle itself. - // but for now harmonia can sign things on serve so this is ok. - "--no-check-sigs", - "--substitute-on-destination", - storePath, - } - - b.logger.Info("importing staged cache path", "target", b.targetStore, "storePath", storePath) - if err := b.runner.Run(ctx, "nix", args...); err != nil { - return fmt.Errorf("nix copy to %s: %w", b.targetStore, err) - } - return nil -} - -func readNarinfoFile(path string) (*narinfo, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - return parseNarinfo(f) -} - -func writeNarinfoFile(path string, body []byte) (int64, error) { - return writeFileAtomic(path, ".tmp-narinfo", func(f *os.File) (int64, error) { - n, err := f.Write(body) - return int64(n), err - }) -} - -func writeFileAtomic(dst, tempPrefix string, write func(*os.File) (int64, error)) (written int64, err error) { - dir := filepath.Dir(dst) - if err := os.MkdirAll(dir, 0o755); err != nil { - return 0, fmt.Errorf("create directory %q: %w", dir, err) - } - - tmp, err := os.CreateTemp(dir, tempPrefix+"-*-"+filepath.Base(dst)) - if err != nil { - return 0, fmt.Errorf("create temporary file in %q: %w", dir, err) - } - tmpName := tmp.Name() - defer func() { - if err != nil { - _ = tmp.Close() - _ = os.Remove(tmpName) - } - }() - - written, err = write(tmp) - if err != nil { - return 0, err - } - if err := tmp.Sync(); err != nil { - return 0, fmt.Errorf("fsync temporary file %q: %w", tmpName, err) - } - if err := tmp.Close(); err != nil { - return 0, fmt.Errorf("close temporary file %q: %w", tmpName, err) - } - if err := os.Rename(tmpName, dst); err != nil { - return 0, fmt.Errorf("rename %q to %q: %w", tmpName, dst, err) - } - - if err := syncDir(dir); err != nil { - return 0, err - } - - return written, nil -} - -func removeFileAndSyncDir(path string) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("remove %q: %w", path, err) - } - return syncDir(filepath.Dir(path)) -} - -func syncDir(dir string) error { - dirFile, err := os.Open(dir) - if err != nil { - return fmt.Errorf("open directory %q: %w", dir, err) - } - defer dirFile.Close() - if err := dirFile.Sync(); err != nil { - return fmt.Errorf("sync directory %q: %w", dir, err) - } - return nil -} diff --git a/spindle/engines/microvm/upload_cache_nix_store_test.go b/spindle/engines/microvm/upload_cache_nix_store_test.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_nix_store_test.go +++ /dev/null @@ -1,496 +0,0 @@ -//go:build linux - -package microvm - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "slices" - "strings" - "sync" - "testing" -) - -const ( - testStoreHash = "0123456789abcdfghijklmnpqrsvwxyz" - testStorePath = "/nix/store/" + testStoreHash + "-abc-output" -) - -func TestUploadCacheBackendSchemeDispatch(t *testing.T) { - staging := t.TempDir() - logger := slog.Default() - - cases := []struct { - uploadURL string - wantErr bool - wantType string - }{ - {"https://cache.example/upload", false, "*microvm.httpUploadBackend"}, - {"http://cache.example/upload", false, "*microvm.httpUploadBackend"}, - {"ssh://cache-host", false, "*microvm.NixStoreUploadBackend"}, - {"ssh-ng://cache-host", false, "*microvm.NixStoreUploadBackend"}, - {"daemon", false, "*microvm.NixStoreUploadBackend"}, - {"local", false, "*microvm.NixStoreUploadBackend"}, - {"ftp://cache.example", true, ""}, - {"/some/path", true, ""}, - } - - for _, tc := range cases { - t.Run(tc.uploadURL, func(t *testing.T) { - backend, err := newUploadCacheBackend(tc.uploadURL, nil, staging, logger) - if tc.wantErr { - if err == nil { - t.Fatalf("expected error for %q", tc.uploadURL) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - got := fmt.Sprintf("%T", backend) - if got != tc.wantType { - t.Fatalf("backend type: got %s, want %s", got, tc.wantType) - } - }) - } -} - -func TestUploadCacheBackendEmptyURL(t *testing.T) { - backend, err := newUploadCacheBackend("", nil, t.TempDir(), slog.Default()) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if backend != nil { - t.Fatalf("expected nil backend for empty URL, got %T", backend) - } -} - -func newTestNixStoreBackend(t *testing.T, target string, runner CommandRunner) (*NixStoreUploadBackend, string) { - t.Helper() - staging := t.TempDir() - if target == "" { - target = "ssh-ng://cache-host" - } - b, err := newNixStoreUploadBackend(target, staging, nil, slog.Default(), runner) - if err != nil { - t.Fatalf("newNixStoreUploadBackend: %v", err) - } - return b, staging -} - -func mustUploadNar(t *testing.T, b *NixStoreUploadBackend, name, body string) { - t.Helper() - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/"+name, strings.NewReader(body))) - if rec.Code != http.StatusOK { - t.Fatalf("upload nar %q: got %d, want 200; body=%q", name, rec.Code, rec.Body.String()) - } -} - -func TestNixStoreBackendNixCacheInfo(t *testing.T) { - b, _ := newTestNixStoreBackend(t, "", nil) - - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/nix-cache-info", nil)) - if rec.Code != http.StatusOK { - t.Fatalf("GET status: got %d, want 200", rec.Code) - } - if !strings.Contains(rec.Body.String(), "StoreDir: /nix/store") { - t.Fatalf("cache info missing StoreDir: %q", rec.Body.String()) - } - - rec = httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodHead, "/nix-cache-info", nil)) - if rec.Code != http.StatusOK { - t.Fatalf("HEAD status: got %d, want 200", rec.Code) - } - if rec.Body.Len() != 0 { - t.Fatalf("HEAD body should be empty, got %q", rec.Body.String()) - } - - rec = httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nix-cache-info", strings.NewReader("ignored"))) - if rec.Code != http.StatusOK { - t.Fatalf("PUT /nix-cache-info status: got %d, want 200", rec.Code) - } -} - -func TestNixStoreBackendRejectsTraversalNar(t *testing.T) { - b, staging := newTestNixStoreBackend(t, "", nil) - - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/../../evil", strings.NewReader("bad"))) - if rec.Code != http.StatusBadRequest { - t.Fatalf("traversal nar status: got %d, want 400", rec.Code) - } - - if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "evil")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("traversal nar escaped staging dir: %v", err) - } -} - -func TestNixStoreBackendRejectsOversizedNarUpload(t *testing.T) { - b, staging := newTestNixStoreBackend(t, "", nil) - b.maxNarUploadSize = 3 - - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/nar/foo.nar", strings.NewReader("four"))) - if rec.Code != http.StatusRequestEntityTooLarge { - t.Fatalf("oversized nar status: got %d, want 413; body=%q", rec.Code, rec.Body.String()) - } - - if _, err := os.Stat(filepath.Join(staging, "nar", "foo.nar")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("oversized nar should not have been staged: %v", err) - } -} - -func TestNixStoreBackendNarinfoRequiresExistingNar(t *testing.T) { - b, _ := newTestNixStoreBackend(t, "", nil) - - narinfo := "StorePath: " + testStorePath + "\nURL: nar/abc.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) - if rec.Code != http.StatusBadRequest { - t.Fatalf("narinfo before nar status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) - } -} - -type fakeRunner struct { - mu sync.Mutex - calls [][]string - nextErr error -} - -func (f *fakeRunner) Run(ctx context.Context, name string, args ...string) error { - f.mu.Lock() - defer f.mu.Unlock() - call := append([]string{name}, args...) - f.calls = append(f.calls, call) - return f.nextErr -} - -func (f *fakeRunner) Calls() [][]string { - f.mu.Lock() - defer f.mu.Unlock() - return slices.Clone(f.calls) -} - -func TestNixStoreBackendImportsNarinfoImmediately(t *testing.T) { - runner := &fakeRunner{} - b, staging := newTestNixStoreBackend(t, "ssh-ng://spindle-upload@cache-host", runner) - - mustUploadNar(t, b, "foo.nar.zst", "nar-body") - - narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) - if rec.Code != http.StatusOK { - t.Fatalf("PUT narinfo status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - - calls := runner.Calls() - if len(calls) != 1 { - t.Fatalf("expected 1 nix copy call, got %d", len(calls)) - } - call := calls[0] - wantFrom := (&url.URL{Scheme: "file", Path: staging}).String() - want := []string{ - "nix", - "copy", - "--from", wantFrom, - "--to", "ssh-ng://spindle-upload@cache-host", - "--no-check-sigs", - "--substitute-on-destination", - testStorePath, - } - if !slices.Equal(call, want) { - t.Fatalf("nix copy args:\n got: %v\nwant: %v", call, want) - } - - data, err := os.ReadFile(filepath.Join(staging, testStoreHash+".narinfo")) - if err != nil { - t.Fatalf("staged narinfo missing: %v", err) - } - if string(data) != narinfo { - t.Fatalf("staged narinfo contents: got %q, want %q", string(data), narinfo) - } -} - -func TestNixStoreBackendBackfillsClosureDepNarinfo(t *testing.T) { - const depHash = "abcdfghijklmnpqrsvwxyz0123456789" - depNarinfo := "StorePath: /nix/store/" + depHash + "-dep\nURL: nar/dep.nar.zst\nNarHash: sha256:dep\nNarSize: 1\n" - - var depRequests int - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/"+depHash+".narinfo" { - depRequests++ - _, _ = io.WriteString(w, depNarinfo) - return - } - w.WriteHeader(http.StatusNotFound) - })) - defer upstream.Close() - - upURL, err := url.Parse(upstream.URL) - if err != nil { - t.Fatal(err) - } - - runner := &fakeRunner{} - staging := t.TempDir() - b, err := newNixStoreUploadBackend("ssh-ng://cache-host", staging, []CacheUpstream{{url: upURL}}, slog.Default(), runner) - if err != nil { - t.Fatalf("newNixStoreUploadBackend: %v", err) - } - - mustUploadNar(t, b, "foo.nar.zst", "nar-body") - - // the top path references the dep, which is absent from staging. - narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\nReferences: " + depHash + "-dep\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) - if rec.Code != http.StatusOK { - t.Fatalf("PUT narinfo status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } - - // the dep narinfo must have been fetched from the upstream and staged... - if depRequests == 0 { - t.Fatalf("expected the dep narinfo to be fetched from the upstream") - } - staged, err := os.ReadFile(filepath.Join(staging, depHash+".narinfo")) - if err != nil { - t.Fatalf("dep narinfo not backfilled into staging: %v", err) - } - if string(staged) != depNarinfo { - t.Fatalf("backfilled dep narinfo contents: got %q, want %q", string(staged), depNarinfo) - } - - // ...and the import still copies just the requested top path. - calls := runner.Calls() - if len(calls) != 1 || calls[0][len(calls[0])-1] != testStorePath { - t.Fatalf("expected a single nix copy for %s, got %v", testStorePath, calls) - } -} - -func TestNixStoreBackendRemovesNarinfoOnImportFailure(t *testing.T) { - runner := &fakeRunner{nextErr: errors.New("nix copy failed")} - b, staging := newTestNixStoreBackend(t, "ssh://cache-host", runner) - - mustUploadNar(t, b, "foo.nar.zst", "nar-body") - - narinfo := "StorePath: " + testStorePath + "\nURL: nar/foo.nar.zst\nNarHash: sha256:abc\nNarSize: 123\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(narinfo))) - if rec.Code != http.StatusBadGateway { - t.Fatalf("failed import status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) - } - - if _, err := os.Stat(filepath.Join(staging, testStoreHash+".narinfo")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("narinfo should be removed after failed import: %v", err) - } -} - -func TestNixStoreBackendNarinfoReadUpstream(t *testing.T) { - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/miss.narinfo" { - w.WriteHeader(http.StatusNotFound) - return - } - _, _ = io.WriteString(w, "StorePath: /nix/store/upstream\nURL: nar/upstream.nar\nNarHash: sha256:up\nNarSize: 1\n") - })) - defer upstream.Close() - - upURL, err := url.Parse(upstream.URL) - if err != nil { - t.Fatal(err) - } - - staging := t.TempDir() - b, err := newNixStoreUploadBackend("ssh://cache-host", staging, []CacheUpstream{{url: upURL}}, slog.Default(), nil) - if err != nil { - t.Fatalf("newNixStoreUploadBackend: %v", err) - } - - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/present.narinfo", nil)) - if rec.Code != http.StatusOK { - t.Fatalf("GET upstream-present narinfo status: got %d, want 200", rec.Code) - } - if !strings.Contains(rec.Body.String(), "/nix/store/upstream") { - t.Fatalf("unexpected upstream narinfo body: %q", rec.Body.String()) - } - - rec = httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/miss.narinfo", nil)) - if rec.Code != http.StatusNotFound { - t.Fatalf("GET upstream-missing narinfo status: got %d, want 404", rec.Code) - } - -} - -func TestNixStoreBackendRejectsInvalidLocalNarinfo(t *testing.T) { - b, staging := newTestNixStoreBackend(t, "", nil) - - if err := os.WriteFile(filepath.Join(staging, testStoreHash+".narinfo"), []byte("not-a-narinfo\n"), 0o644); err != nil { - t.Fatalf("write invalid staged narinfo: %v", err) - } - - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/"+testStoreHash+".narinfo", nil)) - if rec.Code != http.StatusInternalServerError { - t.Fatalf("invalid local narinfo status: got %d, want 500; body=%q", rec.Code, rec.Body.String()) - } -} - -func TestNixStoreBackendNarinfoValidation(t *testing.T) { - b, _ := newTestNixStoreBackend(t, "", nil) - - mustUploadNar(t, b, "x.nar", "x") - - cases := []struct { - name string - body string - wantErr string - }{ - { - name: "missing StorePath", - body: "URL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "StorePath", - }, - { - name: "bad StorePath", - body: "StorePath: /tmp/evil\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "invalid StorePath", - }, - { - name: "malformed StorePath", - body: "StorePath: /nix/store/not-a-real-store-path\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "invalid StorePath", - }, - { - name: "missing URL", - body: "StorePath: " + testStorePath + "\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "URL", - }, - { - name: "absolute URL", - body: "StorePath: " + testStorePath + "\nURL: /etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "URL", - }, - { - name: "traversal URL", - body: "StorePath: " + testStorePath + "\nURL: nar/../../etc/passwd\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "URL", - }, - { - name: "non nar URL", - body: "StorePath: " + testStorePath + "\nURL: nix-cache-info\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "nar/", - }, - { - name: "nested nar URL", - body: "StorePath: " + testStorePath + "\nURL: nar/dir/x.nar\nNarHash: sha256:x\nNarSize: 1\n", - wantErr: "safe nar object path", - }, - { - name: "missing NarHash", - body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarSize: 1\n", - wantErr: "NarHash", - }, - { - name: "bad NarSize", - body: "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: huge\n", - wantErr: "NarSize", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/"+testStoreHash+".narinfo", strings.NewReader(tc.body))) - if rec.Code != http.StatusBadRequest { - t.Fatalf("status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) - } - if !strings.Contains(rec.Body.String(), tc.wantErr) { - t.Fatalf("body %q should mention %q", rec.Body.String(), tc.wantErr) - } - }) - } -} - -func TestNixStoreBackendUploadCacheInfoFileExists(t *testing.T) { - staging := t.TempDir() - if _, err := newNixStoreUploadBackend("ssh://host", staging, nil, slog.Default(), nil); err != nil { - t.Fatalf("newNixStoreUploadBackend: %v", err) - } - data, err := os.ReadFile(filepath.Join(staging, "nix-cache-info")) - if err != nil { - t.Fatalf("nix-cache-info missing: %v", err) - } - if !bytes.Contains(data, []byte("StoreDir: /nix/store")) { - t.Fatalf("unexpected nix-cache-info: %q", string(data)) - } -} - -func TestNixStoreBackendRejectsTraversalNarinfoPath(t *testing.T) { - b, staging := newTestNixStoreBackend(t, "", nil) - - body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/../etc/passwd.narinfo", strings.NewReader(body))) - if rec.Code != http.StatusBadRequest { - t.Fatalf("traversal narinfo status: got %d, want 400", rec.Code) - } - - if _, err := os.Stat(filepath.Join(filepath.Dir(staging), "etc", "passwd.narinfo")); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("traversal narinfo escaped staging dir: %v", err) - } -} - -func TestNixStoreBackendRejectsNarinfoFilenameHashMismatch(t *testing.T) { - b, _ := newTestNixStoreBackend(t, "", nil) - mustUploadNar(t, b, "x.nar", "x") - - body := "StorePath: " + testStorePath + "\nURL: nar/x.nar\nNarHash: sha256:x\nNarSize: 1\n" - rec := httptest.NewRecorder() - b.ServeHTTP(rec, httptest.NewRequest(http.MethodPut, "/11111111111111111111111111111111.narinfo", strings.NewReader(body))) - if rec.Code != http.StatusBadRequest { - t.Fatalf("mismatched narinfo status: got %d, want 400; body=%q", rec.Code, rec.Body.String()) - } - if !strings.Contains(rec.Body.String(), "filename does not match") { - t.Fatalf("unexpected body: %q", rec.Body.String()) - } -} - -func TestParseNarinfoAcceptsLargeReferencesLine(t *testing.T) { - var refs []string - for range 12000 { - refs = append(refs, "0123456789abcdfghijklmnpqrsvwxy-ref") - } - - body := strings.Join([]string{ - "StorePath: " + testStorePath, - "URL: nar/x.nar", - "NarHash: sha256:abc", - "NarSize: 1", - "References: " + strings.Join(refs, " "), - "", - }, "\n") - - info, err := parseNarinfo(strings.NewReader(body)) - if err != nil { - t.Fatalf("parseNarinfo failed for large references line: %v", err) - } - if info.StorePath != testStorePath { - t.Fatalf("StorePath: got %q, want %q", info.StorePath, testStorePath) - } -} diff --git a/spindle/engines/microvm/upload_cache_proxy.go b/spindle/engines/microvm/upload_cache_proxy.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_proxy.go +++ /dev/null @@ -1,160 +0,0 @@ -//go:build linux - -package microvm - -import ( - "context" - "errors" - "fmt" - "log/slog" - "net" - "net/http" - "net/url" - "strings" - "time" - - "github.com/mdlayher/vsock" -) - -type UploadCacheBackend interface { - http.Handler - Close() error -} - -type UploadCacheProxy struct { - port uint32 - - ln *vsock.Listener - server *http.Server - backend UploadCacheBackend -} - -func StartUploadCacheProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []CacheUpstream, stagingDir string, logger *slog.Logger) (*UploadCacheProxy, error) { - if strings.TrimSpace(uploadURL) == "" { - return nil, nil - } - - if logger == nil { - logger = slog.Default() - } - logger = logger.With("where", "upload_cache_proxy", "cid", cid, "uploadURL", uploadURL) - - backend, err := newUploadCacheBackend(uploadURL, readUpstreams, stagingDir, logger) - if err != nil { - return nil, err - } - - ln, port, err := listenRandomVsockUploadPort(ctx) - if err != nil { - return nil, fmt.Errorf("listen for cache upload proxy: %w", err) - } - - proxy := &UploadCacheProxy{ - port: port, - ln: ln, - backend: backend, - } - proxy.server = &http.Server{ - Handler: backend, - Protocols: cacheProxyProtocols(), - ReadHeaderTimeout: 30 * time.Second, - } - - filtered := &cidFilteredVsockListener{ - Listener: ln, - cid: cid, - logger: logger, - } - go func() { - if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { - logger.Warn("upload cache proxy stopped", "port", port, "error", err) - } - }() - - logger.Info("started upload cache proxy", "port", port, "target", uploadURL, "readUpstreams", len(readUpstreams)) - return proxy, nil -} - -func newUploadCacheBackend(uploadURL string, readUpstreams []CacheUpstream, stagingDir string, logger *slog.Logger) (UploadCacheBackend, error) { - if strings.TrimSpace(uploadURL) == "" { - return nil, nil - } - - target, err := url.Parse(uploadURL) - if err != nil { - return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) - } - - switch target.Scheme { - case "http", "https": - if target.Host == "" { - return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) - } - return newHTTPUploadProxyBackend(target, readUpstreams, logger), nil - - case "ssh", "ssh-ng": - return newNixStoreUploadBackend(target.String(), stagingDir, readUpstreams, logger, nil) - - case "": - switch uploadURL { - case "daemon", "local": - return newNixStoreUploadBackend(uploadURL, stagingDir, readUpstreams, logger, nil) - default: - return nil, fmt.Errorf("unsupported upload URL %q", uploadURL) - } - - default: - return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q", uploadURL, target.Scheme) - } -} - -func (p *UploadCacheProxy) Port() uint32 { - if p == nil { - return 0 - } - return p.port -} - -func (p *UploadCacheProxy) Close() error { - if p == nil { - return nil - } - - var closeErr error - if p.server != nil { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) - cancel() - p.server = nil - } - if p.ln != nil { - closeErr = errors.Join(closeErr, p.ln.Close()) - p.ln = nil - } - if p.backend != nil { - closeErr = errors.Join(closeErr, p.backend.Close()) - } - return closeErr -} - -func listenRandomVsockUploadPort(ctx context.Context) (*vsock.Listener, uint32, error) { - var lastErr error - for range 32 { - port, err := randomVsockPort() - if err != nil { - return nil, 0, err - } - ln, err := vsock.ListenContextID(vsock.Host, port, nil) - if err == nil { - return ln, port, nil - } - lastErr = err - - select { - case <-ctx.Done(): - return nil, 0, ctx.Err() - default: - } - } - return nil, 0, fmt.Errorf("listen on random vsock upload port: %w", lastErr) -} diff --git a/spindle/engines/microvm/upload_cache_proxy_test.go b/spindle/engines/microvm/upload_cache_proxy_test.go deleted file mode 100644 --- a/spindle/engines/microvm/upload_cache_proxy_test.go +++ /dev/null @@ -1,130 +0,0 @@ -//go:build linux - -package microvm - -import ( - "io" - "log/slog" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" -) - -func TestUploadProxyRewritesHostAndAuth(t *testing.T) { - var upstreamHost string - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.Host != upstreamHost { - t.Errorf("host: got %q, want %q", req.Host, upstreamHost) - } - if req.URL.Path != "/sub/abc.narinfo" { - t.Errorf("path: got %q, want /sub/abc.narinfo", req.URL.Path) - } - if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { - t.Errorf("basic auth: got %q/%q/%v, want dawn/hunter2/true", user, pass, ok) - } - _, _ = io.WriteString(w, "ok") - })) - defer upstream.Close() - upstreamHost = strings.TrimPrefix(upstream.URL, "http://") - - target, err := url.Parse("http://dawn:woof@" + upstreamHost + "/sub/") - if err != nil { - t.Fatal(err) - } - - req := httptest.NewRequest(http.MethodPut, "http://127.0.0.1:10501/abc.narinfo", strings.NewReader("narinfo")) - req.Host = "127.0.0.1:10501" - rec := httptest.NewRecorder() - uploadProxyHandler(target, nil, slog.Default()).ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) - } -} - -func mustParseURL(t *testing.T, raw string) *url.URL { - t.Helper() - u, err := url.Parse(raw) - if err != nil { - t.Fatalf("parse %q: %v", raw, err) - } - return u -} - -func TestUploadProxySkipsNarinfoAvailableUpstream(t *testing.T) { - var uploadHits int - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - uploadHits++ - w.WriteHeader(http.StatusNotFound) - })) - defer target.Close() - - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if req.URL.Path != "/abc.narinfo" { - t.Errorf("upstream path: got %q, want /abc.narinfo", req.URL.Path) - } - _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") - })) - defer upstream.Close() - - handler := uploadProxyHandler( - mustParseURL(t, target.URL), - []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, - slog.Default(), - ) - - req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200 (so nix treats the path as present and skips upload)", rec.Code) - } - if !strings.Contains(rec.Body.String(), "StorePath: /nix/store/abc") { - t.Fatalf("body: got %q, want the upstream narinfo body", rec.Body.String()) - } -} - -func TestUploadProxyUploadsNarinfoNobodyHas(t *testing.T) { - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(http.StatusNotFound) - })) - defer target.Close() - upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(http.StatusNotFound) - })) - defer upstream.Close() - - handler := uploadProxyHandler( - mustParseURL(t, target.URL), - []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, - slog.Default(), - ) - - req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, req) - - if rec.Code != http.StatusNotFound { - t.Fatalf("status: got %d, want 404 (so nix uploads the path)", rec.Code) - } -} - -func TestUploadProxySkipsNarinfoAlreadyOnTarget(t *testing.T) { - target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") - })) - defer target.Close() - - handler := uploadProxyHandler(mustParseURL(t, target.URL), nil, slog.Default()) - - req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) - rec := httptest.NewRecorder() - handler.ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Fatalf("status: got %d, want 200", rec.Code) - } -} diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go --- a/spindle/engines/microvm/vm.go +++ b/spindle/engines/microvm/vm.go @@ -161,20 +161,20 @@ } type workflowState struct { - ImageSpec ImageSpec - ImageSpecPath string - Config manifestConfig - ConfigKey string - Image string - CacheReadURLs []string - CacheTrustedPublicKeys []string - VM VMHandle - Agent *AgentSession - ReadCache *ReadCacheProxy - UploadCache *UploadCacheProxy - DNSProxy *DNSProxy - WorkDir string - NixOSToplevelCache nixosToplevelCacheStore + ImageSpec ImageSpec + ImageSpecPath string + Config manifestConfig + ConfigKey string + Image string + SubstituterReadURLs []string + SubstituterTrustedPublicKeys []string + VM VMHandle + Agent *AgentSession + Substituter *SubstituterProxy + SubstituterUpload *SubstituterUploadProxy + DNSProxy *DNSProxy + WorkDir string + NixOSToplevels nixosToplevelStore } func (e *Engine) cleanupState(ctx context.Context, wid models.WorkflowId, state *workflowState) error { @@ -191,8 +191,8 @@ } err = errors.Join(err, e.shutdownVM(ctx, wid, state)) err = errors.Join(err, closeIO(&state.Agent)) - err = errors.Join(err, closeIO(&state.ReadCache)) - err = errors.Join(err, closeIO(&state.UploadCache)) + err = errors.Join(err, closeIO(&state.Substituter)) + err = errors.Join(err, closeIO(&state.SubstituterUpload)) err = errors.Join(err, closeIO(&state.DNSProxy)) err = errors.Join(err, removeWorkDir(state)) return err @@ -203,7 +203,7 @@ return nil } - drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) + drainCtx, cancel := context.WithTimeout(ctx, substituterDrainTimeout) defer cancel() if state.Agent != nil { -- tangled.sh