From 67b149bf848a64849bb35ec23d55c121f4971c47 Mon Sep 17 00:00:00 2001 From: dawn Date: Sun, 26 Jul 2026 22:40:53 +0000 Subject: [PATCH] spindle/microvm: cap each workflow vm's cpu and io in its cgroup Signed-off-by: dawn --- docs/DOCS.md | 6 ++++++ spindle/config/config.go | 6 ++++++ spindle/engines/microvm/cgroup.go | 28 ++++++++++++++++++++++++++++ spindle/engines/microvm/cid_test.go | 15 +++++++++++++++ spindle/engines/microvm/engine.go | 20 ++++++++++++++------ 5 file(s) changed, 69 insertion(s)(+), 6 deletion(s)(-) diff --git a/docs/DOCS.md b/docs/DOCS.md --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -1495,6 +1495,12 @@ per workflow cgroup (default: `4096`). - `SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB`: Max swap per workflow cgroup (default: `0`, no swap). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_CPU_MAX_PERCENT`: Max CPU + quota per workflow cgroup, as a percentage of one core + (default: `0`, which caps each VM at its configured vCPU + count, a negative value disables the CPU limit). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_IO_WEIGHT`: IO weight per + workflow cgroup, 1-10000 (default: `0`, IO unlimited). - `SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB`: Memory protected for spindle itself so it isn't OOM-killed before the workflows (default: `512`). diff --git a/spindle/config/config.go b/spindle/config/config.go --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -83,6 +83,12 @@ CgroupParent string `env:"CGROUP_PARENT, default=self"` CgroupPidsMax int64 `env:"CGROUP_PIDS_MAX, default=4096"` CgroupSwapMaxMiB *int64 `env:"CGROUP_SWAP_MAX_MIB"` + // cpu.max quota as a percentage of one core. 0 caps each vm at its + // configured vcpu count, negative disables the limit + CgroupCPUMaxPercent int64 `env:"CGROUP_CPU_MAX_PERCENT, default=0"` + // io.weight for workflow cgroups (1-10000, kernel default 100). 0 leaves + // io unlimited + CgroupIOWeight uint64 `env:"CGROUP_IO_WEIGHT, default=0"` // memory.min that will get assigned to the supervisor (spindle itself) cgroup CgroupSupervisorMemoryMinMiB int64 `env:"CGROUP_SUPERVISOR_MEMORY_MIN_MIB, default=512"` } diff --git a/spindle/engines/microvm/cgroup.go b/spindle/engines/microvm/cgroup.go --- a/spindle/engines/microvm/cgroup.go +++ b/spindle/engines/microvm/cgroup.go @@ -7,6 +7,7 @@ "os" "path/filepath" "regexp" + "strconv" "strings" "syscall" @@ -32,6 +33,12 @@ MemoryMaxMiB int64 SwapMaxMiB *int64 PidsMax int64 + // cpu.max quota as a percentage of one core (100 = one core). <= 0 + // leaves cpu unlimited + CPUQuotaPercent int64 + // io.weight (1-10000). 0 leaves io unlimited, written directly since + // this cgroup2 lib only models io.bfq.weight/io.max + IOWeight uint64 } type CgroupParent struct { @@ -102,6 +109,18 @@ return nil, fmt.Errorf("create cgroup %q: %w", name, err) } + if limits.IOWeight > 0 { + if err := manager.ToggleControllers([]string{"io"}, cgroup2.Enable); err != nil { + _ = manager.Delete() + return nil, fmt.Errorf("enable io controller for cgroup %q: %w", name, err) + } + ioWeightPath := filepath.Join(limits.Parent.mountpoint, strings.TrimPrefix(limits.Parent.group, "/"), name, "io.weight") + if err := os.WriteFile(ioWeightPath, []byte(strconv.FormatUint(limits.IOWeight, 10)), 0); err != nil { + _ = manager.Delete() + return nil, fmt.Errorf("write io.weight for cgroup %q: %w", name, err) + } + } + if logger != nil { logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group) } @@ -126,6 +145,10 @@ } if limits.PidsMax > 0 { resources.Pids = &cgroup2.Pids{Max: limits.PidsMax} + } + if limits.CPUQuotaPercent > 0 { + quota := limits.CPUQuotaPercent * 1000 // 100% of one 100000us period + resources.CPU = &cgroup2.CPU{Max: cgroup2.NewCPUMax("a, nil)} } return resources } @@ -178,6 +201,11 @@ // in the "/" parent's subtree. this fails EBUSY at a populated cgroup // namespace root (no-internal-process constraint); the real root is exempt. func probeRootSubtreeControl(mountpoint string) error { + // cpu/io may be unavailable (eg. no CONFIG_BLK_CGROUP), memory/pids + // are the hard requirement + if err := os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids +cpu +io"), 0); err == nil { + return nil + } return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0) } diff --git a/spindle/engines/microvm/cid_test.go b/spindle/engines/microvm/cid_test.go --- a/spindle/engines/microvm/cid_test.go +++ b/spindle/engines/microvm/cid_test.go @@ -28,3 +28,18 @@ t.Errorf("key not deterministic: %q vs %q", a, again) } } + +func TestCgroupResourcesCPUQuota(t *testing.T) { + r := cgroupResources(CgroupLimits{CPUQuotaPercent: 200}) + if r.CPU == nil { + t.Fatal("cpu quota should produce a cpu controller config") + } + if got := string(r.CPU.Max); got != "200000 100000" { + t.Errorf("cpu.max = %q, want %q", got, "200000 100000") + } + + r = cgroupResources(CgroupLimits{}) + if r.CPU != nil { + t.Errorf("no quota should leave cpu unlimited, got %v", r.CPU) + } +} diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -608,12 +608,20 @@ func (e *Engine) cgroupLimits(wid models.WorkflowId, spec ImageSpec) CgroupLimits { cfg := e.cfg.MicroVMPipelines + cpuQuotaPercent := cfg.CgroupCPUMaxPercent + if cpuQuotaPercent == 0 { + // a vm can only use its vcpus anyway, but without a cap those + // threads still get full host cores and starve every other tenant + cpuQuotaPercent = int64(spec.VCPUs) * 100 + } return CgroupLimits{ - Enabled: cfg.EnableCgroups, - Parent: e.cgroupParent, - Name: "workflow-" + wid.String(), - MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, - SwapMaxMiB: cfg.CgroupSwapMaxMiB, - PidsMax: cfg.CgroupPidsMax, + Enabled: cfg.EnableCgroups, + Parent: e.cgroupParent, + Name: "workflow-" + wid.String(), + MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, + SwapMaxMiB: cfg.CgroupSwapMaxMiB, + PidsMax: cfg.CgroupPidsMax, + CPUQuotaPercent: cpuQuotaPercent, + IOWeight: cfg.CgroupIOWeight, } } -- tangled.sh