Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297package microvm
import ( "errors" "fmt" "log/slog" "os" "path/filepath" "regexp" "strings" "syscall"
cgroups "github.com/containerd/cgroups/v3" "github.com/containerd/cgroups/v3/cgroup2" "github.com/prometheus/procfs")
var ( cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`) cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`))
const ( cgroupParentSelf = "self" supervisorCgroupName = "supervisor")
type CgroupLimits struct { Enabled bool Parent *CgroupParent Name string MemoryMaxMiB int64 SwapMaxMiB *int64 PidsMax int64}
type CgroupParent struct { root *cgroup2.Manager mountpoint string group string}
type CgroupHandle struct { manager *cgroup2.Manager}
func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) { if parent == "" { parent = cgroupParentSelf } if cgroups.Mode() != cgroups.Unified { return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode") }
mountpoint, group, err := resolveCgroupParent(parent) if err != nil { return nil, err } if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil { return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err) }
root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint)) if err != nil { return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err) }
if group != "/" { if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil { return nil, err } } else if err := probeRootSubtreeControl(mountpoint); err != nil { if !errors.Is(err, syscall.EBUSY) { return nil, fmt.Errorf("enable controllers in subtree_control of cgroup root %q: %w", mountpoint, err) } // populated namespace root, not the real root: vacate it too if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil { return nil, err } }
if logger != nil { logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group) } return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil}
func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) { if !limits.Enabled { return nil, nil } if limits.Parent == nil || limits.Parent.root == nil { return nil, fmt.Errorf("cgroup parent is not initialized") } name := sanitizeCgroupName(limits.Name) if name == "" { return nil, fmt.Errorf("cgroup name is empty") }
manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits)) if err != nil { return nil, fmt.Errorf("create cgroup %q: %w", name, err) }
if logger != nil { logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group) } return &CgroupHandle{manager: manager}, nil}
func cgroupResources(limits CgroupLimits) *cgroup2.Resources { resources := &cgroup2.Resources{} if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil { memory := &cgroup2.Memory{} if limits.MemoryMaxMiB > 0 { maxBytes := limits.MemoryMaxMiB * 1024 * 1024 memory.Max = &maxBytes } if limits.SwapMaxMiB != nil { swapBytes := *limits.SwapMaxMiB * 1024 * 1024 memory.Swap = &swapBytes } oomGroup := true memory.OOMGroup = &oomGroup resources.Memory = memory } if limits.PidsMax > 0 { resources.Pids = &cgroup2.Pids{Max: limits.PidsMax} } return resources}
func supervisorResources(memoryMinMiB int64) *cgroup2.Resources { if memoryMinMiB <= 0 { return nil } minBytes := memoryMinMiB * 1024 * 1024 return &cgroup2.Resources{ Memory: &cgroup2.Memory{Min: &minBytes}, }}
func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error { if h == nil || h.manager == nil { return nil } if pid <= 0 { return fmt.Errorf("invalid pid %d", pid) } if err := h.manager.AddProc(uint64(pid)); err != nil { return fmt.Errorf("add pid %d to cgroup: %w", pid, err) } if logger != nil { logger.Info("added process to microVM cgroup", "pid", pid) } return nil}
func (h *CgroupHandle) Close() error { if h == nil || h.manager == nil { return nil } return h.manager.Delete()}
func (h *CgroupHandle) OOMKilled() bool { if h == nil || h.manager == nil { return false } metrics, err := h.manager.Stat() if err != nil || metrics == nil || metrics.MemoryEvents == nil { return false } return metrics.MemoryEvents.OomKill > 0}
// probeRootSubtreeControl enables the domain controllers the engine needs// in the "/" parent's subtree. this fails EBUSY at a populated cgroup// namespace root (no-internal-process constraint); the real root is exempt.func probeRootSubtreeControl(mountpoint string) error { return os.WriteFile(filepath.Join(mountpoint, "cgroup.subtree_control"), []byte("+memory +pids"), 0)}
func resolveCgroupParent(parent string) (string, string, error) { mountpoint, err := cgroup2Mountpoint() if err != nil { return "", "", err }
if parent == "" || parent == cgroupParentSelf { group, err := selfCgroupV2Path() if err != nil { return "", "", err } return mountpoint, group, nil } if !filepath.IsAbs(parent) { return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent) }
cleanParent := filepath.Clean(parent) rel, err := filepath.Rel(mountpoint, cleanParent) if err != nil { return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err) } if rel == ".." || strings.HasPrefix(rel, "../") { return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint) } if rel == "." { return mountpoint, "/", nil }
group := "/" + filepath.ToSlash(rel) if err := cgroup2.VerifyGroupPath(group); err != nil { return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err) } return mountpoint, group, nil}
func cgroup2Mountpoint() (string, error) { mounts, err := procfs.GetMounts() if err != nil { return "", fmt.Errorf("read procfs mountinfo: %w", err) } for _, mount := range mounts { if mount.FSType == "cgroup2" { return mount.MountPoint, nil } } return "", fmt.Errorf("cgroup v2 mountpoint not found")}
func selfCgroupV2Path() (string, error) { self, err := procfs.Self() if err != nil { return "", fmt.Errorf("open procfs self: %w", err) } groups, err := self.Cgroups() if err != nil { return "", fmt.Errorf("read procfs self cgroups: %w", err) } for _, group := range groups { if group.HierarchyID != 0 { continue } path := group.Path if path == "" { path = "/" } if err := cgroup2.VerifyGroupPath(path); err != nil { return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err) } return path, nil } return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry")}
func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error { procs, err := parent.Procs(false) if err != nil { return fmt.Errorf("list parent cgroup processes: %w", err) }
// first create with empty resources supervisor, err := parent.NewChild(supervisorCgroupName, &cgroup2.Resources{}) if err != nil { return fmt.Errorf("create supervisor cgroup: %w", err) }
// move procs for _, pid := range procs { if err := supervisor.AddProc(pid); err != nil { return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err) } }
// now apply resources. we can't do this while parent has procs still if res := supervisorResources(supervisorMemoryMinMiB); res != nil { // we use a "new" parent here, this is so we enable subtree_control. // .Update() does not work here... if _, err = parent.NewChild(supervisorCgroupName, res); err != nil { return fmt.Errorf("apply supervisor cgroup resources: %w", err) } }
if logger != nil && len(procs) > 0 { logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs)) } return nil}
func sanitizeCgroupName(name string) string { name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-") name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-") return strings.Trim(name, "-_.")}