diff --git a/docker-compose.mill.yml b/docker-compose.mill.yml new file mode 100644 index 00000000..6fffb7fe --- /dev/null +++ b/docker-compose.mill.yml @@ -0,0 +1,94 @@ +# Mill-mode overlay for localinfra. +# +# The base docker-compose.yml runs a single standalone spindle. This overlay +# turns that same spindle into a *mill host* (it places jobs instead of running +# them) and adds one *executor* spindle that dials the mill over a websocket and +# runs the real microvm/nixery engines. Repos still point at spindle.* exactly +# as before — only where the job executes changes. +# +# standalone (default): docker compose --profile linux up +# mill + executor: docker compose -f docker-compose.yml -f docker-compose.mill.yml --profile linux up +# +# The shared secret is a dev default; override with SPINDLE_MILL_SHARED_SECRET. + +services: + # the existing front door becomes the mill host. it keeps its identity, + # jetstream subscription and /events + /logs endpoints; its microvm/nixery + # engines are swapped for the mill placement engine. + spindle: + environment: + SPINDLE_ROLE: mill + SPINDLE_MILL_SHARED_SECRET: "${SPINDLE_MILL_SHARED_SECRET:-localmill}" + + # a normal spindle that runs the real engines, but takes its work from the + # mill instead of being pointed at directly. dials out (NAT-friendly), so it + # needs no caddy alias and no PDS record. + spindle-executor: + profiles: ["linux"] + build: + context: . + dockerfile: localinfra/spindle.Dockerfile + restart: unless-stopped + environment: + SPINDLE_ROLE: executor + SPINDLE_SERVER_HOSTNAME: executor.tngl.boltless.dev + SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db + SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev + SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe + SPINDLE_SERVER_DEV: "true" + SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev + SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db + SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev + SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images + SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays + # distinct vsock agent port from the base spindle, in case both ever run + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" + SPINDLE_S3_LOG_BUCKET: "" + SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" + SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 + SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= + SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload + # join the mill instead of being pointed at directly. plain ws to the + # mill container on the internal network (no caddy / TLS in the loop). + SPINDLE_MILL_URL: ws://spindle:6555/mill + SPINDLE_MILL_SHARED_SECRET: "${SPINDLE_MILL_SHARED_SECRET:-localmill}" + devices: + - /dev/vsock:/dev/vsock + - /dev/kvm:/dev/kvm + - /dev/vhost-vsock:/dev/vhost-vsock + - /dev/net/tun:/dev/net/tun + cap_add: + - NET_ADMIN + - SYS_ADMIN + security_opt: + - label=disable + - seccomp=unconfined + volumes: + - spindle-executor-data:/var/lib/spindle + - spindle-executor-logs:/var/log/spindle + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s + depends_on: + plc: + condition: service_started + jetstream: + condition: service_started + init-accounts: + condition: service_completed_successfully + ncps: + condition: service_started + spindle: + condition: service_healthy + networks: [tngl] + +volumes: + spindle-executor-data: + spindle-executor-logs: diff --git a/spindle/mill/integration_test.go b/spindle/mill/integration_test.go new file mode 100644 index 00000000..51890e03 --- /dev/null +++ b/spindle/mill/integration_test.go @@ -0,0 +1,111 @@ +package mill + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "tangled.org/core/api/tangled" + "tangled.org/core/notifier" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/mill/executor" + "tangled.org/core/spindle/models" +) + +func TestEndToEndDummyJob(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + + millDir := t.TempDir() + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) + if err != nil { + t.Fatalf("mill db: %v", err) + } + bn := notifier.New() + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) + mill.Attach(bdb, &bn) + + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) + defer srv.Close() + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + + execDir := t.TempDir() + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) + if err != nil { + t.Fatalf("exec db: %v", err) + } + en := notifier.New() + cfg := &config.Config{} + cfg.Server.LogDir = execDir + cfg.Server.Hostname = "exec-1" + cfg.Mill.URL = wsURL + cfg.Mill.Seats = 2 + + engines := map[string]models.Engine{"dummy": dummy.New(l)} + exec := executor.New(cfg, engines, edb, &en, l) + go exec.Connect(ctx) + + be := NewEngine("dummy", mill) + twf := tangled.Pipeline_Workflow{ + Name: "build", + Raw: "steps:\n - name: hello\n command: echo hi\n", + } + wf, err := be.InitWorkflow(twf, tangled.Pipeline{}) + if err != nil { + t.Fatalf("InitWorkflow: %v", err) + } + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + + placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) + defer placeCancel() + + slot, err := mill.place(placeCtx, "dummy", wid, wf) + if err != nil { + t.Fatalf("place: %v", err) + } + defer slot.Release() + + if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { + t.Fatalf("commitAndWait: %v, want success", err) + } + + if !waitForStatus(t, bdb, wid, "running") { + t.Fatal("mill never saw relayed running status") + } +} + +func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) bool { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + aturi := string(wid.PipelineId.AtUri()) + for time.Now().Before(deadline) { + evs, err := d.GetEvents(0, 1000) + if err != nil { + t.Fatalf("GetEvents: %v", err) + } + for _, ev := range evs { + if ev.Nsid != tangled.PipelineStatusNSID { + continue + } + var st tangled.PipelineStatus + if err := json.Unmarshal(ev.EventJson, &st); err != nil { + continue + } + if st.Pipeline == aturi && st.Workflow == wid.Name && st.Status == want { + return true + } + } + time.Sleep(50 * time.Millisecond) + } + return false +} diff --git a/spindle/server.go b/spindle/server.go index a8d5e3b6..70134277 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -39,6 +39,8 @@ import ( "tangled.org/core/spindle/engines/microvm" "tangled.org/core/spindle/engines/nixery" "tangled.org/core/spindle/git" + "tangled.org/core/spindle/mill" + "tangled.org/core/spindle/mill/executor" "tangled.org/core/spindle/models" "tangled.org/core/spindle/queue" "tangled.org/core/spindle/secrets" @@ -73,27 +75,51 @@ type Spindle struct { motd []byte motdMu sync.RWMutex rootCtx context.Context + + // set only when this spindle hosts the mill / joins one as an executor. + mill *mill.Mill + exec *executor.Executor } // New creates a new Spindle server with the provided configuration and engines. func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { logger := log.FromContext(ctx) + n := notifier.New() + + if cfg.Role == config.RoleExecutor { + if err := cleanupOrphanRepos(ctx, d, logger); err != nil { + return nil, fmt.Errorf("failed to run startup cleanup: %w", err) + } + } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { + return nil, fmt.Errorf("failed to run startup migrations: %w", err) + } + + spindle := &Spindle{ + db: d, + l: logger, + n: &n, + engs: engines, + cfg: cfg, + motd: defaultMotd, + rootCtx: ctx, + } + if cfg.Role == config.RoleExecutor { + return spindle, nil + } e, err := rbac.NewEnforcer(cfg.Server.DBPath) if err != nil { return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) } e.E.EnableAutoSave(true) + spindle.e = e - n := notifier.New() - - var vault secrets.Manager switch cfg.Server.Secrets.Provider { case "openbao": if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") } - vault, err = secrets.NewOpenBaoManager( + spindle.vault, err = secrets.NewOpenBaoManager( cfg.Server.Secrets.OpenBao.ProxyAddr, logger, secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), @@ -103,7 +129,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m } logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) case "sqlite", "": - vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) + spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) if err != nil { return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) } @@ -112,11 +138,8 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) } - if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { - return nil, fmt.Errorf("failed to run startup migrations: %w", err) - } - jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) + spindle.jq = jq logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) collections := []string{ @@ -129,6 +152,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m if err != nil { return nil, fmt.Errorf("failed to setup jetstream client: %w", err) } + spindle.jc = jc jc.AddDid(cfg.Server.Owner) // pull records are created by arbitrary users too, same hack as in tap jc.ExemptCollection(tangled.RepoPullNSID) @@ -152,23 +176,8 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m } } - resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) - - spindle := &Spindle{ - jc: jc, - e: e, - db: d, - l: logger, - n: &n, - engs: engines, - jq: jq, - cfg: cfg, - res: resolver, - verify: repoverify.New(resolver, cfg.Server.Dev), - vault: vault, - motd: defaultMotd, - rootCtx: ctx, - } + spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) + spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) err = e.AddSpindle(rbacDomain) if err != nil { @@ -275,42 +284,52 @@ func (s *Spindle) GetMotdContent() []byte { // Start starts the Spindle server (blocking). func (s *Spindle) Start(ctx context.Context) error { - // starts a job queue runner in the background - s.jq.Start() - defer s.jq.Stop() + // Only standalone runs the local queue. Mill hosts place directly onto + // executors, and executors only run jobs explicitly assigned by a mill. + if s.cfg.Role == config.RoleStandalone { + s.jq.Start() + defer s.jq.Stop() + } + + // An executor dials out to its mill and takes work from it. + if s.exec != nil { + go s.exec.Connect(ctx) + } // Stop vault token renewal if it implements Stopper if stopper, ok := s.vault.(secrets.Stopper); ok { defer stopper.Stop() } - tapCtx, tapCancel := context.WithCancel(ctx) + if s.cfg.Role != config.RoleExecutor { + tapCtx, tapCancel := context.WithCancel(ctx) - if s.cfg.Server.Tap.Embed { - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) - if err != nil { - tapCancel() - return fmt.Errorf("starting embedded tap: %w", err) + if s.cfg.Server.Tap.Embed { + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) + if err != nil { + tapCancel() + return fmt.Errorf("starting embedded tap: %w", err) + } + s.embedTap = emb + defer func() { + tapCancel() + s.embedTap.Shutdown() + }() + + go s.watchTapDrain(tapCtx, tapCancel) + } else { + defer tapCancel() } - s.embedTap = emb - defer func() { - tapCancel() - s.embedTap.Shutdown() + + go func() { + s.l.Info("starting knot event consumer") + s.ks.Start(ctx) }() - go s.watchTapDrain(tapCtx, tapCancel) - } else { - defer tapCancel() + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) + s.tap.Start(tapCtx) } - go func() { - s.l.Info("starting knot event consumer") - s.ks.Start(ctx) - }() - - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) - s.tap.Start(tapCtx) - s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) } @@ -355,23 +374,55 @@ func Run(ctx context.Context) error { return fmt.Errorf("failed to setup db: %w", err) } - nixeryEng, err := nixery.New(ctx, cfg) - if err != nil { - return err + logger := log.FromContext(ctx) + + var engines map[string]models.Engine + var m *mill.Mill + + if cfg.Role == config.RoleMill { + // mill host: register engines that place jobs on executors instead of + // running them. All names share one Mill. + m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ + SharedSecret: cfg.Mill.SharedSecret, + MaxPending: cfg.Mill.MaxPending, + ReconnectGrace: cfg.Mill.ReconnectGrace, + LogDir: cfg.Server.LogDir, + }) + engines = map[string]models.Engine{ + "nixery": mill.NewEngine("nixery", m), + "microvm": mill.NewEngine("microvm", m), + "dummy": mill.NewEngine("dummy", m), + } + } else { + // standalone and executor both run real engines locally. + nixeryEng, err := nixery.New(ctx, cfg) + if err != nil { + return err + } + microvmEng, err := microvm.New(ctx, cfg, d) + if err != nil { + return err + } + engines = map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(logger), + } } - microvmEng, err := microvm.New(ctx, cfg, d) + s, err := New(ctx, cfg, d, engines) if err != nil { return err } - s, err := New(ctx, cfg, d, map[string]models.Engine{ - "nixery": nixeryEng, - "microvm": microvmEng, - "dummy": dummy.New(log.FromContext(ctx)), - }) - if err != nil { - return err + if m != nil { + // resolve the chicken-and-egg: the engines (built above) hold the mill, + // but the mill's db/notifier are created inside New. + m.Attach(s.DB(), s.Notifier()) + s.mill = m + } + if cfg.Role == config.RoleExecutor { + s.exec = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor")) } return s.Start(ctx) @@ -383,9 +434,18 @@ func (s *Spindle) Router() http.Handler { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write(s.GetMotdContent()) }) + if s.cfg.Role == config.RoleExecutor { + return mux + } + mux.HandleFunc("/events", s.Events) mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) + // mill host: executors dial in here (plain ws, shared-secret auth). + if s.mill != nil { + mux.HandleFunc("/mill", s.mill.HandleExecutorConn) + } + mux.Mount("/xrpc", s.XrpcRouter()) return mux } @@ -823,24 +883,34 @@ func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipe workflows[eng] = append(workflows[eng], *ewf) } - // enqueue pipeline - ok := s.jq.Enqueue(repoDid, queue.Job{ - Run: func() error { - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{ - RepoDid: repoDid, - Workflows: workflows, - TrustedSource: trustedSource, - }, pipelineId) - return nil - }, - OnFail: func(jobError error) { - s.l.Error("pipeline run failed", "error", jobError) - }, - }) - if !ok { - return fmt.Errorf("failed to enqueue pipeline: queue is full") + pipeline := &models.Pipeline{ + RepoDid: repoDid, + Workflows: workflows, + TrustedSource: trustedSource, + } + + if s.mill != nil { + // mill host: no bounded pool. Each job blocks in placement + // (AcquireWorkflowSlot) which the user sees as pending; the only + // bound is the mill's maxPending. rootCtx is the long-lived consumer + // context, so the goroutine safely outlives this call. + go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId) + s.l.Info("pipeline handed to mill placement", "id", pipelineId) + } else { + ok := s.jq.Enqueue(repoDid, queue.Job{ + Run: func() error { + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, pipeline, pipelineId) + return nil + }, + OnFail: func(jobError error) { + s.l.Error("pipeline run failed", "error", jobError) + }, + }) + if !ok { + return fmt.Errorf("failed to enqueue pipeline: queue is full") + } + s.l.Info("pipeline enqueued successfully", "id", pipelineId) } - s.l.Info("pipeline enqueued successfully", "id", pipelineId) // after successful enqueue, emit StatusPending for all workflows for _, ewfs := range workflows { diff --git a/spindle/server_test.go b/spindle/server_test.go index 03b1efc4..a762ff87 100644 --- a/spindle/server_test.go +++ b/spindle/server_test.go @@ -1,9 +1,16 @@ package spindle import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" "testing" kgit "tangled.org/core/knotserver/git" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" ) func TestHasSkipCIPushOption(t *testing.T) { @@ -53,3 +60,40 @@ func TestHasSkipCIPushOption(t *testing.T) { }) } } + +func TestExecutorRoleBuildsMinimalSpindle(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "spindle.db") + d, err := db.Make(ctx, dbPath) + if err != nil { + t.Fatalf("db.Make() error = %v", err) + } + + cfg := &config.Config{Role: config.RoleExecutor} + cfg.Server.DBPath = dbPath + cfg.Server.Hostname = "executor.test" + cfg.Server.Tap.Embed = true + + s, err := New(ctx, cfg, d, map[string]models.Engine{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + if s.jc != nil || s.tap != nil || s.e != nil || s.jq != nil || s.ks != nil || s.res != nil || s.vault != nil { + t.Fatal("executor role built coordinator-only spindle dependencies") + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/", nil) + s.Router().ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("root status = %d, want %d", rr.Code, http.StatusOK) + } + + rr = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/xrpc/_health", nil) + s.Router().ServeHTTP(rr, req) + if rr.Code != http.StatusNotFound { + t.Fatalf("executor xrpc status = %d, want %d", rr.Code, http.StatusNotFound) + } +}