From 2321cd925768824ece0f33bcc0059f1204e4e4d9 Mon Sep 17 00:00:00 2001 From: dawn Date: Sun, 19 Jul 2026 02:50:40 +0300 Subject: [PATCH] spindle: implement mill Signed-off-by: dawn --- api/tangled/cbor_gen.go | 83 +- api/tangled/tangledpipeline.go | 1 + buf.gen.yaml | 4 + buf.yaml | 1 + cmd/spindle/main.go | 160 ++ docker-compose.mill.yml | 157 ++ eventstream/eventstream_test.go | 49 + eventstream/store.go | 29 + lexicons/pipeline/pipeline.json | 6 + localinfra/readme.md | 22 +- localinfra/scripts/prepare-spindle-images.sh | 10 + localinfra/spindle.Dockerfile | 5 + netutil/ssrf.go | 6 +- nix/modules/spindle.nix | 1 + spindle/config/config.go | 47 +- spindle/db/db.go | 68 +- spindle/db/events.go | 74 +- spindle/db/mill_state.go | 361 ++++ spindle/db/mill_state_test.go | 394 ++++ spindle/db/mill_tokens.go | 167 ++ spindle/db/mill_tokens_test.go | 272 +++ spindle/engine/engine.go | 82 +- spindle/engine/manifest_test.go | 8 + spindle/engine/placement.go | 10 + spindle/engine/scheduler.go | 20 +- spindle/engine/scheduler_test.go | 84 +- spindle/engine/slot.go | 23 +- spindle/engine/slot_test.go | 47 +- spindle/engines/dummy/engine.go | 18 +- spindle/engines/microvm/budget.go | 7 +- spindle/engines/microvm/engine.go | 8 +- spindle/engines/microvm/image.go | 27 +- spindle/engines/microvm/image_test.go | 58 + .../engines/microvm/placement/placement.go | 15 + .../microvm/placement/placement_test.go | 26 + spindle/engines/microvm/placement_linux.go | 61 + spindle/engines/microvm/qemu.go | 4 + spindle/engines/microvm/vm.go | 20 +- spindle/engines/nixery/engine.go | 3 +- spindle/mill/README.md | 159 ++ spindle/mill/auth_test.go | 482 +++++ spindle/mill/engine.go | 85 + spindle/mill/executor/capability_test.go | 119 ++ spindle/mill/executor/executor.go | 613 ++++++ spindle/mill/executor/observe.go | 251 +++ spindle/mill/executor/outbox.go | 344 ++++ spindle/mill/executor/reserved.go | 37 + spindle/mill/executor/reserved_test.go | 546 ++++++ spindle/mill/handler.go | 183 ++ spindle/mill/integration_test.go | 345 ++++ spindle/mill/lease.go | 186 ++ spindle/mill/mill.go | 1119 +++++++++++ spindle/mill/mill_test.go | 873 +++++++++ spindle/mill/proto/gen/mill.pb.go | 1679 +++++++++++++++++ spindle/mill/proto/protocol.go | 103 + spindle/mill/proto/protocol_test.go | 227 +++ spindle/mill/proto/spindle/mill/v1/mill.proto | 187 ++ spindle/mill/proto/ws.go | 58 + spindle/mill/restore.go | 221 +++ spindle/mill/restore_test.go | 386 ++++ spindle/mill/session.go | 160 ++ spindle/mill/token.go | 23 + spindle/server.go | 374 ++-- spindle/server.go.master | 930 +++++++++ spindle/server.go.mill | 1028 ++++++++++ spindle/server_test.go | 46 + workflow/compile.go | 15 +- workflow/compile_test.go | 45 + workflow/def.go | 21 +- workflow/def_test.go | 13 + 70 files changed, 13016 insertions(+), 280 deletions(-) create mode 100644 docker-compose.mill.yml create mode 100644 spindle/db/mill_state.go create mode 100644 spindle/db/mill_state_test.go create mode 100644 spindle/db/mill_tokens.go create mode 100644 spindle/db/mill_tokens_test.go create mode 100644 spindle/engine/placement.go create mode 100644 spindle/engines/microvm/placement/placement.go create mode 100644 spindle/engines/microvm/placement/placement_test.go create mode 100644 spindle/engines/microvm/placement_linux.go create mode 100644 spindle/mill/README.md create mode 100644 spindle/mill/auth_test.go create mode 100644 spindle/mill/engine.go create mode 100644 spindle/mill/executor/capability_test.go create mode 100644 spindle/mill/executor/executor.go create mode 100644 spindle/mill/executor/observe.go create mode 100644 spindle/mill/executor/outbox.go create mode 100644 spindle/mill/executor/reserved.go create mode 100644 spindle/mill/executor/reserved_test.go create mode 100644 spindle/mill/handler.go create mode 100644 spindle/mill/integration_test.go create mode 100644 spindle/mill/lease.go create mode 100644 spindle/mill/mill.go create mode 100644 spindle/mill/mill_test.go create mode 100644 spindle/mill/proto/gen/mill.pb.go create mode 100644 spindle/mill/proto/protocol.go create mode 100644 spindle/mill/proto/protocol_test.go create mode 100644 spindle/mill/proto/spindle/mill/v1/mill.proto create mode 100644 spindle/mill/proto/ws.go create mode 100644 spindle/mill/restore.go create mode 100644 spindle/mill/restore_test.go create mode 100644 spindle/mill/session.go create mode 100644 spindle/mill/token.go create mode 100644 spindle/server.go.master create mode 100644 spindle/server.go.mill diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go index 36312457..5d8f21d8 100644 --- a/api/tangled/cbor_gen.go +++ b/api/tangled/cbor_gen.go @@ -9749,8 +9749,13 @@ func (t *Pipeline_Workflow) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 5 - if _, err := cw.Write([]byte{164}); err != nil { + if t.RunsOn == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -9838,6 +9843,42 @@ func (t *Pipeline_Workflow) MarshalCBOR(w io.Writer) error { if _, err := cw.WriteString(string(t.Engine)); err != nil { return err } + + // t.RunsOn ([]string) (slice) + if t.RunsOn != nil { + + if len("runsOn") > 1000000 { + return xerrors.Errorf("Value in field \"runsOn\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("runsOn"))); err != nil { + return err + } + if _, err := cw.WriteString(string("runsOn")); err != nil { + return err + } + + if len(t.RunsOn) > 8192 { + return xerrors.Errorf("Slice value in field t.RunsOn was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.RunsOn))); err != nil { + return err + } + for _, v := range t.RunsOn { + if len(v) > 1000000 { + return xerrors.Errorf("Value in field v was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(v))); err != nil { + return err + } + if _, err := cw.WriteString(string(v)); err != nil { + return err + } + + } + } return nil } @@ -9935,6 +9976,46 @@ func (t *Pipeline_Workflow) UnmarshalCBOR(r io.Reader) (err error) { t.Engine = string(sval) } + // t.RunsOn ([]string) (slice) + case "runsOn": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > 8192 { + return fmt.Errorf("t.RunsOn: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.RunsOn = make([]string, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.RunsOn[i] = string(sval) + } + + } + } default: // Field doesn't exist on this type, so ignore it diff --git a/api/tangled/tangledpipeline.go b/api/tangled/tangledpipeline.go index da00f406..2151f331 100644 --- a/api/tangled/tangledpipeline.go +++ b/api/tangled/tangledpipeline.go @@ -90,4 +90,5 @@ type Pipeline_Workflow struct { Engine string `json:"engine" cborgen:"engine"` Name string `json:"name" cborgen:"name"` Raw string `json:"raw" cborgen:"raw"` + RunsOn []string `json:"runsOn,omitempty" cborgen:"runsOn,omitempty"` } diff --git a/buf.gen.yaml b/buf.gen.yaml index 7286491c..7681c279 100644 --- a/buf.gen.yaml +++ b/buf.gen.yaml @@ -9,3 +9,7 @@ plugins: out: shuttle/src/gen opt: - bytes=. + # the fleet protocol is broker<->executor only (both Go); shuttle (Rust) + # only speaks the agent protocol, so keep fleet types out of its gen tree. + exclude_types: + - spindle.mill.v1 diff --git a/buf.yaml b/buf.yaml index 3c20343f..7b664379 100644 --- a/buf.yaml +++ b/buf.yaml @@ -1,5 +1,6 @@ version: v2 modules: - path: spindle/agentproto + - path: spindle/mill/proto deps: - buf.build/bufbuild/protovalidate diff --git a/cmd/spindle/main.go b/cmd/spindle/main.go index c377a7ca..bdbf62ab 100644 --- a/cmd/spindle/main.go +++ b/cmd/spindle/main.go @@ -2,12 +2,18 @@ package main import ( "context" + "fmt" "log/slog" "os" + "strings" + "text/tabwriter" + "time" "github.com/urfave/cli/v3" tlog "tangled.org/core/log" "tangled.org/core/spindle" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/mill" ) func main() { @@ -16,6 +22,7 @@ func main() { Usage: "spindle continuous integration runner", Commands: []*cli.Command{ Command(), + millCommand(), }, DefaultCommand: "run", } @@ -41,3 +48,156 @@ func Command() *cli.Command { }, } } + +// for mill host administration and executor management +func millCommand() *cli.Command { + dbFlag := &cli.StringFlag{ + Name: "db", + Usage: "path to the spindle sqlite db", + Value: "spindle.db", + Sources: cli.EnvVars("SPINDLE_SERVER_DB_PATH"), + } + openDB := func(ctx context.Context, cmd *cli.Command) (*db.DB, error) { + return db.Make(ctx, cmd.String("db")) + } + return &cli.Command{ + Name: "mill", + Usage: "mill host administration", + Commands: []*cli.Command{ + { + Name: "executor", + Usage: "manage executors allowed to join this mill", + Commands: []*cli.Command{ + { + Name: "add", + Usage: "register an executor and print its token", + ArgsUsage: "", + Flags: []cli.Flag{ + dbFlag, + &cli.DurationFlag{ + Name: "ttl", + Usage: "token lifetime (e.g. 720h); omit for no expiry", + }, + &cli.StringSliceFlag{ + Name: "label", + Usage: "authorized labels for this executor", + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor add ") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + token, err := mill.GenerateToken() + if err != nil { + return err + } + var expiresAt *time.Time + if ttl := cmd.Duration("ttl"); ttl > 0 { + exp := time.Now().UTC().Add(ttl) + expiresAt = &exp + } + labels := cmd.StringSlice("label") + if err := d.AddExecutorToken(name, mill.HashToken(token), expiresAt, labels); err != nil { + return fmt.Errorf("registering executor %q: %w", name, err) + } + fmt.Println(token) + return nil + }, + }, + { + Name: "list", + Usage: "list registered executors", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + tokens, err := d.ListExecutorTokens() + if err != nil { + return err + } + w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) + fmt.Fprintln(w, "NAME\tCREATED\tEXPIRES\tLABELS\tQUARANTINE") + for _, t := range tokens { + expires := "never" + if t.ExpiresAt != nil { + expires = t.ExpiresAt.Format(time.RFC3339) + if time.Now().After(*t.ExpiresAt) { + expires += " (expired)" + } + } + labels := strings.Join(t.Labels, ",") + if labels == "" { + labels = "-" + } + quarantine := "-" + if t.QuarantineReason != nil { + quarantine = *t.QuarantineReason + if t.QuarantinedAt != nil { + quarantine = *t.QuarantinedAt + ": " + quarantine + } + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", t.Name, t.CreatedAt, expires, labels, quarantine) + } + return w.Flush() + }, + }, + { + Name: "unquarantine", + Usage: "allow a quarantined executor to reconnect", + ArgsUsage: "", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor unquarantine ") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + ok, err := d.ClearExecutorQuarantine(name) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("no such executor identity %q", name) + } + return nil + }, + }, + { + Name: "revoke", + Usage: "revoke an executor's token", + ArgsUsage: "", + Flags: []cli.Flag{dbFlag}, + Action: func(ctx context.Context, cmd *cli.Command) error { + name := cmd.Args().First() + if name == "" { + return fmt.Errorf("usage: spindle mill executor revoke ") + } + d, err := openDB(ctx, cmd) + if err != nil { + return err + } + ok, err := d.RevokeExecutorToken(name) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("no such executor identity %q", name) + } + return nil + }, + }, + }, + }, + }, + } +} diff --git a/docker-compose.mill.yml b/docker-compose.mill.yml new file mode 100644 index 00000000..413941ff --- /dev/null +++ b/docker-compose.mill.yml @@ -0,0 +1,157 @@ +# turns the primary spindle into a mill host and runs a three-executor fleet +# against it. the executors differ on two axes so placement is testable: +# +# executor labels images can run +# --------- ------------- ----------- ------------------------ +# executor-a linux, fast full image/alpine, image/nixos +# executor-b linux, slow full image/alpine, image/nixos +# executor-c linux, gpu alpine only image/alpine + +x-mill-executor: &mill-executor + profiles: ["linux"] + build: + context: . + dockerfile: localinfra/spindle.Dockerfile + restart: unless-stopped + environment: &mill-executor-env + SPINDLE_ROLE: executor + SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db + SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev + SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe + SPINDLE_SERVER_DEV: "true" + SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev + SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db + SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev + SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images + SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts + SPINDLE_MILL_ARTIFACT_STORE: disk + SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" + SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 + SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= + SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload + # dials the mill container directly using ws + SPINDLE_MILL_URL: ws://spindle:6555/mill + devices: + - /dev/vsock:/dev/vsock + - /dev/kvm:/dev/kvm + - /dev/vhost-vsock:/dev/vhost-vsock + - /dev/net/tun:/dev/net/tun + cap_add: + - NET_ADMIN + - SYS_ADMIN + security_opt: + - label=disable + - seccomp=unconfined + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s + depends_on: &mill-executor-deps + plc: + condition: service_started + jetstream: + condition: service_started + init-accounts: + condition: service_completed_successfully + ncps: + condition: service_started + spindle: + condition: service_healthy + mill-tokens: + condition: service_completed_successfully + networks: [tngl] + +services: + # configures the primary spindle container as a mill host + spindle: + environment: + SPINDLE_ROLE: mill + SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts + SPINDLE_MILL_ARTIFACT_STORE: disk + volumes: + - spindle-artifacts:/var/lib/spindle/artifacts + + mill-tokens: + profiles: ["linux"] + build: + context: . + dockerfile: localinfra/spindle.Dockerfile + restart: "no" + entrypoint: ["/bin/sh", "-c"] + environment: + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db + command: + - | + set -eu + seed() { + spindle mill executor revoke "$$1" >/dev/null 2>&1 || true + args="" + for l in $$(echo "$$2" | tr ',' ' '); do args="$$args --label $$l"; done + spindle mill executor add "$$1" $$args > "/shared/$$1.mill-token" + } + seed executor-a linux,fast + seed executor-b linux,slow + seed executor-c linux,gpu + echo "seeded executor tokens" + volumes: + - spindle-data:/var/lib/spindle + - init-state:/shared + depends_on: + spindle: + condition: service_healthy + networks: [tngl] + + spindle-executor-a: + <<: *mill-executor + environment: + <<: *mill-executor-env + SPINDLE_SERVER_HOSTNAME: executor-a.tngl.boltless.dev + SPINDLE_MILL_LABELS: linux,fast + SPINDLE_MILL_TOKEN_FILE: /shared/executor-a.mill-token + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11241" + volumes: + - spindle-executor-a-data:/var/lib/spindle + - spindle-artifacts:/var/lib/spindle/artifacts + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + + spindle-executor-b: + <<: *mill-executor + environment: + <<: *mill-executor-env + SPINDLE_SERVER_HOSTNAME: executor-b.tngl.boltless.dev + SPINDLE_MILL_LABELS: linux,slow + SPINDLE_MILL_TOKEN_FILE: /shared/executor-b.mill-token + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11242" + volumes: + - spindle-executor-b-data:/var/lib/spindle + - spindle-artifacts:/var/lib/spindle/artifacts + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + + spindle-executor-c: + <<: *mill-executor + environment: + <<: *mill-executor-env + SPINDLE_SERVER_HOSTNAME: executor-c.tngl.boltless.dev + SPINDLE_MILL_LABELS: linux,gpu + SPINDLE_MILL_TOKEN_FILE: /shared/executor-c.mill-token + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11243" + volumes: + - spindle-executor-c-data:/var/lib/spindle + - spindle-artifacts:/var/lib/spindle/artifacts + - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + +volumes: + spindle-executor-a-data: + spindle-executor-b-data: + spindle-executor-c-data: + spindle-artifacts: diff --git a/eventstream/eventstream_test.go b/eventstream/eventstream_test.go index 745d4953..3138fade 100644 --- a/eventstream/eventstream_test.go +++ b/eventstream/eventstream_test.go @@ -293,6 +293,55 @@ func TestInsert_MonotonicCreatedUnderConcurrency(t *testing.T) { } } +func TestHighWaterSeedsClockFromStoredEvents(t *testing.T) { + db, err := sql.Open("sqlite3", t.TempDir()+"/events.db") + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(func() { db.Close() }) + if _, err := db.Exec(`create table events ( + rkey text not null, + nsid text not null, + event text not null, + created integer not null, + primary key (rkey, nsid) + )`); err != nil { + t.Fatalf("schema: %v", err) + } + + stored := time.Now().Add(time.Hour).UnixNano() + if _, err := db.Exec( + `insert into events (rkey, nsid, event, created) values (?, ?, ?, ?)`, + "stored", "sh.tangled.test", "{}", stored, + ); err != nil { + t.Fatalf("seed event: %v", err) + } + + cut, err := HighWater(db) + if err != nil { + t.Fatalf("HighWater() error = %v", err) + } + if cut < stored { + t.Fatalf("HighWater() = %d, want at least stored cursor %d", cut, stored) + } + + n := notifier.New() + if err := Insert(db, Event{ + Rkey: "new", + Nsid: "sh.tangled.test", + EventJson: json.RawMessage("{}"), + }, &n); err != nil { + t.Fatalf("Insert() error = %v", err) + } + events, err := List(db, cut, 10) + if err != nil { + t.Fatalf("List() error = %v", err) + } + if len(events) != 1 || events[0].Rkey != "new" || events[0].Created <= cut { + t.Fatalf("events after cut = %+v, want only new event above %d", events, cut) + } +} + func isCloseErr(err error) bool { if err == nil { return false diff --git a/eventstream/store.go b/eventstream/store.go index c4191220..176527d9 100644 --- a/eventstream/store.go +++ b/eventstream/store.go @@ -19,6 +19,35 @@ var ( lastNanos int64 ) +func HighWater(s Store) (int64, error) { + clockMu.Lock() + defer clockMu.Unlock() + + rows, err := s.Query(`select coalesce(max(created), 0) from events`) + if err != nil { + return 0, err + } + defer rows.Close() + + var created int64 + if !rows.Next() { + if err := rows.Err(); err != nil { + return 0, err + } + return 0, sql.ErrNoRows + } + if err := rows.Scan(&created); err != nil { + return 0, err + } + if err := rows.Err(); err != nil { + return 0, err + } + if created > lastNanos { + lastNanos = created + } + return lastNanos, nil +} + func Insert(s Store, ev Event, n *notifier.Notifier) error { clockMu.Lock() defer clockMu.Unlock() diff --git a/lexicons/pipeline/pipeline.json b/lexicons/pipeline/pipeline.json index 9deb76fd..3256d172 100644 --- a/lexicons/pipeline/pipeline.json +++ b/lexicons/pipeline/pipeline.json @@ -184,6 +184,12 @@ "engine": { "type": "string" }, + "runsOn": { + "type": "array", + "items": { + "type": "string" + } + }, "clone": { "type": "ref", "ref": "#cloneOpts" diff --git a/localinfra/readme.md b/localinfra/readme.md index e0973aa8..2324878d 100644 --- a/localinfra/readme.md +++ b/localinfra/readme.md @@ -51,5 +51,25 @@ To make that work: This writes the image directory under `out/localinfra-spindle-images`. 5. `docker compose up` 6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. - `TANGLED_APPVIEW_HOST` must be a loopback IP with the mapped port (`127.0.0.1:3000`), not `localhost`: atproto's dev OAuth client requires a loopback IP for the redirect URI. If you remap the published appview port, update `TANGLED_APPVIEW_HOST` in `docker-compose.yml` to match. + + +## Mill mode + +The default stack runs one standalone spindle. `docker-compose.mill.yml` is an overlay that splits that into the distributed arch: the primary spindle becomes the mill host (`role=mill`, it only places jobs) and a three-executor fleet (`role=executor`) runs the engines. + +```bash +docker compose -f docker-compose.yml -f docker-compose.mill.yml --profile linux up +``` + +The executors differ on the two axes placement cares about, so you can watch candidates get filtered and ranked: + +| executor | labels | images | runs | +|------------|---------------|-------------|-----------------------------| +| executor-a | `linux, fast` | full | `image/alpine, image/nixos` | +| executor-b | `linux, slow` | full | `image/alpine, image/nixos` | +| executor-c | `linux, gpu` | alpine only | `image/alpine` | + +So `image: nixos` has two candidates, `image: alpine` has three, and `runs_on: [gpu]` pins to executor-c. The alpine-only image set is staged by `prepare-spindle-images.sh` (step 4) alongside the full one, no extra step. + +Each executor needs its own identity (one live session per token), so the `mill-tokens` service registers a token per executor in the mill db and drops it into the shared volume for the executor to read. diff --git a/localinfra/scripts/prepare-spindle-images.sh b/localinfra/scripts/prepare-spindle-images.sh index 93f31cf4..338a1075 100755 --- a/localinfra/scripts/prepare-spindle-images.sh +++ b/localinfra/scripts/prepare-spindle-images.sh @@ -29,4 +29,14 @@ extract_image() { extract_image spindle-nixos-image-tarball nixos-x86_64 nixos extract_image spindle-alpine-image-tarball alpine-x86_64 alpine +# a reduced image set (alpine only) for the mill overlay: an executor mounting +# this advertises image/alpine but not image/nixos, so capability-based +# placement is testable across a mixed fleet +alpine_only="$repo/out/localinfra-spindle-images-alpine" +[ -d "$alpine_only" ] && chmod -R +w "$alpine_only" || true +rm -rf "$alpine_only" +mkdir -p "$alpine_only" +cp -r "$image_root/alpine-x86_64" "$alpine_only/alpine-x86_64" +ln -s alpine-x86_64 "$alpine_only/alpine" + echo "prepared spindle microVM images in $image_root" diff --git a/localinfra/spindle.Dockerfile b/localinfra/spindle.Dockerfile index 007e7d84..b64736e8 100644 --- a/localinfra/spindle.Dockerfile +++ b/localinfra/spindle.Dockerfile @@ -49,6 +49,11 @@ set -eu export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)" : "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}" +# executors read their mill token from a file seeded by the mill-tokens +# service (command substitution strips the trailing newline) +[ -z "${SPINDLE_MILL_SHARED_SECRET:-}" ] && [ -n "${SPINDLE_MILL_TOKEN_FILE:-}" ] && [ -r "${SPINDLE_MILL_TOKEN_FILE}" ] && \ + export SPINDLE_MILL_SHARED_SECRET="$(cat "${SPINDLE_MILL_TOKEN_FILE}")" + mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then diff --git a/netutil/ssrf.go b/netutil/ssrf.go index b0055a1b..6974187d 100644 --- a/netutil/ssrf.go +++ b/netutil/ssrf.go @@ -10,7 +10,7 @@ import ( "github.com/gorilla/websocket" ) -// SSRFDialer returns a net.Dialer that refuses non-public IPs. +// refuses non-public ips to prevent ssrf func SSRFDialer(dev bool) *net.Dialer { if dev { return &net.Dialer{} @@ -18,7 +18,7 @@ func SSRFDialer(dev bool) *net.Dialer { return ssrf.PublicOnlyDialer() } -// SSRFTransport returns an http.Transport that refuses non-public IPs. +// refuses non-public ips to prevent ssrf func SSRFTransport(dev bool) *http.Transport { if dev { return &http.Transport{} @@ -26,7 +26,7 @@ func SSRFTransport(dev bool) *http.Transport { return ssrf.PublicOnlyTransport() } -// SSRFWebsocketDialer returns a websocket.Dialer that refuses non-public IPs. +// refuses non-public ips to prevent ssrf func SSRFWebsocketDialer(dev bool) *websocket.Dialer { dialer := *websocket.DefaultDialer dialer.NetDialContext = SSRFDialer(dev).DialContext diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix index 4b63ebdd..102d14e9 100644 --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -406,6 +406,7 @@ in "SPINDLE_ARTIFACT_STORES_DISK_DIR=${cfg.artifactStores.disk.dir}" "SPINDLE_ARTIFACT_STORES_S3_BUCKET=${cfg.artifactStores.s3.bucket}" "SPINDLE_ARTIFACT_STORES_S3_REGION=${cfg.artifactStores.s3.region}" + "SPINDLE_MILL_ARTIFACT_STORE=s3" ]; ExecStart = "${cfg.package}/bin/spindle"; Restart = "always"; diff --git a/spindle/config/config.go b/spindle/config/config.go index 3af5c906..b2d06b75 100644 --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -2,6 +2,7 @@ package config import ( "context" + "fmt" "time" "github.com/bluesky-social/indigo/atproto/syntax" @@ -77,7 +78,7 @@ type LegacyS3 struct { type MicroVMPipelines struct { ImageDir string `env:"IMAGE_DIR"` - OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live + OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` AgentPort uint32 `env:"AGENT_PORT, default=10240"` EnableKVM bool `env:"ENABLE_KVM, default=true"` @@ -107,13 +108,54 @@ type NixCache struct { UploadURL string `env:"UPLOAD_URL"` } +// governs how spindle places and runs jobs +type Role string + +const ( + RoleStandalone Role = "standalone" + RoleMill Role = "mill" + RoleExecutor Role = "executor" +) + +// fields are selectively active depending on the role +type Mill struct { + URL string `env:"URL"` // mill websocket endpoint dialled by the executor + SharedSecret string `env:"SHARED_SECRET"` // the executor's token for dialing the mill + MaxPending int `env:"MAX_PENDING, default=100"` // mill pending job queue limit + ReconnectGrace time.Duration `env:"RECONNECT_GRACE, default=45s"` // reconnect window before leases are failed + Seats int `env:"SEATS, default=4"` // executor seats advertised to the mill + Labels []string `env:"LABELS"` // executor capability labels + ArtifactStore string `env:"ARTIFACT_STORE"` // store shared by mill and its executors +} + type Config struct { + Role Role `env:"SPINDLE_ROLE, default=standalone"` Server Server `env:",prefix=SPINDLE_SERVER_"` NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` ArtifactStores ArtifactStores `env:",prefix=SPINDLE_ARTIFACT_STORES_"` LegacyS3 LegacyS3 `env:",prefix=SPINDLE_S3_"` + Mill Mill `env:",prefix=SPINDLE_MILL_"` +} + +func (c *Config) validate() error { + switch c.Role { + case RoleStandalone, RoleMill: + if c.Mill.URL != "" { + return fmt.Errorf("SPINDLE_MILL_URL is set but SPINDLE_ROLE=%s; only an executor dials a mill", c.Role) + } + case RoleExecutor: + if c.Mill.URL == "" { + return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_URL (the mill to dial)") + } + if c.Mill.SharedSecret == "" { + return fmt.Errorf("SPINDLE_ROLE=executor requires SPINDLE_MILL_SHARED_SECRET (its executor token)") + } + default: + return fmt.Errorf("unknown SPINDLE_ROLE %q (want standalone, mill, or executor)", c.Role) + } + return nil } func Load(ctx context.Context) (*Config, error) { @@ -122,6 +164,9 @@ func Load(ctx context.Context) (*Config, error) { if err != nil { return nil, err } + if err := cfg.validate(); err != nil { + return nil, err + } return &cfg, nil } diff --git a/spindle/db/db.go b/spindle/db/db.go index cd442912..de8de374 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -79,26 +79,20 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { ); create table if not exists spindle_members ( - -- identifiers for the record id integer primary key autoincrement, did text not null, rkey text not null, - - -- data instance text not null, subject text not null, created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), - - -- constraints unique (did, rkey) ); - -- status event for a single workflow create table if not exists events ( rkey text not null, nsid text not null, - event text not null, -- json - created integer not null -- unix nanos + event text not null, + created integer not null ); create table if not exists nixos_toplevel_cache ( @@ -132,6 +126,50 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { foreign key (pipeline_id) references pipelines(id) on delete cascade ); + create table if not exists mill_executors ( + name text primary key, + token_hash text not null unique, + created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + expires_at text, + labels text, + quarantine_reason text, + quarantined_at text + ); + + create table if not exists mill_leases ( + lease_id text primary key, + node_id text not null, + epoch text not null, + engine text not null, + knot text not null, + rkey text not null, + workflow text not null, + state text not null + ); + + create table if not exists mill_executor_cursors ( + node_id text not null, + epoch text not null, + acked_seqno integer not null, + primary key (node_id, epoch) + ); + + create table if not exists mill_outbox_state ( + epoch text not null, + next_seqno integer not null, + primary key (epoch) + ); + + create table if not exists mill_outbox_rows ( + epoch text not null, + seqno integer not null, + payload blob not null, + byte_size integer not null, + control integer not null, + primary key (epoch, seqno), + foreign key (epoch) references mill_outbox_state(epoch) on delete cascade + ); + create table if not exists mill_artifacts ( id integer primary key autoincrement, lease_id text not null, @@ -140,15 +178,21 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { hash text not null ); + create table if not exists executor_pending_artifacts ( + lease_id text primary key, + workflow text not null, + status text not null, + error text not null default '', + exit_code integer not null default 0, + ref text not null, + hash text not null + ); + create table if not exists migrations ( id integer primary key autoincrement, name text unique ); `) - if err != nil { - return nil, err - } - if err := runMigrations(ctx, conn, logger); err != nil { return nil, err } diff --git a/spindle/db/events.go b/spindle/db/events.go index fa5b0b9b..c300267c 100644 --- a/spindle/db/events.go +++ b/spindle/db/events.go @@ -19,6 +19,10 @@ func (d *DB) GetEvents(cursor int64, limit int) ([]eventstream.Event, error) { return eventstream.List(d, cursor, limit) } +func (d *DB) EventHighWater() (int64, error) { + return eventstream.HighWater(d) +} + func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *notifier.Notifier) error { eventJson, err := json.Marshal(pipeline) if err != nil { @@ -32,38 +36,78 @@ func (d *DB) CreatePipelineEvent(rkey string, pipeline tangled.Pipeline, n *noti return d.insertEvent(event, n) } -func (d *DB) createStatusEvent( - workflowId models.WorkflowId, - statusKind models.StatusKind, - workflowError *string, - exitCode *int64, - n *notifier.Notifier, -) error { - now := time.Now() - pipelineAtUri := workflowId.PipelineId.AtUri() +// leaves created at zero so insertevent stamps the local clock +func statusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) (eventstream.Event, error) { s := tangled.PipelineStatus{ - CreatedAt: now.Format(time.RFC3339), + CreatedAt: time.Now().Format(time.RFC3339), Error: workflowError, ExitCode: exitCode, - Pipeline: string(pipelineAtUri), - Workflow: workflowId.Name, - Status: string(statusKind), + Pipeline: pipelineAtUri, + Workflow: workflow, + Status: status, } eventJson, err := json.Marshal(s) if err != nil { - return err + return eventstream.Event{}, err } - event := eventstream.Event{ + return eventstream.Event{ Rkey: tid.TID(), Nsid: tangled.PipelineStatusNSID, EventJson: eventJson, + }, nil +} + +func (d *DB) createStatusEvent( + workflowId models.WorkflowId, + statusKind models.StatusKind, + workflowError *string, + exitCode *int64, + n *notifier.Notifier, +) error { + event, err := statusEvent(string(workflowId.PipelineId.AtUri()), workflowId.Name, string(statusKind), workflowError, exitCode) + if err != nil { + return err } + return d.insertEvent(event, n) +} +// stamps the mill's own clock so it orders against the cursor like a local write +func (d *DB) InsertEventStatus( + pipelineAtUri string, + workflow string, + status string, + workflowError *string, + exitCode *int64, + n *notifier.Notifier, +) error { + event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) + if err != nil { + return err + } return d.insertEvent(event, n) } +// deleting the lease in the same transaction prevents the terminal event +// from replaying +func (d *DB) CompleteMillLease( + leaseID string, + pipelineAtUri string, + workflow string, + status string, + workflowError *string, + exitCode *int64, + n *notifier.Notifier, +) error { + return d.ApplyEventBatch(n, func(tx *EventBatchTx) error { + if err := tx.InsertStatusEvent(pipelineAtUri, workflow, status, workflowError, exitCode); err != nil { + return err + } + return tx.DeleteLease(leaseID) + }) +} + func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) { pipelineAtUri := workflowId.PipelineId.AtUri() diff --git a/spindle/db/mill_state.go b/spindle/db/mill_state.go new file mode 100644 index 00000000..e7f9ff70 --- /dev/null +++ b/spindle/db/mill_state.go @@ -0,0 +1,361 @@ +package db + +import ( + "database/sql" + "fmt" + + "tangled.org/core/eventstream" + "tangled.org/core/notifier" +) + +// enough to rebuild the fencing token and workflow identity after a restart +type MillLease struct { + LeaseID string + NodeID string + Epoch string + Engine string + Knot string + Rkey string + Workflow string + State string +} + +type ExecutorCursor struct { + NodeID string + Epoch string + AckedSeqno uint64 +} + +type OutboxRow struct { + Epoch string + Seqno uint64 + Payload []byte + ByteSize int64 + Control bool +} +type OutboxDeletion struct { + Rows int64 + Bytes int64 +} + +func (d *DB) SaveMillLease(l MillLease) error { + _, err := d.Exec( + `insert into mill_leases ( + lease_id, node_id, epoch, engine, knot, rkey, workflow, state + ) values (?, ?, ?, ?, ?, ?, ?, ?) + on conflict(lease_id) do update set state = excluded.state`, + l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.Knot, l.Rkey, l.Workflow, l.State, + ) + return err +} + +func (d *DB) DeleteMillLease(leaseID string) error { + _, err := d.Exec(`delete from mill_leases where lease_id = ?`, leaseID) + return err +} + +func (d *DB) ListMillLeases() ([]MillLease, error) { + rows, err := d.Query(` + select lease_id, node_id, epoch, engine, knot, rkey, workflow, state + from mill_leases + `) + if err != nil { + return nil, err + } + defer rows.Close() + + var leases []MillLease + for rows.Next() { + var l MillLease + if err := rows.Scan( + &l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.Knot, &l.Rkey, &l.Workflow, &l.State, + ); err != nil { + return nil, err + } + leases = append(leases, l) + } + return leases, rows.Err() +} + +func (d *DB) ListExecutorCursors() ([]ExecutorCursor, error) { + rows, err := d.Query(`select node_id, epoch, acked_seqno from mill_executor_cursors`) + if err != nil { + return nil, err + } + defer rows.Close() + + var cursors []ExecutorCursor + for rows.Next() { + var c ExecutorCursor + if err := rows.Scan(&c.NodeID, &c.Epoch, &c.AckedSeqno); err != nil { + return nil, err + } + cursors = append(cursors, c) + } + return cursors, rows.Err() +} + +func (d *DB) SetOutboxEpoch(epoch string) error { + tx, err := d.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + if _, err := tx.Exec(`delete from mill_outbox_rows`); err != nil { + return err + } + if _, err := tx.Exec(`delete from mill_outbox_state`); err != nil { + return err + } + if _, err := tx.Exec(`insert into mill_outbox_state (epoch, next_seqno) values (?, 1)`, epoch); err != nil { + return err + } + return tx.Commit() +} + +func (d *DB) AppendOutboxRow(payload []byte, control bool) (uint64, error) { + tx, err := d.Begin() + if err != nil { + return 0, err + } + defer tx.Rollback() + + var epoch string + var nextSeqno uint64 + err = tx.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) + if err == sql.ErrNoRows { + return 0, fmt.Errorf("no outbox epoch set") + } else if err != nil { + return 0, err + } + + byteSize := int64(len(payload)) + controlVal := 0 + if control { + controlVal = 1 + } + + if _, err := tx.Exec( + `insert into mill_outbox_rows (epoch, seqno, payload, byte_size, control) values (?, ?, ?, ?, ?)`, + epoch, nextSeqno, payload, byteSize, controlVal, + ); err != nil { + return 0, err + } + + if _, err := tx.Exec( + `update mill_outbox_state set next_seqno = ? where epoch = ?`, + nextSeqno+1, epoch, + ); err != nil { + return 0, err + } + + if err := tx.Commit(); err != nil { + return 0, err + } + return nextSeqno, nil +} + +func (d *DB) DeleteOutboxPrefix(ackedSeqno uint64) (OutboxDeletion, error) { + tx, err := d.Begin() + if err != nil { + return OutboxDeletion{}, err + } + defer tx.Rollback() + + var epoch string + err = tx.QueryRow(`select epoch from mill_outbox_state limit 1`).Scan(&epoch) + if err == sql.ErrNoRows { + return OutboxDeletion{}, nil + } + if err != nil { + return OutboxDeletion{}, err + } + + var deleted OutboxDeletion + if err := tx.QueryRow(` + select count(*), + coalesce(sum(byte_size), 0) + from mill_outbox_rows + where epoch = ? and seqno <= ? + `, epoch, ackedSeqno).Scan(&deleted.Rows, &deleted.Bytes); err != nil { + return OutboxDeletion{}, err + } + if _, err := tx.Exec( + `delete from mill_outbox_rows where epoch = ? and seqno <= ?`, + epoch, ackedSeqno, + ); err != nil { + return OutboxDeletion{}, err + } + if err := tx.Commit(); err != nil { + return OutboxDeletion{}, err + } + return deleted, nil +} + +func (d *DB) ListOutboxRows() ([]OutboxRow, error) { + rows, err := d.Query(`select epoch, seqno, payload, byte_size, control from mill_outbox_rows order by seqno`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []OutboxRow + for rows.Next() { + var r OutboxRow + var controlVal int + if err := rows.Scan(&r.Epoch, &r.Seqno, &r.Payload, &r.ByteSize, &controlVal); err != nil { + return nil, err + } + r.Control = (controlVal != 0) + out = append(out, r) + } + return out, rows.Err() +} +func (d *DB) ListOutboxRowsAfter(seqno uint64, limit int) ([]OutboxRow, error) { + rows, err := d.Query(` + select epoch, seqno, payload, byte_size, control + from mill_outbox_rows + where epoch = (select epoch from mill_outbox_state limit 1) + and seqno > ? + order by seqno + limit ? + `, seqno, limit) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []OutboxRow + for rows.Next() { + var row OutboxRow + var control int + if err := rows.Scan(&row.Epoch, &row.Seqno, &row.Payload, &row.ByteSize, &control); err != nil { + return nil, err + } + row.Control = control != 0 + out = append(out, row) + } + return out, rows.Err() +} + +func (d *DB) GetOutboxState() (string, uint64, error) { + var epoch string + var nextSeqno uint64 + err := d.QueryRow(`select epoch, next_seqno from mill_outbox_state limit 1`).Scan(&epoch, &nextSeqno) + if err == sql.ErrNoRows { + return "", 0, nil + } + return epoch, nextSeqno, err +} + +type EventBatchTx struct { + tx *sql.Tx + db *DB +} + +func (tx *EventBatchTx) InsertStatusEvent(pipelineAtUri, workflow, status string, workflowError *string, exitCode *int64) error { + event, err := statusEvent(pipelineAtUri, workflow, status, workflowError, exitCode) + if err != nil { + return err + } + return eventstream.Insert(tx.tx, event, nil) +} + +func (tx *EventBatchTx) DeleteLease(leaseID string) error { + _, err := tx.tx.Exec(`delete from mill_leases where lease_id = ?`, leaseID) + return err +} + +func (tx *EventBatchTx) AdvanceCursor(nodeID, epoch string, seqno uint64) error { + _, err := tx.tx.Exec( + `insert into mill_executor_cursors (node_id, epoch, acked_seqno) values (?, ?, ?) + on conflict(node_id, epoch) do update set acked_seqno = excluded.acked_seqno`, + nodeID, epoch, seqno, + ) + return err +} + +func (tx *EventBatchTx) InsertArtifactRef(leaseID, workflow, ref, hash string) error { + _, err := tx.tx.Exec( + `insert into mill_artifacts (lease_id, workflow, ref, hash) + values (?, ?, ?, ?)`, + leaseID, workflow, ref, hash, + ) + return err +} + +type PendingArtifact struct { + LeaseID string + Workflow string + Status string + Error string + ExitCode int64 + Ref string + Hash string +} + +func (d *DB) SavePendingArtifact(leaseID, workflow, status, errStr string, exitCode int64, ref, hash string) error { + _, err := d.Exec( + `insert into executor_pending_artifacts (lease_id, workflow, status, error, exit_code, ref, hash) + values (?, ?, ?, ?, ?, ?, ?) + on conflict(lease_id) do update set + workflow = excluded.workflow, + status = excluded.status, + error = excluded.error, + exit_code = excluded.exit_code, + ref = excluded.ref, + hash = excluded.hash`, + leaseID, workflow, status, errStr, exitCode, ref, hash, + ) + return err +} + +func (d *DB) RemovePendingArtifact(leaseID string) error { + _, err := d.Exec(`delete from executor_pending_artifacts where lease_id = ?`, leaseID) + return err +} + +func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { + rows, err := d.Query(`select lease_id, workflow, status, error, exit_code, ref, hash from executor_pending_artifacts`) + if err != nil { + return nil, err + } + defer rows.Close() + + var res []PendingArtifact + for rows.Next() { + var p PendingArtifact + if err := rows.Scan(&p.LeaseID, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash); err != nil { + return nil, err + } + res = append(res, p) + } + return res, rows.Err() +} + +func (d *DB) ApplyEventBatch(n *notifier.Notifier, fn func(tx *EventBatchTx) error) error { + tx, err := d.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + batchTx := &EventBatchTx{ + tx: tx, + db: d, + } + + if err := fn(batchTx); err != nil { + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + if n != nil { + n.NotifyAll() + } + return nil +} diff --git a/spindle/db/mill_state_test.go b/spindle/db/mill_state_test.go new file mode 100644 index 00000000..1bb47a4e --- /dev/null +++ b/spindle/db/mill_state_test.go @@ -0,0 +1,394 @@ +package db + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "tangled.org/core/notifier" +) + +func TestMillLeaseRoundTrip(t *testing.T) { + d := newTestDB(t) + + lease := MillLease{ + LeaseID: "lease-1", + NodeID: "node-1", + Epoch: "inc-1", + Engine: "dummy", + Knot: "knot.example", + Rkey: "rkey1", + Workflow: "build", + State: "reserved", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + + lease.State = "running" + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease(transition): %v", err) + } + + leases, err := d.ListMillLeases() + if err != nil { + t.Fatalf("ListMillLeases: %v", err) + } + if len(leases) != 1 { + t.Fatalf("ListMillLeases returned %d leases, want 1 (state transition must replace, not duplicate)", len(leases)) + } + if leases[0].LeaseID != lease.LeaseID || leases[0].State != "running" { + t.Fatalf("ListMillLeases[0] = %+v, want %+v", leases[0], lease) + } + + if err := d.DeleteMillLease("lease-1"); err != nil { + t.Fatalf("DeleteMillLease: %v", err) + } + if leases, _ = d.ListMillLeases(); len(leases) != 0 { + t.Fatalf("lease survived deletion: %+v", leases) + } +} + +func TestExecutorCursors(t *testing.T) { + d := newTestDB(t) + + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 5) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 9) }); err != nil { + t.Fatalf("AdvanceCursor(advance): %v", err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 1) }); err != nil { + t.Fatalf("AdvanceCursor(node-2): %v", err) + } + + cursors, err := d.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 2 { + t.Fatalf("ListExecutorCursors = %v, want 2 cursors", cursors) + } + + cursorMap := make(map[string]uint64) + for _, c := range cursors { + cursorMap[c.NodeID+"/"+c.Epoch] = c.AckedSeqno + } + + if cursorMap["node-1/inc-1"] != 9 || cursorMap["node-2/inc-1"] != 1 { + t.Fatalf("unexpected cursors: %v", cursorMap) + } + +} + +func TestCompleteMillLeaseIsAtomic(t *testing.T) { + d := newTestDB(t) + lease := MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + if _, err := d.Exec(` + create trigger reject_mill_lease_delete + before delete on mill_leases + begin + select raise(abort, 'forced delete failure'); + end + `); err != nil { + t.Fatalf("create failure trigger: %v", err) + } + + n := notifier.New() + notifications := n.Subscribe() + defer n.Unsubscribe(notifications) + err := d.CompleteMillLease( + "lease-1", + "at://knot.example/sh.tangled.pipeline/rkey1", + "build", + "failed", + nil, + nil, + &n, + ) + if err == nil { + t.Fatal("CompleteMillLease succeeded despite forced lease deletion failure") + } + var eventCount int + if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { + t.Fatalf("count events after rollback: %v", err) + } + if eventCount != 0 { + t.Fatalf("terminal event count after rollback = %d, want 0", eventCount) + } + if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 1 { + t.Fatalf("leases after rollback = %+v, err = %v; want original lease", leases, listErr) + } + select { + case <-notifications: + t.Fatal("rollback notified event subscribers") + default: + } + + if _, err := d.Exec(`drop trigger reject_mill_lease_delete`); err != nil { + t.Fatalf("drop failure trigger: %v", err) + } + if err := d.CompleteMillLease( + "lease-1", + "at://knot.example/sh.tangled.pipeline/rkey1", + "build", + "failed", + nil, + nil, + &n, + ); err != nil { + t.Fatalf("CompleteMillLease retry: %v", err) + } + if err := d.QueryRow(`select count(*) from events`).Scan(&eventCount); err != nil { + t.Fatalf("count committed events: %v", err) + } + if eventCount != 1 { + t.Fatalf("terminal event count after commit = %d, want 1", eventCount) + } + if leases, listErr := d.ListMillLeases(); listErr != nil || len(leases) != 0 { + t.Fatalf("leases after commit = %+v, err = %v; want none", leases, listErr) + } + select { + case <-notifications: + default: + t.Fatal("committed terminal event did not notify subscribers") + } +} + +func TestRestartPersistence(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "persist.db") + ctx := context.Background() + d, err := Make(ctx, dbPath) + if err != nil { + t.Fatalf("Make: %v", err) + } + + lease := MillLease{ + LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy", + Knot: "k", Rkey: "r", Workflow: "w", State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-p", "inc-p", 42) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + + if err := d.SetOutboxEpoch("inc-p"); err != nil { + t.Fatalf("SetOutboxEpoch: %v", err) + } + if _, err := d.AppendOutboxRow([]byte("hello world"), true); err != nil { + t.Fatalf("AppendOutboxRow: %v", err) + } + + if err := d.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + d2, err := Make(ctx, dbPath) + if err != nil { + t.Fatalf("Make reopen: %v", err) + } + defer d2.Close() + + leases, err := d2.ListMillLeases() + if err != nil { + t.Fatalf("ListMillLeases: %v", err) + } + if len(leases) != 1 || leases[0].LeaseID != "lease-p" || leases[0].Epoch != "inc-p" { + t.Fatalf("unexpected leases: %+v", leases) + } + + cursors, err := d2.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 1 || cursors[0].NodeID != "node-p" || cursors[0].Epoch != "inc-p" || cursors[0].AckedSeqno != 42 { + t.Fatalf("unexpected cursors: %+v", cursors) + } + + inc, nextSeqno, err := d2.GetOutboxState() + if err != nil { + t.Fatalf("GetOutboxState: %v", err) + } + if inc != "inc-p" || nextSeqno != 2 { + t.Fatalf("unexpected outbox state: inc=%q, next=%d", inc, nextSeqno) + } + rows, err := d2.ListOutboxRows() + if err != nil { + t.Fatalf("ListOutboxRows: %v", err) + } + if len(rows) != 1 || string(rows[0].Payload) != "hello world" || !rows[0].Control { + t.Fatalf("unexpected outbox rows: %+v", rows) + } +} + +func TestOutboxPrefixAck(t *testing.T) { + d := newTestDB(t) + + if err := d.SetOutboxEpoch("inc-1"); err != nil { + t.Fatalf("SetOutboxEpoch: %v", err) + } + + o1, err := d.AppendOutboxRow([]byte("msg1"), false) + if err != nil || o1 != 1 { + t.Fatalf("AppendOutboxRow 1: %v, seqno=%d", err, o1) + } + o2, err := d.AppendOutboxRow([]byte("msg2"), false) + if err != nil || o2 != 2 { + t.Fatalf("AppendOutboxRow 2: %v, seqno=%d", err, o2) + } + o3, err := d.AppendOutboxRow([]byte("msg3"), true) + if err != nil || o3 != 3 { + t.Fatalf("AppendOutboxRow 3: %v, seqno=%d", err, o3) + } + + n, err := d.DeleteOutboxPrefix(2) + if err != nil { + t.Fatalf("DeleteOutboxPrefix: %v", err) + } + if n.Rows != 2 || n.Bytes != 8 { + t.Fatalf("deleted prefix = %+v, want 2 rows and 8 bytes", n) + } + + rows, err := d.ListOutboxRows() + if err != nil { + t.Fatalf("ListOutboxRows: %v", err) + } + if len(rows) != 1 || rows[0].Seqno != 3 || string(rows[0].Payload) != "msg3" { + t.Fatalf("expected only msg3 (seqno 3) to remain, got: %+v", rows) + } +} + +func TestCompositeCursors(t *testing.T) { + d := newTestDB(t) + + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 10) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-2", 20) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { return tx.AdvanceCursor("node-2", "inc-1", 5) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + + cursors, err := d.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 3 { + t.Fatalf("expected 3 cursors, got %d", len(cursors)) + } + + cursorMap := make(map[string]uint64) + for _, c := range cursors { + key := c.NodeID + "/" + c.Epoch + cursorMap[key] = c.AckedSeqno + } + + if cursorMap["node-1/inc-1"] != 10 || cursorMap["node-1/inc-2"] != 20 || cursorMap["node-2/inc-1"] != 5 { + t.Fatalf("unexpected cursor values: %v", cursorMap) + } + +} + +func TestBatchRollback(t *testing.T) { + d := newTestDB(t) + + lease := MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "k", Rkey: "r", Workflow: "w", State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + + n := notifier.New() + err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { + if err := tx.DeleteLease("lease-1"); err != nil { + return err + } + if err := tx.AdvanceCursor("node-1", "inc-1", 100); err != nil { + return err + } + return fmt.Errorf("forced batch failure") + }) + + if err == nil { + t.Fatal("expected ApplyEventBatch to return error") + } + + leases, err := d.ListMillLeases() + if err != nil { + t.Fatalf("ListMillLeases: %v", err) + } + if len(leases) != 1 { + t.Fatalf("lease was deleted despite rollback: %+v", leases) + } + + cursors, err := d.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 0 { + t.Fatalf("cursor was advanced despite rollback: %+v", cursors) + } +} + +func TestTerminalCursorAtomicity(t *testing.T) { + d := newTestDB(t) + + lease := MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "k", Rkey: "r", Workflow: "w", State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + + n := notifier.New() + notifications := n.Subscribe() + defer n.Unsubscribe(notifications) + + err := d.ApplyEventBatch(&n, func(tx *EventBatchTx) error { + if err := tx.DeleteLease("lease-1"); err != nil { + return err + } + return tx.AdvanceCursor("node-1", "inc-1", 100) + }) + + if err != nil { + t.Fatalf("ApplyEventBatch: %v", err) + } + + leases, err := d.ListMillLeases() + if err != nil { + t.Fatalf("ListMillLeases: %v", err) + } + if len(leases) != 0 { + t.Fatalf("lease not deleted: %+v", leases) + } + + cursors, err := d.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 1 || cursors[0].AckedSeqno != 100 { + t.Fatalf("cursor not advanced correctly: %+v", cursors) + } + + select { + case <-notifications: + default: + t.Fatal("notifier was not fired after batch commit") + } +} diff --git a/spindle/db/mill_tokens.go b/spindle/db/mill_tokens.go new file mode 100644 index 00000000..69ba351d --- /dev/null +++ b/spindle/db/mill_tokens.go @@ -0,0 +1,167 @@ +package db + +import ( + "database/sql" + "encoding/json" + "slices" + "strings" + "time" +) + +// the raw token is never stored, only its hash +type ExecutorToken struct { + Name string + CreatedAt string + ExpiresAt *time.Time + Labels []string + QuarantineReason *string + QuarantinedAt *string +} + +func (d *DB) AddExecutorToken(name, tokenHash string, expiresAt *time.Time, labels []string) error { + normalized := normalizeLabels(labels) + labelsBytes, err := json.Marshal(normalized) + if err != nil { + return err + } + _, err = d.Exec( + `insert into mill_executors (name, token_hash, expires_at, labels) values (?, ?, ?, ?)`, + name, tokenHash, expiryArg(expiresAt), string(labelsBytes), + ) + return err +} + +func (d *DB) ResolveExecutorToken(tokenHash string) (string, []string, bool, error) { + var name string + var expires sql.NullString + var labelsRaw, quarantineReason sql.NullString + err := d.QueryRow( + `select name, expires_at, labels, quarantine_reason from mill_executors where token_hash = ?`, tokenHash, + ).Scan(&name, &expires, &labelsRaw, &quarantineReason) + if err == sql.ErrNoRows { + return "", nil, false, nil + } + if err != nil { + return "", nil, false, err + } + if quarantineReason.Valid { + return "", nil, false, nil + } + exp, hasExpiry, err := parseExpiry(expires) + if err != nil { + return "", nil, false, err + } + if hasExpiry && time.Now().After(exp) { + return "", nil, false, nil + } + var labels []string + if labelsRaw.Valid && labelsRaw.String != "" { + if err := json.Unmarshal([]byte(labelsRaw.String), &labels); err != nil { + return "", nil, false, err + } + } + return name, labels, true, nil +} + +func (d *DB) QuarantineExecutor(name, reason string) error { + _, err := d.Exec( + `update mill_executors + set quarantine_reason = ?, quarantined_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where name = ?`, + reason, name, + ) + return err +} + +func (d *DB) ClearExecutorQuarantine(name string) (bool, error) { + res, err := d.Exec( + `update mill_executors set quarantine_reason = null, quarantined_at = null where name = ?`, + name, + ) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n > 0, err +} + +func (d *DB) RevokeExecutorToken(name string) (bool, error) { + res, err := d.Exec(`delete from mill_executors where name = ?`, name) + if err != nil { + return false, err + } + n, err := res.RowsAffected() + return n > 0, err +} + +func (d *DB) ListExecutorTokens() ([]ExecutorToken, error) { + rows, err := d.Query(`select name, created_at, expires_at, labels, quarantine_reason, quarantined_at from mill_executors order by name`) + if err != nil { + return nil, err + } + defer rows.Close() + + var out []ExecutorToken + for rows.Next() { + var t ExecutorToken + var expires, labelsRaw, quarantineReason, quarantinedAt sql.NullString + if err := rows.Scan(&t.Name, &t.CreatedAt, &expires, &labelsRaw, &quarantineReason, &quarantinedAt); err != nil { + return nil, err + } + exp, hasExpiry, err := parseExpiry(expires) + if err != nil { + return nil, err + } + if hasExpiry { + t.ExpiresAt = &exp + } + if labelsRaw.Valid && labelsRaw.String != "" { + if err := json.Unmarshal([]byte(labelsRaw.String), &t.Labels); err != nil { + return nil, err + } + } + if quarantineReason.Valid { + t.QuarantineReason = &quarantineReason.String + } + if quarantinedAt.Valid { + t.QuarantinedAt = &quarantinedAt.String + } + out = append(out, t) + } + return out, rows.Err() +} + +func normalizeLabels(labels []string) []string { + var out []string + seen := make(map[string]bool) + for _, l := range labels { + trimmed := strings.TrimSpace(l) + if trimmed == "" { + continue + } + if !seen[trimmed] { + seen[trimmed] = true + out = append(out, trimmed) + } + } + slices.Sort(out) + return out +} + +func expiryArg(t *time.Time) any { + if t == nil { + return nil + } + return t.UTC().Format(time.RFC3339) +} + +func parseExpiry(s sql.NullString) (time.Time, bool, error) { + if !s.Valid { + return time.Time{}, false, nil + } + t, err := time.Parse(time.RFC3339, s.String) + if err != nil { + return time.Time{}, false, err + } + return t, true, nil +} diff --git a/spindle/db/mill_tokens_test.go b/spindle/db/mill_tokens_test.go new file mode 100644 index 00000000..43e1525a --- /dev/null +++ b/spindle/db/mill_tokens_test.go @@ -0,0 +1,272 @@ +package db + +import ( + "slices" + "testing" + "time" +) + +func TestAddExecutorTokenRejectsDuplicateName(t *testing.T) { + d := newTestDB(t) + + if err := d.AddExecutorToken("exec-1", "hash-a", nil, nil); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + if err := d.AddExecutorToken("exec-1", "hash-b", nil, nil); err == nil { + t.Fatal("AddExecutorToken re-registered an existing name; a duplicate must be rejected") + } + + name, _, ok, err := d.ResolveExecutorToken("hash-a") + if err != nil { + t.Fatalf("ResolveExecutorToken(hash-a): %v", err) + } + if !ok || name != "exec-1" { + t.Fatalf("ResolveExecutorToken(hash-a) = (%q, %v), want (exec-1, true)", name, ok) + } + if _, _, ok, _ := d.ResolveExecutorToken("hash-b"); ok { + t.Fatal("rejected duplicate's token resolved; the failed insert leaked a credential") + } +} + +func TestResolveExecutorTokenMissAndHit(t *testing.T) { + d := newTestDB(t) + + name, _, ok, err := d.ResolveExecutorToken("no-such-hash") + if err != nil { + t.Fatalf("ResolveExecutorToken(miss): %v", err) + } + if ok || name != "" { + t.Fatalf("ResolveExecutorToken(miss) = (%q, %v), want (\"\", false)", name, ok) + } + + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + + name, _, ok, err = d.ResolveExecutorToken("hash-1") + if err != nil { + t.Fatalf("ResolveExecutorToken(hit): %v", err) + } + if !ok || name != "exec-1" { + t.Fatalf("ResolveExecutorToken(hash-1) = (%q, %v), want (exec-1, true)", name, ok) + } + + if _, _, ok, _ := d.ResolveExecutorToken("hash-unregistered"); ok { + t.Fatal("ResolveExecutorToken matched an unregistered hash") + } +} + +func TestRevokeExecutorToken(t *testing.T) { + d := newTestDB(t) + + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + + deleted, err := d.RevokeExecutorToken("exec-1") + if err != nil { + t.Fatalf("RevokeExecutorToken: %v", err) + } + if !deleted { + t.Fatal("RevokeExecutorToken reported no deletion for an existing identity") + } + + if _, _, ok, _ := d.ResolveExecutorToken("hash-1"); ok { + t.Fatal("revoked token still resolves; revocation is not enforced") + } + + if deleted, err := d.RevokeExecutorToken("exec-1"); err != nil || deleted { + t.Fatalf("RevokeExecutorToken(already-gone) = (%v, %v), want (false, nil)", deleted, err) + } + + if deleted, err := d.RevokeExecutorToken("ghost"); err != nil || deleted { + t.Fatalf("RevokeExecutorToken(unknown) = (%v, %v), want (false, nil)", deleted, err) + } +} + +func TestExecutorQuarantineIsVisibleAndReversible(t *testing.T) { + d := newTestDB(t) + if err := d.AddExecutorToken("exec-1", "hash-1", nil, nil); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + if err := d.QuarantineExecutor("exec-1", "missed cancel deadline"); err != nil { + t.Fatalf("QuarantineExecutor: %v", err) + } + if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || ok { + t.Fatalf("ResolveExecutorToken(quarantined) = (ok=%v, err=%v), want (false, nil)", ok, err) + } + tokens, err := d.ListExecutorTokens() + if err != nil { + t.Fatalf("ListExecutorTokens: %v", err) + } + if len(tokens) != 1 || tokens[0].QuarantineReason == nil || *tokens[0].QuarantineReason != "missed cancel deadline" || tokens[0].QuarantinedAt == nil { + t.Fatalf("quarantined token not surfaced: %+v", tokens) + } + if cleared, err := d.ClearExecutorQuarantine("exec-1"); err != nil || !cleared { + t.Fatalf("ClearExecutorQuarantine = (%v, %v), want (true, nil)", cleared, err) + } + if _, _, ok, err := d.ResolveExecutorToken("hash-1"); err != nil || !ok { + t.Fatalf("ResolveExecutorToken(cleared) = (ok=%v, err=%v), want (true, nil)", ok, err) + } + if cleared, err := d.ClearExecutorQuarantine("missing"); err != nil || cleared { + t.Fatalf("ClearExecutorQuarantine(missing) = (%v, %v), want (false, nil)", cleared, err) + } +} + +func TestListExecutorTokens(t *testing.T) { + d := newTestDB(t) + + tokens, err := d.ListExecutorTokens() + if err != nil { + t.Fatalf("ListExecutorTokens(empty): %v", err) + } + if len(tokens) != 0 { + t.Fatalf("ListExecutorTokens on empty table = %d rows, want 0", len(tokens)) + } + + // insert out of alphabetical order to test query sorting + for _, name := range []string{"charlie", "alice", "bob"} { + if err := d.AddExecutorToken(name, "hash-"+name, nil, nil); err != nil { + t.Fatalf("AddExecutorToken(%s): %v", name, err) + } + } + + tokens, err = d.ListExecutorTokens() + if err != nil { + t.Fatalf("ListExecutorTokens: %v", err) + } + want := []string{"alice", "bob", "charlie"} + if len(tokens) != len(want) { + t.Fatalf("ListExecutorTokens = %d rows, want %d", len(tokens), len(want)) + } + for i := range want { + if tokens[i].Name != want[i] { + t.Fatalf("ListExecutorTokens[%d].Name = %q, want %q (ordered by name)", i, tokens[i].Name, want[i]) + } + } +} + +// expired tokens must fail closed +func TestResolveExecutorTokenExpiry(t *testing.T) { + cases := []struct { + name string + seqno time.Duration + noExpiry bool + wantOK bool + }{ + {"future expiry resolves", time.Hour, false, true}, + {"past expiry fails closed", -time.Hour, false, false}, + {"nil expiry never expires", 0, true, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := newTestDB(t) + + var expires *time.Time + if !tc.noExpiry { + exp := time.Now().Add(tc.seqno) + expires = &exp + } + if err := d.AddExecutorToken("exec-1", "hash-1", expires, nil); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + + name, _, ok, err := d.ResolveExecutorToken("hash-1") + if err != nil { + t.Fatalf("ResolveExecutorToken: %v", err) + } + if ok != tc.wantOK { + t.Fatalf("ResolveExecutorToken ok = %v, want %v", ok, tc.wantOK) + } + if tc.wantOK && name != "exec-1" { + t.Fatalf("ResolveExecutorToken name = %q, want exec-1", name) + } + if !tc.wantOK && name != "" { + t.Fatalf("ResolveExecutorToken name = %q, want \"\" when failing closed", name) + } + }) + } +} + +func TestResolveExecutorTokenRejectsMalformedExpiry(t *testing.T) { + d := newTestDB(t) + if _, err := d.Exec( + `insert into mill_executors (name, token_hash, expires_at) values (?, ?, ?)`, + "exec-1", "hash-1", "not-a-timestamp", + ); err != nil { + t.Fatalf("insert malformed token: %v", err) + } + + name, _, ok, err := d.ResolveExecutorToken("hash-1") + if err == nil { + t.Fatal("ResolveExecutorToken accepted a malformed non-NULL expiry") + } + if ok || name != "" { + t.Fatalf("ResolveExecutorToken = (%q, %v, %v), want (\"\", false, error)", name, ok, err) + } +} + +// expiry storage is RFC3339 second precision UTC so +// round trips compare to the second +func TestListExecutorTokensSurfacesExpiry(t *testing.T) { + d := newTestDB(t) + + exp := time.Now().Add(24 * time.Hour) + if err := d.AddExecutorToken("expiring", "hash-exp", &exp, nil); err != nil { + t.Fatalf("AddExecutorToken(expiring): %v", err) + } + if err := d.AddExecutorToken("forever", "hash-forever", nil, nil); err != nil { + t.Fatalf("AddExecutorToken(forever): %v", err) + } + + tokens, err := d.ListExecutorTokens() + if err != nil { + t.Fatalf("ListExecutorTokens: %v", err) + } + + got := make(map[string]*time.Time, len(tokens)) + for _, tok := range tokens { + got[tok.Name] = tok.ExpiresAt + } + + e, present := got["forever"] + if !present { + t.Fatal("ListExecutorTokens omitted the non-expiring identity") + } + if e != nil { + t.Fatalf("forever.ExpiresAt = %v, want nil (never expires)", e) + } + + e, present = got["expiring"] + if !present { + t.Fatal("ListExecutorTokens omitted the expiring identity") + } + if e == nil { + t.Fatal("expiring.ExpiresAt = nil, want the stored expiry") + } + if e.Unix() != exp.Unix() { + t.Fatalf("expiring.ExpiresAt = %d (unix), want %d", e.Unix(), exp.Unix()) + } +} + +func TestExecutorTokenLabels(t *testing.T) { + d := newTestDB(t) + + labels := []string{" foo ", "bar", " foo", ""} + if err := d.AddExecutorToken("exec-1", "hash-1", nil, labels); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + + name, resolvedLabels, ok, err := d.ResolveExecutorToken("hash-1") + if err != nil { + t.Fatalf("ResolveExecutorToken: %v", err) + } + if !ok || name != "exec-1" { + t.Fatalf("ResolveExecutorToken: ok=%v, name=%q, want true, exec-1", ok, name) + } + + wantLabels := []string{"bar", "foo"} + if !slices.Equal(resolvedLabels, wantLabels) { + t.Fatalf("resolved labels = %v, want %v", resolvedLabels, wantLabels) + } +} diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index c86aee2f..3fa5a241 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -67,8 +67,34 @@ func writeWfError(db *db.DB, n *notifier.Notifier, l *slog.Logger, wfCtx context } } -type workflowFinalizer interface { - FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error +// the mill streams the executor's real log file into place itself +// a local logger here would only write competing lines +type workflowLoggerProvider interface { + WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger +} + +// for engines that manage status updates outside StartWorkflows +type RemoteStatusEngine interface { + AuthorsRemoteStatus() +} + +func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Notifier, wid models.WorkflowId, err error) { + if errors.Is(err, ErrTimedOut) { + dbErr := database.StatusTimeout(wid, n) + if dbErr != nil { + l.Error("failed to set workflow status to timeout", "wid", wid, "err", dbErr) + } + } else if errors.Is(err, ErrWorkflowCanceled) { + dbErr := database.StatusCancelled(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to cancelled", "wid", wid, "err", dbErr) + } + } else { + dbErr := database.StatusFailed(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) + } + } } func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { @@ -101,7 +127,6 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s wfCounts[wid.String()]++ } } - var wg sync.WaitGroup for eng, wfs := range pipeline.Workflows { workflowTimeout := eng.WorkflowTimeout() @@ -128,14 +153,18 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s l.Info("skipping finished workflow", "wid", wid, "status", st.Status) return } - wfLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues) - if err != nil { + var err error + var wfLogger models.WorkflowLogger + if p, ok := eng.(workflowLoggerProvider); ok { + wfLogger = p.WorkflowLogger(wid) + } else if fileLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues); err != nil { l.Warn("failed to setup step logger; logs will not be persisted", "error", err) wfLogger = models.NullLogger{} } else { l.Info("setup step logger; logs will be persisted", "logDir", cfg.Server.LogDir, "wid", wid) + wfLogger = fileLogger defer archiveWorkflowLog(l, stores, db, cfg.Server.LogDir, wid) - defer wfLogger.Close() + defer fileLogger.Close() } timeoutCtx, timeoutCancel := context.WithTimeout(ctx, workflowTimeout) @@ -156,8 +185,10 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s l.Info("waiting for slot", "wid", wid) slot := WorkflowSlot(NoopSlot{}) + _, remoteStatus := eng.(RemoteStatusEngine) + if s, ok := eng.(WorkflowSlotter); ok { - slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w) + slot, err = s.AcquireWorkflowSlot(wfCtx, wid, &w, Wait) if err != nil { writeWfError(db, n, l, wfCtx, wid, "waiting for slot", err) return @@ -165,10 +196,12 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s } defer slot.Release() - err = db.StatusRunning(wid, n) - if err != nil { - l.Error("failed to set workflow status to running", "wid", wid, "err", err) - return + if !remoteStatus { + err := db.StatusRunning(wid, n) + if err != nil { + l.Error("failed to set workflow status to running", "wid", wid, "err", err) + return + } } err = eng.SetupWorkflow(wfCtx, wid, &w, wfLogger) @@ -178,7 +211,9 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s l.Error("failed to destroy workflow after setup failure", "error", destroyErr) } } - writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) + if !remoteStatus { + writeWfError(db, n, l, wfCtx, wid, "setting up workflow", err) + } return } defer eng.DestroyWorkflow(ctx, wid) @@ -199,26 +234,25 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s } if err != nil { - writeWfError(db, n, l, wfCtx, wid, "running step", err) - return - } - } - - if finalizer, ok := eng.(workflowFinalizer); ok { - if err := finalizer.FinalizeWorkflow(wfCtx, wid, &w, wfLogger); err != nil { - writeWfError(db, n, l, wfCtx, wid, "finalizing", err) + if !remoteStatus { + writeWfError(db, n, l, wfCtx, wid, "running step", err) + } return } } if isCanceled(wfCtx) { - writeWfError(db, n, l, wfCtx, wid, "before success", nil) + if !remoteStatus { + writeWfError(db, n, l, wfCtx, wid, "before success", nil) + } return } - err = db.StatusSuccess(wid, n) - if err != nil { - l.Error("failed to set workflow status to success", "wid", wid, "err", err) + if !remoteStatus { + err = db.StatusSuccess(wid, n) + if err != nil { + l.Error("failed to set workflow status to success", "wid", wid, "err", err) + } } }) } diff --git a/spindle/engine/manifest_test.go b/spindle/engine/manifest_test.go index cb086bf0..0d71b075 100644 --- a/spindle/engine/manifest_test.go +++ b/spindle/engine/manifest_test.go @@ -79,6 +79,14 @@ func TestDescribeManifestError(t *testing.T) { } } +func TestDescribeManifestErrorAcceptsRunsOnGenericWorkflowKey(t *testing.T) { + raw := "engine: microvm\nruns_on: [linux/arm64, kvm]\nimage: nixos\n" + + if err := DescribeManifestError(raw, testManifest{}); err != nil { + t.Fatalf("DescribeManifestError(%q) = %v, want nil", raw, err) + } +} + func TestDescribeManifestErrorNoFalsePositives(t *testing.T) { cases := []string{ // well-formed manifest diff --git a/spindle/engine/placement.go b/spindle/engine/placement.go new file mode 100644 index 00000000..24864e33 --- /dev/null +++ b/spindle/engine/placement.go @@ -0,0 +1,10 @@ +package engine + +import ( + "tangled.org/core/spindle/models" +) + +// checked before an executor accepts and holds a remote lease +type WorkflowPlacementValidator interface { + ValidateWorkflowPlacement(wf *models.Workflow) error +} diff --git a/spindle/engine/scheduler.go b/spindle/engine/scheduler.go index c067a63f..3eff74de 100644 --- a/spindle/engine/scheduler.go +++ b/spindle/engine/scheduler.go @@ -50,7 +50,8 @@ func NewResourceScheduler[R Resources[R]](budget, max R, agingThreshold time.Dur } } -func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) { +// the mill owns the backlog so a NoWait caller must have room immediately or fail +func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R, mode AcquireMode) (WorkflowSlot, error) { if s == nil { return NoopSlot{}, nil } @@ -60,11 +61,18 @@ func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot s.mu.Unlock() return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max) } - if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) { + // NoWait ignores the queue because it never blocks + // Wait only bypasses empty queues to prevent starvation + if s.used.Add(req).Fits(s.budget) && (mode == NoWait || len(s.queue) == 0) { s.used = s.used.Add(req) s.mu.Unlock() return &resourceLease[R]{scheduler: s, req: req}, nil } + if mode == NoWait { + used := s.used + s.mu.Unlock() + return nil, fmt.Errorf("%w: request=%v used=%v budget=%v", ErrNoWorkflowSlots, req, used, s.budget) + } waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()} s.queue = append(s.queue, waiter) @@ -108,7 +116,7 @@ func (s *ResourceScheduler[R]) release(req R) { // start every waiter whose request fits. once a waiter is older than // agingThreshold, count its request as already used so younger waiters -// stop being scheduled ahead of it. +// stop being scheduled ahead of it func (s *ResourceScheduler[R]) schedule() { var reserved R now := s.now() @@ -129,11 +137,7 @@ func (s *ResourceScheduler[R]) schedule() { } func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) { - for i, candidate := range s.queue { - if candidate != waiter { - continue - } + if i := slices.Index(s.queue, waiter); i >= 0 { s.queue = slices.Delete(s.queue, i, i+1) - return } } diff --git a/spindle/engine/scheduler_test.go b/spindle/engine/scheduler_test.go index 99e0630c..714ab32b 100644 --- a/spindle/engine/scheduler_test.go +++ b/spindle/engine/scheduler_test.go @@ -36,7 +36,7 @@ func TestResourceSchedulerZeroLimitsDoNotApply(t *testing.T) { scheduler := NewResourceScheduler(ru{}, ru{}, 0) - slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}) + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, Wait) if err != nil { t.Fatalf("Acquire() error = %v", err) } @@ -48,12 +48,12 @@ func TestResourceSchedulerRejectsRequestsThatCanNeverFit(t *testing.T) { scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) - _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}) + _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, Wait) if !errors.Is(err, ErrNoWorkflowSlots) { t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) } - _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}) + _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}, Wait) if !errors.Is(err, ErrNoWorkflowSlots) { t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) } @@ -64,7 +64,7 @@ func TestResourceSchedulerWaitsUntilResourcesAreReleased(t *testing.T) { scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) - first, err := scheduler.Acquire(context.Background(), ru{a: 1024}) + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, Wait) if err != nil { t.Fatalf("first Acquire() error = %v", err) } @@ -85,7 +85,7 @@ func TestResourceSchedulerReleaseIsIdempotent(t *testing.T) { scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0) - slot, err := scheduler.Acquire(context.Background(), ru{a: 1}) + slot, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) if err != nil { t.Fatalf("Acquire() error = %v", err) } @@ -93,7 +93,7 @@ func TestResourceSchedulerReleaseIsIdempotent(t *testing.T) { slot.Release() slot.Release() - second, err := scheduler.Acquire(context.Background(), ru{a: 1}) + second, err := scheduler.Acquire(context.Background(), ru{a: 1}, Wait) if err != nil { t.Fatalf("Acquire() after double release error = %v", err) } @@ -105,7 +105,7 @@ func TestResourceSchedulerBackfillsPastBlockedHead(t *testing.T) { scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) if err != nil { t.Fatalf("hold Acquire() error = %v", err) } @@ -128,7 +128,7 @@ func TestResourceSchedulerAgingReservesCapacityForBlockedHead(t *testing.T) { fakeNow := time.Now() scheduler.now = func() time.Time { return fakeNow } - hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) if err != nil { t.Fatalf("hold Acquire() error = %v", err) } @@ -151,10 +151,76 @@ func TestResourceSchedulerAgingReservesCapacityForBlockedHead(t *testing.T) { big.Release() } +func TestResourceSchedulerTryRejectsWhenNoRoomNow(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) + + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}, NoWait) + if err != nil { + t.Fatalf("first Acquire(NoWait) error = %v", err) + } + + if _, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) + } + + first.Release() + + second, err := scheduler.Acquire(context.Background(), ru{a: 1}, NoWait) + if err != nil { + t.Fatalf("Acquire(NoWait) after release error = %v", err) + } + second.Release() +} + +func TestResourceSchedulerTryRejectsRequestsThatCanNeverFit(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) + + if _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire(NoWait) error = %v, want ErrNoWorkflowSlots", err) + } +} + +func TestResourceSchedulerTryIgnoresQueuedWaiters(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) + + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}, Wait) + if err != nil { + t.Fatalf("hold Acquire() error = %v", err) + } + defer hold.Release() + + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) + assertAcquireBlocked(t, bigCh) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 256}, NoWait) + if err != nil { + t.Fatalf("Acquire(NoWait) error = %v, want success past queued waiter", err) + } + slot.Release() +} + +func TestResourceSchedulerZeroLimitsTryDoesNotReject(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{}, ru{}, 0) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}, NoWait) + if err != nil { + t.Fatalf("Acquire(NoWait) error = %v", err) + } + slot.Release() +} + func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult { ch := make(chan acquireResult, 1) go func() { - slot, err := scheduler.Acquire(ctx, req) + slot, err := scheduler.Acquire(ctx, req, Wait) ch <- acquireResult{slot: slot, err: err} }() return ch diff --git a/spindle/engine/slot.go b/spindle/engine/slot.go index 667989e5..40bc0cb3 100644 --- a/spindle/engine/slot.go +++ b/spindle/engine/slot.go @@ -13,8 +13,19 @@ type WorkflowSlot interface { Release() } +// governs blocking behaviour when acquiring a slot +type AcquireMode int + +const ( + // blocks until a slot is free + Wait AcquireMode = iota + // fails immediately if full + // executors use this because the mill owns the backlog + NoWait +) + type WorkflowSlotter interface { - AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) + AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) } type releaseFunc func() @@ -41,10 +52,18 @@ func NewSemaphoreSlotter(maxConcurrent int) *SemaphoreSlotter { return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)} } -func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) { +func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode AcquireMode) (WorkflowSlot, error) { if a == nil || a.slots == nil { return NoopSlot{}, nil } + if mode == NoWait { + select { + case a.slots <- struct{}{}: + return releaseFunc(func() { <-a.slots }), nil + default: + return nil, ErrNoWorkflowSlots + } + } select { case a.slots <- struct{}{}: return releaseFunc(func() { <-a.slots }), nil diff --git a/spindle/engine/slot_test.go b/spindle/engine/slot_test.go index 506e8b75..4078fec1 100644 --- a/spindle/engine/slot_test.go +++ b/spindle/engine/slot_test.go @@ -15,7 +15,7 @@ func TestSemaphoreSlotterDisabledDoesNotBlock(t *testing.T) { slotter := NewSemaphoreSlotter(0) for range 10 { - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) if err != nil { t.Fatalf("AcquireWorkflowSlot() error = %v", err) } @@ -28,7 +28,7 @@ func TestSemaphoreSlotterBlocksUntilRelease(t *testing.T) { slotter := NewSemaphoreSlotter(1) - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) if err != nil { t.Fatalf("first AcquireWorkflowSlot() error = %v", err) } @@ -42,7 +42,7 @@ func TestSemaphoreSlotterBlocksUntilRelease(t *testing.T) { acquired := make(chan WorkflowSlot, 1) errs := make(chan error, 1) go func() { - slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) if err != nil { errs <- err return @@ -64,7 +64,7 @@ func TestSemaphoreSlotterHonorsContextCancellation(t *testing.T) { slotter := NewSemaphoreSlotter(1) - first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, Wait) if err != nil { t.Fatalf("first AcquireWorkflowSlot() error = %v", err) } @@ -73,12 +73,49 @@ func TestSemaphoreSlotterHonorsContextCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil) + _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil, Wait) if !errors.Is(err, context.Canceled) { t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err) } } +func TestSemaphoreSlotterTryDisabledDoesNotReject(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(0) + + for range 10 { + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) + if err != nil { + t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v", err) + } + slot.Release() + } +} + +func TestSemaphoreSlotterTryRejectsWhenFull(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(1) + + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) + if err != nil { + t.Fatalf("first AcquireWorkflowSlot(NoWait) error = %v", err) + } + + if _, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait); !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("AcquireWorkflowSlot(NoWait) error = %v, want ErrNoWorkflowSlots", err) + } + + first.Release() + + second, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil, NoWait) + if err != nil { + t.Fatalf("AcquireWorkflowSlot(NoWait) after release error = %v", err) + } + second.Release() +} + func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) { t.Helper() diff --git a/spindle/engines/dummy/engine.go b/spindle/engines/dummy/engine.go index f52026d2..eedc4a17 100644 --- a/spindle/engines/dummy/engine.go +++ b/spindle/engines/dummy/engine.go @@ -8,6 +8,7 @@ import ( "gopkg.in/yaml.v3" "tangled.org/core/api/tangled" + "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" ) @@ -16,7 +17,8 @@ import ( // step output to the workflow logger. Useful for testing pipeline plumbing // without a real execution backend. type DummyEngine struct { - l *slog.Logger + l *slog.Logger + StepDelay time.Duration } func New(l *slog.Logger) *DummyEngine { @@ -79,14 +81,26 @@ func (e *DummyEngine) WorkflowTimeout() time.Duration { return 5 * time.Minute } +// no capacity limit, so always a no-op slot regardless of mode +func (e *DummyEngine) AcquireWorkflowSlot(_ context.Context, _ models.WorkflowId, _ *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { + return engine.NoopSlot{}, nil +} + func (e *DummyEngine) DestroyWorkflow(_ context.Context, wid models.WorkflowId) error { e.l.Info("destroying workflow", "wid", wid) return nil } -func (e *DummyEngine) RunStep(_ context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { +func (e *DummyEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, _ []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { step := w.Steps[idx] e.l.Info("running step", "wid", wid, "step", step.Name(), "command", step.Command()) fmt.Fprintf(wfLogger.DataWriter(idx, "stdout"), "$ %s", step.Command()) + if e.StepDelay > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(e.StepDelay): + } + } return nil } diff --git a/spindle/engines/microvm/budget.go b/spindle/engines/microvm/budget.go index 27383266..a45d834e 100644 --- a/spindle/engines/microvm/budget.go +++ b/spindle/engines/microvm/budget.go @@ -68,7 +68,7 @@ func newVMBudgetConfig(cfg config.MicroVMPipelines) (Resources, Resources, time. return budget, maxReq, cfg.AgingThreshold } -func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { +func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { state, ok := wf.Data.(*workflowState) if !ok || state == nil { return nil, fmt.Errorf("microVM workflow state is not initialized") @@ -77,10 +77,7 @@ func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, return engine.NoopSlot{}, nil } req := resourcesForImage(state.ImageSpec) - if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 { - return nil, fmt.Errorf("microVM resource request must not be negative: %s", req) - } - return e.scheduler.Acquire(ctx, req) + return e.scheduler.Acquire(ctx, req, mode) } func resourcesForImage(spec ImageSpec) Resources { diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go index 021e8558..68a9efe2 100644 --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -53,6 +53,8 @@ type Engine struct { agent *agentHub scheduler *engine.ResourceScheduler[Resources] cgroupParent *CgroupParent + budget Resources + maxWorkflow Resources cleanupMu sync.Mutex cleanup map[string][]cleanupFunc @@ -92,6 +94,8 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB) (*Engine, error) { db: d, scheduler: engine.NewResourceScheduler(budget, max, agingThreshold), cgroupParent: cgroupParent, + budget: budget, + maxWorkflow: max, cleanup: make(map[string][]cleanupFunc), }, nil } @@ -573,10 +577,6 @@ func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) err return cleanupErr } -func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error { - return nil -} - func (e *Engine) WorkflowTimeout() time.Duration { d, err := time.ParseDuration(e.cfg.MicroVMPipelines.WorkflowTimeout) if err != nil { diff --git a/spindle/engines/microvm/image.go b/spindle/engines/microvm/image.go index 7c37152d..4642ae03 100644 --- a/spindle/engines/microvm/image.go +++ b/spindle/engines/microvm/image.go @@ -240,22 +240,23 @@ func imageSpecPath(candidate string) (string, bool, error) { } return "", false, err } - if !info.IsDir() { - return candidate, true, nil + if info.IsDir() { + candidate = filepath.Join(candidate, imageSpecFileName) + info, err = os.Stat(candidate) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", false, fmt.Errorf("microVM image directory %q does not contain %s", filepath.Dir(candidate), imageSpecFileName) + } + return "", false, err + } + if info.IsDir() { + return "", false, fmt.Errorf("microVM image spec %q is a directory", candidate) + } } - spec := filepath.Join(candidate, imageSpecFileName) - info, err = os.Stat(spec) + path, err := filepath.EvalSymlinks(candidate) if err != nil { - if errors.Is(err, os.ErrNotExist) { - return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName) - } return "", false, err } - // this only happens if there is a directory named `spec.json` which would be very silly. - // but better output an error for it anyway :p - if info.IsDir() { - return "", false, fmt.Errorf("microVM image spec %q is a directory", spec) - } - return spec, true, nil + return path, true, nil } diff --git a/spindle/engines/microvm/image_test.go b/spindle/engines/microvm/image_test.go index 1114937c..c73e0ec0 100644 --- a/spindle/engines/microvm/image_test.go +++ b/spindle/engines/microvm/image_test.go @@ -65,6 +65,36 @@ func TestResolveImageConventionalLayouts(t *testing.T) { }) } } +func TestImageSpecPathPinsSymlinkTarget(t *testing.T) { + dir := t.TempDir() + imageA := filepath.Join(dir, "image-a") + imageB := filepath.Join(dir, "image-b") + writeSpecFile(t, filepath.Join(imageA, imageSpecFileName)) + writeSpecFile(t, filepath.Join(imageB, imageSpecFileName)) + + alias := filepath.Join(dir, "default") + if err := os.Symlink(imageA, alias); err != nil { + t.Fatal(err) + } + got, ok, err := imageSpecPath(alias) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("imageSpecPath() did not resolve symlinked image") + } + + if err := os.Remove(alias); err != nil { + t.Fatal(err) + } + if err := os.Symlink(imageB, alias); err != nil { + t.Fatal(err) + } + want := filepath.Join(imageA, imageSpecFileName) + if got != want { + t.Fatalf("imageSpecPath() = %q after alias resolution, want pinned target %q", got, want) + } +} func TestResolveImageDirectoryMissingSpec(t *testing.T) { dir := t.TempDir() @@ -135,3 +165,31 @@ func TestImageSpecRequiresShell(t *testing.T) { t.Fatalf("spec without shell should fail validation, got: %v", err) } } +func TestMkfsExt4ForVolumesSkipsLookupWithoutVolumes(t *testing.T) { + t.Setenv("PATH", "") + path, err := mkfsExt4ForVolumes(nil, "") + if err != nil { + t.Fatalf("mkfsExt4ForVolumes() with no volumes returned error: %v", err) + } + if path != "" { + t.Fatalf("mkfsExt4ForVolumes() = %q with no volumes, want empty path", path) + } + + _, err = mkfsExt4ForVolumes([]Volume{{Image: "workspace"}}, "") + if err == nil || !strings.Contains(err.Error(), "mkfs.ext4") { + t.Fatalf("mkfsExt4ForVolumes() with a volume and no formatter returned %v", err) + } +} + +func TestQEMURunnerValidateRejectsUnsupportedNetworkTypeBeforeHostChecks(t *testing.T) { + spec := validImageSpec() + spec.NetworkInterfaces = []NetworkInterface{{ + Type: "tap", + ID: "net0", + MAC: "02:00:00:00:00:01", + }} + err := (qemuRunner{}).Validate(spec, false) + if err == nil || !strings.Contains(err.Error(), `unsupported microvm network interface type "tap"`) { + t.Fatalf("Validate() error = %v, want unsupported network type", err) + } +} diff --git a/spindle/engines/microvm/placement/placement.go b/spindle/engines/microvm/placement/placement.go new file mode 100644 index 00000000..e0bf1fdd --- /dev/null +++ b/spindle/engines/microvm/placement/placement.go @@ -0,0 +1,15 @@ +package placement + +func IsNativeArchitecture(imageArch, goArch string) bool { + normalize := func(arch string) string { + switch arch { + case "x86_64", "amd64": + return "amd64" + case "aarch64", "arm64": + return "arm64" + default: + return arch + } + } + return imageArch != "" && normalize(imageArch) == normalize(goArch) +} diff --git a/spindle/engines/microvm/placement/placement_test.go b/spindle/engines/microvm/placement/placement_test.go new file mode 100644 index 00000000..995255a3 --- /dev/null +++ b/spindle/engines/microvm/placement/placement_test.go @@ -0,0 +1,26 @@ +package placement + +import ( + "testing" +) + +func TestIsNativeArchitecture(t *testing.T) { + tests := []struct { + image string + host string + want bool + }{ + {image: "x86_64", host: "amd64", want: true}, + {image: "amd64", host: "amd64", want: true}, + {image: "aarch64", host: "arm64", want: true}, + {image: "arm64", host: "arm64", want: true}, + {image: "x86_64", host: "arm64", want: false}, + {image: "aarch64", host: "amd64", want: false}, + {image: "", host: "amd64", want: false}, + } + for _, tt := range tests { + if got := IsNativeArchitecture(tt.image, tt.host); got != tt.want { + t.Errorf("IsNativeArchitecture(%q, %q) = %v, want %v", tt.image, tt.host, got, tt.want) + } + } +} diff --git a/spindle/engines/microvm/placement_linux.go b/spindle/engines/microvm/placement_linux.go new file mode 100644 index 00000000..272b9d50 --- /dev/null +++ b/spindle/engines/microvm/placement_linux.go @@ -0,0 +1,61 @@ +//go:build linux + +package microvm + +import ( + "fmt" + "os/exec" + "runtime" + + "tangled.org/core/spindle/engines/microvm/placement" + "tangled.org/core/spindle/models" +) + +func (e *Engine) ValidateWorkflowPlacement(wf *models.Workflow) error { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil { + return fmt.Errorf("microVM workflow state is not initialized") + } + return e.validateImagePlacement(state.ImageSpec) +} + +func (e *Engine) validateImagePlacement(spec ImageSpec) error { + if err := spec.Validate(); err != nil { + return err + } + if !placement.IsNativeArchitecture(spec.Arch, runtime.GOARCH) { + return fmt.Errorf("microVM image architecture %q is not native to executor architecture %q", spec.Arch, runtime.GOARCH) + } + if err := spec.validateImageFiles(); err != nil { + return err + } + runner, err := runnerFor(spec.RunnerType) + if err != nil { + return err + } + if err := runner.Validate(spec, e.cfg.MicroVMPipelines.EnableKVM); err != nil { + return err + } + if len(spec.Volumes) > 0 { + if _, err := exec.LookPath("mkfs.ext4"); err != nil { + return fmt.Errorf("required host command %q not found in PATH: %w", "mkfs.ext4", err) + } + for _, volume := range spec.Volumes { + if volume.ReadOnly { + return fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image) + } + if volume.FSType != "ext4" { + return fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType) + } + if volume.ImageType != "" && volume.ImageType != "raw" { + return fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType) + } + } + } + + request := resourcesForImage(spec) + if !request.Fits(e.budget) || !request.Fits(e.maxWorkflow) { + return fmt.Errorf("microVM image resources exceed executor limits: request=%v budget=%v max=%v", request, e.budget, e.maxWorkflow) + } + return nil +} diff --git a/spindle/engines/microvm/qemu.go b/spindle/engines/microvm/qemu.go index 2d320b42..eb357105 100644 --- a/spindle/engines/microvm/qemu.go +++ b/spindle/engines/microvm/qemu.go @@ -73,6 +73,10 @@ type QEMUVMHandle struct { type qemuRunner struct{} func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error { + b := newArgBuilder(len(spec.NetworkInterfaces) * 4) + if err := addQEMUNetworkArgs(&b, spec.NetworkInterfaces); err != nil { + return err + } if _, err := exec.LookPath(spec.RunnerCmd()); err != nil { return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err) } diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go index 9d3f41ff..560c7c38 100644 --- a/spindle/engines/microvm/vm.go +++ b/spindle/engines/microvm/vm.go @@ -48,6 +48,17 @@ func prepareWorkDir(workDir string) error { return nil } +func mkfsExt4ForVolumes(volumes []Volume, configured string) (string, error) { + if len(volumes) == 0 || configured != "" { + return configured, nil + } + path, err := exec.LookPath("mkfs.ext4") + if err != nil { + return "", fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) + } + return path, nil +} + func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) { paths := make(map[string]string, len(volumes)) for _, volume := range volumes { @@ -362,12 +373,9 @@ func StartVM(ctx context.Context, cfg VMConfig, logger *slog.Logger) (VMHandle, return nil, err } - mkfsExt4 := cfg.MkfsExt4 - if mkfsExt4 == "" { - mkfsExt4, err = exec.LookPath("mkfs.ext4") - if err != nil { - return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) - } + mkfsExt4, err := mkfsExt4ForVolumes(cfg.Image.Volumes, cfg.MkfsExt4) + if err != nil { + return nil, err } volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4) if err != nil { diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index a06d34bf..fb5232c3 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -198,12 +198,13 @@ func (e *Engine) AcquireWorkflowSlot( ctx context.Context, wid models.WorkflowId, wf *models.Workflow, + mode engine.AcquireMode, ) (engine.WorkflowSlot, error) { if e.slotter == nil { return engine.NoopSlot{}, nil } - return e.slotter.AcquireWorkflowSlot(ctx, wid, wf) + return e.slotter.AcquireWorkflowSlot(ctx, wid, wf, mode) } func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) (err error) { diff --git a/spindle/mill/README.md b/spindle/mill/README.md new file mode 100644 index 00000000..0edc7468 --- /dev/null +++ b/spindle/mill/README.md @@ -0,0 +1,159 @@ +# spindle mill + +This document describes the architecture of the mill, spindle's distributed +job placement layer. A mill is a spindle that doesn't run jobs itself: it +places them on remote executors, which run the real engines (microvm, nixery, +dummy) exactly as they would standalone. + +## The engine mirroring model + +Neither side's execution path changes between local and remote runs. + +On the mill host, engines are registered under the real engine names +("microvm", "nixery", "dummy"), but each is a stand-in that places jobs +remotely instead of running them. `InitWorkflow` on the stand-in parses +nothing, it stashes the raw pipeline and workflow and builds a synthetic +one-step workflow, so everything upstream (trigger handling, pending status, +`TANGLED_*` env) behaves exactly like a local run. The real `InitWorkflow` +runs exactly once, on the executor. + +On the executor, the reserve handler wraps the real engine in a +`reservedEngine`: the slot acquired when the reservation is accepted is the +same slot `StartWorkflows` later runs on. Nothing acquires twice, and from the +engine's point of view the job is indistinguishable from a local one. + +The same mirroring applies to output: the executor's jobs write status rows +and log lines exactly like a standalone spindle, and the mill re-authors them +into its own event stream and log store, so appview and the log endpoints see +no difference between a local and a remote job. + +The only real divergence: secrets are withheld from untrusted pipelines at the +mill, and they cross the wire exactly once, inside `CommitLease`, to the one +node that won the bid. Losing bidders never see them. + +## Placement: labels, seats, and rejection + +Placement decides which executors can run a workflow right now, using one +kind of fact: + +- **labels** are operator-defined strings on the executor's token, checked + against the workflow's `runs_on`. they're for coarse fleet partitioning + ("this pool is for trusted jobs", "this box has a gpu"), nothing more + +Matching is exact intersection: candidate sessions whose snapshot has the +engine available, a free seat, and labels covering `runs_on`. Among the +eligible candidates the mill ranks least-loaded first and bids the top-K +concurrently with `ReserveSeat`. The best-ranked accept wins, losers get +`ReleaseLease` so they free their held seats immediately. If nobody +accepts, the job waits for a change: a new executor connecting, a snapshot +flipping availability, or a lease finishing somewhere. + +Image and arch compatibility is deliberately *not* a mill concern. The mill +never parses an image name or an arch string; the executor re-validates +everything at reserve time against its own disk (engine exists, workflow +parses, the image spec validates and is natively runnable, the runner is +usable) and rejects what it can't run with `incompatible`. A reject rotates +the bid to the next candidate, and if every candidate rejects, the user +gets the collected reasons as the placement error. Multi-arch falls out of +this: an arm64 box simply cannot accept an x86_64 image, so it never holds +one. If you *want* to pin an arch or an image explicitly, label the nodes +and use `runs_on`. + +### Example: an alpine microvm job + +Say a workflow asks for the microvm engine with `image: alpine`, no +`runs_on`, and the fleet looks like this: + +| node | arch | labels | engines | load | +|---------|---------|---------|-------------------|--------| +| ci-1 | x86_64 | [linux] | microvm | busy | +| ci-2 | aarch64 | [linux] | microvm | idle | +| ci-3 | x86_64 | [linux] | microvm, nixery | idle | +| ci-4 | x86_64 | [gpu] | microvm | idle | + +The walkthrough: + +```mermaid +flowchart TD + W["workflow
engine: microvm, image: alpine"] --> F{"candidate filter"} + F -->|"all pass runs_on (empty)"| L{"has microvm
available?"} + L -->|all four| R{"rank by load"} + R -->|"ci-1 busy"| X2["ci-1 ranked last"] + R --> B["bid top-K: ci-2, ci-4, ci-1"] +``` + +Say ci-2's alpine image is actually x86_64-only: the mill offers the bid +anyway, ci-2's reserve-time validation rejects it as incompatible, and the +bid rotates to ci-4. The mill learns "ci-2 can't run alpine" from the +rejection, not from any advertisement, and the reason reaches the user if +every candidate fails the same way. + +The bid then runs concurrently: + +```mermaid +sequenceDiagram + participant M as mill + participant C2 as ci-2 + participant C4 as ci-4 + participant C1 as ci-1 + + par bids + M->>C2: ReserveSeat (raw pipeline+workflow) + M->>C4: ReserveSeat + M->>C1: ReserveSeat + end + C2-->>M: accept (idle) + C4-->>M: accept + C1-->>M: reject (transient, seats full) + Note over M: ci-2 ranked above ci-4,
ci-4 gets ReleaseLease + M->>C2: CommitLease (secrets) + C2-->>M: Committed + M->>C4: ReleaseLease +``` + +Both idle nodes accepted, so rank breaks the tie: ci-2 keeps the seat, ci-4 +frees its immediately, and only ci-2 ever sees the secrets. From here ci-2 +runs the job exactly like a standalone spindle would, booting the alpine +image under QEMU, while the mill blocks on the terminal event. + +## The protocol's durability model + +Executor to mill is a single websocket per node, and everything the executor +reports (status, logs, terminal results) travels as sequenced `Event`s. Two +identifiers keep it all consistent: + +- the **epoch** names one lifetime of the executor process. a restarted + executor connects with a fresh epoch, and anything arriving for an old one + is invalid. leases are bound to node+epoch, so a zombie from a previous + process can't act on them +- the **seqno** is a dense per-epoch counter on events. the executor persists + events in a local outbox before sending and trims it only when the mill + acks. on reconnect it resumes from the mill's acked position and replays. + the mill applies idempotently: replays drop, gaps kill the session + +The mill is equally restartable: leases, acked positions and the canonical +log all live in its db. Restored leases start as orphans and must be +reclaimed by the executor's first snapshot, or a sweep fails them after one +grace window. The invariant throughout: at any moment, for any lease, exactly +one of {executor outbox, mill db} holds the newest state, and the seqno/epoch +pair says who. + +## Leases + +A lease is the mill-side handle for one placed job: + +```mermaid +stateDiagram-v2 + [*] --> reserved: bid won + reserved --> committing: CommitLease sent + committing --> running: Committed + running --> done: terminal arrived + reserved --> done: released / expired + committing --> done: released / expired +``` + +Commit retries ride reconnects: a reservation outlives one disconnect, so a +lost session means wait and retry, not a failed job. What ends a job is the +job timeout, a terminal event, or the executor being declared dead after +reconnect grace expires, at which point every lease the node held fails +(preserving a pending cancellation as the reason). diff --git a/spindle/mill/auth_test.go b/spindle/mill/auth_test.go new file mode 100644 index 00000000..2d823c35 --- /dev/null +++ b/spindle/mill/auth_test.go @@ -0,0 +1,482 @@ +package mill + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "tangled.org/core/notifier" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" + + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +func discardLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func nopEncoder() scriptedEncoder { + return scriptedEncoder(func(*millproto.Message) error { return nil }) +} + +func TestHashToken(t *testing.T) { + const raw = "super-secret-executor-token" + + if HashToken(raw) != HashToken(raw) { + t.Fatal("HashToken is not deterministic; the same token would stop authenticating") + } + if HashToken("token-a") == HashToken("token-b") { + t.Fatal("HashToken collided two distinct tokens") + } + if HashToken(raw) == raw { + t.Fatal("HashToken returned the raw token; a hash leak would expose a usable credential") + } +} + +func TestGenerateTokenDistinct(t *testing.T) { + const n = 100 + seen := make(map[string]struct{}, n) + for i := range n { + tok, err := GenerateToken() + if err != nil { + t.Fatalf("GenerateToken: %v", err) + } + if tok == "" { + t.Fatalf("GenerateToken returned an empty token on call %d", i) + } + if _, dup := seen[tok]; dup { + t.Fatalf("GenerateToken repeated a token after %d calls: %q", i, tok) + } + seen[tok] = struct{}{} + } +} + +func TestAttachSessionRejectsSecondLiveSession(t *testing.T) { + l := discardLogger() + m := New(l, Config{ReconnectGrace: time.Minute}) + + sessionOf := func(node string) *millSession { + m.mu.Lock() + defer m.mu.Unlock() + return m.sessions[node] + } + + sess1 := newSession("node-1", "inc-1", nil, nopEncoder(), l) + if _, ok := m.attachSession(sess1); !ok { + t.Fatal("first attach of a node was rejected; want accept") + } + if sessionOf("node-1") != sess1 { + t.Fatal("first session was not registered as the live session") + } + + sess2 := newSession("node-1", "inc-2", nil, nopEncoder(), l) + if _, ok := m.attachSession(sess2); ok { + t.Fatal("second live attach for an already-live node was accepted; a valid token hijacked the executor") + } + if sessionOf("node-1") != sess1 { + t.Fatal("rejected newcomer evicted the incumbent session") + } + + m.detachSession(sess1) + sess3 := newSession("node-1", "inc-3", nil, nopEncoder(), l) + if _, ok := m.attachSession(sess3); !ok { + t.Fatal("attach during the incumbent's reconnect grace was rejected; want adopt") + } + if sessionOf("node-1") != sess3 { + t.Fatal("adopted session was not installed as the live session") + } +} + +func TestOnAttemptResultIgnoresForeignLease(t *testing.T) { + ctx := context.Background() + l := discardLogger() + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + + m := New(l, Config{ReconnectGrace: time.Minute}) + m.Attach(bdb, &n) + + foreign := newLease("lease-foreign", "node-a", "inc-a", "dummy") + m.mu.Lock() + m.leases[foreign.id] = foreign + m.mu.Unlock() + + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) + m.attachSession(sessB) + _ = m.onEventBatch(sessB, &millv1.EventBatch{ + Epoch: sessB.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: foreign.id, + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }) + + if _, ok := pollTerminal(foreign); ok { + t.Fatal("attempt-result on a foreign lease delivered a terminal; an executor forged another node's job result") + } + if foreign.getState() == leaseDone { + t.Fatal("attempt-result on a foreign lease sealed the lease") + } +} + +func TestOnAttemptResultIgnoresAbsentLease(t *testing.T) { + ctx := context.Background() + l := discardLogger() + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + + m := New(l, Config{ReconnectGrace: time.Minute}) + m.Attach(bdb, &n) + + // bystander lease the reporting node owns, proving an absent-lease stream + // does not spill onto another lease + bystander := newLease("lease-bystander", "node-b", "inc-b", "dummy") + m.mu.Lock() + m.leases[bystander.id] = bystander + m.mu.Unlock() + + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) + m.attachSession(sessB) + + _ = m.onEventBatch(sessB, &millv1.EventBatch{ + Epoch: sessB.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: "lease-nonexistent", + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }) + + if _, ok := pollTerminal(bystander); ok { + t.Fatal("attempt-result for an absent lease delivered a terminal to a bystander lease") + } + if bystander.getState() == leaseDone { + t.Fatal("attempt-result for an absent lease sealed a bystander lease") + } +} + +// owned-lease path proves ignore tests above are not passing merely because +// delivery is broken. correctly owned terminal is delivered +func TestOnAttemptResultDeliversOwnedLease(t *testing.T) { + ctx := context.Background() + l := discardLogger() + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + + m := New(l, Config{ReconnectGrace: time.Minute}) + m.Attach(bdb, &n) + + owned := newLease("lease-owned", "node-b", "inc-b", "dummy") + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + sessB := newSession("node-b", "inc-b", nil, nopEncoder(), l) + m.attachSession(sessB) + _ = m.onEventBatch(sessB, &millv1.EventBatch{ + Epoch: sessB.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: owned.id, + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }) + + res, ok := pollTerminal(owned) + if !ok { + t.Fatal("attempt-result on an owned lease was not delivered") + } + if got := res.GetStatus(); got != millv1.TerminalStatus_SUCCESS { + t.Fatalf("delivered terminal status = %v, want %v", got, millv1.TerminalStatus_SUCCESS) + } + if owned.getState() != leaseDone { + t.Fatal("owned lease was not sealed after its terminal was delivered") + } +} + +func TestOnStatusEventOwnership(t *testing.T) { + ctx := context.Background() + l := discardLogger() + + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + + m := New(l, Config{ReconnectGrace: time.Minute}) + m.Attach(bdb, &n) + + foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy") + foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"} + owned := newLease("lease-owned", "node-z", "inc-z", "dummy") + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"} + m.mu.Lock() + m.leases[foreign.id] = foreign + m.leases[owned.id] = owned + m.mu.Unlock() + + sessY := newSession("node-y", "inc-y", nil, nopEncoder(), l) + m.attachSession(sessY) + sessZ := newSession("node-z", "inc-z", nil, nopEncoder(), l) + m.attachSession(sessZ) + + _ = m.onEventBatch(sessY, &millv1.EventBatch{ + Epoch: sessY.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: foreign.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + }, + }, + }, + }, + }) + if _, err := bdb.GetStatus(foreign.wid); err == nil { + t.Fatal("status stream for a foreign lease authored a status row; an executor forged another pipeline's status") + } + + _ = m.onEventBatch(sessZ, &millv1.EventBatch{ + Epoch: sessZ.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: owned.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + }, + }, + }, + }, + }) + st, err := bdb.GetStatus(owned.wid) + if err != nil { + t.Fatalf("owned status stream did not author a status row: %v", err) + } + if st.Status != "running" { + t.Fatalf("owned status = %q, want %q", st.Status, "running") + } +} + +func setupTestServer(t *testing.T, authorizedLabels []string) (*Mill, *db.DB, *httptest.Server, string) { + ctx := context.Background() + bdb, err := db.Make(ctx, filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + + m := New(discardLogger(), Config{ + ReconnectGrace: time.Minute, + }) + m.Attach(bdb, &n) + + const secret = "test-secret" + if err := bdb.AddExecutorToken("dev-node", HashToken(secret), nil, authorizedLabels); err != nil { + t.Fatalf("AddExecutorToken: %v", err) + } + + server := httptest.NewServer(http.HandlerFunc(m.HandleExecutorConn)) + t.Cleanup(server.Close) + + return m, bdb, server, secret +} + +func TestAuthLabelEscalation(t *testing.T) { + _, _, server, secret := setupTestServer(t, []string{"linux", "amd64"}) + + wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") + + { + header := http.Header{} + header.Set("Authorization", "Bearer bad-token") + _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err == nil { + t.Fatal("expected connection with invalid token to fail") + } + if resp != nil && resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 Unauthorized, got %d", resp.StatusCode) + } + } + + { + header := http.Header{} + header.Set("Authorization", "Bearer "+secret) + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + dec := millproto.NewDecoder(stream) + + hello := &millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux", "gpu"}, + Epoch: "inc-1", + }} + if err := enc.Encode(hello); err != nil { + t.Fatalf("encode hello: %v", err) + } + + _, err = dec.Decode() + if err == nil { + t.Fatal("expected server to close connection for unauthorized label, but got a message") + } + } + + { + header := http.Header{} + header.Set("Authorization", "Bearer "+secret) + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + dec := millproto.NewDecoder(stream) + + hello := &millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + }} + if err := enc.Encode(hello); err != nil { + t.Fatalf("encode hello: %v", err) + } + + msg, err := dec.Decode() + if err != nil { + t.Fatalf("expected resume message, got error: %v", err) + } + res := msg.GetResume() + if res == nil { + t.Fatal("expected Resume message, got nil") + } + if res.GetEpoch() != "inc-1" { + t.Fatalf("expected epoch inc-1, got %q", res.GetEpoch()) + } + } +} + +func TestHandshakeTimeoutAndConcurrency(t *testing.T) { + _, _, server, secret := setupTestServer(t, []string{"linux"}) + wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") + + // executor that never sends Hello is dropped after the 5s pre-hello deadline + { + header := http.Header{} + header.Set("Authorization", "Bearer "+secret) + conn, _, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + + time.Sleep(6 * time.Second) + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + hello := &millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + }} + err = enc.Encode(hello) + dec := millproto.NewDecoder(stream) + _, readErr := dec.Decode() + if readErr == nil { + t.Fatal("expected server to have closed connection due to handshake timeout") + } + } + + // second live session for one identity is rejected with 409 before the ws upgrade + { + header := http.Header{} + header.Set("Authorization", "Bearer "+secret) + + conn1, _, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err != nil { + t.Fatalf("dial 1 failed: %v", err) + } + defer conn1.Close() + + stream1 := millproto.NewWSStream(conn1) + enc1 := millproto.NewEncoder(stream1) + dec1 := millproto.NewDecoder(stream1) + hello1 := &millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + }} + if err := enc1.Encode(hello1); err != nil { + t.Fatalf("encode hello 1: %v", err) + } + _, err = dec1.Decode() + if err != nil { + t.Fatalf("first connection handshake failed: %v", err) + } + + _, resp, err := websocket.DefaultDialer.Dial(wsUrl, header) + if err == nil { + t.Fatal("expected second connection for same live identity to be rejected") + } + if resp != nil && resp.StatusCode != http.StatusConflict { + t.Fatalf("expected 409 Conflict for duplicate session, got %d", resp.StatusCode) + } + } +} diff --git a/spindle/mill/engine.go b/spindle/mill/engine.go new file mode 100644 index 00000000..49f446cc --- /dev/null +++ b/spindle/mill/engine.go @@ -0,0 +1,85 @@ +package mill + +import ( + "context" + "log/slog" + "time" + + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" +) + +// raw pipeline/workflow carried forward, executor runs the real InitWorkflow +type millWorkflowState struct { + RawWorkflow tangled.Pipeline_Workflow + RawPipeline tangled.Pipeline + Lease *RemoteLease +} + +// stand-in for a real engine, registered under the real names +// ("microvm", "nixery"), all sharing one Mill +type Engine struct { + name string + mill *Mill + l *slog.Logger +} + +func (e *Engine) AuthorsRemoteStatus() {} + +func NewEngine(name string, mill *Mill) *Engine { + return &Engine{name: name, mill: mill, l: mill.l.With("engine", "mill:"+name)} +} + +// synthetic one-step workflow so processPipeline injects TANGLED_* env +// and marks pending normally. the real InitWorkflow runs exactly once, on +// the executor inside ReserveSeat, and commit reuses that workflow +func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { + return &models.Workflow{ + Name: twf.Name, + Environment: map[string]string{}, + Steps: []models.Step{remoteStep{}}, + Data: &millWorkflowState{ + RawWorkflow: twf, + RawPipeline: tpl, + }, + }, nil +} + +// no-op logger for the synthetic workflow. the executor's lines stream +// into this wid's log directly, a local logger would just write competing +// lines +func (e *Engine) WorkflowLogger(wid models.WorkflowId) models.WorkflowLogger { + return models.NullLogger{} +} + +// the placement seam, blocks on remote placement which the user sees as +// "pending". only StartWorkflows calls this, always Wait +func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { + return e.mill.place(ctx, e.name, wid, wf) +} + +// no-op. real setup happens on the executor +func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { + e.l.Info("remote job placed, awaiting commit", "wid", wid) + return nil +} + +// hands over the secrets and blocks on the terminal result streamed over the +// session +func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, unlocked []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { + return e.mill.commitAndWait(ctx, w, unlocked) +} + +// deliberately generous, the executor enforces the real timeout. the mill +// only caps a hung or silent executor, true death is caught by reconnect grace +func (e *Engine) WorkflowTimeout() time.Duration { + return e.mill.cfg.JobTimeout +} + +// cancels a still-running attempt. no-op if already terminal +func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { + e.mill.destroy(wid) + return nil +} diff --git a/spindle/mill/executor/capability_test.go b/spindle/mill/executor/capability_test.go new file mode 100644 index 00000000..a9b4cde7 --- /dev/null +++ b/spindle/mill/executor/capability_test.go @@ -0,0 +1,119 @@ +package executor + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "testing" + + "tangled.org/core/api/tangled" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +type placementEngine struct { + *fakeEngine + validationErr error + validated bool +} + +func (e *placementEngine) ValidateWorkflowPlacement(*models.Workflow) error { + e.validated = true + return e.validationErr +} + +func TestHandleReserveRejectsMissingTriggerMetadata(t *testing.T) { + enc := newCaptureEncoder() + e := &Executor{ + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + enc: enc, + seats: 1, + engines: map[string]models.Engine{"microvm": &fakeEngine{}}, + active: make(map[string]*reservation), + } + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) + if err != nil { + t.Fatal(err) + } + tpl, err := json.Marshal(tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + + // engines deref TriggerMetadata unconditionally. this must be a reject, + // not a panic that takes the whole executor down + e.handleReserve(context.Background(), &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: string(twf), + RawPipelineJson: string(tpl), + Knot: "k", + Rkey: "r", + }) + + result := (<-enc.messages).GetReserveResult() + if result == nil { + t.Fatal("handleReserve() did not send ReserveResult") + } + if result.GetAccepted() { + t.Fatal("handleReserve() accepted a pipeline without trigger metadata") + } + if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { + t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) + } + if len(e.active) != 0 { + t.Fatalf("active reservations = %d, want 0", len(e.active)) + } +} + +func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { + enc := newCaptureEncoder() + validationErr := errors.New("image architecture is not native") + eng := &placementEngine{fakeEngine: &fakeEngine{}, validationErr: validationErr} + e := &Executor{ + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + enc: enc, + seats: 1, + engines: map[string]models.Engine{"microvm": eng}, + active: make(map[string]*reservation), + } + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) + if err != nil { + t.Fatal(err) + } + tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + if err != nil { + t.Fatal(err) + } + + e.handleReserve(context.Background(), &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: string(twf), + RawPipelineJson: string(tpl), + Knot: "k", + Rkey: "r", + }) + + result := (<-enc.messages).GetReserveResult() + if result == nil { + t.Fatal("handleReserve() did not send ReserveResult") + } + if result.GetAccepted() { + t.Fatal("handleReserve() accepted placement validation failure") + } + if result.GetRejectClass() != millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { + t.Fatalf("reject class = %v, want incompatible", result.GetRejectClass()) + } + if !eng.validated { + t.Fatal("placement validator was not called") + } + if eng.acquireCalled { + t.Fatal("slot acquisition ran after placement validation failed") + } + if len(e.active) != 0 { + t.Fatalf("active reservations = %d, want 0", len(e.active)) + } +} diff --git a/spindle/mill/executor/executor.go b/spindle/mill/executor/executor.go new file mode 100644 index 00000000..c198fd7f --- /dev/null +++ b/spindle/mill/executor/executor.go @@ -0,0 +1,613 @@ +package executor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "runtime" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "log/slog" + "strings" + + "tangled.org/core/api/tangled" + "tangled.org/core/netutil" + "tangled.org/core/notifier" + "tangled.org/core/spindle/artifactstore" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +const ( + dialBackoffMin = 1 * time.Second + dialBackoffMax = 30 * time.Second + snapshotEvery = 15 * time.Second + defaultSeats = 4 +) + +type Executor struct { + millURL string + token string + nodeID string + seats int + labels []string + + engines map[string]models.Engine + db *db.DB + n *notifier.Notifier + cfg *config.Config + l *slog.Logger + writer artifactstore.Writer + + epoch string + outboxBytes int64 + maxOutboxBytes int64 // 10 MiB default outbox cap + + eventMu sync.Mutex + sendMu sync.Mutex + flushMu sync.Mutex + + sentSeqno uint64 + + connMu sync.Mutex + enc messageEncoder + sessionCancel context.CancelFunc + + mu sync.Mutex + active map[string]*reservation + draining bool + + snapshotMu sync.Mutex + nextSeqno uint64 + + lifecycleCtx context.Context + jobsWG sync.WaitGroup +} + +type reservation struct { + leaseID string + wid models.WorkflowId + realEngine models.Engine + slot engine.WorkflowSlot + wf *models.Workflow + repoDid syntax.DID + vault *memVault + + committed bool + cancelled bool + cancel context.CancelFunc + ttlTimer *time.Timer + stopTail func() +} + +type messageEncoder interface { + Encode(*millproto.Message) error +} + +func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { + seats := defaultSeats + millURL := "" + token := "" + nodeID := "" + var labels []string + if cfg != nil { + if cfg.Mill.Seats > 0 { + seats = cfg.Mill.Seats + } + labels = normalizeLabels(cfg.Mill.Labels) + millURL = cfg.Mill.URL + token = cfg.Mill.SharedSecret + nodeID = cfg.Server.Hostname + } + if d == nil || n == nil { + return nil, fmt.Errorf("executor requires a database and notifier") + } + var writer artifactstore.Writer + if len(writers) > 0 { + writer = writers[0] + } + e := &Executor{ + millURL: millURL, + token: token, + nodeID: nodeID, + seats: seats, + labels: labels, + engines: engines, + db: d, + n: n, + cfg: cfg, + l: l.With("component", "mill.executor"), + writer: writer, + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + } + if err := e.initOutbox(); err != nil { + return nil, fmt.Errorf("initialize executor outbox: %w", err) + } + return e, nil +} + +func (e *Executor) Connect(ctx context.Context) { + e.lifecycleCtx = ctx + sub := e.n.Subscribe() + cursor, err := e.db.EventHighWater() + if err != nil { + e.n.Unsubscribe(sub) + e.l.Error("establish event cursor failed", "err", err) + return + } + e.drainEvents(&cursor) + observerCtx, stopObserver := context.WithCancel(ctx) + observerDone := make(chan struct{}) + go func() { + defer close(observerDone) + e.observeLoop(observerCtx, sub, cursor) + }() + defer e.n.Unsubscribe(sub) + + backoff := dialBackoffMin + for { + if ctx.Err() != nil { + break + } + err := e.runSession(ctx) + if ctx.Err() != nil { + break + } + e.l.Warn("mill session ended; reconnecting", "err", err, "backoff", backoff) + select { + case <-ctx.Done(): + break + case <-time.After(backoff): + } + backoff = min(backoff*2, dialBackoffMax) + } + + e.jobsWG.Wait() + stopObserver() + <-observerDone + e.drainEvents(&cursor) +} + +func (e *Executor) runSession(ctx context.Context) error { + dev := e.cfg == nil || e.cfg.Server.Dev + if _, err := netutil.EnforceWSSURL(e.millURL, dev); err != nil { + return fmt.Errorf("mill url: %w", err) + } + header := http.Header{} + if e.token != "" { + header.Set("Authorization", "Bearer "+e.token) + } + conn, _, err := netutil.SSRFWebsocketDialer(dev).DialContext(ctx, e.millURL, header) + if err != nil { + return fmt.Errorf("dial mill: %w", err) + } + defer conn.Close() + + sessionCtx, cancelSession := context.WithCancel(ctx) + defer cancelSession() + stopClose := context.AfterFunc(sessionCtx, func() { _ = conn.Close() }) + defer stopClose() + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + dec := millproto.NewDecoder(stream) + + hello := &millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion, + Arch: runtime.GOARCH, + Labels: e.labels, + Epoch: e.epoch, + }} + if err := enc.Encode(hello); err != nil { + return fmt.Errorf("send hello: %w", err) + } + + resumeMsg, err := dec.Decode() + if err != nil { + return fmt.Errorf("read resume: %w", err) + } + resume := resumeMsg.GetResume() + if resume == nil { + return fmt.Errorf("expected resume, got something else") + } + if resume.GetEpoch() != e.epoch { + return fmt.Errorf("resume epoch mismatch: got %q, want %q", resume.GetEpoch(), e.epoch) + } + + e.connMu.Lock() + e.sessionCancel = cancelSession + e.enc = enc + e.connMu.Unlock() + defer func() { + e.connMu.Lock() + e.enc = nil + e.sessionCancel = nil + e.connMu.Unlock() + }() + + readErr := make(chan error, 1) + go func() { + for { + msg, err := dec.Decode() + if err != nil { + readErr <- fmt.Errorf("read: %w", err) + return + } + e.dispatch(sessionCtx, msg) + } + }() + + if err := e.replay(resume.GetAckSeqno()); err != nil { + cancelSession() + <-readErr + return fmt.Errorf("replay failed: %w", err) + } + e.pushSnapshot() + e.l.Info("connected to mill", "node", e.nodeID, "resumeFrom", resume.GetAckSeqno()) + + go e.snapshotLoop(sessionCtx, enc) + return <-readErr +} +func (e *Executor) send(msg *millproto.Message) { + e.connMu.Lock() + enc := e.enc + cancel := e.sessionCancel + e.connMu.Unlock() + if enc != nil { + e.sendMu.Lock() + err := enc.Encode(msg) + e.sendMu.Unlock() + if err != nil { + e.l.Error("send failed, ending session", "err", err) + if cancel != nil { + cancel() + } + } + } +} + +func (e *Executor) dispatch(ctx context.Context, msg *millproto.Message) { + switch { + case msg.GetReserveSeat() != nil: + e.handleReserve(ctx, msg.GetReserveSeat()) + case msg.GetCommitLease() != nil: + e.handleCommit(ctx, msg.GetCommitLease()) + case msg.GetReleaseLease() != nil: + e.handleRelease(msg.GetReleaseLease().GetLeaseId()) + case msg.GetCancelAttempt() != nil: + e.handleCancel(msg.GetCancelAttempt().GetLeaseId()) + case msg.GetAck() != nil: + e.handleAck(msg.GetAck()) + default: + e.l.Warn("unhandled incoming message", "type", fmt.Sprintf("%T", msg)) + } +} + +func (e *Executor) sendReject(leaseID string, reason string, class millv1.RejectClass) { + e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ + LeaseId: leaseID, + Accepted: false, + RejectReason: reason, + RejectClass: class, + }}) +} + +func (e *Executor) sendCommitted(leaseID string) { + e.send(&millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) +} + +func (e *Executor) sendCancelAck(leaseID string) { + e.send(&millproto.Message{CancelAck: &millv1.CancelAck{LeaseId: leaseID}}) +} + +func (e *Executor) releaseReservation(cleanup func()) { + if cleanup != nil { + cleanup() + } + e.pushSnapshot() +} + +func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { + reject := func(reason string, class millv1.RejectClass) { + e.sendReject(rs.GetLeaseId(), reason, class) + } + + e.mu.Lock() + draining := e.draining + e.mu.Unlock() + if draining { + reject("draining", millv1.RejectClass_REJECT_CLASS_TRANSIENT) + return + } + + realEngine, ok := e.engines[rs.GetTargetEngine()] + if !ok { + reject("unknown engine "+rs.GetTargetEngine(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + slotter, ok := realEngine.(engine.WorkflowSlotter) + if !ok { + reject("engine does not support workflow slots", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + + var twf tangled.Pipeline_Workflow + if err := json.Unmarshal([]byte(rs.GetRawWorkflowJson()), &twf); err != nil { + reject("bad workflow json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + var tpl tangled.Pipeline + if err := json.Unmarshal([]byte(rs.GetRawPipelineJson()), &tpl); err != nil { + reject("bad pipeline json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + if tpl.TriggerMetadata == nil { + reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + + pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()} + wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name} + + wf, err := realEngine.InitWorkflow(twf, tpl) + if err != nil { + reject("init workflow: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok { + if err := validator.ValidateWorkflowPlacement(wf); err != nil { + reject("validate workflow placement: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + } + if wf.Environment == nil { + wf.Environment = make(map[string]string) + } + maps.Copy(wf.Environment, models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId)) + + slot, err := slotter.AcquireWorkflowSlot(ctx, wid, wf, engine.NoWait) + if err != nil { + class := millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE + if errors.Is(err, engine.ErrNoWorkflowSlots) { + class = millv1.RejectClass_REJECT_CLASS_TRANSIENT + } + reject(err.Error(), class) + return + } + + var repoDid syntax.DID + if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil { + repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid) + } + + res := &reservation{ + leaseID: rs.GetLeaseId(), + wid: wid, + realEngine: realEngine, + slot: slot, + wf: wf, + repoDid: repoDid, + } + + e.snapshotMu.Lock() + e.mu.Lock() + e.active[res.leaseID] = res + res.ttlTimer = time.AfterFunc(ttlDuration(rs.GetTtlSeconds()), func() { e.expireReservation(res.leaseID) }) + e.mu.Unlock() + + e.send(&millproto.Message{ReserveResult: &millv1.ReserveResult{ + LeaseId: rs.GetLeaseId(), + Accepted: true, + }}) + e.pushSnapshotLocked() + e.snapshotMu.Unlock() +} + +func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { + e.mu.Lock() + res := e.active[cl.GetLeaseId()] + if res == nil { + e.mu.Unlock() + e.sendReject(cl.GetLeaseId(), "reservation missing or expired", millv1.RejectClass_REJECT_CLASS_TRANSIENT) + return + } + if res.committed { + e.mu.Unlock() + e.sendCommitted(cl.GetLeaseId()) + return + } + res.committed = true + if res.ttlTimer != nil { + res.ttlTimer.Stop() + } + + jobCtx, cancel := context.WithCancel(e.lifecycleCtx) + res.cancel = cancel + e.mu.Unlock() + + vault := newMemVault(cl.GetSecrets()) + re := newReservedEngine(res.realEngine, res.slot) + pipeline := &models.Pipeline{ + RepoDid: res.repoDid, + Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}}, + TrustedSource: true, + } + + e.startTail(res) + + e.jobsWG.Add(1) + go func() { + defer e.jobsWG.Done() + engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) + }() + + e.sendCommitted(cl.GetLeaseId()) +} + +func (e *Executor) handleRelease(leaseID string) { + cleanup, ok := e.takeUncommittedReservation(leaseID, true) + if !ok { + return + } + e.releaseReservation(cleanup) +} + +func (e *Executor) handleCancel(leaseID string) { + e.mu.Lock() + res := e.active[leaseID] + if res == nil { + e.mu.Unlock() + if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { + e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) + return + } + e.sendCancelAck(leaseID) + return + } + res.cancelled = true + cancel := res.cancel + committed := res.committed + var cleanup func() + if !committed { + cleanup = e.removeReservationLocked(res, true) + } + e.mu.Unlock() + + if !committed { + if err := e.appendTerminal(leaseID, string(models.StatusKindCancelled), nil); err != nil { + e.l.Error("persist cancelled reservation terminal", "lease", leaseID, "err", err) + e.releaseReservation(cleanup) + return + } + e.sendCancelAck(leaseID) + e.releaseReservation(cleanup) + return + } + + e.sendCancelAck(leaseID) + if cancel != nil { + cancel() + } +} + +func (e *Executor) expireReservation(leaseID string) { + cleanup, ok := e.takeUncommittedReservation(leaseID, true) + if !ok { + return + } + e.releaseReservation(cleanup) +} + +func (e *Executor) takeUncommittedReservation(leaseID string, releaseSlot bool) (func(), bool) { + e.mu.Lock() + defer e.mu.Unlock() + res := e.active[leaseID] + if res == nil || res.committed { + return nil, false + } + if res.ttlTimer != nil { + res.ttlTimer.Stop() + } + return e.removeReservationLocked(res, releaseSlot), true +} + +func (e *Executor) removeReservationLocked(res *reservation, releaseSlot bool) func() { + delete(e.active, res.leaseID) + slot := res.slot + return func() { + if releaseSlot && slot != nil { + slot.Release() + } + } +} + +func (e *Executor) snapshotLoop(ctx context.Context, enc *millproto.Encoder) { + ticker := time.NewTicker(snapshotEvery) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + e.pushSnapshot() + } + } +} + +func (e *Executor) pushSnapshot() { + e.snapshotMu.Lock() + defer e.snapshotMu.Unlock() + e.pushSnapshotLocked() +} + +func (e *Executor) pushSnapshotLocked() { + e.mu.Lock() + activeLeases := make([]string, 0, len(e.active)) + for leaseID := range e.active { + activeLeases = append(activeLeases, leaseID) + } + e.mu.Unlock() + + avail := make(map[string]*millv1.EngineAvailability) + for name, eng := range e.engines { + a := &millv1.EngineAvailability{Available: true} + if getter, ok := eng.(interface{ Load() map[string]float64 }); ok { + a.Load = getter.Load() + } + avail[name] = a + } + + e.nextSeqno++ + snap := &millproto.Message{ + NodeSnapshot: &millv1.NodeSnapshot{ + Seqno: e.nextSeqno, + Engines: avail, + ActiveLeaseIds: activeLeases, + }, + } + e.send(snap) +} + +func (e *Executor) Drain() { + e.mu.Lock() + e.draining = true + e.mu.Unlock() + e.pushSnapshot() +} + +func ttlDuration(secs uint32) time.Duration { + if secs == 0 { + return defaultReservationTTL + } + return time.Duration(secs) * time.Second +} + +const defaultReservationTTL = 60 * time.Second + +func normalizeLabels(labels []string) []string { + seen := make(map[string]struct{}, len(labels)) + out := make([]string, 0, len(labels)) + for _, label := range labels { + label = strings.TrimSpace(label) + if label == "" { + continue + } + if _, ok := seen[label]; ok { + continue + } + seen[label] = struct{}{} + out = append(out, label) + } + return out +} diff --git a/spindle/mill/executor/observe.go b/spindle/mill/executor/observe.go new file mode 100644 index 00000000..dc5d1360 --- /dev/null +++ b/spindle/mill/executor/observe.go @@ -0,0 +1,251 @@ +package executor + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "strings" + "sync" + "time" + + "github.com/hpcloud/tail" + "tangled.org/core/api/tangled" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" +) + +func (e *Executor) observeLoop(ctx context.Context, sub <-chan struct{}, cursor int64) { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-sub: + case <-ticker.C: + } + e.drainEvents(&cursor) + } +} + +func (e *Executor) drainEvents(cursor *int64) { + events, err := e.db.GetEvents(*cursor, 128) + if err != nil { + e.l.Error("drain status events failed", "err", err) + return + } + for _, ev := range events { + if ev.Created > *cursor { + *cursor = ev.Created + } + st, ok := parseStatus(ev.EventJson) + if !ok { + continue + } + if err := e.onStatusRow(st); err != nil { + e.l.Error("process status row failed", "err", err) + } + } +} + +func (e *Executor) onStatusRow(st *tangled.PipelineStatus) error { + res := e.reservationFor(st.Pipeline, st.Workflow) + if res == nil { + return nil + } + + if models.StatusKind(st.Status).IsFinish() { + return e.finishJob(res, st) + } + + return e.appendStatus(res.leaseID, st) +} + +func (e *Executor) finishJob(res *reservation, st *tangled.PipelineStatus) error { + e.mu.Lock() + if e.active[res.leaseID] != res { + e.mu.Unlock() + return nil + } + cancelled := res.cancelled + e.mu.Unlock() + + // 1. Finalize log tail first so all log lines precede terminal event + if res.stopTail != nil { + res.stopTail() + } + + terminalStatus := st.Status + if cancelled { + terminalStatus = string(models.StatusKindCancelled) + } + var logDir string + if e.cfg != nil { + logDir = e.cfg.Server.LogDir + } + logPath := models.LogFilePath(logDir, res.wid) + + var errStr string + if st != nil && st.Error != nil { + errStr = *st.Error + } + var exitCode int64 + if st != nil && st.ExitCode != nil { + exitCode = *st.ExitCode + } + + // Calculate SHA256 of log file + hash := "" + if f, err := os.Open(logPath); err == nil { + h := sha256.New() + if _, err := io.Copy(h, f); err == nil { + hash = "sha256:" + hex.EncodeToString(h.Sum(nil)) + } + _ = f.Close() + } + + // refs are opaque keys interpreted by the configured artifact store + ref := "logs/" + res.leaseID + ".log" + + // 2. Persist restart-retryable pending artifact state + if e.db != nil { + _ = e.db.SavePendingArtifact(res.leaseID, res.wid.Name, terminalStatus, errStr, exitCode, ref, hash) + } + + // 3. Upload artifact with non-cancelled cleanup context + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Minute) + defer cancel() + + if e.writer != nil { + if f, err := os.Open(logPath); err == nil { + defer f.Close() + if uploadErr := e.writer.Put(cleanupCtx, ref, f); uploadErr != nil { + e.l.Error("artifact upload failed", "lease", res.leaseID, "err", uploadErr) + return fmt.Errorf("artifact upload: %w", uploadErr) + } + } + } + + // 4. Append terminal event (with LogArtifact) to outbox + if err := e.appendTerminalWithArtifact(res.leaseID, terminalStatus, st, ref, hash); err != nil { + return err + } + + // 5. Remove pending artifact state after terminal outbox append succeeds + if e.db != nil { + _ = e.db.RemovePendingArtifact(res.leaseID) + } + e.mu.Lock() + cleanup := e.removeReservationLocked(res, false) + e.mu.Unlock() + + cleanup() + + e.pushSnapshot() + return nil +} + +func (e *Executor) reservationFor(pipelineAturi, workflow string) *reservation { + e.mu.Lock() + defer e.mu.Unlock() + for _, res := range e.active { + if string(res.wid.PipelineId.AtUri()) == pipelineAturi && res.wid.Name == workflow { + return res + } + } + return nil +} + +func (e *Executor) maskSecrets(res *reservation, text string) string { + if res == nil || res.vault == nil { + return text + } + for _, s := range res.vault.secrets { + if s.Value != "" { + text = strings.ReplaceAll(text, s.Value, "***") + } + } + return text +} + +func (e *Executor) SendLiveLog(leaseID string, raw []byte) error { + e.connMu.Lock() + enc := e.enc + e.connMu.Unlock() + if enc == nil { + return nil + } + return enc.Encode(&millproto.Message{ + LiveLog: &millv1.LiveLog{ + LeaseId: leaseID, + RawJson: raw, + }, + }) +} + +func (e *Executor) startTail(res *reservation) { + path := models.LogFilePath(e.cfg.Server.LogDir, res.wid) + t, err := tail.TailFile(path, tail.Config{ + Follow: true, + ReOpen: true, + MustExist: false, + Location: &tail.SeekInfo{Offset: 0, Whence: io.SeekStart}, + Logger: tail.DiscardingLogger, + }) + if err != nil { + e.l.Error("tail log file failed", "wid", res.wid, "err", err) + return + } + + done := make(chan struct{}) + go func() { + defer close(done) + for line := range t.Lines { + if line == nil || line.Err != nil { + continue + } + masked := e.maskSecrets(res, line.Text) + _ = e.SendLiveLog(res.leaseID, []byte(masked+"\n")) + } + }() + + var once sync.Once + res.stopTail = func() { + once.Do(func() { + _ = t.StopAtEOF() + <-done + }) + } +} + +type memVault struct { + secrets []secrets.UnlockedSecret +} + +func newMemVault(pb []*millv1.Secret) *memVault { + v := &memVault{secrets: make([]secrets.UnlockedSecret, 0, len(pb))} + for _, s := range pb { + v.secrets = append(v.secrets, secrets.UnlockedSecret{ + Key: s.Key, + Value: s.Value, + }) + } + return v +} + +func (v *memVault) GetSecretsUnlocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) { + return v.secrets, nil +} +func (v *memVault) GetSecretsLocked(ctx context.Context, repo secrets.RepoIdentifier) ([]secrets.LockedSecret, error) { + return nil, nil +} +func (v *memVault) AddSecret(ctx context.Context, s secrets.UnlockedSecret) error { return nil } +func (v *memVault) RemoveSecret(ctx context.Context, s secrets.Secret[any]) error { return nil } + +var _ secrets.Manager = (*memVault)(nil) diff --git a/spindle/mill/executor/outbox.go b/spindle/mill/executor/outbox.go new file mode 100644 index 00000000..20ee0650 --- /dev/null +++ b/spindle/mill/executor/outbox.go @@ -0,0 +1,344 @@ +package executor + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "time" + + "google.golang.org/protobuf/proto" + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/db" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +const ( + maxBatchBytes = 4 * 1024 * 1024 + maxBatchEvents = 128 +) + +func generateEpoch() string { + var b [8]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +func (e *Executor) initOutbox() error { + e.eventMu.Lock() + defer e.eventMu.Unlock() + + epoch, _, err := e.db.GetOutboxState() + if err != nil { + return err + } + if epoch != "" { + e.epoch = epoch + } else { + e.epoch = generateEpoch() + if err := e.db.SetOutboxEpoch(e.epoch); err != nil { + return fmt.Errorf("set outbox epoch: %w", err) + } + } + + rows, err := e.db.ListOutboxRows() + if err != nil { + return fmt.Errorf("list outbox rows: %w", err) + } + + for _, r := range rows { + e.outboxBytes += r.ByteSize + } + _ = e.recoverPendingArtifacts() + return nil +} + +func (e *Executor) appendAndSend(leaseID string, payload any, control bool) error { + entry := &millv1.Event{LeaseId: leaseID} + switch payload := payload.(type) { + case *millv1.Event_StatusEvent: + entry.Payload = payload + case *millv1.Event_AttemptResult: + entry.Payload = payload + default: + return fmt.Errorf("unsupported stream payload %T", payload) + } + isTerminal := false + if _, ok := payload.(*millv1.Event_AttemptResult); ok { + isTerminal = true + } + entry.Seqno = ^uint64(0) + wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{ + Epoch: e.epoch, + Events: []*millv1.Event{entry}, + }}) + entry.Seqno = 0 + if wireSize > maxBatchBytes { + return fmt.Errorf("control stream entry exceeds wire limit: %d > %d", wireSize, maxBatchBytes) + } + + encoded, err := proto.Marshal(entry) + if err != nil { + return fmt.Errorf("marshal stream entry: %w", err) + } + + e.eventMu.Lock() + if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { + e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) + e.eventMu.Unlock() + return nil + } + if _, err := e.db.AppendOutboxRow(encoded, control); err != nil { + e.eventMu.Unlock() + return fmt.Errorf("append outbox row: %w", err) + } + e.outboxBytes += int64(len(encoded)) + e.eventMu.Unlock() + + e.sendPending() + return nil +} + +func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) error { + if st.Status != string(models.StatusKindRunning) { + return fmt.Errorf("unsupported nonterminal status %q", st.Status) + } + exit, errStr := parseStatusExitAndError(st) + payload := &millv1.Event_StatusEvent{StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + Error: errStr, + ExitCode: exit, + }} + return e.appendAndSend(leaseID, payload, true) +} + +func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { + return e.appendTerminalWithArtifact(leaseID, status, st, "", "") +} + +func (e *Executor) appendTerminalWithArtifact(leaseID, status string, st *tangled.PipelineStatus, ref, hash string) error { + var terminalStatus millv1.TerminalStatus + switch status { + case string(models.StatusKindSuccess): + terminalStatus = millv1.TerminalStatus_SUCCESS + case string(models.StatusKindFailed): + terminalStatus = millv1.TerminalStatus_FAILED + case string(models.StatusKindTimeout): + terminalStatus = millv1.TerminalStatus_TIMEOUT + case string(models.StatusKindCancelled): + terminalStatus = millv1.TerminalStatus_CANCELLED + default: + return fmt.Errorf("unsupported terminal status %q", status) + } + + exit, errStr := parseStatusExitAndError(st) + var logArtifact *millv1.LogArtifact + if ref != "" { + logArtifact = &millv1.LogArtifact{ + Ref: ref, + Hash: hash, + } + } + payload := &millv1.Event_AttemptResult{AttemptResult: &millv1.AttemptResult{ + Status: terminalStatus, + Error: errStr, + ExitCode: exit, + LogArtifact: logArtifact, + }} + return e.appendAndSend(leaseID, payload, true) +} + +func (e *Executor) recoverPendingArtifacts() error { + if e.db == nil { + return nil + } + pending, err := e.db.ListPendingArtifacts() + if err != nil || len(pending) == 0 { + return err + } + for _, p := range pending { + if p.Ref != "" { + if e.writer == nil { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + var logDir string + if e.cfg != nil { + logDir = e.cfg.Server.LogDir + } + logPath := models.LogFilePath(logDir, models.WorkflowId{Name: p.Workflow}) + f, openErr := os.Open(logPath) + if openErr != nil { + cancel() + continue + } + uploadErr := e.writer.Put(ctx, p.Ref, f) + _ = f.Close() + cancel() + if uploadErr != nil { + continue + } + } + + st := &tangled.PipelineStatus{ + Status: p.Status, + Error: &p.Error, + ExitCode: &p.ExitCode, + } + if err := e.appendTerminalWithArtifact(p.LeaseID, p.Status, st, p.Ref, p.Hash); err == nil { + _ = e.db.RemovePendingArtifact(p.LeaseID) + } + } + return nil +} + +func truncateEventString(value string) string { + if len(value) > 65536 { + return value[:65536] + } + return value +} + +func (e *Executor) sendPending() { + e.connMu.Lock() + enc := e.enc + e.connMu.Unlock() + + if enc == nil { + return + } + + e.flushMu.Lock() + defer e.flushMu.Unlock() + + if err := e.sendPendingLocked(enc); err != nil { + e.l.Error("send pending events failed", "err", err) + } +} + +func (e *Executor) sendPendingLocked(enc messageEncoder) error { + for { + rows, err := e.db.ListOutboxRowsAfter(e.sentSeqno, maxBatchEvents) + if err != nil { + return fmt.Errorf("list outbox rows after %d: %w", e.sentSeqno, err) + } + if len(rows) == 0 { + return nil + } + if err := e.sendRows(enc, rows); err != nil { + return err + } + } +} + +func (e *Executor) sendRows(enc messageEncoder, rows []db.OutboxRow) error { + events := make([]*millv1.Event, 0, len(rows)) + var lastSeqno uint64 + + for _, r := range rows { + ev, err := decodeEvent(r.Payload, r.Seqno) + if err != nil { + return fmt.Errorf("decode event at seqno %d: %w", r.Seqno, err) + } + events = append(events, ev) + lastSeqno = r.Seqno + } + + batch := &millv1.EventBatch{ + Epoch: e.epoch, + Events: events, + } + + e.sendMu.Lock() + defer e.sendMu.Unlock() + + if err := enc.Encode(&millproto.Message{EventBatch: batch}); err != nil { + return fmt.Errorf("encode event batch (seqno %d..%d): %w", rows[0].Seqno, lastSeqno, err) + } + e.sentSeqno = lastSeqno + return nil +} + +func (e *Executor) replay(ackSeqno uint64) error { + e.connMu.Lock() + enc := e.enc + e.connMu.Unlock() + if enc == nil { + return nil + } + + e.flushMu.Lock() + defer e.flushMu.Unlock() + + e.sendMu.Lock() + e.sentSeqno = ackSeqno + e.sendMu.Unlock() + + return e.sendPendingLocked(enc) +} + +func (e *Executor) handleAck(ack *millv1.Ack) { + if ack == nil || ack.GetEpoch() != e.epoch { + return + } + + upTo := ack.GetUpToSeqno() + if upTo == 0 { + return + } + + if err := e.deleteOutboxPrefix(upTo); err != nil { + e.l.Error("delete outbox prefix failed", "upTo", upTo, "err", err) + } +} + +func (e *Executor) subtractOutboxBytes(deleted db.OutboxDeletion) { + e.outboxBytes = max(0, e.outboxBytes-deleted.Bytes) +} + +func parseStatus(raw json.RawMessage) (*tangled.PipelineStatus, bool) { + var st tangled.PipelineStatus + if err := json.Unmarshal(raw, &st); err != nil { + return nil, false + } + return &st, true +} + +func (e *Executor) deleteOutboxPrefix(upTo uint64) error { + e.eventMu.Lock() + defer e.eventMu.Unlock() + + deleted, err := e.db.DeleteOutboxPrefix(upTo) + if err == nil { + e.subtractOutboxBytes(deleted) + } + return err +} + +func parseStatusExitAndError(st *tangled.PipelineStatus) (int64, string) { + if st == nil { + return 0, "" + } + var exit int64 + if st.ExitCode != nil { + exit = *st.ExitCode + } + var errStr string + if st.Error != nil { + errStr = truncateEventString(*st.Error) + } + return exit, errStr +} + +func decodeEvent(payload []byte, seqno uint64) (*millv1.Event, error) { + var ev millv1.Event + if err := proto.Unmarshal(payload, &ev); err != nil { + return nil, err + } + ev.Seqno = seqno + return &ev, nil +} diff --git a/spindle/mill/executor/reserved.go b/spindle/mill/executor/reserved.go new file mode 100644 index 00000000..4a77d926 --- /dev/null +++ b/spindle/mill/executor/reserved.go @@ -0,0 +1,37 @@ +package executor + +import ( + "context" + "fmt" + "sync" + + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" +) + +// wraps a real engine so StartWorkflows gets the slot ReserveSeat already +// acquired, not a second one. everything else delegates, the execution +// path runs exactly like standalone +type reservedEngine struct { + models.Engine + slot engine.WorkflowSlot + once sync.Once +} + +func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine { + return &reservedEngine{Engine: inner, slot: slot} +} + +// hands back the pre-acquired slot exactly once, a second acquire would +// double-count it +func (e *reservedEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, _ engine.AcquireMode) (engine.WorkflowSlot, error) { + var slot engine.WorkflowSlot + e.once.Do(func() { + slot = e.slot + e.slot = nil + }) + if slot == nil { + return nil, fmt.Errorf("reserved slot already consumed") + } + return slot, nil +} diff --git a/spindle/mill/executor/reserved_test.go b/spindle/mill/executor/reserved_test.go new file mode 100644 index 00000000..7fbc9816 --- /dev/null +++ b/spindle/mill/executor/reserved_test.go @@ -0,0 +1,546 @@ +package executor + +import ( + "context" + "encoding/json" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/gorilla/websocket" + "google.golang.org/protobuf/proto" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "tangled.org/core/api/tangled" + "tangled.org/core/notifier" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" +) + +type captureEncoder struct { + messages chan *millproto.Message +} + +func newCaptureEncoder() *captureEncoder { + return &captureEncoder{messages: make(chan *millproto.Message, 4)} +} + +func (e *captureEncoder) Encode(msg *millproto.Message) error { + e.messages <- msg + return nil +} + +type fakeSlot struct{ released int } + +func (s *fakeSlot) Release() { s.released++ } + +type fakeEngine struct { + setupCalled bool + runCalled bool + destroyCalled bool + acquireCalled bool + secrets chan []secrets.UnlockedSecret + done chan struct{} +} + +func (e *fakeEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { + return &models.Workflow{Name: twf.Name}, nil +} +func (e *fakeEngine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, l models.WorkflowLogger) error { + e.setupCalled = true + return nil +} +func (e *fakeEngine) WorkflowTimeout() time.Duration { return 7 * time.Minute } +func (e *fakeEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { + e.destroyCalled = true + return nil +} +func (e *fakeEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, s []secrets.UnlockedSecret, l models.WorkflowLogger) error { + e.runCalled = true + if e.secrets != nil { + e.secrets <- s + } + if e.done != nil { + close(e.done) + } + return nil +} +func (e *fakeEngine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, mode engine.AcquireMode) (engine.WorkflowSlot, error) { + e.acquireCalled = true + return &fakeSlot{}, nil +} + +type fakeStep struct{} + +func (fakeStep) Name() string { return "test" } +func (fakeStep) Command() string { return "true" } +func (fakeStep) Kind() models.StepKind { return models.StepKindUser } + +func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) { + d := testDB(t) + if err := d.Close(); err != nil { + t.Fatal(err) + } + n := notifier.New() + cfg := &config.Config{} + if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil { + t.Fatal("New succeeded with an unavailable outbox database") + } +} + +func TestReservedEngineHandsBackHeldSlotOnce(t *testing.T) { + inner := &fakeEngine{} + slot := &fakeSlot{} + re := newReservedEngine(inner, slot) + + got, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait) + if err != nil { + t.Fatal(err) + } + if got != engine.WorkflowSlot(slot) { + t.Fatal("AcquireWorkflowSlot() did not return the held slot") + } + if inner.acquireCalled { + t.Fatal("wrapper must not call the inner engine's AcquireWorkflowSlot") + } + + if _, err := re.(engine.WorkflowSlotter).AcquireWorkflowSlot(context.Background(), models.WorkflowId{}, nil, engine.Wait); err == nil { + t.Fatal("second AcquireWorkflowSlot() should error") + } +} + +func TestHandleCommitIsIdempotent(t *testing.T) { + enc := newCaptureEncoder() + e := testExecutor(t) + e.enc = enc + e.active["lease-1"] = &reservation{leaseID: "lease-1", committed: true} + + e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "lease-1"}) + msg := <-enc.messages + if got := msg.GetCommitted().GetLeaseId(); got != "lease-1" { + t.Fatalf("Committed lease = %q, want lease-1", got) + } +} + +func TestHandleCommitRejectsMissingReservation(t *testing.T) { + enc := newCaptureEncoder() + e := testExecutor(t) + e.enc = enc + + e.handleCommit(context.Background(), &millv1.CommitLease{LeaseId: "expired"}) + result := (<-enc.messages).GetReserveResult() + if result == nil { + t.Fatal("missing reservation commit did not receive a ReserveResult") + } + if result.GetLeaseId() != "expired" || result.GetAccepted() { + t.Fatalf("ReserveResult = %+v, want correlated rejection", result) + } +} + +func TestHandleCancelFinalizesExpiredReservation(t *testing.T) { + enc := newCaptureEncoder() + e := testExecutor(t) + e.enc = enc + + e.handleCancel("lease-expired") + var ack *millv1.CancelAck + for ack == nil { + select { + case msg := <-enc.messages: + ack = msg.GetCancelAck() + case <-time.After(time.Second): + t.Fatal("cancel acknowledgement timed out") + } + } + if ack.GetLeaseId() != "lease-expired" { + t.Fatalf("CancelAck = %+v, want lease-expired", ack) + } + rows, err := e.db.ListOutboxRows() + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("cancel terminal outbox rows = %d, want 1", len(rows)) + } + var entry millv1.Event + if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { + t.Fatal(err) + } + if got := entry.GetAttemptResult().GetStatus(); got != millv1.TerminalStatus_CANCELLED { + t.Fatalf("cancel terminal = %v, want CANCELLED", got) + } +} + +func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { + d := testDB(t) + n := notifier.New() + enc := newCaptureEncoder() + e := &Executor{ + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, + db: d, + n: &n, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + enc: enc, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + e.lifecycleCtx = context.Background() + + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} + slot := &fakeSlot{} + repoDid, err := syntax.ParseDID("did:web:example.com") + if err != nil { + t.Fatal(err) + } + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + realEngine: inner, + slot: slot, + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, + repoDid: repoDid, + } + e.active[res.leaseID] = res + + e.handleCommit(context.Background(), &millv1.CommitLease{ + LeaseId: res.leaseID, + Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}}, + }) + if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { + t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) + } + select { + case got := <-inner.secrets: + if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" { + t.Fatalf("RunStep secrets = %+v", got) + } + case <-time.After(2 * time.Second): + t.Fatal("RunStep did not receive CommitLease secrets") + } + select { + case <-inner.done: + case <-time.After(2 * time.Second): + t.Fatal("workflow did not finish") + } + if res.stopTail != nil { + res.stopTail() + } + e.jobsWG.Wait() +} + +func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) { + connected := make(chan struct{}) + release := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := websocket.Upgrade(w, r, nil, 1024, 1024) + if err != nil { + return + } + defer conn.Close() + close(connected) + <-release + })) + t.Cleanup(func() { + close(release) + srv.Close() + }) + + e := testSessionExecutor(t, "ws"+strings.TrimPrefix(srv.URL, "http")) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- e.runSession(ctx) }() + <-connected + cancel() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runSession did not return after context cancellation") + } +} + +func testSessionExecutor(t *testing.T, url string) *Executor { + d := testDB(t) + e := &Executor{ + millURL: url, + seats: 1, + engines: make(map[string]models.Engine), + db: d, + cfg: &config.Config{Server: config.Server{Dev: true}}, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + return e +} + +func testExecutor(t *testing.T) *Executor { + d := testDB(t) + e := &Executor{ + db: d, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + return e +} + +func testDB(t *testing.T) *db.DB { + d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) { + d := testDB(t) + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + cancelled: true, + } + e := &Executor{ + db: d, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: map[string]*reservation{res.leaseID: res}, + maxOutboxBytes: 10 * 1024 * 1024, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + + e.finishJob(res, &tangled.PipelineStatus{ + Pipeline: string(res.wid.PipelineId.AtUri()), + Workflow: res.wid.Name, + Status: string(models.StatusKindFailed), + }) + + rows, err := d.ListOutboxRows() + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("outbox rows = %d, want 1", len(rows)) + } + + var entry millv1.Event + if err := proto.Unmarshal(rows[0].Payload, &entry); err != nil { + t.Fatal(err) + } + got := entry.GetAttemptResult().GetStatus() + if got != millv1.TerminalStatus_CANCELLED { + t.Fatalf("terminal status = %v, want CANCELLED", got) + } +} + +func TestReplayRejectsMalformedOutboxRow(t *testing.T) { + d := testDB(t) + e := &Executor{ + db: d, + enc: newCaptureEncoder(), + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + if _, err := d.AppendOutboxRow([]byte("not protobuf"), true); err != nil { + t.Fatal(err) + } + if err := e.replay(0); err == nil { + t.Fatal("replay accepted a malformed row and would leave a permanent seqno gap") + } +} + +func TestSocketCancellationIndependence(t *testing.T) { + d := testDB(t) + n := notifier.New() + e := &Executor{ + db: d, + n: &n, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + + lifecycleCtx, cancelLifecycle := context.WithCancel(context.Background()) + defer cancelLifecycle() + e.lifecycleCtx = lifecycleCtx + + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} + slot := &fakeSlot{} + repoDid, err := syntax.ParseDID("did:web:example.com") + if err != nil { + t.Fatal(err) + } + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + realEngine: inner, + slot: slot, + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, + repoDid: repoDid, + } + e.active[res.leaseID] = res + + sessionCtx, cancelSession := context.WithCancel(lifecycleCtx) + + e.handleCommit(sessionCtx, &millv1.CommitLease{ + LeaseId: "lease-1", + }) + + cancelSession() + + // session disconnect must not cancel the running job + select { + case <-inner.done: + case <-time.After(2 * time.Second): + t.Fatal("workflow did not complete even though websocket session was cancelled") + } + + e.jobsWG.Wait() +} + +func TestMonotonicSnapshots(t *testing.T) { + enc := newCaptureEncoder() + e := &Executor{ + enc: enc, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + } + + e.pushSnapshot() + msg1 := <-enc.messages + seq1 := msg1.GetNodeSnapshot().GetSeqno() + if seq1 != 1 { + t.Fatalf("first seq = %d, want 1", seq1) + } + + e.pushSnapshot() + msg2 := <-enc.messages + seq2 := msg2.GetNodeSnapshot().GetSeqno() + if seq2 != 2 { + t.Fatalf("second seq = %d, want 2", seq2) + } +} + +func TestTimerRace(t *testing.T) { + d := testDB(t) + e := &Executor{ + db: d, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + + twf, _ := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) + tpl, _ := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + + inner := &fakeEngine{} + e.engines = map[string]models.Engine{"microvm": inner} + + e.handleReserve(context.Background(), &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: string(twf), + RawPipelineJson: string(tpl), + TtlSeconds: 1, + }) + + e.mu.Lock() + res := e.active["lease-1"] + e.mu.Unlock() + + if res == nil { + t.Fatal("reservation was not added") + } + + deadline := time.Now().Add(5 * time.Second) + for { + e.mu.Lock() + activeLen := len(e.active) + e.mu.Unlock() + if activeLen == 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("reservation was leaked and never expired") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestStructuredShutdown(t *testing.T) { + d := testDB(t) + n := notifier.New() + e := &Executor{ + db: d, + n: &n, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + e.lifecycleCtx = ctx + + inner := &fakeEngine{secrets: make(chan []secrets.UnlockedSecret, 1), done: make(chan struct{})} + slot := &fakeSlot{} + repoDid, err := syntax.ParseDID("did:web:example.com") + if err != nil { + t.Fatal(err) + } + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + realEngine: inner, + slot: slot, + wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, + repoDid: repoDid, + } + e.active[res.leaseID] = res + + e.handleCommit(ctx, &millv1.CommitLease{ + LeaseId: "lease-1", + }) + + cancel() + + e.jobsWG.Wait() + + select { + case <-inner.done: + default: + t.Fatal("shutdown returned but running job did not finish") + } +} diff --git a/spindle/mill/handler.go b/spindle/mill/handler.go new file mode 100644 index 00000000..b9f2a891 --- /dev/null +++ b/spindle/mill/handler.go @@ -0,0 +1,183 @@ +package mill + +import ( + "github.com/gorilla/websocket" + "io" + "net/http" + "slices" + "strings" + "sync" + "time" + + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +var ( + handshakeSem = make(chan struct{}, 16) + handshakeMu sync.Mutex + inFlightHandshakes = make(map[string]struct{}) +) + +type livenessReader struct { + r io.Reader + conn *websocket.Conn + readTimeout time.Duration +} + +func (lr *livenessReader) Read(p []byte) (int, error) { + if err := lr.conn.SetReadDeadline(time.Now().Add(lr.readTimeout)); err != nil { + return 0, err + } + return lr.r.Read(p) +} + +var upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +// auth before upgrade, a bad token never opens a socket +func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { + name, authorizedLabels, ok := m.authenticate(r) + if !ok { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + select { + case handshakeSem <- struct{}{}: + case <-r.Context().Done(): + return + } + handshakeSlotHeld := true + defer func() { + if handshakeSlotHeld { + <-handshakeSem + } + }() + + // enforces one in-flight handshake and one live session at a time per identity + handshakeMu.Lock() + if _, ok := inFlightHandshakes[name]; ok { + handshakeMu.Unlock() + http.Error(w, "handshake already in progress", http.StatusConflict) + return + } + m.mu.Lock() + old, exists := m.sessions[name] + isLive := exists && old.live(m.cfg.ReconnectGrace) + m.mu.Unlock() + if isLive { + handshakeMu.Unlock() + http.Error(w, "session already active", http.StatusConflict) + return + } + inFlightHandshakes[name] = struct{}{} + handshakeMu.Unlock() + identityHandshakeHeld := true + + defer func() { + if identityHandshakeHeld { + handshakeMu.Lock() + delete(inFlightHandshakes, name) + handshakeMu.Unlock() + } + }() + + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + m.l.Error("fleet ws upgrade failed", "err", err) + return + } + defer conn.Close() + + if err := conn.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil { + m.l.Error("failed to set pre-hello read deadline", "err", err) + return + } + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + dec := millproto.NewDecoder(stream) + + hello, err := dec.Decode() + if err != nil { + m.l.Error("fleet read hello failed", "err", err) + return + } + h := hello.GetHello() + if h == nil { + m.l.Error("fleet first frame was not hello") + return + } + if h.GetProtocolVersion() != millproto.ProtocolVersion { + m.l.Error("fleet protocol version mismatch", "got", h.GetProtocolVersion(), "want", millproto.ProtocolVersion) + return + } + if h.GetEpoch() == "" { + m.l.Error("fleet hello missing epoch") + return + } + + for _, l := range h.GetLabels() { + if !slices.Contains(authorizedLabels, l) { + m.l.Error("executor requested unauthorized label", "label", l, "authorized", authorizedLabels) + return + } + } + + sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l) + sess.closeTransport = conn.Close + sess.labels = h.GetLabels() + + resume, ok := m.attachSession(sess) + if !ok { + m.l.Warn("rejecting duplicate live executor session", "node", name) + return + } + m.l.Info("executor connected", "node", sess.nodeID, "arch", h.GetArch(), "labels", h.GetLabels(), "resume", resume) + + if err := sess.send(&millproto.Message{Resume: &millv1.Resume{Epoch: h.GetEpoch(), AckSeqno: resume}}); err != nil { + m.l.Error("fleet send resume failed", "err", err) + m.detachSession(sess) + return + } + handshakeSlotHeld = false + <-handshakeSem + handshakeMu.Lock() + delete(inFlightHandshakes, name) + handshakeMu.Unlock() + identityHandshakeHeld = false + m.sessionReady(sess) + + readTimeout := m.cfg.ReconnectGrace + if readTimeout <= 0 { + readTimeout = 45 * time.Second + } + liveDec := millproto.NewDecoder(&livenessReader{r: stream, conn: conn, readTimeout: readTimeout}) + + if err := sess.readLoop(m, liveDec); err != nil { + m.l.Debug("session read ended", "node", sess.nodeID, "err", err) + } + m.detachSession(sess) +} + +// identity comes from the token hash. unknown or missing token fails closed +func (m *Mill) authenticate(r *http.Request) (string, []string, bool) { + const prefix = "Bearer " + h := r.Header.Get("Authorization") + if !strings.HasPrefix(h, prefix) { + return "", nil, false + } + token := strings.TrimPrefix(h, prefix) + if token == "" || m.db == nil { + return "", nil, false + } + name, labels, ok, err := m.db.ResolveExecutorToken(HashToken(token)) + if err != nil { + m.l.Error("executor token lookup failed", "err", err) + return "", nil, false + } + return name, labels, ok +} diff --git a/spindle/mill/integration_test.go b/spindle/mill/integration_test.go new file mode 100644 index 00000000..409e0f31 --- /dev/null +++ b/spindle/mill/integration_test.go @@ -0,0 +1,345 @@ +package mill + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "tangled.org/core/api/tangled" + "tangled.org/core/notifier" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/mill/executor" + "tangled.org/core/spindle/models" +) + +func TestEndToEndDummyJob(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + + millDir := t.TempDir() + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) + if err != nil { + t.Fatalf("mill db: %v", err) + } + bn := notifier.New() + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) + mill.Attach(bdb, &bn) + if err := bdb.AddExecutorToken("exec-1", HashToken("test-token"), nil, nil); err != nil { + t.Fatalf("register executor token: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) + defer srv.Close() + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + + execDir := t.TempDir() + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) + if err != nil { + t.Fatalf("exec db: %v", err) + } + en := notifier.New() + cfg := &config.Config{} + cfg.Server.Dev = true + cfg.Server.LogDir = execDir + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") + cfg.Server.Hostname = "exec-1" + cfg.Mill.URL = wsURL + cfg.Mill.Seats = 2 + cfg.Mill.SharedSecret = "test-token" + + dummyEng := dummy.New(l) + dummyEng.StepDelay = 50 * time.Millisecond + engines := map[string]models.Engine{"dummy": dummyEng} + exec, err := executor.New(cfg, engines, edb, &en, l) + if err != nil { + t.Fatalf("executor.New: %v", err) + } + go exec.Connect(ctx) + + be := NewEngine("dummy", mill) + twf := tangled.Pipeline_Workflow{ + Name: "build", + Raw: "steps:\n - name: hello\n command: echo hi\n", + } + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + if err != nil { + t.Fatalf("InitWorkflow: %v", err) + } + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + + placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) + defer placeCancel() + + slot, err := mill.place(placeCtx, "dummy", wid, wf) + if err != nil { + t.Fatalf("place: %v", err) + } + defer slot.Release() + + logPath := models.LogFilePath(millDir, wid) + ch := bn.Subscribe() + defer bn.Unsubscribe(ch) + + sawLogContent := make(chan bool, 1) + go func() { + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + timeout := time.After(10 * time.Second) + for { + select { + case <-ch: + case <-ticker.C: + case <-timeout: + sawLogContent <- false + return + } + data, err := os.ReadFile(logPath) + if err == nil && strings.Contains(string(data), "echo hi") { + sawLogContent <- true + return + } + } + }() + + if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { + t.Fatalf("commitAndWait: %v, want success", err) + } + + if !<-sawLogContent { + t.Fatal("mill live log file never received expected content while job was running") + } + + if !waitForFileRemoval(t, logPath) { + t.Fatalf("mill live log file was not removed after terminal artifact was recorded") + } + + if !waitForStatus(t, bdb, wid, "running") { + events, _ := bdb.GetEvents(0, 1000) + t.Logf("mill events after completion: %+v", events) + t.Fatal("mill never saw streamed running status") + } +} + +func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + + millDir := t.TempDir() + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) + if err != nil { + t.Fatalf("mill db: %v", err) + } + bn := notifier.New() + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) + mill.Attach(bdb, &bn) + if err := bdb.AddExecutorToken("exec-labels", HashToken("test-token"), nil, []string{"linux", "arm64", "gpu"}); err != nil { + t.Fatalf("register executor token: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) + defer srv.Close() + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + + execDir := t.TempDir() + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) + if err != nil { + t.Fatalf("exec db: %v", err) + } + en := notifier.New() + cfg := &config.Config{} + cfg.Server.Dev = true + cfg.Server.LogDir = execDir + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") + cfg.Server.Hostname = "exec-labels" + cfg.Mill.URL = wsURL + cfg.Mill.Seats = 2 + cfg.Mill.SharedSecret = "test-token" + cfg.Mill.Labels = []string{"linux", "arm64", "gpu"} + + engines := map[string]models.Engine{"dummy": dummy.New(l)} + exec, err := executor.New(cfg, engines, edb, &en, l) + if err != nil { + t.Fatalf("executor.New: %v", err) + } + go exec.Connect(ctx) + + if !waitForSessionLabels(t, mill, "exec-labels", []string{"linux", "arm64", "gpu"}) { + t.Fatal("mill session never stored executor labels from hello") + } +} + +func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + + millDir := t.TempDir() + bdb, err := db.Make(ctx, filepath.Join(millDir, "mill.db")) + if err != nil { + t.Fatalf("mill db: %v", err) + } + bn := notifier.New() + mill := New(l, Config{LogDir: millDir, ReconnectGrace: time.Minute, BidTimeout: 2 * time.Second}) + mill.Attach(bdb, &bn) + if err := bdb.AddExecutorToken("exec-x86", HashToken("token-x86"), nil, []string{"linux/amd64", "kvm"}); err != nil { + t.Fatalf("register x86 executor token: %v", err) + } + if err := bdb.AddExecutorToken("exec-arm", HashToken("token-arm"), nil, []string{"linux/arm64", "kvm"}); err != nil { + t.Fatalf("register arm executor token: %v", err) + } + + srv := httptest.NewServer(http.HandlerFunc(mill.HandleExecutorConn)) + defer srv.Close() + wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + + startExecutor := func(name, token string, labels []string) { + t.Helper() + execDir := t.TempDir() + edb, err := db.Make(ctx, filepath.Join(execDir, "exec.db")) + if err != nil { + t.Fatalf("%s exec db: %v", name, err) + } + en := notifier.New() + cfg := &config.Config{} + cfg.Server.Dev = true + cfg.Server.LogDir = execDir + cfg.ArtifactStores.Disk.Dir = filepath.Join(execDir, "artifacts") + cfg.Server.Hostname = name + cfg.Mill.URL = wsURL + cfg.Mill.Seats = 1 + cfg.Mill.SharedSecret = token + cfg.Mill.Labels = labels + + engines := map[string]models.Engine{"dummy": dummy.New(l)} + exec, err := executor.New(cfg, engines, edb, &en, l) + if err != nil { + t.Fatalf("executor.New: %v", err) + } + go exec.Connect(ctx) + } + startExecutor("exec-x86", "token-x86", []string{"linux/amd64", "kvm"}) + startExecutor("exec-arm", "token-arm", []string{"linux/arm64", "kvm"}) + + if !waitForSessionLabels(t, mill, "exec-x86", []string{"linux/amd64", "kvm"}) { + t.Fatal("x86 executor did not connect with labels") + } + if !waitForSessionLabels(t, mill, "exec-arm", []string{"linux/arm64", "kvm"}) { + t.Fatal("arm executor did not connect with labels") + } + + be := NewEngine("dummy", mill) + twf := tangled.Pipeline_Workflow{ + Name: "build-arm", + RunsOn: []string{"linux/arm64"}, + Raw: "steps:\n - name: hello\n command: echo hi\n", + } + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + if err != nil { + t.Fatalf("InitWorkflow: %v", err) + } + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey-arm"}, Name: "build-arm"} + + placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) + defer placeCancel() + + slot, err := mill.place(placeCtx, "dummy", wid, wf) + if err != nil { + t.Fatalf("place: %v", err) + } + defer slot.Release() + + lease := wf.Data.(*millWorkflowState).Lease + if lease == nil { + t.Fatal("place did not attach lease to workflow state") + } + if lease.nodeID != "exec-arm" { + t.Fatalf("placed on %q, want exec-arm", lease.nodeID) + } + + if err := mill.commitAndWait(placeCtx, wf, nil); err != nil { + t.Fatalf("commitAndWait: %v, want success", err) + } +} + +func waitForSessionLabels(t *testing.T, m *Mill, nodeID string, want []string) bool { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + m.mu.Lock() + sess := m.sessions[nodeID] + var got []string + if sess != nil { + got = append([]string(nil), sess.labels...) + } + m.mu.Unlock() + if sameStringMultiset(got, want) { + return true + } + time.Sleep(10 * time.Millisecond) + } + return false +} + +func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) bool { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + aturi := string(wid.PipelineId.AtUri()) + for time.Now().Before(deadline) { + evs, err := d.GetEvents(0, 1000) + if err != nil { + t.Fatalf("GetEvents: %v", err) + } + for _, ev := range evs { + if ev.Nsid != tangled.PipelineStatusNSID { + continue + } + var st tangled.PipelineStatus + if err := json.Unmarshal(ev.EventJson, &st); err != nil { + continue + } + if st.Pipeline == aturi && st.Workflow == wid.Name && st.Status == want { + return true + } + } + time.Sleep(50 * time.Millisecond) + } + return false +} + +func waitForLogFileContent(t *testing.T, path, want string) bool { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + if err == nil && strings.Contains(string(data), want) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +func waitForFileRemoval(t *testing.T, path string) bool { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + _, err := os.Stat(path) + if os.IsNotExist(err) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} diff --git a/spindle/mill/lease.go b/spindle/mill/lease.go new file mode 100644 index 00000000..f4604dc0 --- /dev/null +++ b/spindle/mill/lease.go @@ -0,0 +1,186 @@ +package mill + +import ( + "sync" + + "tangled.org/core/spindle/models" + + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +// the mill's view of a remote attempt +type leaseState int32 + +const ( + // won a bid, executor is holding a seat, not yet committed + leaseReserved leaseState = iota + // CommitLease sent. the executor may already be running, but the mill + // may not have seen Committed yet + leaseCommitting + // CommitLease sent and acked. job is running on the executor + leaseRunning + // terminal result arrived or we gave up, no further action + leaseDone +) + +type cancelAction int + +const ( + cancelNoop cancelAction = iota + cancelLocal + cancelRemote +) + +// mill-side fencing token for one placed job +type RemoteLease struct { + id string + nodeID string + epoch string + engine string + wid models.WorkflowId // job this lease carries, set once placed + // restored after a mill restart. no RunStep waits on it, so terminals + // and death are authored directly. set before publication, never mutated + orphaned bool + claimed bool + cancelAcked bool + cleanedUp bool + cleanupRetry bool + + mu sync.Mutex + state leaseState + cancel bool + reason string + released bool + terminal chan *millv1.AttemptResult // buffered(1), RunStep waits here + + finishMu sync.Mutex +} + +func newLease(id, nodeID, epoch, engine string) *RemoteLease { + return &RemoteLease{ + id: id, + nodeID: nodeID, + epoch: epoch, + engine: engine, + state: leaseReserved, + terminal: make(chan *millv1.AttemptResult, 1), + } +} + +func (l *RemoteLease) setState(s leaseState) { + l.mu.Lock() + l.state = s + l.mu.Unlock() +} + +func (l *RemoteLease) markCommitting() bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.state == leaseDone { + return false + } + if l.state == leaseReserved { + l.state = leaseCommitting + } + return true +} + +func (l *RemoteLease) markRunning() { + l.mu.Lock() + defer l.mu.Unlock() + if l.state != leaseDone { + l.state = leaseRunning + } +} + +func (l *RemoteLease) getState() leaseState { + l.mu.Lock() + defer l.mu.Unlock() + return l.state +} + +// only one caller gets to mark it done +func (l *RemoteLease) markDone() bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.state == leaseDone { + return false + } + l.state = leaseDone + return true +} + +func (l *RemoteLease) requestCancel(reason string) cancelAction { + l.mu.Lock() + defer l.mu.Unlock() + if l.state == leaseDone { + return cancelNoop + } + l.cancel = true + l.reason = reason + if l.state == leaseReserved { + return cancelLocal + } + return cancelRemote +} + +func (l *RemoteLease) cancelRequested() (bool, string) { + l.mu.Lock() + defer l.mu.Unlock() + return l.cancel, l.reason +} + +func (l *RemoteLease) releaseState() (leaseState, bool) { + l.mu.Lock() + defer l.mu.Unlock() + l.released = true + return l.state, l.cancel +} + +func (l *RemoteLease) cleanupReady() bool { + l.mu.Lock() + defer l.mu.Unlock() + return l.released && l.state == leaseDone +} + +func (l *RemoteLease) deliverCancelled(reason string) { + l.deliverTerminal(&millv1.AttemptResult{ + Status: millv1.TerminalStatus_CANCELLED, + Error: reason, + }) +} + +// hands the terminal to a waiting RunStep without blocking. duplicates +// (eg. reconnect replays) just drop, the channel holds one and the lease +// is already done +func (l *RemoteLease) deliverTerminal(res *millv1.AttemptResult) { + if !l.markDone() { + return + } + select { + case l.terminal <- res: + default: + } +} + +// mill's synthetic single step. real steps run on the executor and the +// mill never mirrors them +type remoteStep struct{} + +func (remoteStep) Name() string { return "remote execution" } +func (remoteStep) Command() string { return "" } +func (remoteStep) Kind() models.StepKind { return models.StepKindSystem } + +// what AcquireWorkflowSlot returns. Release unwinds placement +type millSlot struct { + fleet *Mill + lease *RemoteLease + once sync.Once +} + +func (s *millSlot) Release() { + if s == nil { + return + } + s.once.Do(func() { s.fleet.releaseSlot(s) }) +} diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go new file mode 100644 index 00000000..b532a58e --- /dev/null +++ b/spindle/mill/mill.go @@ -0,0 +1,1119 @@ +package mill + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "time" + + "tangled.org/core/notifier" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" + "tangled.org/core/tid" + + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +const ( + defaultReconnectGrace = 45 * time.Second + defaultJobTimeout = 24 * time.Hour + defaultBidTimeout = 5 * time.Second + defaultTopK = 3 + defaultMaxPending = 100 +) + +type Config struct { + // mill appends live-tailed executor lines here so logview can follow running remote jobs. + LogDir string + MaxPending int + ReconnectGrace time.Duration + JobTimeout time.Duration + BidTimeout time.Duration + TopK int + CancelTimeout time.Duration +} + +type Mill struct { + l *slog.Logger + cfg Config + + db *db.DB + n *notifier.Notifier + + mu sync.Mutex + sessions map[string]*millSession + leases map[string]*RemoteLease + reservations map[string]*RemoteLease + nodeSeqno map[string]uint64 + pending int + changeCh chan struct{} // closed and replaced to wake placement waiters + + leaseSeq uint64 +} + +func New(l *slog.Logger, cfg Config) *Mill { + if cfg.ReconnectGrace <= 0 { + cfg.ReconnectGrace = defaultReconnectGrace + } + if cfg.JobTimeout <= 0 { + cfg.JobTimeout = defaultJobTimeout + } + if cfg.BidTimeout <= 0 { + cfg.BidTimeout = defaultBidTimeout + } + if cfg.TopK <= 0 { + cfg.TopK = defaultTopK + } + if cfg.MaxPending <= 0 { + cfg.MaxPending = defaultMaxPending + } + if cfg.CancelTimeout <= 0 { + cfg.CancelTimeout = 10 * time.Second + } + return &Mill{ + l: l, + cfg: cfg, + sessions: make(map[string]*millSession), + leases: make(map[string]*RemoteLease), + reservations: make(map[string]*RemoteLease), + nodeSeqno: make(map[string]uint64), + changeCh: make(chan struct{}), + } +} + +func (m *Mill) Attach(d *db.DB, n *notifier.Notifier) { + m.mu.Lock() + m.db = d + m.n = n + m.mu.Unlock() +} + +func (m *Mill) nextLeaseID() string { + m.mu.Lock() + m.leaseSeq++ + seq := m.leaseSeq + m.mu.Unlock() + return fmt.Sprintf("%s-%d", tid.TID(), seq) +} + +func (m *Mill) dropReservation(id string) { + m.mu.Lock() + delete(m.reservations, id) + m.mu.Unlock() +} + +func (m *Mill) notifyChange() { + m.mu.Lock() + m.notifyChangeLocked() + m.mu.Unlock() +} + +func (m *Mill) notifyChangeLocked() { + close(m.changeCh) + m.changeCh = make(chan struct{}) +} + +func (m *Mill) currentChangeCh() <-chan struct{} { + m.mu.Lock() + defer m.mu.Unlock() + return m.changeCh +} + +func (m *Mill) attachSession(sess *millSession) (uint64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + if old := m.sessions[sess.nodeID]; old != nil { + if old.live(m.cfg.ReconnectGrace) { + // a second live session for the same identity is a hijack + // attempt, reject it + return 0, false + } + if old.graceTimer != nil { + old.graceTimer.Stop() + } + old.close() + if old.disconnected { + m.l.Info("executor reconnected", "node", sess.nodeID) + } else { + m.l.Warn("replacing silent executor session", "node", sess.nodeID) + } + } + m.sessions[sess.nodeID] = sess + // wakes commit retries waiting out reconnect grace + m.notifyChangeLocked() + return m.nodeSeqno[sess.nodeID+"/"+sess.epoch], true +} + +func (m *Mill) touchSession(sess *millSession) bool { + m.mu.Lock() + defer m.mu.Unlock() + if m.sessions[sess.nodeID] != sess || sess.disconnected { + return false + } + sess.lastSeen = time.Now() + return true +} + +func (m *Mill) detachSession(sess *millSession) { + m.mu.Lock() + if m.sessions[sess.nodeID] != sess { + // already replaced by a reconnect + m.mu.Unlock() + sess.close() + return + } + sess.disconnected = true + sess.graceTimer = time.AfterFunc(m.cfg.ReconnectGrace, func() { m.failLeasesAfterGrace(sess) }) + m.mu.Unlock() + + sess.close() + m.l.Warn("executor session lost; entering reconnect grace", "node", sess.nodeID, "grace", m.cfg.ReconnectGrace) + m.notifyChange() +} + +func (m *Mill) sessionReady(sess *millSession) { + for _, lease := range m.cancelledLeasesForNode(sess.nodeID) { + _, reason := lease.cancelRequested() + m.sendCancel(sess, lease, reason) + } + m.notifyChange() +} + +func (m *Mill) cancelledLeasesForNode(nodeID string) []*RemoteLease { + m.mu.Lock() + var candidates []*RemoteLease + for _, lease := range m.leases { + if lease.nodeID == nodeID { + candidates = append(candidates, lease) + } + } + m.mu.Unlock() + + var leases []*RemoteLease + for _, lease := range candidates { + if cancelled, _ := lease.cancelRequested(); cancelled && lease.getState() != leaseDone { + leases = append(leases, lease) + } + } + return leases +} + +// reconnect grace ran out without the executor coming back. fails every +// lease the node held, and reschedules itself if any failure didn't stick +func (m *Mill) failLeasesAfterGrace(sess *millSession) { + m.mu.Lock() + if m.sessions[sess.nodeID] != sess || !sess.disconnected { + // reconnected in the meantime + m.mu.Unlock() + return + } + var dead []*RemoteLease + for _, lease := range m.leases { + if lease.nodeID == sess.nodeID { + dead = append(dead, lease) + } + } + m.mu.Unlock() + + m.l.Warn("executor declared dead; failing its in-flight jobs", "node", sess.nodeID, "jobs", len(dead)) + deadReason := "executor lost" + success := true + for _, lease := range dead { + switch { + case lease.orphaned: + // restored leases have no RunStep waiting, fail them straight + // into the event stream + if err := m.finishOrphan(lease, string(models.StatusKindFailed), &deadReason, nil); err != nil { + m.l.Error("finish orphan after executor loss failed, will retry", "lease", lease.id, "err", err) + success = false + } + default: + // live leases have a blocked RunStep, keep a pending cancellation + // as the terminal reason + status := string(models.StatusKindFailed) + reason := deadReason + if cancelled, cancelReason := lease.cancelRequested(); cancelled { + status = string(models.StatusKindCancelled) + reason = cancelReason + } + if err := m.finishLiveLease(lease, status, reason); err != nil { + m.l.Error("finish lease after executor loss failed, will retry", "lease", lease.id, "err", err) + success = false + } + } + } + + m.mu.Lock() + if !success { + // something didn't finish cleanly, try again shortly. finished + // leases skip themselves on the next pass + sess.graceTimer = time.AfterFunc(5*time.Second, func() { m.failLeasesAfterGrace(sess) }) + m.mu.Unlock() + return + } + + // everything failed cleanly. forget the node and wake placement + if m.sessions[sess.nodeID] == sess { + delete(m.sessions, sess.nodeID) + } + m.mu.Unlock() + m.notifyChange() + m.sweepUnclaimedOrphans() +} + +func (m *Mill) place(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { + m.mu.Lock() + if m.cfg.MaxPending > 0 && m.pending >= m.cfg.MaxPending { + max := m.cfg.MaxPending + cur := m.pending + m.mu.Unlock() + return nil, fmt.Errorf("%w: mill has %d pending jobs (max %d)", engine.ErrNoWorkflowSlots, cur, max) + } + m.pending++ + m.mu.Unlock() + defer func() { + m.mu.Lock() + m.pending-- + m.mu.Unlock() + }() + + for { + if err := ctx.Err(); err != nil { + return nil, err + } + + // grab the channel before bidding. a change mid-bid closes it, so + // the wait below re-bids right away + ch := m.currentChangeCh() + + lease, err := m.bid(ctx, engineName, wid, wf) + if err != nil { + return nil, err + } + if lease != nil { + lease.wid = wid + if err := m.persistLease(lease, leaseRowReserved); err != nil { + m.releaseRemote(lease) + m.mu.Lock() + delete(m.reservations, lease.id) + m.mu.Unlock() + return nil, fmt.Errorf("persist reserved mill lease: %w", err) + } + m.mu.Lock() + delete(m.reservations, lease.id) + m.leases[lease.id] = lease + if st, ok := wf.Data.(*millWorkflowState); ok && st != nil { + st.Lease = lease + } + m.mu.Unlock() + return &millSlot{fleet: m, lease: lease}, nil + } + + // no executor available. wait for a change or ctx + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ch: + } + } +} + +func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId, wf *models.Workflow) (*RemoteLease, error) { + rawPipeline, rawWorkflow, err := marshalJob(wf) + if err != nil { + return nil, err + } + + requiredLabels := requiredLabels(wf) + candidates := m.rankCandidates(engineName, requiredLabels) + if len(candidates) == 0 { + return nil, nil + } + + type bidResult struct { + sess *millSession + lease *RemoteLease + rank int + incompatible bool + reason string + } + limit := m.cfg.TopK + if limit <= 0 { + limit = len(candidates) + } + if len(candidates) > limit { + candidates = candidates[:limit] + } + results := make(chan bidResult, limit) + ask := func(rank int, sess *millSession) { + bidCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) + defer cancel() + leaseID := m.nextLeaseID() + lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName) + m.mu.Lock() + m.reservations[leaseID] = lease + m.mu.Unlock() + msg := &millproto.Message{ReserveSeat: &millv1.ReserveSeat{ + LeaseId: leaseID, + TargetEngine: engineName, + RawPipelineJson: rawPipeline, + RawWorkflowJson: rawWorkflow, + Knot: wid.Knot, + Rkey: wid.Rkey, + TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), + }} + resp, err := sess.request(bidCtx, leaseID, msg) + if err != nil { + m.dropReservation(leaseID) + results <- bidResult{} + return + } + rr := resp.GetReserveResult() + if rr == nil { + m.dropReservation(leaseID) + results <- bidResult{} + return + } + if !rr.GetAccepted() { + m.dropReservation(leaseID) + if rr.GetRejectClass() == millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE { + results <- bidResult{sess: sess, rank: rank, incompatible: true, reason: rr.GetRejectReason()} + return + } + results <- bidResult{} + return + } + results <- bidResult{sess: sess, lease: lease, rank: rank} + } + next := 0 + inFlight := 0 + for next < len(candidates) && inFlight < limit { + inFlight++ + go ask(next, candidates[next]) + next++ + } + + var winner *bidResult + var losers []*RemoteLease + var incompatible []string + // any soft reject (transient or timeout) means the fleet was just + // busy, so an all-incompatible outcome isn't a hard placement error + softReject := false + for inFlight > 0 { + r := <-results + inFlight-- + // incompatible rejects get reported to the user, any other failure + // just means the fleet is busy + if r.incompatible { + if r.reason != "" { + incompatible = append(incompatible, r.reason) + } + } else if r.lease == nil { + softReject = true + } + // a failed bid means ask the next candidate, unless someone won + if r.lease == nil { + for winner == nil && next < len(candidates) && inFlight < limit { + inFlight++ + go ask(next, candidates[next]) + next++ + } + continue + } + // if this bid is worse than the winner it goes to the losers pile + if winner != nil && r.rank >= winner.rank { + losers = append(losers, r.lease) + continue + } + // otherwise it's the new best and the old winner joins the losers + if winner != nil { + losers = append(losers, winner.lease) + } + winner = &r + } + + // let the losers go so they free their seats right away + for _, l := range losers { + m.dropReservation(l.id) + m.releaseRemote(l) + } + + if winner == nil { + if len(incompatible) > 0 && !softReject { + return nil, fmt.Errorf("no compatible executor for %s: %s", engineName, strings.Join(incompatible, "; ")) + } + return nil, nil + } + return winner.lease, nil +} + +// ranks nodes that are least busy first. if a resource is used a lot +// then that node will lose to one that is more even across the board. +func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession { + m.mu.Lock() + defer m.mu.Unlock() + + type ranked struct { + sess *millSession + worst float64 + sum float64 + } + var rs []ranked + for _, s := range m.sessions { + // only live, reporting sessions can take work + if s.disconnected { + continue + } + if s.snapshot == nil { + continue + } + // the engine has to exist and have room right now + ea, ok := s.snapshot.GetEngines()[engineName] + if !ok || !ea.GetAvailable() { + continue + } + // and satisfy the wf's label requirements + if !hasLabels(s.labels, requiredLabels) { + continue + } + worst, sum := loadScore(ea.GetLoad()) + rs = append(rs, ranked{sess: s, worst: worst, sum: sum}) + } + slices.SortStableFunc(rs, func(a, b ranked) int { + if a.worst < b.worst { + return -1 + } + if a.worst > b.worst { + return 1 + } + if a.sum < b.sum { + return -1 + } + if a.sum > b.sum { + return 1 + } + return 0 + }) + + out := make([]*millSession, len(rs)) + for i := range rs { + out[i] = rs[i].sess + } + return out +} + +func loadScore(load map[string]float64) (worst, sum float64) { + for _, v := range load { + if v > worst { + worst = v + } + sum += v + } + return worst, sum +} + +func requiredLabels(wf *models.Workflow) []string { + st, ok := wf.Data.(*millWorkflowState) + if !ok || st == nil { + return nil + } + return st.RawWorkflow.RunsOn +} + +func hasLabels(labels []string, required []string) bool { + for _, want := range required { + if !slices.Contains(labels, want) { + return false + } + } + return true +} + +func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked []secrets.UnlockedSecret) error { + st, ok := wf.Data.(*millWorkflowState) + if !ok || st == nil || st.Lease == nil { + return fmt.Errorf("mill workflow state missing lease") + } + lease := st.Lease + + pbSecrets := make([]*millv1.Secret, len(unlocked)) + for i, s := range unlocked { + pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value} + } + + commit := &millproto.Message{CommitLease: &millv1.CommitLease{ + LeaseId: lease.id, + Secrets: pbSecrets, + }} + + // commit retries ride reconnects, a reservation outlives one + // disconnect. lost session or slow executor just means wait and retry, + // only job timeout or a dead lease stops the loop + for { + if res, ok := pollTerminal(lease); ok { + return terminalError(res.Status) + } + if !lease.markCommitting() { + return engine.ErrWorkflowFailed + } + + sess := m.sessionForNode(lease.nodeID) + if sess == nil { + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { + return err + } + continue + } + + reqCtx, cancel := context.WithTimeout(ctx, m.cfg.BidTimeout) + resp, err := sess.request(reqCtx, lease.id, commit) + cancel() + if err != nil { + switch { + case errors.Is(err, errSessionClosed): + // session died mid-request. wait out the grace, then retry on + // the new one + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { + return err + } + continue + case errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil: + // executor didn't answer in time, but its seat is still held so + // retrying is safe + continue + case errors.Is(err, context.DeadlineExceeded): + // the job ctx itself ran out, a real timeout + return engine.ErrTimedOut + case errors.Is(err, context.Canceled): + return err + default: + m.l.Warn("commit lease send failed; waiting for reconnect", "lease", lease.id, "node", lease.nodeID, "err", err) + if done, err := m.waitCommitRetry(ctx, lease); done || err != nil { + return err + } + continue + } + } + if resp.GetCommitted() == nil { + return engine.ErrWorkflowFailed + } + lease.markRunning() + if err := m.persistLease(lease, leaseRowRunning); err != nil { + m.l.Error("persist running mill lease", "lease", lease.id, "err", err) + } + if cancelled, reason := lease.cancelRequested(); cancelled { + m.sendCancel(sess, lease, reason) + } + break + } + + select { + case res := <-lease.terminal: + return terminalError(res.Status) + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + return engine.ErrTimedOut + } + return ctx.Err() + } +} + +func (m *Mill) waitCommitRetry(ctx context.Context, lease *RemoteLease) (bool, error) { + // grab the channel before checking for a live session again + // a reconnect will still close the channel if it happens in between + ch := m.currentChangeCh() + if m.sessionForNode(lease.nodeID) != nil { + return false, nil + } + select { + case res := <-lease.terminal: + return true, terminalError(res.Status) + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + return true, engine.ErrTimedOut + } + return true, ctx.Err() + case <-ch: + return false, nil + } +} + +func terminalError(status millv1.TerminalStatus) error { + switch status { + case millv1.TerminalStatus_SUCCESS: + return nil + case millv1.TerminalStatus_TIMEOUT: + return engine.ErrTimedOut + case millv1.TerminalStatus_CANCELLED: + return engine.ErrWorkflowCanceled + default: + return engine.ErrWorkflowFailed + } +} + +func pollTerminal(lease *RemoteLease) (*millv1.AttemptResult, bool) { + select { + case res := <-lease.terminal: + return res, true + default: + return nil, false + } +} + +func (m *Mill) destroy(wid models.WorkflowId) { + m.mu.Lock() + var lease *RemoteLease + for _, l := range m.leases { + if l.wid == wid { + lease = l + break + } + } + m.mu.Unlock() + if lease == nil { + return + } + reason := "workflow destroyed" + switch lease.requestCancel(reason) { + case cancelLocal: + if sess := m.sessionForNode(lease.nodeID); sess != nil { + _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) + } + lease.deliverCancelled(reason) + case cancelRemote: + if sess := m.sessionForNode(lease.nodeID); sess != nil { + m.sendCancel(sess, lease, reason) + } + } +} + +func (m *Mill) releaseSlot(s *millSlot) { + lease := s.lease + state, cancelled := lease.releaseState() + if state == leaseReserved { + m.releaseRemote(lease) + } else if cancelled && state != leaseDone { + return + } + if err := m.cleanupLease(lease); err != nil { + m.l.Error("releaseSlot cleanupLease failed", "lease", lease.id, "err", err) + } +} +func (m *Mill) cleanupLease(lease *RemoteLease) error { + lease.finishMu.Lock() + defer lease.finishMu.Unlock() + return m.cleanupLeaseLocked(lease) +} + +func (m *Mill) cleanupLeaseLocked(lease *RemoteLease) error { + if lease.cleanedUp { + return nil + } + if m.db != nil { + if err := m.db.DeleteMillLease(lease.id); err != nil { + m.scheduleCleanupLocked(lease) + return err + } + } + m.mu.Lock() + delete(m.leases, lease.id) + m.mu.Unlock() + lease.cleanedUp = true + m.notifyChange() + return nil +} + +func (m *Mill) scheduleCleanupLocked(lease *RemoteLease) { + if lease.cleanedUp || lease.cleanupRetry { + return + } + lease.cleanupRetry = true + time.AfterFunc(5*time.Second, func() { + lease.finishMu.Lock() + lease.cleanupRetry = false + err := m.cleanupLeaseLocked(lease) + lease.finishMu.Unlock() + if err != nil { + m.l.Error("retry lease cleanup failed", "lease", lease.id, "err", err) + } + }) +} + +func (m *Mill) releaseRemote(lease *RemoteLease) { + lease.setState(leaseDone) + if sess := m.sessionForNode(lease.nodeID); sess != nil { + _ = sess.send(&millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: lease.id}}) + } +} + +func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease, reason string) { + if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{ + LeaseId: lease.id, + Reason: reason, + }}); err != nil { + return + } + time.AfterFunc(m.cfg.CancelTimeout, func() { m.checkCancelDeadline(lease) }) +} + +func (m *Mill) sessionForNode(nodeID string) *millSession { + m.mu.Lock() + defer m.mu.Unlock() + sess := m.sessions[nodeID] + if sess == nil || sess.disconnected { + return nil + } + return sess +} + +func (m *Mill) onSnapshot(sess *millSession, snap *millv1.NodeSnapshot) error { + m.mu.Lock() + if sess.snapshot != nil && snap.Seqno <= sess.snapshot.Seqno { + m.mu.Unlock() + return fmt.Errorf("protocol error: snapshot seqno regression. Got %d, last seen %d", snap.Seqno, sess.snapshot.Seqno) + } + sess.snapshot = snap + m.mu.Unlock() + + if err := m.reconcileLeases(sess, snap.GetActiveLeaseIds()); err != nil { + return err + } + m.mu.Lock() + for _, id := range snap.GetActiveLeaseIds() { + if lease := m.leases[id]; lease != nil && lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { + lease.claimed = true + } + } + m.mu.Unlock() + m.notifyChange() + return nil +} + +func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { + if batch == nil { + return nil + } + + if batch.Epoch != sess.epoch { + return fmt.Errorf("protocol error: batch epoch %q does not match session %q", batch.Epoch, sess.epoch) + } + + m.mu.Lock() + currentKey := sess.nodeID + "/" + sess.epoch + current := m.nodeSeqno[currentKey] + m.mu.Unlock() + + expected := current + 1 + var newEntries []*millv1.Event + for _, entry := range batch.Events { + // reconnect replays old seqnos, drop those + if entry.Seqno <= current { + continue + } + // gap means executor lost rows. so dont apply a partial batch + if entry.Seqno != expected { + return fmt.Errorf("protocol error: gap in stream seqnos. Expected %d, got %d", expected, entry.Seqno) + } + newEntries = append(newEntries, entry) + expected++ + } + + // all replays, still ack so the executor can trim its outbox + if len(newEntries) == 0 { + return m.sendAck(sess, current) + } + + type pendingTerminal struct { + lease *RemoteLease + ar *millv1.AttemptResult + } + var pendingTerminals []pendingTerminal + var artifactLeases []*RemoteLease + finishedInBatch := make(map[string]struct{}) + + var highestSeqno uint64 = current + + applyFunc := func(tx *db.EventBatchTx) error { + for _, entry := range newEntries { + m.mu.Lock() + lease := m.leases[entry.LeaseId] + m.mu.Unlock() + + // events for leases this node doesn't own are skipped but still + // count as processed + if lease == nil || lease.nodeID != sess.nodeID { + highestSeqno = entry.Seqno + continue + } + + // events arriving for a different epoch are invalid (different session) + if lease.epoch != "" && lease.epoch != sess.epoch { + return fmt.Errorf("protocol error: lease %q epoch %q does not match session %q", lease.id, lease.epoch, sess.epoch) + } + + lease.mu.Lock() + state := lease.state + lease.mu.Unlock() + + // done leases can replay terminals on reconnect, skip them + if state == leaseDone { + highestSeqno = entry.Seqno + continue + } + + if _, finished := finishedInBatch[lease.id]; finished { + return fmt.Errorf("protocol error: stream entry follows terminal for lease %q", lease.id) + } + + switch { + case entry.GetStatusEvent() != nil: + ev := entry.GetStatusEvent() + statusStr := string(models.StatusKindRunning) + var errMsg *string + if e := ev.GetError(); e != "" { + errMsg = &e + } + var exitCode *int64 + if c := ev.GetExitCode(); c != 0 { + exitCode = &c + } + pipelineAtUri := string(lease.wid.PipelineId.AtUri()) + if tx != nil { + if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { + return err + } + } + + case entry.GetAttemptResult() != nil: + ar := entry.GetAttemptResult() + statusStr := "success" + switch ar.Status { + case millv1.TerminalStatus_SUCCESS: + statusStr = "success" + case millv1.TerminalStatus_FAILED: + statusStr = "failed" + case millv1.TerminalStatus_TIMEOUT: + statusStr = "timeout" + case millv1.TerminalStatus_CANCELLED: + statusStr = "cancelled" + default: + return fmt.Errorf("protocol error: unsupported terminal status %v", ar.Status) + } + var errMsg *string + if e := ar.GetError(); e != "" { + errMsg = &e + } + var exitCode *int64 + if c := ar.GetExitCode(); c != 0 { + exitCode = &c + } + finishedInBatch[lease.id] = struct{}{} + pipelineAtUri := string(lease.wid.PipelineId.AtUri()) + if tx != nil { + if err := tx.InsertStatusEvent(pipelineAtUri, lease.wid.Name, statusStr, errMsg, exitCode); err != nil { + return err + } + if err := tx.DeleteLease(lease.id); err != nil { + return err + } + if a := ar.GetLogArtifact(); a != nil { + if a.GetRef() == "" { + return fmt.Errorf("protocol error: empty log artifact ref") + } + if !strings.HasPrefix(a.GetHash(), "sha256:") { + return fmt.Errorf("protocol error: invalid log artifact hash %q", a.GetHash()) + } + if tx != nil { + if err := tx.InsertArtifactRef(lease.id, lease.wid.Name, a.GetRef(), a.GetHash()); err != nil { + return err + } + } + artifactLeases = append(artifactLeases, lease) + } + } + pendingTerminals = append(pendingTerminals, pendingTerminal{ + lease: lease, + ar: ar, + }) + } + + highestSeqno = entry.Seqno + } + + if tx != nil { + return tx.AdvanceCursor(sess.nodeID, sess.epoch, highestSeqno) + } + return nil + } + + var err error + if m.db != nil { + err = m.db.ApplyEventBatch(m.n, applyFunc) + } else { + err = applyFunc(nil) + } + + if err != nil { + return err + } + + if m.cfg.LogDir != "" { + for _, lease := range artifactLeases { + path := models.LogFilePath(m.cfg.LogDir, lease.wid) + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + m.l.Warn("failed to remove live log file after artifact recorded", "path", path, "err", err) + } + } + } + + m.mu.Lock() + if highestSeqno > m.nodeSeqno[currentKey] { + m.nodeSeqno[currentKey] = highestSeqno + } + m.mu.Unlock() + + for _, pt := range pendingTerminals { + // orphans have no waiting RunStep, just mark and clean up + if pt.lease.orphaned { + pt.lease.markDone() + _ = m.cleanupLease(pt.lease) + continue + } + pt.lease.deliverTerminal(pt.ar) + if pt.lease.cleanupReady() { + _ = m.cleanupLease(pt.lease) + } + } + + return m.sendAck(sess, highestSeqno) +} + +func (m *Mill) sendAck(sess *millSession, seqno uint64) error { + msg := &millproto.Message{Ack: &millv1.Ack{ + Epoch: sess.epoch, + UpToSeqno: seqno, + }} + if err := sess.send(msg); err != nil { + return fmt.Errorf("send ack message: %w", err) + } + return nil +} +func (m *Mill) onLiveLog(sess *millSession, ll *millv1.LiveLog) error { + if ll == nil || ll.GetLeaseId() == "" { + return nil + } + m.mu.Lock() + lease := m.leases[ll.GetLeaseId()] + m.mu.Unlock() + if lease == nil || lease.nodeID != sess.nodeID { + return nil + } + raw := ll.GetRawJson() + if m.cfg.LogDir == "" || len(raw) == 0 { + if m.n != nil { + m.n.NotifyAll() + } + return nil + } + lease.mu.Lock() + isDone := (lease.state == leaseDone) + lease.mu.Unlock() + if isDone { + if m.n != nil { + m.n.NotifyAll() + } + return nil + } + + logPath := models.LogFilePath(m.cfg.LogDir, lease.wid) + if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil { + m.l.Warn("failed to create log dir", "path", filepath.Dir(logPath), "err", err) + if m.n != nil { + m.n.NotifyAll() + } + return nil + } + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + m.l.Warn("failed to open log file", "path", logPath, "err", err) + if m.n != nil { + m.n.NotifyAll() + } + return nil + } + if _, err := f.Write(raw); err != nil { + m.l.Warn("failed to write log file", "path", logPath, "err", err) + } + _ = f.Close() + + if m.n != nil { + m.n.NotifyAll() + } + return nil +} + +func (m *Mill) onCancelAck(sess *millSession, ca *millv1.CancelAck) { + m.mu.Lock() + lease := m.leases[ca.GetLeaseId()] + m.mu.Unlock() + if lease == nil || lease.nodeID != sess.nodeID { + return + } + lease.mu.Lock() + lease.cancelAcked = true + lease.mu.Unlock() +} + +func (m *Mill) checkCancelDeadline(lease *RemoteLease) { + lease.mu.Lock() + isDone := (lease.state == leaseDone) + isAcked := lease.cancelAcked + lease.mu.Unlock() + + if isDone && isAcked { + return + } + + m.l.Warn("node failed to comply with cancel request within deadline, quarantining", "node", lease.nodeID, "lease", lease.id, "done", isDone, "acked", isAcked) + + reason := fmt.Sprintf("cancel noncompliance for lease %s (done: %t, acked: %t)", lease.id, isDone, isAcked) + if m.db != nil { + if err := m.db.QuarantineExecutor(lease.nodeID, reason); err != nil { + m.l.Error("failed to quarantine executor", "node", lease.nodeID, "err", err) + } + } + + m.mu.Lock() + sess := m.sessions[lease.nodeID] + m.mu.Unlock() + if sess != nil { + sess.close() + } +} + +func marshalJob(wf *models.Workflow) (pipeline string, workflow string, err error) { + st, ok := wf.Data.(*millWorkflowState) + if !ok || st == nil { + return "", "", fmt.Errorf("mill workflow state missing") + } + p, err := json.Marshal(st.RawPipeline) + if err != nil { + return "", "", fmt.Errorf("marshal pipeline: %w", err) + } + w, err := json.Marshal(st.RawWorkflow) + if err != nil { + return "", "", fmt.Errorf("marshal workflow: %w", err) + } + return string(p), string(w), nil +} diff --git a/spindle/mill/mill_test.go b/spindle/mill/mill_test.go new file mode 100644 index 00000000..3234d17d --- /dev/null +++ b/spindle/mill/mill_test.go @@ -0,0 +1,873 @@ +package mill + +import ( + "context" + "errors" + "io" + "log/slog" + "sync" + "testing" + "time" + + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +type scriptedEncoder func(*millproto.Message) error + +func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) } + +func testWorkflow(name string) *models.Workflow { + return &models.Workflow{ + Name: name, + Environment: map[string]string{}, + Steps: []models.Step{remoteStep{}}, + Data: &millWorkflowState{ + RawWorkflow: tangled.Pipeline_Workflow{Name: name}, + RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, + }, + } +} + +func testWorkflowWithRunsOn(name string, runsOn []string) *models.Workflow { + wf := testWorkflow(name) + wf.Data.(*millWorkflowState).RawWorkflow.RunsOn = runsOn + return wf +} + +func addCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, enc messageEncoder) *millSession { + t.Helper() + if enc == nil { + enc = scriptedEncoder(func(*millproto.Message) error { return nil }) + } + sess := newSession(nodeID, "inc-"+nodeID, labels, enc, slog.New(slog.NewTextHandler(io.Discard, nil))) + sess.snapshot = &millv1.NodeSnapshot{ + Seqno: 1, + Engines: map[string]*millv1.EngineAvailability{ + "dummy": {Available: load < 1.0, Load: map[string]float64{"slots": load}}, + }, + } + m.mu.Lock() + m.sessions[nodeID] = sess + m.mu.Unlock() + return sess +} + +func assertRankedNodes(t *testing.T, got []*millSession, want []string) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("rankCandidates() returned %d candidates, want %d: got %v want %v", len(got), len(want), sessionIDs(got), want) + } + for i := range want { + if got[i].nodeID != want[i] { + t.Fatalf("rankCandidates()[%d] = %q, want %q; full order got %v want %v", i, got[i].nodeID, want[i], sessionIDs(got), want) + } + } +} + +func sessionIDs(sessions []*millSession) []string { + out := make([]string, len(sessions)) + for i, sess := range sessions { + out[i] = sess.nodeID + } + return out +} + +func sameStringMultiset(a, b []string) bool { + if len(a) != len(b) { + return false + } + counts := make(map[string]int, len(a)) + for _, s := range a { + counts[s]++ + } + for _, s := range b { + if counts[s] == 0 { + return false + } + counts[s]-- + } + return true +} + +type reserveReply struct { + accepted bool + rejectClass millv1.RejectClass + reason string +} + +func addReplyingCandidateSession(t *testing.T, m *Mill, nodeID string, labels []string, load float64, asked chan<- string, reply reserveReply) *millSession { + t.Helper() + var sess *millSession + sess = addCandidateSession(t, m, nodeID, labels, load, scriptedEncoder(func(msg *millproto.Message) error { + rs := msg.GetReserveSeat() + if rs == nil { + return nil + } + if asked != nil { + asked <- nodeID + } + sess.deliver(rs.GetLeaseId(), &millproto.Message{ReserveResult: &millv1.ReserveResult{ + LeaseId: rs.GetLeaseId(), + Accepted: reply.accepted, + RejectReason: reply.reason, + RejectClass: reply.rejectClass, + }}) + return nil + })) + return sess +} + +func drainAsked(ch <-chan string) []string { + var out []string + for { + select { + case nodeID := <-ch: + out = append(out, nodeID) + default: + return out + } + } +} + +func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second}) + wf := testWorkflow("build") + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = wid + wf.Data.(*millWorkflowState).Lease = lease + + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + var sess1 *millSession + firstCommit := make(chan struct{}) + sess1 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + if msg.GetCommitLease() != nil { + close(firstCommit) + m.detachSession(sess1) + } + return nil + }), l) + m.attachSession(sess1) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- m.commitAndWait(ctx, wf, nil) }() + + select { + case <-firstCommit: + case <-ctx.Done(): + t.Fatal("first commit was not sent") + } + + var sess2 *millSession + sess2 = newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + if msg.GetCommitLease() == nil { + return nil + } + leaseID := msg.GetCommitLease().GetLeaseId() + sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) + _ = m.onEventBatch(sess2, &millv1.EventBatch{ + Epoch: sess2.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: leaseID, + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }) + return nil + }), l) + m.attachSession(sess2) + m.sessionReady(sess2) + + select { + case err := <-done: + if err != nil { + t.Fatalf("commitAndWait() error = %v, want success after reconnect", err) + } + case <-ctx.Done(): + t.Fatal("commitAndWait() did not finish after reconnect") + } +} + +func TestDestroyRunningLeaseDoesNotDropCancelledTerminal(t *testing.T) { + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(l, Config{}) + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = wid + lease.setState(leaseRunning) + + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + m.destroy(wid) + if lease.getState() == leaseDone { + t.Fatal("destroy sealed the lease before the terminal result") + } + + lease.deliverTerminal(&millv1.AttemptResult{ + Status: millv1.TerminalStatus_CANCELLED, + }) + res := <-lease.terminal + if err := terminalError(res.Status); !errors.Is(err, engine.ErrWorkflowCanceled) { + t.Fatalf("terminalError() = %v, want ErrCancelled", err) + } +} + +func TestPlaceBlocksWhenNoCapacity(t *testing.T) { + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(l, Config{}) + wf := testWorkflow("build") + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + + // no executors at all: place must block until ctx expires (user sees pending) + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + _, err := m.place(ctx, "dummy", wid, wf) + if err != context.DeadlineExceeded { + t.Fatalf("place() error = %v, want DeadlineExceeded", err) + } +} + +func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) { + m := New(slog.New(slog.NewTextHandler(io.Discard, nil)), Config{}) + addCandidateSession(t, m, "linux-high", []string{"linux"}, 0.0, nil) + addCandidateSession(t, m, "linux-arm", []string{"linux", "arm64"}, 0.25, nil) + addCandidateSession(t, m, "unlabeled", nil, 0.5, nil) + addCandidateSession(t, m, "linux-arm-gpu", []string{"linux", "arm64", "gpu"}, 0.75, nil) + addCandidateSession(t, m, "linux-arm-full", []string{"linux", "arm64"}, 1.0, nil) + + tests := []struct { + name string + requiredLabels []string + want []string + }{ + { + name: "no required labels keeps old capacity ranking", + want: []string{"linux-high", "linux-arm", "unlabeled", "linux-arm-gpu"}, + }, + { + name: "single required label includes every candidate carrying it", + requiredLabels: []string{"linux"}, + want: []string{"linux-high", "linux-arm", "linux-arm-gpu"}, + }, + { + name: "all required labels must be present", + requiredLabels: []string{"linux", "arm64"}, + want: []string{"linux-arm", "linux-arm-gpu"}, + }, + { + name: "one missing required label excludes the candidate", + requiredLabels: []string{"linux", "arm64", "gpu"}, + want: []string{"linux-arm-gpu"}, + }, + { + name: "unknown required label leaves no candidate", + requiredLabels: []string{"linux", "arm64", "metal"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want) + }) + } +} + +func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) { + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(l, Config{BidTimeout: 10 * time.Millisecond}) + reserveSent := make(chan struct{}, 1) + addCandidateSession(t, m, "linux-only", []string{"linux"}, 0.75, scriptedEncoder(func(msg *millproto.Message) error { + if msg.GetReserveSeat() != nil { + select { + case reserveSent <- struct{}{}: + default: + } + } + return nil + })) + wf := testWorkflowWithRunsOn("build", []string{"linux", "arm64"}) + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) + defer cancel() + _, err := m.place(ctx, "dummy", wid, wf) + if err != context.DeadlineExceeded { + t.Fatalf("place() error = %v, want DeadlineExceeded while job remains pending", err) + } + select { + case <-reserveSent: + t.Fatal("place() sent ReserveSeat to executor missing a required label") + default: + } +} + +func TestMaxPendingRejects(t *testing.T) { + l := slog.New(slog.NewTextHandler(io.Discard, nil)) + m := New(l, Config{MaxPending: 1}) + + m.mu.Lock() + m.pending = 1 + m.mu.Unlock() + + wf2 := testWorkflow("b") + _, err := m.place(context.Background(), "dummy", models.WorkflowId{Name: "b"}, wf2) + if err == nil { + t.Fatal("place() past maxPending should error") + } +} + +func TestCancelledRunningLeaseSurvivesReleaseForReconnectReplay(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = wid + lease.setState(leaseRunning) + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatalf("persistLease: %v", err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + m.destroy(wid) + slot := &millSlot{fleet: m, lease: lease} + slot.Release() + + m.mu.Lock() + _, retained := m.leases[lease.id] + m.mu.Unlock() + if !retained { + t.Fatal("slot release removed a cancellation-requested running lease before its terminal result") + } + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { + t.Fatalf("durable leases after slot release = %+v, err = %v; want retained lease", rows, err) + } + + var sentMu sync.Mutex + var sent []*millproto.Message + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + sentMu.Lock() + sent = append(sent, msg) + sentMu.Unlock() + return nil + }), discardLogger()) + if _, ok := m.attachSession(sess); !ok { + t.Fatal("attachSession rejected reconnect") + } + m.sessionReady(sess) + sentMu.Lock() + var replayed bool + for _, msg := range sent { + if cancel := msg.GetCancelAttempt(); cancel != nil && cancel.GetLeaseId() == lease.id { + replayed = true + } + } + sentMu.Unlock() + if !replayed { + t.Fatal("reconnect did not replay CancelAttempt for retained lease") + } + + if err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: lease.id, + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_CANCELLED, + }, + }, + }, + }, + }); err != nil { + t.Fatalf("onEventBatch: %v", err) + } + m.mu.Lock() + _, retained = m.leases[lease.id] + m.mu.Unlock() + if retained { + t.Fatal("terminal result did not clean retained cancelled lease") + } + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { + t.Fatalf("durable leases after terminal = %+v, err = %v; want none", rows, err) + } + slot.Release() +} + +func TestSessionRequestCancelledBeforeRegistrationOrSend(t *testing.T) { + sent := 0 + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(*millproto.Message) error { + sent++ + return nil + }), discardLogger()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := sess.request(ctx, "lease-1", &millproto.Message{ReleaseLease: &millv1.ReleaseLease{LeaseId: "lease-1"}}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("request error = %v, want context.Canceled", err) + } + if sent != 0 { + t.Fatalf("request sent %d messages for an already-cancelled context, want 0", sent) + } + sess.mu.Lock() + pending := len(sess.pending) + sess.mu.Unlock() + if pending != 0 { + t.Fatalf("request left %d pending waiters, want 0", pending) + } +} + +func TestAttachSessionReplacesSilentIncumbentButRejectsActiveDuplicate(t *testing.T) { + m := New(discardLogger(), Config{ReconnectGrace: time.Minute}) + old := newSession("node-1", "inc-old", nil, nopEncoder(), discardLogger()) + transportClosed := make(chan struct{}) + old.closeTransport = func() error { + close(transportClosed) + return nil + } + if _, ok := m.attachSession(old); !ok { + t.Fatal("first attach rejected") + } + m.mu.Lock() + old.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) + m.mu.Unlock() + + replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) + if _, ok := m.attachSession(replacement); !ok { + t.Fatal("silent incumbent blocked authenticated replacement") + } + select { + case <-transportClosed: + default: + t.Fatal("replacing a silent incumbent did not close its transport") + } + m.mu.Lock() + replacement.lastSeen = time.Now().Add(-2 * m.cfg.ReconnectGrace) + m.mu.Unlock() + if err := replacement.dispatch(m, &millproto.Message{NodeSnapshot: &millv1.NodeSnapshot{Seqno: 1}}); err != nil { + t.Fatalf("periodic snapshot dispatch: %v", err) + } + if _, ok := m.attachSession(newSession("node-1", "inc-dup", nil, nopEncoder(), discardLogger())); ok { + t.Fatal("active replacement did not reject a duplicate session") + } +} + +func TestPlaceReleasesRemoteReservationWhenInitialPersistenceFails(t *testing.T) { + m, bdb := restoreTestMill(t, Config{BidTimeout: time.Second}) + if err := bdb.Close(); err != nil { + t.Fatalf("close db: %v", err) + } + released := make(chan string, 1) + var sess *millSession + sess = addCandidateSession(t, m, "node-1", nil, 0, scriptedEncoder(func(msg *millproto.Message) error { + switch { + case msg.GetReserveSeat() != nil: + leaseID := msg.GetReserveSeat().GetLeaseId() + sess.deliver(leaseID, &millproto.Message{ReserveResult: &millv1.ReserveResult{ + LeaseId: leaseID, + Accepted: true, + }}) + case msg.GetReleaseLease() != nil: + released <- msg.GetReleaseLease().GetLeaseId() + } + return nil + })) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + slot, err := m.place( + ctx, + "dummy", + models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + testWorkflow("build"), + ) + if err == nil { + t.Fatal("place succeeded after reserved lease persistence failed") + } + if slot != nil { + t.Fatalf("place returned slot %T after persistence failure", slot) + } + select { + case leaseID := <-released: + if leaseID == "" { + t.Fatal("ReleaseLease had empty lease id") + } + case <-time.After(time.Second): + t.Fatal("persistence failure did not compensate with ReleaseLease") + } + m.mu.Lock() + leases := len(m.leases) + m.mu.Unlock() + if leases != 0 { + t.Fatalf("mill published %d leases after initial persistence failure, want 0", leases) + } +} + +func TestGapsAndDuplicates(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 0, + LeaseId: owned.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("expected duplicate to be skipped without error, got: %v", err) + } + + err = m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 2, + LeaseId: owned.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, + }, + }, + }, + }) + if err == nil { + t.Fatal("expected error due to seqno gap, got nil") + } +} + +func TestAtomicBatchRollback(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + if _, err := bdb.Exec(` + create trigger reject_status_event + before insert on events + begin + select raise(abort, 'forced status event failure'); + end + `); err != nil { + t.Fatalf("failed to create fail trigger: %v", err) + } + defer bdb.Exec("drop trigger reject_status_event") + + err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: owned.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{Status: millv1.NonterminalStatus_RUNNING}, + }, + }, + }, + }) + if err == nil { + t.Fatal("expected status event insertion to fail due to trigger") + } + + m.mu.Lock() + seqno := m.nodeSeqno["node-1/inc-1"] + m.mu.Unlock() + if seqno != 0 { + t.Fatalf("expected seqno 0 due to rollback, got %d", seqno) + } +} + +func TestTerminalBeforeACK(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + + ackSent := make(chan struct{}) + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + if msg.GetAck() != nil { + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + st, err := bdb.GetStatus(wid) + if err != nil || st.Status != "success" { + t.Errorf("expected terminal status success at ACK time, got status: %v, err: %v", st, err) + } + close(ackSent) + } + return nil + }), discardLogger()) + m.attachSession(sess) + + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: owned.id, + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{Status: millv1.TerminalStatus_SUCCESS}, + }, + }, + }, + }) + if err != nil { + t.Fatalf("onEventBatch: %v", err) + } + + select { + case <-ackSent: + case <-time.After(2 * time.Second): + t.Fatal("ACK was not sent") + } +} + +func TestExecutorRestartEmptySnapshot(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + + lease := newLease("lease-1", "node-1", "inc-old", "dummy") + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatalf("persistLease: %v", err) + } + + sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 1, + ActiveLeaseIds: nil, + }) + if err != nil { + t.Fatalf("onSnapshot: %v", err) + } + + m.mu.Lock() + _, stillActive := m.leases["lease-1"] + m.mu.Unlock() + if stillActive { + t.Fatal("expected old epoch lease to be reconciled and failed") + } +} + +func TestReplacementLostBeforeSnapshotFailsOldEpochLease(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + lease := newLease("lease-1", "node-1", "inc-old", "dummy") + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.setState(leaseRunning) + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatalf("persistLease: %v", err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + replacement := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) + if _, ok := m.attachSession(replacement); !ok { + t.Fatal("attachSession rejected replacement") + } + replacement.disconnected = true + m.failLeasesAfterGrace(replacement) + + m.mu.Lock() + _, stillActive := m.leases[lease.id] + m.mu.Unlock() + if stillActive { + t.Fatal("replacement loss stranded old-epoch lease") + } +} + +func TestSeqRegression(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 5, + }) + if err != nil { + t.Fatalf("first snapshot: %v", err) + } + + err = m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 4, + }) + if err == nil { + t.Fatal("expected seqno regression to be rejected") + } +} + +func TestClaimedSweep(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.orphaned = true + lease.claimed = false + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + err := m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 1, + ActiveLeaseIds: []string{"lease-1"}, + }) + if err != nil { + t.Fatalf("onSnapshot: %v", err) + } + + m.detachSession(sess) + + m.sweepUnclaimedOrphans() + + m.mu.Lock() + _, stillRunning := m.leases["lease-1"] + m.mu.Unlock() + if !stillRunning { + t.Fatal("claimed lease was incorrectly swept by startup sweep") + } +} + +func TestCancelDeadline(t *testing.T) { + m, _ := restoreTestMill(t, Config{CancelTimeout: 50 * time.Millisecond}) + + sessClosed := make(chan struct{}) + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + return nil + }), discardLogger()) + sess.closeTransport = func() error { + close(sessClosed) + return nil + } + m.attachSession(sess) + + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.setState(leaseRunning) + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + m.destroy(lease.wid) + + select { + case <-sessClosed: + case <-time.After(1 * time.Second): + t.Fatal("session was not closed after cancel deadline expiration") + } +} + +func TestCleanupRetry(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + + lease := newLease("lease-1", "node-1", "inc-1", "dummy") + lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatalf("persist lease: %v", err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + if _, err := bdb.Exec(` + create trigger reject_cleanup_delete + before delete on mill_leases + begin + select raise(abort, 'forced delete failure'); + end + `); err != nil { + t.Fatalf("failed to create fail trigger: %v", err) + } + + err := m.cleanupLease(lease) + if err == nil { + t.Fatal("expected cleanupLease to fail") + } + + m.mu.Lock() + _, stillRunning := m.leases["lease-1"] + m.mu.Unlock() + if !stillRunning { + t.Fatal("lease was removed from memory despite cleanup failure") + } + + if _, err := bdb.Exec("drop trigger reject_cleanup_delete"); err != nil { + t.Fatalf("drop trigger: %v", err) + } + + err = m.cleanupLease(lease) + if err != nil { + t.Fatalf("expected retry cleanup to succeed, got: %v", err) + } + + m.mu.Lock() + _, stillRunning = m.leases["lease-1"] + m.mu.Unlock() + if stillRunning { + t.Fatal("lease still in memory after successful cleanup retry") + } +} + +func TestBoundedBidding(t *testing.T) { + m := New(discardLogger(), Config{TopK: 2, BidTimeout: 10 * time.Millisecond}) + + addCandidateSession(t, m, "node-1", nil, 0, nil) + addCandidateSession(t, m, "node-2", nil, 0, nil) + addCandidateSession(t, m, "node-3", nil, 0, nil) + addCandidateSession(t, m, "node-4", nil, 0, nil) + addCandidateSession(t, m, "node-5", nil, 0, nil) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + lease, err := m.bid(ctx, "dummy", models.WorkflowId{}, testWorkflow("build")) + if err != nil { + t.Fatalf("bid: %v", err) + } + if lease != nil { + t.Fatalf("did not expect a lease, got %+v", lease) + } +} diff --git a/spindle/mill/proto/gen/mill.pb.go b/spindle/mill/proto/gen/mill.pb.go new file mode 100644 index 00000000..e31758f2 --- /dev/null +++ b/spindle/mill/proto/gen/mill.pb.go @@ -0,0 +1,1679 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: spindle/mill/v1/mill.proto + +package millv1 + +import ( + _ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type RejectClass int32 + +const ( + RejectClass_REJECT_CLASS_UNSPECIFIED RejectClass = 0 + RejectClass_REJECT_CLASS_TRANSIENT RejectClass = 1 + RejectClass_REJECT_CLASS_INCOMPATIBLE RejectClass = 2 +) + +// Enum value maps for RejectClass. +var ( + RejectClass_name = map[int32]string{ + 0: "REJECT_CLASS_UNSPECIFIED", + 1: "REJECT_CLASS_TRANSIENT", + 2: "REJECT_CLASS_INCOMPATIBLE", + } + RejectClass_value = map[string]int32{ + "REJECT_CLASS_UNSPECIFIED": 0, + "REJECT_CLASS_TRANSIENT": 1, + "REJECT_CLASS_INCOMPATIBLE": 2, + } +) + +func (x RejectClass) Enum() *RejectClass { + p := new(RejectClass) + *p = x + return p +} + +func (x RejectClass) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (RejectClass) Descriptor() protoreflect.EnumDescriptor { + return file_spindle_mill_v1_mill_proto_enumTypes[0].Descriptor() +} + +func (RejectClass) Type() protoreflect.EnumType { + return &file_spindle_mill_v1_mill_proto_enumTypes[0] +} + +func (x RejectClass) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use RejectClass.Descriptor instead. +func (RejectClass) EnumDescriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} +} + +type NonterminalStatus int32 + +const ( + NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED NonterminalStatus = 0 + NonterminalStatus_RUNNING NonterminalStatus = 1 +) + +// Enum value maps for NonterminalStatus. +var ( + NonterminalStatus_name = map[int32]string{ + 0: "NONTERMINAL_STATUS_UNSPECIFIED", + 1: "RUNNING", + } + NonterminalStatus_value = map[string]int32{ + "NONTERMINAL_STATUS_UNSPECIFIED": 0, + "RUNNING": 1, + } +) + +func (x NonterminalStatus) Enum() *NonterminalStatus { + p := new(NonterminalStatus) + *p = x + return p +} + +func (x NonterminalStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (NonterminalStatus) Descriptor() protoreflect.EnumDescriptor { + return file_spindle_mill_v1_mill_proto_enumTypes[1].Descriptor() +} + +func (NonterminalStatus) Type() protoreflect.EnumType { + return &file_spindle_mill_v1_mill_proto_enumTypes[1] +} + +func (x NonterminalStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use NonterminalStatus.Descriptor instead. +func (NonterminalStatus) EnumDescriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} +} + +type TerminalStatus int32 + +const ( + TerminalStatus_TERMINAL_STATUS_UNSPECIFIED TerminalStatus = 0 + TerminalStatus_SUCCESS TerminalStatus = 1 + TerminalStatus_FAILED TerminalStatus = 2 + TerminalStatus_TIMEOUT TerminalStatus = 3 + TerminalStatus_CANCELLED TerminalStatus = 4 +) + +// Enum value maps for TerminalStatus. +var ( + TerminalStatus_name = map[int32]string{ + 0: "TERMINAL_STATUS_UNSPECIFIED", + 1: "SUCCESS", + 2: "FAILED", + 3: "TIMEOUT", + 4: "CANCELLED", + } + TerminalStatus_value = map[string]int32{ + "TERMINAL_STATUS_UNSPECIFIED": 0, + "SUCCESS": 1, + "FAILED": 2, + "TIMEOUT": 3, + "CANCELLED": 4, + } +) + +func (x TerminalStatus) Enum() *TerminalStatus { + p := new(TerminalStatus) + *p = x + return p +} + +func (x TerminalStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TerminalStatus) Descriptor() protoreflect.EnumDescriptor { + return file_spindle_mill_v1_mill_proto_enumTypes[2].Descriptor() +} + +func (TerminalStatus) Type() protoreflect.EnumType { + return &file_spindle_mill_v1_mill_proto_enumTypes[2] +} + +func (x TerminalStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TerminalStatus.Descriptor instead. +func (TerminalStatus) EnumDescriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} +} + +// executor identity, sent on connect +type Hello struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + // GOARCH of the node, informational only + Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"` + // operator-defined labels, matched against runs_on + Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` + Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Hello) Reset() { + *x = Hello{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Hello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Hello) ProtoMessage() {} + +func (x *Hello) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Hello.ProtoReflect.Descriptor instead. +func (*Hello) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{0} +} + +func (x *Hello) GetProtocolVersion() uint32 { + if x != nil { + return x.ProtocolVersion + } + return 0 +} + +func (x *Hello) GetArch() string { + if x != nil { + return x.Arch + } + return "" +} + +func (x *Hello) GetLabels() []string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *Hello) GetEpoch() string { + if x != nil { + return x.Epoch + } + return "" +} + +// reconnect state for an existing epoch +type Resume struct { + state protoimpl.MessageState `protogen:"open.v1"` + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` + AckSeqno uint64 `protobuf:"varint,2,opt,name=ack_seqno,json=ackSeqno,proto3" json:"ack_seqno,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Resume) Reset() { + *x = Resume{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Resume) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Resume) ProtoMessage() {} + +func (x *Resume) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Resume.ProtoReflect.Descriptor instead. +func (*Resume) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{1} +} + +func (x *Resume) GetEpoch() string { + if x != nil { + return x.Epoch + } + return "" +} + +func (x *Resume) GetAckSeqno() uint64 { + if x != nil { + return x.AckSeqno + } + return 0 +} + +// per-engine state of a node +type EngineAvailability struct { + state protoimpl.MessageState `protogen:"open.v1"` + Available bool `protobuf:"varint,1,opt,name=available,proto3" json:"available,omitempty"` + // opaque engine-defined load metrics, higher means more loaded + Load map[string]float64 `protobuf:"bytes,2,rep,name=load,proto3" json:"load,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"fixed64,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EngineAvailability) Reset() { + *x = EngineAvailability{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EngineAvailability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EngineAvailability) ProtoMessage() {} + +func (x *EngineAvailability) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EngineAvailability.ProtoReflect.Descriptor instead. +func (*EngineAvailability) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} +} + +func (x *EngineAvailability) GetAvailable() bool { + if x != nil { + return x.Available + } + return false +} + +func (x *EngineAvailability) GetLoad() map[string]float64 { + if x != nil { + return x.Load + } + return nil +} + +// full node state +type NodeSnapshot struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` + Engines map[string]*EngineAvailability `protobuf:"bytes,2,rep,name=engines,proto3" json:"engines,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // every lease the executor currently holds, reserved or running + ActiveLeaseIds []string `protobuf:"bytes,3,rep,name=active_lease_ids,json=activeLeaseIds,proto3" json:"active_lease_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NodeSnapshot) Reset() { + *x = NodeSnapshot{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NodeSnapshot) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeSnapshot) ProtoMessage() {} + +func (x *NodeSnapshot) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeSnapshot.ProtoReflect.Descriptor instead. +func (*NodeSnapshot) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} +} + +func (x *NodeSnapshot) GetSeqno() uint64 { + if x != nil { + return x.Seqno + } + return 0 +} + +func (x *NodeSnapshot) GetEngines() map[string]*EngineAvailability { + if x != nil { + return x.Engines + } + return nil +} + +func (x *NodeSnapshot) GetActiveLeaseIds() []string { + if x != nil { + return x.ActiveLeaseIds + } + return nil +} + +// a seat reservation, carries no secrets +type ReserveSeat struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + TargetEngine string `protobuf:"bytes,2,opt,name=target_engine,json=targetEngine,proto3" json:"target_engine,omitempty"` + RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` + RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` + // pipeline id, the executor reconstructs the exact WorkflowId from it + Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` + Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` + TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveSeat) Reset() { + *x = ReserveSeat{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveSeat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveSeat) ProtoMessage() {} + +func (x *ReserveSeat) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveSeat.ProtoReflect.Descriptor instead. +func (*ReserveSeat) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{4} +} + +func (x *ReserveSeat) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *ReserveSeat) GetTargetEngine() string { + if x != nil { + return x.TargetEngine + } + return "" +} + +func (x *ReserveSeat) GetRawPipelineJson() string { + if x != nil { + return x.RawPipelineJson + } + return "" +} + +func (x *ReserveSeat) GetRawWorkflowJson() string { + if x != nil { + return x.RawWorkflowJson + } + return "" +} + +func (x *ReserveSeat) GetKnot() string { + if x != nil { + return x.Knot + } + return "" +} + +func (x *ReserveSeat) GetRkey() string { + if x != nil { + return x.Rkey + } + return "" +} + +func (x *ReserveSeat) GetTtlSeconds() uint32 { + if x != nil { + return x.TtlSeconds + } + return 0 +} + +type ReserveResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + Accepted bool `protobuf:"varint,2,opt,name=accepted,proto3" json:"accepted,omitempty"` + RejectReason string `protobuf:"bytes,3,opt,name=reject_reason,json=rejectReason,proto3" json:"reject_reason,omitempty"` + RejectClass RejectClass `protobuf:"varint,4,opt,name=reject_class,json=rejectClass,proto3,enum=spindle.mill.v1.RejectClass" json:"reject_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveResult) Reset() { + *x = ReserveResult{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveResult) ProtoMessage() {} + +func (x *ReserveResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveResult.ProtoReflect.Descriptor instead. +func (*ReserveResult) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{5} +} + +func (x *ReserveResult) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *ReserveResult) GetAccepted() bool { + if x != nil { + return x.Accepted + } + return false +} + +func (x *ReserveResult) GetRejectReason() string { + if x != nil { + return x.RejectReason + } + return "" +} + +func (x *ReserveResult) GetRejectClass() RejectClass { + if x != nil { + return x.RejectClass + } + return RejectClass_REJECT_CLASS_UNSPECIFIED +} + +// a single unlocked secret +type Secret struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Secret) Reset() { + *x = Secret{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Secret) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Secret) ProtoMessage() {} + +func (x *Secret) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Secret.ProtoReflect.Descriptor instead. +func (*Secret) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{6} +} + +func (x *Secret) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Secret) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +// promotes a reservation to a running job and hands over the secrets +type CommitLease struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + Secrets []*Secret `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommitLease) Reset() { + *x = CommitLease{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommitLease) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommitLease) ProtoMessage() {} + +func (x *CommitLease) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommitLease.ProtoReflect.Descriptor instead. +func (*CommitLease) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} +} + +func (x *CommitLease) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *CommitLease) GetSecrets() []*Secret { + if x != nil { + return x.Secrets + } + return nil +} + +type Committed struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Committed) Reset() { + *x = Committed{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Committed) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Committed) ProtoMessage() {} + +func (x *Committed) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Committed.ProtoReflect.Descriptor instead. +func (*Committed) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} +} + +func (x *Committed) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +// drops a reservation that was never committed +type ReleaseLease struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseLease) Reset() { + *x = ReleaseLease{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseLease) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseLease) ProtoMessage() {} + +func (x *ReleaseLease) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead. +func (*ReleaseLease) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} +} + +func (x *ReleaseLease) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +// cancels a running attempt +type CancelAttempt struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CancelAttempt) Reset() { + *x = CancelAttempt{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CancelAttempt) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelAttempt) ProtoMessage() {} + +func (x *CancelAttempt) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead. +func (*CancelAttempt) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} +} + +func (x *CancelAttempt) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *CancelAttempt) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +type CancelAck struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CancelAck) Reset() { + *x = CancelAck{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CancelAck) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CancelAck) ProtoMessage() {} + +func (x *CancelAck) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CancelAck.ProtoReflect.Descriptor instead. +func (*CancelAck) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} +} + +func (x *CancelAck) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +type StatusEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status NonterminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.NonterminalStatus" json:"status,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` + ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StatusEvent) Reset() { + *x = StatusEvent{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StatusEvent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatusEvent) ProtoMessage() {} + +func (x *StatusEvent) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. +func (*StatusEvent) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} +} + +func (x *StatusEvent) GetStatus() NonterminalStatus { + if x != nil { + return x.Status + } + return NonterminalStatus_NONTERMINAL_STATUS_UNSPECIFIED +} + +func (x *StatusEvent) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *StatusEvent) GetExitCode() int64 { + if x != nil { + return x.ExitCode + } + return 0 +} + +type LogArtifact struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ref string `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"` + Hash string `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LogArtifact) Reset() { + *x = LogArtifact{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LogArtifact) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LogArtifact) ProtoMessage() {} + +func (x *LogArtifact) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. +func (*LogArtifact) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} +} + +func (x *LogArtifact) GetRef() string { + if x != nil { + return x.Ref + } + return "" +} + +func (x *LogArtifact) GetHash() string { + if x != nil { + return x.Hash + } + return "" +} + +// terminal outcome of an attempt +type AttemptResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Status TerminalStatus `protobuf:"varint,1,opt,name=status,proto3,enum=spindle.mill.v1.TerminalStatus" json:"status,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` + ExitCode int64 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + LogArtifact *LogArtifact `protobuf:"bytes,4,opt,name=log_artifact,json=logArtifact,proto3" json:"log_artifact,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttemptResult) Reset() { + *x = AttemptResult{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttemptResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttemptResult) ProtoMessage() {} + +func (x *AttemptResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead. +func (*AttemptResult) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} +} + +func (x *AttemptResult) GetStatus() TerminalStatus { + if x != nil { + return x.Status + } + return TerminalStatus_TERMINAL_STATUS_UNSPECIFIED +} + +func (x *AttemptResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *AttemptResult) GetExitCode() int64 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *AttemptResult) GetLogArtifact() *LogArtifact { + if x != nil { + return x.LogArtifact + } + return nil +} + +// live non-replay log frame +type LiveLog struct { + state protoimpl.MessageState `protogen:"open.v1"` + LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + RawJson []byte `protobuf:"bytes,2,opt,name=raw_json,json=rawJson,proto3" json:"raw_json,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LiveLog) Reset() { + *x = LiveLog{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LiveLog) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LiveLog) ProtoMessage() {} + +func (x *LiveLog) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. +func (*LiveLog) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} +} + +func (x *LiveLog) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *LiveLog) GetRawJson() []byte { + if x != nil { + return x.RawJson + } + return nil +} + +// one event in the executor's stream back to the mill +type Event struct { + state protoimpl.MessageState `protogen:"open.v1"` + Seqno uint64 `protobuf:"varint,1,opt,name=seqno,proto3" json:"seqno,omitempty"` + LeaseId string `protobuf:"bytes,2,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` + // Types that are valid to be assigned to Payload: + // + // *Event_StatusEvent + // *Event_AttemptResult + Payload isEvent_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Event) Reset() { + *x = Event{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Event) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Event) ProtoMessage() {} + +func (x *Event) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Event.ProtoReflect.Descriptor instead. +func (*Event) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} +} + +func (x *Event) GetSeqno() uint64 { + if x != nil { + return x.Seqno + } + return 0 +} + +func (x *Event) GetLeaseId() string { + if x != nil { + return x.LeaseId + } + return "" +} + +func (x *Event) GetPayload() isEvent_Payload { + if x != nil { + return x.Payload + } + return nil +} + +func (x *Event) GetStatusEvent() *StatusEvent { + if x != nil { + if x, ok := x.Payload.(*Event_StatusEvent); ok { + return x.StatusEvent + } + } + return nil +} + +func (x *Event) GetAttemptResult() *AttemptResult { + if x != nil { + if x, ok := x.Payload.(*Event_AttemptResult); ok { + return x.AttemptResult + } + } + return nil +} + +type isEvent_Payload interface { + isEvent_Payload() +} + +type Event_StatusEvent struct { + StatusEvent *StatusEvent `protobuf:"bytes,3,opt,name=status_event,json=statusEvent,proto3,oneof"` +} + +type Event_AttemptResult struct { + AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` +} + +func (*Event_StatusEvent) isEvent_Payload() {} + +func (*Event_AttemptResult) isEvent_Payload() {} + +// a flushed bundle of events +type EventBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` + Events []*Event `protobuf:"bytes,2,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EventBatch) Reset() { + *x = EventBatch{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EventBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EventBatch) ProtoMessage() {} + +func (x *EventBatch) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. +func (*EventBatch) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} +} + +func (x *EventBatch) GetEpoch() string { + if x != nil { + return x.Epoch + } + return "" +} + +func (x *EventBatch) GetEvents() []*Event { + if x != nil { + return x.Events + } + return nil +} + +// all events up to and including up_to_seqno are durably processed +type Ack struct { + state protoimpl.MessageState `protogen:"open.v1"` + Epoch string `protobuf:"bytes,1,opt,name=epoch,proto3" json:"epoch,omitempty"` + UpToSeqno uint64 `protobuf:"varint,2,opt,name=up_to_seqno,json=upToSeqno,proto3" json:"up_to_seqno,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ack) Reset() { + *x = Ack{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ack) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ack) ProtoMessage() {} + +func (x *Ack) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ack.ProtoReflect.Descriptor instead. +func (*Ack) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} +} + +func (x *Ack) GetEpoch() string { + if x != nil { + return x.Epoch + } + return "" +} + +func (x *Ack) GetUpToSeqno() uint64 { + if x != nil { + return x.UpToSeqno + } + return 0 +} + +type Message struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hello *Hello `protobuf:"bytes,1,opt,name=hello,proto3" json:"hello,omitempty"` + Resume *Resume `protobuf:"bytes,2,opt,name=resume,proto3" json:"resume,omitempty"` + NodeSnapshot *NodeSnapshot `protobuf:"bytes,3,opt,name=node_snapshot,json=nodeSnapshot,proto3" json:"node_snapshot,omitempty"` + ReserveSeat *ReserveSeat `protobuf:"bytes,4,opt,name=reserve_seat,json=reserveSeat,proto3" json:"reserve_seat,omitempty"` + ReserveResult *ReserveResult `protobuf:"bytes,5,opt,name=reserve_result,json=reserveResult,proto3" json:"reserve_result,omitempty"` + CommitLease *CommitLease `protobuf:"bytes,6,opt,name=commit_lease,json=commitLease,proto3" json:"commit_lease,omitempty"` + Committed *Committed `protobuf:"bytes,7,opt,name=committed,proto3" json:"committed,omitempty"` + ReleaseLease *ReleaseLease `protobuf:"bytes,8,opt,name=release_lease,json=releaseLease,proto3" json:"release_lease,omitempty"` + CancelAttempt *CancelAttempt `protobuf:"bytes,9,opt,name=cancel_attempt,json=cancelAttempt,proto3" json:"cancel_attempt,omitempty"` + CancelAck *CancelAck `protobuf:"bytes,10,opt,name=cancel_ack,json=cancelAck,proto3" json:"cancel_ack,omitempty"` + EventBatch *EventBatch `protobuf:"bytes,11,opt,name=event_batch,json=eventBatch,proto3" json:"event_batch,omitempty"` + Ack *Ack `protobuf:"bytes,12,opt,name=ack,proto3" json:"ack,omitempty"` + LiveLog *LiveLog `protobuf:"bytes,13,opt,name=live_log,json=liveLog,proto3" json:"live_log,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Message) Reset() { + *x = Message{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} +} + +func (x *Message) GetHello() *Hello { + if x != nil { + return x.Hello + } + return nil +} + +func (x *Message) GetResume() *Resume { + if x != nil { + return x.Resume + } + return nil +} + +func (x *Message) GetNodeSnapshot() *NodeSnapshot { + if x != nil { + return x.NodeSnapshot + } + return nil +} + +func (x *Message) GetReserveSeat() *ReserveSeat { + if x != nil { + return x.ReserveSeat + } + return nil +} + +func (x *Message) GetReserveResult() *ReserveResult { + if x != nil { + return x.ReserveResult + } + return nil +} + +func (x *Message) GetCommitLease() *CommitLease { + if x != nil { + return x.CommitLease + } + return nil +} + +func (x *Message) GetCommitted() *Committed { + if x != nil { + return x.Committed + } + return nil +} + +func (x *Message) GetReleaseLease() *ReleaseLease { + if x != nil { + return x.ReleaseLease + } + return nil +} + +func (x *Message) GetCancelAttempt() *CancelAttempt { + if x != nil { + return x.CancelAttempt + } + return nil +} + +func (x *Message) GetCancelAck() *CancelAck { + if x != nil { + return x.CancelAck + } + return nil +} + +func (x *Message) GetEventBatch() *EventBatch { + if x != nil { + return x.EventBatch + } + return nil +} + +func (x *Message) GetAck() *Ack { + if x != nil { + return x.Ack + } + return nil +} + +func (x *Message) GetLiveLog() *LiveLog { + if x != nil { + return x.LiveLog + } + return nil +} + +var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor + +const file_spindle_mill_v1_mill_proto_rawDesc = "" + + "\n" + + "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" + + "\x05Hello\x12)\n" + + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" + + "\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" + + "\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" + + "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" + + "\x06Resume\x12\x1d\n" + + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" + + "\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" + + "\x12EngineAvailability\x12\x1c\n" + + "\tavailable\x18\x01 \x01(\bR\tavailable\x12A\n" + + "\x04load\x18\x02 \x03(\v2-.spindle.mill.v1.EngineAvailability.LoadEntryR\x04load\x1a7\n" + + "\tLoadEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x01R\x05value:\x028\x01\"\xfe\x01\n" + + "\fNodeSnapshot\x12\x1d\n" + + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12D\n" + + "\aengines\x18\x02 \x03(\v2*.spindle.mill.v1.NodeSnapshot.EnginesEntryR\aengines\x12(\n" + + "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + + "\fEnginesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + + "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\x80\x02\n" + + "\vReserveSeat\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + + "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + + "\x11raw_pipeline_json\x18\x03 \x01(\tR\x0frawPipelineJson\x12*\n" + + "\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12\x12\n" + + "\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" + + "\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" + + "\vttl_seconds\x18\a \x01(\rR\n" + + "ttlSeconds\"\xbf\x01\n" + + "\rReserveResult\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + + "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + + "\rreject_reason\x18\x03 \x01(\tR\frejectReason\x12I\n" + + "\freject_class\x18\x04 \x01(\x0e2\x1c.spindle.mill.v1.RejectClassB\b\xbaH\x05\x82\x01\x02\x10\x01R\vrejectClass\"0\n" + + "\x06Secret\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"d\n" + + "\vCommitLease\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" + + "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\"/\n" + + "\tCommitted\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" + + "\fReleaseLease\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"K\n" + + "\rCancelAttempt\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\"/\n" + + "\tCancelAck\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"\x93\x01\n" + + "\vStatusEvent\x12F\n" + + "\x06status\x18\x01 \x01(\x0e2\".spindle.mill.v1.NonterminalStatusB\n" + + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + + "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + + "\texit_code\x18\x03 \x01(\x03R\bexitCode\"E\n" + + "\vLogArtifact\x12\x19\n" + + "\x03ref\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x03ref\x12\x1b\n" + + "\x04hash\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x04hash\"\xd3\x01\n" + + "\rAttemptResult\x12C\n" + + "\x06status\x18\x01 \x01(\x0e2\x1f.spindle.mill.v1.TerminalStatusB\n" + + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + + "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + + "\texit_code\x18\x03 \x01(\x03R\bexitCode\x12?\n" + + "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"Q\n" + + "\aLiveLog\x12\"\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + + "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + + "\x05Event\x12\x1d\n" + + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + + "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + + "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + + "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + + "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + + "\n" + + "EventBatch\x12\x1d\n" + + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x128\n" + + "\x06events\x18\x02 \x03(\v2\x16.spindle.mill.v1.EventB\b\xbaH\x05\x92\x01\x02\b\x01R\x06events\"D\n" + + "\x03Ack\x12\x1d\n" + + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1e\n" + + "\vup_to_seqno\x18\x02 \x01(\x04R\tupToSeqno\"\xb8\a\n" + + "\aMessage\x12,\n" + + "\x05hello\x18\x01 \x01(\v2\x16.spindle.mill.v1.HelloR\x05hello\x12/\n" + + "\x06resume\x18\x02 \x01(\v2\x17.spindle.mill.v1.ResumeR\x06resume\x12B\n" + + "\rnode_snapshot\x18\x03 \x01(\v2\x1d.spindle.mill.v1.NodeSnapshotR\fnodeSnapshot\x12?\n" + + "\freserve_seat\x18\x04 \x01(\v2\x1c.spindle.mill.v1.ReserveSeatR\vreserveSeat\x12E\n" + + "\x0ereserve_result\x18\x05 \x01(\v2\x1e.spindle.mill.v1.ReserveResultR\rreserveResult\x12?\n" + + "\fcommit_lease\x18\x06 \x01(\v2\x1c.spindle.mill.v1.CommitLeaseR\vcommitLease\x128\n" + + "\tcommitted\x18\a \x01(\v2\x1a.spindle.mill.v1.CommittedR\tcommitted\x12B\n" + + "\rrelease_lease\x18\b \x01(\v2\x1d.spindle.mill.v1.ReleaseLeaseR\freleaseLease\x12E\n" + + "\x0ecancel_attempt\x18\t \x01(\v2\x1e.spindle.mill.v1.CancelAttemptR\rcancelAttempt\x129\n" + + "\n" + + "cancel_ack\x18\n" + + " \x01(\v2\x1a.spindle.mill.v1.CancelAckR\tcancelAck\x12<\n" + + "\vevent_batch\x18\v \x01(\v2\x1b.spindle.mill.v1.EventBatchR\n" + + "eventBatch\x12&\n" + + "\x03ack\x18\f \x01(\v2\x14.spindle.mill.v1.AckR\x03ack\x123\n" + + "\blive_log\x18\r \x01(\v2\x18.spindle.mill.v1.LiveLogR\aliveLog:\xa5\x01\xbaH\xa1\x01\"\x9e\x01\n" + + "\x05hello\n" + + "\x06resume\n" + + "\rnode_snapshot\n" + + "\freserve_seat\n" + + "\x0ereserve_result\n" + + "\fcommit_lease\n" + + "\tcommitted\n" + + "\rrelease_lease\n" + + "\x0ecancel_attempt\n" + + "\n" + + "cancel_ack\n" + + "\vevent_batch\n" + + "\x03ack\n" + + "\blive_log\x10\x01*f\n" + + "\vRejectClass\x12\x1c\n" + + "\x18REJECT_CLASS_UNSPECIFIED\x10\x00\x12\x1a\n" + + "\x16REJECT_CLASS_TRANSIENT\x10\x01\x12\x1d\n" + + "\x19REJECT_CLASS_INCOMPATIBLE\x10\x02*D\n" + + "\x11NonterminalStatus\x12\"\n" + + "\x1eNONTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + + "\aRUNNING\x10\x01*f\n" + + "\x0eTerminalStatus\x12\x1f\n" + + "\x1bTERMINAL_STATUS_UNSPECIFIED\x10\x00\x12\v\n" + + "\aSUCCESS\x10\x01\x12\n" + + "\n" + + "\x06FAILED\x10\x02\x12\v\n" + + "\aTIMEOUT\x10\x03\x12\r\n" + + "\tCANCELLED\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3" + +var ( + file_spindle_mill_v1_mill_proto_rawDescOnce sync.Once + file_spindle_mill_v1_mill_proto_rawDescData []byte +) + +func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte { + file_spindle_mill_v1_mill_proto_rawDescOnce.Do(func() { + file_spindle_mill_v1_mill_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc))) + }) + return file_spindle_mill_v1_mill_proto_rawDescData +} + +var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_spindle_mill_v1_mill_proto_goTypes = []any{ + (RejectClass)(0), // 0: spindle.mill.v1.RejectClass + (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus + (TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus + (*Hello)(nil), // 3: spindle.mill.v1.Hello + (*Resume)(nil), // 4: spindle.mill.v1.Resume + (*EngineAvailability)(nil), // 5: spindle.mill.v1.EngineAvailability + (*NodeSnapshot)(nil), // 6: spindle.mill.v1.NodeSnapshot + (*ReserveSeat)(nil), // 7: spindle.mill.v1.ReserveSeat + (*ReserveResult)(nil), // 8: spindle.mill.v1.ReserveResult + (*Secret)(nil), // 9: spindle.mill.v1.Secret + (*CommitLease)(nil), // 10: spindle.mill.v1.CommitLease + (*Committed)(nil), // 11: spindle.mill.v1.Committed + (*ReleaseLease)(nil), // 12: spindle.mill.v1.ReleaseLease + (*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt + (*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck + (*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent + (*LogArtifact)(nil), // 16: spindle.mill.v1.LogArtifact + (*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult + (*LiveLog)(nil), // 18: spindle.mill.v1.LiveLog + (*Event)(nil), // 19: spindle.mill.v1.Event + (*EventBatch)(nil), // 20: spindle.mill.v1.EventBatch + (*Ack)(nil), // 21: spindle.mill.v1.Ack + (*Message)(nil), // 22: spindle.mill.v1.Message + nil, // 23: spindle.mill.v1.EngineAvailability.LoadEntry + nil, // 24: spindle.mill.v1.NodeSnapshot.EnginesEntry +} +var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ + 23, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry + 24, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry + 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass + 9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret + 1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus + 2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus + 16, // 6: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact + 15, // 7: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent + 17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult + 19, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event + 3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello + 4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume + 6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot + 7, // 13: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat + 8, // 14: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult + 10, // 15: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease + 11, // 16: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed + 12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease + 13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt + 14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck + 20, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch + 21, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack + 18, // 22: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog + 5, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name +} + +func init() { file_spindle_mill_v1_mill_proto_init() } +func file_spindle_mill_v1_mill_proto_init() { + if File_spindle_mill_v1_mill_proto != nil { + return + } + file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ + (*Event_StatusEvent)(nil), + (*Event_AttemptResult)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), + NumEnums: 3, + NumMessages: 22, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_spindle_mill_v1_mill_proto_goTypes, + DependencyIndexes: file_spindle_mill_v1_mill_proto_depIdxs, + EnumInfos: file_spindle_mill_v1_mill_proto_enumTypes, + MessageInfos: file_spindle_mill_v1_mill_proto_msgTypes, + }.Build() + File_spindle_mill_v1_mill_proto = out.File + file_spindle_mill_v1_mill_proto_goTypes = nil + file_spindle_mill_v1_mill_proto_depIdxs = nil +} diff --git a/spindle/mill/proto/protocol.go b/spindle/mill/proto/protocol.go new file mode 100644 index 00000000..2e1d1402 --- /dev/null +++ b/spindle/mill/proto/protocol.go @@ -0,0 +1,103 @@ +// package millproto carries the mill<->executor session protocol. a new message +// vocabulary over the same length-prefixed protobuf framing the spindle already +// uses to talk to the microVM guest (see spindle/agentproto). only the framing +// pattern is shared. the messages are entirely separate +package millproto + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "buf.build/go/protovalidate" + "google.golang.org/protobuf/proto" + + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +const ( + ProtocolVersion = 1 + // generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and + // workflow JSON, and streamed log lines can be chunky + MaxMessageBytes = 8 * 1024 * 1024 +) + +type Message = millv1.Message + +var validator protovalidate.Validator + +func init() { + var err error + validator, err = protovalidate.New() + if err != nil { + panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err)) + } +} + +type Encoder struct { + mu sync.Mutex + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(msg *Message) error { + if err := validator.Validate(msg); err != nil { + return fmt.Errorf("validate fleet message: %w", err) + } + + data, err := proto.Marshal(msg) + if err != nil { + return fmt.Errorf("marshal fleet message: %w", err) + } + if len(data) > MaxMessageBytes { + return fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) + } + + // single write of header and payload maps to exactly one websocket binary + // frame when the writer is a ws stream + frame := make([]byte, 4+len(data)) + binary.BigEndian.PutUint32(frame[:4], uint32(len(data))) + copy(frame[4:], data) + + e.mu.Lock() + defer e.mu.Unlock() + _, err = e.w.Write(frame) + return err +} + +type Decoder struct { + r io.Reader +} + +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +func (d *Decoder) Decode() (*Message, error) { + msg := &Message{} + var header [4]byte + if _, err := io.ReadFull(d.r, header[:]); err != nil { + return msg, err + } + + size := binary.BigEndian.Uint32(header[:]) + if size > MaxMessageBytes { + return msg, fmt.Errorf("fleet message exceeded %d bytes", MaxMessageBytes) + } + + data := make([]byte, size) + if _, err := io.ReadFull(d.r, data); err != nil { + return msg, err + } + if err := proto.Unmarshal(data, msg); err != nil { + return msg, fmt.Errorf("parse fleet message: %w", err) + } + if err := validator.Validate(msg); err != nil { + return msg, fmt.Errorf("validate fleet message: %w", err) + } + return msg, nil +} diff --git a/spindle/mill/proto/protocol_test.go b/spindle/mill/proto/protocol_test.go new file mode 100644 index 00000000..0e0b9637 --- /dev/null +++ b/spindle/mill/proto/protocol_test.go @@ -0,0 +1,227 @@ +package millproto + +import ( + "bytes" + "encoding/binary" + "testing" + + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +func TestEncodeDecodeRoundTrip(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + + want := &Message{ + ReserveSeat: &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: `{"name":"build"}`, + Knot: "knot.example", + Rkey: "abc123", + TtlSeconds: 30, + }, + } + if err := enc.Encode(want); err != nil { + t.Fatalf("Encode() error = %v", err) + } + + got, err := NewDecoder(&buf).Decode() + if err != nil { + t.Fatalf("Decode() error = %v", err) + } + rs := got.GetReserveSeat() + if rs == nil { + t.Fatal("decoded message missing reserve_seat") + } + if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 { + t.Fatalf("round-trip mismatch: %+v", rs) + } +} + +func TestDecoderRejectsOversizedMessage(t *testing.T) { + var tooLarge bytes.Buffer + var header [4]byte + binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1) + tooLarge.Write(header[:]) + + if _, err := NewDecoder(&tooLarge).Decode(); err == nil { + t.Fatal("expected oversized message error") + } +} + +func TestValidationRules(t *testing.T) { + tests := []struct { + name string + msg *Message + wantErr bool + }{ + { + name: "valid ack message", + msg: &Message{ + Ack: &millv1.Ack{ + Epoch: "inc-1", + UpToSeqno: 5, + }, + }, + wantErr: false, + }, + { + name: "valid hello message", + msg: &Message{ + Hello: &millv1.Hello{ + ProtocolVersion: 1, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + }, + }, + wantErr: false, + }, + { + name: "invalid message with zero payloads", + msg: &Message{}, + wantErr: true, + }, + { + name: "invalid message with multiple payloads", + msg: &Message{ + Ack: &millv1.Ack{Epoch: "inc-1", UpToSeqno: 5}, + Committed: &millv1.Committed{LeaseId: "x"}, + }, + wantErr: true, + }, + { + name: "invalid ack message missing epoch", + msg: &Message{ + Ack: &millv1.Ack{ + UpToSeqno: 5, + }, + }, + wantErr: true, + }, + { + name: "invalid node snapshot with zero seqno", + msg: &Message{ + NodeSnapshot: &millv1.NodeSnapshot{ + Seqno: 0, + }, + }, + wantErr: true, + }, + { + name: "valid node snapshot with positive seqno", + msg: &Message{ + NodeSnapshot: &millv1.NodeSnapshot{ + Seqno: 1, + }, + }, + wantErr: false, + }, + { + name: "invalid stream batch with zero seqno entry", + msg: &Message{ + EventBatch: &millv1.EventBatch{ + Epoch: "inc-1", + Events: []*millv1.Event{ + { + Seqno: 0, + LeaseId: "lease-1", + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "invalid stream batch with empty entries", + msg: &Message{ + EventBatch: &millv1.EventBatch{ + Epoch: "inc-1", + Events: []*millv1.Event{}, + }, + }, + wantErr: true, + }, + { + name: "invalid reserve result with unknown enum", + msg: &Message{ + ReserveResult: &millv1.ReserveResult{ + LeaseId: "lease-1", + RejectClass: millv1.RejectClass(99), + }, + }, + wantErr: true, + }, + { + name: "invalid stream entry with malformed oneof (empty payload)", + msg: &Message{ + EventBatch: &millv1.EventBatch{ + Epoch: "inc-1", + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: "lease-1", + Payload: nil, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "valid stream batch status event", + msg: &Message{ + EventBatch: &millv1.EventBatch{ + Epoch: "inc-1", + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: "lease-1", + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + }, + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "valid stream batch attempt result", + msg: &Message{ + EventBatch: &millv1.EventBatch{ + Epoch: "inc-1", + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: "lease-1", + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }, + }, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := validator.Validate(tc.msg) + if (err != nil) != tc.wantErr { + t.Fatalf("Validate() error = %v, wantErr = %v", err, tc.wantErr) + } + }) + } +} diff --git a/spindle/mill/proto/spindle/mill/v1/mill.proto b/spindle/mill/proto/spindle/mill/v1/mill.proto new file mode 100644 index 00000000..df71b493 --- /dev/null +++ b/spindle/mill/proto/spindle/mill/v1/mill.proto @@ -0,0 +1,187 @@ +syntax = "proto3"; + +package spindle.mill.v1; + +import "buf/validate/validate.proto"; + +option go_package = "tangled.org/core/spindle/mill/proto/gen;millv1"; + +// executor identity, sent on connect +message Hello { + uint32 protocol_version = 1; + // GOARCH of the node, informational only + string arch = 2; + // operator-defined labels, matched against runs_on + repeated string labels = 3; + string epoch = 4 [(buf.validate.field).string.min_len = 1]; +} + +// reconnect state for an existing epoch +message Resume { + string epoch = 1 [(buf.validate.field).string.min_len = 1]; + uint64 ack_seqno = 2; +} + +// per-engine state of a node +message EngineAvailability { + bool available = 1; + // opaque engine-defined load metrics, higher means more loaded + map load = 2; +} + +// full node state +message NodeSnapshot { + uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; + map engines = 2; + // every lease the executor currently holds, reserved or running + repeated string active_lease_ids = 3; +} + +// a seat reservation, carries no secrets +message ReserveSeat { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; + string target_engine = 2 [(buf.validate.field).string.min_len = 1]; + string raw_pipeline_json = 3; + string raw_workflow_json = 4; + // pipeline id, the executor reconstructs the exact WorkflowId from it + string knot = 5; + string rkey = 6; + uint32 ttl_seconds = 7; +} + +enum RejectClass { + REJECT_CLASS_UNSPECIFIED = 0; + REJECT_CLASS_TRANSIENT = 1; + REJECT_CLASS_INCOMPATIBLE = 2; +} + +message ReserveResult { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; + bool accepted = 2; + string reject_reason = 3; + RejectClass reject_class = 4 [(buf.validate.field).enum.defined_only = true]; +} + +// a single unlocked secret +message Secret { + string key = 1; + string value = 2; +} + +// promotes a reservation to a running job and hands over the secrets +message CommitLease { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; + repeated Secret secrets = 2; +} + +message Committed { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; +} + +// drops a reservation that was never committed +message ReleaseLease { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; +} + +// cancels a running attempt +message CancelAttempt { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; + string reason = 2; +} + +message CancelAck { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; +} + +enum NonterminalStatus { + NONTERMINAL_STATUS_UNSPECIFIED = 0; + RUNNING = 1; +} + +enum TerminalStatus { + TERMINAL_STATUS_UNSPECIFIED = 0; + SUCCESS = 1; + FAILED = 2; + TIMEOUT = 3; + CANCELLED = 4; +} + +message StatusEvent { + NonterminalStatus status = 1 [(buf.validate.field).enum = { + defined_only: true + not_in: 0 + }]; + string error = 2 [(buf.validate.field).string.max_bytes = 65536]; + int64 exit_code = 3; +} + +message LogArtifact { + string ref = 1 [(buf.validate.field).string.min_len = 1]; + string hash = 2 [(buf.validate.field).string.min_len = 1]; +} + +// terminal outcome of an attempt +message AttemptResult { + TerminalStatus status = 1 [(buf.validate.field).enum = { + defined_only: true + not_in: 0 + }]; + string error = 2 [(buf.validate.field).string.max_bytes = 65536]; + int64 exit_code = 3; + LogArtifact log_artifact = 4; +} + +// live non-replay log frame +message LiveLog { + string lease_id = 1 [(buf.validate.field).string.min_len = 1]; + bytes raw_json = 2 [(buf.validate.field).bytes.min_len = 1]; +} + +// one event in the executor's stream back to the mill +message Event { + uint64 seqno = 1 [(buf.validate.field).uint64.gt = 0]; + string lease_id = 2 [(buf.validate.field).string.min_len = 1]; + + oneof payload { + option (buf.validate.oneof).required = true; + StatusEvent status_event = 3; + AttemptResult attempt_result = 4; + } +} + +// a flushed bundle of events +message EventBatch { + string epoch = 1 [(buf.validate.field).string.min_len = 1]; + repeated Event events = 2 [(buf.validate.field).repeated.min_items = 1]; +} + +// all events up to and including up_to_seqno are durably processed +message Ack { + string epoch = 1 [(buf.validate.field).string.min_len = 1]; + uint64 up_to_seqno = 2; +} + +message Message { + option (buf.validate.message).oneof = { + fields: [ + "hello", "resume", "node_snapshot", "reserve_seat", "reserve_result", + "commit_lease", "committed", "release_lease", "cancel_attempt", + "cancel_ack", "event_batch", "ack", "live_log" + ], + required: true + }; + + Hello hello = 1; + Resume resume = 2; + NodeSnapshot node_snapshot = 3; + ReserveSeat reserve_seat = 4; + ReserveResult reserve_result = 5; + CommitLease commit_lease = 6; + Committed committed = 7; + ReleaseLease release_lease = 8; + CancelAttempt cancel_attempt = 9; + CancelAck cancel_ack = 10; + EventBatch event_batch = 11; + Ack ack = 12; + LiveLog live_log = 13; +} diff --git a/spindle/mill/proto/ws.go b/spindle/mill/proto/ws.go new file mode 100644 index 00000000..7d92e851 --- /dev/null +++ b/spindle/mill/proto/ws.go @@ -0,0 +1,58 @@ +package millproto + +import ( + "io" + "sync" + + "github.com/gorilla/websocket" +) + +// adapts a gorilla websocket connection to an io.ReadWriteCloser so the +// length-prefixed fleet framing rides over it. each Encode produces exactly one +// binary frame. the reader reassembles the byte stream across frames +type WSStream struct { + conn *websocket.Conn + + rmu sync.Mutex + r io.Reader // current message reader, advanced as frames are consumed + + wmu sync.Mutex +} + +func NewWSStream(conn *websocket.Conn) *WSStream { + return &WSStream{conn: conn} +} + +func (s *WSStream) Read(p []byte) (int, error) { + s.rmu.Lock() + defer s.rmu.Unlock() + for { + if s.r == nil { + _, r, err := s.conn.NextReader() + if err != nil { + return 0, err + } + s.r = r + } + n, err := s.r.Read(p) + if err == io.EOF { + s.r = nil + if n > 0 { + return n, nil + } + continue + } + return n, err + } +} + +func (s *WSStream) Write(p []byte) (int, error) { + s.wmu.Lock() + defer s.wmu.Unlock() + if err := s.conn.WriteMessage(websocket.BinaryMessage, p); err != nil { + return 0, err + } + return len(p), nil +} + +func (s *WSStream) Close() error { return s.conn.Close() } diff --git a/spindle/mill/restore.go b/spindle/mill/restore.go new file mode 100644 index 00000000..de77c943 --- /dev/null +++ b/spindle/mill/restore.go @@ -0,0 +1,221 @@ +package mill + +import ( + "fmt" + "tangled.org/core/spindle/db" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" + "time" +) + +const ( + leaseRowReserved = "reserved" + leaseRowRunning = "running" +) + +func (m *Mill) persistLease(lease *RemoteLease, state string) error { + if m.db == nil { + return nil + } + return m.db.SaveMillLease(db.MillLease{ + LeaseID: lease.id, + NodeID: lease.nodeID, + Epoch: lease.epoch, + Engine: lease.engine, + Knot: lease.wid.Knot, + Rkey: lease.wid.Rkey, + Workflow: lease.wid.Name, + State: state, + }) +} + +func (m *Mill) RestoreState() error { + if m.db == nil { + return nil + } + cursors, err := m.db.ListExecutorCursors() + if err != nil { + return err + } + rows, err := m.db.ListMillLeases() + if err != nil { + return err + } + + m.mu.Lock() + for _, c := range cursors { + m.nodeSeqno[c.NodeID+"/"+c.Epoch] = c.AckedSeqno + } + for _, r := range rows { + lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine) + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: r.Knot, Rkey: r.Rkey}, + Name: r.Workflow, + } + // restored leases start as orphans, an executor must reclaim it via + // its first snapshot, or the sweep will fail it + lease.orphaned = true + lease.claimed = false + if r.State == leaseRowRunning { + lease.state = leaseRunning + } + m.leases[r.LeaseID] = lease + } + restored := len(rows) + m.mu.Unlock() + + if restored > 0 { + m.l.Info("restored mill leases from previous run", "leases", restored, "cursors", len(cursors)) + // executors get a grace window to reconnect and claim their leases + time.AfterFunc(m.cfg.ReconnectGrace, m.sweepUnclaimedOrphans) + } + return nil +} +func (m *Mill) sweepUnclaimedOrphans() { + m.mu.Lock() + var unclaimed []*RemoteLease + for _, lease := range m.leases { + if !lease.orphaned { + continue + } + if lease.claimed { + continue + } + if lease.getState() == leaseDone { + continue + } + if sess := m.sessions[lease.nodeID]; sess == nil || sess.disconnected { + unclaimed = append(unclaimed, lease) + } + } + m.mu.Unlock() + + retry := false + reason := "executor did not reconnect after mill restart" + for _, lease := range unclaimed { + m.l.Warn("failing unclaimed restored lease", "lease", lease.id, "node", lease.nodeID) + if err := m.finishOrphan(lease, string(models.StatusKindFailed), &reason, nil); err != nil { + m.l.Error("finish unclaimed restored lease", "lease", lease.id, "err", err) + retry = true + } + } + if retry { + time.AfterFunc(5*time.Second, m.sweepUnclaimedOrphans) + } +} + +func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error { + active := make(map[string]struct{}, len(activeLeaseIDs)) + for _, id := range activeLeaseIDs { + active[id] = struct{}{} + } + + known := make(map[string]struct{}) + m.mu.Lock() + var gone []*RemoteLease + for _, lease := range m.leases { + if lease.nodeID == sess.nodeID { + known[lease.id] = struct{}{} + if lease.epoch != sess.epoch { + gone = append(gone, lease) + } else if _, ok := active[lease.id]; !ok { + gone = append(gone, lease) + } + } + } + for _, lease := range m.reservations { + if lease.nodeID == sess.nodeID && lease.epoch == sess.epoch { + known[lease.id] = struct{}{} + } + } + m.mu.Unlock() + var unknown []string + for id := range active { + if _, ok := known[id]; !ok { + unknown = append(unknown, id) + } + } + + for _, lease := range gone { + status := string(models.StatusKindFailed) + reason := "executor no longer holds lease" + if cancelled, cancelReason := lease.cancelRequested(); cancelled { + status = string(models.StatusKindCancelled) + reason = cancelReason + } + m.l.Warn("finishing reconciled lease", "lease", lease.id, "node", sess.nodeID, "leaseInc", lease.epoch, "sessInc", sess.epoch) + if lease.orphaned { + if err := m.finishOrphan(lease, status, &reason, nil); err != nil { + return err + } + } else if err := m.finishLiveLease(lease, status, reason); err != nil { + return err + } + } + for _, id := range unknown { + if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id, Reason: "lease is not owned by this mill"}}); err != nil { + return fmt.Errorf("cancel unknown executor lease %q: %w", id, err) + } + } + return nil +} + +func (m *Mill) completeLeaseRow(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { + if m.db == nil { + return nil + } + return m.db.CompleteMillLease( + lease.id, + string(lease.wid.PipelineId.AtUri()), + lease.wid.Name, + status, + errMsg, + exitCode, + m.n, + ) +} + +func (m *Mill) finishLiveLease(lease *RemoteLease, status, reason string) error { + lease.finishMu.Lock() + defer lease.finishMu.Unlock() + if lease.getState() == leaseDone { + return nil + } + if err := m.completeLeaseRow(lease, status, &reason, nil); err != nil { + return err + } + lease.deliverTerminal(&millv1.AttemptResult{ + Status: mapTerminalStatusString(status), + Error: reason, + }) + return m.cleanupLeaseLocked(lease) +} + +func (m *Mill) finishOrphan(lease *RemoteLease, status string, errMsg *string, exitCode *int64) error { + lease.finishMu.Lock() + defer lease.finishMu.Unlock() + if lease.getState() == leaseDone { + return nil + } + if err := m.completeLeaseRow(lease, status, errMsg, exitCode); err != nil { + return err + } + lease.markDone() + return m.cleanupLeaseLocked(lease) +} + +func mapTerminalStatusString(s string) millv1.TerminalStatus { + switch s { + case "success": + return millv1.TerminalStatus_SUCCESS + case "failed": + return millv1.TerminalStatus_FAILED + case "timeout": + return millv1.TerminalStatus_TIMEOUT + case "cancelled": + return millv1.TerminalStatus_CANCELLED + default: + return millv1.TerminalStatus_TERMINAL_STATUS_UNSPECIFIED + } +} diff --git a/spindle/mill/restore_test.go b/spindle/mill/restore_test.go new file mode 100644 index 00000000..4694173c --- /dev/null +++ b/spindle/mill/restore_test.go @@ -0,0 +1,386 @@ +package mill + +import ( + "context" + "path/filepath" + "testing" + "time" + + "tangled.org/core/notifier" + "tangled.org/core/spindle/db" + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func restoreTestMill(t *testing.T, cfg Config) (*Mill, *db.DB) { + t.Helper() + bdb, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "mill.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { bdb.Close() }) + n := notifier.New() + m := New(discardLogger(), cfg) + m.Attach(bdb, &n) + return m, bdb +} + +func restoredMill(t *testing.T, bdb *db.DB, cfg Config) *Mill { + t.Helper() + n := notifier.New() + m := New(discardLogger(), cfg) + m.Attach(bdb, &n) + if err := m.RestoreState(); err != nil { + t.Fatalf("RestoreState: %v", err) + } + return m +} + +func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) { + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + + if err := bdb.SaveMillLease(db.MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + }); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 7) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + + m2 := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) + + m2.mu.Lock() + lease := m2.leases["lease-1"] + seqno := m2.nodeSeqno["node-1/inc-1"] + m2.mu.Unlock() + + if lease == nil { + t.Fatal("restored mill has no lease-1") + } + if !lease.orphaned { + t.Fatal("restored lease is not orphaned; a terminal would be delivered to a waiter that does not exist") + } + if lease.getState() != leaseRunning { + t.Fatalf("restored lease state = %v, want leaseRunning", lease.getState()) + } + wantWid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} + if lease.wid != wantWid { + t.Fatalf("restored lease wid = %+v, want %+v", lease.wid, wantWid) + } + if seqno != 7 { + t.Fatalf("restored cursor = %d, want 7", seqno) + } +} + +func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + if err := bdb.SaveMillLease(db.MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + }); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + resume, ok := m.attachSession(sess) + if !ok { + t.Fatal("attachSession rejected the reconnecting executor") + } + if resume != 3 { + t.Fatalf("attachSession resume seqno = %d, want restored cursor 3", resume) + } + + _ = m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 4, + LeaseId: "lease-1", + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + }) + + wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} + st, err := bdb.GetStatus(wid) + if err != nil { + t.Fatalf("GetStatus after orphan terminal: %v", err) + } + if st.Status != string(models.StatusKindSuccess) { + t.Fatalf("orphan terminal authored status %q, want success", st.Status) + } + + m.mu.Lock() + _, still := m.leases["lease-1"] + m.mu.Unlock() + if still { + t.Fatal("finished orphan still in the lease map") + } + if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { + t.Fatalf("finished orphan still persisted: %+v", rows) + } +} + +func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) { + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + for _, l := range []db.MillLease{ + {LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowRunning}, + {LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r2", Workflow: "w", State: leaseRowRunning}, + } { + if err := bdb.SaveMillLease(l); err != nil { + t.Fatalf("SaveMillLease(%s): %v", l.LeaseID, err) + } + } + + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 1, + ActiveLeaseIds: []string{"lease-kept"}, + }) + + m.mu.Lock() + _, kept := m.leases["lease-kept"] + _, gone := m.leases["lease-gone"] + m.mu.Unlock() + if !kept { + t.Fatal("reconciliation dropped a lease the executor still holds") + } + if gone { + t.Fatal("reconciliation kept a lease the executor no longer holds") + } + + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r2"}, Name: "w"}) + if err != nil { + t.Fatalf("GetStatus for dropped orphan: %v", err) + } + if st.Status != string(models.StatusKindFailed) { + t.Fatalf("dropped orphan authored status %q, want failed", st.Status) + } +} +func TestSnapshotReconciliationPreservesRequestedCancellation(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + lease := newLease("lease-1", "node-1", "inc-old", "dummy") + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, + Name: "w", + } + lease.setState(leaseRunning) + lease.requestCancel("workflow destroyed") + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatalf("persistLease: %v", err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + sess := newSession("node-1", "inc-new", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + if err := m.onSnapshot(sess, &millv1.NodeSnapshot{Seqno: 1}); err != nil { + t.Fatalf("onSnapshot: %v", err) + } + + st, err := bdb.GetStatus(lease.wid) + if err != nil { + t.Fatalf("GetStatus: %v", err) + } + if st.Status != string(models.StatusKindCancelled) { + t.Fatalf("reconciled status = %q, want cancelled", st.Status) + } +} + +func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) { + m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cancelled := make(chan string, 1) + sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { + if cancel := msg.GetCancelAttempt(); cancel != nil { + cancelled <- cancel.GetLeaseId() + } + return nil + }), discardLogger()) + m.attachSession(sess) + + if err := m.onSnapshot(sess, &millv1.NodeSnapshot{ + Seqno: 1, + ActiveLeaseIds: []string{"executor-only"}, + }); err != nil { + t.Fatalf("onSnapshot: %v", err) + } + select { + case id := <-cancelled: + if id != "executor-only" { + t.Fatalf("cancelled lease = %q, want executor-only", id) + } + default: + t.Fatal("snapshot reconciliation left an executor-only lease running") + } +} + +func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + if err := bdb.SaveMillLease(db.MillLease{ + LeaseID: "lease-1", NodeID: "node-absent", Epoch: "inc-absent", Engine: "dummy", + Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowReserved, + }); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) + m.sweepUnclaimedOrphans() + + m.mu.Lock() + _, still := m.leases["lease-1"] + m.mu.Unlock() + if still { + t.Fatal("sweep kept an orphan whose executor never reconnected") + } + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, Name: "w"}) + if err != nil { + t.Fatalf("GetStatus after sweep: %v", err) + } + if st.Status != string(models.StatusKindFailed) { + t.Fatalf("sweep authored status %q, want failed", st.Status) + } + if rows, _ := bdb.ListMillLeases(); len(rows) != 0 { + t.Fatalf("swept orphan still persisted: %+v", rows) + } +} + +func TestAckSeqnoPersistsCursor(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, + LeaseId: owned.id, + Payload: &millv1.Event_StatusEvent{ + StatusEvent: &millv1.StatusEvent{ + Status: millv1.NonterminalStatus_RUNNING, + }, + }, + }, + }, + }) + if err != nil { + t.Fatalf("onEventBatch: %v", err) + } + + cursors, err := bdb.ListExecutorCursors() + if err != nil { + t.Fatalf("ListExecutorCursors: %v", err) + } + if len(cursors) != 1 || cursors[0].AckedSeqno != 1 { + t.Fatalf("persisted cursor = %+v, want seqno 1", cursors) + } +} + +func TestOrphanTerminalFailureKeepsLeaseAndSeqnoRetryable(t *testing.T) { + _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + if err := bdb.SaveMillLease(db.MillLease{ + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", + Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + }); err != nil { + t.Fatalf("SaveMillLease: %v", err) + } + if err := bdb.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.AdvanceCursor("node-1", "inc-1", 3) }); err != nil { + t.Fatalf("AdvanceCursor: %v", err) + } + if _, err := bdb.Exec(` + create trigger reject_orphan_lease_delete + before delete on mill_leases + begin + select raise(abort, 'forced delete failure'); + end + `); err != nil { + t.Fatalf("create failure trigger: %v", err) + } + + m := restoredMill(t, bdb, Config{ReconnectGrace: time.Minute}) + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + if _, ok := m.attachSession(sess); !ok { + t.Fatal("attachSession rejected reconnect") + } + batch := &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 4, + LeaseId: "lease-1", + Payload: &millv1.Event_AttemptResult{ + AttemptResult: &millv1.AttemptResult{ + Status: millv1.TerminalStatus_SUCCESS, + }, + }, + }, + }, + } + if err := m.onEventBatch(sess, batch); err == nil { + t.Fatal("orphan terminal stream succeeded despite forced transaction failure") + } + + m.mu.Lock() + lease := m.leases["lease-1"] + seqno := m.nodeSeqno["node-1/inc-1"] + m.mu.Unlock() + if lease == nil || lease.getState() == leaseDone { + t.Fatal("failed orphan completion made the in-memory lease unretryable") + } + if seqno != 3 { + t.Fatalf("in-memory stream seqno = %d, want 3", seqno) + } + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 1 { + t.Fatalf("durable leases after transaction rollback = %+v, err = %v; want retained lease", rows, err) + } + var events int + if err := bdb.QueryRow(`select count(*) from events`).Scan(&events); err != nil { + t.Fatalf("count events: %v", err) + } + if events != 0 { + t.Fatalf("terminal events after transaction rollback = %d, want 0", events) + } + + if _, err := bdb.Exec(`drop trigger reject_orphan_lease_delete`); err != nil { + t.Fatalf("drop failure trigger: %v", err) + } + if err := m.onEventBatch(sess, batch); err != nil { + t.Fatalf("retry orphan terminal: %v", err) + } + m.mu.Lock() + _, still := m.leases["lease-1"] + seqno = m.nodeSeqno["node-1/inc-1"] + m.mu.Unlock() + if still { + t.Fatal("successful orphan completion retained in-memory lease") + } + if seqno != 4 { + t.Fatalf("in-memory stream seqno after retry = %d, want 4", seqno) + } + if rows, err := bdb.ListMillLeases(); err != nil || len(rows) != 0 { + t.Fatalf("durable leases after successful retry = %+v, err = %v; want none", rows, err) + } +} diff --git a/spindle/mill/session.go b/spindle/mill/session.go new file mode 100644 index 00000000..4a942ea1 --- /dev/null +++ b/spindle/mill/session.go @@ -0,0 +1,160 @@ +package mill + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + millproto "tangled.org/core/spindle/mill/proto" + millv1 "tangled.org/core/spindle/mill/proto/gen" +) + +var errSessionClosed = errors.New("mill: executor session closed") + +// one live websocket to an executor. many leases and async streams share +// it, so one reader goroutine demuxes by message type and correlates by +// lease ID. never hold locks across decodes +type millSession struct { + nodeID string + epoch string + labels []string + enc messageEncoder + l *slog.Logger + closeTransport func() error + + // snapshot, disconnected and graceTimer are guarded by Mill.mu, the + // fleet ranks across sessions under its own lock + snapshot *millv1.NodeSnapshot + disconnected bool + graceTimer *time.Timer + lastSeen time.Time + + mu sync.Mutex + pending map[string]chan *millproto.Message // maps lease ID to response waiter + + closeOnce sync.Once + closed chan struct{} +} + +type messageEncoder interface { + Encode(*millproto.Message) error +} + +func newSession(nodeID string, epoch string, labels []string, enc messageEncoder, l *slog.Logger) *millSession { + return &millSession{ + nodeID: nodeID, + epoch: epoch, + labels: labels, + enc: enc, + l: l, + pending: make(map[string]chan *millproto.Message), + closed: make(chan struct{}), + lastSeen: time.Now(), + } +} + +// caller holds Mill.mu +func (s *millSession) live(grace time.Duration) bool { + return !s.disconnected && time.Since(s.lastSeen) <= grace +} + +func (s *millSession) send(msg *millproto.Message) error { + return s.enc.Encode(msg) +} + +func (s *millSession) close() { + s.closeOnce.Do(func() { + close(s.closed) + if s.closeTransport != nil { + _ = s.closeTransport() + } + }) +} + +// one-shot waiter for the next response on the lease, cancel unregisters it +func (s *millSession) await(leaseID string) (<-chan *millproto.Message, func()) { + ch := make(chan *millproto.Message, 1) + s.mu.Lock() + s.pending[leaseID] = ch + s.mu.Unlock() + return ch, func() { + s.mu.Lock() + if s.pending[leaseID] == ch { + delete(s.pending, leaseID) + } + s.mu.Unlock() + } +} + +func (s *millSession) deliver(leaseID string, msg *millproto.Message) { + s.mu.Lock() + ch := s.pending[leaseID] + delete(s.pending, leaseID) + s.mu.Unlock() + if ch != nil { + select { + case ch <- msg: + default: + } + } +} + +// sends a message and waits for its response, respecting ctx and session closure +func (s *millSession) request(ctx context.Context, leaseID string, msg *millproto.Message) (*millproto.Message, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ch, cancel := s.await(leaseID) + defer cancel() + + if err := s.send(msg); err != nil { + return nil, err + } + + select { + case resp := <-ch: + return resp, nil + case <-ctx.Done(): + return nil, ctx.Err() + case <-s.closed: + return nil, errSessionClosed + } +} + +// demuxes frames until the decoder errors (connection gone) +func (s *millSession) readLoop(m *Mill, dec *millproto.Decoder) error { + for { + msg, err := dec.Decode() + if err != nil { + return err + } + if err := s.dispatch(m, msg); err != nil { + return err + } + } +} + +func (s *millSession) dispatch(m *Mill, msg *millproto.Message) error { + if !m.touchSession(s) { + return errSessionClosed + } + switch { + case msg.GetNodeSnapshot() != nil: + return m.onSnapshot(s, msg.GetNodeSnapshot()) + case msg.GetReserveResult() != nil: + s.deliver(msg.GetReserveResult().GetLeaseId(), msg) + case msg.GetCommitted() != nil: + s.deliver(msg.GetCommitted().GetLeaseId(), msg) + case msg.GetEventBatch() != nil: + return m.onEventBatch(s, msg.GetEventBatch()) + case msg.GetCancelAck() != nil: + m.onCancelAck(s, msg.GetCancelAck()) + case msg.GetLiveLog() != nil: + return m.onLiveLog(s, msg.GetLiveLog()) + default: + s.l.Warn("session received unexpected message", "node", s.nodeID) + } + return nil +} diff --git a/spindle/mill/token.go b/spindle/mill/token.go new file mode 100644 index 00000000..2fc336bd --- /dev/null +++ b/spindle/mill/token.go @@ -0,0 +1,23 @@ +package mill + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" +) + +func GenerateToken() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b[:]), nil +} + +// hashes the token for persistence and comparison. a database leak never +// exposes a usable token, and lookup avoids a secret-dependent comparison +func HashToken(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} diff --git a/spindle/server.go b/spindle/server.go index a1083db8..a8ea56c6 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -39,6 +39,8 @@ import ( "tangled.org/core/spindle/engines/dummy" "tangled.org/core/spindle/engines/nixery" "tangled.org/core/spindle/git" + "tangled.org/core/spindle/mill" + "tangled.org/core/spindle/mill/executor" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" "tangled.org/core/spindle/xrpc" @@ -63,6 +65,7 @@ type Spindle struct { l *slog.Logger n *notifier.Notifier engs map[string]models.Engine + jobWake chan struct{} cfg *config.Config ks *eventconsumer.Consumer res *idresolver.Resolver @@ -71,30 +74,88 @@ type Spindle struct { motd []byte motdMu sync.RWMutex rootCtx context.Context - jobWake chan struct{} + store artifactstore.Store stores *artifactstore.Stores reader artifactstore.Reader + // set only when this spindle hosts the mill or joins one as an executor + mill *mill.Mill + exec *executor.Executor } // New creates a new Spindle server with the provided configuration and engines. func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { logger := log.FromContext(ctx) + n := notifier.New() + + if cfg.Role == config.RoleExecutor { + if err := cleanupOrphanRepos(ctx, d, logger); err != nil { + return nil, fmt.Errorf("failed to run startup cleanup: %w", err) + } + } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { + return nil, fmt.Errorf("failed to run startup migrations: %w", err) + } + + spindle := &Spindle{ + db: d, + l: logger, + n: &n, + engs: engines, + cfg: cfg, + motd: defaultMotd, + rootCtx: ctx, + jobWake: make(chan struct{}, 1), + } + diskFallback := "" + if cfg.Role == config.RoleStandalone { + diskFallback = cfg.Server.LogDir + if cfg.ArtifactStores.Disk.Dir == "" { + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") + } + } + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) + if err != nil { + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) + } + spindle.stores = stores + if cfg.LegacyS3.LogBucket != "" { + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") + } + if cfg.Role == config.RoleStandalone { + spindle.reader = stores + } else { + name := cfg.Mill.ArtifactStore + if name == "" { + names := stores.Names() + if len(names) != 1 { + return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) + } + name = names[0] + logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) + } + store, ok := stores.Store(name) + if !ok { + return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) + } + spindle.store = store + spindle.reader = store + } + if cfg.Role == config.RoleExecutor { + return spindle, nil + } e, err := rbac.NewEnforcer(cfg.Server.DBPath) if err != nil { return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) } e.E.EnableAutoSave(true) + spindle.e = e - n := notifier.New() - - var vault secrets.Manager switch cfg.Server.Secrets.Provider { case "openbao": if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") } - vault, err = secrets.NewOpenBaoManager( + spindle.vault, err = secrets.NewOpenBaoManager( cfg.Server.Secrets.OpenBao.ProxyAddr, logger, secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), @@ -104,7 +165,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m } logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) case "sqlite", "": - vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) + spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) if err != nil { return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) } @@ -113,10 +174,6 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) } - if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { - return nil, fmt.Errorf("failed to run startup migrations: %w", err) - } - collections := []string{ tangled.SpindleMemberNSID, tangled.RepoNSID, @@ -127,6 +184,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m if err != nil { return nil, fmt.Errorf("failed to setup jetstream client: %w", err) } + spindle.jc = jc jc.AddDid(cfg.Server.Owner) // pull records are created by arbitrary users too, same hack as in tap jc.ExemptCollection(tangled.RepoPullNSID) @@ -150,36 +208,8 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m } } - resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) - - spindle := &Spindle{ - jc: jc, - e: e, - db: d, - l: logger, - n: &n, - engs: engines, - cfg: cfg, - res: resolver, - verify: repoverify.New(resolver, cfg.Server.Dev), - vault: vault, - motd: defaultMotd, - rootCtx: ctx, - jobWake: make(chan struct{}, 1), - } - diskFallback := cfg.Server.LogDir - if cfg.ArtifactStores.Disk.Dir == "" { - logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") - } - stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) - if err != nil { - return nil, fmt.Errorf("failed to setup artifact stores: %w", err) - } - spindle.stores = stores - spindle.reader = stores - if cfg.LegacyS3.LogBucket != "" { - logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") - } + spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) + spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) err = e.AddSpindle(rbacDomain) if err != nil { @@ -208,8 +238,6 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m ccfg.Logger = log.SubLogger(logger, "eventconsumer") ccfg.ProcessFunc = spindle.processKnotStream ccfg.CursorStore = cursorStore - ccfg.WorkerCount = 16 - ccfg.QueueSize = 200 if cfg.Server.Dev { ccfg.RetryInterval = 5 * time.Second ccfg.MaxRetryInterval = 10 * time.Second @@ -228,7 +256,6 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m ccfg.Sources[src] = struct{}{} } spindle.ks = eventconsumer.NewConsumer(*ccfg) - if cfg.Server.Tap.Embed { pw, err := randomAdminPassword() if err != nil { @@ -241,8 +268,6 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m return spindle, nil } - -// DB returns the database instance. func (s *Spindle) DB() *db.DB { return s.db } @@ -281,43 +306,50 @@ func (s *Spindle) GetMotdContent() []byte { return s.motd } -// Start starts the Spindle server (blocking). +// runs the server. blocks func (s *Spindle) Start(ctx context.Context) error { - // starts a job queue runner in the background + // only standalone runs the local queue. mill hosts place directly onto + // executors, and executors only run jobs explicitly assigned by a mill s.StartJobWorkers(ctx) - // Stop vault token renewal if it implements Stopper + // an executor dials out to its mill and takes work from it + if s.exec != nil { + go s.exec.Connect(ctx) + } + if stopper, ok := s.vault.(secrets.Stopper); ok { defer stopper.Stop() } - tapCtx, tapCancel := context.WithCancel(ctx) + if s.cfg.Role != config.RoleExecutor { + tapCtx, tapCancel := context.WithCancel(ctx) - if s.cfg.Server.Tap.Embed { - emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) - if err != nil { - tapCancel() - return fmt.Errorf("starting embedded tap: %w", err) + if s.cfg.Server.Tap.Embed { + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) + if err != nil { + tapCancel() + return fmt.Errorf("starting embedded tap: %w", err) + } + s.embedTap = emb + defer func() { + tapCancel() + s.embedTap.Shutdown() + }() + + go s.watchTapDrain(tapCtx, tapCancel) + } else { + defer tapCancel() } - s.embedTap = emb - defer func() { - tapCancel() - s.embedTap.Shutdown() + + go func() { + s.l.Info("starting knot event consumer") + s.ks.Start(ctx) }() - go s.watchTapDrain(tapCtx, tapCancel) - } else { - defer tapCancel() + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) + s.tap.Start(tapCtx) } - go func() { - s.l.Info("starting knot event consumer") - s.ks.Start(ctx) - }() - - s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) - s.tap.Start(tapCtx) - s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) } @@ -362,23 +394,60 @@ func Run(ctx context.Context) error { return fmt.Errorf("failed to setup db: %w", err) } - nixeryEng, err := nixery.New(ctx, cfg) - if err != nil { - return err + logger := log.FromContext(ctx) + + var engines map[string]models.Engine + var m *mill.Mill + + if cfg.Role == config.RoleMill { + // mill host: register engines that place jobs on executors instead of + // running them. all names share one Mill + m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ + LogDir: cfg.Server.LogDir, + MaxPending: cfg.Mill.MaxPending, + ReconnectGrace: cfg.Mill.ReconnectGrace, + }) + engines = map[string]models.Engine{ + "nixery": mill.NewEngine("nixery", m), + "microvm": mill.NewEngine("microvm", m), + "dummy": mill.NewEngine("dummy", m), + } + } else { + // standalone and executor both run real engines locally. + nixeryEng, err := nixery.New(ctx, cfg) + if err != nil { + return err + } + microvmEng, err := newMicrovmEngine(ctx, cfg, d) + if err != nil { + return err + } + engines = map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(logger), + } } - microvmEng, err := newMicrovmEngine(ctx, cfg, d) + s, err := New(ctx, cfg, d, engines) if err != nil { return err } - s, err := New(ctx, cfg, d, map[string]models.Engine{ - "nixery": nixeryEng, - "microvm": microvmEng, - "dummy": dummy.New(log.FromContext(ctx)), - }) - if err != nil { - return err + if m != nil { + // resolve the chicken-and-egg: the engines (built above) hold the mill, + // but the mill's db/notifier are created inside New + m.Attach(s.DB(), s.Notifier()) + s.mill = m + if err := m.RestoreState(); err != nil { + return fmt.Errorf("restoring mill state: %w", err) + } + } + if cfg.Role == config.RoleExecutor { + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) + if err != nil { + return err + } } return s.Start(ctx) @@ -390,9 +459,18 @@ func (s *Spindle) Router() http.Handler { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { w.Write(s.GetMotdContent()) }) + if s.cfg.Role == config.RoleExecutor { + return mux + } + mux.HandleFunc("/events", s.Events) mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) + // mill host: executors dial in here (plain ws, shared-secret auth) + if s.mill != nil { + mux.HandleFunc("/mill", s.mill.HandleExecutorConn) + } + mux.Mount("/xrpc", s.XrpcRouter()) return mux } @@ -452,6 +530,10 @@ func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Sourc // NOTE: we are blindly trusting the knot that it will return only repos it own repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) repoPath := s.newRepoPath(repoDid) + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { + return fmt.Errorf("sync git repo: %w", err) + } + l.Info("synced git repo") triggerRepo, err := s.buildTriggerRepo(ctx, repo) if err != nil { @@ -764,6 +846,65 @@ func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev strin return rawPipeline, nil } +// newRepoPath creates a path to store repository by its did and rkey. +// The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey +func (s *Spindle) newRepoPath(repo syntax.DID) string { + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) +} + +func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { + scheme := "https://" + if s.cfg.Server.Dev { + scheme = "http://" + } + return fmt.Sprintf("%s%s/%s", scheme, knot, did) +} + +const RequiredVersion = "2.49.0" + +func ensureGitVersion() error { + v, err := git.Version() + if err != nil { + return fmt.Errorf("fetching git version: %w", err) + } + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) + } + return nil +} + +func (s *Spindle) configureOwner() error { + cfgOwner := s.cfg.Server.Owner + + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) + if err != nil { + return err + } + + switch len(existing) { + case 0: + // no owner configured, continue + case 1: + // find existing owner + existingOwner := existing[0] + + // no ownership change, this is okay + if existingOwner == s.cfg.Server.Owner { + break + } + + // remove existing owner + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) + if err != nil { + return nil + } + default: + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) + } + + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) +} + func (s *Spindle) StartJobWorkers(ctx context.Context) { for range s.cfg.Server.MaxJobCount { go func() { @@ -872,76 +1013,3 @@ func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipe } return nil } - -// newRepoPath creates a path to store repository by its did and rkey. -// The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey -func (s *Spindle) newRepoPath(repo syntax.DID) string { - return filepath.Join(s.cfg.Server.RepoDir, repo.String()) -} - -func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { - scheme := "https://" - if s.cfg.Server.Dev { - scheme = "http://" - } - return fmt.Sprintf("%s%s/%s", scheme, knot, did) -} - -const RequiredVersion = "2.49.0" - -func ensureGitVersion() error { - v, err := git.Version() - if err != nil { - return fmt.Errorf("fetching git version: %w", err) - } - if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { - return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) - } - return nil -} - -func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { - if repo.RepoDid == nil || *repo.RepoDid == "" { - return "", fmt.Errorf("pipeline trigger missing repoDid") - } - repoDid, err := syntax.ParseDID(*repo.RepoDid) - if err != nil { - return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) - } - if _, err := s.db.GetRepoByDid(repoDid); err != nil { - return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) - } - return repoDid, nil -} - -func (s *Spindle) configureOwner() error { - cfgOwner := s.cfg.Server.Owner - - existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) - if err != nil { - return err - } - - switch len(existing) { - case 0: - // no owner configured, continue - case 1: - // find existing owner - existingOwner := existing[0] - - // no ownership change, this is okay - if existingOwner == s.cfg.Server.Owner { - break - } - - // remove existing owner - err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) - if err != nil { - return nil - } - default: - return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) - } - - return s.e.AddSpindleOwner(rbacDomain, cfgOwner) -} diff --git a/spindle/server.go.master b/spindle/server.go.master new file mode 100644 index 00000000..7de46342 --- /dev/null +++ b/spindle/server.go.master @@ -0,0 +1,930 @@ +package spindle + +import ( + "context" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "path/filepath" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/hashicorp/go-version" + "tangled.org/core/api/tangled" + "tangled.org/core/eventconsumer" + "tangled.org/core/eventconsumer/cursor" + "tangled.org/core/eventstream" + "tangled.org/core/idresolver" + "tangled.org/core/jetstream" + knotdb "tangled.org/core/knotserver/db" + kgit "tangled.org/core/knotserver/git" + "tangled.org/core/log" + "tangled.org/core/notifier" + "tangled.org/core/rbac" + "tangled.org/core/repoident" + "tangled.org/core/repoverify" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/engines/nixery" + "tangled.org/core/spindle/git" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/xrpc" + "tangled.org/core/tid" + "tangled.org/core/workflow" + "tangled.org/core/xrpc/serviceauth" +) + +//go:embed motd +var defaultMotd []byte + +const ( + rbacDomain = "thisserver" +) + +type Spindle struct { + jc *jetstream.JetstreamClient + tap *Tap + embedTap *embeddedTap + db *db.DB + e *rbac.Enforcer + l *slog.Logger + n *notifier.Notifier + engs map[string]models.Engine + cfg *config.Config + ks *eventconsumer.Consumer + res *idresolver.Resolver + verify repoverify.Verifier + vault secrets.Manager + motd []byte + motdMu sync.RWMutex + rootCtx context.Context + jobWake chan struct{} +} + +// New creates a new Spindle server with the provided configuration and engines. +func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { + logger := log.FromContext(ctx) + + e, err := rbac.NewEnforcer(cfg.Server.DBPath) + if err != nil { + return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) + } + e.E.EnableAutoSave(true) + + n := notifier.New() + + var vault secrets.Manager + switch cfg.Server.Secrets.Provider { + case "openbao": + if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { + return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") + } + vault, err = secrets.NewOpenBaoManager( + cfg.Server.Secrets.OpenBao.ProxyAddr, + logger, + secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), + ) + if err != nil { + return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) + } + logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) + case "sqlite", "": + vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) + if err != nil { + return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) + } + logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) + default: + return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) + } + + if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { + return nil, fmt.Errorf("failed to run startup migrations: %w", err) + } + + collections := []string{ + tangled.SpindleMemberNSID, + tangled.RepoNSID, + tangled.RepoCollaboratorNSID, + tangled.RepoPullNSID, + } + jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) + if err != nil { + return nil, fmt.Errorf("failed to setup jetstream client: %w", err) + } + jc.AddDid(cfg.Server.Owner) + // pull records are created by arbitrary users too, same hack as in tap + jc.ExemptCollection(tangled.RepoPullNSID) + + // Check if the spindle knows about any Dids; + dids, err := d.GetAllDids() + if err != nil { + return nil, fmt.Errorf("failed to get all dids: %w", err) + } + for _, d := range dids { + jc.AddDid(d) + } + + knownRepos, err := d.AllRepos() + if err != nil { + return nil, fmt.Errorf("failed to get known repos: %w", err) + } + for _, r := range knownRepos { + if r.Owner != "" { + jc.AddDid(r.Owner.String()) + } + } + + resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) + + spindle := &Spindle{ + jc: jc, + e: e, + db: d, + l: logger, + n: &n, + engs: engines, + cfg: cfg, + res: resolver, + verify: repoverify.New(resolver, cfg.Server.Dev), + vault: vault, + motd: defaultMotd, + rootCtx: ctx, + jobWake: make(chan struct{}, 1), + } + + err = e.AddSpindle(rbacDomain) + if err != nil { + return nil, fmt.Errorf("failed to set rbac domain: %w", err) + } + err = spindle.configureOwner() + if err != nil { + return nil, err + } + logger.Info("owner set", "did", cfg.Server.Owner) + + cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) + if err != nil { + return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) + } + + err = jc.StartJetstream(ctx, spindle.ingest()) + if err != nil { + return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) + } + + // spindle listen to knot stream for sh.tangled.git.refUpdate + // which will sync the local workflow files in spindle and enqueues the + // pipeline job for on-push workflows + ccfg := eventconsumer.NewConsumerConfig() + ccfg.Logger = log.SubLogger(logger, "eventconsumer") + ccfg.ProcessFunc = spindle.processKnotStream + ccfg.CursorStore = cursorStore + ccfg.WorkerCount = 16 + ccfg.QueueSize = 200 + if cfg.Server.Dev { + ccfg.RetryInterval = 5 * time.Second + ccfg.MaxRetryInterval = 10 * time.Second + } else { + ccfg.RetryInterval = 1 * time.Minute + ccfg.MaxRetryInterval = 10 * time.Minute + } + knownKnots, err := d.Knots() + if err != nil { + return nil, err + } + for _, knot := range knownKnots { + logger.Info("adding source start", "knot", knot) + src := eventconsumer.NewKnotSource(knot) + eventconsumer.MigrateLegacyCursor(cursorStore, src) + ccfg.Sources[src] = struct{}{} + } + spindle.ks = eventconsumer.NewConsumer(*ccfg) + + if cfg.Server.Tap.Embed { + pw, err := randomAdminPassword() + if err != nil { + return nil, err + } + cfg.Server.Tap.AdminPassword = pw + logger.Info("embedded tap: using random admin password") + } + spindle.tap = NewTapClient(spindle) + + return spindle, nil +} + +// DB returns the database instance. +func (s *Spindle) DB() *db.DB { + return s.db +} + +// Engines returns the map of available engines. +func (s *Spindle) Engines() map[string]models.Engine { + return s.engs +} + +// Vault returns the secrets manager instance. +func (s *Spindle) Vault() secrets.Manager { + return s.vault +} + +// Notifier returns the notifier instance. +func (s *Spindle) Notifier() *notifier.Notifier { + return s.n +} + +// Enforcer returns the RBAC enforcer instance. +func (s *Spindle) Enforcer() *rbac.Enforcer { + return s.e +} + +// SetMotdContent sets custom MOTD content, replacing the embedded default. +func (s *Spindle) SetMotdContent(content []byte) { + s.motdMu.Lock() + defer s.motdMu.Unlock() + s.motd = content +} + +// GetMotdContent returns the current MOTD content. +func (s *Spindle) GetMotdContent() []byte { + s.motdMu.RLock() + defer s.motdMu.RUnlock() + return s.motd +} + +// Start starts the Spindle server (blocking). +func (s *Spindle) Start(ctx context.Context) error { + // starts a job queue runner in the background + s.StartJobWorkers(ctx) + + // Stop vault token renewal if it implements Stopper + if stopper, ok := s.vault.(secrets.Stopper); ok { + defer stopper.Stop() + } + + tapCtx, tapCancel := context.WithCancel(ctx) + + if s.cfg.Server.Tap.Embed { + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) + if err != nil { + tapCancel() + return fmt.Errorf("starting embedded tap: %w", err) + } + s.embedTap = emb + defer func() { + tapCancel() + s.embedTap.Shutdown() + }() + + go s.watchTapDrain(tapCtx, tapCancel) + } else { + defer tapCancel() + } + + go func() { + s.l.Info("starting knot event consumer") + s.ks.Start(ctx) + }() + + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) + s.tap.Start(tapCtx) + + s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) + return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) +} + +func (s *Spindle) declareTapInterest(ctx context.Context) { + repos, err := s.db.AllRepos() + if err != nil { + s.l.Warn("tap declare: failed to load known repos", "err", err) + return + } + seen := make(map[syntax.DID]struct{}, len(repos)) + dids := make([]syntax.DID, 0, len(repos)) + for _, r := range repos { + if r.Owner == "" { + continue + } + if _, ok := seen[r.Owner]; ok { + continue + } + seen[r.Owner] = struct{}{} + dids = append(dids, r.Owner) + } + if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { + s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) + return + } + s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) +} + +func Run(ctx context.Context) error { + cfg, err := config.Load(ctx) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + if err := ensureGitVersion(); err != nil { + return fmt.Errorf("ensuring git version: %w", err) + } + + d, err := db.Make(ctx, cfg.Server.DBPath) + if err != nil { + return fmt.Errorf("failed to setup db: %w", err) + } + + nixeryEng, err := nixery.New(ctx, cfg) + if err != nil { + return err + } + + microvmEng, err := newMicrovmEngine(ctx, cfg, d) + if err != nil { + return err + } + + s, err := New(ctx, cfg, d, map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(log.FromContext(ctx)), + }) + if err != nil { + return err + } + + return s.Start(ctx) +} + +func (s *Spindle) Router() http.Handler { + mux := chi.NewRouter() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write(s.GetMotdContent()) + }) + mux.HandleFunc("/events", s.Events) + mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) + + mux.Mount("/xrpc", s.XrpcRouter()) + return mux +} + +func (s *Spindle) XrpcRouter() http.Handler { + serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) + + l := log.SubLogger(s.l, "xrpc") + + x := xrpc.Xrpc{ + Logger: l, + Db: s.db, + Enforcer: s.e, + Engines: s.engs, + Config: s.cfg, + Resolver: s.res, + Vault: s.vault, + Notifier: s.Notifier(), + ServiceAuth: serviceAuth, + Trigger: s, + } + + return x.Router() +} + +func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { + l := log.FromContext(ctx).With("handler", "processKnotStream") + l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) + if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { + return s.ingestKnotCollaborator(ctx, l, src, msg) + } + if msg.Nsid == tangled.GitRefUpdateNSID { + event := tangled.GitRefUpdate{} + if err := json.Unmarshal(msg.EventJson, &event); err != nil { + l.Error("error unmarshalling", "err", err) + return err + } + l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) + l.Debug("debug") + + repoDid := syntax.DID(event.Repo) + repo, err := s.db.GetRepoByDid(repoDid) + if err != nil { + return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) + } + + if src.Host != repo.Knot { + return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) + } + + if kgit.HasSkipCIPushOption(event.PushOptions) { + l.Info("push event requested ci skip, skipping the event") + return nil + } + + // NOTE: we are blindly trusting the knot that it will return only repos it own + repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) + repoPath := s.newRepoPath(repoDid) + + triggerRepo, err := s.buildTriggerRepo(ctx, repo) + if err != nil { + return fmt.Errorf("building trigger repo: %w", err) + } + + trigger := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + Ref: event.Ref, + OldSha: event.OldSha, + NewSha: event.NewSha, + }, + Repo: triggerRepo, + } + + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) + if err != nil { + return err + } + if pipelineId.Rkey == "" { + l.Info("no workflow matched 'push' trigger, skipping the event") + return nil + } + l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) + } + + return nil +} + +func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { + var rec knotdb.RepoCollaboratorUpdate + if err := json.Unmarshal(msg.EventJson, &rec); err != nil { + l.Error("error unmarshalling collaboratorUpdate", "err", err) + return err + } + + subject, err := syntax.ParseDID(rec.Subject) + if err != nil { + l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) + return nil + } + repoDid, err := syntax.ParseDID(rec.Repo) + if err != nil { + l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) + return nil + } + + repo, err := s.db.GetRepoByDid(repoDid) + if errors.Is(err, sql.ErrNoRows) { + l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) + return nil + } + if err != nil { + return fmt.Errorf("lookup repo %s: %w", repoDid, err) + } + if src.Host != repo.Knot { + l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) + return nil + } + + switch rec.Op { + case knotdb.AclOpAdd: + if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { + return fmt.Errorf("add collaborator policy: %w", err) + } + if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { + return fmt.Errorf("track collaborator: %w", err) + } + l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) + case knotdb.AclOpRemove: + if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { + return fmt.Errorf("remove collaborator policy: %w", err) + } + if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { + return fmt.Errorf("delete collaborator row: %w", err) + } + l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) + default: + return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) + } + return nil +} + +// buildTriggerRepo gathers trigger metadata, resolving default branch from the knot +func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { + rkey := string(repo.Rkey) + repoDid := repo.RepoDid.String() + return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil +} + +func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { + scheme := "https" + if s.cfg.Server.Dev { + scheme = "http" + } + client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} + + // this should maybe (?) be in the refUpdate event itself to save a roundtrip + defaultBranch := "" + if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { + defaultBranch = out.Name + } + + var rkeyPtr *string + if rkey != "" { + rkeyPtr = &rkey + } + return &tangled.Pipeline_TriggerRepo{ + Did: did, + Knot: knot, + Repo: rkeyPtr, + RepoDid: &repoDid, + DefaultBranch: defaultBranch, + } +} + +func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { + if trigger == nil { + return nil, nil + } + if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { + return trigger.Repo, nil + } + repoDid, err := syntax.ParseDID(*trigger.SourceRepo) + if err != nil { + return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) + } + return s.resolveSourceRepoInfo(ctx, repoDid) +} + +// resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. +func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { + repo, err := s.db.GetRepoByDid(repoDid) + if err == nil { + return s.buildTriggerRepo(ctx, repo) + } + + // verify repo, we don't want git sync to point to arbitrary endpoints + res, err := s.verify(ctx, repoident.RepoDid(repoDid)) + if err != nil { + return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) + } + return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil +} + +// runPipeline compiles and enqueues the pipeline for the given revision. +// sourceRepo is the resolved repo the code was checked out from, forwarded to +// processPipeline for env vars. +func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { + l := log.FromContext(ctx) + + compiler := workflow.Compiler{ + ChangedFiles: changedFiles, + Trigger: trigger, + } + + rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) + if err != nil { + return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) + } + if len(rawPipeline) == 0 { + return models.PipelineId{}, nil + } + + tpl := compiler.Compile(compiler.Parse(rawPipeline)) + // todo(dawn): pass compile error to workflow log + for _, w := range compiler.Diagnostics.Errors { + l.Error(w.String()) + } + for _, w := range compiler.Diagnostics.Warnings { + l.Warn(w.String()) + } + + if len(only) > 0 { + tpl.Workflows = filterWorkflows(tpl.Workflows, only) + } + if len(tpl.Workflows) == 0 { + return models.PipelineId{}, nil + } + + pipelineId := models.PipelineId{ + Knot: trigger.Repo.Knot, + Rkey: tid.TID(), + } + if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { + return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) + } + err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) + return pipelineId, err +} + +// filterWorkflows filters workflows to the requested names +func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { + allowed := make(map[string]struct{}, len(only)) + for _, n := range only { + allowed[n] = struct{}{} + } + var filtered []*tangled.Pipeline_Workflow + for _, w := range workflows { + if w == nil { + continue + } + if _, ok := allowed[w.Name]; ok { + filtered = append(filtered, w) + } + } + return filtered +} + +// TriggerManual dispatches a pipeline at sha, authorized against and recorded +// under repoDid. sourceRepo, pull, and inputs are optional trigger payload. +func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { + repo, err := s.db.GetRepoByDid(repoDid) + if err != nil { + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) + } + + triggerRepo, err := s.buildTriggerRepo(ctx, repo) + if err != nil { + return "", fmt.Errorf("building trigger repo: %w", err) + } + + trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} + if pull.IsPullRequest { + var pullAt *string + if pull.Pull != "" { + pullAtStr := pull.Pull.String() + pullAt = &pullAtStr + } + trigger.Kind = string(workflow.TriggerKindPullRequest) + trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ + SourceBranch: pull.SourceBranch, + TargetBranch: pull.TargetBranch, + SourceSha: sha, + Pull: pullAt, + } + } else { + var refPtr *string + if ref != "" { + refPtr = &ref + } + trigger.Kind = string(workflow.TriggerKindManual) + trigger.Manual = &tangled.Pipeline_ManualTriggerData{ + Sha: sha, + Ref: refPtr, + Inputs: inputs, + } + } + + repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) + repoPath := s.newRepoPath(repoDid) + sourceInfo := triggerRepo // default: code comes from the repo itself + if sourceRepo != "" && sourceRepo != repoDid { + sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) + if err != nil { + return "", err + } + sourceRepoStr := sourceRepo.String() + trigger.SourceRepo = &sourceRepoStr + repoCloneUri = models.BuildRepoURL(sourceInfo) + repoPath = s.newRepoPath(sourceRepo) + } + + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) + if err != nil { + return "", err + } + if pipelineId.Rkey == "" { + return "", xrpc.ErrNoMatchingWorkflows + } + return pipelineId.AtUri(), nil +} + +func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { + if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { + return nil, fmt.Errorf("syncing git repo: %w", err) + } + gr, err := kgit.Open(repoPath, rev) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) + if errors.Is(err, object.ErrDirectoryNotFound) { + // return empty RawPipeline when directory doesn't exist + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("loading file tree: %w", err) + } + + var rawPipeline workflow.RawPipeline + for _, e := range workflowDir { + if !e.IsFile() { + continue + } + + fpath := filepath.Join(workflow.WorkflowDir, e.Name) + contents, err := gr.RawContent(fpath) + if err != nil { + return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) + } + + rawPipeline = append(rawPipeline, workflow.RawWorkflow{ + Name: e.Name, + Contents: contents, + }) + } + + return rawPipeline, nil +} + +func (s *Spindle) StartJobWorkers(ctx context.Context) { + for range s.cfg.Server.MaxJobCount { + go func() { + for { + job, err := s.db.DequeueJob(ctx) + if err != nil { + s.l.Error("failed to dequeue job", "error", err) + } + if job == nil { + // sleep until a new job wakes us + select { + case <-ctx.Done(): + return + case <-s.jobWake: + } + continue + } + s.runJob(ctx, job) + } + }() + } +} + +func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { + pipelineId := models.PipelineId{ + Knot: job.PipelineIdKnot, + Rkey: job.PipelineIdRkey, + } + + pipelineEnv := models.PipelineEnvVarsForSource(job.Tpl.TriggerMetadata, pipelineId, job.SourceRepo) + trustedSource := true + if tm := job.Tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && + *tm.SourceRepo != "" && *tm.SourceRepo != job.RepoDid { + trustedSource = false + } + + initTpl := job.Tpl + if job.SourceRepo != nil && job.Tpl.TriggerMetadata != nil { + tm := *job.Tpl.TriggerMetadata + tm.Repo = job.SourceRepo + initTpl.TriggerMetadata = &tm + } + + workflows := make(map[models.Engine][]models.Workflow) + for _, w := range job.Tpl.Workflows { + if w == nil { + continue + } + eng, ok := s.engs[w.Engine] + if !ok { + _ = s.db.StatusFailed(models.WorkflowId{ + PipelineId: pipelineId, + Name: w.Name, + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) + continue + } + + ewf, err := eng.InitWorkflow(*w, initTpl) + if err != nil { + _ = s.db.StatusFailed(models.WorkflowId{ + PipelineId: pipelineId, + Name: w.Name, + }, fmt.Sprintf("init workflow: %s", err), -1, s.n) + continue + } + + if ewf.Environment == nil { + ewf.Environment = make(map[string]string) + } + maps.Copy(ewf.Environment, pipelineEnv) + workflows[eng] = append(workflows[eng], *ewf) + } + + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.rootCtx, &models.Pipeline{ + RepoDid: syntax.DID(job.RepoDid), + Workflows: workflows, + TrustedSource: trustedSource, + }, pipelineId) +} + +// enqueues the workflows in tpl. +func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { + err := s.db.EnqueueJob(s.rootCtx, repoDid.String(), pipelineId, sourceRepo, tpl) + if err != nil { + return fmt.Errorf("failed to enqueue durable job: %w", err) + } + s.l.Info("pipeline enqueued successfully to db", "id", pipelineId) + + // wake up an idle worker to pick up more jobs if any + select { + case s.jobWake <- struct{}{}: + default: + } + + // pipelines visible from now on, they are sitting in queue + for _, w := range tpl.Workflows { + if w == nil { + continue + } + if err := s.db.StatusPending(models.WorkflowId{ + PipelineId: pipelineId, + Name: w.Name, + }, s.n); err != nil { + return fmt.Errorf("db.StatusPending: %w", err) + } + } + return nil +} + +// newRepoPath creates a path to store repository by its did and rkey. +// The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey +func (s *Spindle) newRepoPath(repo syntax.DID) string { + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) +} + +func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { + scheme := "https://" + if s.cfg.Server.Dev { + scheme = "http://" + } + return fmt.Sprintf("%s%s/%s", scheme, knot, did) +} + +const RequiredVersion = "2.49.0" + +func ensureGitVersion() error { + v, err := git.Version() + if err != nil { + return fmt.Errorf("fetching git version: %w", err) + } + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) + } + return nil +} + +func (s *Spindle) resolvePipelineRepoDid(repo *tangled.Pipeline_TriggerRepo) (syntax.DID, error) { + if repo.RepoDid == nil || *repo.RepoDid == "" { + return "", fmt.Errorf("pipeline trigger missing repoDid") + } + repoDid, err := syntax.ParseDID(*repo.RepoDid) + if err != nil { + return "", fmt.Errorf("parse repoDid %s: %w", *repo.RepoDid, err) + } + if _, err := s.db.GetRepoByDid(repoDid); err != nil { + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) + } + return repoDid, nil +} + +func (s *Spindle) configureOwner() error { + cfgOwner := s.cfg.Server.Owner + + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) + if err != nil { + return err + } + + switch len(existing) { + case 0: + // no owner configured, continue + case 1: + // find existing owner + existingOwner := existing[0] + + // no ownership change, this is okay + if existingOwner == s.cfg.Server.Owner { + break + } + + // remove existing owner + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) + if err != nil { + return nil + } + default: + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) + } + + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) +} diff --git a/spindle/server.go.mill b/spindle/server.go.mill new file mode 100644 index 00000000..65c0bafd --- /dev/null +++ b/spindle/server.go.mill @@ -0,0 +1,1028 @@ +package spindle + +import ( + "context" + "database/sql" + _ "embed" + "encoding/json" + "errors" + "fmt" + "log/slog" + "maps" + "net/http" + "path/filepath" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/hashicorp/go-version" + "tangled.org/core/api/tangled" + "tangled.org/core/eventconsumer" + "tangled.org/core/eventconsumer/cursor" + "tangled.org/core/eventstream" + "tangled.org/core/idresolver" + "tangled.org/core/jetstream" + knotdb "tangled.org/core/knotserver/db" + kgit "tangled.org/core/knotserver/git" + "tangled.org/core/log" + "tangled.org/core/notifier" + "tangled.org/core/rbac" + "tangled.org/core/repoident" + "tangled.org/core/repoverify" + "tangled.org/core/spindle/artifactstore" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/engines/nixery" + "tangled.org/core/spindle/git" + "tangled.org/core/spindle/mill" + "tangled.org/core/spindle/mill/executor" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/queue" + "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/xrpc" + "tangled.org/core/tid" + "tangled.org/core/workflow" + "tangled.org/core/xrpc/serviceauth" +) + +//go:embed motd +var defaultMotd []byte + +const ( + rbacDomain = "thisserver" +) + +type Spindle struct { + jc *jetstream.JetstreamClient + tap *Tap + embedTap *embeddedTap + db *db.DB + e *rbac.Enforcer + l *slog.Logger + n *notifier.Notifier + engs map[string]models.Engine + jq *queue.Queue + cfg *config.Config + ks *eventconsumer.Consumer + res *idresolver.Resolver + verify repoverify.Verifier + vault secrets.Manager + motd []byte + motdMu sync.RWMutex + rootCtx context.Context + store artifactstore.Store + stores *artifactstore.Stores + reader artifactstore.Reader + // set only when this spindle hosts the mill or joins one as an executor + mill *mill.Mill + exec *executor.Executor +} + +// New creates a new Spindle server with the provided configuration and engines. +func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { + logger := log.FromContext(ctx) + n := notifier.New() + + if cfg.Role == config.RoleExecutor { + if err := cleanupOrphanRepos(ctx, d, logger); err != nil { + return nil, fmt.Errorf("failed to run startup cleanup: %w", err) + } + } else if err := runStartupMigrations(ctx, d, cfg.Server.Tap.Embed, cfg.Server.Tap.DBPath, logger); err != nil { + return nil, fmt.Errorf("failed to run startup migrations: %w", err) + } + + spindle := &Spindle{ + db: d, + l: logger, + n: &n, + engs: engines, + cfg: cfg, + motd: defaultMotd, + rootCtx: ctx, + } + diskFallback := "" + if cfg.Role == config.RoleStandalone { + diskFallback = cfg.Server.LogDir + if cfg.ArtifactStores.Disk.Dir == "" { + logger.Warn("using SPINDLE_SERVER_LOG_DIR as the implicit disk artifact store; configure SPINDLE_ARTIFACT_STORES_DISK_DIR explicitly") + } + } + stores, err := artifactstore.NewStores(cfg.ArtifactStores, diskFallback, cfg.LegacyS3.LogBucket) + if err != nil { + return nil, fmt.Errorf("failed to setup artifact stores: %w", err) + } + spindle.stores = stores + if cfg.LegacyS3.LogBucket != "" { + logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") + } + if cfg.Role == config.RoleStandalone { + spindle.reader = stores + } else { + name := cfg.Mill.ArtifactStore + if name == "" { + names := stores.Names() + if len(names) != 1 { + return nil, fmt.Errorf("%s requires SPINDLE_MILL_ARTIFACT_STORE when %d artifact stores are configured", cfg.Role, len(names)) + } + name = names[0] + logger.Warn("SPINDLE_MILL_ARTIFACT_STORE is not set; inferred the only configured store", "store", name) + } + store, ok := stores.Store(name) + if !ok { + return nil, fmt.Errorf("SPINDLE_MILL_ARTIFACT_STORE=%q is not configured", name) + } + spindle.store = store + spindle.reader = store + } + if cfg.Role == config.RoleExecutor { + return spindle, nil + } + + e, err := rbac.NewEnforcer(cfg.Server.DBPath) + if err != nil { + return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) + } + e.E.EnableAutoSave(true) + spindle.e = e + + switch cfg.Server.Secrets.Provider { + case "openbao": + if cfg.Server.Secrets.OpenBao.ProxyAddr == "" { + return nil, fmt.Errorf("openbao proxy address is required when using openbao secrets provider") + } + spindle.vault, err = secrets.NewOpenBaoManager( + cfg.Server.Secrets.OpenBao.ProxyAddr, + logger, + secrets.WithMountPath(cfg.Server.Secrets.OpenBao.Mount), + ) + if err != nil { + return nil, fmt.Errorf("failed to setup openbao secrets provider: %w", err) + } + logger.Info("using openbao secrets provider", "proxy_address", cfg.Server.Secrets.OpenBao.ProxyAddr, "mount", cfg.Server.Secrets.OpenBao.Mount) + case "sqlite", "": + spindle.vault, err = secrets.NewSQLiteManager(cfg.Server.DBPath, secrets.WithTableName("secrets")) + if err != nil { + return nil, fmt.Errorf("failed to setup sqlite secrets provider: %w", err) + } + logger.Info("using sqlite secrets provider", "path", cfg.Server.DBPath) + default: + return nil, fmt.Errorf("unknown secrets provider: %s", cfg.Server.Secrets.Provider) + } + + if cfg.Role == config.RoleStandalone { + spindle.jq = queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) + logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) + } + + collections := []string{ + tangled.SpindleMemberNSID, + tangled.RepoNSID, + tangled.RepoCollaboratorNSID, + tangled.RepoPullNSID, + } + jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, log.SubLogger(logger, "jetstream"), d, true, true) + if err != nil { + return nil, fmt.Errorf("failed to setup jetstream client: %w", err) + } + spindle.jc = jc + jc.AddDid(cfg.Server.Owner) + // pull records are created by arbitrary users too, same hack as in tap + jc.ExemptCollection(tangled.RepoPullNSID) + + // Check if the spindle knows about any Dids; + dids, err := d.GetAllDids() + if err != nil { + return nil, fmt.Errorf("failed to get all dids: %w", err) + } + for _, d := range dids { + jc.AddDid(d) + } + + knownRepos, err := d.AllRepos() + if err != nil { + return nil, fmt.Errorf("failed to get known repos: %w", err) + } + for _, r := range knownRepos { + if r.Owner != "" { + jc.AddDid(r.Owner.String()) + } + } + + spindle.res = idresolver.DefaultResolver(cfg.Server.PlcUrl) + spindle.verify = repoverify.New(spindle.res, cfg.Server.Dev) + + err = e.AddSpindle(rbacDomain) + if err != nil { + return nil, fmt.Errorf("failed to set rbac domain: %w", err) + } + err = spindle.configureOwner() + if err != nil { + return nil, err + } + logger.Info("owner set", "did", cfg.Server.Owner) + + cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath) + if err != nil { + return nil, fmt.Errorf("failed to setup sqlite3 cursor store: %w", err) + } + + err = jc.StartJetstream(ctx, spindle.ingest()) + if err != nil { + return nil, fmt.Errorf("failed to start jetstream consumer: %w", err) + } + + // spindle listen to knot stream for sh.tangled.git.refUpdate + // which will sync the local workflow files in spindle and enqueues the + // pipeline job for on-push workflows + ccfg := eventconsumer.NewConsumerConfig() + ccfg.Logger = log.SubLogger(logger, "eventconsumer") + ccfg.ProcessFunc = spindle.processKnotStream + ccfg.CursorStore = cursorStore + if cfg.Server.Dev { + ccfg.RetryInterval = 5 * time.Second + ccfg.MaxRetryInterval = 10 * time.Second + } else { + ccfg.RetryInterval = 1 * time.Minute + ccfg.MaxRetryInterval = 10 * time.Minute + } + knownKnots, err := d.Knots() + if err != nil { + return nil, err + } + for _, knot := range knownKnots { + logger.Info("adding source start", "knot", knot) + src := eventconsumer.NewKnotSource(knot) + eventconsumer.MigrateLegacyCursor(cursorStore, src) + ccfg.Sources[src] = struct{}{} + } + spindle.ks = eventconsumer.NewConsumer(*ccfg) + if cfg.Server.Tap.Embed { + pw, err := randomAdminPassword() + if err != nil { + return nil, err + } + cfg.Server.Tap.AdminPassword = pw + logger.Info("embedded tap: using random admin password") + } + spindle.tap = NewTapClient(spindle) + + return spindle, nil +} +func (s *Spindle) DB() *db.DB { + return s.db +} +func (s *Spindle) Queue() *queue.Queue { + return s.jq +} + +// Engines returns the map of available engines. +func (s *Spindle) Engines() map[string]models.Engine { + return s.engs +} + +// Vault returns the secrets manager instance. +func (s *Spindle) Vault() secrets.Manager { + return s.vault +} + +// Notifier returns the notifier instance. +func (s *Spindle) Notifier() *notifier.Notifier { + return s.n +} + +// Enforcer returns the RBAC enforcer instance. +func (s *Spindle) Enforcer() *rbac.Enforcer { + return s.e +} + +// SetMotdContent sets custom MOTD content, replacing the embedded default. +func (s *Spindle) SetMotdContent(content []byte) { + s.motdMu.Lock() + defer s.motdMu.Unlock() + s.motd = content +} + +// GetMotdContent returns the current MOTD content. +func (s *Spindle) GetMotdContent() []byte { + s.motdMu.RLock() + defer s.motdMu.RUnlock() + return s.motd +} + +// runs the server. blocks +func (s *Spindle) Start(ctx context.Context) error { + // only standalone runs the local queue. mill hosts place directly onto + // executors, and executors only run jobs explicitly assigned by a mill + if s.cfg.Role == config.RoleStandalone { + if s.jq != nil { + s.jq.Start() + defer s.jq.Stop() + } + } + + // an executor dials out to its mill and takes work from it + if s.exec != nil { + go s.exec.Connect(ctx) + } + + if stopper, ok := s.vault.(secrets.Stopper); ok { + defer stopper.Stop() + } + + if s.cfg.Role != config.RoleExecutor { + tapCtx, tapCancel := context.WithCancel(ctx) + + if s.cfg.Server.Tap.Embed { + emb, err := startEmbeddedTap(tapCtx, s.cfg, log.SubLogger(s.l, "embedtap")) + if err != nil { + tapCancel() + return fmt.Errorf("starting embedded tap: %w", err) + } + s.embedTap = emb + defer func() { + tapCancel() + s.embedTap.Shutdown() + }() + + go s.watchTapDrain(tapCtx, tapCancel) + } else { + defer tapCancel() + } + + go func() { + s.l.Info("starting knot event consumer") + s.ks.Start(ctx) + }() + + s.l.Info("starting tap client", "url", s.cfg.Server.Tap.Url) + s.tap.Start(tapCtx) + } + + s.l.Info("starting spindle server", "address", s.cfg.Server.ListenAddr) + return http.ListenAndServe(s.cfg.Server.ListenAddr, s.Router()) +} + +func (s *Spindle) declareTapInterest(ctx context.Context) { + repos, err := s.db.AllRepos() + if err != nil { + s.l.Warn("tap declare: failed to load known repos", "err", err) + return + } + seen := make(map[syntax.DID]struct{}, len(repos)) + dids := make([]syntax.DID, 0, len(repos)) + for _, r := range repos { + if r.Owner == "" { + continue + } + if _, ok := seen[r.Owner]; ok { + continue + } + seen[r.Owner] = struct{}{} + dids = append(dids, r.Owner) + } + if err := s.tap.AddOwnerDIDs(ctx, dids); err != nil { + s.l.Warn("tap declare: AddRepos rejected", "count", len(dids), "err", err) + return + } + s.l.Info("tap declare: known owner DIDs registered", "count", len(dids)) +} + +func Run(ctx context.Context) error { + cfg, err := config.Load(ctx) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + if err := ensureGitVersion(); err != nil { + return fmt.Errorf("ensuring git version: %w", err) + } + + d, err := db.Make(ctx, cfg.Server.DBPath) + if err != nil { + return fmt.Errorf("failed to setup db: %w", err) + } + + logger := log.FromContext(ctx) + + var engines map[string]models.Engine + var m *mill.Mill + + if cfg.Role == config.RoleMill { + // mill host: register engines that place jobs on executors instead of + // running them. all names share one Mill + m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ + LogDir: cfg.Server.LogDir, + MaxPending: cfg.Mill.MaxPending, + ReconnectGrace: cfg.Mill.ReconnectGrace, + }) + engines = map[string]models.Engine{ + "nixery": mill.NewEngine("nixery", m), + "microvm": mill.NewEngine("microvm", m), + "dummy": mill.NewEngine("dummy", m), + } + } else { + // standalone and executor both run real engines locally. + nixeryEng, err := nixery.New(ctx, cfg) + if err != nil { + return err + } + microvmEng, err := newMicrovmEngine(ctx, cfg, d) + if err != nil { + return err + } + engines = map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(logger), + } + } + + s, err := New(ctx, cfg, d, engines) + if err != nil { + return err + } + + if m != nil { + // resolve the chicken-and-egg: the engines (built above) hold the mill, + // but the mill's db/notifier are created inside New + m.Attach(s.DB(), s.Notifier()) + s.mill = m + if err := m.RestoreState(); err != nil { + return fmt.Errorf("restoring mill state: %w", err) + } + } + if cfg.Role == config.RoleExecutor { + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) + if err != nil { + return err + } + } + + return s.Start(ctx) +} + +func (s *Spindle) Router() http.Handler { + mux := chi.NewRouter() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write(s.GetMotdContent()) + }) + if s.cfg.Role == config.RoleExecutor { + return mux + } + + mux.HandleFunc("/events", s.Events) + mux.HandleFunc("/logs/{knot}/{rkey}/{name}", s.Logs) + + // mill host: executors dial in here (plain ws, shared-secret auth) + if s.mill != nil { + mux.HandleFunc("/mill", s.mill.HandleExecutorConn) + } + + mux.Mount("/xrpc", s.XrpcRouter()) + return mux +} + +func (s *Spindle) XrpcRouter() http.Handler { + serviceAuth := serviceauth.NewServiceAuth(s.l, s.res.Directory(), s.cfg.Server.Did().String()) + + l := log.SubLogger(s.l, "xrpc") + + x := xrpc.Xrpc{ + Logger: l, + Db: s.db, + Enforcer: s.e, + Engines: s.engs, + Config: s.cfg, + ArtifactReader: s.reader, + Resolver: s.res, + Vault: s.vault, + Notifier: s.Notifier(), + ServiceAuth: serviceAuth, + Trigger: s, + } + + return x.Router() +} + +func (s *Spindle) processKnotStream(ctx context.Context, src eventconsumer.Source, msg eventstream.Event) error { + l := log.FromContext(ctx).With("handler", "processKnotStream") + l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) + if msg.Nsid == knotdb.RepoCollaboratorUpdateNSID { + return s.ingestKnotCollaborator(ctx, l, src, msg) + } + if msg.Nsid == tangled.GitRefUpdateNSID { + event := tangled.GitRefUpdate{} + if err := json.Unmarshal(msg.EventJson, &event); err != nil { + l.Error("error unmarshalling", "err", err) + return err + } + l = l.With("repo", event.Repo, "ref", event.Ref, "newSha", event.NewSha) + l.Debug("debug") + + repoDid := syntax.DID(event.Repo) + repo, err := s.db.GetRepoByDid(repoDid) + if err != nil { + return fmt.Errorf("unknown repoDid %s: %w", repoDid, err) + } + + if src.Host != repo.Knot { + return fmt.Errorf("repo knot does not match event source: %s != %s", src.Host, repo.Knot) + } + + if kgit.HasSkipCIPushOption(event.PushOptions) { + l.Info("push event requested ci skip, skipping the event") + return nil + } + + // NOTE: we are blindly trusting the knot that it will return only repos it own + repoCloneUri := s.newRepoCloneUrl(src.Host, repoDid) + repoPath := s.newRepoPath(repoDid) + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { + return fmt.Errorf("sync git repo: %w", err) + } + l.Info("synced git repo") + + triggerRepo, err := s.buildTriggerRepo(ctx, repo) + if err != nil { + return fmt.Errorf("building trigger repo: %w", err) + } + + trigger := tangled.Pipeline_TriggerMetadata{ + Kind: string(workflow.TriggerKindPush), + Push: &tangled.Pipeline_PushTriggerData{ + Ref: event.Ref, + OldSha: event.OldSha, + NewSha: event.NewSha, + }, + Repo: triggerRepo, + } + + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, event.ChangedFiles, repoCloneUri, repoPath, event.NewSha, nil, triggerRepo) + if err != nil { + return err + } + if pipelineId.Rkey == "" { + l.Info("no workflow matched 'push' trigger, skipping the event") + return nil + } + l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) + } + + return nil +} + +func (s *Spindle) ingestKnotCollaborator(ctx context.Context, l *slog.Logger, src eventconsumer.Source, msg eventstream.Event) error { + var rec knotdb.RepoCollaboratorUpdate + if err := json.Unmarshal(msg.EventJson, &rec); err != nil { + l.Error("error unmarshalling collaboratorUpdate", "err", err) + return err + } + + subject, err := syntax.ParseDID(rec.Subject) + if err != nil { + l.Info("skipping collaboratorUpdate with malformed subject", "subject", rec.Subject, "err", err) + return nil + } + repoDid, err := syntax.ParseDID(rec.Repo) + if err != nil { + l.Info("skipping collaboratorUpdate with malformed repo", "repo", rec.Repo, "err", err) + return nil + } + + repo, err := s.db.GetRepoByDid(repoDid) + if errors.Is(err, sql.ErrNoRows) { + l.Info("skipping collaboratorUpdate for unknown repo", "repo", repoDid) + return nil + } + if err != nil { + return fmt.Errorf("lookup repo %s: %w", repoDid, err) + } + if src.Host != repo.Knot { + l.Warn("dropping collaboratorUpdate from non-owning knot", "src", src.Host, "repoKnot", repo.Knot) + return nil + } + + switch rec.Op { + case knotdb.AclOpAdd: + if err := s.e.AddCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { + return fmt.Errorf("add collaborator policy: %w", err) + } + if err := s.db.AddKnotCollaborator(repoDid, subject); err != nil { + return fmt.Errorf("track collaborator: %w", err) + } + l.Info("added knot-managed collaborator", "subject", subject, "repo", repoDid) + case knotdb.AclOpRemove: + if err := s.e.RemoveCollaborator(subject.String(), rbac.ThisServer, repoDid.String()); err != nil { + return fmt.Errorf("remove collaborator policy: %w", err) + } + if err := s.db.DeleteRepoCollaboratorBySubjectRepo(subject, repoDid); err != nil { + return fmt.Errorf("delete collaborator row: %w", err) + } + l.Info("removed knot-managed collaborator", "subject", subject, "repo", repoDid) + default: + return fmt.Errorf("collaboratorUpdate unknown op %q", rec.Op) + } + return nil +} + +// buildTriggerRepo gathers trigger metadata, resolving default branch from the knot +func (s *Spindle) buildTriggerRepo(ctx context.Context, repo *db.Repo) (*tangled.Pipeline_TriggerRepo, error) { + rkey := string(repo.Rkey) + repoDid := repo.RepoDid.String() + return s.buildTriggerRepoFrom(ctx, repo.Knot, repo.Owner.String(), rkey, repoDid), nil +} + +func (s *Spindle) buildTriggerRepoFrom(ctx context.Context, knot, did, rkey, repoDid string) *tangled.Pipeline_TriggerRepo { + scheme := "https" + if s.cfg.Server.Dev { + scheme = "http" + } + client := &indigoxrpc.Client{Host: fmt.Sprintf("%s://%s", scheme, knot)} + + // this should maybe (?) be in the refUpdate event itself to save a roundtrip + defaultBranch := "" + if out, err := tangled.RepoGetDefaultBranch(ctx, client, repoDid); err == nil { + defaultBranch = out.Name + } + + var rkeyPtr *string + if rkey != "" { + rkeyPtr = &rkey + } + return &tangled.Pipeline_TriggerRepo{ + Did: did, + Knot: knot, + Repo: rkeyPtr, + RepoDid: &repoDid, + DefaultBranch: defaultBranch, + } +} + +func (s *Spindle) resolvePipelineSourceRepo(ctx context.Context, trigger *tangled.Pipeline_TriggerMetadata) (*tangled.Pipeline_TriggerRepo, error) { + if trigger == nil { + return nil, nil + } + if trigger.SourceRepo == nil || *trigger.SourceRepo == "" { + return trigger.Repo, nil + } + repoDid, err := syntax.ParseDID(*trigger.SourceRepo) + if err != nil { + return nil, fmt.Errorf("parse sourceRepo %s: %w", *trigger.SourceRepo, err) + } + return s.resolveSourceRepoInfo(ctx, repoDid) +} + +// resolveSourceRepoInfo resolves trigger-repo metadata for a source repo DID. +func (s *Spindle) resolveSourceRepoInfo(ctx context.Context, repoDid syntax.DID) (*tangled.Pipeline_TriggerRepo, error) { + repo, err := s.db.GetRepoByDid(repoDid) + if err == nil { + return s.buildTriggerRepo(ctx, repo) + } + + // verify repo, we don't want git sync to point to arbitrary endpoints + res, err := s.verify(ctx, repoident.RepoDid(repoDid)) + if err != nil { + return nil, fmt.Errorf("verify sourceRepo %s: %w", repoDid, err) + } + return s.buildTriggerRepoFrom(ctx, res.KnotURL.Host, res.OwnerDid.String(), res.Rkey, repoDid.String()), nil +} + +// runPipeline compiles and enqueues the pipeline for the given revision. +// sourceRepo is the resolved repo the code was checked out from, forwarded to +// processPipeline for env vars. +func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger tangled.Pipeline_TriggerMetadata, changedFiles []string, repoCloneUri, repoPath, rev string, only []string, sourceRepo *tangled.Pipeline_TriggerRepo) (models.PipelineId, error) { + l := log.FromContext(ctx) + + compiler := workflow.Compiler{ + ChangedFiles: changedFiles, + Trigger: trigger, + } + + rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) + if err != nil { + return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) + } + if len(rawPipeline) == 0 { + return models.PipelineId{}, nil + } + + tpl := compiler.Compile(compiler.Parse(rawPipeline)) + // todo(dawn): pass compile error to workflow log + for _, w := range compiler.Diagnostics.Errors { + l.Error(w.String()) + } + for _, w := range compiler.Diagnostics.Warnings { + l.Warn(w.String()) + } + + if len(only) > 0 { + tpl.Workflows = filterWorkflows(tpl.Workflows, only) + } + if len(tpl.Workflows) == 0 { + return models.PipelineId{}, nil + } + + pipelineId := models.PipelineId{ + Knot: trigger.Repo.Knot, + Rkey: tid.TID(), + } + if err := s.db.CreatePipelineEvent(pipelineId.Rkey, tpl, s.n); err != nil { + return models.PipelineId{}, fmt.Errorf("creating pipeline event: %w", err) + } + err = s.processPipeline(repoDid, tpl, pipelineId, sourceRepo) + return pipelineId, err +} + +// filterWorkflows filters workflows to the requested names +func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*tangled.Pipeline_Workflow { + allowed := make(map[string]struct{}, len(only)) + for _, n := range only { + allowed[n] = struct{}{} + } + var filtered []*tangled.Pipeline_Workflow + for _, w := range workflows { + if w == nil { + continue + } + if _, ok := allowed[w.Name]; ok { + filtered = append(filtered, w) + } + } + return filtered +} + +// TriggerManual dispatches a pipeline at sha, authorized against and recorded +// under repoDid. sourceRepo, pull, and inputs are optional trigger payload. +func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { + repo, err := s.db.GetRepoByDid(repoDid) + if err != nil { + return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) + } + + triggerRepo, err := s.buildTriggerRepo(ctx, repo) + if err != nil { + return "", fmt.Errorf("building trigger repo: %w", err) + } + + trigger := tangled.Pipeline_TriggerMetadata{Repo: triggerRepo} + if pull.IsPullRequest { + var pullAt *string + if pull.Pull != "" { + pullAtStr := pull.Pull.String() + pullAt = &pullAtStr + } + trigger.Kind = string(workflow.TriggerKindPullRequest) + trigger.PullRequest = &tangled.Pipeline_PullRequestTriggerData{ + SourceBranch: pull.SourceBranch, + TargetBranch: pull.TargetBranch, + SourceSha: sha, + Pull: pullAt, + } + } else { + var refPtr *string + if ref != "" { + refPtr = &ref + } + trigger.Kind = string(workflow.TriggerKindManual) + trigger.Manual = &tangled.Pipeline_ManualTriggerData{ + Sha: sha, + Ref: refPtr, + Inputs: inputs, + } + } + + repoCloneUri := s.newRepoCloneUrl(repo.Knot, repoDid) + repoPath := s.newRepoPath(repoDid) + sourceInfo := triggerRepo // default: code comes from the repo itself + if sourceRepo != "" && sourceRepo != repoDid { + sourceInfo, err = s.resolveSourceRepoInfo(ctx, sourceRepo) + if err != nil { + return "", err + } + sourceRepoStr := sourceRepo.String() + trigger.SourceRepo = &sourceRepoStr + repoCloneUri = models.BuildRepoURL(sourceInfo) + repoPath = s.newRepoPath(sourceRepo) + } + + pipelineId, err := s.runPipeline(ctx, repoDid, trigger, nil, repoCloneUri, repoPath, sha, workflows, sourceInfo) + if err != nil { + return "", err + } + if pipelineId.Rkey == "" { + return "", xrpc.ErrNoMatchingWorkflows + } + return pipelineId.AtUri(), nil +} + +func (s *Spindle) loadPipeline(ctx context.Context, repoUri, repoPath, rev string) (workflow.RawPipeline, error) { + if err := git.SparseSyncGitRepo(ctx, repoUri, repoPath, rev); err != nil { + return nil, fmt.Errorf("syncing git repo: %w", err) + } + gr, err := kgit.Open(repoPath, rev) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) + if errors.Is(err, object.ErrDirectoryNotFound) { + // return empty RawPipeline when directory doesn't exist + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("loading file tree: %w", err) + } + + var rawPipeline workflow.RawPipeline + for _, e := range workflowDir { + if !e.IsFile() { + continue + } + + fpath := filepath.Join(workflow.WorkflowDir, e.Name) + contents, err := gr.RawContent(fpath) + if err != nil { + return nil, fmt.Errorf("reading raw content of '%s': %w", fpath, err) + } + + rawPipeline = append(rawPipeline, workflow.RawWorkflow{ + Name: e.Name, + Contents: contents, + }) + } + + return rawPipeline, nil +} + +// processPipeline enqueues the workflows in tpl. +func (s *Spindle) processPipeline(repoDid syntax.DID, tpl tangled.Pipeline, pipelineId models.PipelineId, sourceRepo *tangled.Pipeline_TriggerRepo) error { + // derive security-relevant things like whether this run is trusted and can be passed + // secrets to from the original metadata. + pipelineEnv := models.PipelineEnvVarsForSource(tpl.TriggerMetadata, pipelineId, sourceRepo) + trustedSource := true + if tm := tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && + *tm.SourceRepo != "" && *tm.SourceRepo != repoDid.String() { + trustedSource = false + } + + // swap the repo with our sourceRepo if we are running a pipeline on a fork. + // the metadata stays the same. we check whether the repo is trusted above, + // so this only affects the clone URL. + initTpl := tpl + if sourceRepo != nil && tpl.TriggerMetadata != nil { + tm := *tpl.TriggerMetadata + tm.Repo = sourceRepo + initTpl.TriggerMetadata = &tm + } + + // filter & init workflows + workflows := make(map[models.Engine][]models.Workflow) + for _, w := range tpl.Workflows { + if w == nil { + continue + } + eng, ok := s.engs[w.Engine] + if !ok { + err := s.db.StatusFailed(models.WorkflowId{ + PipelineId: pipelineId, + Name: w.Name, + }, fmt.Sprintf("unknown engine %#v", w.Engine), -1, s.n) + if err != nil { + return fmt.Errorf("db.StatusFailed: %w", err) + } + + continue + } + + ewf, err := eng.InitWorkflow(*w, initTpl) + if err != nil { + err = s.db.StatusFailed(models.WorkflowId{ + PipelineId: pipelineId, + Name: w.Name, + }, fmt.Sprintf("init workflow: %s", err), -1, s.n) + if err != nil { + return fmt.Errorf("db.StatusFailed: %w", err) + } + + continue + } + + // inject TANGLED_* env vars after InitWorkflow + // This prevents user-defined env vars from overriding them + if ewf.Environment == nil { + ewf.Environment = make(map[string]string) + } + maps.Copy(ewf.Environment, pipelineEnv) + + workflows[eng] = append(workflows[eng], *ewf) + } + + pipeline := &models.Pipeline{ + RepoDid: repoDid, + Workflows: workflows, + TrustedSource: trustedSource, + } + + if s.mill != nil { + // mill host: no bounded pool. each job blocks in placement + // (AcquireWorkflowSlot) which the user sees as pending. the only bound + // is the mill's maxPending. rootCtx is the long-lived consumer context, + // so the goroutine safely outlives this call + go engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) + s.l.Info("pipeline handed to mill placement", "id", pipelineId) + } else if s.jq != nil { + ok := s.jq.Enqueue(repoDid, queue.Job{ + Run: func() error { + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, pipeline, pipelineId) + return nil + }, + OnFail: func(jobError error) { + s.l.Error("pipeline run failed", "error", jobError) + }, + }) + if !ok { + return fmt.Errorf("failed to enqueue pipeline: queue is full") + } + s.l.Info("pipeline enqueued successfully", "id", pipelineId) + } else { + return fmt.Errorf("no queue or mill available to process pipeline") + } + + // after successful enqueue, emit StatusPending for all workflows + for _, ewfs := range workflows { + for _, ewf := range ewfs { + err := s.db.StatusPending(models.WorkflowId{ + PipelineId: pipelineId, + Name: ewf.Name, + }, s.n) + if err != nil { + return fmt.Errorf("db.StatusPending: %w", err) + } + } + } + return nil +} + +// newRepoPath creates a path to store repository by its did and rkey. +// The path format would be: `/data/repos/did:plc:foo/sh.tangled.repo/repo-rkey +func (s *Spindle) newRepoPath(repo syntax.DID) string { + return filepath.Join(s.cfg.Server.RepoDir, repo.String()) +} + +func (s *Spindle) newRepoCloneUrl(knot string, did syntax.DID) string { + scheme := "https://" + if s.cfg.Server.Dev { + scheme = "http://" + } + return fmt.Sprintf("%s%s/%s", scheme, knot, did) +} + +const RequiredVersion = "2.49.0" + +func ensureGitVersion() error { + v, err := git.Version() + if err != nil { + return fmt.Errorf("fetching git version: %w", err) + } + if v.LessThan(version.Must(version.NewVersion(RequiredVersion))) { + return fmt.Errorf("installed git version %q is not supported, Spindle requires git version >= %q", v, RequiredVersion) + } + return nil +} + +func (s *Spindle) configureOwner() error { + cfgOwner := s.cfg.Server.Owner + + existing, err := s.e.GetSpindleUsersByRole("server:owner", rbacDomain) + if err != nil { + return err + } + + switch len(existing) { + case 0: + // no owner configured, continue + case 1: + // find existing owner + existingOwner := existing[0] + + // no ownership change, this is okay + if existingOwner == s.cfg.Server.Owner { + break + } + + // remove existing owner + err = s.e.RemoveSpindleOwner(rbacDomain, existingOwner) + if err != nil { + return nil + } + default: + return fmt.Errorf("more than one owner in DB, try deleting %q and starting over", s.cfg.Server.DBPath) + } + + return s.e.AddSpindleOwner(rbacDomain, cfgOwner) +} diff --git a/spindle/server_test.go b/spindle/server_test.go index 03b1efc4..c1f9350c 100644 --- a/spindle/server_test.go +++ b/spindle/server_test.go @@ -1,9 +1,16 @@ package spindle import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" "testing" kgit "tangled.org/core/knotserver/git" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" ) func TestHasSkipCIPushOption(t *testing.T) { @@ -53,3 +60,42 @@ func TestHasSkipCIPushOption(t *testing.T) { }) } } + +func TestExecutorRoleBuildsMinimalSpindle(t *testing.T) { + ctx := context.Background() + dbPath := filepath.Join(t.TempDir(), "spindle.db") + d, err := db.Make(ctx, dbPath) + if err != nil { + t.Fatalf("db.Make() error = %v", err) + } + + cfg := &config.Config{Role: config.RoleExecutor} + cfg.Server.DBPath = dbPath + cfg.Server.Hostname = "executor.test" + cfg.Server.Tap.Embed = true + cfg.ArtifactStores.Disk.Dir = t.TempDir() + cfg.Mill.ArtifactStore = "disk" + + s, err := New(ctx, cfg, d, map[string]models.Engine{}) + if err != nil { + t.Fatalf("New() error = %v", err) + } + + if s.jc != nil || s.tap != nil || s.e != nil || s.ks != nil || s.res != nil || s.vault != nil { + t.Fatal("executor role built coordinator-only spindle dependencies") + } + + rr := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/", nil) + s.Router().ServeHTTP(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("root status = %d, want %d", rr.Code, http.StatusOK) + } + + rr = httptest.NewRecorder() + req = httptest.NewRequest(http.MethodGet, "/xrpc/_health", nil) + s.Router().ServeHTTP(rr, req) + if rr.Code != http.StatusNotFound { + t.Fatalf("executor xrpc status = %d, want %d", rr.Code, http.StatusNotFound) + } +} diff --git a/workflow/compile.go b/workflow/compile.go index 61bd6e55..d02ff9b9 100644 --- a/workflow/compile.go +++ b/workflow/compile.go @@ -112,8 +112,6 @@ func (compiler *Compiler) Compile(p Pipeline) tangled.Pipeline { } func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow { - cw := &tangled.Pipeline_Workflow{} - matched, err := w.Match(compiler.Trigger, compiler.ChangedFiles) if err != nil { compiler.Diagnostics.AddError( @@ -134,18 +132,19 @@ func (compiler *Compiler) compileWorkflow(w Workflow) *tangled.Pipeline_Workflow // validate clone options compiler.analyzeCloneOptions(w) - cw.Name = w.Name - if w.Engine == "" { compiler.Diagnostics.AddError(w.Name, MissingEngine) return nil } - cw.Engine = w.Engine - cw.Raw = w.Raw - o := w.CloneOpts.AsRecord() - cw.Clone = &o + cw := &tangled.Pipeline_Workflow{ + Clone: &o, + Engine: w.Engine, + Name: w.Name, + Raw: w.Raw, + RunsOn: w.RunsOn, + } return cw } diff --git a/workflow/compile_test.go b/workflow/compile_test.go index 45428552..a8354ad7 100644 --- a/workflow/compile_test.go +++ b/workflow/compile_test.go @@ -1,6 +1,7 @@ package workflow import ( + "encoding/json" "strings" "testing" @@ -41,6 +42,50 @@ func TestCompileWorkflow_MatchingWorkflowWithSteps(t *testing.T) { assert.False(t, c.Diagnostics.IsErr()) } +func TestCompileWorkflow_RunsOnLabelsPersistWithoutFanout(t *testing.T) { + wf := Workflow{ + Name: ".tangled/workflows/arm64.yml", + Engine: "microvm", + When: when, + RunsOn: []string{"linux/arm64", "kvm"}, + } + + c := Compiler{Trigger: trigger} + cp := c.Compile([]Workflow{wf}) + + assert.Len(t, cp.Workflows, 1) + assert.Equal(t, []string{"linux/arm64", "kvm"}, cp.Workflows[0].RunsOn) + + raw, err := json.Marshal(cp.Workflows[0]) + assert.NoError(t, err) + + var persisted map[string]any + assert.NoError(t, json.Unmarshal(raw, &persisted)) + assert.Equal(t, []any{"linux/arm64", "kvm"}, persisted["runsOn"]) +} + +func TestCompileWorkflow_LegacyWorkflowOmitsRunsOn(t *testing.T) { + wf := Workflow{ + Name: ".tangled/workflows/legacy.yml", + Engine: "microvm", + When: when, + } + + c := Compiler{Trigger: trigger} + cp := c.Compile([]Workflow{wf}) + + assert.Len(t, cp.Workflows, 1) + assert.Empty(t, cp.Workflows[0].RunsOn) + + raw, err := json.Marshal(cp.Workflows[0]) + assert.NoError(t, err) + + var persisted map[string]any + assert.NoError(t, json.Unmarshal(raw, &persisted)) + _, ok := persisted["runsOn"] + assert.False(t, ok, "legacy workflow JSON should not gain runsOn") +} + func TestCompileWorkflow_TriggerMismatch(t *testing.T) { wf := Workflow{ Name: ".tangled/workflows/mismatch.yml", diff --git a/workflow/def.go b/workflow/def.go index ddd9b7d0..055fc043 100644 --- a/workflow/def.go +++ b/workflow/def.go @@ -27,6 +27,7 @@ type ( Workflow struct { Name string `yaml:"-"` // name of the workflow file Engine string `yaml:"engine"` + RunsOn []string `yaml:"runs_on"` When []Constraint `yaml:"when"` CloneOpts CloneOpts `yaml:"clone"` Raw string `yaml:"-"` @@ -121,15 +122,15 @@ func (w *Workflow) Match(trigger tangled.Pipeline_TriggerMetadata, changedFiles } func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFiles []string) (bool, error) { - match := true - // manual triggers always pass this constraint if trigger.Manual != nil { return true, nil } // apply event constraints - match = match && c.MatchEvent(trigger.Kind) + if !c.MatchEvent(trigger.Kind) { + return false, nil + } // apply branch constraints for PRs if trigger.PullRequest != nil { @@ -137,7 +138,9 @@ func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFile if err != nil { return false, err } - match = match && matched + if !matched { + return false, nil + } } // apply ref constraints for pushes @@ -146,7 +149,9 @@ func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFile if err != nil { return false, err } - match = match && matched + if !matched { + return false, nil + } } // apply paths filter: if specified, at least one changed file must match @@ -155,10 +160,12 @@ func (c *Constraint) Match(trigger tangled.Pipeline_TriggerMetadata, changedFile if err != nil { return false, err } - match = match && matched + if !matched { + return false, nil + } } - return match, nil + return true, nil } // matchesAnyFile returns true if any file in files matches any of the glob patterns. diff --git a/workflow/def_test.go b/workflow/def_test.go index 6f246f9c..165115d8 100644 --- a/workflow/def_test.go +++ b/workflow/def_test.go @@ -23,6 +23,19 @@ when: assert.False(t, wf.CloneOpts.Skip, "Skip should default to false") } +func TestUnmarshalWorkflowWithRunsOnLabels(t *testing.T) { + yamlData := ` +engine: microvm +runs_on: [linux/arm64, kvm] +when: + - event: push` + + wf, err := FromFile("test.yml", []byte(yamlData)) + assert.NoError(t, err) + + assert.Equal(t, []string{"linux/arm64", "kvm"}, wf.RunsOn) +} + func TestUnmarshalCloneFalse(t *testing.T) { yamlData := ` when: -- 2.51.2