From afd329ee545e2cb628bc48eef8638df7fe4a5583 Mon Sep 17 00:00:00 2001 From: dawn Date: Fri, 12 Jun 2026 11:45:50 +0000 Subject: [PATCH] spindle,shuttle: microvm engine init spindle,shuttle,nix: add an alpine microvm image spindle/microvm: allow user defined binary caches in workflows shuttle,nix/microvm: get rid of the hacky nix config parsing / rendering, use nix directly so we can access module options spindle/engine: generalize scheduler out of microvm, make it work-conserving with aging and per-user fairness spindle/microvm: add resource budget limits and optional cgroup enforcement Signed-off-by: dawn --- .gitignore | 2 ++ Cargo.lock | 298 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 ++- buf.gen.yaml | 11 +++++++++++ buf.lock | 6 ++++++ buf.yaml | 5 +++++ docker-compose.yml | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------ flake.lock | 44 +++++++++++++++++++++++++++++++++++++++++--- flake.nix | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- go.mod | 35 ++++++++++++++++++++++++++++------- go.sum | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------ docs/DOCS.md | 348 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------- eventconsumer/consumer.go | 7 +------ eventconsumer/consumer_test.go | 6 +----- eventconsumer/source.go | 40 +++++++++++++++++++++++----------------- eventconsumer/upgrade_test.go | 1 - hostutil/hostutil.go | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ localinfra/Caddyfile | 10 ++++++++++ localinfra/appview.Dockerfile | 4 ++++ localinfra/knot.Dockerfile | 4 ++++ localinfra/knotmirror.Dockerfile | 5 +---- localinfra/readme.md | 13 ++++++++----- localinfra/spindle.Dockerfile | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nix/gomod2nix.toml | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------- nix/vm.nix | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++---- shuttle/Cargo.toml | 22 ++++++++++++++++++++++ shuttle/README.md | 1 + spindle/server.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------- appview/pipelines/logs.go | 13 ++++++++----- appview/pipelines/pipelines.go | 17 ++++++++++++++--- appview/state/knotstream.go | 2 +- appview/state/spindlestream.go | 2 +- appview/state/spindlestream_test.go | 3 +-- appview/state/streams.go | 2 -- blog/posts/spindle-microvm.md | 277 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/spindle-microvm-run/main.go | 26 ++++++++++++++++++++++++++ cmd/spindle-microvm-run/main_linux.go | 357 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/spindle-microvm-run/unsupported.go | 13 +++++++++++++ cmd/spindle/main.go | 25 ++++++++++++++++++++++--- knotmirror/hostutil/hostutil.go | 56 -------------------------------------------------------- knotmirror/xrpc/sync_request_crawl.go | 2 +- localinfra/scripts/init-accounts.sh | 1 + localinfra/scripts/prepare-spindle-images.sh | 32 ++++++++++++++++++++++++++++++++ nix/microvm/base.nix | 309 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nix/microvm/qemu.nix | 25 +++++++++++++++++++++++++ nix/microvm/spindle-vm.nix | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ nix/microvm/user-config.nix | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nix/modules/shuttle.nix | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nix/modules/spindle.nix | 288 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------ nix/pkgs/shuttle.nix | 23 +++++++++++++++++++++++ nix/pkgs/spindle-alpine-image.nix | 222 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nix/pkgs/spindle-nixos-image.nix | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/activation.rs | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/command.rs | 298 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/dns_proxy.rs | 255 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/exec.rs | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/host_proxy.rs | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/logging.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/main.rs | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/nix_config.rs | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/protocol.rs | 212 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/session.rs | 244 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/agentproto/protocol.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/agentproto/protocol_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/config/config.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------- spindle/db/db.go | 6 ++++++ spindle/db/nixos_toplevel_cache.go | 41 +++++++++++++++++++++++++++++++++++++++++ spindle/engine/engine.go | 36 +++++++++++++++++++++++++++++++----- spindle/engine/scheduler.go | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engine/scheduler_test.go | 200 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engine/slot.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engine/slot_test.go | 112 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/models/clone.go | 31 ++++++++++++++++++------------- spindle/models/clone_test.go | 10 +++++----- spindle/models/pipeline_env.go | 4 ++-- spindle/models/pipeline_env_test.go | 17 ++++++++--------- spindle/queue/queue.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- spindle/queue/queue_test.go | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/pipelines/ssh/tui.go | 2 +- shuttle/src/cache/mod.rs | 724 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/cache/read_proxy.rs | 27 +++++++++++++++++++++++++++ shuttle/src/cache/write_proxy.rs | 22 ++++++++++++++++++++++ shuttle/src/gen/file_descriptor_set.bin | 0 spindle/agentproto/gen/agent.pb.go | 1102 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/README.md | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/agent.go | 368 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/args.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/bench-boot.sh | 38 ++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/budget.go | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/cgroup.go | 267 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/cgroup_oom_test.go | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/cgroup_test.go | 39 +++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/dns_proxy.go | 380 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/dns_proxy_test.go | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/engine.go | 517 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/engine_test.go | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/image.go | 254 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/image_test.go | 127 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/models.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/models_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/netns_wrapper.sh.tmpl | 25 +++++++++++++++++++++++++ spindle/engines/microvm/networking.go | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/nixos_toplevel_cache.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/qemu.go | 676 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/read_cache_proxy.go | 427 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/read_cache_proxy_test.go | 171 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/runner.go | 24 ++++++++++++++++++++++++ spindle/engines/microvm/start-test-cache.sh | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/test-spindle-microvm.sh | 850 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/upload_cache_proxy.go | 192 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/upload_cache_proxy_test.go | 128 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/microvm/vm.go | 370 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ spindle/engines/nixery/engine.go | 28 ++++++++++++++++++++++++---- spindle/agentproto/spindle/agent/v1/agent.proto | 110 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ shuttle/src/gen/spindle/agent/v1/spindle.agent.v1.rs | 146 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 115 file(s) changed, 14089 insertion(s)(+), 388 deletion(s)(-) diff --git a/.gitignore b/.gitignore --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ *.db-* .bin/ appview/pages/static/* +spindle/spindle +spindle/spindle-microvm-run result !.gitkeep !appview/pages/static/topbar-search.js diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -117,6 +117,23 @@ ] [[package]] +name = "antlr4rust" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093d520274bfff7278d776f7ea12981a0a0a6f96db90964658e0f38fc6e9a6a6" +dependencies = [ + "better_any", + "bit-set", + "byteorder", + "lazy_static", + "murmur3", + "once_cell", + "parking_lot", + "typed-arena", + "uuid", +] + +[[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -270,6 +287,27 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "better_any" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4372b9543397a4b86050cc5e7ee36953edf4bac9518e8a774c2da694977fb6e4" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" [[package]] name = "bitflags" @@ -655,6 +693,24 @@ "jobserver", "libc", "shlex", +] + +[[package]] +name = "cel" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca1e5eda1b0f8476181bed1bfc9232a91d62ff0b9f1bc0e48afff3cbcb5b0b5c" +dependencies = [ + "antlr4rust", + "base64", + "chrono", + "lazy_static", + "nom", + "paste", + "regex", + "serde", + "serde_json", + "thiserror 1.0.69", ] [[package]] @@ -1281,6 +1337,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" @@ -2263,6 +2325,15 @@ ] [[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] name = "miette" version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2340,6 +2411,21 @@ ] [[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmur3" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a198f9589efc03f544388dfc4a19fe8af4323662b62f598b8dcfdac62c14771c" +dependencies = [ + "byteorder", +] + +[[package]] name = "murmurhash32" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2370,6 +2456,19 @@ "wasm-bindgen", "wasm-bindgen-futures", "web-time", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", ] [[package]] @@ -2445,6 +2544,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] [[package]] name = "ordered-float" @@ -2568,6 +2676,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] [[package]] name = "phf" @@ -2735,6 +2854,124 @@ "syn", "version_check", "yansi", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-protovalidate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3807965edce2730437ec19f99eae81064d152421c4c875a3e8f9e80cf2dd234" +dependencies = [ + "cel", + "chrono", + "fluent-uri", + "prost", + "prost-protovalidate-types", + "prost-reflect", + "prost-types", + "regex", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "prost-protovalidate-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb5c8151aa3da0ea9ba5d91b170c7763b6c7c7352c3643d57b2b28de29afde9" +dependencies = [ + "prost", + "prost-build", + "prost-reflect", + "prost-reflect-build", + "prost-types", + "thiserror 2.0.18", +] + +[[package]] +name = "prost-reflect" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590aa145fee8f7a26b5a6055365e7c5e89a5c1caae9869de76ec0ee73181a2f9" +dependencies = [ + "base64", + "prost", + "prost-reflect-derive", + "prost-types", + "serde", + "serde-value", +] + +[[package]] +name = "prost-reflect-build" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8214ae2c30bbac390db0134d08300e770ef89b6d4e5abf855e8d300eded87e28" +dependencies = [ + "prost-build", + "prost-reflect", +] + +[[package]] +name = "prost-reflect-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b6d90e29fa6c0d13c2c19ba5e4b3fb0efbf5975d27bcf4e260b7b15455bcabe" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", ] [[package]] @@ -3248,6 +3485,16 @@ ] [[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + +[[package]] name = "serde_bytes" version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3421,6 +3668,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shuttle" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "nix", + "once_cell", + "prost", + "prost-protovalidate", + "prost-reflect", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-vsock", + "tracing", + "tracing-subscriber", +] [[package]] name = "signal-hook-registry" @@ -3706,7 +3973,7 @@ dependencies = [ "fnv", "nom", - "ordered-float", + "ordered-float 5.3.0", "serde", "serde_json", ] @@ -4017,6 +4284,19 @@ ] [[package]] +name = "tokio-vsock" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b319ef9394889dab2e1b4f0085b45ba11d0c79dc9d1a9d1afc057d009d0f1c7" +dependencies = [ + "bytes", + "futures", + "libc", + "tokio", + "vsock", +] + +[[package]] name = "toml" version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4238,6 +4518,12 @@ ] [[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] name = "typeid" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4369,6 +4655,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsock" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205" +dependencies = [ + "libc", + "nix", +] [[package]] name = "walkdir" diff --git a/Cargo.toml b/Cargo.toml --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["bobbin/crates/*"] +members = ["bobbin/crates/*", "shuttle"] exclude = ["sites"] [workspace.package] @@ -14,6 +14,7 @@ "oppiliappan ", "Anirudh Oppiliappan ", "eti ", + "dawn ", ] [workspace.lints.rust] diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,11 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: . + opt: + - paths=import + - module=tangled.org/core + - local: protoc-gen-prost + out: shuttle/src/gen + opt: + - bytes=. diff --git a/buf.lock b/buf.lock new file mode 100644 --- /dev/null +++ b/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/bufbuild/protovalidate + commit: 50325440f8f24053b047484a6bf60b76 + digest: b5:74cb6f5c0853c3c10aafc701614194bbd63326bdb8ef4068214454b8894b03ba4113e04b3a33a8321cdf05336e37db4dc14a5e2495db8462566914f36086ba31 diff --git a/buf.yaml b/buf.yaml new file mode 100644 --- /dev/null +++ b/buf.yaml @@ -0,0 +1,5 @@ +version: v2 +modules: + - path: spindle/agentproto +deps: + - buf.build/bufbuild/protovalidate diff --git a/docker-compose.yml b/docker-compose.yml --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ PDS_URL: http://pds:3000 OWNER_USER: alice KNOT_HOSTNAME: knot.tngl.boltless.dev + SPINDLE_HOSTNAME: spindle.tngl.boltless.dev volumes: - ./localinfra/scripts/init-accounts.sh:/init.sh:ro - init-state:/shared @@ -85,7 +86,7 @@ JETSTREAM_WS_URL: wss://pds.tngl.boltless.dev/xrpc/com.atproto.sync.subscribeRepos volumes: - jetstream-data:/data - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: pds: condition: service_healthy @@ -113,7 +114,7 @@ - knot-data:/home/git - knot-ssh-keys:/etc/ssh/keys - init-state:/shared:ro - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:5555/"] interval: 2s @@ -131,6 +132,57 @@ condition: service_completed_successfully networks: [tngl] + spindle: + build: + context: . + dockerfile: localinfra/spindle.Dockerfile + restart: unless-stopped + environment: + SPINDLE_SERVER_HOSTNAME: spindle.tngl.boltless.dev + SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db + SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev + SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe + SPINDLE_SERVER_DEV: "true" + SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev + SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db + SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev + SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images + SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" + SPINDLE_S3_LOG_BUCKET: "" + devices: + - /dev/vsock:/dev/vsock + - /dev/kvm:/dev/kvm + - /dev/vhost-vsock:/dev/vhost-vsock + - /dev/net/tun:/dev/net/tun + cap_add: + - NET_ADMIN + - SYS_ADMIN + security_opt: + - label=disable + - seccomp=unconfined + volumes: + - spindle-data:/var/lib/spindle + - spindle-logs:/var/log/spindle + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s + depends_on: + plc: + condition: service_started + jetstream: + condition: service_started + init-accounts: + condition: service_completed_successfully + networks: [tngl] + knotmirror-tap: image: ghcr.io/bluesky-social/indigo/tap:sha-4f47add43060c27e8a37d9d76482ecddf001fcd8 # 0.1.10 restart: unless-stopped @@ -144,7 +196,7 @@ TAP_RESYNC_PARALLELISM: "10" TAP_RETRY_TIMEOUT: 60s volumes: - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: postgres: condition: service_started @@ -171,7 +223,7 @@ MIRROR_RESYNC_PARALLELISM: "4" volumes: - knotmirror-data:/data - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:7000/"] interval: 2s @@ -202,7 +254,8 @@ - ./blog/templates:/build/blog/templates:ro - ./blog/posts:/build/blog/posts:ro - ./appview/pages/static:/build/appview/pages/static - command: ["-i", "input.css", "-o", "appview/pages/static/tw.css", "--watch=always"] + command: + ["-i", "input.css", "-o", "appview/pages/static/tw.css", "--watch=always"] network_mode: none appview: @@ -226,7 +279,7 @@ - go-mod-cache:/go/mod - appview-data:/var/lib/appview - init-state:/shared:ro - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro depends_on: redis: condition: service_started @@ -267,6 +320,8 @@ knot-data: knot-ssh-keys: knotmirror-data: + spindle-data: + spindle-logs: init-state: go-cache: go-mod-cache: diff --git a/flake.lock b/flake.lock --- a/flake.lock +++ b/flake.lock @@ -117,14 +117,14 @@ "ibm-plex-mono-src": { "flake": false, "locked": { - "lastModified": 1731402384, + "lastModified": 1731402378, "narHash": "sha256-OwUmrPfEehLDz0fl2ChYLK8FQM2p0G1+EMrGsYEq+6g=", "type": "tarball", - "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + "url": "https://github.com/IBM/plex/releases/download/@ibm/plex-mono@1.1.0/ibm-plex-mono.zip" }, "original": { "type": "tarball", - "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + "url": "https://github.com/IBM/plex/releases/download/@ibm/plex-mono@1.1.0/ibm-plex-mono.zip" } }, "indigo": { @@ -181,6 +181,27 @@ "url": "https://cdn.jsdelivr.net/npm/mermaid@11.12.3/dist/mermaid.min.js" } }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1779970379, + "narHash": "sha256-ZHsxoYXXnfJtMVh1/yY+1Eh9hHcPBhE28Qvinauh+BQ=", + "owner": "microvm-nix", + "repo": "microvm.nix", + "rev": "0d49083ba2d7419b22908ac392777c16df9a032e", + "type": "github" + }, + "original": { + "owner": "microvm-nix", + "repo": "microvm.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1771848320, @@ -210,6 +231,7 @@ "inter-fonts-src": "inter-fonts-src", "lucide-src": "lucide-src", "mermaid-src": "mermaid-src", + "microvm": "microvm", "nixpkgs": "nixpkgs", "sqlite-lib-src": "sqlite-lib-src" } @@ -229,6 +251,22 @@ "ref": "nightly", "repo": "rust-analyzer", "type": "github" + } + }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1778940603, + "narHash": "sha256-voSM8dZNlaOWN3kbYFky+FNY6fFQOEw0xF+ZMpZKkCQ=", + "ref": "refs/heads/main", + "rev": "367dd227f539267eae2b62770b4c17b88ac8c1f1", + "revCount": 1265, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" } }, "sqlite-lib-src": { diff --git a/flake.nix b/flake.nix --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,10 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + microvm = { + url = "github:microvm-nix/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; fenix = { url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -69,6 +73,7 @@ ibm-plex-mono-src, actor-typeahead-src, mermaid-src, + microvm, ... }: let supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; @@ -84,13 +89,41 @@ root = ./.; fileset = fs.difference (fs.intersection (fs.gitTracked ./.) (fs.fileFilter (file: !(file.hasExt "nix")) ./.)) (fs.maybeMissing ./.jj); }; + rustSrc = let + fs = pkgs.lib.fileset; + in + fs.toSource { + root = ./.; + fileset = + fs.intersection + (fs.fromSource self.src) + (fs.unions [ + ./Cargo.toml + ./Cargo.lock + ./shuttle + ./bobbin + ]); + }; buildGoApplication = (self.callPackage "${gomod2nix}/builder" { gomod2nix = gomod2nix.legacyPackages.${pkgs.stdenv.hostPlatform.system}.gomod2nix; }).buildGoApplication; rustPlatform = pkgs.makeRustPlatform { - inherit (fenix.packages.${pkgs.system}.stable) rustc cargo; + inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo; }; + rustPlatformStatic = let + system = pkgs.stdenv.hostPlatform.system; + muslTarget = pkgs.pkgsStatic.stdenv.hostPlatform.rust.rustcTarget; + toolchain = fenix.packages.${system}.combine [ + fenix.packages.${system}.stable.cargo + fenix.packages.${system}.stable.rustc + fenix.packages.${system}.targets.${muslTarget}.stable.rust-std + ]; + in + pkgs.pkgsStatic.makeRustPlatform { + cargo = toolchain; + rustc = toolchain; + }; modules = ./nix/gomod2nix.toml; sqlite-lib = self.callPackage ./nix/pkgs/sqlite-lib.nix { inherit sqlite-lib-src; @@ -107,6 +140,13 @@ inherit (pkgs) pagefind; }; spindle = self.callPackage ./nix/pkgs/spindle.nix {}; + shuttle = self.callPackage ./nix/pkgs/shuttle.nix { + src = self.rustSrc; + }; + shuttle-static = self.callPackage ./nix/pkgs/shuttle.nix { + src = self.rustSrc; + rustPlatform = self.rustPlatformStatic; + }; knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {}; knot = self.callPackage ./nix/pkgs/knot.nix {}; dolly = self.callPackage ./nix/pkgs/dolly.nix {}; @@ -116,11 +156,12 @@ }); in { overlays.default = final: prev: { - inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly tap knotmirror bobbin; + inherit (mkPackageSet final) lexgen goat sqlite-lib spindle shuttle knot-unwrapped knot appview docs dolly tap knotmirror bobbin; }; packages = forAllSystems (system: let pkgs = nixpkgsFor.${system}; + linuxPkgs = nixpkgsFor."x86_64-linux"; packages = mkPackageSet pkgs; staticPackages = mkPackageSet pkgs.pkgsStatic; crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic; @@ -137,6 +178,8 @@ knot-unwrapped sqlite-lib docs + shuttle + shuttle-static dolly tap knotmirror @@ -189,6 +232,48 @@ # }; }; }; + + spindle-nixos-image = linuxPkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix { + nixosSystem = self.nixosConfigurations.spindle-nixos; + }; + spindle-nixos-image-tarball = linuxPkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out . + ''; + + spindle-alpine-image = let + branch = "3.24"; + version = "${branch}.0"; + arch = "x86_64"; + cdn = "https://dl-cdn.alpinelinux.org/alpine/v${branch}/releases/${arch}"; + + shuttle = (mkPackageSet linuxPkgs).shuttle-static; + in + linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix { + inherit arch shuttle; + repositories = [ + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/main" + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/community" + ]; + rootfs = linuxPkgs.fetchurl { + url = "${cdn}/alpine-minirootfs-${version}-${arch}.tar.gz"; + hash = "sha256-3poRwODn6clNs+2K97RQ6vwLE2h71+kZnVUFDyCqCok="; + }; + kernel = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/vmlinuz-virt"; + hash = "sha256-Hmv5Ancgx1w+0NeRcfIbV5HuQMqXldB8fG4E3F6irpA="; + }; + initramfs = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/initramfs-virt"; + hash = "sha256-ZCWGSaVMOYOmLz1Gwsf2RhYarMqk+tFVA6MMDWiHVJQ="; + }; + modloop = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/modloop-virt"; + hash = "sha256-p3yO7yU28k04iT01sOzhDmEYi+Yl7VZs5r3RYsWCBX0="; + }; + }; + spindle-alpine-image-tarball = linuxPkgs.runCommand "spindle-alpine-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-alpine-image} -h -czf $out . + ''; }); defaultPackage = forAllSystems (system: self.packages.${system}.appview); devShells = forAllSystems (system: let @@ -219,6 +304,15 @@ pkgs.redis pkgs.worker-build pkgs.cargo-generate + pkgs.qemu + pkgs.cdrkit + pkgs.parted + pkgs.buf + pkgs.protobuf + pkgs.protoc-gen-prost + pkgs.protoc-gen-prost-crate + pkgs.protoc-gen-prost-serde + pkgs.protoc-gen-go (fenix.packages.${system}.combine [ fenix.packages.${system}.stable.cargo fenix.packages.${system}.stable.rustc @@ -231,6 +325,10 @@ packages'.lexgen packages'.treefmt-wrapper packages'.tap + pkgs.e2fsprogs + pkgs.slirp4netns + pkgs.iproute2 + pkgs.util-linux ]; shellHook = '' mkdir -p appview/pages/static @@ -305,6 +403,24 @@ cd "$docsOut" exec ${pkgs.python3}/bin/python3 -m http.server 1414 ''); + }; + regenerate-proto = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "regenerate-proto"; + runtimeInputs = with pkgs; [git buf coreutils]; + text = '' + rootDir=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + cd "$rootDir" + echo ">>> regenerating protobuf files.." + buf generate + echo ">>> generating file descriptor set for shuttle..." + buf build -o shuttle/src/gen/file_descriptor_set.bin + echo ">>> done" + ''; + }) + + "/bin/regenerate-proto"; }; vm = let guestSystem = @@ -409,7 +525,27 @@ services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle; }; + nixosModules.shuttle = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/shuttle.nix]; + + services.tangled.shuttle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle; + }; + + nixosModules.spindle-nixos = import ./nix/microvm/spindle-vm.nix {inherit self microvm;} ./nix/microvm/qemu.nix; formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper); + + nixosConfigurations = let + spindleNixosBase = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [self.nixosModules.spindle-nixos]; + }; + in { + spindle-nixos = spindleNixosBase; + }; }; } diff --git a/go.mod b/go.mod --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ go 1.25.0 require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 + buf.build/go/protovalidate v1.2.0 github.com/Blank-Xu/sql-adapter v1.1.1 github.com/adrg/frontmatter v0.2.0 github.com/alecthomas/assert/v2 v2.11.0 @@ -27,9 +29,11 @@ github.com/charmbracelet/ssh v0.0.0-20250128164007-98fd5ae11894 github.com/charmbracelet/wish v1.4.7 github.com/cloudflare/cloudflare-go/v6 v6.7.0 + github.com/containerd/cgroups/v3 v3.1.3 github.com/cyphar/filepath-securejoin v0.4.1 github.com/dgraph-io/ristretto v0.2.0 github.com/did-method-plc/go-didplc v0.2.2 + github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 github.com/docker/docker v28.2.2+incompatible @@ -49,11 +53,14 @@ github.com/jackc/pgx/v5 v5.8.0 github.com/landlock-lsm/go-landlock v0.8.1 github.com/mattn/go-sqlite3 v1.14.34 + github.com/mdlayher/vsock v1.3.0 + github.com/miekg/dns v1.1.72 github.com/microcosm-cc/bluemonday v1.0.27 github.com/multiformats/go-multihash v0.2.3 github.com/openbao/openbao/api/v2 v2.3.0 github.com/posthog/posthog-go v1.5.5 github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/procfs v0.19.2 github.com/redis/go-redis/v9 v9.7.3 github.com/resend/resend-go/v3 v3.5.0 github.com/sethvargo/go-envconfig v1.1.0 @@ -67,16 +74,18 @@ github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab go.abhg.dev/goldmark/mermaid v0.6.0 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.51.0 golang.org/x/image v0.31.0 - golang.org/x/net v0.50.0 - golang.org/x/sync v0.19.0 - golang.org/x/sys v0.41.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.20.0 + golang.org/x/sys v0.45.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( + cel.dev/expr v0.25.1 // indirect dario.cat/mergo v1.0.1 // indirect github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -85,6 +94,7 @@ github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect github.com/alecthomas/repr v0.5.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect @@ -134,6 +144,7 @@ github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/termios v0.1.0 // indirect github.com/charmbracelet/x/windows v0.2.0 // indirect + github.com/cilium/ebpf v0.16.0 // indirect github.com/clipperhouse/displaywidth v0.9.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.5.0 // indirect @@ -141,9 +152,11 @@ github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/creack/pty v1.1.21 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -163,6 +176,7 @@ github.com/go-redis/cache/v9 v9.0.0 // indirect github.com/go-test/deep v1.1.1 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect @@ -170,6 +184,7 @@ github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/cel-go v0.28.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect @@ -219,10 +234,12 @@ github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mdlayher/socket v0.6.0 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -240,6 +257,7 @@ github.com/onsi/gomega v1.37.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -247,11 +265,11 @@ github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.19.2 // indirect github.com/puzpuzpuz/xsync/v4 v4.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect @@ -276,9 +294,12 @@ go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.12.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + golang.org/x/tools v0.44.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -23,6 +29,8 @@ github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= @@ -120,6 +128,8 @@ github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2 h1:63+EsT7kltod8g1eA0eNuvq1q9ANJWRdxlLeJjJDVYY= github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2/go.mod h1:VG/LeqLGNI3Ew7lsYixajnZGFfWPv144qbUddh+Oyag= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -183,6 +193,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= @@ -193,12 +205,16 @@ github.com/cloudflare/circl v1.6.2-0.20250618153321-aa837fd1539d/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cloudflare/cloudflare-go/v6 v6.7.0 h1:MP6Xy5WmsyrxgTxoLeq/vraqR0nbTtXoHhW4vAYc4SY= github.com/cloudflare/cloudflare-go/v6 v6.7.0/go.mod h1:Lj3MUqjvKctXRpdRhLQxZYRrNZHuRs0XYuH8JtQGyoI= +github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6dyLYXQ= +github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= @@ -221,6 +237,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/did-method-plc/go-didplc v0.2.2 h1:53HFhTT8NCAeFmZ6fdIZCf3PGDvj7A3cDjzOOEqn5XM= github.com/did-method-plc/go-didplc v0.2.2/go.mod h1:bKdJ21irnwNHgVLWWL32zUWqZueXYbJRUcxplZghByo= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e h1:SCnqm8SjSa0QqRxXbo5YY//S+OryeJioe17nK+iDZpg= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e/go.mod h1:o129ljs6alsIQTc8d6eweihqpmmrbxZ2g1jhgjhPykI= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= @@ -285,6 +305,8 @@ github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -301,6 +323,9 @@ github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= @@ -325,6 +350,8 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= +github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -465,6 +492,10 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -522,8 +553,16 @@ github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= +github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= +github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI= +github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -534,6 +573,8 @@ github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -606,6 +647,8 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= @@ -642,6 +685,8 @@ github.com/resend/resend-go/v3 v3.5.0/go.mod h1:iI7VA0NoGjWvsNii5iNC5Dy0llsI3HncXPejhniYzwE= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -770,6 +815,8 @@ go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -777,8 +824,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= @@ -793,6 +840,8 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -813,8 +862,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -822,8 +871,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -845,6 +894,7 @@ golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -855,8 +905,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -866,8 +916,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -878,8 +928,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -899,6 +949,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/docs/DOCS.md b/docs/DOCS.md --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -760,7 +760,8 @@ directory at the root of your repository, and are defined using YAML. -The fields are: +A workflow has a set of common fields that apply no matter +which engine you pick: - [Trigger](#trigger): A **required** field that defines when a workflow should be triggered. @@ -768,12 +769,14 @@ engine a workflow should run on. - [Clone options](#clone-options): An **optional** field that defines how the repository should be cloned. -- [Dependencies](#dependencies): An **optional** field that - allows you to list dependencies you may need. - [Environment](#environment): An **optional** field that allows you to define environment variables. - [Steps](#steps): An **optional** field that allows you to define what steps should run in the workflow. + +On top of these, each engine has its own options for things +like dependencies and images. See [Engines](#engines) for +the per-engine fields. ### Trigger @@ -853,12 +856,21 @@ search for packages on https://search.nixos.org, and there's a pretty good chance the package(s) you're looking for will be there. + See [Nixery engine](#nixery-engine). +- `microvm`: Runs the whole workflow inside its own + microVM. Has configuration features for NixOS images + that will let you enable services, do Docker-in-VM, etc. + See [microVM engine](#microvm-engine). Example: ```yaml engine: "nixery" ``` + +Each engine also adds its own workflow fields (dependencies, +images, services, and so on). These are documented under +[Engines](#engines). ### Clone options @@ -890,41 +902,6 @@ depth: 1 submodules: false ``` - -### Dependencies - -Usually when you're running a workflow, you'll need -additional dependencies. The `dependencies` field lets you -define which dependencies to get, and from where. It's a -key-value map, with the key being the registry to fetch -dependencies from, and the value being the list of -dependencies to fetch. - -The registry URL syntax can be found [on the nix -manual](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-registry-add). - -Say you want to fetch Node.js and Go from `nixpkgs`, and a -package called `my_pkg` you've made from your own registry -at your repository at -`https://tangled.org/@example.com/my_pkg`. You can define -those dependencies like so: - -```yaml -dependencies: - # nixpkgs - nixpkgs: - - nodejs - - go - # unstable - nixpkgs/nixpkgs-unstable: - - bun - # custom registry - git+https://tangled.org/@example.com/my_pkg: - - my_pkg -``` - -Now these dependencies are available to use in your -workflow! ### Environment @@ -992,9 +969,9 @@ - `command`: This field allows you to define a command to run in that step. The step is run in a Bash shell, and the logs from the command will be visible in the pipelines - page on the Tangled website. The - [dependencies](#dependencies) you added will be available - to use here. + page on the Tangled website. Any dependencies you added in + your engine's section (see [Engines](#engines)) will be + available to use here. - `environment`: Similar to the global [environment](#environment) config, this **optional** field is a key-value map that allows you to set @@ -1018,7 +995,49 @@ NODE_ENV: "production" ``` -### Complete workflow +## Engines + +The common fields above apply to every workflow. Each engine +then adds its own fields on top. Pick an engine with the +[`engine`](#engine) field and use the matching section below. + +### Nixery engine + +#### Dependencies + +When you're running a workflow you'll usually need additional +dependencies. The `dependencies` field lets you define which +dependencies to get, and from where. It's a key-value map, +with the key being the registry to fetch dependencies from, +and the value being the list of dependencies to fetch. + +The registry URL syntax can be found [on the nix +manual](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-registry-add). + +Say you want to fetch Node.js and Go from `nixpkgs`, and a +package called `my_pkg` you've made from your own registry +at your repository at +`https://tangled.org/@example.com/my_pkg`. You can define +those dependencies like so: + +```yaml +dependencies: + # nixpkgs + nixpkgs: + - nodejs + - go + # unstable + nixpkgs/nixpkgs-unstable: + - bun + # custom registry + git+https://tangled.org/@example.com/my_pkg: + - my_pkg +``` + +Now these dependencies are available to use in your +workflow! + +#### Complete nixery workflow ```yaml # .tangled/workflows/build.yml @@ -1068,12 +1087,116 @@ the one [Tangled uses to build the project](https://tangled.org/@tangled.org/core/blob/master/.tangled/workflows/build.yml). +### microVM engine + +#### Image + +A workflow picks the image to boot with the top-level `image` +field: + +```yaml +engine: microvm +image: nixos +``` + +There are two flavours of images: + +- **NixOS images** (e.g. `nixos`): the whole guest is built + with Nix, so you can configure it from the workflow file + itself. The `dependencies`, `services`, `virtualisation`, + `registry` and `caches` fields below are all understood + here, and the guest builds and activates that configuration + before any of your steps run. +- **Non-NixOS images** (e.g. `alpine`): there's no NixOS to + configure, so the workflow-level config fields above have + no effect. You still get a full machine to run steps in. + +The available image names depend on what the spindle operator +has installed. `nixos` and `alpine` are examples. If `image` +is omitted, the spindle's configured default image is used. + +#### Dependencies + +On the microVM engine, `dependencies` is a flat list of +packages that get added to the guest's `PATH` (via +`environment.systemPackages`). This field only applies to +**NixOS images**, for other images you can use the package +manager included in a step. + +A bare name like `go` is looked up in nixpkgs. You can also +point at any flake with the `flakeref#attr` syntax, so +`github:nixos/nixpkgs#hello` pulls `hello` straight out of +that flake. + +```yaml +dependencies: + - go + - github:nixos/nixpkgs#hello +``` + +#### Registry + +The `registry` field remaps flake references, the same way +`nix registry` does. This lets you pin or alias the flakes +used by `dependencies`. + +For example, pin `nixpkgs` to `nixos-unstable` so that the +bare `go` above resolves from unstable, and alias your own +flake so you can use `myflake#tool` in `dependencies`: + +```yaml +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + myflake: github:me/x +``` + +#### Caches + +The `caches` field is a map of Nix binary cache URL to its +trusted public key. These are fed into the spindle's read +proxy, so the guest can substitute prebuilt paths from them +instead of building everything from scratch. + +```yaml +caches: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" +``` + +#### Services and virtualisation + +The `services` and `virtualisation` fields are passed straight +through to NixOS. Anything you could write under +`services.*` or `virtualisation.*` in a NixOS configuration, +you can write here, and it's brought up before any of your +steps run. + +As a convenience, `true` works as shorthand for +`.enable = true` anywhere an `enable` option exists (e.g. +`virtualisation.docker: true`). + +```yaml +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true +``` + ## Self-hosting guide ### Prerequisites - Go -- Docker (the only supported backend currently) +- For the **nixery** engine: Docker (or Podman with Docker + compatibility enabled). +- For the **microVM** engine: a Linux host with KVM, plus the + microVM host dependencies described in [Running microVM + workflows](#running-microvm-workflows). ### Configuration @@ -1089,6 +1212,57 @@ - `SPINDLE_SERVER_DOCKER_SOCKET`: Path to Docker socket to expose to invoked Spindle containers (default: `""`). - `SPINDLE_PIPELINES_NIXERY`: The Nixery URL (default: `"nixery.tangled.sh"`). - `SPINDLE_PIPELINES_WORKFLOW_TIMEOUT`: The default workflow timeout (default: `"5m"`). + +For the microVM engine, the following are also available +(prefix `SPINDLE_MICROVM_PIPELINES_`): + +- `SPINDLE_MICROVM_PIPELINES_IMAGE_DIR`: Directory containing + microVM images (**required** to use the engine). See + [Running microVM workflows](#running-microvm-workflows). +- `SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE`: Image used when a + workflow doesn't set `image` (default: `"nixos-x86_64"`). +- `SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR`: Where per-workflow + temporary disks are created (default: the system temp dir). +- `SPINDLE_MICROVM_PIPELINES_ENABLE_KVM`: Use KVM hardware + acceleration (default: `true`). Without KVM, guests fall + back to slow software emulation. +- `SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT`: Default + workflow timeout (default: `"5m"`). + +Optional resource limits (a value of `0` disables that +limit). The limits cap usage across all running microVM +workflows: + +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB` +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS` +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB` + +Optional cgroup enforcement: + +- `SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS`: Place each + workflow's QEMU and slirp4netns in a per-workflow cgroup= + (default: `false`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT`: Parent cgroup; + `self` resolves the spindle service's own cgroup (default: + `"self"`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX`: Max processes + per workflow cgroup (default: `4096`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB`: Max swap + per workflow cgroup (default: `0`, no swap). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB`: + Memory protected for spindle itself so it isn't OOM-killed + before the workflows (default: `512`). + +To push paths built inside microVMs back to a shared Nix +cache (and read from it), configure the cache (prefix +`SPINDLE_NIX_CACHE_`): + +- `SPINDLE_NIX_CACHE_READ_URLS`: Comma-separated binary cache + URLs the guest reads from. +- `SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS`: Comma-separated + trusted public keys for those caches. +- `SPINDLE_NIX_CACHE_UPLOAD_URL`: Cache URL that paths built + in the guest are uploaded to. ### Running spindle @@ -1122,6 +1296,70 @@ Spindle will now start, connect to the Jetstream server, and begin processing pipelines. +### Running microVM workflows + +The microVM engine needs a few extra things on the host, and +it needs images to boot. + +#### Host dependencies + +microVM workflows depend on a handful of host tools and +devices. spindle checks for the ones an image needs right +before it launches, so a missing dependency surfaces as a +clear error. You'll need: + +- `qemu`: the runner. The QEMU binary for the image's arch + must be present (e.g. `qemu-system-x86_64`). +- `mkfs.ext4` (from `e2fsprogs`): to format the per-workflow + writable volumes. +- [`slirp4netns`](https://github.com/rootless-containers/slirp4netns#install), + `ip` (from `iproute2`), `mount` and `unshare` (from `util-linux`): + used to sandbox guest networking. +- `/dev/kvm`: for hardware acceleration (unless you disable + KVM with `SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=false`). +- `/dev/vhost-vsock`: the guest agent talks to spindle over + vsock. + +On NixOS, the [spindle +module](https://tangled.org/tangled.org/core/blob/master/nix/modules/spindle.nix) +puts `qemu`, `e2fsprogs`, `slirp4netns`, `iproute2` and +`util-linux` on the service's `PATH` for you. + +#### Building images + +Images are built with Nix. The flake exposes packages for the +two stock images (use the `-tarball` prefixed ones for a gzipped +tarball you can copy to another host): + +```shell +# a NixOS image +nix build .#spindle-nixos-image +# an Alpine image +nix build .#spindle-alpine-image +``` + +#### Installing images + +Spindle looks for images in +`SPINDLE_MICROVM_PIPELINES_IMAGE_DIR`. An image is resolved by +the name a workflow puts in its `image` field, matched +literally against what's on disk: + +1. a directory `/` containing a `spec.json` (next to the + kernel/initrd/store-disk), or +2. a flat `.json` self-contained spec. + +Resolution depends only on the name and what's on disk, never +on the host doing the resolving, so the same workflow resolves +to the same image on every spindle. If you keep multiple +arches side by side, you can name them `-` (e.g. +`nixos-x86_64`, `alpine-aarch64`); the suffix is just part of +the name. To make a name like `nixos` work if you are hosting +multiple arches, you can use symlinks. + +On NixOS, you'll most likely want to use `systemd.tmpfiles.rules` +to set these up declaratively. + ## Architecture Spindle is a small CI runner service. Here's a high-level overview of how it operates: @@ -1135,16 +1373,24 @@ - The spindle engine then handles execution of the pipeline, with results and logs beamed on the spindle event stream over WebSocket -### The engine +### The engines -At present, the only supported backend is Docker (and Podman, if Docker -compatibility is enabled, so that `/run/docker.sock` is created). spindle -executes each step in the pipeline in a fresh container, with state persisted -across steps within the `/tangled/workspace` directory. +Spindle has two execution backends, picked per-workflow with +the [`engine`](#engine) field: -The base image for the container is constructed on the fly using -[Nixery](https://nixery.dev), which is handy for caching layers for frequently -used packages. +- **nixery**: executes each step in a fresh Docker container + (Podman works too, if Docker compatibility is enabled so + that `/run/docker.sock` is created), with state persisted + across steps within the `/tangled/workspace` directory. The + base image for the container is constructed on the fly using + [Nixery](https://nixery.dev), which is/rhandy for caching + layers for frequently used packages. +- **microvm**: runs the whole workflow inside its own + microVM, supporting different images, with extra + configuration for NixOS images (e.g. services in workflow file) + See the [engine + README](https://tangled.org/tangled.org/core/blob/master/spindle/engines/microvm/README.md) + for the architecture in depth. The pipeline manifest is [specified here](https://docs.tangled.org/spindles.html#pipelines). diff --git a/eventconsumer/consumer.go b/eventconsumer/consumer.go --- a/eventconsumer/consumer.go +++ b/eventconsumer/consumer.go @@ -5,7 +5,6 @@ "encoding/json" "log/slog" "net/http" - "net/url" "sync" "time" @@ -29,7 +28,6 @@ QueueSize int Logger *slog.Logger CursorStore cursor.Store - URLFunc func(Source, int64) (*url.URL, error) Dialer *websocket.Dialer RequestHeader http.Header @@ -92,9 +90,6 @@ } if cfg.CursorStore == nil { cfg.CursorStore = &cursor.MemoryStore{} - } - if cfg.URLFunc == nil { - cfg.URLFunc = DefaultURL(false) } dialer := cfg.Dialer if dialer == nil { @@ -263,7 +258,7 @@ func (c *Consumer) runConnection(ctx context.Context, source Source) error { cursor := c.cfg.CursorStore.Get(source.Key()) - u, err := c.cfg.URLFunc(source, cursor) + u, err := source.URL(cursor) if err != nil { return err } diff --git a/eventconsumer/consumer_test.go b/eventconsumer/consumer_test.go --- a/eventconsumer/consumer_test.go +++ b/eventconsumer/consumer_test.go @@ -69,7 +69,7 @@ srv := httptest.NewServer(mux) t.Cleanup(srv.Close) addr := strings.TrimPrefix(srv.URL, "http://") - return Source{Kind: "test", Host: addr}, &n + return Source{Kind: "test", Host: addr, NoTLS: true}, &n } func TestConsumer_DrainAdvancesCursor(t *testing.T) { @@ -95,7 +95,6 @@ QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -158,7 +157,6 @@ QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -203,7 +201,6 @@ QueueSize: 8, ConnectionTimeout: 2 * time.Second, CursorStore: &cursor.MemoryStore{}, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -249,7 +246,6 @@ QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) diff --git a/eventconsumer/source.go b/eventconsumer/source.go --- a/eventconsumer/source.go +++ b/eventconsumer/source.go @@ -5,6 +5,7 @@ "strconv" "tangled.org/core/eventconsumer/cursor" + "tangled.org/core/hostutil" ) type Kind string @@ -15,12 +16,19 @@ ) type Source struct { - Kind Kind - Host string + Kind Kind + Host string + NoTLS bool // use TLS by default } -func NewKnotSource(host string) Source { return Source{Kind: KindKnot, Host: host} } -func NewSpindleSource(host string) Source { return Source{Kind: KindSpindle, Host: host} } +func NewKnotSource(host string) Source { + host, noTLS, _ := hostutil.ParseHostname(host) + return Source{Kind: KindKnot, Host: host, NoTLS: noTLS} +} +func NewSpindleSource(host string) Source { + host, noTLS, _ := hostutil.ParseHostname(host) + return Source{Kind: KindSpindle, Host: host, NoTLS: noTLS} +} func (s Source) Key() string { return string(s.Kind) + ":" + s.Host } @@ -33,21 +41,19 @@ } } -func DefaultURL(dev bool) func(Source, int64) (*url.URL, error) { +func (s Source) URL(cursor int64) (*url.URL, error) { scheme := "wss" - if dev { + if s.NoTLS { scheme = "ws" } - return func(s Source, cursor int64) (*url.URL, error) { - u, err := url.Parse(scheme + "://" + s.Host + "/events") - if err != nil { - return nil, err - } - if cursor != 0 { - q := url.Values{} - q.Add("cursor", strconv.FormatInt(cursor, 10)) - u.RawQuery = q.Encode() - } - return u, nil + u, err := url.Parse(scheme + "://" + s.Host + "/events") + if err != nil { + return nil, err } + if cursor != 0 { + q := url.Values{} + q.Add("cursor", strconv.FormatInt(cursor, 10)) + u.RawQuery = q.Encode() + } + return u, nil } diff --git a/eventconsumer/upgrade_test.go b/eventconsumer/upgrade_test.go --- a/eventconsumer/upgrade_test.go +++ b/eventconsumer/upgrade_test.go @@ -39,7 +39,6 @@ QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) diff --git a/hostutil/hostutil.go b/hostutil/hostutil.go new file mode 100644 --- /dev/null +++ b/hostutil/hostutil.go @@ -0,0 +1,80 @@ +package hostutil + +import ( + "fmt" + "net/url" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func ParseHostname(raw string) (hostname string, noSSL bool, err error) { + // handle case of bare hostname + if !strings.Contains(raw, "://") { + if strings.HasPrefix(raw, "localhost:") { + raw = "http://" + raw + } else { + raw = "https://" + raw + } + } + + u, err := url.Parse(raw) + if err != nil { + return "", false, fmt.Errorf("not a valid host URL: %w", err) + } + + switch u.Scheme { + case "https", "wss": + noSSL = false + case "http", "ws": + noSSL = true + default: + return "", false, fmt.Errorf("unsupported URL scheme: %s", u.Scheme) + } + + // 'localhost' (exact string) is allowed *with* a required port number; SSL is optional + if u.Hostname() == "localhost" { + if u.Port() == "" || !strings.HasPrefix(u.Host, "localhost:") { + return "", false, fmt.Errorf("port number is required for localhost") + } + return u.Host, noSSL, nil + } + + // port numbers not allowed otherwise + if u.Port() != "" { + return "", false, fmt.Errorf("port number not allowed for non-local names") + } + + // check it is a real hostname (eg, not IP address or single-word alias) + h, err := syntax.ParseHandle(u.Host) + if err != nil { + return "", false, fmt.Errorf("not a public hostname") + } + + // lower-case in response + return h.Normalize().String(), noSSL, nil +} + +func EnsureHttpScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "http://" + hostname, nil + } else { + return "https://" + hostname, nil + } +} + +func EnsureWsScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "ws://" + hostname, nil + } else { + return "wss://" + hostname, nil + } +} diff --git a/localinfra/Caddyfile b/localinfra/Caddyfile --- a/localinfra/Caddyfile +++ b/localinfra/Caddyfile @@ -37,6 +37,16 @@ reverse_proxy knot:5555 } +# spindle +http://spindle.tngl.boltless.dev { + reverse_proxy spindle:6555 +} + +spindle.tngl.boltless.dev { + tls internal + reverse_proxy spindle:6555 +} + # knotmirror mirror.tngl.boltless.dev { tls internal diff --git a/localinfra/appview.Dockerfile b/localinfra/appview.Dockerfile --- a/localinfra/appview.Dockerfile +++ b/localinfra/appview.Dockerfile @@ -41,6 +41,10 @@ [ -r /shared/label-defaults ] && export TANGLED_LABEL_DEFAULTS="$(cat /shared/label-defaults)" [ -r /shared/label-gfi ] && export TANGLED_LABEL_GFI="$(cat /shared/label-gfi)" +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + exec air -c /src/.air/appview.toml EOF RUN chmod +x /usr/local/bin/appview-entrypoint.sh diff --git a/localinfra/knot.Dockerfile b/localinfra/knot.Dockerfile --- a/localinfra/knot.Dockerfile +++ b/localinfra/knot.Dockerfile @@ -84,6 +84,10 @@ [ -f /etc/ssh/keys/ssh_host_ecdsa_key ] || ssh-keygen -t ecdsa -f /etc/ssh/keys/ssh_host_ecdsa_key -q -N "" [ -f /etc/ssh/keys/ssh_host_ed25519_key ] || ssh-keygen -t ed25519 -f /etc/ssh/keys/ssh_host_ed25519_key -q -N "" +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + /usr/sbin/sshd -D -e & exec su-exec git /usr/local/bin/knot server EOF diff --git a/localinfra/knotmirror.Dockerfile b/localinfra/knotmirror.Dockerfile --- a/localinfra/knotmirror.Dockerfile +++ b/localinfra/knotmirror.Dockerfile @@ -14,13 +14,10 @@ RUN apk add --no-cache git tini ca-certificates -# Trust dev CA in the system bundle so git/curl/openssl all accept caddy certs. -COPY localinfra/certs/root.crt /usr/local/share/ca-certificates/caddy.crt -RUN update-ca-certificates COPY --from=build /knotmirror /usr/local/bin/knotmirror EXPOSE 7000 ENTRYPOINT ["/sbin/tini", "--"] -CMD ["/usr/local/bin/knotmirror", "serve"] +CMD ["sh", "-c", "if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then update-ca-certificates; fi && exec /usr/local/bin/knotmirror serve"] diff --git a/localinfra/readme.md b/localinfra/readme.md --- a/localinfra/readme.md +++ b/localinfra/readme.md @@ -14,12 +14,10 @@ - atproto_pds () - jetstream () - knot () +- spindle () - knotmirror () - appview () (live reloading) - caddy reverse proxy - -> [!NOTE] -> Spindle is not included yet. ## Setup @@ -44,5 +42,10 @@ ``` - Depending on your browser you may have to import the certificate into your browser profiles too as some have their own certs do not use your system ones 3. run `./localinfra/scripts/appview-static-files.sh` -4. `docker compose up` -5. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. +4. Prepare the spindle microVM images: + ```bash + ./localinfra/scripts/prepare-spindle-images.sh + ``` + This writes the image directory under `out/localinfra-spindle-images`. +5. `docker compose up` +6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. diff --git a/localinfra/spindle.Dockerfile b/localinfra/spindle.Dockerfile new file mode 100644 --- /dev/null +++ b/localinfra/spindle.Dockerfile @@ -0,0 +1,66 @@ +# Development only. Not for production use. + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git build-base sqlite-dev + +ENV CGO_ENABLED=1 +ENV GOCACHE=/go/cache +ENV GOMODCACHE=/go/mod + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/cache \ + --mount=type=cache,target=/go/mod \ + go mod download + +COPY . . +RUN --mount=type=cache,target=/go/cache \ + --mount=type=cache,target=/go/mod \ + go build -tags libsqlite3 -o /out/spindle ./cmd/spindle && \ + go build -tags libsqlite3 -o /out/spindle-microvm-run ./cmd/spindle-microvm-run + +FROM alpine:3.20 + +RUN apk add --no-cache \ + bash \ + ca-certificates \ + e2fsprogs \ + git \ + iproute2 \ + qemu-system-x86_64 \ + shadow \ + slirp4netns \ + sqlite-libs \ + tini \ + util-linux + + +COPY --from=builder /out/spindle /usr/local/bin/spindle +COPY --from=builder /out/spindle-microvm-run /usr/local/bin/spindle-microvm-run +RUN chmod 0755 /usr/local/bin/spindle /usr/local/bin/spindle-microvm-run + +COPY <<'EOF' /usr/local/bin/spindle-entrypoint.sh +#!/bin/sh +set -eu + +[ -z "${SPINDLE_SERVER_OWNER:-}" ] && [ -r /shared/owner-did ] && \ + export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)" +: "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}" + +mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle + +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + +exec /usr/local/bin/spindle run +EOF +RUN chmod +x /usr/local/bin/spindle-entrypoint.sh + +VOLUME /var/lib/spindle +EXPOSE 6555 + +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/usr/local/bin/spindle-entrypoint.sh"] diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -1,6 +1,15 @@ schema = 3 [mod] + [mod."buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go"] + version = "v1.36.11-20260415201107-50325440f8f2.1" + hash = "sha256-oyhP92KT9a++f5riA9sI1myID7MR39AzHuT3cKx5hgg=" + [mod."buf.build/go/protovalidate"] + version = "v1.2.0" + hash = "sha256-4+42DSj7LAdJyPPxif3b5QRH5MCjedudgylTHBibZlQ=" + [mod."cel.dev/expr"] + version = "v0.25.1" + hash = "sha256-TEdMxFUPK7IZuCXMufwCkbN+ZZIXSQclljIybFZcByo=" [mod."dario.cat/mergo"] version = "v1.0.1" hash = "sha256-wcG6+x0k6KzOSlaPA+1RFxa06/RIAePJTAjjuhLbImw=" @@ -38,6 +47,9 @@ [mod."github.com/anmitsu/go-shlex"] version = "v0.0.0-20200514113438-38f4b401e2be" hash = "sha256-L3Ak4X2z7WXq7vMKuiHCOJ29nlpajUQ08Sfb9T0yP54=" + [mod."github.com/antlr4-go/antlr/v4"] + version = "v4.13.1" + hash = "sha256-beAuxHNRUuhzcSJUh/8ztVf1zCUiaT72fg2Jvx0AuNQ=" [mod."github.com/avast/retry-go/v4"] version = "v4.6.1" hash = "sha256-PeZc8k4rDV64+k8nZt/oy1YNVbLevltXP3ZD1jf6Z6k=" @@ -244,6 +256,9 @@ [mod."github.com/charmbracelet/x/windows"] version = "v0.2.0" hash = "sha256-pDAd1E5w66E/d3vuTyzgnW+W/KegZ2sxQQMfoEn7S1A=" + [mod."github.com/cilium/ebpf"] + version = "v0.16.0" + hash = "sha256-xACuieGmiUUjoTT/9MpvPBNexp98S/AZbLxm5f9nqDk=" [mod."github.com/clipperhouse/displaywidth"] version = "v0.9.0" hash = "sha256-9CNyTZPSncKQ7Y0my9DR4WYXDjtDHYNL512D691WDAM=" @@ -259,6 +274,9 @@ [mod."github.com/cloudflare/cloudflare-go/v6"] version = "v6.7.0" hash = "sha256-ycQpx1II/JgBgrCRwY5qiVKStGv5wuCANy1091sJ5Zw=" + [mod."github.com/containerd/cgroups/v3"] + version = "v3.1.3" + hash = "sha256-1a5heWXIzME7iMu2L35OBiAOi2Z/gnpg2fjvP6On9sM=" [mod."github.com/containerd/errdefs"] version = "v1.0.0" hash = "sha256-wMZGoeqvRhuovYCJx0Js4P3qFCNTZ/6Atea/kNYoPMI=" @@ -268,6 +286,9 @@ [mod."github.com/containerd/log"] version = "v0.1.0" hash = "sha256-vuE6Mie2gSxiN3jTKTZovjcbdBd1YEExb7IBe3GM+9s=" + [mod."github.com/coreos/go-systemd/v22"] + version = "v22.5.0" + hash = "sha256-E2zXikbmIQImghstLUWuey1YgA0Folu3F+fi5k4hCxA=" [mod."github.com/creack/pty"] version = "v1.1.21" hash = "sha256-pjGw6wQlrVhN65XaIxZueNJqnXThGu00u24rKOLzxS0=" @@ -286,6 +307,12 @@ [mod."github.com/did-method-plc/go-didplc"] version = "v0.2.2" hash = "sha256-TF5vdW1U2q5F23ELmbqhdvpNgDQjBs9i/CWJlzqWNKs=" + [mod."github.com/digitalocean/go-libvirt"] + version = "v0.0.0-20220804181439-8648fbde413e" + hash = "sha256-xgRZCefeUNM76M9ht5FCgCf4dnwAdeP/r+FZlJ2gmeY=" + [mod."github.com/digitalocean/go-qemu"] + version = "v0.0.0-20250212194115-ee9b0668d242" + hash = "sha256-AzY84aq9CqGqYwmGkKoi4PX08E/hRQC46mrZwKSMwXE=" [mod."github.com/distribution/reference"] version = "v0.6.0" hash = "sha256-gr4tL+qz4jKyAtl8LINcxMSanztdt+pybj1T+2ulQv4=" @@ -368,6 +395,9 @@ [mod."github.com/goccy/go-json"] version = "v0.10.5" hash = "sha256-/EtlGihP0/7oInzMC5E0InZ4b5Ad3s4xOpqotloi3xw=" + [mod."github.com/godbus/dbus/v5"] + version = "v5.1.0" + hash = "sha256-xOCMJpQK3KTmHTPn/CdqI4j0eENCtMmJDgAIoYqYOEY=" [mod."github.com/gogo/protobuf"] version = "v1.3.2" hash = "sha256-pogILFrrk+cAtb0ulqn9+gRZJ7sGnnLLdtqITvxvG6c=" @@ -389,6 +419,9 @@ [mod."github.com/golang/snappy"] version = "v0.0.4" hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA=" + [mod."github.com/google/cel-go"] + version = "v0.28.0" + hash = "sha256-86MSoJX3fovcAWir41Z34nM0HcaDh0yRA+lIiVA1gEM=" [mod."github.com/google/go-querystring"] version = "v1.1.0" hash = "sha256-itsKgKghuX26czU79cK6C2n+lc27jm5Dw1XbIRgwZJY=" @@ -569,9 +602,18 @@ [mod."github.com/mattn/go-sqlite3"] version = "v1.14.34" hash = "sha256-PGOevTQb3gNdGu4H2bhvXJlntkMkDt+UhuiK1IpuorQ=" + [mod."github.com/mdlayher/socket"] + version = "v0.6.0" + hash = "sha256-Qpwu6m0PqySAGxSkHfME00YLO+lhBc825EGqASDpob8=" + [mod."github.com/mdlayher/vsock"] + version = "v1.3.0" + hash = "sha256-rkN/QRzs5hZ3idZ131yrNEkwfe+cyRkAizumJMQmFLQ=" [mod."github.com/microcosm-cc/bluemonday"] version = "v1.0.27" hash = "sha256-EZSya9FLPQ83CL7N2cZy21fdS35hViTkiMK5f3op8Es=" + [mod."github.com/miekg/dns"] + version = "v1.1.72" + hash = "sha256-i5TwAlGQjz6lF2SNc7MWuTLmxR/7f2Uomco3V0rKaRU=" [mod."github.com/minio/sha256-simd"] version = "v1.0.1" hash = "sha256-4hfGDIQaWq8fvtGzHDhoK9v2IocXnJY7OAL6saMJbmA=" @@ -584,6 +626,9 @@ [mod."github.com/moby/sys/atomicwriter"] version = "v0.1.0" hash = "sha256-i46GNrsICnJ0AYkN+ocbVZ2GNTQVEsrVX5WcjKzjtBM=" + [mod."github.com/moby/sys/userns"] + version = "v0.1.0" + hash = "sha256-zwXKyEZIH/FZjSVuSGmtwThDxPutj1pY+N6Ae6oVPuc=" [mod."github.com/moby/term"] version = "v0.5.2" hash = "sha256-/G20jUZKx36ktmPU/nEw/gX7kRTl1Dbu7zvNBYNt4xU=" @@ -641,6 +686,9 @@ [mod."github.com/opencontainers/image-spec"] version = "v1.1.1" hash = "sha256-bxBjtl+6846Ed3QHwdssOrNvlHV6b+Dn17zPISSQGP8=" + [mod."github.com/opencontainers/runtime-spec"] + version = "v1.3.0" + hash = "sha256-B2QF7FlUYZDL9eNA0+JD7WasnBryMXNIDbdSGS4MMG4=" [mod."github.com/opentracing/opentracing-go"] version = "v1.2.1-0.20220228012449-10b1cf09e00b" hash = "sha256-77oWcDviIoGWHVAotbgmGRpLGpH5AUy+pM15pl3vRrw=" @@ -693,6 +741,9 @@ [mod."github.com/sethvargo/go-envconfig"] version = "v1.1.0" hash = "sha256-WelRHfyZG9hrA4fbQcfBawb2ZXBQNT1ourEYHzQdZ4w=" + [mod."github.com/sirupsen/logrus"] + version = "v1.9.3" + hash = "sha256-EnxsWdEUPYid+aZ9H4/iMTs1XMvCLbXZRDyvj89Ebms=" [mod."github.com/spaolacci/murmur3"] version = "v1.1.0" hash = "sha256-RWD4PPrlAsZZ8Xy356MBxpj+/NZI7w2XOU14Ob7/Y9M=" @@ -793,32 +844,44 @@ version = "v2.4.3" hash = "sha256-WqfrOUQFvfuORgl1yyVOcsEXU/vwWQHkcVWx3vCxvaw=" [mod."golang.org/x/crypto"] - version = "v0.48.0" - hash = "sha256-uBIGGSGmWWklRxX6XTOqUECzz165UFY9Y99Ka3pLKAw=" + version = "v0.51.0" + hash = "sha256-/R74sc1mcOaOuBeXRQzrXrHAgA5VhNWc6SfQJaxb17U=" [mod."golang.org/x/exp"] version = "v0.0.0-20260112195511-716be5621a96" hash = "sha256-rWqwXzLvvhcI/ZkOQMqCXMKI5FAuHd9YNoKTXujmboA=" [mod."golang.org/x/image"] version = "v0.31.0" hash = "sha256-ZFTlu9+4QToPPLA8C5UcG2eq/lQylq81RoG/WtYo9rg=" + [mod."golang.org/x/mod"] + version = "v0.35.0" + hash = "sha256-ICEQxokHywOFInDPqoP+go9l1tZSz3roknF5SXPtNV4=" [mod."golang.org/x/net"] - version = "v0.50.0" - hash = "sha256-A3tvRuVotO4d8S1FX9ri9CpMJacrFJmHebLJ5m9b+Ss=" + version = "v0.55.0" + hash = "sha256-Phi2mSmBGOJcvqPPAit3uqF3UP8SKRI9dHj6yTM3s5s=" [mod."golang.org/x/sync"] - version = "v0.19.0" - hash = "sha256-RbRZ+sKZUurOczGhhzOoY/sojTlta3H9XjL4PXX/cno=" + version = "v0.20.0" + hash = "sha256-ybcjhCfK6lroUM0yswUvWooW8MOQZBXyiSqoxG6Uy0Y=" [mod."golang.org/x/sys"] - version = "v0.41.0" - hash = "sha256-owjs3/IzAKfFlIz1U1fiHSfl2+bTUhaXTyWEjL5SWHk=" + version = "v0.45.0" + hash = "sha256-hkBoNazrDA67ER6sWhb+EKxx9nJ24+nz3zGy+zT5Hvw=" [mod."golang.org/x/text"] - version = "v0.34.0" - hash = "sha256-wGKd1JkeiFROibvo2kkAuQ7JajSIfV4utGaoGbTQhQM=" + version = "v0.37.0" + hash = "sha256-8XDOnlPIybcDRy89fkjG5VqtIt5Ku+LmaqYhgKl7i1E=" [mod."golang.org/x/time"] version = "v0.12.0" hash = "sha256-Cp3oxrCMH2wyxjzr5SHVmyhgaoUuSl56Uy00Q7DYEpw=" + [mod."golang.org/x/tools"] + version = "v0.44.0" + hash = "sha256-xuj5FLtSJsAojLLTLXtPdLAIFNTKoVFbDMuqRXmj2W4=" [mod."golang.org/x/xerrors"] version = "v0.0.0-20240903120638-7835f813f4da" hash = "sha256-bE7CcrnAvryNvM26ieJGXqbAtuLwHaGcmtVMsVnksqo=" + [mod."google.golang.org/genproto/googleapis/api"] + version = "v0.0.0-20260209200024-4cfbd4190f57" + hash = "sha256-2C7DZwLpDDdmUhVUcRDaotbtkhQFOQ9a1SsdVC8lOqc=" + [mod."google.golang.org/genproto/googleapis/rpc"] + version = "v0.0.0-20260209200024-4cfbd4190f57" + hash = "sha256-gdgUw1LzgVOrarF1cGBUI9uoaR/d6lur2RwxUDKnOZA=" [mod."google.golang.org/protobuf"] version = "v1.36.11" hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE=" diff --git a/nix/vm.nix b/nix/vm.nix --- a/nix/vm.nix +++ b/nix/vm.nix @@ -4,6 +4,8 @@ hostSystem, self, }: let + lib = nixpkgs.lib; + envVar = name: let var = builtins.getEnv name; in @@ -19,8 +21,21 @@ plcUrl = envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory"; jetstream = envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe"; + + checkFile = value: path: + if builtins.pathExists path + then lib.hasPrefix value (builtins.readFile path) + else false; + _nestedVirt = + (checkFile "1" /sys/module/kvm_amd/parameters/nested) + || (checkFile "Y" /sys/module/kvm_intel/parameters/nested); + nestedVirtWarning = '' + KVM nested virtualisation is not enabled on this host. + You should enable it if you can for better performance when testing the QEMU spindle engine! + ''; + nestedVirt = lib.warnIf (!_nestedVirt) nestedVirtWarning _nestedVirt; in - nixpkgs.lib.nixosSystem { + lib.nixosSystem { inherit system; modules = [ self.nixosModules.knot @@ -36,9 +51,11 @@ host.pkgs = import nixpkgs {system = hostSystem;}; graphics = false; - memorySize = 2048; - diskSize = 10 * 1024; + memorySize = 3072; + diskSize = 20 * 1024; cores = 2; + qemu.options = lib.optionals nestedVirt ["-enable-kvm" "-cpu host"]; + forwardPorts = [ # ssh { @@ -101,6 +118,12 @@ }; }; }; + systemd.tmpfiles.rules = [ + "L+ /var/lib/spindle/images/nixos-x86_64 - - - - ${self.packages.${system}.spindle-nixos-image}" + "L+ /var/lib/spindle/images/nixos - - - - /var/lib/spindle/images/nixos-x86_64" + "L+ /var/lib/spindle/images/alpine-x86_64 - - - - ${self.packages.${system}.spindle-alpine-image}" + "L+ /var/lib/spindle/images/alpine - - - - /var/lib/spindle/images/alpine-x86_64" + ]; # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall networking.firewall.enable = false; services.timesyncd.enable = lib.mkForce true; @@ -141,7 +164,30 @@ pipelines = { logBucket = envVarOr "SPINDLE_S3_LOG_BUCKET" ""; + microvm = { + enableKVM = nestedVirt; + }; }; + + cache = { + readUrls = ["http://127.0.0.1:8501"]; + trustedPublicKeys = ["cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU="]; + uploadUrl = "http://127.0.0.1:8501/upload"; + }; + }; + services.ncps = { + enable = true; + cache = { + allowPutVerb = true; + allowDeleteVerb = true; + hostName = "cache.local"; + secretKeyPath = pkgs.writeText "ncps-secret-key" "cache.local:hay0+jvBNguou2tNt19FvrBCogHwHc+mqQe3bww5ZX4XtiqkzO4F0gth3+/7Axk3ZgrELOWJdDIWZ57M7HDutQ=="; + upstream = { + urls = ["https://cache.nixos.org"]; + publicKeys = ["cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="]; + }; + }; + server.addr = "127.0.0.1:8501"; }; services.postgresql = { enable = true; @@ -191,7 +237,7 @@ }; in { knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir; - spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath); + spindle = mkDataSyncScripts "/mnt/spindle-data" (dirOf config.services.tangled.spindle.server.dbPath); knotmirror.after = ["postgresql.target"]; tap-knotmirror.after = ["postgresql.target"]; }; diff --git a/shuttle/Cargo.toml b/shuttle/Cargo.toml new file mode 100644 --- /dev/null +++ b/shuttle/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "shuttle" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1" +base64 = "0.22" +nix = { version = "0.31", features = ["fs", "process", "reboot", "signal", "user"] } +prost = "0.14" +prost-reflect = "0.16" +prost-protovalidate = "0.3" +once_cell = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tempfile = "3" +tokio = { workspace = true, features = ["fs", "net", "process"] } +tokio-vsock = "0.7.2" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/shuttle/README.md b/shuttle/README.md new file mode 100644 --- /dev/null +++ b/shuttle/README.md @@ -0,0 +1,1 @@ +shuttle is a tiny agent for micro VM guests that handles the communication between host <-> guest. diff --git a/spindle/server.go b/spindle/server.go --- a/spindle/server.go +++ b/spindle/server.go @@ -9,6 +9,7 @@ "maps" "net/http" "sync" + "time" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-chi/chi/v5" @@ -25,6 +26,7 @@ "tangled.org/core/spindle/db" "tangled.org/core/spindle/engine" "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/engines/microvm" "tangled.org/core/spindle/engines/nixery" "tangled.org/core/spindle/models" "tangled.org/core/spindle/queue" @@ -41,33 +43,27 @@ ) type Spindle struct { - jc *jetstream.JetstreamClient - tap *Tap - embedTap *embeddedTap - db *db.DB - e *rbac.Enforcer - l *slog.Logger - n *notifier.Notifier - engs map[string]models.Engine - jq *queue.Queue - cfg *config.Config - ks *eventconsumer.Consumer - res *idresolver.Resolver - vault secrets.Manager - motd []byte - motdMu sync.RWMutex - workflowSem chan struct{} - rootCtx context.Context + jc *jetstream.JetstreamClient + tap *Tap + embedTap *embeddedTap + db *db.DB + e *rbac.Enforcer + l *slog.Logger + n *notifier.Notifier + engs map[string]models.Engine + jq *queue.Queue + cfg *config.Config + ks *eventconsumer.Consumer + res *idresolver.Resolver + vault secrets.Manager + motd []byte + motdMu sync.RWMutex + rootCtx context.Context } // New creates a new Spindle server with the provided configuration and engines. -func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engine) (*Spindle, error) { +func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { logger := log.FromContext(ctx) - - d, err := db.Make(ctx, cfg.Server.DBPath) - if err != nil { - return nil, fmt.Errorf("failed to setup db: %w", err) - } e, err := rbac.NewEnforcer(cfg.Server.DBPath) if err != nil { @@ -109,9 +105,6 @@ jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) - workflowSem := make(chan struct{}, cfg.Server.MaxConcurrentWorkflows) - logger.Info("initialized workflow semaphore", "maxConcurrentWorkflows", cfg.Server.MaxConcurrentWorkflows) - collections := []string{ tangled.SpindleMemberNSID, tangled.RepoNSID, @@ -145,19 +138,18 @@ resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) spindle := &Spindle{ - jc: jc, - e: e, - db: d, - l: logger, - n: &n, - engs: engines, - jq: jq, - cfg: cfg, - res: resolver, - vault: vault, - motd: defaultMotd, - workflowSem: workflowSem, - rootCtx: ctx, + jc: jc, + e: e, + db: d, + l: logger, + n: &n, + engs: engines, + jq: jq, + cfg: cfg, + res: resolver, + vault: vault, + motd: defaultMotd, + rootCtx: ctx, } err = e.AddSpindle(rbacDomain) @@ -185,9 +177,15 @@ // job in the above registered queue. ccfg := eventconsumer.NewConsumerConfig() ccfg.Logger = log.SubLogger(logger, "eventconsumer") - ccfg.URLFunc = eventconsumer.DefaultURL(cfg.Server.Dev) ccfg.ProcessFunc = spindle.processPipeline ccfg.CursorStore = cursorStore + if cfg.Server.Dev { + ccfg.RetryInterval = 5 * time.Second + ccfg.MaxRetryInterval = 10 * time.Second + } else { + ccfg.RetryInterval = 1 * time.Minute + ccfg.MaxRetryInterval = 10 * time.Minute + } knownKnots, err := d.Knots() if err != nil { return nil, err @@ -330,14 +328,25 @@ return fmt.Errorf("failed to load config: %w", err) } + d, err := db.Make(ctx, cfg.Server.DBPath) + if err != nil { + return fmt.Errorf("failed to setup db: %w", err) + } + nixeryEng, err := nixery.New(ctx, cfg) if err != nil { return err } - s, err := New(ctx, cfg, map[string]models.Engine{ - "nixery": nixeryEng, - "dummy": dummy.New(log.FromContext(ctx)), + microvmEng, err := microvm.New(ctx, cfg, d) + if err != nil { + return err + } + + s, err := New(ctx, cfg, d, map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(log.FromContext(ctx)), }) if err != nil { return err @@ -413,7 +422,7 @@ workflows := make(map[models.Engine][]models.Workflow) // Build pipeline environment variables once for all workflows - pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev) + pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId) for _, w := range tpl.Workflows { if w != nil { @@ -467,9 +476,9 @@ } } - ok := s.jq.Enqueue(queue.Job{ + ok := s.jq.Enqueue(repoDid, queue.Job{ Run: func() error { - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.workflowSem, ctx, &models.Pipeline{ + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{ RepoDid: repoDid, Workflows: workflows, }, pipelineId) diff --git a/appview/pipelines/logs.go b/appview/pipelines/logs.go --- a/appview/pipelines/logs.go +++ b/appview/pipelines/logs.go @@ -2,12 +2,14 @@ import ( "html/template" + "path" "regexp" "strings" terminal "github.com/buildkite/terminal-to-html/v3" "github.com/gorilla/websocket" "tangled.org/core/appview/pages/markup" + "tangled.org/core/hostutil" ) // matches any ANSI escape sequence: ESC [ m @@ -79,10 +81,11 @@ } } -func SpindleURL(dev bool, spindle, knot, rkey, workflow string) string { - scheme := "wss" - if dev { - scheme = "ws" +func SpindleURL(spindle, knot, rkey, workflow string) string { + url, err := hostutil.EnsureWsScheme(spindle) + if err != nil { + return "" } - return scheme + "://" + strings.Join([]string{spindle, "logs", knot, rkey, workflow}, "/") + + return url + path.Join("/logs", knot, rkey, workflow) } diff --git a/appview/pipelines/pipelines.go b/appview/pipelines/pipelines.go --- a/appview/pipelines/pipelines.go +++ b/appview/pipelines/pipelines.go @@ -18,6 +18,7 @@ "tangled.org/core/appview/pages" "tangled.org/core/appview/reporesolver" "tangled.org/core/eventconsumer" + "tangled.org/core/hostutil" "tangled.org/core/idresolver" "tangled.org/core/orm" "tangled.org/core/rbac" @@ -222,7 +223,11 @@ return } - url := SpindleURL(p.config.Core.Dev, spindle, knot, rkey, workflow) + url := SpindleURL(spindle, knot, rkey, workflow) + if url == "" { + http.Error(w, "invalid spindle hostname", http.StatusBadRequest) + return + } l = l.With("url", url) clientConn, err := upgrader.Upgrade(w, r, nil) @@ -421,11 +426,17 @@ return } + hostname, noTLS, err := hostutil.ParseHostname(spindle) + if err != nil { + http.Error(w, "invalid spindle hostname", http.StatusBadRequest) + return + } + spindleClient, err := p.oauth.ServiceClient( r, - oauth.WithService(f.Spindle), + oauth.WithService(hostname), oauth.WithLxm(tangled.PipelineCancelPipelineNSID), - oauth.WithDev(p.config.Core.Dev), + oauth.WithDev(noTLS), oauth.WithTimeout(time.Second*30), // workflow cleanup usually takes time ) diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -56,7 +56,7 @@ return bootstrapStream( ctx, "knotstream", ec.KindKnot, hosts, c.Redis.Addr, - c.Knotstream, c.Core.Dev, + c.Knotstream, knotIngester(d, acl, enforcer, posthog, notifier, c.Core.Dev, c, cfClient), ), nil } diff --git a/appview/state/spindlestream.go b/appview/state/spindlestream.go --- a/appview/state/spindlestream.go +++ b/appview/state/spindlestream.go @@ -33,7 +33,7 @@ return bootstrapStream( ctx, "spindlestream", ec.KindSpindle, hosts, c.Redis.Addr, - c.Spindlestream, c.Core.Dev, + c.Spindlestream, spindleIngester(d, pn), ), nil } diff --git a/appview/state/spindlestream_test.go b/appview/state/spindlestream_test.go --- a/appview/state/spindlestream_test.go +++ b/appview/state/spindlestream_test.go @@ -55,7 +55,7 @@ }) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) - source := ec.Source{Kind: "test", Host: strings.TrimPrefix(srv.URL, "http://")} + source := ec.Source{Kind: "test", Host: strings.TrimPrefix(srv.URL, "http://"), NoTLS: true} appviewDB, err := db.Make(ctx, filepath.Join(t.TempDir(), "appview.db")) if err != nil { @@ -72,7 +72,6 @@ QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: &cursor.MemoryStore{}, - URLFunc: ec.DefaultURL(true), Logger: logger, } c := ec.NewConsumer(cfg) diff --git a/appview/state/streams.go b/appview/state/streams.go --- a/appview/state/streams.go +++ b/appview/state/streams.go @@ -17,7 +17,6 @@ hosts []string, redisAddr string, streamCfg config.ConsumerConfig, - dev bool, processFn ec.ProcessFunc, ) *ec.Consumer { logger := log.SubLogger(log.FromContext(ctx), name) @@ -41,7 +40,6 @@ WorkerCount: streamCfg.WorkerCount, QueueSize: streamCfg.QueueSize, Logger: logger, - URLFunc: ec.DefaultURL(dev), CursorStore: &cursorStore, }) } diff --git a/blog/posts/spindle-microvm.md b/blog/posts/spindle-microvm.md new file mode 100644 --- /dev/null +++ b/blog/posts/spindle-microvm.md @@ -0,0 +1,277 @@ +--- +atroot: true +template: +slug: spindle-microvm +title: How the microVM engine comes together +subtitle: spindle has a microVM engine now! +date: 2026-06-16 +image: https://assets.tangled.network/blog/seed.png +authors: + - name: dawn + email: dawn@tangled.org + handle: ptr.pet +--- + +Since launching, [spindle](/ci) has run your CI inside Docker containers created +with nixery. That's been mostly okay if you are doing simple things, but if you +wanted to do anything more outside the box (maybe you wanted some services, or +to build & test containers inside), or if you wanted to use Nix inside it (which +is rough :P), it wouldn't meet your needs. That changes today! + +spindle gains a microVM engine. Each workflow gets its own little virtual +machine. You get a full environment inside your workflows that you can do +whatever you want with without any of the roughness of nixery containers. +Alongside this, you also get the ability to configure *services* that a workflow +will have (on the NixOS image), so that means you can easily have postgres, +Docker, and so on that will be alive through the workflow. + +## what's in a microVM + +A microVM is just a VM with most of the boring parts removed. There's no BIOS, +no PCI bus to probe, no emulated graphics card, none of the slow legacy stuff a +normal QEMU machine drags along for example. You get virtio devices and not much +else, which means it boots very quickly and uses very little memory. Right now +QEMU is the only runner we support, but the engine is written so that other +runners (firecracker for example) can slot in later. + +Inside the guest there's a small piece of software we call the agent. Spindle +never SSHes in or runs commands "from the outside"; instead the agent dials back +to spindle over vsock the moment it boots, says hello, and from then on every +step of your workflow is sent to it as a message. The agent runs the command as +an unprivileged user, streams stdout and stderr back, and reports the exit code. +The host side of this lives in +[`spindle`](https://tangled.org/tangled.org/core/tree/master/spindle/engines/microvm/agent.go) +and the guest side is a little Rust binary called +[`shuttle`](https://tangled.org/tangled.org/core/tree/master/shuttle). +(`shuttle` implements +[`agentproto`](https://tangled.org/tangled.org/core/tree/master/spindle/) which +is the protocol used by `spindle`. Technically speaking anyone could implement +this and, assuming side effects hold, you could have your own agent!) + +## two kinds of images + +There are two "flavours" of image you can boot, and they're aimed at fairly +different people. + +The first is **NixOS images**. These are the interesting ones: because the whole +guest is built with Nix, you can configure it from your workflow file directly. +Things like `dependencies`, `services`, `virtualisation` (e.g. Docker), +`registry` and `caches` are all written right there in the YAML, and the guest +agent builds and activates that config before any of your steps run. If we've +built that exact base plus config before, spindle can just hand the guest a +store path to realize (fetching from whatever cache `spindle` has configured) +instead of rebuilding it, so the second run is quick. + +The second is **non-NixOS images**, which today just means Alpine, but can be +anything. You don't get the workflow-level NixOS config here (there's no NixOS +to configure), but if Nix happens to exist inside the image, like it does in our +Alpine one, it can still talk to the spindle Nix cache just fine. + +### example nixos workflow + +If you've used spindle before this will look familiar, it's the same manifest you +already know, just with a few extra keys that the NixOS image understands. Here's +a workflow that needs postgres to test against and Docker to build an image: + +```yaml +# .tangled/workflows/test.yaml +engine: microvm + +when: + - event: ["push", "pull_request"] + branch: ["master"] + +image: nixos + +dependencies: + - go + - github:nixos/nixpkgs#hello + +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + +caches: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true + +steps: + - name: run tests + environment: + PGHOST: /run/postgresql + command: | + docker build -t app . + psql -c "select 1" + go test ./... +``` + +`dependencies` are packages that are added to `environment.systemPackages` (so, +`PATH`). A bare name like `go` is looked up in nixpkgs (same as regular +spindle), but you can also point at any flake with the `flakeref#attr` syntax, +so `github:nixos/nixpkgs#hello` pulls `hello` straight out of that flake. +`registry` is how you remap the global refs: here we pin `nixpkgs` to +`nixos-unstable`, so now the bare `go` above resolves from unstable. You can +alias your own flakes the same way (`myflake: github:me/x`, then `myflake#tool` +in `dependencies`). `caches` is a map of binary cache URL to its trusted public +key, and they get wired into the read proxy (more on that later), so the guest +can substitute prebuilt paths from them instead of building everything from +scratch. + +`services` and `virtualisation` are the interesting parts: they're passed +straight through to NixOS, so anything you could write in a NixOS config you can +write here. `services.postgresql.enable` brings postgres up before any of your +steps run. Since steps run as the `spindle-workflow` user, naming a database +after that user with `ensureDBOwnership` is the easy path to a working db - +postgres peer auth maps the unix user straight to the matching role, so `psql` +connects over the socket with no password and no extra setup (this name-matching +is a NixOS requirement for `ensureDBOwnership`, if you want a differently named +db you'd grant access yourself). `virtualisation.docker: true` is shorthand for +`virtualisation.docker.enable = true`, which gets you a real Docker daemon +inside the VM. By the time your first step runs, postgres is listening and the +Docker socket is there, no sidecar dance, it's just part of the machine. + +(`true` works as shorthand for `.enable = true` anywhere an `enable` option +exists, so most "just turn this on" services are a one-liner!) + +## building the images + +Image builds are done with Nix. For NixOS we lean on +[microvm.nix](https://github.com/microvm-nix/microvm.nix) and layer our own bits +on top (stripping down kernel modules, configuring users, etc.). For Alpine +there's a smallish Nix definition that fetches the kernel, the initrd and the +kernel modules, sets up an init script that configures the machine on boot, +copies in the dependencies we want (`nix`, `git`, etc.) and compresses the whole +rootfs into a squashfs. + +None of this *has* to be Nix, though. As far as spindle is concerned an image is +valid as long as a few things hold: a guest agent (that implements `agentproto`) +is present and gets started on boot, a `spindle-workflow` user exists, and the +work directory is set up at `/workspace`. That can be built however you like. + +## finding an image + +Every built image ships a `spec.json` next to its artifacts. The spec is the +whole contract: where the kernel and initrd and read-only store disk live, the +boot args, how much memory and how many vCPUs to give it, the shell to run steps +in, the writable volumes, the network interfaces, and the runner-specific knobs +(machine type, CPU, extra QEMU args). NixOS images also carry a `baseConfigHash` +identifying the base config baked in (this is the hash of +`nixosSystem.config.system.build.toplevel.outPath`). + +A workflow picks an image with the `image` key at the top level. The name is +matched literally against what's on disk, we look for a directory called +`` with a `spec.json` in it, then fall back to a flat `.json`. The +nice property here is that resolution depends *only* on the name and what's on +disk, never on the host doing the resolving, so the same workflow resolves to +the same image on every spindle. If an operator keeps multiple arches side by +side they can name them `nixos-x86_64`, `alpine-aarch64` and so on (that suffix +is just part of the name, it's not handled specially). If you want, for example, +`nixos` to work, you can just symlink `nixos` to `nixos-x86_64`. + +Right before launch we double-check the referenced files actually exist +and that the host has the tools we need: `mkfs.ext4` for the volumes, the +QEMU binary for the spec's arch, `/dev/kvm` and `/dev/vhost-vsock`, plus +the `ip` / `mount` / `slirp4netns` / `unshare` toolchain if the image +wants networking. + +## the life of a workflow + +A workflow moves through a handful of stages: it gets parsed and its +image resolved, it waits for a slot, it gets set up, its steps ran, and +then everything is torn down. + +The waiting bit matters a lot. Each image declares how much memory, how many +vCPUs and how much disk it needs, and a workflow has to acquire a slot from a +resource scheduler before anything boots. The scheduler is work-conserving with +aging and per-user fairness, so one person submitting a hundred jobs won't +starve everyone else, and slots don't sit idle if there's work that fits in the +budget. + +Once a slot is acquired, we do the setup. Spindle allocates a random vsock CID +for the guest and registers it with the agent hub. It creates the per-workflow +work directory, starts the two cache proxies (more on those later), then creates +the VM: writable volumes become sparse files formatted ext4, the store disk is +attached read-only, and QEMU is started with `-sandbox on`, `-nodefaults`, no +display, no monitor, etc. with serial / `virtio_console` output to a log file +and a QMP socket for control. + +Then we wait for the machine. We poll QMP until QEMU says the guest is running, +then wait for the agent's handshake to arrive over vsock from the CID we expect. +The agent tells us its protocol and versions, and spindle sends back the job id, +the trusted cache public keys and the cache proxy ports, NixOS config if already +cached... From there steps run one at a time as `$shell -lc `, as the +unprivileged workflow user in `/workspace/repo`, with the right environment and +any unlocked secrets. + +Timeouts are cooperative: we work out a deadline from the workflow timeout +and ship it to the guest, with a little grace on our side so the guest +gets a chance to report the timeout itself rather than us just yanking the +machine out from under it. And if the VM crashes mid-step we tail the +serial and QEMU logs into the step's stderr, because "guest agent +connection lost: EOF" is a genuinely useless thing to read at 2am. + +Teardown is the same whether the workflow passed, failed or timed out: +drain any pending Nix cache uploads, ask the agent to power off, wait for +QEMU to exit (falling back to a QMP `system_powerdown`, and finally a +kill if it's being stubborn), then close the proxies and remove the work +directory. + +## locking down the network + +A VM that can reach the host's local network is a VM that can reach things it +has no business reaching. So QEMU doesn't run in the host's network namespace at +all. We `unshare` into fresh user, net and mount namespaces first. Inside that +namespace a small wrapper bind-mounts a resolv.conf pointing at the slirp DNS +and installs blackhole routes for every special-use IP range (RFC 6890, so +private networks, link-local, loopback, etc.) before it execs QEMU. +`slirp4netns` then provides outbound connectivity for that namespace, with +`--disable-host-loopback`, sandbox and seccomp all on. The guest itself sits +behind a *second* layer of QEMU user-mode networking inside that namespace. All +of this is done without needing any privileges! + +## budgets and cgroups + +The scheduler's budget is bookkeeping on its own, it tracks what it's handed +out, and the runner (QEMU) will ensure that a workflow only gets those. But +optionally the whole thing (QEMU and slirp4netns both) gets placed in a +per-workflow cgroup with memory, swap etc. limits, which is an extra +enforcement layer on top. A nice side effect is when the cgroup OOM-kills the VM +we can see that it was an OOM and report it as such, instead of surfacing it as +a generic crash and leaving you guessing. + +The spindle itself also gets a cgroup, which means that in a host OOM situation, +it should be the workflows that die first, not the spindle itself. + +## the nix cache, both ways + +The two proxies I mentioned during setup are how the guest talks to spindle's +Nix cache, and they run on the host so the guest never needs credentials or +direct network access to do it. Like the agent, they also use vsock to +communicate with the spindle. + +The read proxy fans out to the configured substituters plus any caches you +listed in your workflow, so when the guest needs to realize a store path it asks +the proxy and the proxy fetches it. The request is sent concurrently to the read +caches, so the one that answers it first wins. + +The upload proxy goes the other way: any path built inside the guest gets pushed +back out to spindle's Nix cache (if one is configured), so the next workflow +that needs it doesn't have to build it again. Any paths that already exist on +any of the configured read caches won't be uploaded. Built paths are queued by +the agent and are immediately uploaded. If any paths are still left when we +reach VM teardown, the workflow will wait until everything is uploaded. + +## in the future + +todo + +Feel free to come and ask any questions you might have on https://chat.tangled.sh! diff --git a/cmd/spindle-microvm-run/main.go b/cmd/spindle-microvm-run/main.go new file mode 100644 --- /dev/null +++ b/cmd/spindle-microvm-run/main.go @@ -0,0 +1,26 @@ +//go:build linux + +package main + +import ( + "context" + "log/slog" + "os" + + tlog "tangled.org/core/log" +) + +func main() { + cmd := SpindleMicroVMRunCommand() + + logger := tlog.New("spindle-microvm-run") + slog.SetDefault(logger) + + ctx := context.Background() + ctx = tlog.IntoContext(ctx, logger) + + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) + os.Exit(-1) + } +} diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go new file mode 100644 --- /dev/null +++ b/cmd/spindle-microvm-run/main_linux.go @@ -0,0 +1,357 @@ +//go:build linux + +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "net" + "os" + "time" + + "github.com/mdlayher/vsock" + "github.com/urfave/cli/v3" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engines/microvm" +) + +func SpindleMicroVMRunCommand() *cli.Command { + return &cli.Command{ + Name: "spindle-microvm-run", + Usage: "launch the Spindle base microVM and run one command over vsock", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "image-spec", + Sources: cli.EnvVars("SPINDLE_MICROVM_IMAGE_SPEC"), + Usage: "path to microVM image spec JSON", + }, + &cli.StringFlag{ + Name: "mkfs-ext4", + Usage: "override mkfs.ext4 binary", + }, + &cli.StringFlag{ + Name: "work-dir", + Usage: "directory for per-run disks and sockets", + }, + &cli.UintFlag{ + Name: "cid", + Usage: "guest vsock CID; defaults to a random high CID", + }, + &cli.UintFlag{ + Name: "port", + Value: 10240, + Usage: "host vsock port to listen on", + }, + &cli.UintFlag{ + Name: "memory-mib", + Usage: "override the guest memory size in MiB (defaults to the image spec)", + }, + &cli.BoolFlag{ + Name: "disable-kvm", + Usage: "run without -enable-kvm even if /dev/kvm is available", + }, + &cli.BoolFlag{ + Name: "dev", + Usage: "enable dev mode (allows host network access, disables SSL verification)", + }, + &cli.DurationFlag{ + Name: "qmp-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qmp to become ready", + }, + &cli.DurationFlag{ + Name: "accept-timeout", + Value: 15 * time.Second, + Usage: "how long to wait for the guest agent after qemu starts", + }, + &cli.DurationFlag{ + Name: "exec-timeout", + Value: 30 * time.Second, + Usage: "timeout for the guest command", + }, + &cli.DurationFlag{ + Name: "cache-drain-timeout", + Value: 5 * time.Minute, + Usage: "how long to wait for queued cache uploads after the guest command exits", + }, + &cli.DurationFlag{ + Name: "shutdown-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qemu to exit after guest powerdown", + }, + &cli.StringFlag{ + Name: "cwd", + Usage: "guest working directory", + }, + &cli.StringSliceFlag{ + Name: "cache-read-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_READ_URLS"), + Usage: "Nix binary cache URL to pass to the guest; repeatable", + }, + &cli.StringSliceFlag{ + Name: "cache-trusted-public-key", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS"), + Usage: "Nix binary cache public key to trust in the guest; repeatable", + }, + &cli.StringFlag{ + Name: "cache-upload-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_UPLOAD_URL"), + Usage: "optional cache upload URL for guest-built store paths", + }, + &cli.StringFlag{ + Name: "activate-config", + Usage: "JSON user config to activate before exec (e.g. '{\"services\":{\"openssh\":{\"enable\":true}}}')", + }, + &cli.StringFlag{ + Name: "db", + Usage: "path to sqlite database for config cache", + }, + }, + Action: runMicroVMRunDev, + } +} + +func runMicroVMRunDev(ctx context.Context, cmd *cli.Command) error { + imageSpecPath := cmd.String("image-spec") + if imageSpecPath == "" { + return fmt.Errorf("--image-spec or SPINDLE_MICROVM_IMAGE_SPEC is required") + } + + imageSpec, err := microvm.LoadImageSpec(imageSpecPath) + if err != nil { + return err + } + + port := uint32(cmd.Uint("port")) + // tell the guest which host vsock port to dial back on. shuttle reads the + // cmdline params this is so we can run multiple of this process + // concurrently, because otherwise it listens on a specific vsock port, and + // we cant bind to the same port twice... + imageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", imageSpec.BootArgs, port) + if mib := cmd.Uint("memory-mib"); mib > 0 { + imageSpec.MemoryMiB = int(mib) + } + ln, err := vsock.Listen(port, nil) + if err != nil { + return fmt.Errorf("listen on vsock port %d: %w", port, err) + } + defer ln.Close() + + vm, err := microvm.StartVM(ctx, microvm.VMConfig{ + Image: imageSpec, + BootTimeout: cmd.Duration("qmp-timeout"), + CID: uint32(cmd.Uint("cid")), + EnableKVM: !cmd.Bool("disable-kvm"), + MkfsExt4: cmd.String("mkfs-ext4"), + WorkDir: cmd.String("work-dir"), + Dev: cmd.Bool("dev"), + }, slog.Default()) + if err != nil { + return err + } + defer vm.Close() + + logs := vm.Logs() + fmt.Fprintf(os.Stderr, "microvm started: cid=%d work-dir=%s serial-log=%s qemu-log=%s\n", + vm.CID(), + vm.WorkDir(), + logs.Serial, + logs.Extra["qemu"], + ) + + logger := slog.Default() + + if cmd.Duration("accept-timeout") > 0 { + if err := ln.SetDeadline(time.Now().Add(cmd.Duration("accept-timeout"))); err != nil { + return fmt.Errorf("set accept deadline: %w", err) + } + } + + argv := cmd.Args().Slice() + if len(argv) == 0 { + argv = []string{"/run/current-system/sw/bin/echo", "hello-from-spindle"} + } + jobID := "spindle-microvm-run" + execID := "dev-1" + + fmt.Fprintf(os.Stderr, "listening for agent on %s\n", ln.Addr()) + conn, err := acceptExpectedVsockConn(ln, vm.CID(), logger) + if err != nil { + return fmt.Errorf("accept agent connection: %w", err) + } + defer conn.Close() + + upstreams, err := microvm.BuildCacheUpstreams(cmd.StringSlice("cache-read-url"), nil) + if err != nil { + return fmt.Errorf("build cache upstreams: %w", err) + } + + var readCache *microvm.ReadCacheProxy + if len(cmd.StringSlice("cache-read-url")) > 0 { + var err error + readCache, err = microvm.StartReadCacheProxy(ctx, vm.CID(), upstreams, logger) + if err != nil { + return fmt.Errorf("start read cache proxy: %w", err) + } + defer readCache.Close() + } + + var uploadCache *microvm.UploadCacheProxy + if cmd.String("cache-upload-url") != "" { + var err error + uploadCache, err = microvm.StartUploadCacheProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, logger) + if err != nil { + return fmt.Errorf("start upload cache proxy: %w", err) + } + defer uploadCache.Close() + } + dnsProxy, err := microvm.StartDNSProxy(ctx, vm.CID(), logger) + if err != nil { + return fmt.Errorf("start dns proxy: %w", err) + } + defer dnsProxy.Close() + + session := microvm.NewAgentSession(conn, logger) + + initCtx, cancelInit := context.WithTimeout(ctx, 30*time.Second) + defer cancelInit() + if err := session.Init(initCtx, &agentv1.Init{ + JobId: jobID, + CacheTrustedPublicKeys: cmd.StringSlice("cache-trusted-public-key"), + CacheReadProxyPort: readCache.Port(), + CacheUploadProxyPort: uploadCache.Port(), + DnsProxyPort: dnsProxy.Port(), + }); err != nil { + return fmt.Errorf("init agent: %w", err) + } + + execCtx := ctx + if cmd.Duration("exec-timeout") > 0 { + var cancel context.CancelFunc + execCtx, cancel = context.WithTimeout(ctx, cmd.Duration("exec-timeout")) + defer cancel() + } + + if cmd.String("activate-config") != "" { + actCtx := execCtx + baseHash, err := microvm.BaseConfigHash(imageSpec) + if err != nil { + return fmt.Errorf("calculate base config hash: %w", err) + } + + var d *db.DB + var configKey string + var cachedToplevel string + if cmd.String("db") != "" { + d, err = db.Make(ctx, cmd.String("db")) + if err != nil { + return fmt.Errorf("failed to open database: %w", err) + } + defer d.Close() + + configKey, err = microvm.BuildConfigKey(imageSpec, cmd.String("activate-config")) + if err != nil { + return fmt.Errorf("calculate config key: %w", err) + } + + record, err := d.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("lookup config cache: %w", err) + } + } else { + cachedToplevel = record.Toplevel + fmt.Printf("realizing cached NixOS config %s\n", cachedToplevel) + } + } + + result, err := session.ActivateConfig(actCtx, "dev-activate", &agentv1.ActivateConfig{ + ConfigKey: configKey, + BaseConfigHash: baseHash, + UserConfig: cmd.String("activate-config"), + Toplevel: cachedToplevel, + }) + if err != nil { + return fmt.Errorf("activate config: %w", err) + } + fmt.Fprintf(os.Stderr, "activated config toplevel: %s\n", result.Toplevel) + + if d != nil && cachedToplevel == "" && result.Toplevel != "" && configKey != "" { + err = d.SaveNixOSToplevelCacheRecord(configKey, result.Toplevel) + if err != nil { + return fmt.Errorf("save config cache: %w", err) + } + } + } + + exitCode, err := session.Exec(execCtx, microvm.AgentExec{ + ID: execID, + ExecStart: &agentv1.ExecStart{ + Argv: argv, + Cwd: cmd.String("cwd"), + }, + Stdout: os.Stdout, + Stderr: os.Stderr, + }) + if err != nil { + return err + } + + if uploadCache != nil { + drainCtx := ctx + if cmd.Duration("cache-drain-timeout") > 0 { + var cancel context.CancelFunc + drainCtx, cancel = context.WithTimeout(ctx, cmd.Duration("cache-drain-timeout")) + defer cancel() + } + uploaded, err := session.Drain(drainCtx) + if err != nil { + return err + } + fmt.Printf("cache uploaded: %d\n", uploaded) + } + + // mirror the engine shutdown order: ask the agent to power off first, + // then fall back to qemu powerdown / kill + shutdownCtx, cancel := context.WithTimeout(context.Background(), cmd.Duration("shutdown-timeout")) + defer cancel() + poweredOff := false + if err := session.Poweroff(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "agent poweroff: %s\n", err) + } else if err := vm.WaitContext(shutdownCtx); err == nil { + poweredOff = true + } + if !poweredOff { + if err := vm.Shutdown(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "microvm shutdown fallback: %s\n", err) + } + } + + if exitCode != 0 { + return fmt.Errorf("guest command exited with code %d", exitCode) + } + return nil +} + +func acceptExpectedVsockConn(ln *vsock.Listener, allowedCID uint32, logger *slog.Logger) (net.Conn, error) { + for { + conn, err := ln.Accept() + if err != nil { + return nil, err + } + if allowedCID == 0 { + return conn, nil + } + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == allowedCID { + return conn, nil + } + remote := conn.RemoteAddr() + _ = conn.Close() + logger.Warn("dropped agent connection from unexpected cid", "remote", remote, "expected", allowedCID) + } +} diff --git a/cmd/spindle-microvm-run/unsupported.go b/cmd/spindle-microvm-run/unsupported.go new file mode 100644 --- /dev/null +++ b/cmd/spindle-microvm-run/unsupported.go @@ -0,0 +1,13 @@ +//go:build !linux + +package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Fprintf(os.Stderr, "spindle-microvm-run is only supported on Linux\n") + os.Exit(-1) +} diff --git a/cmd/spindle/main.go b/cmd/spindle/main.go --- a/cmd/spindle/main.go +++ b/cmd/spindle/main.go @@ -5,20 +5,39 @@ "log/slog" "os" + "github.com/urfave/cli/v3" tlog "tangled.org/core/log" "tangled.org/core/spindle" ) func main() { + cmd := &cli.Command{ + Name: "spindle", + Usage: "spindle continuous integration runner", + Commands: []*cli.Command{ + Command(), + }, + DefaultCommand: "run", + } + logger := tlog.New("spindle") slog.SetDefault(logger) ctx := context.Background() ctx = tlog.IntoContext(ctx, logger) - err := spindle.Run(ctx) - if err != nil { - logger.Error("error running spindle", "error", err) + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) os.Exit(-1) + } +} + +func Command() *cli.Command { + return &cli.Command{ + Name: "run", + Usage: "run the spindle server", + Action: func(ctx context.Context, cmd *cli.Command) error { + return spindle.Run(ctx) + }, } } diff --git a/knotmirror/hostutil/hostutil.go b/knotmirror/hostutil/hostutil.go deleted file mode 100644 --- a/knotmirror/hostutil/hostutil.go +++ /dev/null @@ -1,56 +0,0 @@ -package hostutil - -import ( - "fmt" - "net/url" - "strings" - - "github.com/bluesky-social/indigo/atproto/syntax" -) - -func ParseHostname(raw string) (hostname string, noSSL bool, err error) { - // handle case of bare hostname - if !strings.Contains(raw, "://") { - if strings.HasPrefix(raw, "localhost:") { - raw = "http://" + raw - } else { - raw = "https://" + raw - } - } - - u, err := url.Parse(raw) - if err != nil { - return "", false, fmt.Errorf("not a valid host URL: %w", err) - } - - switch u.Scheme { - case "https", "wss": - noSSL = false - case "http", "ws": - noSSL = true - default: - return "", false, fmt.Errorf("unsupported URL scheme: %s", u.Scheme) - } - - // 'localhost' (exact string) is allowed *with* a required port number; SSL is optional - if u.Hostname() == "localhost" { - if u.Port() == "" || !strings.HasPrefix(u.Host, "localhost:") { - return "", false, fmt.Errorf("port number is required for localhost") - } - return u.Host, noSSL, nil - } - - // port numbers not allowed otherwise - if u.Port() != "" { - return "", false, fmt.Errorf("port number not allowed for non-local names") - } - - // check it is a real hostname (eg, not IP address or single-word alias) - h, err := syntax.ParseHandle(u.Host) - if err != nil { - return "", false, fmt.Errorf("not a public hostname") - } - - // lower-case in response - return h.Normalize().String(), noSSL, nil -} diff --git a/knotmirror/xrpc/sync_request_crawl.go b/knotmirror/xrpc/sync_request_crawl.go --- a/knotmirror/xrpc/sync_request_crawl.go +++ b/knotmirror/xrpc/sync_request_crawl.go @@ -11,8 +11,8 @@ "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/xrpc" "tangled.org/core/api/tangled" + "tangled.org/core/hostutil" "tangled.org/core/knotmirror/db" - "tangled.org/core/knotmirror/hostutil" "tangled.org/core/knotmirror/models" ) diff --git a/localinfra/scripts/init-accounts.sh b/localinfra/scripts/init-accounts.sh --- a/localinfra/scripts/init-accounts.sh +++ b/localinfra/scripts/init-accounts.sh @@ -187,5 +187,6 @@ JWT=$(login "$OWNER_DID") put_record "$JWT" "$OWNER_DID" "sh.tangled.knot" $KNOT_HOSTNAME "{\"createdAt\": \"${CREATED_AT}\"}" +put_record "$JWT" "$OWNER_DID" "sh.tangled.spindle" "$SPINDLE_HOSTNAME" "{\"createdAt\": \"${CREATED_AT}\"}" printf 'done.\n' >&2 diff --git a/localinfra/scripts/prepare-spindle-images.sh b/localinfra/scripts/prepare-spindle-images.sh new file mode 100644 --- /dev/null +++ b/localinfra/scripts/prepare-spindle-images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "$0")/../.." && pwd) +image_root="${1:-$repo/out/localinfra-spindle-images}" + +mkdir -p "$image_root" + +extract_image() { + local package="$1" + local name="$2" + shift 2 + + local tarball + tarball=$(nix build "$repo#$package" --no-link --print-out-paths) + + [ -d "$image_root/$name" ] && chmod -R +w "$image_root/$name" || true + rm -rf "$image_root/$name" + mkdir -p "$image_root/$name" + tar -C "$image_root/$name" -xzf "$tarball" + + local alias + for alias in "$@"; do + rm -rf "$image_root/$alias" + ln -s "$name" "$image_root/$alias" + done +} + +extract_image spindle-nixos-image-tarball nixos-x86_64 nixos +extract_image spindle-alpine-image-tarball alpine-x86_64 alpine + +echo "prepared spindle microVM images in $image_root" diff --git a/nix/microvm/base.nix b/nix/microvm/base.nix new file mode 100644 --- /dev/null +++ b/nix/microvm/base.nix @@ -0,0 +1,309 @@ +{ + config, + pkgs, + lib, + ... +}: let + # these are modules / module trees we keep. everything else is pruned. + # this is a "cheap" way to save on what we ship, we don't have to recompile anything. + # this saves about 118mb! + keepTrees = [ + "crypto" + "lib" + "arch" + "drivers/virtio" + # net: the firewall modprobes across netfilter/ipv4/ipv6; docker adds the + # bridge/llc/802(stp)/xfrm machinery + NAT targets in netfilter. + "net/core" + "net/netfilter" + "net/ipv4" + "net/ipv6" + "net/packet" + "net/sched" + "net/vmw_vsock" + "net/bridge" + "net/llc" + "net/802" + "net/xfrm" + "fs/configfs" + "fs/autofs" + "fs/nls" + "fs/unicode" + ]; + keepMods = [ + # boot + storage + common workflow filesystems + "erofs" + "ext4" + "jbd2" + "mbcache" + "overlay" + "fuse" + "loop" + # "btrfs" + # "xfs" + # "f2fs" + # "vfat" + # "exfat" + "squashfs" + "isofs" + "dm-mod" + "zram" + # virtio devices the runner exposes + "virtio" + "virtio_mmio" + "virtio_pci" + "virtio_blk" + "virtio_net" + "virtio_rng" + "virtio_console" + "vsock_loopback" + "vmw_vsock_virtio_transport" + "vmw_vsock_virtio_transport_common" + # container networking (docker default bridge + common custom networks) + "veth" + "tun" + "tap" + "bridge" + "br_netfilter" + "macvlan" + "ipvlan" + "vxlan" + "geneve" + "dummy" + "wireguard" + ]; + keepTreesFile = pkgs.writeText "keep-trees" (lib.concatStringsSep "\n" keepTrees); + keepModsFile = pkgs.writeText "keep-mods" (lib.concatStringsSep "\n" keepMods); + slimModulesScript = pkgs.writeText "slim-modules.py" '' + import os, shutil, sys + + src, dst, trees_file, mods_file = sys.argv[1:5] + KEEP_TREES = open(trees_file).read().split() + KEEP_MODS = open(mods_file).read().split() + + def norm(name): + return name.replace("-", "_") + + kerneldir = os.path.join(src, "kernel") + + bypath, byname = {}, {} + for root, _, files in os.walk(kerneldir): + for f in files: + if ".ko" not in f: + continue + ap = os.path.join(root, f) + rel = os.path.relpath(ap, src) + bypath[rel] = ap + byname[norm(f.split(".ko")[0])] = rel + + deps = {} + with open(os.path.join(src, "modules.dep")) as fh: + for line in fh: + if ":" in line: + mod, rest = line.split(":", 1) + deps[mod.strip()] = rest.split() + + keep = set() + def add(rel): + if rel in keep or rel not in bypath: + return + keep.add(rel) + for dep in deps.get(rel, []): + add(dep) + + for tree in KEEP_TREES: + for root, _, files in os.walk(os.path.join(kerneldir, tree)): + for f in files: + if ".ko" in f: + add(os.path.relpath(os.path.join(root, f), src)) + for mod in KEEP_MODS: + rel = byname.get(norm(mod)) + if rel: + add(rel) + + for rel in keep: + target = os.path.join(dst, rel) + os.makedirs(os.path.dirname(target), exist_ok=True) + shutil.copy2(bypath[rel], target) + print(f"kept {len(keep)} of {len(bypath)} modules") + ''; + slimKernelModules = + pkgs.runCommand "${config.boot.kernelPackages.kernel.name}-modules-microvm" + {nativeBuildInputs = [pkgs.python3 pkgs.kmod];} + '' + src=${lib.getOutput "modules" config.boot.kernelPackages.kernel}/lib/modules + ver=$(ls "$src") + mkdir -p "$out/lib/modules/$ver" + for f in "$src/$ver"/modules.builtin* "$src/$ver"/modules.order; do + [ -e "$f" ] && cp "$f" "$out/lib/modules/$ver/" + done + python3 ${slimModulesScript} "$src/$ver" "$out/lib/modules/$ver" ${keepTreesFile} ${keepModsFile} + # regen modules.dep + depmod -b "$out" "$ver" + ''; +in { + system.stateVersion = "26.05"; + + # actually use our slimmed down modules set + system.modulesTree = lib.mkForce ([slimKernelModules] ++ config.boot.extraModulePackages); + + boot.initrd.includeDefaultModules = lib.mkForce false; + boot.initrd.availableKernelModules = lib.mkForce []; + boot.initrd.kernelModules = lib.mkForce [ + "virtio_pci" + "virtio_mmio" + "virtio_blk" + "virtio_console" + "erofs" + "ext4" + "overlay" + ]; + boot.kernelModules = ["loop"]; + + # some zram to help situations where burst memory usage causes OOM + zramSwap = { + enable = true; + algorithm = "zstd"; + memoryPercent = 50; + }; + + programs.nano.enable = false; + # we are on a microvm we don't need the hardware map + environment.etc."udev/hwdb.bin".enable = lib.mkForce false; + + networking.hostName = "spindle-microvm"; + networking.useDHCP = false; + systemd.network.networks."40-eth0" = { + matchConfig.Name = "eth0"; + address = ["10.0.3.15/24"]; + gateway = ["10.0.3.2"]; + dns = ["127.0.0.1"]; + }; + networking.nameservers = ["127.0.0.1"]; + + # this is disabled by microvm optimizations but we do need it + system.switch.enable = lib.mkForce true; + + # don't install docs or any xdg things, not necessary + documentation.enable = false; + xdg.mime.enable = false; + xdg.icons.enable = false; + xdg.sounds.enable = false; + + users.groups.spindle-workflow = { + gid = 970; + }; + users.users.spindle-workflow = { + isSystemUser = true; + uid = 970; + group = "spindle-workflow"; + home = "/workspace"; + createHome = false; + }; + users.users.spindle-workflow.extraGroups = lib.mkIf config.virtualisation.docker.enable [ + "docker" + ]; + virtualisation.docker.listenOptions = [ + "/run/docker.sock" + "/var/run/docker.sock" + ]; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + trusted-users = ["root"]; + allowed-users = ["spindle-workflow"]; + }; + registry.nixpkgs.to = { + type = "path"; + path = pkgs.path; + }; + extraOptions = '' + extra-experimental-features = nix-command flakes + !include /run/spindle/nix.conf + ''; + nixPath = ["nixpkgs=${config.nix.registry.nixpkgs.to.path}"]; + }; + + systemd.tmpfiles.rules = [ + "d /run/spindle 0755 root root -" + "d /workspace 0755 spindle-workflow spindle-workflow -" + "d /workspace/repo 0755 spindle-workflow spindle-workflow -" + ]; + + # add any common packages / services here + environment.systemPackages = [pkgs.gitMinimal]; + # disable default nixos packages ([perl rsync strace]) + environment.defaultPackages = []; + # this removed nixos-rebuild-ng and nixos-generate-config, which lets us + # remove python3 closure (~107MB) + system.disableInstallerTools = true; + + # a single volume that will back /workspace, /var, and the nix store + microvm.storeOnDisk = true; + microvm.storeDiskType = "erofs"; + # lz4hc, not zstd: the stock nixpkgs kernel builds erofs without + # CONFIG_EROFS_FS_ZIP_ZSTD, so a zstd image fails to mount at boot ("algorithm + # 3 isn't enabled on this kernel"); only lz4 is guaranteed. -Efragments and + # -Ededupe are omitted because microvm.nix falls back to single-threaded + # erofs-utils when either is present, which makes image builds really slow. + # for now, we take the compression hit, which isn't too much anyway. + # todo(dawn): the remaining big save needs a custom guest kernel (we'd want a + # binary cache first so downstream users don't rebuild it every time): enable + # EROFS_FS_ZIP_ZSTD for a better-compressing store-disk, build the essentials + # (virtio/erofs/ext4/overlay/netfilter) in as =y, and strip the kernel image + # itself. the modules tree is already pruned without a recompile, see + # slimKernelModules above. + microvm.storeDiskErofsFlags = [ + "-zlz4hc" + "-Eztailpacking" + "-C131072" # bigger compression window lets lz4hc compress better (~47mb) + ]; + microvm.writableStoreOverlay = "/persist/rw-store"; + microvm.volumes = [ + { + image = "persist.img"; + mountPoint = "/persist"; + size = 1024 * 16; + fsType = "ext4"; + } + ]; + + # /persist must be mounted before the writable store overlay activates + fileSystems."/persist".neededForBoot = true; + + fileSystems."/workspace" = { + device = "/persist/workspace"; + fsType = "none"; + options = ["bind"]; + depends = ["/persist"]; + }; + # bind mounting /var is important since docker etc. can't use overlayfs + # (overlayfs on overlayfs does not work) + fileSystems."/var" = { + device = "/persist/var"; + fsType = "none"; + options = ["bind"]; + depends = ["/persist"]; + }; + + # create bind sources before local-fs.target, which means we have to do this + # at initrd time + boot.initrd.systemd.enable = true; + boot.initrd.systemd.tmpfiles.settings."00-persist-layout" = { + "/sysroot/persist/rw-store".d = { + mode = "0755"; + }; + "/sysroot/persist/workspace".d = { + mode = "0755"; + user = "spindle-workflow"; + group = "spindle-workflow"; + }; + "/sysroot/persist/var".d = { + mode = "0755"; + }; + }; +} diff --git a/nix/microvm/qemu.nix b/nix/microvm/qemu.nix new file mode 100644 --- /dev/null +++ b/nix/microvm/qemu.nix @@ -0,0 +1,25 @@ +{...}: { + microvm = { + hypervisor = "qemu"; + qemu.machine = "microvm"; + + optimize.enable = true; + + vcpu = 2; + # don't set to 2048, https://github.com/microvm-nix/microvm.nix/issues/171 + mem = 2049; + + interfaces = [ + { + type = "user"; + id = "net0"; + mac = "02:00:00:00:10:01"; + } + ]; + vsock.cid = 3; + + socket = "control.socket"; + }; + + boot.kernelModules = ["vsock_loopback"]; +} diff --git a/nix/microvm/spindle-vm.nix b/nix/microvm/spindle-vm.nix new file mode 100644 --- /dev/null +++ b/nix/microvm/spindle-vm.nix @@ -0,0 +1,48 @@ +{ + self, + microvm, +}: runnerModule: { + imports = [ + microvm.nixosModules.microvm + ./base.nix + runnerModule + self.nixosModules.shuttle + ({pkgs, ...}: { + services.tangled.shuttle.enable = true; + + environment.etc = { + "spindle/nixos/base.nix".source = ./base.nix; + "spindle/nixos/runner.nix".source = runnerModule; + "spindle/nixos/shuttle.nix".text = '' + { config, lib, pkgs, ... }: + { + imports = [${../modules/shuttle.nix}]; + services.tangled.shuttle.package = lib.mkDefault ${self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle}; + } + ''; + "spindle/nixos/user-config.nix".source = ./user-config.nix; + "spindle/nixos/microvm".source = microvm; + # pkgs.path is fine here because we pass the nixpkgs source into the vm in ./base.nix + "spindle/nixos/default.nix".text = '' + let + nixpkgs = ${pkgs.path}; + nixos = import (nixpkgs + "/nixos") { + system = "${pkgs.stdenv.hostPlatform.system}"; + configuration = { + imports = [ + /etc/spindle/nixos/microvm/nixos-modules/microvm/default.nix + /etc/spindle/nixos/base.nix + /etc/spindle/nixos/runner.nix + /etc/spindle/nixos/shuttle.nix + /etc/spindle/nixos/user-config.nix + ]; + services.tangled.shuttle.enable = true; + }; + }; + in + nixos.system + ''; + }; + }) + ]; +} diff --git a/nix/microvm/user-config.nix b/nix/microvm/user-config.nix new file mode 100644 --- /dev/null +++ b/nix/microvm/user-config.nix @@ -0,0 +1,110 @@ +{ + pkgs, + lib, + options, + ... +} @ args: let + configPath = /run/spindle/user-config/config.json; + userConfig = + args.userConfig + or ( + if builtins.pathExists configPath + then lib.importJSON configPath + else {} + ); + + registry = userConfig.registry or {}; + + # registry targets may be structured attrs or flake ref strings; strings are + # parsed by nix itself in getFlake. flakeRefToString rejects unforced attr + # values, hence the toJSON round-trip + toRefString = target: + if builtins.isAttrs target + then builtins.flakeRefToString (builtins.fromJSON (builtins.toJSON target)) + else target; + + # user registry entries shadow the system registry (which pins nixpkgs) + getFlake = ref: builtins.getFlake (toRefString (registry.${ref} or ref)); + + # "flakeref#attr" or a bare attr looked up in nixpkgs. nixpkgs refs use the + # already-evaluated pkgs directly instead of re-evaluating via getFlake, + # unless the user remapped nixpkgs in their registry + resolvePackage = ref: let + parts = lib.splitString "#" ref; + hasAttr = lib.length parts > 1; + flakeRef = + if hasAttr + then lib.head parts + else "nixpkgs"; + pkgName = + if hasAttr + then lib.elemAt parts 1 + else ref; + system = pkgs.stdenv.hostPlatform.system; + flake = getFlake flakeRef; + notFound = throw "Package ${pkgName} not found in ${flakeRef}"; + in + if flakeRef == "nixpkgs" && !(registry ? nixpkgs) + then pkgs.${pkgName} or notFound + else flake.legacyPackages.${system}.${pkgName} or flake.packages.${system}.${pkgName} or notFound; + + # strings are resolved as package references only where the option type + # actually expects packages; everything else passes through untouched + resolveForType = type: v: + if type.name == "package" && builtins.isString v + then resolvePackage v + # path-typed options (e.g. services.udev.packages) accept derivations via + # coercion; "#" disambiguates flake refs from actual paths, which are + # always absolute + else if type.name == "path" && builtins.isString v && lib.hasInfix "#" v && !lib.hasPrefix "/" v + then resolvePackage v + else if type.name == "nullOr" && v != null + then resolveForType type.nestedTypes.elemType v + else if type.name == "listOf" && builtins.isList v + then map (resolveForType type.nestedTypes.elemType) v + else if (type.name == "attrsOf" || type.name == "lazyAttrsOf") && builtins.isAttrs v + then builtins.mapAttrs (_: resolveForType type.nestedTypes.elemType) v + else if type.name == "submodule" && builtins.isAttrs v + then resolveOptions (type.getSubOptions []) v + else v; + + resolveOptions = opts: builtins.mapAttrs (name: resolveValue (opts.${name} or null)); + + resolveValue = opt: v: + if !builtins.isAttrs opt + then v + else if lib.isOption opt + then resolveForType opt.type v + else if builtins.isAttrs v + then resolveOptions opt v + else v; + + # `foo = true` is shorthand for `foo.enable = true`, but only when an + # enable option actually exists under foo + hasEnableOption = opt: + builtins.isAttrs opt + && ( + if lib.isOption opt + then (opt.type.getSubOptions opt.loc) ? enable + else opt ? enable && lib.isOption opt.enable + ); + + normalize = opts: name: v: let + opt = opts.${name} or null; + in + if builtins.isBool v && hasEnableOption opt + then {enable = v;} + else resolveValue opt v; +in { + nix.registry = builtins.mapAttrs (name: _: + lib.mkForce { + to = { + type = "path"; + path = (getFlake name).outPath; + }; + }) + registry; + environment.systemPackages = map resolvePackage (userConfig.dependencies or []); + services = builtins.mapAttrs (normalize (options.services or {})) (userConfig.services or {}); + virtualisation = builtins.mapAttrs (normalize (options.virtualisation or {})) (userConfig.virtualisation or {}); +} diff --git a/nix/modules/shuttle.nix b/nix/modules/shuttle.nix new file mode 100644 --- /dev/null +++ b/nix/modules/shuttle.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.services.tangled.shuttle; + + postBuildHook = pkgs.writeShellApplication { + name = "spindle-post-build-hook"; + text = '' + set -f + + if [ -z "''${OUT_PATHS:-}" ]; then + exit 0 + fi + + # OUT_PATHS is intentionally split into individual store paths for the agent + # shellcheck disable=SC2086 + exec ${cfg.package}/bin/shuttle enqueue-built-paths $OUT_PATHS + ''; + }; +in { + options.services.tangled.shuttle = { + enable = lib.mkEnableOption "the shuttle guest agent"; + + package = lib.mkOption { + type = lib.types.package; + description = "package providing the shuttle executable."; + }; + }; + + config = lib.mkIf cfg.enable { + nix.settings.post-build-hook = "${postBuildHook}/bin/spindle-post-build-hook"; + + systemd.services.shuttle = { + description = "shuttle guest agent"; + wantedBy = ["multi-user.target"]; + wants = ["network-online.target"]; + after = [ + "local-fs.target" + "network-online.target" + ]; + before = ["nix-daemon.service"]; + restartIfChanged = false; + environment = { + NIX_PATH = lib.concatStringsSep ":" config.nix.nixPath; + }; + serviceConfig = { + Type = "simple"; + ExecStart = "${cfg.package}/bin/shuttle"; + Restart = "always"; + RestartSec = "1s"; + }; + }; + }; +} diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -1,6 +1,7 @@ { config, lib, + pkgs, ... }: let cfg = config.services.tangled.spindle; @@ -75,12 +76,6 @@ description = "Maximum number of jobs queue up"; }; - maxConcurrentWorkflows = mkOption { - type = types.int; - default = 8; - description = "Maximum number of workflow containers running simultaneously (controls total memory usage)"; - }; - secrets = { provider = mkOption { type = types.str; @@ -136,28 +131,155 @@ }; pipelines = { - nixery = mkOption { - type = types.str; - default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet - description = "Nixery instance to use"; - }; - - workflowTimeout = mkOption { - type = types.str; - default = "5m"; - description = "Timeout for each step of a pipeline"; - }; - - maxJobMemoryMb = mkOption { - type = types.int; - default = 6144; - description = "Memory limit per workflow container in MiB (default 6 GiB)"; - }; - logBucket = mkOption { type = types.str; default = "tangled-logs"; description = "S3 bucket for workflow logs"; + }; + workflowTimeout = mkOption { + type = types.str; + default = "5m"; + description = "Timeout for each workflow step"; + }; + + nixery = { + nixery = mkOption { + type = types.str; + default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet + description = "Nixery instance to use"; + }; + + maxJobMemoryMb = mkOption { + type = types.int; + default = 6144; + description = "Memory limit per nixery workflow container in MiB (default 6 GiB)"; + }; + maxConcurrentWorkflows = mkOption { + type = types.int; + default = 8; + description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit."; + }; + }; + + microvm = { + enableKVM = mkOption { + type = types.bool; + default = true; + description = "Enable KVM hardware acceleration"; + }; + + imageDir = mkOption { + type = types.str; + default = "/var/lib/spindle/images"; + description = "Directory containing microVM image spec JSONs or image spec directories"; + }; + overlayDir = mkOption { + type = types.str; + default = "/tmp"; + description = "Directory to store microVM temporary overlay files"; + }; + defaultImage = mkOption { + type = types.str; + default = "nixos"; + description = "Default microVM image spec to use if none is specified in workflow"; + }; + agentPort = mkOption { + type = types.port; + default = 10240; + description = "Host vsock port the microVM agent connects back to"; + }; + + limits = { + total = { + memoryMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit."; + }; + vcpus = mkOption { + type = types.int; + default = 0; + description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit."; + }; + diskMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit."; + }; + }; + + workflow = { + memoryMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit."; + }; + vcpus = mkOption { + type = types.int; + default = 0; + description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit."; + }; + diskMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit."; + }; + }; + }; + + cgroup = { + enable = mkOption { + type = types.bool; + default = false; + description = "Enable cgroup v2 containment for microVM processes."; + }; + parent = mkOption { + type = types.str; + default = "self"; + description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup."; + }; + pidsMax = mkOption { + type = types.int; + default = 4096; + description = "Maximum number of processes allowed in each microVM workflow cgroup."; + }; + swapMaxMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap."; + }; + supervisorMinMiB = mkOption { + type = types.int; + default = 512; + description = '' + Amount of memory in MiB that will be protected by the cgroup for the spindle + (allowing it to not get OOMed first.) + ''; + }; + }; + }; + }; + + cache = { + readUrls = mkOption { + type = types.listOf types.str; + default = []; + example = ["http://ncps.internal:8501"]; + description = "Nix binary cache URLs the Spindle guest should read from."; + }; + + trustedPublicKeys = mkOption { + type = types.listOf types.str; + default = []; + example = ["ncps.internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]; + description = "Public keys trusted for the configured Nix binary caches."; + }; + + uploadUrl = mkOption { + type = types.str; + default = ""; + example = "http://ncps.internal:8501/upload"; + description = "Optional cache upload URL used by live cache import paths."; }; }; @@ -177,45 +299,91 @@ }; }; - config = mkIf cfg.enable { - virtualisation.docker.enable = true; + config = let + deps = [ + pkgs.qemu + pkgs.e2fsprogs + pkgs.slirp4netns + pkgs.iproute2 + pkgs.util-linux + ]; + in + mkIf cfg.enable { + environment.systemPackages = [ + (pkgs.writeShellScriptBin "spindle" '' + export PATH="${lib.makeBinPath deps}:$PATH" + ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"} + ${lib.concatMapStringsSep "\n" ( + e: "export ${e}" + ) + config.systemd.services.spindle.serviceConfig.Environment} + exec ${cfg.package}/bin/spindle "$@" + '') + ]; - systemd.services.spindle = { - description = "spindle service"; - after = ["network.target" "docker.service"]; - wantedBy = ["multi-user.target"]; - serviceConfig = { - LogsDirectory = "spindle"; - StateDirectory = "spindle"; - EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + virtualisation.docker.enable = true; - Environment = [ - "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}" - "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}" - "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}" - "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}" - "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}" - "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" - "SPINDLE_SERVER_OWNER=${cfg.server.owner}" - "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}" - "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}" - "SPINDLE_SERVER_MAX_CONCURRENT_WORKFLOWS=${toString cfg.server.maxConcurrentWorkflows}" - "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" - "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" - "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" - "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}" - "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}" - "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}" - "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}" - "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}" - "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" - "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" - "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.maxJobMemoryMb}" - "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" + systemd.services.spindle = { + description = "spindle service"; + after = [ + "network.target" + "docker.service" ]; - ExecStart = "${cfg.package}/bin/spindle"; - Restart = "always"; + wantedBy = ["multi-user.target"]; + path = deps; + serviceConfig = { + LogsDirectory = "spindle"; + StateDirectory = "spindle"; + Delegate = cfg.pipelines.microvm.cgroup.enable; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + + Environment = [ + "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}" + "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}" + "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}" + "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}" + "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}" + "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" + "SPINDLE_SERVER_OWNER=${cfg.server.owner}" + "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}" + "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}" + "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" + "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" + "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" + "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}" + "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}" + "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}" + "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}" + "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}" + "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}" + "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" + "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}" + "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}" + "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}" + "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}" + "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}" + "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}" + "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}" + "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.cache.readUrls}" + "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.cache.trustedPublicKeys}" + "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.cache.uploadUrl}" + "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" + ]; + ExecStart = "${cfg.package}/bin/spindle"; + Restart = "always"; + }; }; }; - }; } diff --git a/nix/pkgs/shuttle.nix b/nix/pkgs/shuttle.nix new file mode 100644 --- /dev/null +++ b/nix/pkgs/shuttle.nix @@ -0,0 +1,23 @@ +{ + rustPlatform, + src, + protobuf, + ... +}: let + flags = ["--bin" "shuttle" "--package" "shuttle"]; +in + rustPlatform.buildRustPackage { + pname = "shuttle"; + version = "0.1.0"; + + inherit src; + + cargoLock.lockFile = "${src}/Cargo.lock"; + + nativeBuildInputs = [ + protobuf + ]; + + cargoBuildFlags = flags; + cargoTestFlags = flags; + } diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix new file mode 100644 --- /dev/null +++ b/nix/pkgs/spindle-alpine-image.nix @@ -0,0 +1,222 @@ +{ + pkgsStatic, + runCommand, + writeText, + squashfsTools, + shuttle, + binutils, + publicsuffix-list, + rootfs, + kernel, + initramfs, + modloop, + repositories, + arch ? "x86_64", +}: let + nix = pkgsStatic.nixStatic; + bash = pkgsStatic.bashNonInteractive; + git = + (pkgsStatic.gitMinimal.override { + curl = pkgsStatic.curlMinimal; + pythonSupport = false; + withManual = false; + nlsSupport = false; + }).overrideAttrs (old: { + doCheck = false; + doInstallCheck = false; + configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"]; + }); + guestTools = [nix bash git]; + + # run by busybox at sysinit + setupScript = writeText "spindle-setup" '' + #!/bin/sh + + mountpoint -q /proc || mount -t proc proc /proc + mountpoint -q /sys || mount -t sysfs sys /sys + mountpoint -q /dev || mount -t devtmpfs dev /dev + mountpoint -q /dev/pts || { + mkdir -p /dev/pts + mount -t devpts devpts /dev/pts + } + mountpoint -q /dev/shm || { + mkdir -p /dev/shm + mount -t tmpfs -o mode=1777 shm /dev/shm + } + mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run + mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp + + # the initramfs mdev leaves these 0660, which breaks non-root workflows + chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null + + modprobe vmw_vsock_virtio_transport + # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock + modprobe vsock_loopback + modprobe ext4 + + # /dev/vda is the squashfs root; the first spindle volume backs /workspace + if [ -b /dev/vdb ]; then + mount -t ext4 /dev/vdb /workspace + mkdir -p /workspace/repo + chown spindle-workflow:spindle-workflow /workspace /workspace/repo + fi + + ip link set lo up + ip link set eth0 up + ip addr add 10.0.3.15/24 dev eth0 + ip route add default via 10.0.3.2 + hostname -F /etc/hostname + ''; + + inittab = writeText "inittab" '' + ::sysinit:/sbin/spindle-setup + ::respawn:/usr/local/bin/nix-daemon + ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle + ::ctrlaltdel:/sbin/reboot + ''; + + profileScript = writeText "spindle-profile" '' + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt + export NIX_REMOTE=daemon + ''; + + # mirror nix/microvm/base.nix and nix/modules/shuttle.nix + nixConf = writeText "nix.conf" '' + experimental-features = nix-command flakes + trusted-users = root + allowed-users = spindle-workflow + post-build-hook = /usr/libexec/spindle-post-build-hook + !include /run/spindle/nix.conf + ''; + + apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n"); + + postBuildHook = writeText "spindle-post-build-hook" '' + #!/bin/sh + set -f + + if [ -z "''${OUT_PATHS:-}" ]; then + exit 0 + fi + + # OUT_PATHS is intentionally split into individual store paths + exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS + ''; + + imageSpecJSON = writeText "spec.json" ( + builtins.toJSON { + inherit arch; + bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init"; + kernel = "kernel"; + initrd = "initrd"; + runnerType = "qemu"; + runnerConfig = { + cpu = "host,+x2apic,-sgx"; + machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; + console = "hvc0"; + extraArgs = []; + }; + memoryMiB = 2048; + storeDisk = "store-disk"; + storeDiskType = "squashfs"; + vcpus = 2; + shell = "/usr/local/bin/bash"; + networkInterfaces = [ + { + type = "slirp4netns"; + id = "net0"; + mac = "02:00:00:00:10:01"; + } + ]; + volumes = [ + { + fsType = "ext4"; + image = "workspace.img"; + imageType = "raw"; + mountPoint = "/workspace"; + readOnly = false; + sizeMiB = 1024 * 10; + } + ]; + } + ); +in + runCommand "spindle-alpine-image-${arch}" { + nativeBuildInputs = [squashfsTools binutils]; + } '' + mkdir -p rootfs + tar -xzpf ${rootfs} -C rootfs + + # kernel modules from modloop (ships its own modules.dep, no depmod needed) + unsquashfs -q -d modloop ${modloop} + mkdir -p rootfs/lib/modules + cp -a modloop/modules/* rootfs/lib/modules/ + + install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle + install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup + install -D -m 0644 ${inittab} rootfs/etc/inittab + install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh + install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf + install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook + + # install dependencies + # we only copy binaries + libexec for minimal deps so the image size doesn't + # increase so much (if we copy the whole guestTools closure for example, it + # doubles the disk size) + mkdir -p rootfs/nix/store rootfs/usr/local/bin + for pkg in ${toString guestTools}; do + for bin in "$pkg/bin/"*; do + [[ -e "$bin" ]] || continue + name=$(basename "$bin") + # we resolve symlinks as to copy the actual binaries + if [[ -L "$bin" ]]; then + real=$(readlink "$bin") + else + real="$bin" + fi + # handle symlinks properly + if [[ "$real" != /nix/store* ]]; then + ln -vsf "$real" "rootfs/usr/local/bin/$name" + else + cp -v "$real" "rootfs/usr/local/bin/$name" + fi + done + # libexec has binaries used by packages even if statically compiled + if [[ -d "$pkg/libexec" ]]; then + mkdir -p "rootfs$pkg" + cp -av "$pkg/libexec" "rootfs$pkg/" + fi + done + # this is necessary for nix to work, it is not a library but nix hardcodes + # it in it's binary + cp -rv ${publicsuffix-list} rootfs/nix/store/ + + # scripts commonly hardcode #!/bin/bash + ln -sf ${bash}/bin/bash rootfs/bin/bash + + echo "spindle-microvm" > rootfs/etc/hostname + printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf + install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories + + echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd + echo "spindle-workflow:x:970:" >> rootfs/etc/group + echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow + mkdir -p rootfs/workspace + + # setup nix build users for the daemon + members="" + for i in $(seq 1 8); do + echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd + echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow + members="$members''${members:+,}nixbld$i" + done + echo "nixbld:x:30000:$members" >> rootfs/etc/group + + mkdir -p "$out" + mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \ + -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth + cp ${kernel} "$out/kernel" + cp ${initramfs} "$out/initrd" + cp ${imageSpecJSON} "$out/spec.json" + '' diff --git a/nix/pkgs/spindle-nixos-image.nix b/nix/pkgs/spindle-nixos-image.nix new file mode 100644 --- /dev/null +++ b/nix/pkgs/spindle-nixos-image.nix @@ -0,0 +1,58 @@ +{ + pkgs, + lib, + nixosSystem, +}: let + system = nixosSystem.pkgs.stdenv.hostPlatform.qemuArch; + microvm = nixosSystem.config.microvm; + baseConfigHash = lib.pipe nixosSystem.config.system.build.toplevel.outPath [ + (lib.strings.removePrefix "/nix/store/") + (lib.strings.splitString "-") + lib.head + ]; + imageSpecJSON = pkgs.writeText "spec.json" ( + builtins.toJSON { + arch = system; + bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 ${lib.concatStringsSep " " microvm.kernelParams}"; + kernel = "kernel"; + initrd = "initrd"; + runnerType = "qemu"; + runnerConfig = { + cpu = "host,+x2apic,-sgx"; + machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; + console = "hvc0"; + extraArgs = []; + }; + memoryMiB = microvm.mem; + storeDisk = "store-disk"; + storeDiskType = microvm.storeDiskType; + vcpus = microvm.vcpu; + shell = "/run/current-system/sw/bin/bash"; + baseConfigHash = baseConfigHash; + networkInterfaces = + map (interface: { + type = "slirp4netns"; + id = interface.id; + mac = interface.mac; + }) + microvm.interfaces; + volumes = + map (volume: { + fsType = volume.fsType; + image = volume.image; + imageType = volume.imageType; + mountPoint = volume.mountPoint; + readOnly = volume.readOnly; + sizeMiB = volume.size; + }) + microvm.volumes; + } + ); +in + pkgs.runCommand "spindle-nixos-image-${system}" {} '' + mkdir -p "$out" + cp ${imageSpecJSON} "$out/spec.json" + ln -s ${microvm.kernel}/bzImage "$out/kernel" + ln -s ${microvm.initrdPath} "$out/initrd" + ln -s ${microvm.storeDisk} "$out/store-disk" + '' diff --git a/shuttle/src/activation.rs b/shuttle/src/activation.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/activation.rs @@ -0,0 +1,146 @@ +use crate::command::{self, Spec, run_capture}; +use crate::nix_config::{SPINDLE_RUN_DIR, nix_executable}; +use crate::protocol::{self, Message, v1}; +use anyhow::{Context, Result}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::sync::mpsc::Sender; +use tracing::info; + +const USER_CONFIG_DIR: &str = "/run/spindle/user-config"; + +pub async fn run(id: String, req: v1::ActivateConfig, out: Sender) { + let config_key = req.config_key.clone(); + let result = activate(&req).await; + let msg = Message { + id, + activate_config_result: Some(v1::ActivateConfigResult { + config_key, + toplevel: (result.as_ref()) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(), + error: protocol::error_or_empty(result.err().map(|e| format!("{e:#}"))), + }), + ..Default::default() + }; + let _ = out.send(msg).await; +} + +async fn activate(req: &v1::ActivateConfig) -> Result { + let need_build = req.toplevel.is_empty(); + let timeout = (req.timeout_seconds > 0) + .then(|| Duration::from_secs(u64::from(req.timeout_seconds))) + .or_else(|| need_build.then_some(Duration::from_secs(10 * 60))) + .unwrap_or(Duration::from_secs(2 * 60)); + + let toplevel = if need_build { + build_toplevel(req, timeout).await? + } else { + realise_toplevel(&req.toplevel, timeout).await? + }; + + if !toplevel.starts_with("/nix/store/") { + anyhow::bail!("config toplevel {toplevel:?} is not a nix store path"); + } + + switch_to_configuration(&toplevel, timeout).await?; + info!( + config_key = %req.config_key, + base_config_hash = %req.base_config_hash, + ?toplevel, + "activated NixOS config" + ); + Ok(toplevel) +} + +async fn build_toplevel(req: &v1::ActivateConfig, timeout: Duration) -> Result { + let user_config = (req.user_config.is_empty()) + .then_some("{}") + .unwrap_or_else(|| &req.user_config); + + info!("writing user config to {USER_CONFIG_DIR}/config.json"); + write_user_config(user_config).context("write user config")?; + + info!("running nix build command for user config toplevel..."); + let output = run_capture( + Spec::new(nix_executable()) + .args([ + "build", + "--no-link", + "--show-trace", + "--json", + "--file", + "/etc/spindle/nixos/default.nix", + ]) + .cwd(SPINDLE_RUN_DIR) + .timeout(timeout), + ) + .await?; + + if !output.success() { + anyhow::bail!( + "nix config build failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + #[derive(Debug, serde::Deserialize)] + struct NixBuildResult { + outputs: NixBuildOutputs, + } + #[derive(Debug, serde::Deserialize)] + struct NixBuildOutputs { + out: PathBuf, + } + let [result] = serde_json::from_slice::<[NixBuildResult; 1]>(&output.stdout) + .context("parse nix build --json output")?; + Ok(result.outputs.out) +} + +fn write_user_config(user_config: &str) -> Result<()> { + fs::create_dir_all(USER_CONFIG_DIR).with_context(|| format!("create {USER_CONFIG_DIR}"))?; + + let config_path = format!("{USER_CONFIG_DIR}/config.json"); + fs::write(&config_path, user_config).with_context(|| format!("write {config_path}"))?; + Ok(()) +} + +async fn realise_toplevel(toplevel: &str, timeout: Duration) -> Result { + if !toplevel.starts_with("/nix/store/") { + anyhow::bail!("cached config toplevel {toplevel:?} is not a nix store path"); + } + let output = command::run_capture( + Spec::new(nix_executable()) + .args(["build", "--no-link", "--show-trace", toplevel]) + .timeout(timeout), + ) + .await?; + if !output.success() { + anyhow::bail!( + "realise cached config failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + Ok(PathBuf::from(toplevel)) +} + +async fn switch_to_configuration(toplevel: &Path, timeout: Duration) -> Result<()> { + info!("switching to new configuration: {:?}", toplevel); + let switch = toplevel.join("bin/switch-to-configuration"); + let output = run_capture(Spec::new(switch).args(["test"]).timeout(timeout)).await?; + if !output.success() { + anyhow::bail!( + "switch-to-configuration test failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + Ok(()) +} diff --git a/shuttle/src/command.rs b/shuttle/src/command.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/command.rs @@ -0,0 +1,298 @@ +use anyhow::{Context, Result}; +use nix::sys::signal::{Signal, kill}; +use nix::unistd::{Gid, Pid, Uid, User, getgrouplist, setgid, setgroups, setuid}; +use std::ffi::{CString, OsStr, OsString}; +use std::io; +use std::os::unix::process::ExitStatusExt; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::process::{Child, Command}; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::task::JoinHandle; +use tracing::warn; + +#[derive(Clone, Debug)] +pub struct Spec { + pub program: OsString, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: Option, + pub timeout: Option, + pub uid: Option, + pub gid: Option, +} + +impl Spec { + pub fn new(program: impl Into) -> Self { + Self { + program: program.into(), + args: Vec::new(), + env: Vec::new(), + cwd: None, + timeout: None, + uid: None, + gid: None, + } + } + + pub fn arg(mut self, arg: impl Into) -> Self { + self.args.push(arg.into()); + self + } + + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + pub fn envs(mut self, env: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.env.extend( + env.into_iter() + .map(|(key, value)| (key.into(), value.into())), + ); + self + } + + pub fn cwd(mut self, cwd: impl Into) -> Self { + self.cwd = Some(cwd.into()); + self + } + + pub fn timeout(mut self, timeout: Duration) -> Self { + self.timeout = Some(timeout); + self + } + + pub fn run_as(mut self, uid: u32, gid: u32) -> Self { + self.uid = Some(uid); + self.gid = Some(gid); + self + } +} + +#[derive(Clone, Debug)] +pub struct ExitResult { + pub exit_code: i32, + pub error: Option, + pub timed_out: bool, +} + +#[derive(Clone, Debug)] +pub struct CaptureOutput { + pub exit: ExitResult, + pub stdout: Vec, + pub stderr: Vec, +} + +impl CaptureOutput { + pub fn success(&self) -> bool { + self.exit.exit_code == 0 && self.exit.error.is_none() + } + + pub fn combined_lossy(&self) -> String { + let mut data = self.stdout.clone(); + data.extend_from_slice(&self.stderr); + String::from_utf8_lossy(&data).trim().to_owned() + } +} + +#[derive(Clone, Copy, Debug)] +pub enum OutKind { + Stdout, + Stderr, +} + +#[derive(Clone, Debug)] +pub struct OutData { + pub data: Vec, + pub kind: OutKind, +} + +pub struct StreamingCommand { + events: Receiver, + exit: JoinHandle>, +} + +impl StreamingCommand { + pub fn into_parts(self) -> (Receiver, JoinHandle>) { + (self.events, self.exit) + } +} + +pub async fn run_capture(spec: Spec) -> Result { + let running = spawn_streaming(spec)?; + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let (mut events, exit_task) = running.into_parts(); + + while let Some(event) = events.recv().await { + match event.kind { + OutKind::Stdout => stdout.extend_from_slice(&event.data), + OutKind::Stderr => stderr.extend_from_slice(&event.data), + } + } + + let exit = exit_task + .await + .unwrap_or_else(|error| Err(anyhow::anyhow!("command supervisor failed: {error}")))?; + Ok(CaptureOutput { + exit, + stdout, + stderr, + }) +} + +pub fn spawn_streaming(mut spec: Spec) -> Result { + let mut child = spawn(&mut spec)?; + let stdout = child.stdout.take().context("stdout pipe missing")?; + let stderr = child.stderr.take().context("stderr pipe missing")?; + + let (events_tx, events_rx) = mpsc::channel(64); + let stdout_thread = spawn_reader(stdout, events_tx.clone(), OutKind::Stdout); + let stderr_thread = spawn_reader(stderr, events_tx.clone(), OutKind::Stderr); + drop(events_tx); + + let exit = tokio::spawn(async move { + let exit = wait_child(&mut child, spec.timeout).await; + + // ensure all output is observed before exiting + // this assumes children dont daemonize and hold onto the stdout/err + stdout_thread.await.context("stdout reader task failed")?; + stderr_thread.await.context("stderr reader task failed")?; + + Ok(exit) + }); + + Ok(StreamingCommand { + events: events_rx, + exit, + }) +} + +fn spawn(spec: &mut Spec) -> Result { + let mut cmd = Command::new(&spec.program); + cmd.args(&spec.args) + .envs(spec.env.iter().map(|(key, value)| (key, value))) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + if let Some(cwd) = &spec.cwd { + cmd.current_dir(cwd); + } + + // don't use rust's .uid() / .gid() methods here because they clear + // supplemantary groups, which means for example adding a user to "docker" + // group won't actually let it access the sock. + // https://github.com/rust-lang/rust/issues/90747 + if let (Some(uid), Some(gid)) = (spec.uid, spec.gid) { + let username = User::from_uid(Uid::from_raw(uid)) + .ok() + .flatten() + .map(|u| u.name) + .with_context(|| format!("lookup passwd entry for uid {uid}"))?; + let cname = CString::new(username) + .with_context(|| format!("username for uid {uid} contained a null byte"))?; + // resolve groups beforehand so we don't have to read /etc/group in the pre_exec + let groups = + getgrouplist(&cname, Gid::from_raw(gid)).context("resolve supplementary groups")?; + // SAFETY: pre_exec runs between fork and execve in the child. + // we only call async-signal-safe syscalls and we don't touch any + // shared state, no allocator, no mutexes, no globals. + unsafe { + cmd.pre_exec(move || { + setgroups(&groups).map_err(io::Error::from)?; + setgid(Gid::from_raw(gid)).map_err(io::Error::from)?; + setuid(Uid::from_raw(uid)).map_err(io::Error::from)?; + Ok(()) + }); + } + } + + // allow us to kill this whole process tree on deadline + cmd.process_group(0); + + cmd.spawn() + .with_context(|| format!("spawn {}", display_os(&spec.program))) +} + +async fn wait_child(child: &mut Child, timeout: Option) -> ExitResult { + let wait = child.wait(); + let status = match timeout { + Some(timeout) => match tokio::time::timeout(timeout, wait).await { + Ok(status) => status, + Err(_) => { + if let Some(pid) = child.id() + && let Err(error) = kill(Pid::from_raw(-(pid as i32)), Signal::SIGKILL) + { + warn!(pid, %error, "failed to kill process group"); + } + let _ = child.wait().await; + return ExitResult { + exit_code: 124, + error: Some("command timed out".to_owned()), + timed_out: true, + }; + } + }, + None => wait.await, + }; + + match status { + Ok(status) => ExitResult { + exit_code: status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(1), + error: None, + timed_out: false, + }, + Err(error) => ExitResult { + exit_code: 1, + error: Some(error.to_string()), + timed_out: false, + }, + } +} + +fn spawn_reader( + mut reader: impl AsyncRead + Unpin + Send + 'static, + events: Sender, + kind: OutKind, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut buf = [0_u8; 32 * 1024]; + loop { + match reader.read(&mut buf).await { + Ok(0) => return, + Ok(n) => { + let event = OutData { + data: buf[..n].to_vec(), + kind, + }; + if events.send(event).await.is_err() { + return; + } + } + Err(error) => { + warn!(%error, "failed to read command stream"); + return; + } + } + } + }) +} + +fn display_os(value: &OsStr) -> String { + value.to_string_lossy().into_owned() +} diff --git a/shuttle/src/dns_proxy.rs b/shuttle/src/dns_proxy.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/dns_proxy.rs @@ -0,0 +1,255 @@ +use anyhow::{Context, Result}; +use std::io; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream, UdpSocket}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +const DEFAULT_DNS_PROXY_ADDR: &str = "127.0.0.1:53"; +const SHUTTLE_DNS_PROXY_ADDR_ENV: &str = "SHUTTLE_DNS_PROXY_ADDR"; + +const MAX_DNS_MESSAGE_BYTES: usize = u16::MAX as usize; +const DNS_IO_TIMEOUT: Duration = Duration::from_secs(10); +const DNS_TCP_IDLE_TIMEOUT: Duration = Duration::from_secs(120); + +// implements a proxy that sends dns requests to the spindle and +// lets the spindle resolve any queries, and streams the response back. +// +// we use this because the way spindle isolates QEMU VMs is unreliable +// to rely on. if we unblock the blackholed-routes for private nameservers +// like slirp one, we risk leaking internal DNS zones, even if the guest +// can't connect to them. there is also potentially DNS rebinding issues. +// and other slirp4netns quirks... +// +// this way we also get to filter the DNS queries very easily, so we can +// make sure we remove everything that would leak a host information. +pub struct DnsProxy { + handles: Vec>, +} + +#[derive(Clone)] +struct HostDnsClient { + host_cid: u32, + host_port: u32, +} + +impl DnsProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let addr = std::env::var(SHUTTLE_DNS_PROXY_ADDR_ENV) + .unwrap_or_else(|_| DEFAULT_DNS_PROXY_ADDR.to_owned()); + + let udp = Arc::new( + UdpSocket::bind(&addr) + .await + .with_context(|| format!("bind dns udp listener {addr}"))?, + ); + + let tcp = TcpListener::bind(&addr) + .await + .with_context(|| format!("bind dns tcp listener {addr}"))?; + + let host = HostDnsClient { + host_cid, + host_port, + }; + + let handles = vec![ + tokio::spawn(udp_loop(udp, host.clone())), + tokio::spawn(tcp_loop(tcp, host)), + ]; + + info!(%addr, host_cid, host_port, "dns proxy ready"); + Ok(Some(Self { handles })) + } +} + +impl Drop for DnsProxy { + fn drop(&mut self) { + for handle in self.handles.drain(..) { + handle.abort(); + } + } +} + +impl HostDnsClient { + async fn query(&self, query: Vec) -> Result> { + match self.query_once(&query).await { + Ok(response) => Ok(response), + Err(first_error) => self.query_once(&query).await.with_context(|| { + format!("dns host query failed after retry; first error: {first_error:#}") + }), + } + } + + async fn query_once(&self, query: &[u8]) -> Result> { + let addr = VsockAddr::new(self.host_cid, self.host_port); + + let mut host = tokio::time::timeout(DNS_IO_TIMEOUT, VsockStream::connect(addr)) + .await + .context("dns host connect timed out")? + .with_context(|| { + format!( + "dial host dns proxy cid={} port={}", + self.host_cid, self.host_port + ) + })?; + + tokio::time::timeout(DNS_IO_TIMEOUT, async { + write_dns_packet(&mut host, query) + .await + .context("write dns query to host")?; + + read_dns_packet(&mut host) + .await + .context("read dns response from host")? + .context("host dns proxy closed without response") + }) + .await + .context("dns host query timed out")? + } +} + +async fn udp_loop(socket: Arc, host: HostDnsClient) { + let mut buf = vec![0; MAX_DNS_MESSAGE_BYTES]; + let mut tasks = JoinSet::new(); + + loop { + tokio::select! { + received = socket.recv_from(&mut buf) => match received { + Ok((len, peer)) => { + let query = buf[..len].to_vec(); + let socket = socket.clone(); + let host = host.clone(); + + tasks.spawn(async move { + if let Err(error) = handle_udp_query(socket, peer, query, host).await { + warn!(%peer, %error, "dns udp query failed"); + } + }); + } + Err(error) => warn!(%error, "dns udp recv failed"), + }, + + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_dns_task_result(result); + } + } + } +} + +async fn handle_udp_query( + socket: Arc, + peer: SocketAddr, + query: Vec, + host: HostDnsClient, +) -> Result<()> { + let response = host.query(query).await?; + + socket + .send_to(&response, peer) + .await + .context("send dns udp response")?; + + Ok(()) +} + +async fn tcp_loop(listener: TcpListener, host: HostDnsClient) { + let mut tasks = JoinSet::new(); + + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, peer)) => { + let host = host.clone(); + + tasks.spawn(async move { + if let Err(error) = handle_tcp_conn(conn, host).await { + warn!(%peer, %error, "dns tcp connection failed"); + } + }); + } + Err(error) => warn!(%error, "dns tcp accept failed"), + }, + + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_dns_task_result(result); + } + } + } +} + +async fn handle_tcp_conn(mut tcp: TcpStream, host: HostDnsClient) -> Result<()> { + loop { + let query = tokio::time::timeout(DNS_TCP_IDLE_TIMEOUT, read_dns_packet(&mut tcp)) + .await + .context("dns tcp idle timeout")? + .context("read dns tcp query")?; + + let Some(query) = query else { + return Ok(()); + }; + + let response = host.query(query).await?; + + write_dns_packet(&mut tcp, &response) + .await + .context("write dns tcp response")?; + } +} + +async fn read_dns_packet(reader: &mut R) -> io::Result>> +where + R: AsyncRead + Unpin, +{ + let mut len_buf = [0; 2]; + + match reader.read_exact(&mut len_buf).await { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + } + + let len = u16::from_be_bytes(len_buf) as usize; + if len == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "empty dns packet", + )); + } + + let mut packet = vec![0; len]; + reader.read_exact(&mut packet).await?; + Ok(Some(packet)) +} + +async fn write_dns_packet(writer: &mut W, packet: &[u8]) -> io::Result<()> +where + W: AsyncWrite + Unpin, +{ + if packet.is_empty() || packet.len() > MAX_DNS_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid dns packet size {}", packet.len()), + )); + } + + writer + .write_all(&(packet.len() as u16).to_be_bytes()) + .await?; + writer.write_all(packet).await?; + writer.flush().await +} + +fn log_dns_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "dns proxy task failed"); + } +} diff --git a/shuttle/src/exec.rs b/shuttle/src/exec.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/exec.rs @@ -0,0 +1,188 @@ +use crate::command::{self, OutKind, Spec}; +use crate::protocol::{self, Message, v1}; +use nix::unistd::{Group, User}; +use std::ffi::OsString; +use std::time::Duration; +use tokio::sync::mpsc::Sender; +use tracing::info; + +const DEFAULT_USER: &str = "spindle-workflow"; + +pub async fn run(id: String, req: v1::ExecStart, out: Sender) { + let send_exit = async |exit_code: i32, error: Option, timed_out: bool| { + let msg = Message { + id: id.clone(), + exec_exit: Some(v1::ExecExit { + exit_code, + error: protocol::error_or_empty(error), + timed_out, + }), + ..Default::default() + }; + let _ = out.send(msg).await; + }; + + if req.argv.is_empty() { + send_exit(127, Some("missing argv".to_owned()), false).await; + return; + } + + let user = if req.user.is_empty() { + DEFAULT_USER + } else { + req.user.as_str() + }; + let run_as = match resolve_user(user) { + Ok(run_as) => run_as, + Err(err) => { + send_exit(127, Some(err), false).await; + return; + } + }; + + let mut spec = Spec::new(req.argv[0].clone()) + .args(req.argv[1..].iter().cloned()) + .envs(parse_env(&req.env)) + .run_as(run_as.uid, run_as.gid); + if !req.cwd.is_empty() { + spec = spec.cwd(req.cwd.clone()); + } + let timeout = + (req.timeout_seconds > 0).then(|| Duration::from_secs(u64::from(req.timeout_seconds))); + if let Some(timeout) = timeout { + spec = spec.timeout(timeout); + } + + info!( + %id, + user = %run_as.name, + uid = run_as.uid, + gid = run_as.gid, + argv = ?req.argv, + cwd = ?req.cwd, + "starting exec" + ); + + let cmd = match command::spawn_streaming(spec) { + Ok(cmd) => cmd, + Err(err) => { + send_exit(127, Some(err.to_string()), false).await; + return; + } + }; + let (mut events, exit_task) = cmd.into_parts(); + while let Some(event) = events.recv().await { + let data = String::from_utf8_lossy(&event.data).into_owned(); + let output = match event.kind { + OutKind::Stdout => Message { + id: id.clone(), + exec_stdout: Some(v1::ExecStdout { data }), + ..Default::default() + }, + OutKind::Stderr => Message { + id: id.clone(), + exec_stderr: Some(v1::ExecStderr { data }), + ..Default::default() + }, + }; + let _ = out.send(output).await; + } + let exit = match exit_task + .await + .unwrap_or_else(|error| Err(anyhow::anyhow!("command supervisor failed: {error}"))) + { + Ok(exit) => exit, + Err(err) => { + send_exit(127, Some(err.to_string()), false).await; + return; + } + }; + + send_exit(exit.exit_code, exit.error, exit.timed_out).await +} + +#[derive(Clone, Debug)] +struct ResolvedUser { + name: String, + uid: u32, + gid: u32, +} + +fn resolve_user(spec: &str) -> Result { + let spec = spec.trim(); + if spec.is_empty() { + return resolve_user(DEFAULT_USER); + } + + let (user_part, group_part) = spec + .split_once(':') + .map(|(user, group)| (user, Some(group))) + .unwrap_or((spec, None)); + + let mut user = lookup_user(user_part)?; + if let Some(group) = group_part.filter(|group| !group.is_empty()) { + user.gid = lookup_group(group)?; + } + + if user.uid == 0 || user.gid == 0 { + return Err(format!("refusing to run exec as privileged user {spec:?}")); + } + + Ok(user) +} + +fn lookup_user(name: &str) -> Result { + match User::from_name(name) { + Ok(Some(user)) => Ok(ResolvedUser { + name: name.to_owned(), + uid: user.uid.as_raw(), + gid: user.gid.as_raw(), + }), + Ok(None) => { + let uid = name + .parse::() + .map_err(|_| format!("workflow user {name:?} was not found"))?; + Ok(ResolvedUser { + name: name.to_owned(), + uid, + gid: uid, + }) + } + Err(error) => Err(format!("lookup workflow user {name:?}: {error}")), + } +} + +fn lookup_group(name: &str) -> Result { + match Group::from_name(name) { + Ok(Some(group)) => Ok(group.gid.as_raw()), + Ok(None) => name + .parse::() + .map_err(|_| format!("workflow group {name:?} was not found")), + Err(error) => Err(format!("lookup workflow group {name:?}: {error}")), + } +} + +fn parse_env(values: &[String]) -> Vec<(OsString, OsString)> { + values + .iter() + .filter_map(|value| value.split_once('=')) + .map(|(key, value)| (OsString::from(key), OsString::from(value))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refuses_root_exec_user() { + let err = resolve_user("root").unwrap_err(); + assert!(err.contains("refusing to run exec as privileged user")); + } + + #[test] + fn refuses_root_exec_group() { + let err = resolve_user("65534:0").unwrap_err(); + assert!(err.contains("refusing to run exec as privileged user")); + } +} diff --git a/shuttle/src/host_proxy.rs b/shuttle/src/host_proxy.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/host_proxy.rs @@ -0,0 +1,92 @@ +use anyhow::{Context, Result}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +// this implements a vsock <-> tcp proxy for communicating with spindle +pub struct VsockTcpProxy { + url: String, + handle: JoinHandle<()>, +} + +impl VsockTcpProxy { + pub async fn start( + name: &'static str, + bind_addr: &str, + host_cid: u32, + host_port: u32, + ) -> Result { + if host_port == 0 { + anyhow::bail!("port 0 cant be requested"); + } + + let listener = TcpListener::bind(bind_addr) + .await + .with_context(|| format!("bind {name} listener {bind_addr}"))?; + let local_addr = listener + .local_addr() + .with_context(|| format!("{name} local address"))?; + let url = format!("http://{local_addr}"); + + let handle = tokio::spawn(async move { + accept_loop(name, listener, host_cid, host_port).await; + }); + + info!(%url, host_cid, host_port, "{name} ready"); + Ok(Self { url, handle }) + } + + pub fn url(&self) -> &str { + &self.url + } +} + +impl Drop for VsockTcpProxy { + fn drop(&mut self) { + self.handle.abort(); + } +} + +async fn accept_loop(name: &'static str, listener: TcpListener, host_cid: u32, host_port: u32) { + let mut tasks = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, _addr)) => { + tasks.spawn(async move { + if let Err(error) = proxy_conn(name, conn, host_cid, host_port).await { + warn!(%error, "{name} connection failed"); + } + }); + } + Err(error) => warn!(%error, "{name} accept failed"), + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_proxy_task_result(result); + } + } + } +} + +fn log_proxy_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "proxy task failed"); + } +} + +async fn proxy_conn( + name: &'static str, + mut tcp: TcpStream, + host_cid: u32, + host_port: u32, +) -> Result<()> { + let mut host = VsockStream::connect(VsockAddr::new(host_cid, host_port)) + .await + .with_context(|| format!("dial host {name} cid={host_cid} port={host_port}"))?; + + tokio::io::copy_bidirectional(&mut tcp, &mut host) + .await + .context("proxy connection copy")?; + Ok(()) +} diff --git a/shuttle/src/logging.rs b/shuttle/src/logging.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/logging.rs @@ -0,0 +1,49 @@ +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt::MakeWriter; + +pub fn init() { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(ConsoleAndStderr) + .init(); +} + +#[derive(Clone, Copy, Debug)] +struct ConsoleAndStderr; + +struct TeeWriter { + stderr: io::Stderr, + console: Option, +} + +impl<'a> MakeWriter<'a> for ConsoleAndStderr { + type Writer = TeeWriter; + + fn make_writer(&'a self) -> Self::Writer { + TeeWriter { + stderr: io::stderr(), + console: OpenOptions::new().write(true).open("/dev/console").ok(), + } + } +} + +impl Write for TeeWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.stderr.write_all(buf)?; + if let Some(console) = &mut self.console { + console.write_all(buf)?; + } + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.stderr.flush()?; + if let Some(console) = &mut self.console { + console.flush()?; + } + Ok(()) + } +} diff --git a/shuttle/src/main.rs b/shuttle/src/main.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/main.rs @@ -0,0 +1,62 @@ +#![cfg(target_os = "linux")] + +mod activation; +mod cache; +mod command; +mod dns_proxy; +mod exec; +mod host_proxy; +mod logging; +mod nix_config; +mod protocol; +mod session; + +use std::env; +use std::time::Duration; +use tracing::warn; + +#[macro_export] +macro_rules! cfg { + (@val $key:expr) => { + std::env::var(concat!("SHUTTLE_", $key)) + }; + ($key:expr, $default:expr) => { + cfg!(@val $key) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or($default.to_owned()) + .into() + }; +} + +fn cmdline_param(key: &str) -> Option { + let cmdline = std::fs::read_to_string("/proc/cmdline").ok()?; + cmdline + .split_whitespace() + .find_map(|tok| Some(tok.strip_prefix(key)?.strip_prefix('=')?.to_owned())) +} + +#[tokio::main] +async fn main() { + logging::init(); + + let args: Vec = env::args().collect(); + if args.get(1).map(String::as_str) == Some("enqueue-built-paths") { + cache::enqueue_built_paths(&args[2..]).await; + return; + } + + let port: u32 = cfg!(@val "VSOCK_PORT") + .ok() + .or_else(|| cmdline_param("shuttle.vsock_port")) + .and_then(|s| s.parse().ok()) + .unwrap_or(protocol::DEFAULT_PORT); + let host_cid: u32 = cfg!("HOST_CID", tokio_vsock::VMADDR_CID_HOST); + + loop { + if let Err(error) = session::run(host_cid, port).await { + warn!(host_cid, port, %error, "agent session failed"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} diff --git a/shuttle/src/nix_config.rs b/shuttle/src/nix_config.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/nix_config.rs @@ -0,0 +1,223 @@ +use crate::command::{self, Spec}; +use crate::protocol::v1; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::fs; +use std::io::Write as _; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tempfile::Builder; +use tracing::{info, warn}; + +pub const SPINDLE_RUN_DIR: &str = "/run/spindle"; +pub const SPINDLE_NIX_CONFIG: &str = "/run/spindle/nix.conf"; +pub const SPINDLE_CACHE_CONFIG: &str = "/run/spindle/cache.json"; +pub const SYSTEMCTL_EXECUTABLE: &str = "/run/current-system/sw/bin/systemctl"; + +// nix lives in different places depending on the guest OS (NixOS system +// profile vs. plain /usr/local on e.g. alpine) +pub fn nix_executable() -> &'static str { + static NIX: once_cell::sync::Lazy<&'static str> = once_cell::sync::Lazy::new(|| { + let paths = [ + "/run/current-system/sw/bin/nix", + "/usr/local/bin/nix", + "/usr/bin/nix", + ]; + for candidate in paths { + if Path::new(candidate).exists() { + return candidate; + } + } + "/run/current-system/sw/bin/nix" + }); + &NIX +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct RuntimeCacheConfig { + pub read_urls: Vec, + pub trusted_public_keys: Vec, +} + +// configures nix daemon with the configuration passed from host +pub async fn configure(init: &v1::Init, read_proxy_url: &str) -> Result { + let read_urls = vec![read_proxy_url.to_owned()]; + let cfg = RuntimeCacheConfig { + read_urls, + trusted_public_keys: clean_strings(&init.cache_trusted_public_keys), + }; + + if cfg.read_urls.is_empty() && cfg.trusted_public_keys.is_empty() { + remove_if_exists(SPINDLE_NIX_CONFIG)?; + remove_if_exists(SPINDLE_CACHE_CONFIG)?; + return Ok(cfg); + } + + fs::create_dir_all(SPINDLE_RUN_DIR).with_context(|| format!("create {SPINDLE_RUN_DIR}"))?; + + let cache_json = serde_json::to_vec_pretty(&cfg)?; + write_file_atomic(SPINDLE_CACHE_CONFIG, &cache_json, 0o600)?; + + let mut nix_conf = String::new(); + if !cfg.read_urls.is_empty() { + writeln!( + &mut nix_conf, + "extra-substituters = {}", + cfg.read_urls.join(" ") + ) + .unwrap(); + } + if !cfg.trusted_public_keys.is_empty() { + writeln!( + &mut nix_conf, + "extra-trusted-public-keys = {}", + cfg.trusted_public_keys.join(" ") + ) + .unwrap(); + } + + if nix_conf.is_empty() { + remove_if_exists(SPINDLE_NIX_CONFIG)?; + return Ok(cfg); + } + + write_file_atomic(SPINDLE_NIX_CONFIG, nix_conf.as_bytes(), 0o644)?; + restart_nix_daemon().await; + info!( + read_urls = ?cfg.read_urls, + trusted_public_keys = cfg.trusted_public_keys.len(), + "configured nix cache" + ); + + Ok(cfg) +} + +pub fn clean_strings(values: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::with_capacity(values.len()); + + for value in values { + let value = value.trim(); + if value.is_empty() || !seen.insert(value.to_owned()) { + continue; + } + out.push(value.to_owned()); + } + + out +} + +pub fn clean_store_paths(values: &[String]) -> Vec { + clean_strings(values) + .into_iter() + .filter(|value| value.starts_with("/nix/store/")) + .collect() +} + +pub async fn nix_version() -> String { + let spec = Spec::new(nix_executable()) + .arg("--version") + .timeout(Duration::from_secs(1)); + + let Ok(output) = command::run_capture(spec).await else { + return String::new(); + }; + if !output.success() { + return String::new(); + } + + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +fn write_file_atomic(path: impl AsRef, data: &[u8], mode: u32) -> Result<()> { + let path = path.as_ref(); + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + let prefix = path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| format!(".{name}.tmp-")) + .unwrap_or_else(|| ".tmp-".to_owned()); + + let mut tmp = Builder::new() + .prefix(&prefix) + .permissions(fs::Permissions::from_mode(mode)) + .tempfile_in(dir) + .with_context(|| format!("create temp file for {}", path.display()))?; + + // no separate sync here because we don't need to be crash-safe (this is an + // ephemeral vm) only atomicity is needed + tmp.write_all(data) + .with_context(|| format!("write temp file for {}", path.display()))?; + tmp.persist(path) + .map(|_| ()) + .map_err(|err| err.error) + .with_context(|| format!("install {}", path.display())) +} + +const NIX_DAEMON_SOCKET: &str = "/nix/var/nix/daemon-socket/socket"; + +async fn restart_nix_daemon() { + let systemd = Path::new(SYSTEMCTL_EXECUTABLE).exists(); + let spec = if systemd { + Spec::new(SYSTEMCTL_EXECUTABLE) + .args(["try-restart", "nix-daemon.service"]) + .timeout(Duration::from_secs(5)) + } else { + // on non-systemd we can just kill the daemon and it should restart + Spec::new("pkill") + .args(["-f", "nix-daemon"]) + .timeout(Duration::from_secs(5)) + }; + + match command::run_capture(spec).await { + Ok(output) if output.success() => { + if !systemd { + // init has to respawn the daemon before any step needs it + wait_for_nix_daemon_socket(Duration::from_secs(5)).await; + } + } + // pkill exits 1 when nothing matched, ie. no daemon to restart + Ok(output) if !systemd && output.exit.exit_code == 1 => { + info!("no nix-daemon running, skipping restart") + } + Ok(output) => warn!( + exit_code = output.exit.exit_code, + error = ?output.exit.error, + output = %output.combined_lossy(), + "nix-daemon restart failed" + ), + Err(error) => warn!(%error, "nix-daemon restart failed"), + } +} + +async fn wait_for_nix_daemon_socket(timeout: Duration) { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if tokio::net::UnixStream::connect(NIX_DAEMON_SOCKET) + .await + .is_ok() + { + return; + } + if tokio::time::Instant::now() >= deadline { + warn!( + socket = NIX_DAEMON_SOCKET, + "nix-daemon did not come back after restart" + ); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn remove_if_exists(path: impl AsRef) -> Result<()> { + let path: PathBuf = path.as_ref().to_owned(); + match fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("remove {}", path.display())), + } +} diff --git a/shuttle/src/protocol.rs b/shuttle/src/protocol.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/protocol.rs @@ -0,0 +1,212 @@ +use once_cell::sync::Lazy; +use prost::Message as ProstMessage; +use prost_reflect::DescriptorPool; +use std::io; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub mod v1 { + include!("gen/spindle/agent/v1/spindle.agent.v1.rs"); +} + +pub use v1::Message; + +pub static DESCRIPTOR_POOL: Lazy = Lazy::new(|| { + let bytes = include_bytes!("gen/file_descriptor_set.bin"); + DescriptorPool::decode(&bytes[..]).unwrap() +}); + +macro_rules! impl_reflect { + ($($t:ident),* $(,)?) => { + $( + impl prost_reflect::ReflectMessage for v1::$t { + fn descriptor(&self) -> prost_reflect::MessageDescriptor { + DESCRIPTOR_POOL + .get_message_by_name(concat!("spindle.agent.v1.", stringify!($t))) + .unwrap() + } + } + )* + }; +} + +impl_reflect!( + Hello, + Init, + ExecStart, + ExecStdout, + ExecStderr, + ExecExit, + ActivateConfig, + ActivateConfigResult, + BuiltPaths, + CacheDrain, + CacheDrainResult, + Poweroff, + PoweroffResult, + Message, +); + +pub const PROTOCOL_VERSION: u32 = 1; +pub const DEFAULT_PORT: u32 = 10240; +pub const MAX_MESSAGE_BYTES: usize = 1024 * 1024; + +#[macro_export] +macro_rules! on_payload { + (ref $msg:expr, { $( $field:ident => $body:expr ),* $(,)? }) => { + #[allow(unused_variables)] + $(if let Some(ref $field) = $msg.$field { Some($body) } else)* { None } + }; + ($msg:expr, { $( $field:ident => $body:expr ),* $(,)? }) => { + $(if let Some($field) = $msg.$field { Some($body) } else )* { None } + }; +} + +pub fn kind(msg: &Message) -> &'static str { + // todo(dawn): maybe eventually we should have a custom protoc plugin for + // generating an enum, right now not worth it, when we have more needs for + // it imo we can consider it again + on_payload!(ref msg, { + hello => "hello", + init => "init", + exec_start => "exec_start", + exec_stdout => "exec_stdout", + exec_stderr => "exec_stderr", + exec_exit => "exec_exit", + activate_config => "activate_config", + activate_config_result => "activate_config_result", + built_paths => "built_paths", + cache_drain => "cache_drain", + cache_drain_result => "cache_drain_result", + poweroff => "poweroff", + poweroff_result => "poweroff_result", + }) + .unwrap_or_else(|| unreachable!("validated message has no payload")) +} + +pub fn error_or_empty(error: Option) -> String { + error.filter(|error| !error.is_empty()).unwrap_or_default() +} + +pub async fn write_message(writer: &mut W, msg: &Message) -> io::Result<()> { + if let Err(err) = prost_protovalidate::validate(msg) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("validate agent message: {err}"), + )); + } + + let mut data = Vec::new(); + msg.encode(&mut data) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + if data.len() > MAX_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("agent message exceeded {MAX_MESSAGE_BYTES} bytes"), + )); + } + + writer.write_all(&(data.len() as u32).to_be_bytes()).await?; + writer.write_all(&data).await?; + writer.flush().await +} + +pub async fn read_message(reader: &mut R) -> io::Result> { + let Some(header) = read_header(reader).await? else { + return Ok(None); + }; + let size = u32::from_be_bytes(header) as usize; + if size > MAX_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("agent message exceeded {MAX_MESSAGE_BYTES} bytes"), + )); + } + + let mut data = vec![0; size]; + reader.read_exact(&mut data).await?; + let msg = Message::decode(&data[..]) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + + if let Err(err) = prost_protovalidate::validate(&msg) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("validate agent message: {err}"), + )); + } + + Ok(Some(msg)) +} + +async fn read_header(reader: &mut R) -> io::Result> { + let mut header = [0; 4]; + let mut read = 0; + while read < header.len() { + match reader.read(&mut header[read..]).await { + Ok(0) if read == 0 => return Ok(None), + Ok(0) => { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "partial agent message header", + )); + } + Ok(n) => read += n, + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => return Err(err), + } + } + Ok(Some(header)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn round_trips_protobuf_message() { + let msg = Message { + id: "built-paths".to_owned(), + built_paths: Some(v1::BuiltPaths { + paths: vec!["/nix/store/abc-package".to_owned()], + reason: "post_build_hook".to_owned(), + }), + ..Default::default() + }; + + let mut encoded = Vec::new(); + write_message(&mut encoded, &msg).await.unwrap(); + + let decoded = read_message(&mut &encoded[..]).await.unwrap().unwrap(); + assert!(decoded.built_paths.is_some()); + if let Some(p) = decoded.built_paths { + assert_eq!(p.paths, ["/nix/store/abc-package"]); + assert_eq!(p.reason, "post_build_hook"); + } + } + + #[test] + fn validates_messages() { + // 1. valid message (exactly one field set) + let valid = Message { + id: "test-1".to_owned(), + hello: Some(v1::Hello::default()), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&valid).is_ok()); + + // 2. invalid message (zero fields set) + let invalid_zero = Message { + id: "test-2".to_owned(), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&invalid_zero).is_err()); + + // 3. invalid message (multiple fields set) + let invalid_multi = Message { + id: "test-3".to_owned(), + hello: Some(v1::Hello::default()), + init: Some(v1::Init::default()), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&invalid_multi).is_err()); + } +} diff --git a/shuttle/src/session.rs b/shuttle/src/session.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/session.rs @@ -0,0 +1,244 @@ +use crate::cache::{CacheUploadManager, ReadCacheProxy, WriteCacheProxy}; +use crate::command::Spec; +use crate::dns_proxy::DnsProxy; +use crate::exec; +use crate::nix_config::{self, SYSTEMCTL_EXECUTABLE}; +use crate::on_payload; +use crate::protocol::{self, Message, v1}; +use crate::{activation, command}; +use anyhow::{Context, Result, bail}; +use std::time::Duration; +use tokio::io::{AsyncWrite, BufReader}; +use tokio::sync::mpsc::{self, Sender}; +use tokio::task::{JoinError, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +pub async fn run(host_cid: u32, port: u32) -> Result<()> { + let mut conn = VsockStream::connect(VsockAddr::new(host_cid, port)) + .await + .with_context(|| format!("dial host vsock cid={host_cid} port={port}"))?; + + send_hello(&mut conn).await?; + + let (reader_conn, writer_conn) = tokio::io::split(conn); + let (out_tx, out_rx) = mpsc::channel::(256); + let writer = tokio::spawn(async move { writer_loop(writer_conn, out_rx).await }); + let mut reader = BufReader::new(reader_conn); + + let init = match protocol::read_message(&mut reader).await? { + Some(Message { + init: Some(init), .. + }) => init, + Some(other) => bail!("expected init, got {}", protocol::kind(&other)), + None => bail!("read init: EOF"), + }; + info!(job_id = %init.job_id, "received init"); + + let read_proxy = ReadCacheProxy::start(host_cid, init.cache_read_proxy_port) + .await + .context("start read cache proxy")?; + let write_proxy = WriteCacheProxy::start(host_cid, init.cache_upload_proxy_port) + .await + .context("start write cache proxy")?; + let _dns_proxy = DnsProxy::start(host_cid, init.dns_proxy_port) + .await + .context("start dns proxy")?; + let _cache_cfg = nix_config::configure( + &init, + read_proxy.as_ref().map(ReadCacheProxy::url).unwrap_or(""), + ) + .await + .context("configure nix cache")?; + let uploader = CacheUploadManager::start( + write_proxy.as_ref().map(WriteCacheProxy::url).unwrap_or(""), + out_tx.clone(), + ) + .await + .context("start cache upload manager")?; + + let mut tasks = JoinSet::new(); + let read_result: Result<()> = loop { + tokio::select! { + read = protocol::read_message(&mut reader) => match read { + Ok(Some(msg)) => spawn_message_task(&mut tasks, msg, &out_tx, uploader.clone()), + Ok(None) => break Ok(()), + Err(error) => break Err(error).context("read message"), + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_task_result(result, false); + } + } + }; + + tasks.abort_all(); + while let Some(result) = tasks.join_next().await { + log_task_result(result, true); + } + + drop(out_tx); + let _ = writer.await; + read_result?; + Ok(()) +} + +fn spawn_message_task( + tasks: &mut JoinSet<()>, + msg: Message, + out_tx: &Sender, + uploader: Option, +) { + let kind = protocol::kind(&msg); + let handle = on_payload!(msg, { + activate_config => tasks.spawn(activation::run(msg.id, activate_config, out_tx.clone())), + exec_start => tasks.spawn(exec::run(msg.id, exec_start, out_tx.clone())), + cache_drain => tasks.spawn(run_cache_drain(msg.id, cache_drain, out_tx.clone(), uploader)), + poweroff => tasks.spawn(run_poweroff(msg.id, poweroff, out_tx.clone())), + }); + if handle.is_none() { + warn!(kind, "ignoring unsupported message"); + } +} + +fn log_task_result(result: Result<(), JoinError>, shutting_down: bool) { + match result { + Ok(()) => {} + Err(error) if shutting_down && error.is_cancelled() => {} + Err(error) => warn!(%error, "session handler task failed"), + } +} + +async fn writer_loop(mut conn: W, mut rx: mpsc::Receiver) +where + W: AsyncWrite + Unpin, +{ + while let Some(msg) = rx.recv().await { + if let Err(error) = protocol::write_message(&mut conn, &msg).await { + warn!(%error, "failed to write protocol message"); + break; + } + } +} + +async fn send_hello(conn: &mut VsockStream) -> Result<()> { + let boot_id = tokio::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .await + .unwrap_or_default() + .trim() + .to_owned(); + let nix_version = nix_config::nix_version().await; + + let hello_payload = v1::Hello { + protocol_version: protocol::PROTOCOL_VERSION, + agent_version: env!("CARGO_PKG_VERSION").to_string(), + boot_id: boot_id.clone(), + nix_version: nix_version.clone(), + }; + info!( + protocol = hello_payload.protocol_version, + version = %hello_payload.agent_version, + boot = %hello_payload.boot_id, + nix = %hello_payload.nix_version, + "sent hello" + ); + let hello = Message { + id: "hello".to_owned(), + hello: Some(hello_payload), + ..Default::default() + }; + + protocol::write_message(conn, &hello) + .await + .context("send hello")?; + Ok(()) +} + +async fn run_cache_drain( + id: String, + req: v1::CacheDrain, + out: Sender, + uploader: Option, +) { + let timeout = + (req.timeout_seconds > 0).then(|| Duration::from_secs(u64::from(req.timeout_seconds))); + let stats = match uploader.as_ref() { + Some(uploader) => uploader.drain(timeout).await, + None => Default::default(), + }; + + if let Some(error) = &stats.last_error { + warn!( + %id, + pending = stats.pending, + active = stats.active, + uploaded = stats.uploaded, + failed = stats.failed, + %error, + "cache drain completed with error" + ); + } else { + info!( + %id, + uploaded = stats.uploaded, + failed = stats.failed, + "cache drain completed" + ); + } + + let result = Message { + id, + cache_drain_result: Some(v1::CacheDrainResult { + error: protocol::error_or_empty(stats.last_error), + cache_queued: stats.pending, + cache_active: stats.active, + cache_uploaded: stats.uploaded, + cache_failed: stats.failed, + }), + ..Default::default() + }; + let _ = out.send(result).await; +} + +async fn run_poweroff(id: String, _req: v1::Poweroff, out: Sender) { + let result = Message { + id, + poweroff_result: Some(v1::PoweroffResult { + error: String::new(), + }), + ..Default::default() + }; + let _ = out.send(result).await; + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(100)).await; + + // prefer a clean shutdown through the init system when one is around + // (systemd on NixOS, busybox/openrc elsewhere), then fall back to the + // raw reboot(2) syscall on minimal guests + for poweroff in [SYSTEMCTL_EXECUTABLE, "/sbin/poweroff", "/usr/sbin/poweroff"] { + if !std::path::Path::new(poweroff).exists() { + continue; + } + let mut spec = Spec::new(poweroff).timeout(Duration::from_secs(5)); + if poweroff == SYSTEMCTL_EXECUTABLE { + spec = spec.args(["poweroff"]); + } + match command::run_capture(spec).await { + Ok(output) if output.success() => return, + Ok(output) => warn!( + %poweroff, + exit_code = output.exit.exit_code, + error = ?output.exit.error, + output = %output.combined_lossy(), + "poweroff command failed" + ), + Err(error) => warn!(%poweroff, %error, "poweroff command failed"), + } + } + + // only ever returns on failure + let error = + nix::sys::reboot::reboot(nix::sys::reboot::RebootMode::RB_POWER_OFF).unwrap_err(); + warn!(%error, "reboot(RB_POWER_OFF) syscall failed"); + }); +} diff --git a/spindle/agentproto/protocol.go b/spindle/agentproto/protocol.go new file mode 100644 --- /dev/null +++ b/spindle/agentproto/protocol.go @@ -0,0 +1,98 @@ +package agentproto + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "google.golang.org/protobuf/proto" + + "buf.build/go/protovalidate" + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +const ( + ProtocolVersion = 1 + DefaultPort = 10240 + MaxMessageBytes = 1024 * 1024 +) + +type Message = agentv1.Message + +var validator protovalidate.Validator + +func init() { + var err error + validator, err = protovalidate.New() + if err != nil { + panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err)) + } +} + +type Encoder struct { + mu sync.Mutex + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(msg *Message) error { + if err := validator.Validate(msg); err != nil { + return fmt.Errorf("validate agent message: %w", err) + } + + data, err := proto.Marshal(msg) + if err != nil { + return fmt.Errorf("marshal agent message: %w", err) + } + if len(data) > MaxMessageBytes { + return fmt.Errorf("agent message exceeded %d bytes", MaxMessageBytes) + } + + var header [4]byte + binary.BigEndian.PutUint32(header[:], uint32(len(data))) + + e.mu.Lock() + defer e.mu.Unlock() + if _, err := e.w.Write(header[:]); err != nil { + return err + } + _, err = e.w.Write(data) + return err +} + +type Decoder struct { + r io.Reader +} + +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +func (d *Decoder) Decode() (*Message, error) { + msg := &Message{} + var header [4]byte + if _, err := io.ReadFull(d.r, header[:]); err != nil { + return msg, err + } + + size := binary.BigEndian.Uint32(header[:]) + if size > MaxMessageBytes { + return msg, fmt.Errorf("agent message exceeded %d bytes", MaxMessageBytes) + } + + data := make([]byte, size) + if _, err := io.ReadFull(d.r, data); err != nil { + return msg, err + } + if err := proto.Unmarshal(data, msg); err != nil { + return msg, fmt.Errorf("parse agent message: %w", err) + } + if err := validator.Validate(msg); err != nil { + return msg, fmt.Errorf("validate agent message: %w", err) + } + return msg, nil +} diff --git a/spindle/agentproto/protocol_test.go b/spindle/agentproto/protocol_test.go new file mode 100644 --- /dev/null +++ b/spindle/agentproto/protocol_test.go @@ -0,0 +1,57 @@ +package agentproto + +import ( + "bytes" + "encoding/binary" + "testing" + + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +func TestDecoderRejectsOversizedMessage(t *testing.T) { + var tooLarge bytes.Buffer + var header [4]byte + binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1) + tooLarge.Write(header[:]) + + _, err := NewDecoder(&tooLarge).Decode() + if err == nil { + t.Fatal("expected oversized message error") + } +} + +func TestValidation(t *testing.T) { + // 1. Valid message (exactly one of the payload fields is set) + validMsg := &Message{ + Id: "test-1", + Hello: &agentv1.Hello{ + ProtocolVersion: 1, + AgentVersion: "1.0", + }, + } + if err := validator.Validate(validMsg); err != nil { + t.Fatalf("expected valid message to pass validation, got: %v", err) + } + + // 2. Invalid message: zero payloads set + invalidZeroMsg := &Message{ + Id: "test-2", + } + if err := validator.Validate(invalidZeroMsg); err == nil { + t.Fatal("expected message with zero payloads to fail validation") + } + + // 3. Invalid message: multiple payloads set + invalidMultiMsg := &Message{ + Id: "test-3", + Hello: &agentv1.Hello{ + ProtocolVersion: 1, + }, + Init: &agentv1.Init{ + JobId: "job-1", + }, + } + if err := validator.Validate(invalidMultiMsg); err == nil { + t.Fatal("expected message with multiple payloads to fail validation") + } +} diff --git a/spindle/config/config.go b/spindle/config/config.go --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -2,6 +2,7 @@ import ( "context" + "time" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/sethvargo/go-envconfig" @@ -9,20 +10,20 @@ ) type Server struct { - ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"` - DBPath string `env:"DB_PATH, default=spindle.db"` - Hostname string `env:"HOSTNAME, required"` - JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"` - Tap Tap `env:",prefix=TAP_"` - PlcUrl string `env:"PLC_URL, default=https://plc.directory"` - Dev bool `env:"DEV, default=false"` - Owner string `env:"OWNER, required"` - Secrets Secrets `env:",prefix=SECRETS_"` - LogDir string `env:"LOG_DIR, default=/var/log/spindle"` - QueueSize int `env:"QUEUE_SIZE, default=100"` - MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time - MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) - DockerSocket string `env:"DOCKER_SOCKET"` // path to a docker socket to expose to workflow containers + ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"` + DBPath string `env:"DB_PATH, default=spindle.db"` + Hostname string `env:"HOSTNAME, required"` + JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"` + Tap Tap `env:",prefix=TAP_"` + PlcUrl string `env:"PLC_URL, default=https://plc.directory"` + Dev bool `env:"DEV, default=false"` + DevExtraHosts []string `env:"DEV_EXTRA_HOSTS"` + Owner string `env:"OWNER, required"` + Secrets Secrets `env:",prefix=SECRETS_"` + LogDir string `env:"LOG_DIR, default=/var/log/spindle"` + QueueSize int `env:"QUEUE_SIZE, default=100"` + MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time + DockerSocket string `env:"DOCKER_SOCKET"` // path to a docker socket to expose to workflow containers } type Tap struct { @@ -49,19 +50,54 @@ } type NixeryPipelines struct { - Nixery string `env:"NIXERY, default=nixery.tangled.sh"` - WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` - MaxJobMemoryMB int64 `env:"MAX_JOB_MEMORY_MB, default=6144"` // per-container memory limit in MiB (default 6 GiB) + Nixery string `env:"NIXERY, default=nixery.tangled.sh"` + WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` + MaxJobMemoryMB int64 `env:"MAX_JOB_MEMORY_MB, default=6144"` // per-container memory limit in MiB (default 6 GiB) + MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) } type S3 struct { LogBucket string `env:"LOG_BUCKET"` } +type MicroVMPipelines struct { + ImageDir string `env:"IMAGE_DIR, required"` + OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live + DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` + AgentPort uint32 `env:"AGENT_PORT, default=10240"` + EnableKVM bool `env:"ENABLE_KVM, default=true"` + WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` + + MaxTotalMemoryMiB int64 `env:"MAX_TOTAL_MEMORY_MIB, default=0"` + MaxTotalVCPUs int64 `env:"MAX_TOTAL_VCPUS, default=0"` + MaxTotalDiskMiB int64 `env:"MAX_TOTAL_DISK_MIB, default=0"` + + MaxWorkflowMemoryMiB int64 `env:"MAX_WORKFLOW_MEMORY_MIB, default=0"` + MaxWorkflowVCPUs int64 `env:"MAX_WORKFLOW_VCPUS, default=0"` + MaxWorkflowDiskMiB int64 `env:"MAX_WORKFLOW_DISK_MIB, default=0"` + + AgingThreshold time.Duration `env:"AGING_THRESHOLD, default=30s"` + + EnableCgroups bool `env:"ENABLE_CGROUPS, default=false"` + CgroupParent string `env:"CGROUP_PARENT, default=self"` + CgroupPidsMax int64 `env:"CGROUP_PIDS_MAX, default=4096"` + CgroupSwapMaxMiB *int64 `env:"CGROUP_SWAP_MAX_MIB"` + // memory.min that will get assigned to the supervisor (spindle itself) cgroup + CgroupSupervisorMemoryMinMiB int64 `env:"CGROUP_SUPERVISOR_MEMORY_MIN_MIB, default=512"` +} + +type NixCache struct { + ReadURLs []string `env:"READ_URLS"` + TrustedPublicKeys []string `env:"TRUSTED_PUBLIC_KEYS"` + UploadURL string `env:"UPLOAD_URL"` +} + type Config struct { - Server Server `env:",prefix=SPINDLE_SERVER_"` - NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` - S3 S3 `env:",prefix=SPINDLE_S3_"` + Server Server `env:",prefix=SPINDLE_SERVER_"` + NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` + MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` + NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` + S3 S3 `env:",prefix=SPINDLE_S3_"` } func Load(ctx context.Context) (*Config, error) { diff --git a/spindle/db/db.go b/spindle/db/db.go --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -101,6 +101,12 @@ created integer not null -- unix nanos ); + create table if not exists nixos_toplevel_cache ( + config_key text primary key, + toplevel text not null, + updated_at text not null + ); + create table if not exists migrations ( id integer primary key autoincrement, name text unique diff --git a/spindle/db/nixos_toplevel_cache.go b/spindle/db/nixos_toplevel_cache.go new file mode 100644 --- /dev/null +++ b/spindle/db/nixos_toplevel_cache.go @@ -0,0 +1,41 @@ +package db + +import ( + "time" +) + +type NixOSToplevelCacheRecord struct { + ConfigKey string + Toplevel string + UpdatedAt time.Time +} + +func (d *DB) GetNixOSToplevelCacheRecord(configKey string) (*NixOSToplevelCacheRecord, error) { + var record NixOSToplevelCacheRecord + var updatedAtStr string + err := d.QueryRow( + `select config_key, toplevel, updated_at from nixos_toplevel_cache where config_key = ?`, + configKey, + ).Scan(&record.ConfigKey, &record.Toplevel, &updatedAtStr) + if err != nil { + return nil, err + } + updatedAt, err := time.Parse(time.RFC3339, updatedAtStr) + if err != nil { + return nil, err + } + record.UpdatedAt = updatedAt + return &record, nil +} + +func (d *DB) SaveNixOSToplevelCacheRecord(configKey, toplevel string) error { + _, err := d.Exec( + `insert into nixos_toplevel_cache (config_key, toplevel, updated_at) + values (?, ?, ?) + on conflict(config_key) do update set + toplevel = excluded.toplevel, + updated_at = excluded.updated_at`, + configKey, toplevel, time.Now().UTC().Format(time.RFC3339), + ) + return err +} diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -20,7 +20,11 @@ ErrWorkflowFailed = errors.New("workflow failed") ) -func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, workflowSem chan struct{}, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { +type workflowFinalizer interface { + FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error +} + +func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { l.Info("starting all workflows in parallel", "pipeline", pipelineId) // extract secrets @@ -74,15 +78,27 @@ defer wfLogger.Close() } + l.Info("waiting for slot", "wid", wid) + slot := WorkflowSlot(NoopSlot{}) + if s, ok := eng.(WorkflowSlotter); ok { + var err error + slot, err = s.AcquireWorkflowSlot(ctx, wid, &w) + if err != nil { + l.Error("failed to acquire slot", "wid", wid, "err", err) + dbErr := db.StatusFailed(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) + } + return + } + } + defer slot.Release() + err = db.StatusRunning(wid, n) if err != nil { l.Error("failed to set workflow status to running", "wid", wid, "err", err) return } - - // acquire semaphore slot before starting the container - workflowSem <- struct{}{} - defer func() { <-workflowSem }() err = eng.SetupWorkflow(ctx, wid, &w, wfLogger) if err != nil { @@ -134,6 +150,16 @@ if dbErr != nil { l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) } + } + return + } + } + + if finalizer, ok := eng.(workflowFinalizer); ok { + if err := finalizer.FinalizeWorkflow(ctx, wid, &w, wfLogger); err != nil { + dbErr := db.StatusFailed(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) } return } diff --git a/spindle/engine/scheduler.go b/spindle/engine/scheduler.go new file mode 100644 --- /dev/null +++ b/spindle/engine/scheduler.go @@ -0,0 +1,139 @@ +package engine + +import ( + "context" + "fmt" + "slices" + "sync" + "time" +) + +const defaultAgingThreshold = 30 * time.Second + +type Resources[Self any] interface { + Fits(Self) bool + Add(Self) Self + Sub(Self) Self +} + +type ResourceScheduler[R Resources[R]] struct { + mu sync.Mutex + budget R + max R + used R + queue []*resourceWaiter[R] + now func() time.Time // get time now, is a field for mocking + agingThreshold time.Duration +} + +type resourceWaiter[R Resources[R]] struct { + req R + ready chan struct{} + enqueuedAt time.Time +} + +type resourceLease[R Resources[R]] struct { + scheduler *ResourceScheduler[R] + req R + once sync.Once +} + +func NewResourceScheduler[R Resources[R]](budget, max R, agingThreshold time.Duration) *ResourceScheduler[R] { + if agingThreshold <= 0 { + agingThreshold = defaultAgingThreshold + } + return &ResourceScheduler[R]{ + budget: budget, + max: max, + now: time.Now, + agingThreshold: agingThreshold, + } +} + +func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) { + if s == nil { + return NoopSlot{}, nil + } + + s.mu.Lock() + if !req.Fits(s.budget) || !req.Fits(s.max) { + s.mu.Unlock() + return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max) + } + if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) { + s.used = s.used.Add(req) + s.mu.Unlock() + return &resourceLease[R]{scheduler: s, req: req}, nil + } + + waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()} + s.queue = append(s.queue, waiter) + s.schedule() + s.mu.Unlock() + + select { + case <-waiter.ready: + return &resourceLease[R]{scheduler: s, req: req}, nil + case <-ctx.Done(): + s.mu.Lock() + select { + case <-waiter.ready: + // undo committed resources, schedule already did that + s.used = s.used.Sub(req) + default: + // still in queue, just remove + s.remove(waiter) + } + s.schedule() + s.mu.Unlock() + return nil, ctx.Err() + } +} + +func (l *resourceLease[R]) Release() { + if l == nil || l.scheduler == nil { + return + } + l.once.Do(func() { + l.scheduler.release(l.req) + }) +} + +func (s *ResourceScheduler[R]) release(req R) { + s.mu.Lock() + defer s.mu.Unlock() + s.used = s.used.Sub(req) + s.schedule() +} + +// start every waiter whose request fits. once a waiter is older than +// agingThreshold, count its request as already used so younger waiters +// stop being scheduled ahead of it. +func (s *ResourceScheduler[R]) schedule() { + var reserved R + now := s.now() + i := 0 + for i < len(s.queue) { + w := s.queue[i] + if s.used.Add(reserved).Add(w.req).Fits(s.budget) { + s.queue = slices.Delete(s.queue, i, i+1) + s.used = s.used.Add(w.req) + close(w.ready) + continue + } + if now.Sub(w.enqueuedAt) >= s.agingThreshold { + reserved = reserved.Add(w.req) + } + i++ + } +} + +func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) { + for i, candidate := range s.queue { + if candidate != waiter { + continue + } + s.queue = slices.Delete(s.queue, i, i+1) + return + } +} diff --git a/spindle/engine/scheduler_test.go b/spindle/engine/scheduler_test.go new file mode 100644 --- /dev/null +++ b/spindle/engine/scheduler_test.go @@ -0,0 +1,200 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// resources for testing +type ru struct{ a, b int64 } + +func (r ru) Fits(limit ru) bool { + if limit.a > 0 && r.a > limit.a { + return false + } + if limit.b > 0 && r.b > limit.b { + return false + } + return true +} +func (r ru) Add(o ru) ru { return ru{r.a + o.a, r.b + o.b} } +func (r ru) Sub(o ru) ru { return ru{max(0, r.a-o.a), max(0, r.b-o.b)} } +func (r ru) String() string { + return fmt.Sprintf("a=%d b=%d", r.a, r.b) +} + +type acquireResult struct { + slot WorkflowSlot + err error +} + +func TestResourceSchedulerZeroLimitsDoNotApply(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{}, ru{}, 0) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}) + if err != nil { + t.Fatalf("Acquire() error = %v", err) + } + slot.Release() +} + +func TestResourceSchedulerRejectsRequestsThatCanNeverFit(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) + + _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}) + if !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) + } + + _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}) + if !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) + } +} + +func TestResourceSchedulerWaitsUntilResourcesAreReleased(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) + + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}) + if err != nil { + t.Fatalf("first Acquire() error = %v", err) + } + defer first.Release() + + ch := acquireAsync(context.Background(), scheduler, ru{a: 1}) + assertAcquireBlocked(t, ch) + + first.Release() + first = NoopSlot{} + + second := waitAcquireOK(t, ch) + second.Release() +} + +func TestResourceSchedulerReleaseIsIdempotent(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 1}) + if err != nil { + t.Fatalf("Acquire() error = %v", err) + } + + slot.Release() + slot.Release() + + second, err := scheduler.Acquire(context.Background(), ru{a: 1}) + if err != nil { + t.Fatalf("Acquire() after double release error = %v", err) + } + second.Release() +} + +func TestResourceSchedulerBackfillsPastBlockedHead(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill + + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + if err != nil { + t.Fatalf("hold Acquire() error = %v", err) + } + defer hold.Release() + + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) + assertAcquireBlocked(t, bigCh) + + smallCh := acquireAsync(context.Background(), scheduler, ru{a: 256}) + small := waitAcquireOK(t, smallCh) + small.Release() + + assertAcquireBlocked(t, bigCh) +} + +func TestResourceSchedulerAgingReservesCapacityForBlockedHead(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 10*time.Millisecond) + fakeNow := time.Now() + scheduler.now = func() time.Time { return fakeNow } + + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + if err != nil { + t.Fatalf("hold Acquire() error = %v", err) + } + + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) + assertAcquireBlocked(t, bigCh) + + fakeNow = fakeNow.Add(time.Second) + + // big is now aged and reserves its 768. a 256 request would fit + // alongside the held 512, but the reservation blocks it. + smallCh := acquireAsync(context.Background(), scheduler, ru{a: 256}) + assertAcquireBlocked(t, smallCh) + + hold.Release() + + big := waitAcquireOK(t, bigCh) + small := waitAcquireOK(t, smallCh) + small.Release() + big.Release() +} + +func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult { + ch := make(chan acquireResult, 1) + go func() { + slot, err := scheduler.Acquire(ctx, req) + ch <- acquireResult{slot: slot, err: err} + }() + return ch +} + +func assertAcquireBlocked(t *testing.T, ch <-chan acquireResult) { + t.Helper() + + select { + case res := <-ch: + if res.slot != nil { + res.slot.Release() + } + t.Fatalf("Acquire() returned before resources were available: err=%v", res.err) + case <-time.After(25 * time.Millisecond): + } +} + +func waitAcquireOK(t *testing.T, ch <-chan acquireResult) WorkflowSlot { + t.Helper() + + res := waitAcquireResult(t, ch) + if res.err != nil { + t.Fatalf("Acquire() error = %v", res.err) + } + if res.slot == nil { + t.Fatal("Acquire() returned nil slot") + } + return res.slot +} + +func waitAcquireResult(t *testing.T, ch <-chan acquireResult) acquireResult { + t.Helper() + + select { + case res := <-ch: + return res + case <-time.After(time.Second): + t.Fatal("timed out waiting for Acquire() result") + } + + return acquireResult{} +} diff --git a/spindle/engine/slot.go b/spindle/engine/slot.go new file mode 100644 --- /dev/null +++ b/spindle/engine/slot.go @@ -0,0 +1,54 @@ +package engine + +import ( + "context" + "errors" + + "tangled.org/core/spindle/models" +) + +var ErrNoWorkflowSlots = errors.New("no workflow slots available") + +type WorkflowSlot interface { + Release() +} + +type WorkflowSlotter interface { + AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) +} + +type releaseFunc func() + +func (f releaseFunc) Release() { + if f != nil { + f() + } +} + +type NoopSlot struct{} + +func (NoopSlot) Release() {} + +// limit by concurrent workflow count +type SemaphoreSlotter struct { + slots chan struct{} +} + +func NewSemaphoreSlotter(maxConcurrent int) *SemaphoreSlotter { + if maxConcurrent <= 0 { + return &SemaphoreSlotter{} + } + return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)} +} + +func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) { + if a == nil || a.slots == nil { + return NoopSlot{}, nil + } + select { + case a.slots <- struct{}{}: + return releaseFunc(func() { <-a.slots }), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/spindle/engine/slot_test.go b/spindle/engine/slot_test.go new file mode 100644 --- /dev/null +++ b/spindle/engine/slot_test.go @@ -0,0 +1,112 @@ +package engine + +import ( + "context" + "errors" + "testing" + "time" + + "tangled.org/core/spindle/models" +) + +func TestSemaphoreSlotterDisabledDoesNotBlock(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(0) + + for range 10 { + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("AcquireWorkflowSlot() error = %v", err) + } + slot.Release() + } +} + +func TestSemaphoreSlotterBlocksUntilRelease(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(1) + + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("first AcquireWorkflowSlot() error = %v", err) + } + releasedFirst := false + defer func() { + if !releasedFirst { + first.Release() + } + }() + + acquired := make(chan WorkflowSlot, 1) + errs := make(chan error, 1) + go func() { + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + errs <- err + return + } + acquired <- slot + }() + + assertNotAcquired(t, acquired, errs) + + first.Release() + releasedFirst = true + + second := waitForSlot(t, acquired, errs) + second.Release() +} + +func TestSemaphoreSlotterHonorsContextCancellation(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(1) + + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("first AcquireWorkflowSlot() error = %v", err) + } + defer first.Release() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err) + } +} + +func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) { + t.Helper() + + select { + case slot := <-acquired: + slot.Release() + t.Fatal("AcquireWorkflowSlot() acquired a slot before one was released") + case err := <-errs: + t.Fatalf("AcquireWorkflowSlot() returned unexpected error: %v", err) + case <-time.After(25 * time.Millisecond): + } +} + +func waitForSlot(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) WorkflowSlot { + t.Helper() + + select { + case slot := <-acquired: + return slot + case err := <-errs: + t.Fatalf("AcquireWorkflowSlot() returned error: %v", err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for slot acquisition") + } + + return nil +} + +func zeroWorkflowID() models.WorkflowId { + return models.WorkflowId{} +} diff --git a/spindle/models/clone.go b/spindle/models/clone.go --- a/spindle/models/clone.go +++ b/spindle/models/clone.go @@ -5,6 +5,7 @@ "strings" "tangled.org/core/api/tangled" + "tangled.org/core/hostutil" "tangled.org/core/workflow" ) @@ -55,7 +56,7 @@ } } - repoURL := BuildRepoURL(tr.Repo, dev) + repoURL := BuildRepoURL(tr.Repo) var cloneOpts tangled.Pipeline_CloneOpts if twf.Clone != nil { @@ -63,13 +64,20 @@ } fetchArgs := buildFetchArgs(cloneOpts, commitSHA) + // In dev mode we point at Caddy via host-gateway with a self-signed cert, + // so skip the TLS check for the fetch call. + fetchCmd := "git fetch" + if dev { + fetchCmd = "git -c http.sslVerify=false fetch" + } + return CloneStep{ kind: StepKindSystem, name: "Clone repository into workspace", commands: []string{ "git init", fmt.Sprintf("git remote add origin %s", repoURL), - fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")), + fmt.Sprintf("%s %s", fetchCmd, strings.Join(fetchArgs, " ")), "git checkout FETCH_HEAD", }, } @@ -102,21 +110,18 @@ } // BuildRepoURL constructs the repository URL from repo metadata. -func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo, devMode bool) string { - scheme := "https://" - if devMode { - scheme = "http://" +func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo) string { + if repo == nil { + return "" } - // Get host from knot - host := repo.Knot - - // In dev mode, replace localhost with host.docker.internal for Docker networking - if devMode && strings.Contains(host, "localhost") { - host = strings.ReplaceAll(host, "localhost", "host.docker.internal") + host, noSSL, _ := hostutil.ParseHostname(repo.Knot) + scheme := "https" + if noSSL { + scheme = "http" } - return fmt.Sprintf("%s%s/%s", scheme, host, *repo.RepoDid) + return fmt.Sprintf("%s://%s/%s", scheme, host, *repo.RepoDid) } // buildFetchArgs constructs the arguments for git fetch based on clone options diff --git a/spindle/models/clone_test.go b/spindle/models/clone_test.go --- a/spindle/models/clone_test.go +++ b/spindle/models/clone_test.go @@ -166,6 +166,7 @@ func TestBuildCloneStep_DevMode(t *testing.T) { twf := tangled.Pipeline_Workflow{ + Engine: "nixery", Clone: &tangled.Pipeline_CloneOpts{ Depth: 1, Skip: false, @@ -177,7 +178,7 @@ NewSha: "abc123", }, Repo: &tangled.Pipeline_TriggerRepo{ - Knot: "localhost:3000", + Knot: "knot.tngl.boltless.dev", Did: "did:plc:user123", Repo: sp("my-repo"), RepoDid: sp("did:plc:boltless"), @@ -186,11 +187,10 @@ step := BuildCloneStep(twf, tr, true) - // In dev mode, should use http:// and replace localhost with host.docker.internal + // In dev mode, sslVerify should be disabled allCmds := strings.Join(step.Commands(), " ") - expectedURL := "http://host.docker.internal:3000/did:plc:boltless" - if !strings.Contains(allCmds, expectedURL) { - t.Errorf("Expected dev mode URL '%s' in commands", expectedURL) + if !strings.Contains(allCmds, "git -c http.sslVerify=false fetch") { + t.Error("Expected sslVerify to be disabled in dev mode clone commands") } } diff --git a/spindle/models/pipeline_env.go b/spindle/models/pipeline_env.go --- a/spindle/models/pipeline_env.go +++ b/spindle/models/pipeline_env.go @@ -10,7 +10,7 @@ // PipelineEnvVars extracts environment variables from pipeline trigger metadata. // These are framework-provided variables that are injected into workflow steps. -func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId, devMode bool) map[string]string { +func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId) map[string]string { if tr == nil { return nil } @@ -34,7 +34,7 @@ env["TANGLED_REPO_REPO_DID"] = *tr.Repo.RepoDid } env["TANGLED_REPO_DEFAULT_BRANCH"] = tr.Repo.DefaultBranch - env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo, devMode) + env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo) } switch workflow.TriggerKind(tr.Kind) { diff --git a/spindle/models/pipeline_env_test.go b/spindle/models/pipeline_env_test.go --- a/spindle/models/pipeline_env_test.go +++ b/spindle/models/pipeline_env_test.go @@ -27,7 +27,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check standard CI variable if env["CI"] != "true" { @@ -90,7 +90,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) if env["TANGLED_REF"] != "refs/tags/v1.2.3" { t.Errorf("Expected TANGLED_REF='refs/tags/v1.2.3', got '%s'", env["TANGLED_REF"]) @@ -123,7 +123,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check ref variables for PR if env["TANGLED_REF"] != "refs/heads/feature-branch" { @@ -179,7 +179,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check manual input variables if env["TANGLED_INPUT_VERSION"] != "1.0.0" { @@ -216,10 +216,9 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, true) + env := PipelineEnvVars(tr, id) - // Dev mode should use http:// and replace localhost with host.docker.internal - expectedURL := "http://host.docker.internal:3000/did:plc:boltless" + expectedURL := "http://localhost:3000/did:plc:boltless" if env["TANGLED_REPO_URL"] != expectedURL { t.Errorf("Expected TANGLED_REPO_URL='%s', got '%s'", expectedURL, env["TANGLED_REPO_URL"]) } @@ -230,7 +229,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(nil, id, false) + env := PipelineEnvVars(nil, id) if env != nil { t.Error("Expected nil env for nil trigger") @@ -252,7 +251,7 @@ Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Should still have repo variables if env["TANGLED_REPO_KNOT"] != "example.com" { diff --git a/spindle/queue/queue.go b/spindle/queue/queue.go --- a/spindle/queue/queue.go +++ b/spindle/queue/queue.go @@ -1,7 +1,10 @@ package queue import ( + "slices" "sync" + + "github.com/bluesky-social/indigo/atproto/syntax" ) type Job struct { @@ -9,26 +12,43 @@ OnFail func(error) } +type ownedJob struct { + owner syntax.DID + job Job +} + +// prefers users with fewer running jobs, otherwise it's FIFO type Queue struct { - jobs chan Job + mu sync.Mutex + cond *sync.Cond + queue []ownedJob + running map[syntax.DID]int + maxSize int workers int + stopped bool wg sync.WaitGroup } func NewQueue(queueSize, numWorkers int) *Queue { - return &Queue{ - jobs: make(chan Job, queueSize), + q := &Queue{ + maxSize: queueSize, workers: numWorkers, + running: make(map[syntax.DID]int), } + q.cond = sync.NewCond(&q.mu) + return q } -func (q *Queue) Enqueue(job Job) bool { - select { - case q.jobs <- job: - return true - default: +// todo(dawn): add a per-user cap so a single user can't fill the queue +func (q *Queue) Enqueue(owner syntax.DID, job Job) bool { + q.mu.Lock() + defer q.mu.Unlock() + if q.stopped || len(q.queue) >= q.maxSize { return false } + q.queue = append(q.queue, ownedJob{owner: owner, job: job}) + q.cond.Signal() + return true } func (q *Queue) Start() { @@ -40,16 +60,68 @@ func (q *Queue) worker() { defer q.wg.Done() - for job := range q.jobs { - if err := job.Run(); err != nil { - if job.OnFail != nil { - job.OnFail(err) - } + for { + picked, ok := q.takeNext() + if !ok { + return } + + err := picked.job.Run() + if err != nil && picked.job.OnFail != nil { + picked.job.OnFail(err) + } + + q.finish(picked.owner) + } +} + +// get or wait for the next job +func (q *Queue) takeNext() (ownedJob, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + for len(q.queue) == 0 && !q.stopped { + q.cond.Wait() // waiting for jobs + } + if q.stopped && len(q.queue) == 0 { + return ownedJob{}, false // no jobs are left and the queue is stopped + } + + idx := q.pickBest() + picked := q.queue[idx] + q.queue = slices.Delete(q.queue, idx, idx+1) + q.running[picked.owner]++ + + return picked, true +} + +// index of the queued job whose owner has the fewest currently-running jobs, +// tiebreaking by arrival order. +func (q *Queue) pickBest() int { + best := 0 + for idx, job := range q.queue { + if q.running[job.owner] < q.running[q.queue[best].owner] { + best = idx + } + } + return best +} + +// called when finishing a job +func (q *Queue) finish(owner syntax.DID) { + q.mu.Lock() + defer q.mu.Unlock() + + q.running[owner]-- + if q.running[owner] <= 0 { + delete(q.running, owner) } } func (q *Queue) Stop() { - close(q.jobs) + q.mu.Lock() + q.stopped = true + q.cond.Broadcast() + q.mu.Unlock() q.wg.Wait() } diff --git a/spindle/queue/queue_test.go b/spindle/queue/queue_test.go new file mode 100644 --- /dev/null +++ b/spindle/queue/queue_test.go @@ -0,0 +1,103 @@ +package queue + +import ( + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +const ( + alice = syntax.DID("did:plc:alice") + eve = syntax.DID("did:plc:eve") + dawn = syntax.DID("did:plc:dawn") +) + +func TestQueueDrainsAllJobs(t *testing.T) { + t.Parallel() + + q := NewQueue(10, 2) + q.Start() + + var mu sync.Mutex + var ran []string + done := make(chan struct{}, 5) + + for _, name := range []string{"a", "b", "c", "d", "e"} { + q.Enqueue(dawn, Job{Run: func() error { + mu.Lock() + ran = append(ran, name) + mu.Unlock() + done <- struct{}{} + return nil + }}) + } + + for range 5 { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("timed out waiting for jobs to finish") + } + } + + q.Stop() + + if len(ran) != 5 { + t.Fatalf("expected 5 jobs, ran %d", len(ran)) + } +} + +func TestQueueRejectsWhenFull(t *testing.T) { + t.Parallel() + + // no workers, so the queue never drains + q := NewQueue(2, 0) + + if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("first Enqueue() returned false, want true") + } + if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("second Enqueue() returned false, want true") + } + if q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("third Enqueue() returned true on full queue, want false") + } +} + +func TestQueuePrefersOwnerWithFewestRunning(t *testing.T) { + t.Parallel() + + // 2 workers. alice gets the first slot; while she's holding it, eve's + // job should win the second slot over alice's own queued waiters. + q := NewQueue(20, 2) + + releaseAlice1 := make(chan struct{}) + gotEve := make(chan struct{}, 1) + + q.Enqueue(alice, Job{Run: func() error { + <-releaseAlice1 + return nil + }}) + // alice queues two more + q.Enqueue(alice, Job{Run: func() error { return nil }}) + q.Enqueue(alice, Job{Run: func() error { return nil }}) + // eve queues one + q.Enqueue(eve, Job{Run: func() error { + gotEve <- struct{}{} + return nil + }}) + + q.Start() + + // eve should run while alice's first is still held + select { + case <-gotEve: + case <-time.After(time.Second): + t.Fatal("eve's job did not run while alice was blocked") + } + + close(releaseAlice1) + q.Stop() +} diff --git a/appview/pipelines/ssh/tui.go b/appview/pipelines/ssh/tui.go --- a/appview/pipelines/ssh/tui.go +++ b/appview/pipelines/ssh/tui.go @@ -113,7 +113,7 @@ if !ok || len(ws.Data) == 0 { return logDoneMsg{workflow: workflow} } - url := pipelines.SpindleURL(m.server.config.Core.Dev, ws.Data[0].Spindle, m.pipeline.Knot, m.pipeline.Rkey, workflow) + url := pipelines.SpindleURL(ws.Data[0].Spindle, m.pipeline.Knot, m.pipeline.Rkey, workflow) conn, _, err := websocket.DefaultDialer.Dial(url, nil) if err != nil { return logDoneMsg{workflow: workflow, err: fmt.Errorf("connecting to spindle: %w", err)} diff --git a/shuttle/src/cache/mod.rs b/shuttle/src/cache/mod.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/cache/mod.rs @@ -0,0 +1,724 @@ +use crate::command::{self, Spec}; +use crate::nix_config::{SPINDLE_RUN_DIR, clean_store_paths, nix_executable}; +use crate::protocol::{Message, v1}; +use anyhow::{Context, Result}; +use nix::unistd::{Group, chown}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Write as FmtWrite}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read}; +use std::net::Shutdown; +use std::os::unix::fs::OpenOptionsExt; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::sync::{Semaphore, mpsc, oneshot, watch}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VMADDR_CID_LOCAL, VsockAddr, VsockListener, VsockStream}; +use tracing::{info, warn}; + +mod read_proxy; +mod write_proxy; + +pub use read_proxy::ReadCacheProxy; +pub use write_proxy::WriteCacheProxy; + +const UPLOAD_QUEUE_CAPACITY: usize = 128; +const CONNECTION_WORKERS: usize = 4; +const CACHE_ENQUEUE_IO_TIMEOUT: Duration = Duration::from_secs(2); +const DEFAULT_CACHE_ENQUEUE_PORT: u32 = 10241; +const SHUTTLE_CACHE_ENQUEUE_PORT_ENV: &str = "SHUTTLE_CACHE_VSOCK_PORT"; +const NIX_BUILD_GROUP: &str = "nixbld"; +const SPINDLE_HOOK_TOKEN: &str = "/run/spindle/hook-token"; + +#[derive(Clone, Debug, Default)] +pub struct CacheStats { + pub pending: u32, + pub active: u32, + pub uploaded: u32, + pub failed: u32, + pub last_error: Option, +} + +#[derive(Debug, Default)] +struct CacheState { + stats: CacheStats, + enqueue_active: u32, + stopped: bool, +} + +impl CacheState { + fn snapshot(&self) -> CacheSnapshot { + CacheSnapshot { + stats: self.stats.clone(), + enqueue_active: self.enqueue_active, + stopped: self.stopped, + } + } +} + +#[derive(Clone, Debug, Default)] +struct CacheSnapshot { + stats: CacheStats, + enqueue_active: u32, + stopped: bool, +} + +impl CacheSnapshot { + fn is_idle(&self) -> bool { + self.stats.pending == 0 && self.stats.active == 0 && self.enqueue_active == 0 + } +} + +#[derive(Clone)] +pub struct CacheUploadManager { + inner: Arc, +} + +struct CacheUploadInner { + cmd_tx: mpsc::Sender, + stats_rx: watch::Receiver, + handles: Mutex>>, +} + +struct UploadJob { + paths: Vec, + count: u32, +} + +enum Cmd { + EnqueueStarted, + EnqueueFinished, + Enqueue { + paths: Vec, + reply: oneshot::Sender>, + }, + UploadStarted { + count: u32, + }, + UploadFinished { + count: u32, + error: Option, + }, + UploadWorkerStopped, + Stop, +} + +impl CacheUploadManager { + pub async fn start(upload_url: &str, event_tx: mpsc::Sender) -> Result> { + if upload_url.is_empty() { + // nothing to upload to, so don't require the guest-local vsock + // listener (vsock_loopback) or the nix post-build hook + info!("no cache upload url configured, cache uploads disabled"); + return Ok(None); + } + let token = create_hook_token().context("create cache hook token")?; + let port = cache_enqueue_port(); + let listener = VsockListener::bind(VsockAddr::new(VMADDR_CID_LOCAL, port)) + .with_context(|| format!("listen on guest-local vsock port {port}"))?; + + let (cmd_tx, cmd_rx) = mpsc::channel::(UPLOAD_QUEUE_CAPACITY); + let (upload_tx, upload_rx) = mpsc::channel::(UPLOAD_QUEUE_CAPACITY); + let (stats_tx, stats_rx) = watch::channel(CacheSnapshot::default()); + + let mut handles = Vec::with_capacity(3); + + handles.push(tokio::spawn(async move { + cache_manager_loop(cmd_rx, upload_tx, stats_tx).await; + })); + + let upload_cmd_tx = cmd_tx.clone(); + let upload_url = upload_url.to_owned(); + handles.push(tokio::spawn(async move { + upload_loop(upload_rx, upload_cmd_tx, upload_url).await; + })); + + let accept_cmd_tx = cmd_tx.clone(); + handles.push(tokio::spawn(async move { + accept_loop(listener, token, event_tx, accept_cmd_tx).await; + })); + + info!( + port, + workers = CONNECTION_WORKERS, + "cache upload queue ready" + ); + let inner = Arc::new(CacheUploadInner { + cmd_tx, + stats_rx, + handles: Mutex::new(handles), + }); + Ok(Some(Self { inner })) + } + + pub async fn drain(&self, timeout: Option) -> CacheStats { + let mut stats_rx = self.inner.stats_rx.clone(); + let wait = async { + loop { + let snapshot = stats_rx.borrow_and_update().clone(); + if snapshot.is_idle() || snapshot.stopped { + return snapshot.stats; + } + if stats_rx.changed().await.is_err() { + let mut stats = stats_rx.borrow().stats.clone(); + stats.last_error = Some("cache manager stopped".to_owned()); + return stats; + } + } + }; + + match timeout { + Some(timeout) => match tokio::time::timeout(timeout, wait).await { + Ok(stats) => stats, + Err(_) => { + let mut stats = self.inner.stats_rx.borrow().stats.clone(); + stats.last_error = Some("cache drain timed out".to_owned()); + stats + } + }, + None => wait.await, + } + } +} + +impl Drop for CacheUploadInner { + fn drop(&mut self) { + let _ = self.cmd_tx.try_send(Cmd::Stop); + if let Ok(mut handles) = self.handles.lock() { + for handle in handles.drain(..) { + handle.abort(); + } + } + let _ = fs::remove_file(SPINDLE_HOOK_TOKEN); + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct EnqueueBuiltPathsRequest { + token: String, + paths: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct EnqueueBuiltPathsResponse { + queued: usize, + #[serde(default, skip_serializing_if = "String::is_empty")] + error: String, +} + +#[derive(Debug)] +enum JsonLineError { + Empty, + TimedOut, + Io(io::Error), + Json(serde_json::Error), +} + +impl JsonLineError { + fn enqueue_request_message(self) -> String { + match self { + Self::Empty => "empty cache enqueue request".to_owned(), + Self::TimedOut => "cache enqueue read timed out".to_owned(), + Self::Io(error) => error.to_string(), + Self::Json(error) => error.to_string(), + } + } +} + +impl fmt::Display for JsonLineError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => f.write_str("empty message"), + Self::TimedOut => f.write_str("timed out"), + Self::Io(error) => error.fmt(f), + Self::Json(error) => error.fmt(f), + } + } +} + +async fn cache_manager_loop( + mut cmd_rx: mpsc::Receiver, + upload_tx: mpsc::Sender, + stats_tx: watch::Sender, +) { + let mut state = CacheState::default(); + + let publish_state = |state: &CacheState| { + let _ = stats_tx.send(state.snapshot()); + }; + + while let Some(command) = cmd_rx.recv().await { + match command { + Cmd::EnqueueStarted => { + state.enqueue_active += 1; + publish_state(&state); + } + Cmd::EnqueueFinished => { + decrement_counter(&mut state.enqueue_active, 1, "cache enqueue active"); + publish_state(&state); + } + Cmd::Enqueue { paths, reply } => { + let result = enqueue_upload_job(&mut state, &upload_tx, paths); + publish_state(&state); + let _ = reply.send(result); + } + Cmd::UploadStarted { count } => { + decrement_counter(&mut state.stats.pending, count, "cache pending uploads"); + state.stats.active += count; + publish_state(&state); + } + Cmd::UploadFinished { count, error } => { + decrement_counter(&mut state.stats.active, count, "cache active uploads"); + if let Some(error) = error { + state.stats.failed += count; + state.stats.last_error = Some(error); + } else { + state.stats.uploaded += count; + } + publish_state(&state); + } + Cmd::UploadWorkerStopped => { + state.stopped = true; + publish_state(&state); + } + Cmd::Stop => { + state.stopped = true; + publish_state(&state); + break; + } + } + } + + state.stopped = true; + publish_state(&state); +} + +fn enqueue_upload_job( + state: &mut CacheState, + upload_tx: &mpsc::Sender, + paths: Vec, +) -> Result { + if state.stopped { + return Err("cache manager stopped".to_owned()); + } + + let count = paths.len() as u32; + if count == 0 { + return Ok(0); + } + + match upload_tx.try_send(UploadJob { paths, count }) { + Ok(()) => { + state.stats.pending += count; + Ok(count as usize) + } + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => { + state.stopped = true; + Err("cache upload worker stopped".to_owned()) + } + } +} + +fn decrement_counter(counter: &mut u32, count: u32, name: &'static str) { + match counter.checked_sub(count) { + Some(value) => *counter = value, + None => { + warn!(name, current = *counter, count, "cache counter underflow"); + *counter = 0; + } + } +} + +async fn upload_loop( + mut upload_rx: mpsc::Receiver, + cmd_tx: mpsc::Sender, + upload_url: String, +) { + while let Some(job) = upload_rx.recv().await { + let cmd = Cmd::UploadStarted { count: job.count }; + if cmd_tx.send(cmd).await.is_err() { + break; + } + + let error = upload_paths(&upload_url, &job.paths) + .await + .err() + .map(|error| error.to_string()); + + let cmd = Cmd::UploadFinished { + count: job.count, + error, + }; + if cmd_tx.send(cmd).await.is_err() { + break; + } + } + + let _ = cmd_tx.send(Cmd::UploadWorkerStopped).await; +} + +// runs nix copy against the write cache proxy, which goes to the spindle +// and spindle will then forward the request to the actual binary cache +async fn upload_paths(upload_url: &str, paths: &[String]) -> Result<()> { + fn add_query_param(url: &str, key: &str, value: &str) -> String { + let separator = url.contains('?').then_some('&').unwrap_or('?'); + format!("{}{}{}={}", url, separator, key, value) + } + + if paths.is_empty() || upload_url.is_empty() { + return Ok(()); + } + + // we use zstd 3 because its the best usually. it is faster than no + // compression also because of IO savings + let dest_url = add_query_param(upload_url, "compression", "zstd"); + let dest_url = add_query_param(&dest_url, "compression-level", "3"); + let dest_url = add_query_param(&dest_url, "parallel-compression", "true"); + + let spec = Spec::new(nix_executable()) + .args(["copy", "--to", &dest_url]) + .args(paths.iter().cloned()) + .timeout(Duration::from_secs(10 * 60)); + + let output = command::run_capture(spec).await.context("run nix copy")?; + if !output.success() { + anyhow::bail!( + "nix copy failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + info!(paths = paths.len(), %upload_url, "uploaded cache paths"); + Ok(()) +} + +async fn accept_loop( + listener: VsockListener, + token: String, + event_tx: mpsc::Sender, + cmd_tx: mpsc::Sender, +) { + let permits = Arc::new(Semaphore::new(CONNECTION_WORKERS)); + let mut tasks = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, _addr)) => { + let Ok(permit) = permits.clone().try_acquire_owned() else { + tasks.spawn(async move { + let mut conn = conn; + write_enqueue_response( + &mut conn, + 0, + Some("cache enqueue workers are busy".to_owned()), + ) + .await; + }); + warn!("cache enqueue dropped because workers are busy"); + continue; + }; + + let worker_cmd_tx = cmd_tx.clone(); + if let Err(error) = start_enqueue_request(&worker_cmd_tx) { + tasks.spawn(async move { + let mut conn = conn; + write_enqueue_response(&mut conn, 0, Some(error)).await; + }); + continue; + } + + let worker_token = token.clone(); + let worker_event_tx = event_tx.clone(); + tasks.spawn(async move { + let _permit = permit; + handle_enqueue_conn( + conn, + &worker_token, + &worker_event_tx, + &worker_cmd_tx, + ) + .await; + let _ = worker_cmd_tx.send(Cmd::EnqueueFinished).await; + }); + } + Err(error) => { + if error.kind() != io::ErrorKind::Interrupted { + warn!(%error, "cache enqueue accept failed"); + } + } + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_enqueue_task_result(result); + } + } + } +} + +fn log_enqueue_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "cache enqueue task failed"); + } +} + +async fn handle_enqueue_conn( + mut conn: VsockStream, + expected_token: &str, + event_tx: &mpsc::Sender, + cmd_tx: &mpsc::Sender, +) { + let req: EnqueueBuiltPathsRequest = match read_enqueue_request(&mut conn).await { + Ok(req) => req, + Err(error) => { + write_enqueue_response(&mut conn, 0, Some(error)).await; + return; + } + }; + + if req.token != expected_token { + write_enqueue_response(&mut conn, 0, Some("invalid cache enqueue token".to_owned())).await; + return; + } + + match enqueue_paths(cmd_tx, req.paths).await { + Ok(queued) => { + send_built_paths_event(event_tx, queued.event_paths).await; + write_enqueue_response(&mut conn, queued.count, None).await; + } + Err(error) => write_enqueue_response(&mut conn, 0, Some(error)).await, + } +} + +fn start_enqueue_request(cmd_tx: &mpsc::Sender) -> Result<(), String> { + match cmd_tx.try_send(Cmd::EnqueueStarted) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => Err("cache upload worker stopped".to_owned()), + } +} + +async fn read_enqueue_request(conn: &mut VsockStream) -> Result { + read_json_line(conn) + .await + .map_err(JsonLineError::enqueue_request_message) +} + +async fn read_json_line(conn: &mut VsockStream) -> Result +where + T: DeserializeOwned, +{ + let mut data = Vec::new(); + let mut reader = BufReader::new(conn); + let bytes_read = tokio::time::timeout( + CACHE_ENQUEUE_IO_TIMEOUT, + reader.read_until(b'\n', &mut data), + ) + .await + .map_err(|_| JsonLineError::TimedOut)? + .map_err(JsonLineError::Io)?; + + if bytes_read == 0 { + return Err(JsonLineError::Empty); + } + + serde_json::from_slice(&data).map_err(JsonLineError::Json) +} + +struct QueuedPaths { + count: usize, + event_paths: Vec, +} + +async fn enqueue_paths( + cmd_tx: &mpsc::Sender, + paths: Vec, +) -> Result { + let paths = clean_store_paths(&paths); + let event_paths = paths.clone(); + if paths.is_empty() { + return Ok(QueuedPaths { + count: 0, + event_paths, + }); + } + + let (reply, queued) = oneshot::channel(); + match cmd_tx.try_send(Cmd::Enqueue { paths, reply }) { + Ok(()) => match queued.await { + Ok(Ok(count)) => Ok(QueuedPaths { count, event_paths }), + Ok(Err(error)) => Err(error), + Err(_) => Err("cache upload worker stopped".to_owned()), + }, + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => Err("cache upload worker stopped".to_owned()), + } +} + +async fn send_built_paths_event(event_tx: &mpsc::Sender, paths: Vec) { + if paths.is_empty() { + return; + } + + let msg = Message { + id: "built-paths".to_owned(), + built_paths: Some(v1::BuiltPaths { + paths, + reason: "post_build_hook".to_owned(), + }), + ..Default::default() + }; + let _ = event_tx.send(msg).await; +} + +async fn write_enqueue_response(conn: &mut VsockStream, queued: usize, error: Option) { + let response = EnqueueBuiltPathsResponse { + queued, + error: error.unwrap_or_default(), + }; + let _ = write_json_line(conn, &response).await; +} + +async fn write_json_line(conn: &mut VsockStream, value: &T) -> Result<(), JsonLineError> +where + T: Serialize + ?Sized, +{ + let data = serde_json::to_vec(value).map_err(JsonLineError::Json)?; + tokio::time::timeout(CACHE_ENQUEUE_IO_TIMEOUT, async { + conn.write_all(&data).await?; + conn.write_all(b"\n").await?; + VsockStream::shutdown(conn, Shutdown::Write) + }) + .await + .map_err(|_| JsonLineError::TimedOut)? + .map_err(JsonLineError::Io) +} + +// we use a loopback vsock here since its better than having to do the whole http song and dance! +pub async fn enqueue_built_paths(paths: &[String]) { + let paths = clean_store_paths(paths); + if paths.is_empty() { + return; + } + + let token = match read_hook_token() { + Ok(token) => token, + Err(_) => return, + }; + + if token.is_empty() { + return; + } + + let mut conn = + match VsockStream::connect(VsockAddr::new(VMADDR_CID_LOCAL, cache_enqueue_port())).await { + Ok(conn) => conn, + Err(error) => { + warn!(paths = paths.len(), %error, "cache enqueue unavailable"); + return; + } + }; + + let request = EnqueueBuiltPathsRequest { token, paths }; + match write_json_line(&mut conn, &request).await { + Ok(()) => {} + Err(JsonLineError::Json(error)) => { + warn!(%error, "cache enqueue encode failed"); + return; + } + Err(JsonLineError::TimedOut) => { + warn!("cache enqueue write timed out"); + return; + } + Err(error) => { + warn!(%error, "cache enqueue write failed"); + return; + } + } + + let response: EnqueueBuiltPathsResponse = match read_json_line(&mut conn).await { + Ok(response) => response, + Err(JsonLineError::Empty) => { + warn!("cache enqueue ack was empty"); + return; + } + Err(JsonLineError::TimedOut) => { + warn!("cache enqueue ack timed out"); + return; + } + Err(error) => { + warn!(%error, "cache enqueue ack failed"); + return; + } + }; + + if !response.error.is_empty() { + warn!(error = %response.error, "cache enqueue rejected"); + return; + } + + info!(queued = response.queued, "cache paths enqueued"); +} + +fn cache_enqueue_port() -> u32 { + std::env::var(SHUTTLE_CACHE_ENQUEUE_PORT_ENV) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_CACHE_ENQUEUE_PORT) +} + +fn create_hook_token() -> Result { + use std::io::Write; + + fs::create_dir_all(SPINDLE_RUN_DIR).with_context(|| format!("create {SPINDLE_RUN_DIR}"))?; + let token = random_token().context("generate hook token")?; + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o640) + .open(SPINDLE_HOOK_TOKEN) + .with_context(|| format!("create {SPINDLE_HOOK_TOKEN}"))?; + allow_nix_build_group(SPINDLE_HOOK_TOKEN)?; + file.write_all(token.as_bytes()) + .with_context(|| format!("write {SPINDLE_HOOK_TOKEN}"))?; + file.write_all(b"\n") + .with_context(|| format!("write {SPINDLE_HOOK_TOKEN}"))?; + Ok(token) +} + +fn allow_nix_build_group(path: &str) -> Result<()> { + let Some(group) = + Group::from_name(NIX_BUILD_GROUP).with_context(|| format!("lookup {NIX_BUILD_GROUP}"))? + else { + warn!( + group = NIX_BUILD_GROUP, + "nix build group not found; cache hook token remains root-only" + ); + return Ok(()); + }; + + chown(path, None, Some(group.gid)).with_context(|| format!("chown {path} to {NIX_BUILD_GROUP}")) +} + +fn read_hook_token() -> Result { + fs::read_to_string(SPINDLE_HOOK_TOKEN) + .map(|token| token.trim().to_owned()) + .with_context(|| format!("read {SPINDLE_HOOK_TOKEN}")) +} + +fn random_token() -> Result { + let mut bytes = [0_u8; 32]; + File::open("/dev/urandom") + .context("open /dev/urandom")? + .read_exact(&mut bytes) + .context("read /dev/urandom")?; + + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(&mut token, "{byte:02x}").unwrap(); + } + Ok(token) +} diff --git a/shuttle/src/cache/read_proxy.rs b/shuttle/src/cache/read_proxy.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/cache/read_proxy.rs @@ -0,0 +1,27 @@ +use crate::host_proxy::VsockTcpProxy; +use anyhow::Result; + +const DEFAULT_CACHE_READ_PROXY_ADDR: &str = "127.0.0.1:10500"; +const SHUTTLE_CACHE_READ_PROXY_ADDR_ENV: &str = "SHUTTLE_CACHE_READ_PROXY_ADDR"; + +pub struct ReadCacheProxy { + inner: VsockTcpProxy, +} + +impl ReadCacheProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let addr = std::env::var(SHUTTLE_CACHE_READ_PROXY_ADDR_ENV) + .unwrap_or_else(|_| DEFAULT_CACHE_READ_PROXY_ADDR.to_owned()); + + let inner = VsockTcpProxy::start("read cache proxy", &addr, host_cid, host_port).await?; + Ok(Some(Self { inner })) + } + + pub fn url(&self) -> &str { + self.inner.url() + } +} diff --git a/shuttle/src/cache/write_proxy.rs b/shuttle/src/cache/write_proxy.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/cache/write_proxy.rs @@ -0,0 +1,22 @@ +use crate::host_proxy::VsockTcpProxy; +use anyhow::Result; + +pub struct WriteCacheProxy { + inner: VsockTcpProxy, +} + +impl WriteCacheProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let inner = + VsockTcpProxy::start("write cache proxy", "127.0.0.1:0", host_cid, host_port).await?; + Ok(Some(Self { inner })) + } + + pub fn url(&self) -> &str { + self.inner.url() + } +} diff --git a/shuttle/src/gen/file_descriptor_set.bin b/shuttle/src/gen/file_descriptor_set.bin new file mode 100644 --- /dev/null +++ b/shuttle/src/gen/file_descriptor_set.bin diff --git a/spindle/agentproto/gen/agent.pb.go b/spindle/agentproto/gen/agent.pb.go new file mode 100644 --- /dev/null +++ b/spindle/agentproto/gen/agent.pb.go @@ -0,0 +1,1102 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: spindle/agent/v1/agent.proto + +package agentv1 + +import ( + _ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Hello struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + AgentVersion string `protobuf:"bytes,2,opt,name=agent_version,json=agentVersion,proto3" json:"agent_version,omitempty"` + BootId string `protobuf:"bytes,3,opt,name=boot_id,json=bootId,proto3" json:"boot_id,omitempty"` + NixVersion string `protobuf:"bytes,4,opt,name=nix_version,json=nixVersion,proto3" json:"nix_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Hello) Reset() { + *x = Hello{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Hello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Hello) ProtoMessage() {} + +func (x *Hello) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Hello.ProtoReflect.Descriptor instead. +func (*Hello) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{0} +} + +func (x *Hello) GetProtocolVersion() uint32 { + if x != nil { + return x.ProtocolVersion + } + return 0 +} + +func (x *Hello) GetAgentVersion() string { + if x != nil { + return x.AgentVersion + } + return "" +} + +func (x *Hello) GetBootId() string { + if x != nil { + return x.BootId + } + return "" +} + +func (x *Hello) GetNixVersion() string { + if x != nil { + return x.NixVersion + } + return "" +} + +type Init struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + CacheTrustedPublicKeys []string `protobuf:"bytes,2,rep,name=cache_trusted_public_keys,json=cacheTrustedPublicKeys,proto3" json:"cache_trusted_public_keys,omitempty"` + CacheReadProxyPort uint32 `protobuf:"varint,3,opt,name=cache_read_proxy_port,json=cacheReadProxyPort,proto3" json:"cache_read_proxy_port,omitempty"` + CacheUploadProxyPort uint32 `protobuf:"varint,4,opt,name=cache_upload_proxy_port,json=cacheUploadProxyPort,proto3" json:"cache_upload_proxy_port,omitempty"` + DnsProxyPort uint32 `protobuf:"varint,5,opt,name=dns_proxy_port,json=dnsProxyPort,proto3" json:"dns_proxy_port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Init) Reset() { + *x = Init{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Init) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Init) ProtoMessage() {} + +func (x *Init) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Init.ProtoReflect.Descriptor instead. +func (*Init) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{1} +} + +func (x *Init) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *Init) GetCacheTrustedPublicKeys() []string { + if x != nil { + return x.CacheTrustedPublicKeys + } + return nil +} + +func (x *Init) GetCacheReadProxyPort() uint32 { + if x != nil { + return x.CacheReadProxyPort + } + return 0 +} + +func (x *Init) GetCacheUploadProxyPort() uint32 { + if x != nil { + return x.CacheUploadProxyPort + } + return 0 +} + +func (x *Init) GetDnsProxyPort() uint32 { + if x != nil { + return x.DnsProxyPort + } + return 0 +} + +type ExecStart struct { + state protoimpl.MessageState `protogen:"open.v1"` + Argv []string `protobuf:"bytes,1,rep,name=argv,proto3" json:"argv,omitempty"` + Env []string `protobuf:"bytes,2,rep,name=env,proto3" json:"env,omitempty"` + Cwd string `protobuf:"bytes,3,opt,name=cwd,proto3" json:"cwd,omitempty"` + User string `protobuf:"bytes,4,opt,name=user,proto3" json:"user,omitempty"` + TimeoutSeconds uint32 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStart) Reset() { + *x = ExecStart{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStart) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStart) ProtoMessage() {} + +func (x *ExecStart) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStart.ProtoReflect.Descriptor instead. +func (*ExecStart) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{2} +} + +func (x *ExecStart) GetArgv() []string { + if x != nil { + return x.Argv + } + return nil +} + +func (x *ExecStart) GetEnv() []string { + if x != nil { + return x.Env + } + return nil +} + +func (x *ExecStart) GetCwd() string { + if x != nil { + return x.Cwd + } + return "" +} + +func (x *ExecStart) GetUser() string { + if x != nil { + return x.User + } + return "" +} + +func (x *ExecStart) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type ExecStdout struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStdout) Reset() { + *x = ExecStdout{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStdout) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStdout) ProtoMessage() {} + +func (x *ExecStdout) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStdout.ProtoReflect.Descriptor instead. +func (*ExecStdout) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *ExecStdout) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +type ExecStderr struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStderr) Reset() { + *x = ExecStderr{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStderr) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStderr) ProtoMessage() {} + +func (x *ExecStderr) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStderr.ProtoReflect.Descriptor instead. +func (*ExecStderr) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{4} +} + +func (x *ExecStderr) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +type ExecExit struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExitCode int32 `protobuf:"varint,1,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` + // set when the guest killed the step on its own timeout timer, so the host + // can classify it as a timeout rather than inferring failure from exit_code. + TimedOut bool `protobuf:"varint,3,opt,name=timed_out,json=timedOut,proto3" json:"timed_out,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecExit) Reset() { + *x = ExecExit{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecExit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecExit) ProtoMessage() {} + +func (x *ExecExit) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecExit.ProtoReflect.Descriptor instead. +func (*ExecExit) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{5} +} + +func (x *ExecExit) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *ExecExit) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *ExecExit) GetTimedOut() bool { + if x != nil { + return x.TimedOut + } + return false +} + +type ActivateConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfigKey string `protobuf:"bytes,1,opt,name=config_key,json=configKey,proto3" json:"config_key,omitempty"` + BaseConfigHash string `protobuf:"bytes,2,opt,name=base_config_hash,json=baseConfigHash,proto3" json:"base_config_hash,omitempty"` + UserConfig string `protobuf:"bytes,3,opt,name=user_config,json=userConfig,proto3" json:"user_config,omitempty"` + Toplevel string `protobuf:"bytes,4,opt,name=toplevel,proto3" json:"toplevel,omitempty"` + TimeoutSeconds uint32 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActivateConfig) Reset() { + *x = ActivateConfig{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActivateConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActivateConfig) ProtoMessage() {} + +func (x *ActivateConfig) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActivateConfig.ProtoReflect.Descriptor instead. +func (*ActivateConfig) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{6} +} + +func (x *ActivateConfig) GetConfigKey() string { + if x != nil { + return x.ConfigKey + } + return "" +} + +func (x *ActivateConfig) GetBaseConfigHash() string { + if x != nil { + return x.BaseConfigHash + } + return "" +} + +func (x *ActivateConfig) GetUserConfig() string { + if x != nil { + return x.UserConfig + } + return "" +} + +func (x *ActivateConfig) GetToplevel() string { + if x != nil { + return x.Toplevel + } + return "" +} + +func (x *ActivateConfig) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type ActivateConfigResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfigKey string `protobuf:"bytes,1,opt,name=config_key,json=configKey,proto3" json:"config_key,omitempty"` + Toplevel string `protobuf:"bytes,2,opt,name=toplevel,proto3" json:"toplevel,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActivateConfigResult) Reset() { + *x = ActivateConfigResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActivateConfigResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActivateConfigResult) ProtoMessage() {} + +func (x *ActivateConfigResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActivateConfigResult.ProtoReflect.Descriptor instead. +func (*ActivateConfigResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{7} +} + +func (x *ActivateConfigResult) GetConfigKey() string { + if x != nil { + return x.ConfigKey + } + return "" +} + +func (x *ActivateConfigResult) GetToplevel() string { + if x != nil { + return x.Toplevel + } + return "" +} + +func (x *ActivateConfigResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +type BuiltPaths struct { + state protoimpl.MessageState `protogen:"open.v1"` + Paths []string `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BuiltPaths) Reset() { + *x = BuiltPaths{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BuiltPaths) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BuiltPaths) ProtoMessage() {} + +func (x *BuiltPaths) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BuiltPaths.ProtoReflect.Descriptor instead. +func (*BuiltPaths) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{8} +} + +func (x *BuiltPaths) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + +func (x *BuiltPaths) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +type CacheDrain struct { + state protoimpl.MessageState `protogen:"open.v1"` + TimeoutSeconds uint32 `protobuf:"varint,1,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheDrain) Reset() { + *x = CacheDrain{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheDrain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheDrain) ProtoMessage() {} + +func (x *CacheDrain) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheDrain.ProtoReflect.Descriptor instead. +func (*CacheDrain) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{9} +} + +func (x *CacheDrain) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type CacheDrainResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + CacheQueued uint32 `protobuf:"varint,2,opt,name=cache_queued,json=cacheQueued,proto3" json:"cache_queued,omitempty"` + CacheActive uint32 `protobuf:"varint,3,opt,name=cache_active,json=cacheActive,proto3" json:"cache_active,omitempty"` + CacheUploaded uint32 `protobuf:"varint,4,opt,name=cache_uploaded,json=cacheUploaded,proto3" json:"cache_uploaded,omitempty"` + CacheFailed uint32 `protobuf:"varint,5,opt,name=cache_failed,json=cacheFailed,proto3" json:"cache_failed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheDrainResult) Reset() { + *x = CacheDrainResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheDrainResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheDrainResult) ProtoMessage() {} + +func (x *CacheDrainResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheDrainResult.ProtoReflect.Descriptor instead. +func (*CacheDrainResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *CacheDrainResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *CacheDrainResult) GetCacheQueued() uint32 { + if x != nil { + return x.CacheQueued + } + return 0 +} + +func (x *CacheDrainResult) GetCacheActive() uint32 { + if x != nil { + return x.CacheActive + } + return 0 +} + +func (x *CacheDrainResult) GetCacheUploaded() uint32 { + if x != nil { + return x.CacheUploaded + } + return 0 +} + +func (x *CacheDrainResult) GetCacheFailed() uint32 { + if x != nil { + return x.CacheFailed + } + return 0 +} + +type Poweroff struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Poweroff) Reset() { + *x = Poweroff{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Poweroff) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Poweroff) ProtoMessage() {} + +func (x *Poweroff) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Poweroff.ProtoReflect.Descriptor instead. +func (*Poweroff) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{11} +} + +type PoweroffResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PoweroffResult) Reset() { + *x = PoweroffResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PoweroffResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PoweroffResult) ProtoMessage() {} + +func (x *PoweroffResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PoweroffResult.ProtoReflect.Descriptor instead. +func (*PoweroffResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *PoweroffResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +type Message struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Hello *Hello `protobuf:"bytes,2,opt,name=hello,proto3" json:"hello,omitempty"` + Init *Init `protobuf:"bytes,3,opt,name=init,proto3" json:"init,omitempty"` + ExecStart *ExecStart `protobuf:"bytes,4,opt,name=exec_start,json=execStart,proto3" json:"exec_start,omitempty"` + ExecStdout *ExecStdout `protobuf:"bytes,5,opt,name=exec_stdout,json=execStdout,proto3" json:"exec_stdout,omitempty"` + ExecStderr *ExecStderr `protobuf:"bytes,6,opt,name=exec_stderr,json=execStderr,proto3" json:"exec_stderr,omitempty"` + ExecExit *ExecExit `protobuf:"bytes,7,opt,name=exec_exit,json=execExit,proto3" json:"exec_exit,omitempty"` + ActivateConfig *ActivateConfig `protobuf:"bytes,8,opt,name=activate_config,json=activateConfig,proto3" json:"activate_config,omitempty"` + ActivateConfigResult *ActivateConfigResult `protobuf:"bytes,9,opt,name=activate_config_result,json=activateConfigResult,proto3" json:"activate_config_result,omitempty"` + BuiltPaths *BuiltPaths `protobuf:"bytes,10,opt,name=built_paths,json=builtPaths,proto3" json:"built_paths,omitempty"` + CacheDrain *CacheDrain `protobuf:"bytes,11,opt,name=cache_drain,json=cacheDrain,proto3" json:"cache_drain,omitempty"` + CacheDrainResult *CacheDrainResult `protobuf:"bytes,12,opt,name=cache_drain_result,json=cacheDrainResult,proto3" json:"cache_drain_result,omitempty"` + Poweroff *Poweroff `protobuf:"bytes,13,opt,name=poweroff,proto3" json:"poweroff,omitempty"` + PoweroffResult *PoweroffResult `protobuf:"bytes,14,opt,name=poweroff_result,json=poweroffResult,proto3" json:"poweroff_result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Message) Reset() { + *x = Message{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{13} +} + +func (x *Message) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Message) GetHello() *Hello { + if x != nil { + return x.Hello + } + return nil +} + +func (x *Message) GetInit() *Init { + if x != nil { + return x.Init + } + return nil +} + +func (x *Message) GetExecStart() *ExecStart { + if x != nil { + return x.ExecStart + } + return nil +} + +func (x *Message) GetExecStdout() *ExecStdout { + if x != nil { + return x.ExecStdout + } + return nil +} + +func (x *Message) GetExecStderr() *ExecStderr { + if x != nil { + return x.ExecStderr + } + return nil +} + +func (x *Message) GetExecExit() *ExecExit { + if x != nil { + return x.ExecExit + } + return nil +} + +func (x *Message) GetActivateConfig() *ActivateConfig { + if x != nil { + return x.ActivateConfig + } + return nil +} + +func (x *Message) GetActivateConfigResult() *ActivateConfigResult { + if x != nil { + return x.ActivateConfigResult + } + return nil +} + +func (x *Message) GetBuiltPaths() *BuiltPaths { + if x != nil { + return x.BuiltPaths + } + return nil +} + +func (x *Message) GetCacheDrain() *CacheDrain { + if x != nil { + return x.CacheDrain + } + return nil +} + +func (x *Message) GetCacheDrainResult() *CacheDrainResult { + if x != nil { + return x.CacheDrainResult + } + return nil +} + +func (x *Message) GetPoweroff() *Poweroff { + if x != nil { + return x.Poweroff + } + return nil +} + +func (x *Message) GetPoweroffResult() *PoweroffResult { + if x != nil { + return x.PoweroffResult + } + return nil +} + +var File_spindle_agent_v1_agent_proto protoreflect.FileDescriptor + +const file_spindle_agent_v1_agent_proto_rawDesc = "" + + "\n" + + "\x1cspindle/agent/v1/agent.proto\x12\x10spindle.agent.v1\x1a\x1bbuf/validate/validate.proto\"\x91\x01\n" + + "\x05Hello\x12)\n" + + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12#\n" + + "\ragent_version\x18\x02 \x01(\tR\fagentVersion\x12\x17\n" + + "\aboot_id\x18\x03 \x01(\tR\x06bootId\x12\x1f\n" + + "\vnix_version\x18\x04 \x01(\tR\n" + + "nixVersion\"\xe8\x01\n" + + "\x04Init\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\x129\n" + + "\x19cache_trusted_public_keys\x18\x02 \x03(\tR\x16cacheTrustedPublicKeys\x121\n" + + "\x15cache_read_proxy_port\x18\x03 \x01(\rR\x12cacheReadProxyPort\x125\n" + + "\x17cache_upload_proxy_port\x18\x04 \x01(\rR\x14cacheUploadProxyPort\x12$\n" + + "\x0edns_proxy_port\x18\x05 \x01(\rR\fdnsProxyPort\"\x80\x01\n" + + "\tExecStart\x12\x12\n" + + "\x04argv\x18\x01 \x03(\tR\x04argv\x12\x10\n" + + "\x03env\x18\x02 \x03(\tR\x03env\x12\x10\n" + + "\x03cwd\x18\x03 \x01(\tR\x03cwd\x12\x12\n" + + "\x04user\x18\x04 \x01(\tR\x04user\x12'\n" + + "\x0ftimeout_seconds\x18\x05 \x01(\rR\x0etimeoutSeconds\" \n" + + "\n" + + "ExecStdout\x12\x12\n" + + "\x04data\x18\x01 \x01(\tR\x04data\" \n" + + "\n" + + "ExecStderr\x12\x12\n" + + "\x04data\x18\x01 \x01(\tR\x04data\"Z\n" + + "\bExecExit\x12\x1b\n" + + "\texit_code\x18\x01 \x01(\x05R\bexitCode\x12\x14\n" + + "\x05error\x18\x02 \x01(\tR\x05error\x12\x1b\n" + + "\ttimed_out\x18\x03 \x01(\bR\btimedOut\"\xbf\x01\n" + + "\x0eActivateConfig\x12\x1d\n" + + "\n" + + "config_key\x18\x01 \x01(\tR\tconfigKey\x12(\n" + + "\x10base_config_hash\x18\x02 \x01(\tR\x0ebaseConfigHash\x12\x1f\n" + + "\vuser_config\x18\x03 \x01(\tR\n" + + "userConfig\x12\x1a\n" + + "\btoplevel\x18\x04 \x01(\tR\btoplevel\x12'\n" + + "\x0ftimeout_seconds\x18\x05 \x01(\rR\x0etimeoutSeconds\"g\n" + + "\x14ActivateConfigResult\x12\x1d\n" + + "\n" + + "config_key\x18\x01 \x01(\tR\tconfigKey\x12\x1a\n" + + "\btoplevel\x18\x02 \x01(\tR\btoplevel\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error\":\n" + + "\n" + + "BuiltPaths\x12\x14\n" + + "\x05paths\x18\x01 \x03(\tR\x05paths\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\"5\n" + + "\n" + + "CacheDrain\x12'\n" + + "\x0ftimeout_seconds\x18\x01 \x01(\rR\x0etimeoutSeconds\"\xb8\x01\n" + + "\x10CacheDrainResult\x12\x14\n" + + "\x05error\x18\x01 \x01(\tR\x05error\x12!\n" + + "\fcache_queued\x18\x02 \x01(\rR\vcacheQueued\x12!\n" + + "\fcache_active\x18\x03 \x01(\rR\vcacheActive\x12%\n" + + "\x0ecache_uploaded\x18\x04 \x01(\rR\rcacheUploaded\x12!\n" + + "\fcache_failed\x18\x05 \x01(\rR\vcacheFailed\"\n" + + "\n" + + "\bPoweroff\"&\n" + + "\x0ePoweroffResult\x12\x14\n" + + "\x05error\x18\x01 \x01(\tR\x05error\"\xa8\b\n" + + "\aMessage\x12\x17\n" + + "\x02id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x02id\x12-\n" + + "\x05hello\x18\x02 \x01(\v2\x17.spindle.agent.v1.HelloR\x05hello\x12*\n" + + "\x04init\x18\x03 \x01(\v2\x16.spindle.agent.v1.InitR\x04init\x12:\n" + + "\n" + + "exec_start\x18\x04 \x01(\v2\x1b.spindle.agent.v1.ExecStartR\texecStart\x12=\n" + + "\vexec_stdout\x18\x05 \x01(\v2\x1c.spindle.agent.v1.ExecStdoutR\n" + + "execStdout\x12=\n" + + "\vexec_stderr\x18\x06 \x01(\v2\x1c.spindle.agent.v1.ExecStderrR\n" + + "execStderr\x127\n" + + "\texec_exit\x18\a \x01(\v2\x1a.spindle.agent.v1.ExecExitR\bexecExit\x12I\n" + + "\x0factivate_config\x18\b \x01(\v2 .spindle.agent.v1.ActivateConfigR\x0eactivateConfig\x12\\\n" + + "\x16activate_config_result\x18\t \x01(\v2&.spindle.agent.v1.ActivateConfigResultR\x14activateConfigResult\x12=\n" + + "\vbuilt_paths\x18\n" + + " \x01(\v2\x1c.spindle.agent.v1.BuiltPathsR\n" + + "builtPaths\x12=\n" + + "\vcache_drain\x18\v \x01(\v2\x1c.spindle.agent.v1.CacheDrainR\n" + + "cacheDrain\x12P\n" + + "\x12cache_drain_result\x18\f \x01(\v2\".spindle.agent.v1.CacheDrainResultR\x10cacheDrainResult\x126\n" + + "\bpoweroff\x18\r \x01(\v2\x1a.spindle.agent.v1.PoweroffR\bpoweroff\x12I\n" + + "\x0fpoweroff_result\x18\x0e \x01(\v2 .spindle.agent.v1.PoweroffResultR\x0epoweroffResult:\xb9\x01\xbaH\xb5\x01\"\xb2\x01\n" + + "\x05hello\n" + + "\x04init\n" + + "\n" + + "exec_start\n" + + "\vexec_stdout\n" + + "\vexec_stderr\n" + + "\texec_exit\n" + + "\x0factivate_config\n" + + "\x16activate_config_result\n" + + "\vbuilt_paths\n" + + "\vcache_drain\n" + + "\x12cache_drain_result\n" + + "\bpoweroff\n" + + "\x0fpoweroff_result\x10\x01B1Z/tangled.org/core/spindle/agentproto/gen;agentv1b\x06proto3" + +var ( + file_spindle_agent_v1_agent_proto_rawDescOnce sync.Once + file_spindle_agent_v1_agent_proto_rawDescData []byte +) + +func file_spindle_agent_v1_agent_proto_rawDescGZIP() []byte { + file_spindle_agent_v1_agent_proto_rawDescOnce.Do(func() { + file_spindle_agent_v1_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_agent_v1_agent_proto_rawDesc), len(file_spindle_agent_v1_agent_proto_rawDesc))) + }) + return file_spindle_agent_v1_agent_proto_rawDescData +} + +var file_spindle_agent_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_spindle_agent_v1_agent_proto_goTypes = []any{ + (*Hello)(nil), // 0: spindle.agent.v1.Hello + (*Init)(nil), // 1: spindle.agent.v1.Init + (*ExecStart)(nil), // 2: spindle.agent.v1.ExecStart + (*ExecStdout)(nil), // 3: spindle.agent.v1.ExecStdout + (*ExecStderr)(nil), // 4: spindle.agent.v1.ExecStderr + (*ExecExit)(nil), // 5: spindle.agent.v1.ExecExit + (*ActivateConfig)(nil), // 6: spindle.agent.v1.ActivateConfig + (*ActivateConfigResult)(nil), // 7: spindle.agent.v1.ActivateConfigResult + (*BuiltPaths)(nil), // 8: spindle.agent.v1.BuiltPaths + (*CacheDrain)(nil), // 9: spindle.agent.v1.CacheDrain + (*CacheDrainResult)(nil), // 10: spindle.agent.v1.CacheDrainResult + (*Poweroff)(nil), // 11: spindle.agent.v1.Poweroff + (*PoweroffResult)(nil), // 12: spindle.agent.v1.PoweroffResult + (*Message)(nil), // 13: spindle.agent.v1.Message +} +var file_spindle_agent_v1_agent_proto_depIdxs = []int32{ + 0, // 0: spindle.agent.v1.Message.hello:type_name -> spindle.agent.v1.Hello + 1, // 1: spindle.agent.v1.Message.init:type_name -> spindle.agent.v1.Init + 2, // 2: spindle.agent.v1.Message.exec_start:type_name -> spindle.agent.v1.ExecStart + 3, // 3: spindle.agent.v1.Message.exec_stdout:type_name -> spindle.agent.v1.ExecStdout + 4, // 4: spindle.agent.v1.Message.exec_stderr:type_name -> spindle.agent.v1.ExecStderr + 5, // 5: spindle.agent.v1.Message.exec_exit:type_name -> spindle.agent.v1.ExecExit + 6, // 6: spindle.agent.v1.Message.activate_config:type_name -> spindle.agent.v1.ActivateConfig + 7, // 7: spindle.agent.v1.Message.activate_config_result:type_name -> spindle.agent.v1.ActivateConfigResult + 8, // 8: spindle.agent.v1.Message.built_paths:type_name -> spindle.agent.v1.BuiltPaths + 9, // 9: spindle.agent.v1.Message.cache_drain:type_name -> spindle.agent.v1.CacheDrain + 10, // 10: spindle.agent.v1.Message.cache_drain_result:type_name -> spindle.agent.v1.CacheDrainResult + 11, // 11: spindle.agent.v1.Message.poweroff:type_name -> spindle.agent.v1.Poweroff + 12, // 12: spindle.agent.v1.Message.poweroff_result:type_name -> spindle.agent.v1.PoweroffResult + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_spindle_agent_v1_agent_proto_init() } +func file_spindle_agent_v1_agent_proto_init() { + if File_spindle_agent_v1_agent_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_agent_v1_agent_proto_rawDesc), len(file_spindle_agent_v1_agent_proto_rawDesc)), + NumEnums: 0, + NumMessages: 14, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_spindle_agent_v1_agent_proto_goTypes, + DependencyIndexes: file_spindle_agent_v1_agent_proto_depIdxs, + MessageInfos: file_spindle_agent_v1_agent_proto_msgTypes, + }.Build() + File_spindle_agent_v1_agent_proto = out.File + file_spindle_agent_v1_agent_proto_goTypes = nil + file_spindle_agent_v1_agent_proto_depIdxs = nil +} diff --git a/spindle/engines/microvm/README.md b/spindle/engines/microvm/README.md new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/README.md @@ -0,0 +1,208 @@ +# spindle microVM engine + +This document describes the architecture of the microvm engine for spindle. In +short it allows the spindle to spin up microvm guests, and implements a guest +[agent protocol](../../agentproto) for communicating with those guests (via the +[shuttle](../../../shuttle) implementation of that proto). It implements some +fairly simple resource budgeting and optionally sets up cgroups for better +enforcing resource limits, and hardens the VM network access. It has Nix cache +integration for any paths built in the VM, those will get pushed to a Nix cache +by the spindle (if one is configured). The runner is abstracted behind an +interface; right now only the QEMU microVM impl is supported, but others (e.g. +firecracker) can slot in later. + +Currently two kinds of images are supported: + +- NixOS images: these allow configuration such as `dependencies`, `services`, + `virtualisation`, `registry`, `caches` in the workflow file itself. The guest + agent will build (or if it's cached, spindle will send the store path for + realization) and activate it before any workflow steps are ran. +- Non-NixOS: this is mainly just Alpine for now, but can be anything else. + Workflow-level configuration like NixOS aren't supported while using these. If + Nix exists inside the image (like in our Alpine image) it will still be able + to make use of the spindle cache. + +(For testing, you can run `bash spindle/engines/microvm/test-spindle-microvm.sh` +from repo root. These test the Alpine & NixOS, and features like if Docker +works, public internet is reachable, and so on.) + +## Image builds + +Image builds right now are done via Nix: + +- For NixOS, we use [microvm.nix](https://github.com/microvm-nix/microvm.nix), + and layer our own configs on-top, see [here](../../../nix/microvm). +- For Alpine we have a small-ish Nix definition that includes fetching the + kernel, initrd, kernel modules; setting up the init script that configures the + VM proper; copying dependencies (like `nix` or `git`) into a rootfs and + creating a squashfs from it. + +This does not mean it *has* to be done via Nix, as long as your images are what +spindle expects, they should work. That is: +- a guest agent is present inside of the image and when that image boots it will + get started, +- `spindle-workflow` user exists, +- and the work directory is configured (`/workspace`). + +## Image discovery + +Each built image ships with a `spec.json` next to its artifacts. This spec +describes everything needed to run the image: the kernel, initrd and read-only +store disk paths, boot args, memory/vCPU sizing, the shell used for workflow +steps, writable volumes, network interfaces, and runner-specific config (machine +type, CPU, extra args for QEMU). NixOS images also carry a `baseConfigHash` +identifying the base configuration baked into the image. + +An image lives in the configured image directory either as a directory +containing a `spec.json` (alongside the kernel/initrd/store-disk artifacts) or, +for a self-contained spec, as a flat `.json` file. An operator keeping +multiple arches side by side can name them `-` (eg. `nixos-x86_64`, +`alpine-aarch64`); that arch suffix is just part of the name, not something +resolution infers. + +A workflow names an image with the `image` key at top-level (falling back to +`SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE` if unset). The name is matched +literally: we look for `` (a directory with a `spec.json`) then +`.json`. Resolution depends only on the name and what is on disk, never on +the host, so the same workflow resolves identically on every spindle. If for +example an operator wants `nixos` to work, they can symlink `nixos` to +`nixos-x86_64`. + +The spec is validated at resolve time (required fields, positive sizes etc.), +and right before launch we also check the referenced files actually exist on +disk and that the host has the commands we need: `mkfs.ext4` for volume +formatting, plus whatever the selected runner requires. For QEMU that's the QEMU +binary for the spec's arch, `/dev/vhost-vsock`, `/dev/kvm` (if KVM is enabled), +and the `ip`, `mount`, `slirp4netns`, `unshare` toolchain when the image has +network interfaces. + +## microVM lifecycle + +```mermaid +flowchart LR + Init["InitWorkflow
parse manifest, resolve image, build steps"] + Acquire["AcquireWorkflowSlot
queue until resources fit budget"] + Setup["SetupWorkflow
proxies, VM, agent handshake"] + Run["RunStep ×N
exec via agent"] + Destroy["DestroyWorkflow
drain cache, poweroff, cleanup"] + + Init --> Acquire --> Setup --> Run --> Destroy +``` + +While a workflow is running, things look like this (everything inside the cgroup +box is what gets resource-limited): + +```mermaid +flowchart LR + subgraph Host["spindle host"] + Hub["agent hub"] + ReadProxy["read cache proxy"] + UploadProxy["upload cache proxy"] + subgraph Cgroup["per-workflow cgroup"] + QEMU["qemu"] + Slirp["slirp4netns"] + end + end + + subgraph Guest["guest"] + Agent["guest agent"] + end + + Agent -->|"vsock"| Hub + Agent -->|substitutions| ReadProxy + Agent -->|built paths| UploadProxy + QEMU --- Guest + Slirp -->|outbound only| Internet["the internet"] + ReadProxy --> Substituters["upstream caches"] + UploadProxy --> NixCache["spindle nix cache"] +``` + +`InitWorkflow` parses the workflow manifest, resolves the image, and assembles +the step list: the clone step first, then (for NixOS images with a workflow +config) a "NixOS config activation" system step, then the user steps. Before any +of this actually runs the workflow has to acquire a slot from the resource +scheduler, each image declares its memory/vCPUs/disk and workflows queue until +their request fits within the configured budget. The scheduler is +work-conserving with aging and per-user fairness, so one user submitting a pile +of jobs won't starve everyone else, and slots don't sit idle while there's +queued work that fits in the budget. + +### Configuration + +Setup allocates a random vsock CID for the guest and registers it with the agent +hub, which listens on a single host vsock port. Incoming agent connections are +matched to workflows by CID, anything with an unknown CID is dropped. It then +creates a per-workflow work directory and starts three host-side proxies the guest +reaches over vsock: a read cache proxy (fronting the configured Nix substituters +plus any workflow-level `caches`) and an upload cache proxy (for pushing paths +built in the guest to the spindle's cache), plus a DNS proxy that resolves +through the host's resolver and filters private/special-purpose address answers. + +Then the VM itself. Writable volumes from the spec are created as sparse files +and formatted ext4, the store disk is attached read-only. QEMU runs with +`-sandbox on`, `-nodefaults`, no display/monitor, etc., serial output to a log +file, and a QMP socket for control. + +For network hardening: if the image has network interfaces, QEMU doesn't run in +the host network namespace at all. We `unshare` into fresh user/net/mount +namespaces, and a small wrapper script inside the namespace bind-mounts a +resolv.conf that disables qemu's slirp DNS and adds blackhole routes for every +special-use IPv4/IPv6 range (RFC 6890, so private networks, link-local, +loopback, CGNAT, multicast, ULAs and so on) before exec'ing QEMU. `slirp4netns` +(with `--disable-host-loopback`, sandbox and seccomp enabled) then provides +outbound connectivity for the namespace. The guest's `/etc/resolv.conf` points +at shuttle on localhost; shuttle forwards DNS packets over vsock to the +host-side DNS proxy. The guest sits behind a second layer of QEMU user-mode +networking inside that namespace, so guest traffic can only ever reach the +outside world, never the host or anything on its local networks. + +Optionally the whole thing (QEMU and slirp4netns) is placed in a per-workflow +cgroup with memory, swap and pids limits, so the budget above is actually +enforced and not just bookkeeping. That also allows us to, for example, if the +cgroup OOM-kills the VM we can detect that and report it as such instead of a +generic crash. The spindle supervisor itself also gets a cgroup with a +protected `memory.min`, so under host memory pressure it's the workflows that +get OOM-killed first, not spindle. + +### Boot - run - death + +Once QEMU is up we poll the QMP socket until it accepts a connection and reports +the guest as running, then wait for the guest agent to send handshake message +over vsock from the expected CID. It reports its protocol and versions, and +spindle sends it the job id, trusted cache public keys, and the cache/DNS proxy +ports. + +First the activation step is ran (if on a NixOS image and the workflow is +configured with anything), spindle sends the user config (or a cached toplevel +store path, if we've built this exact base + config combo before) and the agent +builds and activates it before the user steps run. Afterwards, each step is sent +as an exec request (`$shell -lc ` as an unprivileged workflow user in +`/workspace/repo`, with workflow/step environment and unlocked secrets), and +stdout/stderr stream back as messages until an exit message arrives. Timeouts +are cooperative: we derive a deadline from the workflow timeout and ship it to +the guest, with a little grace on the host side so the guest gets to report the +timeout itself. While a step runs we also watch for the VM crashing, if it does +we tail the serial (and qemu) logs into the step's stderr so you get something +more useful than "guest agent connection lost: EOF". + +Teardown is same whether the workflow succeeded, failed or timed out: drain the +guest's pending Nix cache uploads, ask the agent to power off and wait for QEMU +to exit (falling back to QMP `system_powerdown` and finally a kill if it +doesn't), then close the proxies and remove the work directory. + +### Nix cache + +The two host-side proxies are how the guest talks to spindle's Nix cache without +ever needing credentials or direct network access; like the agent they reach the +host over vsock. + +The read proxy fronts the configured substituters plus any workflow-level +`caches`. When the guest needs to realize a store path it asks the proxy, which +queries the read caches concurrently and returns the first successful response, +with a 404 only winning if every upstream returns 404. + +The upload proxy goes the other way: paths built inside the guest are pushed to +spindle's configured upload cache (if any) so the next workflow that needs them +doesn't rebuild. Paths already present on any configured read cache are skipped. +The agent queues built paths and they're uploaded eagerly as they appear; any +still in flight at teardown block the drain step until they finish. diff --git a/spindle/engines/microvm/agent.go b/spindle/engines/microvm/agent.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/agent.go @@ -0,0 +1,368 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/mdlayher/vsock" + + "tangled.org/core/spindle/agentproto" + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +const guestWorkflowUser = "spindle-workflow" + +var errGuestTimedOut = errors.New("guest reported step timed out") + +type agentHub struct { + l *slog.Logger + ln *vsock.Listener + pending map[uint32]chan net.Conn + mu sync.Mutex +} + +func newAgentHub(port uint32, l *slog.Logger) (*agentHub, error) { + ln, err := vsock.Listen(port, nil) + if err != nil { + return nil, fmt.Errorf("listen for agent on vsock port %d: %w", port, err) + } + h := &agentHub{ + l: l, + ln: ln, + pending: make(map[uint32]chan net.Conn), + } + go h.acceptLoop() + return h, nil +} + +func (h *agentHub) expect(cid uint32) (<-chan net.Conn, func(), error) { + h.mu.Lock() + defer h.mu.Unlock() + if _, exists := h.pending[cid]; exists { + return nil, nil, fmt.Errorf("already waiting for agent cid %d", cid) + } + ch := make(chan net.Conn, 1) + h.pending[cid] = ch + unregister := func() { + h.mu.Lock() + delete(h.pending, cid) + h.mu.Unlock() + close(ch) + for conn := range ch { + if conn != nil { + _ = conn.Close() + } + } + } + return ch, unregister, nil +} + +func (h *agentHub) acceptLoop() { + for { + conn, err := h.ln.Accept() + if err != nil { + h.l.Error("agent vsock accept failed", "error", err) + return + } + + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if !ok { + h.l.Warn("agent connection has unexpected remote address", "remote", conn.RemoteAddr()) + _ = conn.Close() + continue + } + + h.mu.Lock() + ch, ok := h.pending[addr.ContextID] + if ok { + delete(h.pending, addr.ContextID) + } + h.mu.Unlock() + + // todo: if / when we add agent recovery (reconnect) we should add a + // boot-initialized session credential to prevent random connections... + // checking cid here works to ensure for now since we dont attempt to + // reconnect, so we block anything else thats not expected (and agent + // runs first in the boot sequence always). + if !ok { + h.l.Warn("dropping agent connection for unknown cid", "cid", addr.ContextID) + _ = conn.Close() + continue + } + + select { + case ch <- conn: + default: + _ = conn.Close() + } + } +} + +type AgentExec struct { + *agentv1.ExecStart + ID string + Stdout io.Writer + Stderr io.Writer +} + +type AgentSession struct { + conn net.Conn + enc *agentproto.Encoder + dec *agentproto.Decoder + l *slog.Logger + mu sync.Mutex +} + +func NewAgentSession(conn net.Conn, l *slog.Logger) *AgentSession { + return &AgentSession{ + conn: conn, + enc: agentproto.NewEncoder(conn), + dec: agentproto.NewDecoder(conn), + l: l, + } +} + +func (s *AgentSession) Init(ctx context.Context, init *agentv1.Init) error { + s.mu.Lock() + defer s.mu.Unlock() + + hello, err := s.decode(ctx) + if err != nil { + return fmt.Errorf("read agent hello: %w", err) + } + helloPayload := hello.Hello + if helloPayload == nil { + return fmt.Errorf("expected agent hello, got nil") + } + s.l.Info("agent connected", "protocol", helloPayload.ProtocolVersion, "version", helloPayload.AgentVersion, "boot", helloPayload.BootId, "nix", helloPayload.NixVersion) + + if err := s.enc.Encode(&agentproto.Message{ + Id: "init", + Init: init, + }); err != nil { + return fmt.Errorf("send agent init: %w", err) + } + return nil +} + +func (s *AgentSession) Exec(ctx context.Context, exec AgentExec) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if exec.ID == "" { + return 0, fmt.Errorf("empty ID passed to Exec") + } + + if exec.ExecStart.TimeoutSeconds == 0 { + exec.ExecStart.TimeoutSeconds = timeoutSeconds(ctx, guestTimeoutGrace) + } + + if err := s.enc.Encode(&agentproto.Message{ + Id: exec.ID, + ExecStart: exec.ExecStart, + }); err != nil { + return 0, fmt.Errorf("send exec_start: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return 0, err + } + if msg.BuiltPaths == nil && msg.Id != exec.ID { + continue + } + + if p := msg.ExecStdout; p != nil { + _, _ = io.WriteString(exec.Stdout, p.Data) + } else if p := msg.ExecStderr; p != nil { + _, _ = io.WriteString(exec.Stderr, p.Data) + } else if p := msg.BuiltPaths; p != nil { + // s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths)) + } else if p := msg.ExecExit; p != nil { + if p.Error != "" { + s.l.Warn("guest exec error", "id", msg.Id, "error", p.Error) + } + if p.TimedOut { + return int(p.ExitCode), errGuestTimedOut + } + return int(p.ExitCode), nil + } + } +} + +func (s *AgentSession) ActivateConfig(ctx context.Context, id string, req *agentv1.ActivateConfig) (*agentv1.ActivateConfigResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if id == "" { + return nil, fmt.Errorf("empty ID passed to ActivateConfig") + } + if req.TimeoutSeconds == 0 { + req.TimeoutSeconds = timeoutSeconds(ctx, guestTimeoutGrace) + } + if err := s.enc.Encode(&agentproto.Message{ + Id: id, + ActivateConfig: req, + }); err != nil { + return nil, fmt.Errorf("send activate_config: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return nil, err + } + if msg.BuiltPaths == nil && msg.Id != id { + continue + } + + if p := msg.BuiltPaths; p != nil { + // s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths)) + } else if p := msg.ActivateConfigResult; p != nil { + if p.Error != "" { + return nil, fmt.Errorf("activate config failed: %s", p.Error) + } + if p.Toplevel == "" { + return nil, fmt.Errorf("activate config returned empty toplevel") + } + return p, nil + } + } +} + +func (s *AgentSession) Poweroff(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + + id := "poweroff" + if err := s.enc.Encode(&agentproto.Message{ + Id: id, + Poweroff: &agentv1.Poweroff{}, + }); err != nil { + return fmt.Errorf("send poweroff: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return err + } + if msg.Id != id { + continue + } + p := msg.PoweroffResult + if p == nil { + continue + } + if p.Error != "" { + return fmt.Errorf("guest poweroff failed: %s", p.Error) + } + return nil + } +} + +func (s *AgentSession) Drain(ctx context.Context) (uint32, error) { + s.mu.Lock() + defer s.mu.Unlock() + + drainID := "cache-drain" + if err := s.enc.Encode(&agentproto.Message{ + Id: drainID, + CacheDrain: &agentv1.CacheDrain{ + TimeoutSeconds: timeoutSeconds(ctx, 0), + }, + }); err != nil { + return 0, fmt.Errorf("send cache_drain: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return 0, err + } + if msg.Id != drainID { + continue + } + p := msg.CacheDrainResult + if p == nil { + continue + } + s.l.Info("cache drain complete", "uploaded", p.CacheUploaded, "failed", p.CacheFailed, "queued", p.CacheQueued, "active", p.CacheActive) + if p.Error != "" { + return 0, fmt.Errorf("cache drain failed: %s", p.Error) + } + if p.CacheFailed > 0 { + return 0, fmt.Errorf("cache drain failed for %d paths", p.CacheFailed) + } + if p.CacheQueued > 0 || p.CacheActive > 0 { + return 0, fmt.Errorf("cache drain incomplete: queued=%d active=%d", p.CacheQueued, p.CacheActive) + } + return p.CacheUploaded, nil + } +} + +func (s *AgentSession) decode(ctx context.Context) (*agentproto.Message, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if deadline, ok := ctx.Deadline(); ok { + _ = s.conn.SetReadDeadline(deadline) + } else { + _ = s.conn.SetReadDeadline(time.Time{}) + } + + // a blocked vsock read wont wake up just from the ctx being cancelled, + // only a deadline will wake it up, so if the VM crashes mid-step the read would + // hang until workflow timeout. so we will set a deadline in the past to cancel it. + // + // we set a deadline here instead of closing the connection, this is the long-lived + // connection that everything reuses, so we only really want to interrupt it for this + // current read. this also lands as a timeout error which the netErr.Timeout() check + // below maps to ctx.Err() correctly + stop := context.AfterFunc(ctx, func() { + _ = s.conn.SetReadDeadline(time.Now()) + }) + defer stop() + + msg, err := s.dec.Decode() + if err != nil { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() && ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("read agent message: %w", err) + } + return msg, nil +} + +func (s *AgentSession) Close() error { + if s == nil || s.conn == nil { + return nil + } + return s.conn.Close() +} + +// this pulls the deadline from the context and converts it to what the +// agentproto expects +func timeoutSeconds(ctx context.Context, lead time.Duration) uint32 { + deadline, ok := ctx.Deadline() + if !ok { + return 0 + } + seconds := int64((time.Until(deadline) - lead).Round(time.Second) / time.Second) + if seconds < 1 { + return 1 + } + if seconds > int64(^uint32(0)) { + return ^uint32(0) + } + return uint32(seconds) +} diff --git a/spindle/engines/microvm/args.go b/spindle/engines/microvm/args.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/args.go @@ -0,0 +1,69 @@ +package microvm + +import ( + "fmt" + "strings" +) + +type argBuilder struct { + args []string +} + +func newArgBuilder(capacity int) argBuilder { + return argBuilder{ + args: make([]string, 0, capacity), + } +} + +func (b *argBuilder) Add(args ...string) *argBuilder { + b.args = append(b.args, args...) + return b +} + +func (b *argBuilder) Flag(name string) *argBuilder { + b.args = append(b.args, name) + return b +} + +func (b *argBuilder) Opt(name, value string) *argBuilder { + b.args = append(b.args, name, value) + return b +} + +func (b *argBuilder) Optf(name, format string, values ...any) *argBuilder { + return b.Opt(name, fmt.Sprintf(format, values...)) +} + +func (b *argBuilder) Args() []string { + args := make([]string, len(b.args)) + copy(args, b.args) + return args +} + +type optionBuilder struct { + parts []string +} + +func newOptionBuilder(capacity int) optionBuilder { + return optionBuilder{ + parts: make([]string, 0, capacity), + } +} + +func (b *optionBuilder) Add(parts ...string) *optionBuilder { + b.parts = append(b.parts, parts...) + return b +} + +func (b *optionBuilder) KV(key, value string) *optionBuilder { + b.parts = append(b.parts, key+"="+value) + return b +} + +func (b *optionBuilder) KVf(key, format string, values ...any) *optionBuilder { + return b.KV(key, fmt.Sprintf(format, values...)) +} + +func (b optionBuilder) String() string { + return strings.Join(b.parts, ",") +} diff --git a/spindle/engines/microvm/bench-boot.sh b/spindle/engines/microvm/bench-boot.sh new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/bench-boot.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# quick boot-time benchmark for the spindle nixos microvm. +# boots N times running a trivial command, reports wall-clock + systemd-analyze. +# needs: sudo modprobe vhost_vsock +set -euo pipefail + +N="${1:-5}" +cd "$(git rev-parse --show-toplevel)" + +strip_ansi() { sed -E "s/$(printf '\033')\[[0-9;]*[a-zA-Z]//g; s/$(printf '\033')\([a-zA-Z]//g"; } + +echo ">>> building runner + image" +nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run +TARBALL=$(nix build .#spindle-nixos-image-tarball --no-link --print-out-paths) + +WORK=$(mktemp -d -t spindle-bench-XXXXXX) +trap 'chmod -R +w "$WORK" 2>/dev/null || true; rm -rf "$WORK"' EXIT +mkdir -p "$WORK/image" +tar -C "$WORK/image" -xzf "$TARBALL" +SPEC="$WORK/image/spec.json" + +echo ">>> systemd-analyze breakdown" +spindle/spindle-microvm-run --image-spec "$SPEC" --work-dir "$WORK/analyze" --exec-timeout 60s -- \ + /run/current-system/sw/bin/systemd-analyze time 2>/dev/null | strip_ansi | grep -i startup || true + +echo ">>> $N timed boot+exec(true) runs" +total=0 +for i in $(seq 1 "$N"); do + start=$EPOCHREALTIME + spindle/spindle-microvm-run --image-spec "$SPEC" --work-dir "$WORK/run$i" --exec-timeout 60s -- \ + /run/current-system/sw/bin/true >/dev/null 2>&1 + end=$EPOCHREALTIME + ms=$(( (${end%.*} - ${start%.*}) * 1000 + (10#${end#*.} - 10#${start#*.}) / 1000 )) + echo " run $i: ${ms}ms" + total=$((total + ms)) + rm -rf "$WORK/run$i" +done +echo ">>> mean wall-clock: $((total / N))ms over $N runs" diff --git a/spindle/engines/microvm/budget.go b/spindle/engines/microvm/budget.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/budget.go @@ -0,0 +1,91 @@ +package microvm + +import ( + "context" + "fmt" + "time" + + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" +) + +type Resources struct { + MemoryMiB int64 + VCPUs int64 + DiskMiB int64 +} + +func (r Resources) Fits(limit Resources) bool { + if limit.MemoryMiB > 0 && r.MemoryMiB > limit.MemoryMiB { + return false + } + if limit.VCPUs > 0 && r.VCPUs > limit.VCPUs { + return false + } + if limit.DiskMiB > 0 && r.DiskMiB > limit.DiskMiB { + return false + } + return true +} + +func (r Resources) Add(other Resources) Resources { + return Resources{ + MemoryMiB: r.MemoryMiB + other.MemoryMiB, + VCPUs: r.VCPUs + other.VCPUs, + DiskMiB: r.DiskMiB + other.DiskMiB, + } +} + +func (r Resources) Sub(other Resources) Resources { + return Resources{ + MemoryMiB: max(0, r.MemoryMiB-other.MemoryMiB), + VCPUs: max(0, r.VCPUs-other.VCPUs), + DiskMiB: max(0, r.DiskMiB-other.DiskMiB), + } +} + +func (r Resources) String() string { + return fmt.Sprintf("memory=%dMiB vcpus=%d disk=%dMiB", r.MemoryMiB, r.VCPUs, r.DiskMiB) +} + +func newVMBudgetConfig(cfg config.MicroVMPipelines) (Resources, Resources, time.Duration) { + budget := Resources{ + MemoryMiB: cfg.MaxTotalMemoryMiB, + VCPUs: cfg.MaxTotalVCPUs, + DiskMiB: cfg.MaxTotalDiskMiB, + } + maxReq := Resources{ + MemoryMiB: cfg.MaxWorkflowMemoryMiB, + VCPUs: cfg.MaxWorkflowVCPUs, + DiskMiB: cfg.MaxWorkflowDiskMiB, + } + return budget, maxReq, cfg.AgingThreshold +} + +func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil { + return nil, fmt.Errorf("microVM workflow state is not initialized") + } + if e.scheduler == nil { + return engine.NoopSlot{}, nil + } + req := resourcesForImage(state.ImageSpec) + if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 { + return nil, fmt.Errorf("microVM resource request must not be negative: %s", req) + } + return e.scheduler.Acquire(ctx, req) +} + +func resourcesForImage(spec ImageSpec) Resources { + var diskMiB int64 + for _, volume := range spec.Volumes { + diskMiB += volume.SizeMiB + } + return Resources{ + MemoryMiB: int64(spec.MemoryMiB), + VCPUs: int64(spec.VCPUs), + DiskMiB: diskMiB, + } +} diff --git a/spindle/engines/microvm/cgroup.go b/spindle/engines/microvm/cgroup.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/cgroup.go @@ -0,0 +1,267 @@ +package microvm + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + + cgroups "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/prometheus/procfs" +) + +var ( + cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`) + cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`) +) + +const ( + cgroupParentSelf = "self" + supervisorCgroupName = "supervisor" +) + +type CgroupLimits struct { + Enabled bool + Parent *CgroupParent + Name string + MemoryMaxMiB int64 + SwapMaxMiB *int64 + PidsMax int64 +} + +type CgroupParent struct { + root *cgroup2.Manager + mountpoint string + group string +} + +type CgroupHandle struct { + manager *cgroup2.Manager +} + +func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) { + if parent == "" { + parent = cgroupParentSelf + } + if cgroups.Mode() != cgroups.Unified { + return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode") + } + + mountpoint, group, err := resolveCgroupParent(parent) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil { + return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err) + } + + root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint)) + if err != nil { + return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err) + } + + if group != "/" { + if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil { + return nil, err + } + } + + if logger != nil { + logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group) + } + return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil +} + +func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) { + if !limits.Enabled { + return nil, nil + } + if limits.Parent == nil || limits.Parent.root == nil { + return nil, fmt.Errorf("cgroup parent is not initialized") + } + name := sanitizeCgroupName(limits.Name) + if name == "" { + return nil, fmt.Errorf("cgroup name is empty") + } + + manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits)) + if err != nil { + return nil, fmt.Errorf("create cgroup %q: %w", name, err) + } + + if logger != nil { + logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group) + } + return &CgroupHandle{manager: manager}, nil +} + +func cgroupResources(limits CgroupLimits) *cgroup2.Resources { + resources := &cgroup2.Resources{} + if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil { + memory := &cgroup2.Memory{} + if limits.MemoryMaxMiB > 0 { + maxBytes := limits.MemoryMaxMiB * 1024 * 1024 + memory.Max = &maxBytes + } + if limits.SwapMaxMiB != nil { + swapBytes := *limits.SwapMaxMiB * 1024 * 1024 + memory.Swap = &swapBytes + } + oomGroup := true + memory.OOMGroup = &oomGroup + resources.Memory = memory + } + if limits.PidsMax > 0 { + resources.Pids = &cgroup2.Pids{Max: limits.PidsMax} + } + return resources +} + +func supervisorResources(memoryMinMiB int64) *cgroup2.Resources { + if memoryMinMiB <= 0 { + return nil + } + minBytes := memoryMinMiB * 1024 * 1024 + return &cgroup2.Resources{ + Memory: &cgroup2.Memory{Min: &minBytes}, + } +} + +func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error { + if h == nil || h.manager == nil { + return nil + } + if pid <= 0 { + return fmt.Errorf("invalid pid %d", pid) + } + if err := h.manager.AddProc(uint64(pid)); err != nil { + return fmt.Errorf("add pid %d to cgroup: %w", pid, err) + } + if logger != nil { + logger.Info("added process to microVM cgroup", "pid", pid) + } + return nil +} + +func (h *CgroupHandle) Close() error { + if h == nil || h.manager == nil { + return nil + } + return h.manager.Delete() +} + +func (h *CgroupHandle) OOMKilled() bool { + if h == nil || h.manager == nil { + return false + } + metrics, err := h.manager.Stat() + if err != nil || metrics == nil || metrics.MemoryEvents == nil { + return false + } + return metrics.MemoryEvents.OomKill > 0 +} + +func resolveCgroupParent(parent string) (string, string, error) { + mountpoint, err := cgroup2Mountpoint() + if err != nil { + return "", "", err + } + + if parent == "" || parent == cgroupParentSelf { + group, err := selfCgroupV2Path() + if err != nil { + return "", "", err + } + return mountpoint, group, nil + } + if !filepath.IsAbs(parent) { + return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent) + } + + cleanParent := filepath.Clean(parent) + rel, err := filepath.Rel(mountpoint, cleanParent) + if err != nil { + return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err) + } + if rel == ".." || strings.HasPrefix(rel, "../") { + return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint) + } + if rel == "." { + return mountpoint, "/", nil + } + + group := "/" + filepath.ToSlash(rel) + if err := cgroup2.VerifyGroupPath(group); err != nil { + return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err) + } + return mountpoint, group, nil +} + +func cgroup2Mountpoint() (string, error) { + mounts, err := procfs.GetMounts() + if err != nil { + return "", fmt.Errorf("read procfs mountinfo: %w", err) + } + for _, mount := range mounts { + if mount.FSType == "cgroup2" { + return mount.MountPoint, nil + } + } + return "", fmt.Errorf("cgroup v2 mountpoint not found") +} + +func selfCgroupV2Path() (string, error) { + self, err := procfs.Self() + if err != nil { + return "", fmt.Errorf("open procfs self: %w", err) + } + groups, err := self.Cgroups() + if err != nil { + return "", fmt.Errorf("read procfs self cgroups: %w", err) + } + for _, group := range groups { + if group.HierarchyID != 0 { + continue + } + path := group.Path + if path == "" { + path = "/" + } + if err := cgroup2.VerifyGroupPath(path); err != nil { + return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err) + } + return path, nil + } + return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry") +} + +func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error { + supervisor, err := parent.NewChild(supervisorCgroupName, supervisorResources(supervisorMemoryMinMiB)) + if err != nil { + return fmt.Errorf("create supervisor cgroup: %w", err) + } + + procs, err := parent.Procs(false) + if err != nil { + return fmt.Errorf("list parent cgroup processes: %w", err) + } + for _, pid := range procs { + if err := supervisor.AddProc(pid); err != nil { + return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err) + } + } + + if logger != nil && len(procs) > 0 { + logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs)) + } + return nil +} + +func sanitizeCgroupName(name string) string { + name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-") + name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-") + return strings.Trim(name, "-_.") +} diff --git a/spindle/engines/microvm/cgroup_oom_test.go b/spindle/engines/microvm/cgroup_oom_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/cgroup_oom_test.go @@ -0,0 +1,126 @@ +package microvm + +import ( + "log/slog" + "os" + "os/exec" + "runtime" + "testing" + "time" + + cgroups "github.com/containerd/cgroups/v3" +) + +const memhogEnv = "SPINDLE_CGROUP_MEMHOG" + +func TestMain(m *testing.M) { + if os.Getenv(memhogEnv) == "1" { + runMemhogChild() + return + } + os.Exit(m.Run()) +} + +// this will allocate memory in steps until either the cgroup kills the process +// this is running on, or if the limit is reached. the limit is there so that if +// the cgroup somehow does not work, we don't kill the host and can observe that +// failure. +func runMemhogChild() { + var b [1]byte + _, _ = os.Stdin.Read(b[:]) + + const chunk = 4 << 20 // 4 MiB + const limit = 512 << 20 // safety cap + hold := make([][]byte, 0, limit/chunk) + for total := 0; total < limit; total += chunk { + c := make([]byte, chunk) + for i := range c { + c[i] = 1 // fault the pages in so they count against memory.current + } + hold = append(hold, c) + time.Sleep(5 * time.Millisecond) + } + runtime.KeepAlive(hold) + os.Exit(0) +} + +// creates a cgroup parent, adds a memory limited child to it, and creates a +// process that hogs memory and observes if it OOMs or not. +// +// run with: +// +// SPINDLE_CGROUP_INTEGRATION=1 systemd-run --user --scope -p Delegate=yes \ +// go test -run TestCgroupOOMEnforcement ./spindle/engines/microvm/ +func TestCgroupOOMEnforcement(t *testing.T) { + if os.Getenv("SPINDLE_CGROUP_INTEGRATION") != "1" { + t.Skip("see test doc comment on how to run") + } + if cgroups.Mode() != cgroups.Unified { + t.Skip("requires cgroup v2 unified mode") + } + + logger := slog.Default() + + parent, err := initCgroupParent(cgroupParentSelf, 0, logger) + if err != nil { + t.Skipf("cannot initialize cgroup parent (need cgroup v2 delegation): %v", err) + } + + swap := int64(0) // disable swap so the limit forces an OOM promptly + handle, err := prepareCgroup(CgroupLimits{ + Enabled: true, + Parent: parent, + Name: "cgtest-oom", + MemoryMaxMiB: 64, + SwapMaxMiB: &swap, + PidsMax: 256, + }, logger) + if err != nil { + t.Skipf("cannot create a memory-limited child cgroup (need the memory controller delegated): %v", err) + } + if handle == nil { + t.Fatal("prepareCgroup returned a nil handle for enabled limits") + } + t.Cleanup(func() { _ = handle.Close() }) + + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), memhogEnv+"=1") + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + + if err := handle.AddProcess(cmd.Process.Pid, logger); err != nil { + t.Fatalf("add memhog to cgroup: %v", err) + } + + // let the child process start allocating memory + if _, err := stdin.Write([]byte("g")); err != nil { + t.Fatalf("release memhog: %v", err) + } + _ = stdin.Close() + + waitErr := make(chan error, 1) + go func() { waitErr <- cmd.Wait() }() + + select { + case err := <-waitErr: + if err == nil { + t.Fatal("memhog exited cleanly: the cgroup memory limit was not enforced") + } + t.Logf("memhog died as expected: %v", err) + case <-time.After(30 * time.Second): + t.Fatal("memhog did not die within 30s, cgroup memory limit not enforced") + } + + if !handle.OOMKilled() { + t.Fatal("OOMKilled() is false after the memhog was killed, memory.events oom_kill was not observed") + } +} diff --git a/spindle/engines/microvm/cgroup_test.go b/spindle/engines/microvm/cgroup_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/cgroup_test.go @@ -0,0 +1,39 @@ +package microvm + +import ( + "testing" +) + +func TestSanitizeCgroupName(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"workflow-abc123", "workflow-abc123"}, + {"a/b:c", "a-b-c"}, + {"--lead--", "lead"}, + {"a__b..c", "a-b-c"}, + {"keep.dots_and-dashes", "keep.dots_and-dashes"}, + {"", ""}, + {"///", ""}, + } + for _, tc := range cases { + if got := sanitizeCgroupName(tc.in); got != tc.want { + t.Errorf("sanitizeCgroupName(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestCgroupResourcesSwapOnlyStillSetsMemory(t *testing.T) { + swap := int64(8) + r := cgroupResources(CgroupLimits{SwapMaxMiB: &swap}) + if r.Memory == nil { + t.Fatal("a swap limit alone should still produce a memory controller config") + } + if r.Memory.Max != nil { + t.Errorf("memory max should be unset when only swap is limited, got %v", *r.Memory.Max) + } + if r.Memory.Swap == nil || *r.Memory.Swap != 8*1024*1024 { + t.Errorf("swap = %v, want %d bytes", r.Memory.Swap, 8*1024*1024) + } +} diff --git a/spindle/engines/microvm/dns_proxy.go b/spindle/engines/microvm/dns_proxy.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/dns_proxy.go @@ -0,0 +1,380 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "sync" + "time" + + "github.com/miekg/dns" +) + +const ( + dnsProxyIOTimeout = 10 * time.Second + dnsProxyIdleTimeout = 30 * time.Second + dnsProxyShutdownTimeout = 10 * time.Second + dnsProxyMaxConnections = 64 + dnsProxyMaxTCPQueries = 128 + dnsProxyResolvConfPath = "/etc/resolv.conf" +) + +type DNSProxy struct { + port uint32 + srv *dns.Server + + closeOnce sync.Once + closeErr error +} + +func StartDNSProxy(ctx context.Context, cid uint32, logger *slog.Logger) (*DNSProxy, error) { + if ctx == nil { + ctx = context.Background() + } + + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "dns_proxy", "cid", cid) + + ln, port, err := listenRandomVsockPort(ctx) + if err != nil { + return nil, fmt.Errorf("listen for dns proxy: %w", err) + } + + resolver, err := newHostDNSResolver(dnsProxyResolvConfPath, logger) + if err != nil { + _ = ln.Close() + return nil, err + } + + listener := newLimitedListener( + &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + }, + dnsProxyMaxConnections, + logger, + ) + + proxy := &DNSProxy{ + port: port, + srv: &dns.Server{ + Net: "tcp", + Listener: listener, + Handler: dns.HandlerFunc(resolver.ServeDNS), + ReadTimeout: dnsProxyIOTimeout, + WriteTimeout: dnsProxyIOTimeout, + IdleTimeout: func() time.Duration { return dnsProxyIdleTimeout }, + MaxTCPQueries: dnsProxyMaxTCPQueries, + MsgInvalidFunc: func(_ []byte, err error) { + logger.Warn("dns proxy invalid message", "error", err) + }, + }, + } + + go func() { + <-ctx.Done() + _ = proxy.Close() + }() + + go func() { + if err := proxy.srv.ActivateAndServe(); err != nil && !errors.Is(err, net.ErrClosed) { + logger.Warn("dns proxy stopped", "error", err) + } + }() + + logger.Info("started dns proxy", "port", port) + return proxy, nil +} + +func (p *DNSProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *DNSProxy) Close() error { + if p == nil || p.srv == nil { + return nil + } + + p.closeOnce.Do(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), dnsProxyShutdownTimeout) + defer cancel() + + p.closeErr = p.srv.ShutdownContext(shutdownCtx) + }) + return p.closeErr +} + +type limitedListener struct { + net.Listener + slots chan struct{} + logger *slog.Logger +} + +func newLimitedListener(listener net.Listener, limit int, logger *slog.Logger) net.Listener { + if limit <= 0 { + return listener + } + return &limitedListener{ + Listener: listener, + slots: make(chan struct{}, limit), + logger: logger, + } +} + +func (l *limitedListener) Accept() (net.Conn, error) { + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + select { + case l.slots <- struct{}{}: + return &limitedConn{ + Conn: conn, + release: func() { + <-l.slots + }, + }, nil + default: + l.logger.Warn("dns proxy dropped connection because workers are busy") + _ = conn.Close() + } + } +} + +type limitedConn struct { + net.Conn + once sync.Once + release func() +} + +func (c *limitedConn) Close() error { + err := c.Conn.Close() + c.once.Do(c.release) + return err +} + +type hostDNSResolver struct { + upstreams []string + attempts int + timeout time.Duration + logger *slog.Logger +} + +func newHostDNSResolver(path string, logger *slog.Logger) (*hostDNSResolver, error) { + config, err := dns.ClientConfigFromFile(path) + if err != nil { + return nil, fmt.Errorf("read host resolv.conf: %w", err) + } + if len(config.Servers) == 0 { + return nil, fmt.Errorf("host resolv.conf has no nameservers") + } + + port := config.Port + if port == "" { + port = "53" + } + + upstreams := make([]string, 0, len(config.Servers)) + for _, server := range config.Servers { + upstreams = append(upstreams, net.JoinHostPort(server, port)) + } + + timeout := time.Duration(config.Timeout) * time.Second + if timeout <= 0 { + timeout = dnsProxyIOTimeout + } + + return &hostDNSResolver{ + upstreams: upstreams, + attempts: max(config.Attempts, 1), + timeout: timeout, + logger: logger, + }, nil +} + +func (r *hostDNSResolver) ServeDNS(w dns.ResponseWriter, req *dns.Msg) { + resp, err := r.exchange(req) + if err != nil { + r.logger.Warn( + "dns upstream exchange failed", + "question", dnsQuestionLogValue(req), + "error", err, + ) + if err := w.WriteMsg(rcodeResponse(req, dns.RcodeServerFailure)); err != nil { + r.logger.Warn("dns proxy response write failed", "error", err) + } + return + } + + filterDNSResponse(resp) + + if err := w.WriteMsg(resp); err != nil { + r.logger.Warn("dns proxy response write failed", "error", err) + } +} + +func (r *hostDNSResolver) exchange(req *dns.Msg) (*dns.Msg, error) { + var errs []error + + for range r.attempts { + for _, upstream := range r.upstreams { + resp, err := exchangeDNSAt(req, upstream, r.timeout) + if err == nil { + return resp, nil + } + errs = append(errs, fmt.Errorf("%s: %w", upstream, err)) + } + } + + return nil, errors.Join(errs...) +} + +func exchangeDNSAt(req *dns.Msg, addr string, timeout time.Duration) (*dns.Msg, error) { + resp, _, err := (&dns.Client{Net: "udp", Timeout: timeout}).Exchange(req, addr) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("empty udp response") + } + if !resp.Truncated { + return resp, nil + } + + resp, _, err = (&dns.Client{Net: "tcp", Timeout: timeout}).Exchange(req, addr) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("empty tcp response") + } + return resp, nil +} + +func filterDNSResponse(msg *dns.Msg) { + if msg == nil { + return + } + msg.Answer = filterDNSRRs(msg.Answer) + msg.Ns = filterDNSRRs(msg.Ns) + msg.Extra = filterDNSRRs(msg.Extra) +} + +func filterDNSRRs(rrs []dns.RR) []dns.RR { + filtered := rrs[:0] + for _, rr := range rrs { + if rr := filterDNSRR(rr); rr != nil { + filtered = append(filtered, rr) + } + } + return filtered +} + +func filterDNSRR(rr dns.RR) dns.RR { + switch rr := rr.(type) { + case *dns.A: + if isBlockedNamespaceIP(rr.A) { + return nil + } + case *dns.AAAA: + if isBlockedNamespaceIP(rr.AAAA) { + return nil + } + case *dns.SVCB: + filterSVCBValues(&rr.Value) + case *dns.HTTPS: + filterSVCBValues(&rr.Value) + } + return rr +} + +// this removes any blocked namespaces in ipv4/v6 hints +func filterSVCBValues(values *[]dns.SVCBKeyValue) { + filtered := (*values)[:0] + for _, value := range *values { + switch value := value.(type) { + case *dns.SVCBIPv4Hint: + value.Hint = filterDNSIPs(value.Hint) + if len(value.Hint) == 0 { + continue + } + case *dns.SVCBIPv6Hint: + value.Hint = filterDNSIPs(value.Hint) + if len(value.Hint) == 0 { + continue + } + } + filtered = append(filtered, value) + } + *values = filtered +} + +func filterDNSIPs(ips []net.IP) []net.IP { + filtered := ips[:0] + for _, ip := range ips { + if !isBlockedNamespaceIP(ip) { + filtered = append(filtered, ip) + } + } + return filtered +} + +func isBlockedNamespaceIP(ip net.IP) bool { + if ip == nil { + return true + } + if ip4 := ip.To4(); ip4 != nil { + return isBlockedByNamespaceNets(ip4, 32) + } + return isBlockedByNamespaceNets(ip, 128) +} + +func isBlockedByNamespaceNets(ip net.IP, bits int) bool { + for _, blockedNet := range blockedNamespaceNets { + if blockedNet == nil { + continue + } + + _, blockedBits := blockedNet.Mask.Size() + if blockedBits != bits { + continue + } + if blockedNet.Contains(ip) { + return true + } + } + return false +} + +func rcodeResponse(req *dns.Msg, rcode int) *dns.Msg { + resp := new(dns.Msg) + if req == nil { + resp.Rcode = rcode + return resp + } + resp.SetRcode(req, rcode) + return resp +} + +func dnsQuestionLogValue(msg *dns.Msg) string { + if msg == nil || len(msg.Question) == 0 { + return "" + } + + q := msg.Question[0] + qtype := dns.TypeToString[q.Qtype] + if qtype == "" { + qtype = fmt.Sprintf("TYPE%d", q.Qtype) + } + return fmt.Sprintf("%s/%s", q.Name, qtype) +} diff --git a/spindle/engines/microvm/dns_proxy_test.go b/spindle/engines/microvm/dns_proxy_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/dns_proxy_test.go @@ -0,0 +1,73 @@ +package microvm + +import ( + "net" + "testing" + + "github.com/miekg/dns" +) + +func TestFilterDNSResponseDropsBlockedAddressRecords(t *testing.T) { + msg := new(dns.Msg) + msg.Answer = []dns.RR{ + &dns.CNAME{Hdr: dns.RR_Header{Name: "cache.example.", Rrtype: dns.TypeCNAME, Class: dns.ClassINET}, Target: "edge.example."}, + &dns.A{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("1.1.1.1")}, + &dns.A{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("10.0.0.1")}, + &dns.AAAA{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("2606:4700:4700::1111")}, + &dns.AAAA{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("fd7a:115c:a1e0::53")}, + } + msg.Extra = []dns.RR{ + &dns.A{Hdr: dns.RR_Header{Name: "private.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("192.168.1.2")}, + &dns.A{Hdr: dns.RR_Header{Name: "public.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("8.8.8.8")}, + } + + filterDNSResponse(msg) + + if len(msg.Answer) != 3 { + t.Fatalf("filtered answer len = %d, want 3: %#v", len(msg.Answer), msg.Answer) + } + if _, ok := msg.Answer[0].(*dns.CNAME); !ok { + t.Fatalf("answer[0] = %T, want CNAME", msg.Answer[0]) + } + if a, ok := msg.Answer[1].(*dns.A); !ok || !a.A.Equal(net.ParseIP("1.1.1.1")) { + t.Fatalf("answer[1] = %#v, want public A", msg.Answer[1]) + } + if aaaa, ok := msg.Answer[2].(*dns.AAAA); !ok || !aaaa.AAAA.Equal(net.ParseIP("2606:4700:4700::1111")) { + t.Fatalf("answer[2] = %#v, want public AAAA", msg.Answer[2]) + } + if len(msg.Extra) != 1 { + t.Fatalf("filtered extra len = %d, want 1: %#v", len(msg.Extra), msg.Extra) + } +} + +func TestFilterDNSResponseFiltersSVCBAddressHints(t *testing.T) { + msg := new(dns.Msg) + msg.Answer = []dns.RR{ + &dns.HTTPS{ + SVCB: dns.SVCB{ + Hdr: dns.RR_Header{Name: "svc.example.", Rrtype: dns.TypeHTTPS, Class: dns.ClassINET}, + Priority: 1, + Target: ".", + Value: []dns.SVCBKeyValue{ + &dns.SVCBIPv4Hint{Hint: []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("8.8.8.8")}}, + &dns.SVCBIPv6Hint{Hint: []net.IP{net.ParseIP("fd7a:115c:a1e0::53"), net.ParseIP("2001:4860:4860::8888")}}, + }, + }, + }, + } + + filterDNSResponse(msg) + + https := msg.Answer[0].(*dns.HTTPS) + if len(https.Value) != 2 { + t.Fatalf("https values len = %d, want 2", len(https.Value)) + } + ipv4 := https.Value[0].(*dns.SVCBIPv4Hint) + if len(ipv4.Hint) != 1 || !ipv4.Hint[0].Equal(net.ParseIP("8.8.8.8")) { + t.Fatalf("ipv4 hints = %v, want [8.8.8.8]", ipv4.Hint) + } + ipv6 := https.Value[1].(*dns.SVCBIPv6Hint) + if len(ipv6.Hint) != 1 || !ipv6.Hint[0].Equal(net.ParseIP("2001:4860:4860::8888")) { + t.Fatalf("ipv6 hints = %v, want [2001:4860:4860::8888]", ipv6.Hint) + } +} diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/engine.go @@ -0,0 +1,517 @@ +package microvm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "slices" + "sync" + "sync/atomic" + "time" + + "gopkg.in/yaml.v3" + + "tangled.org/core/api/tangled" + "tangled.org/core/log" + "tangled.org/core/spindle/agentproto" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" +) + +const ( + guestWorkDir = "/workspace/repo" + activationStepAction = "activate-config" + agentAcceptTimeout = 2 * time.Minute + agentHandshakeTimeout = 30 * time.Second + cacheDrainTimeout = 5 * time.Minute + vmShutdownTimeout = 10 * time.Second + guestTimeoutGrace = 5 * time.Second +) + +type cleanupFunc func(context.Context) error + +type Engine struct { + l *slog.Logger + cfg *config.Config + db *db.DB + agent *agentHub + scheduler *engine.ResourceScheduler[Resources] + cgroupParent *CgroupParent + + cleanupMu sync.Mutex + cleanup map[string][]cleanupFunc +} + +type Step struct { + name string + kind models.StepKind + command string + environment map[string]string + action string + config manifestConfig + configKey string +} + +func (s Step) Name() string { return s.name } +func (s Step) Command() string { return s.command } +func (s Step) Kind() models.StepKind { return s.kind } + +func New(ctx context.Context, cfg *config.Config, d *db.DB) (*Engine, error) { + l := log.FromContext(ctx).With("component", "engine.microvm") + port := cfg.MicroVMPipelines.AgentPort + if port == 0 { + port = agentproto.DefaultPort + } + agent, err := newAgentHub(port, l) + if err != nil { + return nil, err + } + budget, max, agingThreshold := newVMBudgetConfig(cfg.MicroVMPipelines) + l.Info("initialized microVM workflow budget", "budget", budget.String(), "maxWorkflow", max.String(), "agingThreshold", agingThreshold) + + var cgroupParent *CgroupParent + if cfg.MicroVMPipelines.EnableCgroups { + cgroupParent, err = initCgroupParent(cfg.MicroVMPipelines.CgroupParent, cfg.MicroVMPipelines.CgroupSupervisorMemoryMinMiB, l) + if err != nil { + return nil, err + } + } + + return &Engine{ + l: l, + cfg: cfg, + db: d, + agent: agent, + scheduler: engine.NewResourceScheduler[Resources](budget, max, agingThreshold), + cgroupParent: cgroupParent, + cleanup: make(map[string][]cleanupFunc), + }, nil +} + +func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { + swf := &models.Workflow{} + var dwf manifestWorkflow + + if err := yaml.Unmarshal([]byte(twf.Raw), &dwf); err != nil { + return nil, err + } + + for _, dstep := range dwf.Steps { + swf.Steps = append(swf.Steps, Step{ + name: dstep.Name, + kind: models.StepKindUser, + command: dstep.Command, + environment: dstep.Environment, + }) + } + swf.Name = twf.Name + swf.Environment = dwf.Environment + + if tpl.TriggerMetadata != nil { + if clone := models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev); clone.Command() != "" { + swf.Steps = append([]models.Step{clone}, swf.Steps...) + } + } + + imageSpec, imageSpecPath, imageName, err := e.resolveImage(dwf.Image) + if err != nil { + return nil, err + } + configKey := "" + config := manifestConfig{ + Services: dwf.Services, + Virtualisation: dwf.Virtualisation, + Dependencies: dwf.Dependencies, + Registry: dwf.Registry, + } + if config.Enabled() { + if !imageSpec.SupportsConfigActivation() { + return nil, fmt.Errorf( + "microVM image %q is not a NixOS image: services, virtualisation, dependencies and registry workflow options require a NixOS image", + imageName, + ) + } + var err error + configKey, err = buildConfigKey(imageSpec, config) + if err != nil { + return nil, fmt.Errorf("build config key: %w", err) + } + activationStep := Step{ + name: "NixOS config activation", + kind: models.StepKindSystem, + command: "activate nixos config", + action: activationStepAction, + config: config, + configKey: configKey, + } + + insertAt := 0 + if len(swf.Steps) > 0 && swf.Steps[0].Kind() == models.StepKindSystem { + insertAt = 1 + } + swf.Steps = append(swf.Steps, nil) + copy(swf.Steps[insertAt+1:], swf.Steps[insertAt:]) + swf.Steps[insertAt] = activationStep + } + + cacheURLs, cacheKeys, err := workflowCaches(dwf.Caches) + if err != nil { + return nil, err + } + + swf.Data = &workflowState{ + ImageSpec: imageSpec, + ImageSpecPath: imageSpecPath, + Config: config, + ConfigKey: configKey, + Image: imageName, + CacheReadURLs: cacheURLs, + CacheTrustedPublicKeys: cacheKeys, + NixOSToplevelCache: newNixOSToplevelCacheStore(e.db), + } + return swf, nil +} + +func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { + l := e.l.With("workflow", wid) + setupStep := Step{name: "microVM setup", kind: models.StepKindSystem} + + wfLogger.ControlWriter(-1, setupStep, models.StepStatusStart).Write([]byte{0}) + defer wfLogger.ControlWriter(-1, setupStep, models.StepStatusEnd).Write([]byte{0}) + + state, ok := wf.Data.(*workflowState) + if !ok || state == nil { + return fmt.Errorf("workflow state is not initialized") + } + + cid, err := AllocateCID() + if err != nil { + return err + } + connCh, unregister, err := e.agent.expect(cid) + if err != nil { + return err + } + defer unregister() + + workDirBase := e.cfg.MicroVMPipelines.OverlayDir + if workDirBase == "" { + workDirBase = os.TempDir() + } + workDir, err := os.MkdirTemp(workDirBase, "spindle-microvm-"+wid.String()+"-*") + if err != nil { + return fmt.Errorf("create workflow microVM directory: %w", err) + } + state.WorkDir = workDir + + setupDone := false + defer func() { + if setupDone { + return + } + if err := e.cleanupState(context.Background(), wid, state); err != nil { + l.Error("failed to cleanup failed setup", "error", err) + } + }() + + upstreams, err := BuildCacheUpstreams(e.cfg.NixCache.ReadURLs, state.CacheReadURLs) + if err != nil { + return err + } + readCache, err := StartReadCacheProxy(ctx, cid, upstreams, l) + if err != nil { + return err + } + state.ReadCache = readCache + uploadCache, err := StartUploadCacheProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, l) + if err != nil { + return err + } + state.UploadCache = uploadCache + dnsProxy, err := StartDNSProxy(ctx, cid, l) + if err != nil { + return err + } + state.DNSProxy = dnsProxy + + port := e.cfg.MicroVMPipelines.AgentPort + if port == 0 { + port = agentproto.DefaultPort + } + state.ImageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", state.ImageSpec.BootArgs, port) + + fmt.Fprintf(wfLogger.DataWriter(-1, "stdout"), "starting microVM image %s\n", state.Image) + l.Info("starting microVM workflow", "image", state.Image, "imageSpec", state.ImageSpecPath, "cid", cid, "workDir", workDir) + + var vm VMHandle + vm, err = StartVM(ctx, VMConfig{ + Image: state.ImageSpec, + CID: cid, + EnableKVM: e.cfg.MicroVMPipelines.EnableKVM, + WorkDir: workDir, + Cgroup: e.cgroupLimits(wid, state.ImageSpec), + Dev: e.cfg.Server.Dev, + }, l) + if err != nil { + return err + } + state.VM = vm + + acceptCtx, cancelAccept := context.WithTimeout(ctx, agentAcceptTimeout) + defer cancelAccept() + conn, err := waitAgentConn(acceptCtx, connCh) + if err != nil { + return err + } + + agentSession := NewAgentSession(conn, l) + initCtx, cancelInit := context.WithTimeout(ctx, agentHandshakeTimeout) + defer cancelInit() + if err := agentSession.Init(initCtx, &agentv1.Init{ + JobId: wid.String(), + CacheTrustedPublicKeys: append(slices.Clone(e.cfg.NixCache.TrustedPublicKeys), state.CacheTrustedPublicKeys...), + CacheReadProxyPort: readCache.Port(), + CacheUploadProxyPort: uploadCache.Port(), + DnsProxyPort: dnsProxy.Port(), + }); err != nil { + _ = agentSession.Close() + return err + } + state.Agent = agentSession + wf.Data = state + + e.registerCleanup(wid, func(ctx context.Context) error { + return e.cleanupState(ctx, wid, state) + }) + setupDone = true + + fmt.Fprintf(wfLogger.DataWriter(-1, "stdout"), + "agent connected; serial log: %s\n", vm.Logs().Serial, + ) + return nil +} + +func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { + state, ok := w.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + stderr := wfLogger.DataWriter(idx, "stderr") + + execCtx, vmExited, cancelWatch := watchVMExit(ctx, state.VM) + defer cancelWatch() + + step := w.Steps[idx] + if s, ok := step.(Step); ok && s.action == activationStepAction { + err := e.activateConfig(execCtx, wid, state, s, wfLogger.DataWriter(idx, "stdout")) + return e.classifyStepError(ctx, wid, step, state, stderr, vmExited, err) + } + env := []string{ + "HOME=/workspace", + "LOGNAME=" + guestWorkflowUser, + "PATH=/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "USER=" + guestWorkflowUser, + } + for k, v := range w.Environment { + env = append(env, k+"="+v) + } + for _, s := range secrets { + env = append(env, s.Key+"="+s.Value) + } + if s, ok := step.(Step); ok { + for k, v := range s.environment { + env = append(env, k+"="+v) + } + } + + stdout := wfLogger.DataWriter(idx, "stdout") + exitCode, err := state.Agent.Exec(execCtx, AgentExec{ + ID: fmt.Sprintf("%s-%d", wid.String(), idx), + ExecStart: &agentv1.ExecStart{ + Argv: []string{state.ImageSpec.Shell, "-lc", step.Command()}, + Env: env, + Cwd: guestWorkDir, + User: guestWorkflowUser, + // timeout not set here, Exec will fill it + }, + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + return e.classifyStepError(ctx, wid, step, state, stderr, vmExited, err) + } + + if exitCode != 0 { + e.l.Debug("step exited non-zero", "workflow", wid, "step", step.Name(), "exitCode", exitCode) + return engine.ErrWorkflowFailed + } + return nil +} + +// reads the vm serial logs so we report the tail of that as an error instead of +// just "guest agent connection lost: EOF" +func (e *Engine) classifyStepError(ctx context.Context, wid models.WorkflowId, step models.Step, state *workflowState, stderr io.Writer, vmExited *atomic.Bool, err error) error { + if err == nil { + return nil + } + l := e.l.With("workflow", wid, "step", step.Name()) + + if vmExited != nil && vmExited.Load() { + reason := "microVM exited unexpectedly" + oom := state.VM != nil && state.VM.OOMKilled() + if oom { + reason = "microVM killed by OOM (cgroup memory limit exceeded)" + } + if detail := vmCrashLog(state.VM); detail != "" { + fmt.Fprintf(stderr, "%s:\n%s\n", reason, detail) + l.Debug(reason, "oom", oom, "detail", detail) + } else { + fmt.Fprintln(stderr, reason) + l.Debug(reason, "oom", oom) + } + return errors.New(reason + "; see workflow logs for serial output") + } + + if errors.Is(err, errGuestTimedOut) || ctx.Err() != nil { + l.Debug("step timed out", "guestReported", errors.Is(err, errGuestTimedOut)) + return engine.ErrTimedOut + } + + // the agent connection dropped while qemu stayed up (eg. the guest kernel + // OOM-killed the agent or a guest panic), so surface serial logs, those + // will be more helpful. + if detail := vmCrashLog(state.VM); detail != "" { + fmt.Fprintf(stderr, "step failed (%v):\n%s\n", err, detail) + l.Debug("step failed", "error", err, "detail", detail) + } else { + l.Debug("step failed", "error", err) + } + return err +} + +func (e *Engine) activateConfig(ctx context.Context, wid models.WorkflowId, state *workflowState, step Step, out io.Writer) error { + cfg := step.config + if !cfg.Enabled() { + return nil + } + + configKey := step.configKey + if configKey == "" { + configKey = state.ConfigKey + } + + userConfigJSON, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("encode user config: %w", err) + } + + var cachedToplevel string + if configKey != "" { + if record, ok, err := state.NixOSToplevelCache.Lookup(configKey); err != nil { + return err + } else if ok { + cachedToplevel = record.Toplevel + fmt.Fprintf(out, "realizing cached NixOS config %s\n", cachedToplevel) + } + } + if cachedToplevel == "" { + fmt.Fprintf(out, "building NixOS config from user config\n") + } + + baseHash, err := BaseConfigHash(state.ImageSpec) + if err != nil { + return fmt.Errorf("calculate base config hash: %w", err) + } + + result, err := state.Agent.ActivateConfig(ctx, fmt.Sprintf("%s-config", wid.String()), &agentv1.ActivateConfig{ + ConfigKey: configKey, + BaseConfigHash: baseHash, + UserConfig: string(userConfigJSON), + Toplevel: cachedToplevel, + }) + if err != nil { + return err + } + fmt.Fprintf(out, "activated NixOS config toplevel %s\n", result.Toplevel) + + if cachedToplevel != "" || configKey == "" { + return nil + } + if e.cfg.NixCache.UploadURL == "" { + e.l.Warn("not committing config cache metadata: no upload URL configured", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel) + return nil + } + + drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) + defer cancel() + if _, err := state.Agent.Drain(drainCtx); err != nil { + return fmt.Errorf("drain config cache uploads before metadata commit: %w", err) + } + if err := state.NixOSToplevelCache.Commit(configKey, result.Toplevel); err != nil { + return err + } + fmt.Fprintf(out, "committed config cache metadata %s -> %s\n", configKey, result.Toplevel) + return nil +} + +func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { + fns := e.drainCleanups(wid) + + var cleanupErr error + for i := len(fns) - 1; i >= 0; i-- { + if err := fns[i](ctx); err != nil { + e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) + cleanupErr = errors.Join(cleanupErr, err) + } + } + return cleanupErr +} + +func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error { + return nil +} + +func (e *Engine) WorkflowTimeout() time.Duration { + d, err := time.ParseDuration(e.cfg.MicroVMPipelines.WorkflowTimeout) + if err != nil { + d = 5 * time.Minute + } + return d + guestTimeoutGrace +} + +func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { + e.cleanupMu.Lock() + defer e.cleanupMu.Unlock() + key := wid.String() + e.cleanup[key] = append(e.cleanup[key], fn) +} + +func (e *Engine) drainCleanups(wid models.WorkflowId) []cleanupFunc { + e.cleanupMu.Lock() + defer e.cleanupMu.Unlock() + key := wid.String() + fns := e.cleanup[key] + delete(e.cleanup, key) + return fns +} + +func (e *Engine) cgroupLimits(wid models.WorkflowId, spec ImageSpec) CgroupLimits { + cfg := e.cfg.MicroVMPipelines + return CgroupLimits{ + Enabled: cfg.EnableCgroups, + Parent: e.cgroupParent, + Name: "workflow-" + wid.String(), + MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, + SwapMaxMiB: cfg.CgroupSwapMaxMiB, + PidsMax: cfg.CgroupPidsMax, + } +} diff --git a/spindle/engines/microvm/engine_test.go b/spindle/engines/microvm/engine_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/engine_test.go @@ -0,0 +1,116 @@ +package microvm + +import ( + "encoding/json" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/config" +) + +func writeTestImageSpec(t *testing.T, dir, name string, spec ImageSpec) { + t.Helper() + data, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name+".json"), data, 0o644); err != nil { + t.Fatal(err) + } +} + +func testEngine(t *testing.T, imageDir string) *Engine { + t.Helper() + return &Engine{ + l: slog.Default(), + cfg: &config.Config{ + MicroVMPipelines: config.MicroVMPipelines{ + ImageDir: imageDir, + DefaultImage: "alpine", + }, + }, + } +} + +func TestInitWorkflowRejectsConfigOnNonNixOSImage(t *testing.T) { + dir := t.TempDir() + writeTestImageSpec(t, dir, "alpine", validImageSpec()) + + e := testEngine(t, dir) + _, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: alpine +dependencies: + - nixpkgs#hello +steps: + - name: hello + command: hello +`, + }, tangled.Pipeline{}) + if err == nil { + t.Fatal("expected error for NixOS config options on a non-NixOS image") + } + if !strings.Contains(err.Error(), "NixOS") { + t.Fatalf("error should mention NixOS images, got: %v", err) + } +} + +func TestInitWorkflowPlainStepsOnNonNixOSImage(t *testing.T) { + dir := t.TempDir() + writeTestImageSpec(t, dir, "alpine", validImageSpec()) + + e := testEngine(t, dir) + wf, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: alpine +steps: + - name: hello + command: echo hello +`, + }, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if len(wf.Steps) != 1 { + t.Fatalf("expected exactly the user step, got %d steps", len(wf.Steps)) + } + state, ok := wf.Data.(*workflowState) + if !ok { + t.Fatal("workflow data is not workflowState") + } + if state.ConfigKey != "" { + t.Fatalf("non-NixOS workflow should not have a config key, got %q", state.ConfigKey) + } +} + +func TestInitWorkflowConfigOnNixOSImage(t *testing.T) { + dir := t.TempDir() + spec := validImageSpec() + spec.BaseConfigHash = "abcdef123456" + writeTestImageSpec(t, dir, "nixos", spec) + + e := testEngine(t, dir) + wf, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: nixos +dependencies: + - nixpkgs#hello +steps: + - name: hello + command: hello +`, + }, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if len(wf.Steps) != 2 { + t.Fatalf("expected activation step + user step, got %d steps", len(wf.Steps)) + } + if step, ok := wf.Steps[0].(Step); !ok || step.action != activationStepAction { + t.Fatalf("first step should be the activation step, got %+v", wf.Steps[0]) + } +} diff --git a/spindle/engines/microvm/image.go b/spindle/engines/microvm/image.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/image.go @@ -0,0 +1,254 @@ +package microvm + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const imageSpecFileName = "spec.json" + +type RunnerConfig struct { + CPU string `json:"cpu,omitempty"` + Machine string `json:"machine,omitempty"` + Console string `json:"console,omitempty"` + ExtraArgs []string `json:"extraArgs,omitempty"` +} + +type ImageSpec struct { + Arch string `json:"arch"` + BootArgs string `json:"bootArgs"` + Initrd string `json:"initrd"` + Kernel string `json:"kernel"` + RunnerType string `json:"runnerType"` + RunnerConfig RunnerConfig `json:"runnerConfig"` + MemoryMiB int `json:"memoryMiB"` + NetworkInterfaces []NetworkInterface `json:"networkInterfaces"` + StoreDisk string `json:"storeDisk"` + StoreDiskType string `json:"storeDiskType"` + // baseConfigHash identifies the base nixos configuration baked into the + // image. its only for nixos images as other images won't have a system + // to rebuild. + BaseConfigHash string `json:"baseConfigHash,omitempty"` + // shell is the login shell used to run workflow step commands in the guest. + Shell string `json:"shell"` + VCPUs int `json:"vcpus"` + Volumes []Volume `json:"volumes"` +} + +func (s ImageSpec) SupportsConfigActivation() bool { + return s.BaseConfigHash != "" +} + +type NetworkInterface struct { + Type string `json:"type"` + ID string `json:"id"` + MAC string `json:"mac"` +} + +type Volume struct { + FSType string `json:"fsType"` + Image string `json:"image"` + ImageType string `json:"imageType"` + MountPoint string `json:"mountPoint"` + ReadOnly bool `json:"readOnly"` + SizeMiB int64 `json:"sizeMiB"` +} + +func LoadImageSpec(path string) (ImageSpec, error) { + data, err := os.ReadFile(path) + if err != nil { + return ImageSpec{}, fmt.Errorf("read microvm image spec: %w", err) + } + + var spec ImageSpec + if err := json.Unmarshal(data, &spec); err != nil { + return ImageSpec{}, fmt.Errorf("parse microvm image spec: %w", err) + } + + base := filepath.Dir(path) + spec.Kernel = resolveImageSpecPath(base, spec.Kernel) + spec.Initrd = resolveImageSpecPath(base, spec.Initrd) + spec.StoreDisk = resolveImageSpecPath(base, spec.StoreDisk) + + if err := spec.Validate(); err != nil { + return ImageSpec{}, err + } + return spec, nil +} + +func (s ImageSpec) Validate() error { + if s.Kernel == "" { + return fmt.Errorf("microvm image spec missing kernel") + } + if s.Initrd == "" { + return fmt.Errorf("microvm image spec missing initrd") + } + if s.StoreDisk == "" { + return fmt.Errorf("microvm image spec missing storeDisk") + } + if s.BootArgs == "" { + return fmt.Errorf("microvm image spec missing bootArgs") + } + if s.Shell == "" { + return fmt.Errorf("microvm image spec missing shell") + } + if s.RunnerType == "qemu" || s.RunnerType == "" { + if s.RunnerConfig.Machine == "" { + return fmt.Errorf("microvm image spec missing runnerConfig.machine for qemu runner") + } + } + if s.MemoryMiB <= 0 { + return fmt.Errorf("microvm image spec memoryMiB must be positive") + } + if s.VCPUs <= 0 { + return fmt.Errorf("microvm image spec vcpus must be positive") + } + for _, networkInterface := range s.NetworkInterfaces { + if networkInterface.Type == "" { + return fmt.Errorf("microvm image spec network interface missing type") + } + if networkInterface.ID == "" { + return fmt.Errorf("microvm image spec network interface missing id") + } + if networkInterface.MAC == "" { + return fmt.Errorf("microvm image spec network interface %q missing mac", networkInterface.ID) + } + } + for _, volume := range s.Volumes { + if volume.Image == "" { + return fmt.Errorf("microvm image spec volume missing image") + } + if volume.FSType == "" { + return fmt.Errorf("microvm image spec volume %q missing fsType", volume.Image) + } + if volume.SizeMiB <= 0 { + return fmt.Errorf("microvm image spec volume %q sizeMiB must be positive", volume.Image) + } + } + return nil +} + +func (s ImageSpec) RunnerCmd() string { + switch s.RunnerType { + case "qemu", "": + return "qemu-system-" + s.Arch + case "firecracker": + return "firecracker" + default: + return "" + } +} + +// also see Runner.Validate for where Runner specific files are validated +func (s ImageSpec) validateImageFiles() error { + required := map[string]string{ + "kernel": s.Kernel, + "initrd": s.Initrd, + "storeDisk": s.StoreDisk, + } + for name, path := range required { + if !filepath.IsAbs(path) { + continue + } + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("required image spec file %s not found at %q: %w", name, path, err) + } + } + + return nil +} + +func resolveImageSpecPath(base, path string) string { + if path == "" || filepath.IsAbs(path) { + return path + } + return filepath.Join(base, path) +} + +func (e *Engine) resolveImage(name string) (ImageSpec, string, string, error) { + name = strings.TrimSpace(name) + if name == "" { + name = strings.TrimSpace(e.cfg.MicroVMPipelines.DefaultImage) + } + if name == "" { + return ImageSpec{}, "", "", fmt.Errorf("no image specified in workflow and SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE is not set") + } + if !isPlainImageName(name) { + return ImageSpec{}, "", "", fmt.Errorf("invalid microVM image name %q: must be a plain name, not a path", name) + } + + candidates := imageCandidates(e.cfg.MicroVMPipelines.ImageDir, name) + for _, candidate := range candidates { + path, ok, err := imageSpecPath(candidate) + if err != nil { + return ImageSpec{}, "", "", err + } + if !ok { + continue + } + imageSpec, err := LoadImageSpec(path) + if err != nil { + return ImageSpec{}, "", "", err + } + return imageSpec, path, name, nil + } + + return ImageSpec{}, "", "", fmt.Errorf("microVM image %q was not found; looked in: %s", name, strings.Join(candidates, ", ")) +} + +// check if image name is not a path +func isPlainImageName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + if filepath.IsAbs(name) || strings.ContainsRune(name, '/') || strings.ContainsRune(name, filepath.Separator) { + return false + } + return true +} + +// returns candidates, which is either a directory or spec file itself +func imageCandidates(imageDir, name string) []string { + if imageDir == "" { + return nil + } + return []string{ + filepath.Join(imageDir, name), + filepath.Join(imageDir, name+".json"), + } +} + +// resolve the candidate to a spec: +// - first check if its a file, if yes, return +// - otherwise assume its a directory and check and return `/spec.json` +func imageSpecPath(candidate string) (string, bool, error) { + info, err := os.Stat(candidate) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", false, nil + } + return "", false, err + } + if !info.IsDir() { + return candidate, true, nil + } + + spec := filepath.Join(candidate, imageSpecFileName) + info, err = os.Stat(spec) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName) + } + return "", false, err + } + // this only happens if there is a directory named `spec.json` which would be very silly. + // but better output an error for it anyway :p + if info.IsDir() { + return "", false, fmt.Errorf("microVM image spec %q is a directory", spec) + } + return spec, true, nil +} diff --git a/spindle/engines/microvm/image_test.go b/spindle/engines/microvm/image_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/image_test.go @@ -0,0 +1,127 @@ +package microvm + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeSpecFile(t *testing.T, path string) { + t.Helper() + data, err := json.Marshal(validImageSpec()) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } +} + +func TestResolveImageConventionalLayouts(t *testing.T) { + cases := []struct { + name string + layout func(t *testing.T, dir string) + }{ + { + name: "directory with spec.json", + layout: func(t *testing.T, dir string) { + writeSpecFile(t, filepath.Join(dir, "nixos", "spec.json")) + }, + }, + { + name: "flat .json", + layout: func(t *testing.T, dir string) { + writeSpecFile(t, filepath.Join(dir, "nixos.json")) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + tc.layout(t, dir) + + e := testEngine(t, dir) + spec, path, name, err := e.resolveImage("nixos") + if err != nil { + t.Fatalf("resolveImage: %v", err) + } + if name != "nixos" { + t.Fatalf("name = %q, want nixos", name) + } + if !strings.HasPrefix(path, dir) { + t.Fatalf("resolved path %q not under image dir %q", path, dir) + } + if spec.Shell == "" { + t.Fatal("resolved spec not loaded") + } + }) + } +} + +func TestResolveImageDirectoryMissingSpec(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "nixos"), 0o755); err != nil { + t.Fatal(err) + } + + e := testEngine(t, dir) + _, _, _, err := e.resolveImage("nixos") + if err == nil || !strings.Contains(err.Error(), imageSpecFileName) { + t.Fatalf("directory without %s should error, got: %v", imageSpecFileName, err) + } +} + +func TestResolveImageRejectsPaths(t *testing.T) { + e := testEngine(t, t.TempDir()) + for _, name := range []string{"/etc/passwd", "../evil", "sub/evil", "..", "."} { + if _, _, _, err := e.resolveImage(name); err == nil || !strings.Contains(err.Error(), "must be a plain name") { + t.Fatalf("name %q should be rejected as a path, got: %v", name, err) + } + } +} + +func validImageSpec() ImageSpec { + return ImageSpec{ + Arch: "x86_64", + BootArgs: "console=ttyS0", + Initrd: "initrd", + Kernel: "kernel", + RunnerConfig: RunnerConfig{ + Machine: "microvm", + }, + MemoryMiB: 2048, + Shell: "/bin/sh", + StoreDisk: "store-disk", + VCPUs: 2, + } +} + +func TestImageSpecValidateWithoutBaseConfigHash(t *testing.T) { + spec := validImageSpec() + if err := spec.Validate(); err != nil { + t.Fatalf("non-NixOS image spec should validate: %v", err) + } + if spec.SupportsConfigActivation() { + t.Fatal("spec without baseConfigHash should not support config activation") + } + + spec.BaseConfigHash = "abcdef" + if !spec.SupportsConfigActivation() { + t.Fatal("spec with baseConfigHash should support config activation") + } +} + +func TestImageSpecRequiresShell(t *testing.T) { + spec := validImageSpec() + spec.Shell = "" + err := spec.Validate() + if err == nil || !strings.Contains(err.Error(), "shell") { + t.Fatalf("spec without shell should fail validation, got: %v", err) + } +} diff --git a/spindle/engines/microvm/models.go b/spindle/engines/microvm/models.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/models.go @@ -0,0 +1,48 @@ +package microvm + +import ( + "fmt" + "slices" +) + +type manifestWorkflow struct { + Image string `yaml:"image"` + Services map[string]any `yaml:"services"` + Virtualisation map[string]any `yaml:"virtualisation"` + Dependencies []string `yaml:"dependencies"` + Registry map[string]any `yaml:"registry"` + Environment map[string]string `yaml:"environment"` + Caches map[string]string `yaml:"caches"` + Steps []struct { + Name string `yaml:"name"` + Command string `yaml:"command"` + Environment map[string]string `yaml:"environment"` + } `yaml:"steps"` +} + +// flattens the caches map into sorted substituter URLs and trusted public keys +func workflowCaches(caches map[string]string) (urls []string, keys []string, err error) { + for cacheURL, key := range caches { + urls = append(urls, cacheURL) + if key != "" { + keys = append(keys, key) + } + } + if _, err := parseCacheUpstreams(urls); err != nil { + return nil, nil, fmt.Errorf("caches: %w", err) + } + slices.Sort(urls) + slices.Sort(keys) + return urls, keys, nil +} + +type manifestConfig struct { + Services map[string]any `yaml:"services" json:"services,omitempty"` + Virtualisation map[string]any `yaml:"virtualisation" json:"virtualisation,omitempty"` + Dependencies []string `yaml:"dependencies" json:"dependencies,omitempty"` + Registry map[string]any `yaml:"registry" json:"registry,omitempty"` +} + +func (c manifestConfig) Enabled() bool { + return len(c.Services) > 0 || len(c.Virtualisation) > 0 || len(c.Dependencies) > 0 || len(c.Registry) > 0 +} diff --git a/spindle/engines/microvm/models_test.go b/spindle/engines/microvm/models_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/models_test.go @@ -0,0 +1,41 @@ +package microvm + +import ( + "slices" + "testing" +) + +func TestWorkflowCaches(t *testing.T) { + urls, keys, err := workflowCaches(map[string]string{ + "https://hydra.nixos.org/": "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=", + "https://cache.garnix.io/": "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=", + "https://unsigned.example/": "", + }) + if err != nil { + t.Fatal(err) + } + + wantURLs := []string{ + "https://cache.garnix.io/", + "https://hydra.nixos.org/", + "https://unsigned.example/", + } + if !slices.Equal(urls, wantURLs) { + t.Fatalf("urls: got %v, want %v", urls, wantURLs) + } + wantKeys := []string{ + "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=", + "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=", + } + if !slices.Equal(keys, wantKeys) { + t.Fatalf("keys: got %v, want %v", keys, wantKeys) + } +} + +func TestWorkflowCachesRejectsBadURLs(t *testing.T) { + for _, bad := range []string{"ftp://cache.example/", "not a url"} { + if _, _, err := workflowCaches(map[string]string{bad: ""}); err == nil { + t.Errorf("workflowCaches(%q): expected error, got nil", bad) + } + } +} diff --git a/spindle/engines/microvm/netns_wrapper.sh.tmpl b/spindle/engines/microvm/netns_wrapper.sh.tmpl new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/netns_wrapper.sh.tmpl @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +pid_file="$1"; shift +ip="$1"; shift +mount="$1"; shift +resolv_conf="$1"; shift + +printf '%s\n' "$$" > "$pid_file" + +while ! "$ip" link show {{.TapName}} >/dev/null 2>&1; do + sleep 0.02 +done + +"$mount" --bind "$resolv_conf" /etc/resolv.conf + +# fail if we can't add a route, ipv6 is skipped if not enabled on host +for route in{{range .BlockedRoutes}} {{.}}{{end}}; do + case "$route" in + *:*) [ -d /proc/sys/net/ipv6 ] || continue ;; + esac + "$ip" route add blackhole "$route" +done + +exec "$@" diff --git a/spindle/engines/microvm/networking.go b/spindle/engines/microvm/networking.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/networking.go @@ -0,0 +1,152 @@ +package microvm + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "log/slog" + "net" + "os" + "os/exec" + "text/template" +) + +// https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +// https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +// https://datatracker.ietf.org/doc/rfc6890/ +var blockedNamespaceRoutes = []string{ + "0.0.0.0/8", // unspecified / "this network" addresses + "10.0.0.0/8", // private network + "100.64.0.0/10", // shared carrier-grade nat space + "127.0.0.0/8", // loopback + "169.254.0.0/16", // link-local / autoconfiguration + "172.16.0.0/12", // private network + "192.0.0.0/24", // ietf protocol assignments + "192.0.2.0/24", // documentation / examples + "192.88.99.0/24", // deprecated 6to4 relay anycast + "192.168.0.0/16", // private network + "198.18.0.0/15", // benchmarking / testing + "198.51.100.0/24", // documentation / examples + "203.0.113.0/24", // documentation / examples + "224.0.0.0/4", // multicast + "240.0.0.0/4", // reserved / future use, includes limited broadcast + "::/128", // unspecified address + "::1/128", // loopback + "::ffff:0:0/96", // ipv4-mapped addresses + "64:ff9b::/96", // ipv4/ipv6 translation prefix + "100::/64", // discard-only prefix + "2001::/23", // ietf protocol assignments + "2001:db8::/32", // documentation / examples + "2002::/16", // deprecated 6to4 addressing + "fc00::/7", // unique local addresses + "fe80::/10", // link-local unicast + "ff00::/8", // multicast +} + +var blockedNamespaceNets = func() []*net.IPNet { + nets := make([]*net.IPNet, 0, len(blockedNamespaceRoutes)) + for _, route := range blockedNamespaceRoutes { + _, ipnet, err := net.ParseCIDR(route) + if err != nil { + panic(fmt.Sprintf("parse blocked route %q: %v", route, err)) + } + nets = append(nets, ipnet) + } + return nets +}() + +//go:embed netns_wrapper.sh.tmpl +var netnsWrapperTemplate string + +type netnsWrapperData struct { + TapName string + BlockedRoutes []string +} + +func writeNetnsWrapper(path string, dev bool) error { + tmpl, err := template.New("netns-wrapper").Parse(netnsWrapperTemplate) + if err != nil { + return fmt.Errorf("parse qemu network namespace wrapper template: %w", err) + } + + var script bytes.Buffer + + var routes []string + if !dev { + routes = blockedNamespaceRoutes + } + + err = tmpl.Execute(&script, netnsWrapperData{ + TapName: netnsTapName, + BlockedRoutes: routes, + }) + if err != nil { + return fmt.Errorf("render qemu network namespace wrapper template: %w", err) + } + + if err := os.WriteFile(path, script.Bytes(), 0o700); err != nil { + return fmt.Errorf("write qemu network namespace wrapper: %w", err) + } + + return nil +} + +type slirpNamespace struct { + spec ImageSpec + pidFile string + dev bool +} + +func (n *slirpNamespace) Start(ctx context.Context, logFile *os.File, logger *slog.Logger) (*exec.Cmd, *os.File, error) { + pid, err := waitForPIDFile(ctx, n.pidFile) + if err != nil { + return nil, nil, err + } + + exitR, exitW, err := os.Pipe() + if err != nil { + return nil, nil, fmt.Errorf("create slirp4netns exit pipe: %w", err) + } + defer exitR.Close() // always close our read end; child gets it via ExtraFiles dup + + var ok bool + defer func() { + if !ok { + _ = exitW.Close() + } + }() + + slirpPath, err := exec.LookPath("slirp4netns") + if err != nil { + return nil, nil, fmt.Errorf("slirp4netns command not found in PATH: %w", err) + } + + args := []string{ + "--configure", + "--mtu=" + netnsMTU, + } + if !n.dev { + args = append(args, "--disable-host-loopback") + } + args = append(args, + "--enable-sandbox", + "--enable-seccomp", + "--exit-fd=3", + "--cidr="+outerSlirpCIDR, + pid, + netnsTapName, + ) + + cmd := exec.CommandContext(ctx, slirpPath, args...) + cmd.ExtraFiles = []*os.File{exitR} + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Start(); err != nil { + return nil, nil, fmt.Errorf("start slirp4netns: %w", err) + } + logger.Info("started slirp4netns network namespace", "pid", pid, "cidr", outerSlirpCIDR, "tap", netnsTapName) + + ok = true + return cmd, exitW, nil +} diff --git a/spindle/engines/microvm/nixos_toplevel_cache.go b/spindle/engines/microvm/nixos_toplevel_cache.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/nixos_toplevel_cache.go @@ -0,0 +1,100 @@ +package microvm + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "tangled.org/core/spindle/db" +) + +const nixosToplevelCacheSchemaVersion = 1 + +type nixosToplevelCacheRecord struct { + ConfigKey string `json:"config_key"` + Toplevel string `json:"toplevel"` + UpdatedAt time.Time `json:"updated_at"` +} + +type nixosToplevelCacheStore struct { + db *db.DB +} + +func newNixOSToplevelCacheStore(d *db.DB) nixosToplevelCacheStore { + return nixosToplevelCacheStore{db: d} +} + +func (s nixosToplevelCacheStore) Lookup(configKey string) (nixosToplevelCacheRecord, bool, error) { + if s.db == nil { + return nixosToplevelCacheRecord{}, false, nil + } + r, err := s.db.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nixosToplevelCacheRecord{}, false, nil + } + return nixosToplevelCacheRecord{}, false, err + } + return nixosToplevelCacheRecord{ + ConfigKey: r.ConfigKey, + Toplevel: r.Toplevel, + UpdatedAt: r.UpdatedAt, + }, true, nil +} + +func (s nixosToplevelCacheStore) Commit(configKey, toplevel string) error { + if configKey == "" { + return fmt.Errorf("config key is empty") + } + if toplevel == "" { + return fmt.Errorf("config toplevel is empty") + } + if s.db == nil { + return nil + } + return s.db.SaveNixOSToplevelCacheRecord(configKey, toplevel) +} + +func BaseConfigHash(imageSpec ImageSpec) (string, error) { + if imageSpec.BaseConfigHash == "" { + return "", fmt.Errorf("microvm image spec missing baseConfigHash") + } + return imageSpec.BaseConfigHash, nil +} + +func userConfigHash(cfg manifestConfig) string { + data, _ := json.Marshal(cfg) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func buildConfigKey(imageSpec ImageSpec, cfg manifestConfig) (string, error) { + baseHash, err := BaseConfigHash(imageSpec) + if err != nil { + return "", err + } + payload := struct { + Schema int `json:"schema"` + BaseConfig string `json:"base_config"` + UserConfig string `json:"user_config"` + }{ + Schema: nixosToplevelCacheSchemaVersion, + BaseConfig: baseHash, + UserConfig: userConfigHash(cfg), + } + data, _ := json.Marshal(payload) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func BuildConfigKey(imageSpec ImageSpec, userConfigJSON string) (string, error) { + var cfg manifestConfig + if err := json.Unmarshal([]byte(userConfigJSON), &cfg); err != nil { + return "", err + } + return buildConfigKey(imageSpec, cfg) +} diff --git a/spindle/engines/microvm/qemu.go b/spindle/engines/microvm/qemu.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/qemu.go @@ -0,0 +1,676 @@ +package microvm + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/digitalocean/go-qemu/qmp" + "github.com/google/uuid" +) + +const ( + defaultQMPTimeout = 10 * time.Second + outerSlirpCIDR = "10.0.2.0/24" + innerSlirpNet = "10.0.3.0/24" + innerSlirpHost = "10.0.3.2" + innerSlirpDNS = "10.0.3.3" + innerSlirpDHCP = "10.0.3.15" + netnsTapName = "tap0" + netnsMTU = "65520" +) + +type QEMUConfig struct { + Image ImageSpec + BootTimeout time.Duration + CID uint32 + EnableKVM bool + QEMULogPath string + QMPPath string + SerialLogPath string + WorkDir string + VolumePaths map[string]string + VolumeBaseName string + Cgroup CgroupLimits + Dev bool +} + +type QEMUVMHandle struct { + cid uint32 + Process *os.Process + qemuLogPath string + QMPMon *qmp.SocketMonitor + QMPPath string + serialLogPath string + workDir string + + cmd *exec.Cmd + done chan struct{} + qemuLogFile *os.File + cgroup *CgroupHandle + slirpCmd *exec.Cmd + slirpExit *os.File + waitErr error + waitErrMu sync.Mutex +} + +type qemuRunner struct{} + +func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error { + if _, err := exec.LookPath(spec.RunnerCmd()); err != nil { + return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err) + } + if _, err := os.Stat("/dev/vhost-vsock"); err != nil { + return fmt.Errorf("microvm requires /dev/vhost-vsock for vhost-vsock-device: %w", err) + } + if enableKVM { + if _, err := os.Stat("/dev/kvm"); err != nil { + return fmt.Errorf("microvm KVM was requested but /dev/kvm is not accessible: %w", err) + } + } + if len(spec.NetworkInterfaces) > 0 { + if _, err := os.Stat("/dev/net/tun"); err != nil { + return fmt.Errorf("microvm slirp4netns networking requires /dev/net/tun: %w", err) + } + for _, cmd := range []string{"ip", "mount", "slirp4netns", "unshare"} { + if _, err := exec.LookPath(cmd); err != nil { + return fmt.Errorf("required host command %q not found in PATH: %w", cmd, err) + } + } + } + return nil +} + +func (qemuRunner) Start(ctx context.Context, cfg VMConfig, volumePaths map[string]string, logger *slog.Logger) (VMHandle, error) { + bootTimeout := cfg.BootTimeout + if bootTimeout == 0 { + bootTimeout = 10 * time.Second + } + return StartQEMU(ctx, QEMUConfig{ + Image: cfg.Image, + BootTimeout: bootTimeout, + CID: cfg.CID, + EnableKVM: cfg.EnableKVM, + WorkDir: cfg.WorkDir, + VolumePaths: volumePaths, + Cgroup: cfg.Cgroup, + Dev: cfg.Dev, + }, logger) +} + +func StartQEMU(ctx context.Context, cfg QEMUConfig, logger *slog.Logger) (VMHandle, error) { + if logger == nil { + logger = slog.Default() + } + + workDir := cfg.WorkDir + + handle := &QEMUVMHandle{ + workDir: workDir, + } + + var ok bool + defer func() { + if !ok { + _ = handle.Close() + } + }() + + cid := cfg.CID + if cid == 0 { + var err error + cid, err = AllocateCID() + if err != nil { + return nil, err + } + } + if cid < minGuestCID { + return nil, fmt.Errorf("guest CID must be >= %d", minGuestCID) + } + handle.cid = cid + + volumePaths := cfg.VolumePaths + + qemuLogPath := cfg.QEMULogPath + if qemuLogPath == "" { + qemuLogPath = filepath.Join(workDir, "qemu.log") + } + qemuLogFile, err := createParentedFile(qemuLogPath) + if err != nil { + return nil, err + } + handle.qemuLogPath = qemuLogPath + handle.qemuLogFile = qemuLogFile + + serialLogPath := cfg.SerialLogPath + if serialLogPath == "" { + serialLogPath = filepath.Join(workDir, "serial.log") + } + if err := os.MkdirAll(filepath.Dir(serialLogPath), 0o755); err != nil { + return nil, fmt.Errorf("create serial log directory: %w", err) + } + handle.serialLogPath = serialLogPath + + qmpPath := cfg.QMPPath + if qmpPath == "" { + qmpPath = filepath.Join(workDir, "qmp.sock") + } + handle.QMPPath = qmpPath + + qemuCmd := cfg.Image.RunnerCmd() + qemuBinary, err := exec.LookPath(qemuCmd) + if err != nil { + return nil, fmt.Errorf("%s command not found in PATH: %w", qemuCmd, err) + } + + args, err := qemuArgs(qemuArgsConfig{ + Image: cfg.Image, + CID: cid, + EnableKVM: cfg.EnableKVM, + QMPPath: qmpPath, + SerialLogPath: serialLogPath, + VolumePaths: volumePaths, + }) + if err != nil { + return nil, err + } + + cmd, slirpNet, err := qemuCommand(ctx, qemuBinary, args, cfg.Image, workDir, cfg.Dev) + if err != nil { + return nil, err + } + cmd.Env = append(os.Environ(), "TMPDIR="+workDir) + cmd.Stdout = qemuLogFile + cmd.Stderr = qemuLogFile + + cgroup, err := prepareCgroup(cfg.Cgroup, logger) + if err != nil { + return nil, err + } + handle.cgroup = cgroup + + logger.Info("starting qemu microvm", "cid", cid, "workDir", workDir, "serialLog", serialLogPath, "qmp", qmpPath) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("starting qemu: %w", err) + } + handle.cmd = cmd + handle.Process = cmd.Process + handle.done = make(chan struct{}) + go func() { + err := cmd.Wait() + handle.waitErrMu.Lock() + handle.waitErr = err + handle.waitErrMu.Unlock() + close(handle.done) + }() + + if err := cgroup.AddProcess(cmd.Process.Pid, logger); err != nil { + return nil, err + } + + if slirpNet != nil { + handle.slirpCmd, handle.slirpExit, err = slirpNet.Start(ctx, qemuLogFile, logger) + if err != nil { + return nil, err + } + if handle.slirpCmd != nil && handle.slirpCmd.Process != nil { + if err := cgroup.AddProcess(handle.slirpCmd.Process.Pid, logger); err != nil { + return nil, err + } + } + } + + qmpTimeout := cfg.BootTimeout + if qmpTimeout == 0 { + qmpTimeout = defaultQMPTimeout + } + if err := handle.waitForQMP(ctx, qmpTimeout); err != nil { + return nil, err + } + + status, err := handle.QMPQueryStatus() + if err != nil { + return nil, err + } + if status != "running" { + return nil, fmt.Errorf("qemu guest not running (status: %s)", status) + } + logger.Info("qemu microvm running", "cid", cid, "status", status) + + ok = true + return handle, nil +} + +func (h *QEMUVMHandle) Wait() error { + if h == nil || h.done == nil { + return nil + } + <-h.done + h.waitErrMu.Lock() + defer h.waitErrMu.Unlock() + return h.waitErr +} + +func (h *QEMUVMHandle) WaitContext(ctx context.Context) error { + if h == nil || h.done == nil { + return nil + } + select { + case <-h.done: + h.waitErrMu.Lock() + defer h.waitErrMu.Unlock() + return h.waitErr + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *QEMUVMHandle) Kill() error { + if h == nil || h.Process == nil { + return nil + } + return h.Process.Kill() +} + +func (h *QEMUVMHandle) Shutdown(ctx context.Context) error { + if h == nil { + return nil + } + if h.QMPMon != nil { + if err := h.QMPSystemPowerdown(); err != nil { + return err + } + } + if h.done == nil { + return nil + } + select { + case <-h.done: + return h.Wait() + case <-ctx.Done(): + _ = h.Kill() + _ = h.Wait() + return ctx.Err() + } +} + +func (h *QEMUVMHandle) Close() error { + if h == nil { + return nil + } + + var closeErr error + if h.QMPMon != nil { + closeErr = errors.Join(closeErr, h.QMPMon.Disconnect()) + h.QMPMon = nil + } + if h.Process != nil { + _ = h.Process.Kill() + _ = h.Wait() + } + if h.slirpExit != nil { + _ = h.slirpExit.Close() + h.slirpExit = nil + } + if h.slirpCmd != nil && h.slirpCmd.Process != nil { + _ = h.slirpCmd.Process.Kill() + _ = h.slirpCmd.Wait() + h.slirpCmd = nil + } + if h.qemuLogFile != nil { + closeErr = errors.Join(closeErr, h.qemuLogFile.Close()) + h.qemuLogFile = nil + } + if h.cgroup != nil { + closeErr = errors.Join(closeErr, h.cgroup.Close()) + h.cgroup = nil + } + return closeErr +} + +func (h *QEMUVMHandle) QMPRun(command qmp.Command) ([]byte, error) { + if h == nil || h.QMPMon == nil { + return nil, fmt.Errorf("qmp monitor is not connected") + } + data, err := json.Marshal(command) + if err != nil { + return nil, err + } + return h.QMPMon.Run(data) +} + +func (h *QEMUVMHandle) QMPQueryStatus() (string, error) { + raw, err := h.QMPRun(qmp.Command{Execute: "query-status"}) + if err != nil { + return "", fmt.Errorf("qmp query-status failed: %w", err) + } + + var resp struct { + Return struct { + Status string `json:"status"` + } `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return "", fmt.Errorf("qmp query-status parse: %w", err) + } + return resp.Return.Status, nil +} + +func (h *QEMUVMHandle) QMPSystemPowerdown() error { + _, err := h.QMPRun(qmp.Command{Execute: "system_powerdown"}) + return err +} + +func (h *QEMUVMHandle) Logs() VMLogs { + if h == nil { + return VMLogs{} + } + return VMLogs{ + Serial: h.serialLogPath, + Extra: map[string]string{ + "qemu": h.qemuLogPath, + }, + } +} + +func (h *QEMUVMHandle) CID() uint32 { + if h == nil { + return 0 + } + return h.cid +} + +func (h *QEMUVMHandle) WorkDir() string { + if h == nil { + return "" + } + return h.workDir +} + +func (h *QEMUVMHandle) OOMKilled() bool { + if h == nil { + return false + } + return h.cgroup.OOMKilled() +} + +func (h *QEMUVMHandle) waitForQMP(ctx context.Context, timeout time.Duration) error { + qmpCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var lastErr error + for { + mon, err := qmp.NewSocketMonitor("unix", h.QMPPath, 2*time.Second) + if err == nil { + if err = mon.Connect(); err == nil { + h.QMPMon = mon + return nil + } + _ = mon.Disconnect() + } + lastErr = err + + select { + case <-qmpCtx.Done(): + return fmt.Errorf("qmp connect timeout: %w", lastErr) + case <-h.done: + return fmt.Errorf("qemu exited before qmp was ready: %w", h.Wait()) + case <-time.After(25 * time.Millisecond): + } + } +} + +func qemuCommand( + ctx context.Context, + qemuBinary string, + args []string, + spec ImageSpec, + workDir string, + dev bool, +) (*exec.Cmd, *slirpNamespace, error) { + if len(spec.NetworkInterfaces) == 0 { + return exec.CommandContext(ctx, qemuBinary, args...), nil, nil + } + + ipPath, err := exec.LookPath("ip") + if err != nil { + return nil, nil, fmt.Errorf("ip command not found in PATH: %w", err) + } + mountPath, err := exec.LookPath("mount") + if err != nil { + return nil, nil, fmt.Errorf("mount command not found in PATH: %w", err) + } + unsharePath, err := exec.LookPath("unshare") + if err != nil { + return nil, nil, fmt.Errorf("unshare command not found in PATH: %w", err) + } + + pidFile, resolvPath, wrapperPath, err := prepareQEMUNetnsFiles(workDir, dev) + if err != nil { + return nil, nil, err + } + + cmdArgs := append([]string{ + "--user", + "--map-root-user", + "--net", + "--mount", + "--propagation", "private", + "--", + wrapperPath, + pidFile, + ipPath, + mountPath, + resolvPath, + qemuBinary, + }, args...) + + cmd := exec.CommandContext(ctx, unsharePath, cmdArgs...) + + return cmd, &slirpNamespace{ + spec: spec, + pidFile: pidFile, + dev: dev, + }, nil +} + +func prepareQEMUNetnsFiles(workDir string, dev bool) (pidFile, resolvPath, wrapperPath string, err error) { + pidFile = filepath.Join(workDir, "qemu-netns.pid") + resolvPath = filepath.Join(workDir, "qemu-netns-resolv.conf") + wrapperPath = filepath.Join(workDir, "qemu-netns-wrapper") + + // the guest resolves through shuttle on 127.0.0.1. keep qemu's slirp DNS + // pointed at an unroutable local resolver inside this network namespace so + // direct guest queries to 10.0.3.3 don't bypass the shuttle dns policy. + if err := os.WriteFile(resolvPath, []byte("nameserver 127.0.0.1\n"), 0o644); err != nil { + return "", "", "", fmt.Errorf("write qemu network namespace resolv.conf: %w", err) + } + + if err := writeNetnsWrapper(wrapperPath, dev); err != nil { + return "", "", "", fmt.Errorf("write qemu network namespace wrapper: %w", err) + } + + return pidFile, resolvPath, wrapperPath, nil +} + +type qemuArgsConfig struct { + Image ImageSpec + CID uint32 + EnableKVM bool + QMPPath string + SerialLogPath string + VolumePaths map[string]string +} + +func qemuArgs(cfg qemuArgsConfig) ([]string, error) { + uuid := uuid.New() + + b := newArgBuilder(64) + + addQEMUMachineArgs(&b, cfg, uuid) + addQEMUStoreArgs(&b, cfg) + + if cfg.EnableKVM { + addQEMUKVMArgs(&b, cfg.Image) + } + + if err := addQEMUVolumeArgs(&b, cfg); err != nil { + return nil, err + } + + if err := addQEMUNetworkArgs(&b, cfg.Image.NetworkInterfaces); err != nil { + return nil, err + } + + b.Optf("-device", "vhost-vsock-device,guest-cid=%d", cfg.CID) + + if len(cfg.Image.RunnerConfig.ExtraArgs) > 0 { + b.Add(cfg.Image.RunnerConfig.ExtraArgs...) + } + + return b.Args(), nil +} + +func addQEMUMachineArgs(b *argBuilder, cfg qemuArgsConfig, uuid uuid.UUID) { + if cfg.Image.RunnerConfig.Machine != "" { + b.Opt("-M", cfg.Image.RunnerConfig.Machine) + } + b.Optf("-m", "%dM", cfg.Image.MemoryMiB) + b.Opt("-smp", strconv.Itoa(cfg.Image.VCPUs)) + + b.Add( + "-nodefaults", + "-no-user-config", + "-no-reboot", + ) + + b.Opt("-kernel", cfg.Image.Kernel) + b.Opt("-initrd", cfg.Image.Initrd) + + b.Opt("-device", "virtio-rng-device") + + b.Optf("-smbios", "type=1,uuid=%s", uuid) + b.Opt("-serial", "file:"+cfg.SerialLogPath) + + // use virtio console if requsted. this is faster than the serial UART logging + // because serial has a higher cost when being accesssed. we still have to + // support serial itself for early kernel boot but thats OK. + if cfg.Image.RunnerConfig.Console == "hvc0" { + b.Optf("-chardev", "file,id=virtiocon0,path=%s,append=on", cfg.SerialLogPath) + b.Add("-device", "virtio-serial-device") + b.Opt("-device", "virtconsole,chardev=virtiocon0") + } + b.Opt("-display", "none") + b.Opt("-monitor", "none") + b.Opt("-append", cfg.Image.BootArgs) + + b.Opt("-sandbox", "on") + b.Optf("-qmp", "unix:%s,server,nowait", cfg.QMPPath) +} + +func addQEMUStoreArgs(b *argBuilder, cfg qemuArgsConfig) { + drive := newOptionBuilder(8) + drive.KV("id", "store") + drive.KV("format", "raw") + drive.Add("read-only=on") + drive.KV("file", cfg.Image.StoreDisk) + drive.Add("if=none") + drive.Add("aio=io_uring") + + b.Opt("-drive", drive.String()) + b.Opt("-device", "virtio-blk-device,drive=store") +} + +func addQEMUKVMArgs(b *argBuilder, image ImageSpec) { + b.Flag("-enable-kvm") + if image.RunnerConfig.CPU != "" { + b.Opt("-cpu", image.RunnerConfig.CPU) + } + b.Opt("-device", "i8042") +} + +func addQEMUVolumeArgs(b *argBuilder, cfg qemuArgsConfig) error { + for index, volume := range cfg.Image.Volumes { + path := cfg.VolumePaths[volume.Image] + if path == "" { + return fmt.Errorf("missing prepared path for volume %q", volume.Image) + } + + driveID := fmt.Sprintf("volume%d", index) + + drive := newOptionBuilder(10) + drive.KV("id", driveID) + drive.KV("format", "raw") + drive.Add("read-only=off") + drive.KV("file", path) + drive.Add("if=none") + drive.Add("aio=io_uring") + drive.Add("discard=unmap") + drive.Add("cache=none") + + b.Opt("-drive", drive.String()) + b.Optf("-device", "virtio-blk-device,drive=%s", driveID) + } + + return nil +} + +func addQEMUNetworkArgs(b *argBuilder, interfaces []NetworkInterface) error { + for _, networkInterface := range interfaces { + if networkInterface.Type != "slirp4netns" { + return fmt.Errorf("unsupported microvm network interface type %q", networkInterface.Type) + } + + netdevOpts := newOptionBuilder(6) + netdevOpts.Add("user") + netdevOpts.KV("id", networkInterface.ID) + netdevOpts.KV("net", innerSlirpNet) + netdevOpts.KV("host", innerSlirpHost) + netdevOpts.KV("dns", innerSlirpDNS) + netdevOpts.KV("dhcpstart", innerSlirpDHCP) + + b.Opt("-netdev", netdevOpts.String()) + b.Optf( + "-device", "virtio-net-device,netdev=%s,mac=%s", + networkInterface.ID, networkInterface.MAC, + ) + } + + return nil +} + +func waitForPIDFile(ctx context.Context, path string) (string, error) { + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + + for { + data, err := os.ReadFile(path) + if err == nil { + pid := strings.TrimSpace(string(data)) + if pid != "" { + return pid, nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("read qemu network namespace pid: %w", err) + } + + select { + case <-waitCtx.Done(): + return "", fmt.Errorf("waiting for qemu network namespace pid: %w", waitCtx.Err()) + case <-ticker.C: + } + } +} diff --git a/spindle/engines/microvm/read_cache_proxy.go b/spindle/engines/microvm/read_cache_proxy.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/read_cache_proxy.go @@ -0,0 +1,427 @@ +package microvm + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" + "syscall" + "time" + + "github.com/mdlayher/vsock" +) + +const ( + readCacheProxyPortMin = 20000 + readCacheProxyPortMax = 60000 +) + +type ReadCacheProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server +} + +func StartReadCacheProxy(ctx context.Context, cid uint32, upstreams []CacheUpstream, logger *slog.Logger) (*ReadCacheProxy, error) { + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "read_cache", "cid", cid) + + if len(upstreams) == 0 { + return nil, nil + } + + ln, port, err := listenRandomVsockPort(ctx) + if err != nil { + return nil, err + } + + proxy := &ReadCacheProxy{ + port: port, + ln: ln, + } + proxy.server = &http.Server{ + Handler: cacheProxyHandler(upstreams, logger), + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 10 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("proxy stopped", "cid", cid, "port", port, "error", err) + } + }() + + logger.Info("started proxy", "cid", cid, "port", port, "upstreams", len(upstreams)) + return proxy, nil +} + +func (p *ReadCacheProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *ReadCacheProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + return closeErr +} + +type cidFilteredVsockListener struct { + *vsock.Listener + cid uint32 + logger *slog.Logger +} + +func (l *cidFilteredVsockListener) Accept() (net.Conn, error) { + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == l.cid { + return conn, nil + } + + l.logger.Warn("dropping proxy connection from unexpected cid", "remote", conn.RemoteAddr(), "expectedCID", l.cid) + _ = conn.Close() + } +} + +func parseCacheUpstreams(raw []string) ([]*url.URL, error) { + upstreams := make([]*url.URL, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, value := range raw { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + + parsed, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("parse URL %q: %w", value, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("URL %q uses unsupported scheme %q", value, parsed.Scheme) + } + if parsed.Host == "" { + return nil, fmt.Errorf("URL %q is missing host", value) + } + upstreams = append(upstreams, parsed) + } + return upstreams, nil +} + +type CacheUpstream struct { + url *url.URL + // guarded upstreams come from the workflow file + // requests to them are refused for special-purpose address ranges + guarded bool +} + +func BuildCacheUpstreams(rawTrusted, rawGuarded []string) ([]CacheUpstream, error) { + trusted, err := parseCacheUpstreams(rawTrusted) + if err != nil { + return nil, err + } + guarded, err := parseCacheUpstreams(rawGuarded) + if err != nil { + return nil, err + } + return mergeCacheUpstreams(trusted, guarded), nil +} + +func mergeCacheUpstreams(trusted, guarded []*url.URL) []CacheUpstream { + merged := make([]CacheUpstream, 0, len(trusted)+len(guarded)) + seen := make(map[string]struct{}, len(trusted)+len(guarded)) + for _, u := range trusted { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, CacheUpstream{url: u}) + } + for _, u := range guarded { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, CacheUpstream{url: u, guarded: true}) + } + return merged +} + +func listenRandomVsockPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.Listen(port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock port: %w", lastErr) +} + +func randomVsockPort() (uint32, error) { + var data [4]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, fmt.Errorf("allocate read vsock port: %w", err) + } + span := uint32(readCacheProxyPortMax - readCacheProxyPortMin) + return readCacheProxyPortMin + binary.BigEndian.Uint32(data[:])%span, nil +} + +var proxyTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// for guarded upstreams, this will refuse requests made to blocked addresses +var guardedProxyTransport = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: refuseSpecialPurposeAddrs, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// this should run after dns resolution, so it should cover any rebinding tricks +func refuseSpecialPurposeAddrs(network, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("split dial address %q: %w", address, err) + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("refusing to dial non-IP address %q", host) + } + for _, ipnet := range blockedNamespaceNets { + if ipnet.Contains(ip) { + return fmt.Errorf("refusing to dial %s: %s is blocked for workflow caches", ip, ipnet) + } + } + return nil +} + +// the proxy is the cache as far as the guest is concerned, so we answer +// /nix-cache-info ourselves instead of racing the upstreams for it. merging +// those also doesn't make any sense (none of the options make sense for +// merging) +const nixCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n" + +func cacheProxyHandler(upstreams []CacheUpstream, logger *slog.Logger) http.Handler { + proxy := &httputil.ReverseProxy{ + // nothing to do here: the racing transport builds the full URL per + // upstream, it just needs the guest's path/query left intact + Rewrite: func(*httputil.ProxyRequest) {}, + ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), + Transport: ¶llelRacingTransport{ + upstreams: upstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + }, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/nix-cache-info" { + w.Header().Set("Content-Type", "text/x-nix-cache-info") + _, _ = io.WriteString(w, nixCacheInfo) + return + } + proxy.ServeHTTP(w, r) + }) +} + +func cacheProxyProtocols() *http.Protocols { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetUnencryptedHTTP2(true) + return protocols +} + +func mergeQuery(base, extra string) string { + switch { + case base == "": + return extra + case extra == "": + return base + default: + return base + "&" + extra + } +} + +type parallelRacingTransport struct { + upstreams []CacheUpstream + underlying http.RoundTripper + guardedUnderlying http.RoundTripper + logger *slog.Logger +} + +func (t *parallelRacingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + type result struct { + resp *http.Response + err error + is404 bool + idx int + } + + resCh := make(chan result, len(t.upstreams)) + cancels := make([]context.CancelFunc, len(t.upstreams)) + var wg sync.WaitGroup + + for i, upstream := range t.upstreams { + wg.Add(1) + ctx, cancel := context.WithCancel(req.Context()) + cancels[i] = cancel + + go func(idx int, target CacheUpstream, uCtx context.Context) { + defer wg.Done() + + raceReq := req.Clone(uCtx) + // rewrite to the target, joining the upstream's base path/query + // with what the guest asked for + raceReq.URL.Scheme = target.url.Scheme + raceReq.URL.Host = target.url.Host + raceReq.URL.Path = strings.TrimSuffix(target.url.Path, "/") + req.URL.Path + raceReq.URL.RawQuery = mergeQuery(target.url.RawQuery, req.URL.RawQuery) + // Host wins over URL.Host for the outgoing Host header, and the + // reverse proxy preserves the guest's (127.0.0.1:), which + // host-routed upstreams like fastly reject with a 421 + raceReq.Host = target.url.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.url.User; user != nil { + password, _ := user.Password() + raceReq.SetBasicAuth(user.Username(), password) + } + + rt := t.underlying + if target.guarded { + rt = t.guardedUnderlying + } + resp, err := rt.RoundTrip(raceReq) + if err != nil { + resCh <- result{err: err, idx: idx} + return + } + if resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() // don't care about the body of a 404 + resCh <- result{is404: true, idx: idx} + return + } + if resp.StatusCode >= 400 { + // an erroring upstream must not win over a healthy one + _ = resp.Body.Close() + resCh <- result{err: fmt.Errorf("upstream returned status %d", resp.StatusCode), idx: idx} + return + } + // yay, ok + resCh <- result{resp: resp, idx: idx} + }(i, upstream, ctx) + } + + go func() { + wg.Wait() + close(resCh) + }() + + var total404s int + for res := range resCh { + if res.is404 { + total404s++ + if total404s == len(t.upstreams) { + for _, cancel := range cancels { + cancel() + } + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader("404 nix path not found")), + Header: make(http.Header), + Request: req, + }, nil + } + continue + } + + if res.err != nil { + if !errors.Is(res.err, context.Canceled) { + t.logger.Warn("upstream failed", + "path", req.URL.Path, + "error", res.err, + ) + } + continue + } + + // cancel other requests + for i, cancel := range cancels { + if i != res.idx { + cancel() + } + } + return res.resp, nil + } + + for _, cancel := range cancels { + cancel() + } + return nil, errors.New("all upstreams failed or timed out") +} diff --git a/spindle/engines/microvm/read_cache_proxy_test.go b/spindle/engines/microvm/read_cache_proxy_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/read_cache_proxy_test.go @@ -0,0 +1,171 @@ +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestCacheProxyFallsBackOnNotFound(t *testing.T) { + first := httptest.NewServer(http.NotFoundHandler()) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Fatalf("path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "ok") + })) + defer second.Close() + + upstreams, err := parseCacheUpstreams([]string{first.URL, second.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyServesNixCacheInfoItself(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("upstream should not be hit, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/nix-cache-info", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != nixCacheInfo { + t.Fatalf("body: got %q, want %q", got, nixCacheInfo) + } +} + +func TestCacheProxyErrorStatusDoesNotWinRace(t *testing.T) { + erroring := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Error(w, "misdirected", http.StatusMisdirectedRequest) + })) + defer erroring.Close() + healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + time.Sleep(50 * time.Millisecond) // lose the race to the erroring upstream + _, _ = io.WriteString(w, "ok") + })) + defer healthy.Close() + + upstreams, err := parseCacheUpstreams([]string{erroring.URL, healthy.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyJoinsSubpathQueryAndAuth(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/sub/cache/abc.narinfo" { + t.Errorf("path: got %q, want /sub/cache/abc.narinfo", req.URL.Path) + } + if got := req.URL.Query().Get("token"); got != "s3cret" { + t.Errorf("token: got %q, want s3cret", got) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/woof/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + + upstreamURL := "http://dawn:woof@" + strings.TrimPrefix(upstream.URL, "http://") + "/sub/cache/?token=s3cret" + upstreams, err := parseCacheUpstreams([]string{upstreamURL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyGuardedUpstreamCannotReachBlockedRanges(t *testing.T) { + // httptest listens on 127.0.0.1, which is in the blocked ranges; reaching + // it would mean a workflow-defined cache can hit the host's loopback + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("guarded upstream on loopback should not be reachable, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(nil, upstreams), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestCacheProxyRewritesHostHeader(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10500/abc.narinfo", nil) + req.Host = "127.0.0.1:10500" + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} diff --git a/spindle/engines/microvm/runner.go b/spindle/engines/microvm/runner.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/runner.go @@ -0,0 +1,24 @@ +package microvm + +import ( + "context" + "fmt" + "log/slog" +) + +type Runner interface { + // check the host has what this backend needs for spec. + Validate(spec ImageSpec, enableKVM bool) error + Start(ctx context.Context, cfg VMConfig, volumePaths map[string]string, logger *slog.Logger) (VMHandle, error) +} + +func runnerFor(runnerType string) (Runner, error) { + switch runnerType { + case "qemu", "": + return qemuRunner{}, nil + case "firecracker": + return nil, fmt.Errorf("runner type %q not implemented yet", runnerType) + default: + return nil, fmt.Errorf("unsupported runner type %q", runnerType) + } +} diff --git a/spindle/engines/microvm/start-test-cache.sh b/spindle/engines/microvm/start-test-cache.sh new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/start-test-cache.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# start a local ncps binary cache +# usage: ./start-test-cache.sh [port] + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [ncps-port]" + exit 1 +fi + +TEST_DIR="$(mkdir -p "$1" && cd "$1" && pwd)" +PORT="${2:-8501}" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +SECRET_KEY_PATH="$TEST_DIR/test-cache-key.secret" +PUBLIC_KEY_PATH="$TEST_DIR/test-cache-key.pub" +DB_PATH="$TEST_DIR/ncps.sqlite" +CONFIG_PATH="$TEST_DIR/ncps-config.yaml" +STORAGE_DIR="$TEST_DIR/storage" +ENV_PATH="$TEST_DIR/env.sh" +PID_PATH="$TEST_DIR/ncps.pid" + +mkdir -p "$STORAGE_DIR" + +echo "generating binary cache keys.." +nix-store --generate-binary-cache-key test-cache-key "$SECRET_KEY_PATH" "$PUBLIC_KEY_PATH" +PUBKEY_VAL=$(cat "$PUBLIC_KEY_PATH") + +echo "initializing ncps db..." +nix shell nixpkgs#dbmate --command dbmate \ + --migrations-dir "$(nix build --no-link --print-out-paths nixpkgs#ncps)/share/ncps/db/migrations/sqlite" \ + -u "sqlite:$DB_PATH" \ + up + +echo "writing ncps configuration..." +cat < "$CONFIG_PATH" +cache: + allow-delete-verb: true + allow-put-verb: true + hostname: "cache.local" + database-url: "sqlite:$DB_PATH" + secret-key-path: "$SECRET_KEY_PATH" + sign-narinfo: true + storage: + local: "$STORAGE_DIR" + upstream: + urls: + - https://cache.nixos.org + public-keys: + - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +server: + addr: "127.0.0.1:$PORT" +EOF + +echo "starting ncps on port $PORT..." +export CACHE_ALLOW_PUT_VERB=true +nix shell nixpkgs#ncps --command ncps serve --config "$CONFIG_PATH" & +NCPS_PID=$! +echo "$NCPS_PID" > "$PID_PATH" + +# wait for connection +for i in {1..30}; do + if curl -s "http://127.0.0.1:$PORT/nix-cache-info" > /dev/null; then + echo "ncps is healthy." + break + fi + sleep 0.5 + if ! kill -0 "$NCPS_PID" 2>/dev/null; then + echo "ncps exited unexpectedly during startup." + exit 1 + fi +done + +cat < "$ENV_PATH" +export CACHE_PUBKEY="$PUBKEY_VAL" +export CACHE_PORT="$PORT" +export CACHE_URL="http://127.0.0.1:$PORT" +export CACHE_UPLOAD_URL="http://127.0.0.1:$PORT/upload" +export CACHE_SECRET_KEY_PATH="$SECRET_KEY_PATH" +export NCPS_PID="$NCPS_PID" +export TEST_DIR="$TEST_DIR" +EOF + +echo "cache server started successfully. source $ENV_PATH to use, and kill PID $NCPS_PID or check $PID_PATH to stop it." diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -0,0 +1,850 @@ +#!/usr/bin/env bash +set -euo pipefail +# note: needs `sudo modprobe vhost_vsock`! + +log() { + printf "\n\033[1;36m>>> %s\033[0m\n" "$*" +} + +strip_ansi() { + local esc + esc=$(printf '\033') + sed -E "s/${esc}\[[0-9;]*[a-zA-Z]//g; s/${esc}\([a-zA-Z]//g" "$@" +} + +declare -a TEST_NAMES=() +declare -a TEST_STATUSES=() +declare -a TEST_TIMES=() + +get_time_ms() { + local t="${EPOCHREALTIME:-}" + if [[ "$t" == *.* ]]; then + local secs="${t%.*}" + local subs="${t#*.}" + subs="${subs:0:3}" + while [ "${#subs}" -lt 3 ]; do + subs="${subs}0" + done + echo "${secs}${subs}" + else + echo "$(date +%s)000" + fi +} + +format_duration() { + local ms=$1 + local secs=$((ms / 1000)) + local rem=$((ms % 1000)) + printf "%d.%03ds" "$secs" "$rem" +} + +print_summary() { + if [ "${#TEST_NAMES[@]}" -eq 0 ]; then + return + fi + printf "\n" + log "test summary" + echo "=========================================" + local passed_count=0 + local failed_count=0 + local total_time=0 + for i in "${!TEST_NAMES[@]}"; do + local name="${TEST_NAMES[$i]}" + local status="${TEST_STATUSES[$i]}" + local duration_ms="${TEST_TIMES[$i]}" + local duration_str + duration_str=$(format_duration "$duration_ms") + + local status_color="\033[0;32m" + if [ "$status" = "Failed" ]; then + status_color="\033[0;31m" + failed_count=$((failed_count + 1)) + else + passed_count=$((passed_count + 1)) + fi + total_time=$((total_time + duration_ms)) + + printf " %-30s %b%-8b\033[0m %s\n" "$name" "$status_color" "$status" "$duration_str" + done + echo "-----------------------------------------" + local total_tests="${#TEST_NAMES[@]}" + local total_time_str + total_time_str=$(format_duration "$total_time") + printf " total: %d tests, %d passed, %d failed\n" "$total_tests" "$passed_count" "$failed_count" + printf " total execution time: %s\n" "$total_time_str" + echo "=========================================" +} + +JOBS="${JOBS:-4}" +while [[ $# -gt 0 ]]; do + case "$1" in + -j | --jobs) + JOBS="$2" + shift 2 + ;; + --jobs=*) + JOBS="${1#*=}" + shift + ;; + --only) + TEST_ONLY="$2" + shift 2 + ;; + --only=*) + TEST_ONLY="${1#*=}" + shift + ;; + *) + echo "unknown argument: $1" >&2 + echo "usage: $0 [-j N|--jobs N] [--only TEST]" >&2 + exit 1 + ;; + esac +done +if ! [[ "$JOBS" =~ ^[0-9]+$ ]] || [ "$JOBS" -lt 1 ]; then + echo "error: --jobs must be a positive integer (got '$JOBS')" >&2 + exit 1 +fi + +pick_free_port() { + local port + for _ in $(seq 1 50); do + port=$(((RANDOM % 16384) + 20000)) + if ! (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + echo "$port" + return 0 + fi + done + echo "error: could not find a free port for the cache" >&2 + return 1 +} + +SUCCESS=0 +rm -rf /tmp/test-spindle-microvm-logs + +log "setup local cache & temp environment" +TEMP_DIR=$(mktemp -d -t test-spindle-microvm-XXXXXX) + +log "build spindle & microvm image tarball" +nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run +TARBALL_PATH=$(nix build .#spindle-nixos-image-tarball --no-link --print-out-paths) +mkdir -p "$TEMP_DIR/image" +tar -C "$TEMP_DIR/image" -xzf "$TARBALL_PATH" +IMAGE_SPEC_JSON="$TEMP_DIR/image/spec.json" + +log "build alpine microvm image tarball" +ALPINE_TARBALL_PATH=$(nix build .#spindle-alpine-image-tarball --no-link --print-out-paths) +mkdir -p "$TEMP_DIR/alpine-image" +tar -C "$TEMP_DIR/alpine-image" -xzf "$ALPINE_TARBALL_PATH" +ALPINE_IMAGE_SPEC_JSON="$TEMP_DIR/alpine-image/spec.json" + +kill_temp_dir_procs() { + if [ -f "$TEMP_DIR/ncps.pid" ]; then + kill "$(cat "$TEMP_DIR/ncps.pid")" 2>/dev/null || true + fi + pkill -TERM -f "$TEMP_DIR" 2>/dev/null || true + local i + for i in $(seq 1 20); do + pgrep -f "$TEMP_DIR" >/dev/null 2>&1 || break + sleep 0.25 + done + pkill -KILL -f "$TEMP_DIR" 2>/dev/null || true +} + +collect_logs() { + echo "test failed. copying logs to /tmp/test-spindle-microvm-logs" + mkdir -p /tmp/test-spindle-microvm-logs + local f + for f in "$TEMP_DIR"/*; do + [ -f "$f" ] && cp "$f" /tmp/test-spindle-microvm-logs/ + done + local work logf + for work in "$TEMP_DIR"/work-*; do + [ -d "$work" ] || continue + for logf in "$work"/*.log; do + [ -f "$logf" ] || continue + strip_ansi "$logf" > "/tmp/test-spindle-microvm-logs/$(basename "$work")-$(basename "$logf")" + done + done +} + +CLEANED=0 +cleanup() { + [ "$CLEANED" -eq 1 ] && return + CLEANED=1 + + print_summary + log "cleaning up..." + + local jobs_pids + jobs_pids=$(jobs -p) + [ -n "$jobs_pids" ] && kill $jobs_pids 2>/dev/null || true + + kill_temp_dir_procs + + [ "$SUCCESS" -ne 1 ] && collect_logs + + chmod -R +w "$TEMP_DIR" 2>/dev/null || true + rm -rf "$TEMP_DIR" + echo "done" +} +trap cleanup EXIT +# route signals through the EXIT trap so an interrupt still tears down VMs. +trap 'exit 130' INT +trap 'exit 143' TERM + +CACHE_PORT=$(pick_free_port) +./spindle/engines/microvm/start-test-cache.sh "$TEMP_DIR" "$CACHE_PORT" +source "$TEMP_DIR/env.sh" + +run_vm() { + local name="" + local timeout="60s" + local upload=0 + local activate="" + local no_cache=0 + local db="" + local spec="$IMAGE_SPEC_JSON" + + while [[ $# -gt 0 ]]; do + case "$1" in + --spec) + spec="$2" + shift 2 + ;; + --name) + name="$2" + shift 2 + ;; + --timeout) + timeout="$2" + shift 2 + ;; + --upload) + upload=1 + shift + ;; + --activate) + activate="$2" + shift 2 + ;; + --no-cache) + no_cache=1 + shift + ;; + --db) + db="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "unknown argument: $1" >&2 + exit 1 + ;; + esac + done + + local work_dir="$TEMP_DIR/work-${name}" + mkdir -p "$work_dir" + + local args=( + --image-spec "$spec" + --work-dir "$work_dir" + --exec-timeout "$timeout" + --port "${SPINDLE_TEST_VSOCK_PORT:-10240}" + --memory-mib 2049 + ) + + if [ "$no_cache" -eq 0 ]; then + args+=( + --cache-read-url "$CACHE_URL" + --cache-trusted-public-key "$CACHE_PUBKEY" + ) + fi + + if [ "$upload" -eq 1 ]; then + args+=( + --cache-upload-url "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" + ) + fi + + if [ -n "$activate" ]; then + args+=( + --activate-config "$activate" + ) + fi + + if [ -n "$db" ]; then + args+=( + --db "$db" + ) + fi + + local out + if ! out=$(spindle/spindle-microvm-run "${args[@]}" -- "$@" 2>&1); then + echo "$out" | strip_ansi >&2 + strip_ansi "$work_dir/serial.log" >&2 + strip_ansi "$work_dir/qemu.log" >&2 + return 1 + fi + echo "$out" +} + +run_test_job() { + local name="$1" + local func="$2" + local port="$3" + export SPINDLE_TEST_VSOCK_PORT="$port" + + local logfile="$TEMP_DIR/test-${name}.log" + local start + start=$(get_time_ms) + log "[$name] start (vsock port $port)" + + local status="Passed" + if ! "$func" > "$logfile" 2>&1; then + status="Failed" + fi + + local duration_ms=$(($(get_time_ms) - start)) + printf '%s\t%s\n' "$status" "$duration_ms" > "$TEMP_DIR/test-${name}.status" + + local duration_str + duration_str=$(format_duration "$duration_ms") + if [ "$status" = "Failed" ]; then + printf "\n\033[0;31m>>> [%s] FAILED (%s)\033[0m\n" "$name" "$duration_str" + strip_ansi "$logfile" || true + else + printf "\n\033[0;32m>>> [%s] passed (%s)\033[0m\n" "$name" "$duration_str" + fi +} + +# schedules every selected test across at most $JOBS concurrent VMs (each on its +# own vsock port), then aggregates the per-test status files into the summary +# arrays. returns 1 if any test failed, 0 otherwise. +run_tests() { + local base_port=10240 + local idx=0 + local running=0 + + for func in "${TESTS[@]}"; do + local name="${func#test_}" + name="${name//_/-}" + if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then + continue + fi + + run_test_job "$name" "$func" "$((base_port + idx))" & + idx=$((idx + 1)) + running=$((running + 1)) + + if [ "$running" -ge "$JOBS" ]; then + wait -n || true + running=$((running - 1)) + fi + done + wait + + local failed=0 + for func in "${TESTS[@]}"; do + local name="${func#test_}" + name="${name//_/-}" + if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then + continue + fi + + local statusfile="$TEMP_DIR/test-${name}.status" + if [ ! -f "$statusfile" ]; then + TEST_NAMES+=("$name") + TEST_STATUSES+=("Failed") + TEST_TIMES+=(0) + failed=1 + continue + fi + + local status duration_ms + IFS=$'\t' read -r status duration_ms < "$statusfile" + TEST_NAMES+=("$name") + TEST_STATUSES+=("$status") + TEST_TIMES+=("$duration_ms") + if [ "$status" = "Failed" ]; then + failed=1 + fi + done + + return "$failed" +} + +test_realize() { + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "test-file" "hello from cache"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + local out + out=$(run_vm --name "realize" --timeout "60s" -- /run/current-system/sw/bin/bash -lc ' +set -euo pipefail +store_path=$1 +cache_url=$(sed -n "s/^extra-substituters = //p" /run/spindle/nix.conf) +cache_url=${cache_url%% *} +if [ -z "$cache_url" ]; then + echo "error: cache URL not found in /run/spindle/nix.conf" >&2 + exit 1 +fi + +http_version=$(/run/current-system/sw/bin/curl --http2-prior-knowledge -fsS -o /dev/null -w "%{http_version}" "$cache_url/nix-cache-info") +echo "http_version=$http_version" +case "$http_version" in + 2|2.0) ;; + *) + echo "error: cache proxy did not negotiate HTTP/2 (got $http_version)" >&2 + exit 1 + ;; +esac + +/run/current-system/sw/bin/nix-store --realise "$store_path" >/dev/null +' bash "$test_store_path") || return 1 + + if ! echo "$out" | strip_ansi | grep -q -E "^http_version=2(\\.0)?$"; then + echo "error: cache proxy did not report HTTP/2" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: store path realized from cache and cache proxy accepted cleartext HTTP/2" +} + +test_build_upload() { + local nix_expr='with import {}; writeText "uploaded-test-file" "hello from vm upload"' + local out + out=$(run_vm --name "build-upload" --timeout "120s" --upload -- /run/current-system/sw/bin/bash -l -c "nix-build -E '$nix_expr' --no-out-link") || return 1 + + local built_path + built_path=$(echo "$out" | strip_ansi | grep -v '\.drv' | grep -o '/nix/store/[a-z0-9]*-uploaded-test-file' | head -n 1 || true) + if [ -z "$built_path" ]; then + echo "error: could not find built store path in vm output" >&2 + return 1 + fi + echo "extracted path: $built_path" + + local hash + hash=$(basename "$built_path" | cut -d'-' -f1) + if ! curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then + echo "error: built store path was not uploaded to the binary cache" >&2 + return 1 + fi + echo "success: store path uploaded to cache" +} + +test_networking() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --name "networking" --timeout "120s" --no-cache -- /run/current-system/sw/bin/bash -c "/run/current-system/sw/bin/curl -I --connect-timeout 1 -m 1 http://10.0.2.2:$CACHE_PORT; /run/current-system/sw/bin/nix-store --realise $hello_path") || return 1 + + if echo "$out" | grep -qi -E "unreachable|timeout|failed to connect|timed out" || echo "$out" | grep -q "exited with code"; then + echo "success: host network access blocked" + else + echo "error: guest vm accessed host network or returned unexpected output" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: guest vm reached the internet and substituted hello" +} + +test_substitution_and_no_upload() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --name "nixpkgs-hello" --timeout "180s" --upload -- /run/current-system/sw/bin/nix-store --realise "$hello_path") || return 1 + + # Check that it was substituted from our proxy + if ! echo "$out" | strip_ansi | grep -q -E "copying path.*hello"; then + echo "error: hello package was not substituted (or output mismatch)" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + # Check that nothing was uploaded to the cache + if ! echo "$out" | strip_ansi | grep -q "cache uploaded: 0"; then + echo "error: hello package substitution triggered cache upload" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + echo "success: hello package substituted from upstream cache and was not uploaded" +} + +# a pinned registry, reused by the dependency and registry-pin tests. +ACTIVATION_REGISTRY='"registry": { + "nixpkgs": "github:nixos/nixpkgs/nixos-unstable", + "my-nixpkgs": "nixpkgs" + }' + +test_activation_services() { + local config='{ + "services": { + "openssh": { + "enable": true, + "authorizedKeysFiles": ["/etc/ssh/authorized_keys"] + } + } + }' + local out + out=$(run_vm --name "activation-services" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + if ! echo "$out" | strip_ansi | grep -q "^active$"; then + echo "error: sshd not active after activation" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: openssh service active after activation" +} + +test_activation_dependencies() { + # cowsay as a bare dependency (resolved via the pinned nixpkgs registry); + # hello via the github flakeref and (separately) the my-nixpkgs alias. + local config='{ + '"$ACTIVATION_REGISTRY"', + "dependencies": [ + "cowsay", + "github:nixos/nixpkgs#hello", + "my-nixpkgs#hello" + ] + }' + local out + out=$(run_vm --name "activation-dependencies" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +cowsay "registry pin ok" >/dev/null && echo "cowsay=ran" +echo "hello=$(hello)" +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -qF "cowsay=ran"; then + echo "error: bare dependency 'cowsay' (resolved via the pinned nixpkgs registry) did not run" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -qF "hello=Hello, world!"; then + echo "error: hello dependency (github flakeref + my-nixpkgs alias) did not run" >&2 + echo "$clean" >&2 + return 1 + fi + echo "success: bare, flakeref, and aliased dependencies all resolved and ran" +} + +test_activation_registry_pin() { + # the nixpkgs the image itself was built from; the registry override must NOT + # resolve to this. deterministic (locked in the repo flake), so safe to compare. + local base_nixpkgs + base_nixpkgs=$(nix eval --raw --impure --expr '(builtins.getFlake (toString ./.)).inputs.nixpkgs.outPath') + + local config='{ + '"$ACTIVATION_REGISTRY"' + }' + # the pinned nixpkgs must reach the system nix config: resolving it via the + # flakes CLI must not error "is not locked", and it must win the nixPath. + local out + out=$(run_vm --name "activation-registry-pin" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +echo "lib_version=$(nix eval --raw nixpkgs#lib.version)" +echo "nix_path=$(nix eval --raw --impure --expr "toString ")" +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -qE "lib_version=[0-9]"; then + echo "error: guest could not resolve nixpkgs#lib.version from the user registry (locked-ref failure?)" >&2 + echo "$clean" >&2 + return 1 + fi + local guest_nixpath + guest_nixpath=$(echo "$clean" | sed -n 's/^nix_path=//p' | head -n1) + if [ -z "$guest_nixpath" ] || [ "$guest_nixpath" = "$base_nixpkgs" ]; then + echo "error: guest nixPath did not resolve to the registry override (got '$guest_nixpath', base '$base_nixpkgs')" >&2 + return 1 + fi + echo "success: pinned nixpkgs registry reached the guest nix config (flakes CLI + nixPath)" +} + +test_activation_cache_substitution() { + # a unique path that only exists in the configured (workflow) cache; the host + # seeds it so the guest can prove it substitutes through the read proxy. + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "activation-cache-test" "hello from the workflow cache"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + # a trivial config: this test only cares that the read proxy serves the + # workflow-cache path during an activated run. + local out + out=$(run_vm --name "activation-cache-substitution" --timeout "300s" --activate '{}' -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +store_path=$1 +# the unique path only exists in the configured cache, so realising it proves it +# was substituted through the read proxy and not built or found elsewhere. +nix-store --realise "$store_path" >/dev/null +echo "substituted=$(cat "$store_path")" +' bash "$test_store_path") || return 1 + + if ! echo "$out" | strip_ansi | grep -qF "substituted=hello from the workflow cache"; then + echo "error: unique path was not substituted from the configured cache" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: unique path substituted from the workflow cache through the read proxy" +} + +test_activation_docker() { + local config='{ + "virtualisation": { + "docker": { "enable": true } + } + }' + # docker.service is up, but the daemon socket can lag a beat behind activation; + # wait for it to answer, then pull+run a real image. this drives the slimmed + # kernel modules: overlay.ko storage plus bridge/iptables networking out of the + # pruned tree, with outbound DNS/network over the guest slirp link. + local out + out=$(run_vm --name "activation-docker" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +echo "docker_unit=$(systemctl is-active docker)" +for i in $(seq 1 60); do docker info >/dev/null 2>&1 && break; sleep 1; done +docker info >/dev/null +echo "storage_driver=$(docker info --format "{{.Driver}}")" +docker run --rm alpine cat /etc/alpine-release | sed "s/^/alpine_release=/" +docker run --rm alpine echo container-ran-ok +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -q "^docker_unit=active$"; then + echo "error: docker service not active after activation" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -qE "storage_driver=overlay(2|fs)"; then + echo "error: docker is not using an overlay storage driver (overlay.ko missing?)" >&2 + return 1 + fi + if ! echo "$clean" | grep -qE "alpine_release=[0-9]+\."; then + echo "error: failed to pull and read the alpine image" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "container-ran-ok"; then + echo "error: command did not run inside the alpine container" >&2 + return 1 + fi + echo "success: docker service active, pulled and ran an alpine container on the overlay storage driver" +} + +test_activation_cached_realize() { + local config='{ + "services": { + "openssh": { + "enable": true, + "authorizedKeysFiles": ["/etc/ssh/authorized_keys"] + } + } + }' + local db_path="$TEMP_DIR/activation-cached.db" + + # first run: build the config, upload its closure, and record the toplevel in + # the db. nothing cached yet, so this builds from scratch. + local out + out=$(run_vm --name "activation-cached-first" --timeout "600s" --activate "$config" --db "$db_path" --upload -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + if ! echo "$out" | strip_ansi | grep -q "^active$"; then + echo "error: sshd not active after first activation" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + # second run: same config + db, no upload. must realize the recorded toplevel + # from the cache instead of rebuilding, and the cached system must come up. + out=$(run_vm --name "activation-cached-second" --timeout "300s" --activate "$config" --db "$db_path" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -q "realizing cached NixOS config"; then + echo "error: second run did not realize cached configuration" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "^active$"; then + echo "error: sshd not active after cached config activation" >&2 + echo "$clean" >&2 + return 1 + fi + echo "success: second run realized the cached NixOS config from the cache and sshd came up" +} + +test_alpine() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine" --timeout "180s" --no-cache -- /bin/sh -lc ' +set -eu +export HOME=/workspace +hello_path=$1 +echo "release=$(cat /etc/alpine-release)" +echo "user=$(id -un)" +git version +bash -c "echo bash=\$BASH_VERSION" +touch /workspace/write-test +echo "workspace writable" +git ls-remote https://tangled.org/@tangled.org/core HEAD >/dev/null +echo "git over https ok" +apk add make +echo "apk ok" +# substitute a real package from cache.nixos.org over HTTPS and run it +nix-store --realise "$hello_path" >/dev/null +echo "ran=$("$hello_path/bin/hello")" +' sh "$hello_path") || return 1 + + echo "$out" | strip_ansi >&2 + for needle in "release=" "user=spindle-workflow" "git version" "bash=" "workspace writable" "git over https ok" "apk ok" "ran=Hello, world!"; do + if ! echo "$out" | strip_ansi | grep -q "$needle"; then + echo "error: alpine guest output missing $needle" >&2 + return 1 + fi + done + echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned + installed over the network, and substituted+ran a package from cache.nixos.org over HTTPS" +} + +# asserts a store path's narinfo shows up in the local cache, retrying briefly +# since the post-build-hook enqueues uploads asynchronously. +cache_has_path() { + local path="$1" + local hash + hash=$(basename "$path" | cut -d'-' -f1) + local i + for i in $(seq 1 20); do + if curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then + return 0 + fi + sleep 0.5 + done + return 1 +} + +test_alpine_nix() { + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "alpine-nix-test" "hello from cache to alpine"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + # exercise the full local-cache path: daemon connectivity, substitution, + # store-db queries, and a build via *both* the classic (nix-build) and the + # new flakes/nix-command (nix build) frontends. the two build derivations + # use distinct names so we can confirm each got uploaded back to the cache. + local out + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-nix" --timeout "180s" --upload -- /bin/sh -lc ' +set -eu +export HOME=/workspace +store_path=$1 + +echo "nix_version=$(nix --version | head -n1)" +{ nix store info >/dev/null 2>&1 || nix store ping >/dev/null 2>&1; } && echo "daemon=ok" + +# substitute a path from the cache and query the store db about it +nix-store --realise "$store_path" >/dev/null +echo "substituted=$(cat "$store_path")" +echo "requisites=$(nix-store --query --requisites "$store_path" | wc -l | tr -d " ")" +nix path-info --json "$store_path" >/dev/null && echo "path_info=ok" + +# build via the new CLI; the substituted path is declared as a real input +# (builtins.storePath) so nix must realise it into the build sandbox first. +# heredoc is unquoted (the outer guest script is single-quoted, so a quoted +# delimiter would close it), hence \$ escapes what nix/the builder must expand. +export DEP="$store_path" +cat > /workspace/new.nix < \$out" ]; +} +NIXEOF +new_path=$(nix build --impure --file /workspace/new.nix --no-link --print-out-paths) +echo "new_path=$new_path" +echo "new_content=$(tr "\n" "|" < "$new_path")" + +# build via the classic CLI +cat > /workspace/old.nix < \$out" ]; +} +NIXEOF +old_path=$(nix-build /workspace/old.nix --no-out-link) +echo "old_path=$old_path" +' sh "$test_store_path") || return 1 + + echo "$out" | strip_ansi >&2 + local clean + clean=$(echo "$out" | strip_ansi) + + local needle + for needle in "daemon=ok" "substituted=hello from cache to alpine" "path_info=ok"; do + if ! echo "$clean" | grep -q "$needle"; then + echo "error: alpine nix output missing '$needle'" >&2 + return 1 + fi + done + if ! echo "$clean" | grep -qE "requisites=[1-9][0-9]*"; then + echo "error: store db query returned no requisites for the substituted path" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "new_content=via-nix-build-with-dep"; then + echo "error: 'nix build' (new CLI) did not realise its substituted dependency and build" >&2 + return 1 + fi + + local new_path old_path + new_path=$(echo "$clean" | grep -o 'new_path=/nix/store/[a-z0-9]*-alpine-nix-build-new' | cut -d= -f2) + old_path=$(echo "$clean" | grep -o 'old_path=/nix/store/[a-z0-9]*-alpine-nix-build-old' | cut -d= -f2) + if [ -z "$new_path" ] || [ -z "$old_path" ]; then + echo "error: could not extract both built store paths from alpine guest output" >&2 + return 1 + fi + if ! cache_has_path "$new_path"; then + echo "error: nix-build (new CLI) output was not uploaded to the cache" >&2 + return 1 + fi + if ! cache_has_path "$old_path"; then + echo "error: nix-build (classic CLI) output was not uploaded to the cache" >&2 + return 1 + fi + echo "success: alpine guest substituted, queried the store db, built via both CLIs, and uploaded both outputs" +} + +TESTS=( + test_alpine + test_alpine_nix + test_realize + test_build_upload + test_networking + test_substitution_and_no_upload + test_activation_services + test_activation_dependencies + test_activation_registry_pin + test_activation_cache_substitution + test_activation_docker + test_activation_cached_realize +) + +log "running ${#TESTS[@]} tests" +if ! run_tests; then + exit 1 +fi + +SUCCESS=1 +log "passed!!" diff --git a/spindle/engines/microvm/upload_cache_proxy.go b/spindle/engines/microvm/upload_cache_proxy.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_proxy.go @@ -0,0 +1,192 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" + + "github.com/mdlayher/vsock" +) + +type UploadCacheProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server +} + +func StartUploadCacheProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []CacheUpstream, logger *slog.Logger) (*UploadCacheProxy, error) { + if strings.TrimSpace(uploadURL) == "" { + return nil, nil + } + + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "upload_cache_proxy", "cid", cid, "uploadURL", uploadURL) + + target, err := url.Parse(uploadURL) + if err != nil { + return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) + } + if target.Scheme != "http" && target.Scheme != "https" { + return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q (must be http or https)", uploadURL, target.Scheme) + } + if target.Host == "" { + return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) + } + + ln, port, err := listenRandomVsockUploadPort(ctx) + if err != nil { + return nil, fmt.Errorf("listen for cache upload proxy: %w", err) + } + + proxy := &UploadCacheProxy{ + port: port, + ln: ln, + } + proxy.server = &http.Server{ + Handler: uploadProxyHandler(target, readUpstreams, logger), + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 30 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("upload cache proxy stopped", "port", port, "error", err) + } + }() + + logger.Info("started upload cache proxy", "port", port, "target", uploadURL, "readUpstreams", len(readUpstreams)) + return proxy, nil +} + +func (p *UploadCacheProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *UploadCacheProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + return closeErr +} + +func uploadProxyHandler(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) http.Handler { + rp := httputil.NewSingleHostReverseProxy(target) + rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) + + origDirector := rp.Director + rp.Director = func(req *http.Request) { + origDirector(req) + // ensure host matches target + req.Host = target.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.User; user != nil { + password, _ := user.Password() + req.SetBasicAuth(user.Username(), password) + } + } + + // before uploading, nix copy asks the destination whether it already has each + // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer + // that check across the upload target *and* the read caches: if any of them + // already serves the path there is no point uploading it (the guest would + // just substitute it from there anyway). + narinfoUpstreams := append([]CacheUpstream{{url: target}}, readUpstreams...) + exists := ¶llelRacingTransport{ + upstreams: narinfoUpstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isNarinfoExistenceCheck(r) { + serveNarinfoExistence(w, r, exists, logger) + return + } + rp.ServeHTTP(w, r) + }) +} + +func isNarinfoExistenceCheck(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + return strings.HasSuffix(r.URL.Path, ".narinfo") +} + +func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { + probe := r.Clone(r.Context()) + probe.RequestURI = "" + + resp, err := exists.RoundTrip(probe) + if err != nil { + logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) + w.WriteHeader(http.StatusNotFound) + return + } + defer resp.Body.Close() + + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { + logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) + } +} + +func listenRandomVsockUploadPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.Listen(port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock upload port: %w", lastErr) +} diff --git a/spindle/engines/microvm/upload_cache_proxy_test.go b/spindle/engines/microvm/upload_cache_proxy_test.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_proxy_test.go @@ -0,0 +1,128 @@ +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestUploadProxyRewritesHostAndAuth(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + if req.URL.Path != "/sub/abc.narinfo" { + t.Errorf("path: got %q, want /sub/abc.narinfo", req.URL.Path) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/hunter2/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + target, err := url.Parse("http://dawn:woof@" + upstreamHost + "/sub/") + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodPut, "http://127.0.0.1:10501/abc.narinfo", strings.NewReader("narinfo")) + req.Host = "127.0.0.1:10501" + rec := httptest.NewRecorder() + uploadProxyHandler(target, nil, slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +func TestUploadProxySkipsNarinfoAvailableUpstream(t *testing.T) { + var uploadHits int + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + uploadHits++ + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Errorf("upstream path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200 (so nix treats the path as present and skips upload)", rec.Code) + } + if !strings.Contains(rec.Body.String(), "StorePath: /nix/store/abc") { + t.Fatalf("body: got %q, want the upstream narinfo body", rec.Body.String()) + } +} + +func TestUploadProxyUploadsNarinfoNobodyHas(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status: got %d, want 404 (so nix uploads the path)", rec.Code) + } +} + +func TestUploadProxySkipsNarinfoAlreadyOnTarget(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer target.Close() + + handler := uploadProxyHandler(mustParseURL(t, target.URL), nil, slog.Default()) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200", rec.Code) + } +} diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go new file mode 100644 --- /dev/null +++ b/spindle/engines/microvm/vm.go @@ -0,0 +1,370 @@ +package microvm + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "math" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "time" + + "tangled.org/core/spindle/models" +) + +const ( + minGuestCID = 3 + vmCrashLogTailBytes = 4096 +) + +func AllocateCID() (uint32, error) { + var data [4]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, fmt.Errorf("allocate guest CID: %w", err) + } + return minGuestCID + binary.BigEndian.Uint32(data[:])%60000, nil +} + +func prepareWorkDir(workDir string) error { + if workDir == "" { + return fmt.Errorf("microvm work directory is required") + } + if err := os.MkdirAll(workDir, 0o755); err != nil { + return fmt.Errorf("create microvm work directory: %w", err) + } + return nil +} + +func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) { + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if volume.ReadOnly { + return nil, fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image) + } + if volume.FSType != "ext4" { + return nil, fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType) + } + if volume.ImageType != "" && volume.ImageType != "raw" { + return nil, fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType) + } + + path := filepath.Join(workDir, filepath.Base(volume.Image)) + if err := createSparseFile(path, volume.SizeMiB); err != nil { + return nil, err + } + noJournal := volume.MountPoint == "/workspace" + if err := runMkfsExt4(ctx, mkfsExt4, path, noJournal); err != nil { + return nil, err + } + paths[volume.Image] = path + } + return paths, nil +} + +func createSparseFile(path string, sizeMiB int64) error { + if sizeMiB <= 0 { + return fmt.Errorf("sparse file %q size must be positive", path) + } + if sizeMiB > math.MaxInt64/(1024*1024) { + return fmt.Errorf("sparse file %q size is too large", path) + } + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("create sparse file %q: %w", path, err) + } + defer file.Close() + + if err := file.Truncate(sizeMiB * 1024 * 1024); err != nil { + return fmt.Errorf("resize sparse file %q: %w", path, err) + } + return nil +} + +func runMkfsExt4(ctx context.Context, mkfsExt4, path string, noJournal bool) error { + if mkfsExt4 == "" { + return fmt.Errorf("mkfs.ext4 path is required") + } + args := []string{"-F"} + if noJournal { + args = append(args, "-O", "^has_journal") + } + args = append(args, path) + + cmd := exec.CommandContext(ctx, mkfsExt4, args...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("mkfs.ext4 %q: %w: %s", path, err, strings.TrimSpace(string(output))) + } + return nil +} + +func createParentedFile(path string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("create log directory: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return nil, fmt.Errorf("create log file %q: %w", path, err) + } + return file, nil +} + +type VMLogs struct { + Serial string + Extra map[string]string +} + +type VMHandle interface { + Shutdown(ctx context.Context) error + WaitContext(ctx context.Context) error + Close() error + Logs() VMLogs + CID() uint32 + WorkDir() string + OOMKilled() bool +} + +type VMConfig struct { + Image ImageSpec + CID uint32 + EnableKVM bool + WorkDir string + Cgroup CgroupLimits + + BootTimeout time.Duration + MkfsExt4 string + Dev bool +} + +type workflowState struct { + ImageSpec ImageSpec + ImageSpecPath string + Config manifestConfig + ConfigKey string + Image string + CacheReadURLs []string + CacheTrustedPublicKeys []string + VM VMHandle + Agent *AgentSession + ReadCache *ReadCacheProxy + UploadCache *UploadCacheProxy + DNSProxy *DNSProxy + WorkDir string + NixOSToplevelCache nixosToplevelCacheStore +} + +func (e *Engine) cleanupState(ctx context.Context, wid models.WorkflowId, state *workflowState) error { + if state == nil { + return nil + } + + ctx = context.WithoutCancel(ctx) + + var err error + err = errors.Join(err, e.drainNixCache(ctx, state)) + err = errors.Join(err, e.shutdownVM(ctx, wid, state)) + err = errors.Join(err, closeIO(&state.Agent)) + err = errors.Join(err, closeIO(&state.ReadCache)) + err = errors.Join(err, closeIO(&state.UploadCache)) + err = errors.Join(err, closeIO(&state.DNSProxy)) + err = errors.Join(err, removeWorkDir(state)) + return err +} + +func (e *Engine) drainNixCache(ctx context.Context, state *workflowState) error { + if state.Agent == nil || e.cfg.NixCache.UploadURL == "" { + return nil + } + + drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) + defer cancel() + + if _, err := state.Agent.Drain(drainCtx); err != nil { + return fmt.Errorf("drain nix cache: %w", err) + } + return nil +} + +func (e *Engine) shutdownVM(ctx context.Context, wid models.WorkflowId, state *workflowState) error { + if state.VM == nil { + return nil + } + + var err error + + if state.Agent != nil { + gracefulCtx, cancel := context.WithTimeout(ctx, vmShutdownTimeout) + poweredOff, poweroffErr := e.poweroffViaAgent(gracefulCtx, wid, state) + cancel() + + err = errors.Join(err, poweroffErr) + if poweredOff { + return errors.Join(err, closeIO(&state.VM)) + } + } + + fallbackCtx, cancel := context.WithTimeout(ctx, vmShutdownTimeout) + defer cancel() + + if shutdownErr := state.VM.Shutdown(fallbackCtx); shutdownErr != nil { + e.l.Warn("microVM shutdown fallback failed", "workflow", wid, "error", shutdownErr) + err = errors.Join(err, shutdownErr) + } + + return errors.Join(err, closeIO(&state.VM)) +} + +func (e *Engine) poweroffViaAgent(ctx context.Context, wid models.WorkflowId, state *workflowState) (bool, error) { + if err := state.Agent.Poweroff(ctx); err != nil { + e.l.Warn("agent poweroff request failed", "workflow", wid, "error", err) + return false, err + } + + if err := state.VM.WaitContext(ctx); err != nil { + e.l.Warn("agent poweroff did not stop microVM", "workflow", wid, "error", err) + return false, nil + } + + return true, nil +} + +// helper for closing io interfaces, sets to nil to prevent double-close +func closeIO[T io.Closer](field *T) error { + closer := *field + var zero T + *field = zero + if any(closer) == any(zero) { + return nil + } + return closer.Close() +} + +func removeWorkDir(state *workflowState) error { + if state.WorkDir == "" { + return nil + } + + err := os.RemoveAll(state.WorkDir) + state.WorkDir = "" + return err +} + +// returns a context derived from ctx that is cancelled either when ctx itself +// is cancelled or when the microVM exits on its own. the returned flag reports +// whether the VM exited (as opposed to ctx being cancelled for another reason, +// e.g. the workflow timeout), letting callers tell a crash apart from a +// timeout. cancel must be called to release the watcher goroutine. +func watchVMExit(ctx context.Context, vm VMHandle) (context.Context, *atomic.Bool, context.CancelFunc) { + exited := &atomic.Bool{} + watchCtx, cancel := context.WithCancel(ctx) + if vm == nil { + return watchCtx, exited, cancel + } + go func() { + _ = vm.WaitContext(watchCtx) // returns when VM exits or watchCtx is cancelled + if watchCtx.Err() == nil { + exited.Store(true) + cancel() // don't forget to cancel the watchCtx... + } + }() + return watchCtx, exited, cancel +} + +func vmCrashLog(vm VMHandle) string { + if vm == nil { + return "" + } + logs := vm.Logs() + + var b strings.Builder + if tail := tailFile(logs.Serial, vmCrashLogTailBytes); tail != "" { + fmt.Fprintf(&b, "==== serial log ====\n%s\n", tail) + } + for _, name := range slices.Sorted(maps.Keys(logs.Extra)) { + if tail := tailFile(logs.Extra[name], vmCrashLogTailBytes); tail != "" { + fmt.Fprintf(&b, "==== %s log ====\n%s\n", name, tail) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func tailFile(path string, max int64) string { + if path == "" { + return "" + } + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + if info, err := f.Stat(); err == nil && info.Size() > max { + if _, err := f.Seek(-max, io.SeekEnd); err != nil { + return "" + } + } + data, err := io.ReadAll(f) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func waitAgentConn(ctx context.Context, connCh <-chan net.Conn) (net.Conn, error) { + select { + case conn := <-connCh: + if conn == nil { + return nil, fmt.Errorf("agent connection closed before setup") + } + return conn, nil + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for agent: %w", ctx.Err()) + } +} + +func StartVM(ctx context.Context, cfg VMConfig, logger *slog.Logger) (VMHandle, error) { + if logger == nil { + logger = slog.Default() + } + + runner, err := runnerFor(cfg.Image.RunnerType) + if err != nil { + return nil, err + } + if err := cfg.Image.Validate(); err != nil { + return nil, err + } + if err := cfg.Image.validateImageFiles(); err != nil { + return nil, err + } + if err := runner.Validate(cfg.Image, cfg.EnableKVM); err != nil { + return nil, err + } + + if err := prepareWorkDir(cfg.WorkDir); err != nil { + return nil, err + } + + mkfsExt4 := cfg.MkfsExt4 + if mkfsExt4 == "" { + mkfsExt4, err = exec.LookPath("mkfs.ext4") + if err != nil { + return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) + } + } + volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4) + if err != nil { + return nil, err + } + + return runner.Start(ctx, cfg, volumePaths, logger) +} diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -39,6 +39,8 @@ l *slog.Logger cfg *config.Config + slotter engine.WorkflowSlotter + cleanupMu sync.Mutex cleanup map[string][]cleanupFunc } @@ -168,14 +170,27 @@ l := log.FromContext(ctx).With("component", "spindle") e := &Engine{ - docker: dcli, - l: l, - cfg: cfg, + docker: dcli, + l: l, + cfg: cfg, + slotter: engine.NewSemaphoreSlotter(cfg.NixeryPipelines.MaxConcurrentWorkflows), } e.cleanup = make(map[string][]cleanupFunc) return e, nil +} + +func (e *Engine) AcquireWorkflowSlot( + ctx context.Context, + wid models.WorkflowId, + wf *models.Workflow, +) (engine.WorkflowSlot, error) { + if e.slotter == nil { + return engine.NoopSlot{}, nil + } + + return e.slotter.AcquireWorkflowSlot(ctx, wid, wf) } func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { @@ -235,6 +250,11 @@ l.Info("creating container") wfLogger.DataWriter(setupStepIdx, "stdout").Write([]byte("Creating container...")) + extraHosts := []string{"host.docker.internal:host-gateway"} + for _, h := range e.cfg.Server.DevExtraHosts { + extraHosts = append(extraHosts, h+":host-gateway") + } + resp, err := e.docker.ContainerCreate(ctx, &container.Config{ Image: addl.image, Cmd: []string{"cat"}, @@ -265,7 +285,7 @@ CapDrop: []string{"ALL"}, CapAdd: []string{"CAP_DAC_OVERRIDE", "CAP_CHOWN", "CAP_FOWNER", "CAP_SETUID", "CAP_SETGID"}, SecurityOpt: []string{"no-new-privileges"}, - ExtraHosts: []string{"host.docker.internal:host-gateway"}, + ExtraHosts: extraHosts, Resources: container.Resources{ Memory: e.cfg.NixeryPipelines.MaxJobMemoryMB * 1024 * 1024, }, diff --git a/spindle/agentproto/spindle/agent/v1/agent.proto b/spindle/agentproto/spindle/agent/v1/agent.proto new file mode 100644 --- /dev/null +++ b/spindle/agentproto/spindle/agent/v1/agent.proto @@ -0,0 +1,110 @@ +syntax = "proto3"; + +package spindle.agent.v1; + +import "buf/validate/validate.proto"; + +option go_package = "tangled.org/core/spindle/agentproto/gen;agentv1"; + +message Hello { + uint32 protocol_version = 1; + string agent_version = 2; + string boot_id = 3; + string nix_version = 4; +} + +message Init { + string job_id = 1; + repeated string cache_trusted_public_keys = 2; + uint32 cache_read_proxy_port = 3; + uint32 cache_upload_proxy_port = 4; + uint32 dns_proxy_port = 5; +} + +message ExecStart { + repeated string argv = 1; + repeated string env = 2; + string cwd = 3; + string user = 4; + uint32 timeout_seconds = 5; +} + +message ExecStdout { + string data = 1; +} + +message ExecStderr { + string data = 1; +} + +message ExecExit { + int32 exit_code = 1; + string error = 2; + // set when the guest killed the step on its own timeout timer, so the host + // can classify it as a timeout rather than inferring failure from exit_code. + bool timed_out = 3; +} + +message ActivateConfig { + string config_key = 1; + string base_config_hash = 2; + string user_config = 3; + string toplevel = 4; + uint32 timeout_seconds = 5; +} + +message ActivateConfigResult { + string config_key = 1; + string toplevel = 2; + string error = 3; +} + +message BuiltPaths { + repeated string paths = 1; + string reason = 2; +} + +message CacheDrain { + uint32 timeout_seconds = 1; +} + +message CacheDrainResult { + string error = 1; + uint32 cache_queued = 2; + uint32 cache_active = 3; + uint32 cache_uploaded = 4; + uint32 cache_failed = 5; +} + +message Poweroff {} + +message PoweroffResult { + string error = 1; +} + +message Message { + option (buf.validate.message).oneof = { + fields: [ + "hello", "init", "exec_start", "exec_stdout", "exec_stderr", "exec_exit", + "activate_config", "activate_config_result", "built_paths", "cache_drain", + "cache_drain_result", "poweroff", "poweroff_result" + ], + required: true + }; + + string id = 1 [(buf.validate.field).string.min_len = 1]; + + Hello hello = 2; + Init init = 3; + ExecStart exec_start = 4; + ExecStdout exec_stdout = 5; + ExecStderr exec_stderr = 6; + ExecExit exec_exit = 7; + ActivateConfig activate_config = 8; + ActivateConfigResult activate_config_result = 9; + BuiltPaths built_paths = 10; + CacheDrain cache_drain = 11; + CacheDrainResult cache_drain_result = 12; + Poweroff poweroff = 13; + PoweroffResult poweroff_result = 14; +} diff --git a/shuttle/src/gen/spindle/agent/v1/spindle.agent.v1.rs b/shuttle/src/gen/spindle/agent/v1/spindle.agent.v1.rs new file mode 100644 --- /dev/null +++ b/shuttle/src/gen/spindle/agent/v1/spindle.agent.v1.rs @@ -0,0 +1,146 @@ +// @generated +// This file is @generated by prost-build. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Hello { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, + #[prost(string, tag = "2")] + pub agent_version: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub boot_id: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub nix_version: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Init { + #[prost(string, tag = "1")] + pub job_id: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "2")] + pub cache_trusted_public_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(uint32, tag = "3")] + pub cache_read_proxy_port: u32, + #[prost(uint32, tag = "4")] + pub cache_upload_proxy_port: u32, + #[prost(uint32, tag = "5")] + pub dns_proxy_port: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStart { + #[prost(string, repeated, tag = "1")] + pub argv: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, repeated, tag = "2")] + pub env: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub cwd: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub user: ::prost::alloc::string::String, + #[prost(uint32, tag = "5")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStdout { + #[prost(string, tag = "1")] + pub data: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStderr { + #[prost(string, tag = "1")] + pub data: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecExit { + #[prost(int32, tag = "1")] + pub exit_code: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + /// set when the guest killed the step on its own timeout timer, so the host + /// can classify it as a timeout rather than inferring failure from exit_code. + #[prost(bool, tag = "3")] + pub timed_out: bool, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ActivateConfig { + #[prost(string, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub base_config_hash: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub user_config: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub toplevel: ::prost::alloc::string::String, + #[prost(uint32, tag = "5")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ActivateConfigResult { + #[prost(string, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub toplevel: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BuiltPaths { + #[prost(string, repeated, tag = "1")] + pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "2")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheDrain { + #[prost(uint32, tag = "1")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheDrainResult { + #[prost(string, tag = "1")] + pub error: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub cache_queued: u32, + #[prost(uint32, tag = "3")] + pub cache_active: u32, + #[prost(uint32, tag = "4")] + pub cache_uploaded: u32, + #[prost(uint32, tag = "5")] + pub cache_failed: u32, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Poweroff {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PoweroffResult { + #[prost(string, tag = "1")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Message { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub hello: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub init: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub exec_start: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub exec_stdout: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub exec_stderr: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub exec_exit: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub activate_config: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub activate_config_result: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub built_paths: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub cache_drain: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub cache_drain_result: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub poweroff: ::core::option::Option, + #[prost(message, optional, tag = "14")] + pub poweroff_result: ::core::option::Option, +} +// @@protoc_insertion_point(module) -- tangled.sh