diff --git a/.gitignore b/.gitignore index 86a919b8..3b80065c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ tmp *.db-* .bin/ appview/pages/static/* +spindle/spindle +spindle/spindle-microvm-run result !.gitkeep !appview/pages/static/topbar-search.js diff --git a/Cargo.lock b/Cargo.lock index 860c0dfd..14f07b6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -116,6 +116,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "antlr4rust" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093d520274bfff7278d776f7ea12981a0a0a6f96db90964658e0f38fc6e9a6a6" +dependencies = [ + "better_any", + "bit-set", + "byteorder", + "lazy_static", + "murmur3", + "once_cell", + "parking_lot", + "typed-arena", + "uuid", +] + [[package]] name = "anyhow" version = "1.0.102" @@ -271,6 +288,27 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "better_any" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4372b9543397a4b86050cc5e7ee36953edf4bac9518e8a774c2da694977fb6e4" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.11.1" @@ -657,6 +695,24 @@ dependencies = [ "shlex", ] +[[package]] +name = "cel" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca1e5eda1b0f8476181bed1bfc9232a91d62ff0b9f1bc0e48afff3cbcb5b0b5c" +dependencies = [ + "antlr4rust", + "base64", + "chrono", + "lazy_static", + "nom", + "paste", + "regex", + "serde", + "serde_json", + "thiserror 1.0.69", +] + [[package]] name = "census" version = "0.4.2" @@ -1282,6 +1338,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -2262,6 +2324,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "miette" version = "7.6.0" @@ -2339,6 +2410,21 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmur3" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a198f9589efc03f544388dfc4a19fe8af4323662b62f598b8dcfdac62c14771c" +dependencies = [ + "byteorder", +] + [[package]] name = "murmurhash32" version = "0.3.1" @@ -2372,6 +2458,19 @@ dependencies = [ "web-time", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + [[package]] name = "nom" version = "7.1.3" @@ -2446,6 +2545,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-float" version = "5.3.0" @@ -2569,6 +2677,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "phf" version = "0.11.3" @@ -2737,6 +2856,124 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-protovalidate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3807965edce2730437ec19f99eae81064d152421c4c875a3e8f9e80cf2dd234" +dependencies = [ + "cel", + "chrono", + "fluent-uri", + "prost", + "prost-protovalidate-types", + "prost-reflect", + "prost-types", + "regex", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "prost-protovalidate-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddb5c8151aa3da0ea9ba5d91b170c7763b6c7c7352c3643d57b2b28de29afde9" +dependencies = [ + "prost", + "prost-build", + "prost-reflect", + "prost-reflect-build", + "prost-types", + "thiserror 2.0.18", +] + +[[package]] +name = "prost-reflect" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590aa145fee8f7a26b5a6055365e7c5e89a5c1caae9869de76ec0ee73181a2f9" +dependencies = [ + "base64", + "prost", + "prost-reflect-derive", + "prost-types", + "serde", + "serde-value", +] + +[[package]] +name = "prost-reflect-build" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8214ae2c30bbac390db0134d08300e770ef89b6d4e5abf855e8d300eded87e28" +dependencies = [ + "prost-build", + "prost-reflect", +] + +[[package]] +name = "prost-reflect-derive" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b6d90e29fa6c0d13c2c19ba5e4b3fb0efbf5975d27bcf4e260b7b15455bcabe" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost", +] + [[package]] name = "quick_cache" version = "0.6.22" @@ -3247,6 +3484,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -3422,6 +3669,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shuttle" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "nix", + "once_cell", + "prost", + "prost-protovalidate", + "prost-reflect", + "serde", + "serde_json", + "tempfile", + "tokio", + "tokio-vsock", + "tracing", + "tracing-subscriber", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -3706,7 +3973,7 @@ checksum = "dfadb8526b6da90704feb293b0701a6aae62ea14983143344be2dc5ce30f1d82" dependencies = [ "fnv", "nom", - "ordered-float", + "ordered-float 5.3.0", "serde", "serde_json", ] @@ -4016,6 +4283,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-vsock" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b319ef9394889dab2e1b4f0085b45ba11d0c79dc9d1a9d1afc057d009d0f1c7" +dependencies = [ + "bytes", + "futures", + "libc", + "tokio", + "vsock", +] + [[package]] name = "toml" version = "0.9.12+spec-1.1.0" @@ -4237,6 +4517,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typeid" version = "1.0.3" @@ -4370,6 +4656,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsock" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205" +dependencies = [ + "libc", + "nix", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/Cargo.toml b/Cargo.toml index 123a783f..c21e2de8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["bobbin/crates/*"] +members = ["bobbin/crates/*", "shuttle"] exclude = ["sites"] [workspace.package] @@ -14,6 +14,7 @@ authors = [ "oppiliappan ", "Anirudh Oppiliappan ", "eti ", + "dawn ", ] [workspace.lints.rust] diff --git a/appview/pipelines/logs.go b/appview/pipelines/logs.go index 7a4c78b2..172e501e 100644 --- a/appview/pipelines/logs.go +++ b/appview/pipelines/logs.go @@ -2,12 +2,14 @@ package pipelines import ( "html/template" + "path" "regexp" "strings" terminal "github.com/buildkite/terminal-to-html/v3" "github.com/gorilla/websocket" "tangled.org/core/appview/pages/markup" + "tangled.org/core/hostutil" ) // matches any ANSI escape sequence: ESC [ m @@ -79,10 +81,11 @@ func ReadLogs(conn *websocket.Conn, ch chan LogEvent) { } } -func SpindleURL(dev bool, spindle, knot, rkey, workflow string) string { - scheme := "wss" - if dev { - scheme = "ws" +func SpindleURL(spindle, knot, rkey, workflow string) string { + url, err := hostutil.EnsureWsScheme(spindle) + if err != nil { + return "" } - return scheme + "://" + strings.Join([]string{spindle, "logs", knot, rkey, workflow}, "/") + + return url + path.Join("/logs", knot, rkey, workflow) } diff --git a/appview/pipelines/pipelines.go b/appview/pipelines/pipelines.go index 76149a64..a22de03f 100644 --- a/appview/pipelines/pipelines.go +++ b/appview/pipelines/pipelines.go @@ -18,6 +18,7 @@ import ( "tangled.org/core/appview/pages" "tangled.org/core/appview/reporesolver" "tangled.org/core/eventconsumer" + "tangled.org/core/hostutil" "tangled.org/core/idresolver" "tangled.org/core/orm" "tangled.org/core/rbac" @@ -222,7 +223,11 @@ func (p *Pipelines) Logs(w http.ResponseWriter, r *http.Request) { return } - url := SpindleURL(p.config.Core.Dev, spindle, knot, rkey, workflow) + url := SpindleURL(spindle, knot, rkey, workflow) + if url == "" { + http.Error(w, "invalid spindle hostname", http.StatusBadRequest) + return + } l = l.With("url", url) clientConn, err := upgrader.Upgrade(w, r, nil) @@ -421,11 +426,17 @@ func (p *Pipelines) Cancel(w http.ResponseWriter, r *http.Request) { return } + hostname, noTLS, err := hostutil.ParseHostname(spindle) + if err != nil { + http.Error(w, "invalid spindle hostname", http.StatusBadRequest) + return + } + spindleClient, err := p.oauth.ServiceClient( r, - oauth.WithService(f.Spindle), + oauth.WithService(hostname), oauth.WithLxm(tangled.PipelineCancelPipelineNSID), - oauth.WithDev(p.config.Core.Dev), + oauth.WithDev(noTLS), oauth.WithTimeout(time.Second*30), // workflow cleanup usually takes time ) diff --git a/appview/pipelines/ssh/tui.go b/appview/pipelines/ssh/tui.go index 01cc44cd..66f7d9d6 100644 --- a/appview/pipelines/ssh/tui.go +++ b/appview/pipelines/ssh/tui.go @@ -113,7 +113,7 @@ func (m *pipelineModel) connectCmd(workflow string) tea.Cmd { if !ok || len(ws.Data) == 0 { return logDoneMsg{workflow: workflow} } - url := pipelines.SpindleURL(m.server.config.Core.Dev, ws.Data[0].Spindle, m.pipeline.Knot, m.pipeline.Rkey, workflow) + url := pipelines.SpindleURL(ws.Data[0].Spindle, m.pipeline.Knot, m.pipeline.Rkey, workflow) conn, _, err := websocket.DefaultDialer.Dial(url, nil) if err != nil { return logDoneMsg{workflow: workflow, err: fmt.Errorf("connecting to spindle: %w", err)} diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go index 5cf824ba..df81c304 100644 --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -56,7 +56,7 @@ func Knotstream(ctx context.Context, c *config.Config, d *db.DB, acl *knotacl.Se return bootstrapStream( ctx, "knotstream", ec.KindKnot, hosts, c.Redis.Addr, - c.Knotstream, c.Core.Dev, + c.Knotstream, knotIngester(d, acl, enforcer, posthog, notifier, c.Core.Dev, c, cfClient), ), nil } diff --git a/appview/state/spindlestream.go b/appview/state/spindlestream.go index ebe2643a..6cfd5aba 100644 --- a/appview/state/spindlestream.go +++ b/appview/state/spindlestream.go @@ -33,7 +33,7 @@ func Spindlestream(ctx context.Context, c *config.Config, d *db.DB, enforcer *rb return bootstrapStream( ctx, "spindlestream", ec.KindSpindle, hosts, c.Redis.Addr, - c.Spindlestream, c.Core.Dev, + c.Spindlestream, spindleIngester(d, pn), ), nil } diff --git a/appview/state/spindlestream_test.go b/appview/state/spindlestream_test.go index 05cccf57..8b515a19 100644 --- a/appview/state/spindlestream_test.go +++ b/appview/state/spindlestream_test.go @@ -55,7 +55,7 @@ func TestColdStart_SpindleEventsRebuildPipelineStatuses(t *testing.T) { }) srv := httptest.NewServer(mux) t.Cleanup(srv.Close) - source := ec.Source{Kind: "test", Host: strings.TrimPrefix(srv.URL, "http://")} + source := ec.Source{Kind: "test", Host: strings.TrimPrefix(srv.URL, "http://"), NoTLS: true} appviewDB, err := db.Make(ctx, filepath.Join(t.TempDir(), "appview.db")) if err != nil { @@ -72,7 +72,6 @@ func TestColdStart_SpindleEventsRebuildPipelineStatuses(t *testing.T) { QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: &cursor.MemoryStore{}, - URLFunc: ec.DefaultURL(true), Logger: logger, } c := ec.NewConsumer(cfg) diff --git a/appview/state/streams.go b/appview/state/streams.go index 59fabf68..4e7c748a 100644 --- a/appview/state/streams.go +++ b/appview/state/streams.go @@ -17,7 +17,6 @@ func bootstrapStream( hosts []string, redisAddr string, streamCfg config.ConsumerConfig, - dev bool, processFn ec.ProcessFunc, ) *ec.Consumer { logger := log.SubLogger(log.FromContext(ctx), name) @@ -41,7 +40,6 @@ func bootstrapStream( WorkerCount: streamCfg.WorkerCount, QueueSize: streamCfg.QueueSize, Logger: logger, - URLFunc: ec.DefaultURL(dev), CursorStore: &cursorStore, }) } diff --git a/blog/posts/spindle-microvm.md b/blog/posts/spindle-microvm.md new file mode 100644 index 00000000..e010fa41 --- /dev/null +++ b/blog/posts/spindle-microvm.md @@ -0,0 +1,277 @@ +--- +atroot: true +template: +slug: spindle-microvm +title: How the microVM engine comes together +subtitle: spindle has a microVM engine now! +date: 2026-06-16 +image: https://assets.tangled.network/blog/seed.png +authors: + - name: dawn + email: dawn@tangled.org + handle: ptr.pet +--- + +Since launching, [spindle](/ci) has run your CI inside Docker containers created +with nixery. That's been mostly okay if you are doing simple things, but if you +wanted to do anything more outside the box (maybe you wanted some services, or +to build & test containers inside), or if you wanted to use Nix inside it (which +is rough :P), it wouldn't meet your needs. That changes today! + +spindle gains a microVM engine. Each workflow gets its own little virtual +machine. You get a full environment inside your workflows that you can do +whatever you want with without any of the roughness of nixery containers. +Alongside this, you also get the ability to configure *services* that a workflow +will have (on the NixOS image), so that means you can easily have postgres, +Docker, and so on that will be alive through the workflow. + +## what's in a microVM + +A microVM is just a VM with most of the boring parts removed. There's no BIOS, +no PCI bus to probe, no emulated graphics card, none of the slow legacy stuff a +normal QEMU machine drags along for example. You get virtio devices and not much +else, which means it boots very quickly and uses very little memory. Right now +QEMU is the only runner we support, but the engine is written so that other +runners (firecracker for example) can slot in later. + +Inside the guest there's a small piece of software we call the agent. Spindle +never SSHes in or runs commands "from the outside"; instead the agent dials back +to spindle over vsock the moment it boots, says hello, and from then on every +step of your workflow is sent to it as a message. The agent runs the command as +an unprivileged user, streams stdout and stderr back, and reports the exit code. +The host side of this lives in +[`spindle`](https://tangled.org/tangled.org/core/tree/master/spindle/engines/microvm/agent.go) +and the guest side is a little Rust binary called +[`shuttle`](https://tangled.org/tangled.org/core/tree/master/shuttle). +(`shuttle` implements +[`agentproto`](https://tangled.org/tangled.org/core/tree/master/spindle/) which +is the protocol used by `spindle`. Technically speaking anyone could implement +this and, assuming side effects hold, you could have your own agent!) + +## two kinds of images + +There are two "flavours" of image you can boot, and they're aimed at fairly +different people. + +The first is **NixOS images**. These are the interesting ones: because the whole +guest is built with Nix, you can configure it from your workflow file directly. +Things like `dependencies`, `services`, `virtualisation` (e.g. Docker), +`registry` and `caches` are all written right there in the YAML, and the guest +agent builds and activates that config before any of your steps run. If we've +built that exact base plus config before, spindle can just hand the guest a +store path to realize (fetching from whatever cache `spindle` has configured) +instead of rebuilding it, so the second run is quick. + +The second is **non-NixOS images**, which today just means Alpine, but can be +anything. You don't get the workflow-level NixOS config here (there's no NixOS +to configure), but if Nix happens to exist inside the image, like it does in our +Alpine one, it can still talk to the spindle Nix cache just fine. + +### example nixos workflow + +If you've used spindle before this will look familiar, it's the same manifest you +already know, just with a few extra keys that the NixOS image understands. Here's +a workflow that needs postgres to test against and Docker to build an image: + +```yaml +# .tangled/workflows/test.yaml +engine: microvm + +when: + - event: ["push", "pull_request"] + branch: ["master"] + +image: nixos + +dependencies: + - go + - github:nixos/nixpkgs#hello + +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + +caches: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true + +steps: + - name: run tests + environment: + PGHOST: /run/postgresql + command: | + docker build -t app . + psql -c "select 1" + go test ./... +``` + +`dependencies` are packages that are added to `environment.systemPackages` (so, +`PATH`). A bare name like `go` is looked up in nixpkgs (same as regular +spindle), but you can also point at any flake with the `flakeref#attr` syntax, +so `github:nixos/nixpkgs#hello` pulls `hello` straight out of that flake. +`registry` is how you remap the global refs: here we pin `nixpkgs` to +`nixos-unstable`, so now the bare `go` above resolves from unstable. You can +alias your own flakes the same way (`myflake: github:me/x`, then `myflake#tool` +in `dependencies`). `caches` is a map of binary cache URL to its trusted public +key, and they get wired into the read proxy (more on that later), so the guest +can substitute prebuilt paths from them instead of building everything from +scratch. + +`services` and `virtualisation` are the interesting parts: they're passed +straight through to NixOS, so anything you could write in a NixOS config you can +write here. `services.postgresql.enable` brings postgres up before any of your +steps run. Since steps run as the `spindle-workflow` user, naming a database +after that user with `ensureDBOwnership` is the easy path to a working db - +postgres peer auth maps the unix user straight to the matching role, so `psql` +connects over the socket with no password and no extra setup (this name-matching +is a NixOS requirement for `ensureDBOwnership`, if you want a differently named +db you'd grant access yourself). `virtualisation.docker: true` is shorthand for +`virtualisation.docker.enable = true`, which gets you a real Docker daemon +inside the VM. By the time your first step runs, postgres is listening and the +Docker socket is there, no sidecar dance, it's just part of the machine. + +(`true` works as shorthand for `.enable = true` anywhere an `enable` option +exists, so most "just turn this on" services are a one-liner!) + +## building the images + +Image builds are done with Nix. For NixOS we lean on +[microvm.nix](https://github.com/microvm-nix/microvm.nix) and layer our own bits +on top (stripping down kernel modules, configuring users, etc.). For Alpine +there's a smallish Nix definition that fetches the kernel, the initrd and the +kernel modules, sets up an init script that configures the machine on boot, +copies in the dependencies we want (`nix`, `git`, etc.) and compresses the whole +rootfs into a squashfs. + +None of this *has* to be Nix, though. As far as spindle is concerned an image is +valid as long as a few things hold: a guest agent (that implements `agentproto`) +is present and gets started on boot, a `spindle-workflow` user exists, and the +work directory is set up at `/workspace`. That can be built however you like. + +## finding an image + +Every built image ships a `spec.json` next to its artifacts. The spec is the +whole contract: where the kernel and initrd and read-only store disk live, the +boot args, how much memory and how many vCPUs to give it, the shell to run steps +in, the writable volumes, the network interfaces, and the runner-specific knobs +(machine type, CPU, extra QEMU args). NixOS images also carry a `baseConfigHash` +identifying the base config baked in (this is the hash of +`nixosSystem.config.system.build.toplevel.outPath`). + +A workflow picks an image with the `image` key at the top level. The name is +matched literally against what's on disk, we look for a directory called +`` with a `spec.json` in it, then fall back to a flat `.json`. The +nice property here is that resolution depends *only* on the name and what's on +disk, never on the host doing the resolving, so the same workflow resolves to +the same image on every spindle. If an operator keeps multiple arches side by +side they can name them `nixos-x86_64`, `alpine-aarch64` and so on (that suffix +is just part of the name, it's not handled specially). If you want, for example, +`nixos` to work, you can just symlink `nixos` to `nixos-x86_64`. + +Right before launch we double-check the referenced files actually exist +and that the host has the tools we need: `mkfs.ext4` for the volumes, the +QEMU binary for the spec's arch, `/dev/kvm` and `/dev/vhost-vsock`, plus +the `ip` / `mount` / `slirp4netns` / `unshare` toolchain if the image +wants networking. + +## the life of a workflow + +A workflow moves through a handful of stages: it gets parsed and its +image resolved, it waits for a slot, it gets set up, its steps ran, and +then everything is torn down. + +The waiting bit matters a lot. Each image declares how much memory, how many +vCPUs and how much disk it needs, and a workflow has to acquire a slot from a +resource scheduler before anything boots. The scheduler is work-conserving with +aging and per-user fairness, so one person submitting a hundred jobs won't +starve everyone else, and slots don't sit idle if there's work that fits in the +budget. + +Once a slot is acquired, we do the setup. Spindle allocates a random vsock CID +for the guest and registers it with the agent hub. It creates the per-workflow +work directory, starts the two cache proxies (more on those later), then creates +the VM: writable volumes become sparse files formatted ext4, the store disk is +attached read-only, and QEMU is started with `-sandbox on`, `-nodefaults`, no +display, no monitor, etc. with serial / `virtio_console` output to a log file +and a QMP socket for control. + +Then we wait for the machine. We poll QMP until QEMU says the guest is running, +then wait for the agent's handshake to arrive over vsock from the CID we expect. +The agent tells us its protocol and versions, and spindle sends back the job id, +the trusted cache public keys and the cache proxy ports, NixOS config if already +cached... From there steps run one at a time as `$shell -lc `, as the +unprivileged workflow user in `/workspace/repo`, with the right environment and +any unlocked secrets. + +Timeouts are cooperative: we work out a deadline from the workflow timeout +and ship it to the guest, with a little grace on our side so the guest +gets a chance to report the timeout itself rather than us just yanking the +machine out from under it. And if the VM crashes mid-step we tail the +serial and QEMU logs into the step's stderr, because "guest agent +connection lost: EOF" is a genuinely useless thing to read at 2am. + +Teardown is the same whether the workflow passed, failed or timed out: +drain any pending Nix cache uploads, ask the agent to power off, wait for +QEMU to exit (falling back to a QMP `system_powerdown`, and finally a +kill if it's being stubborn), then close the proxies and remove the work +directory. + +## locking down the network + +A VM that can reach the host's local network is a VM that can reach things it +has no business reaching. So QEMU doesn't run in the host's network namespace at +all. We `unshare` into fresh user, net and mount namespaces first. Inside that +namespace a small wrapper bind-mounts a resolv.conf pointing at the slirp DNS +and installs blackhole routes for every special-use IP range (RFC 6890, so +private networks, link-local, loopback, etc.) before it execs QEMU. +`slirp4netns` then provides outbound connectivity for that namespace, with +`--disable-host-loopback`, sandbox and seccomp all on. The guest itself sits +behind a *second* layer of QEMU user-mode networking inside that namespace. All +of this is done without needing any privileges! + +## budgets and cgroups + +The scheduler's budget is bookkeeping on its own, it tracks what it's handed +out, and the runner (QEMU) will ensure that a workflow only gets those. But +optionally the whole thing (QEMU and slirp4netns both) gets placed in a +per-workflow cgroup with memory, swap etc. limits, which is an extra +enforcement layer on top. A nice side effect is when the cgroup OOM-kills the VM +we can see that it was an OOM and report it as such, instead of surfacing it as +a generic crash and leaving you guessing. + +The spindle itself also gets a cgroup, which means that in a host OOM situation, +it should be the workflows that die first, not the spindle itself. + +## the nix cache, both ways + +The two proxies I mentioned during setup are how the guest talks to spindle's +Nix cache, and they run on the host so the guest never needs credentials or +direct network access to do it. Like the agent, they also use vsock to +communicate with the spindle. + +The read proxy fans out to the configured substituters plus any caches you +listed in your workflow, so when the guest needs to realize a store path it asks +the proxy and the proxy fetches it. The request is sent concurrently to the read +caches, so the one that answers it first wins. + +The upload proxy goes the other way: any path built inside the guest gets pushed +back out to spindle's Nix cache (if one is configured), so the next workflow +that needs it doesn't have to build it again. Any paths that already exist on +any of the configured read caches won't be uploaded. Built paths are queued by +the agent and are immediately uploaded. If any paths are still left when we +reach VM teardown, the workflow will wait until everything is uploaded. + +## in the future + +todo + +Feel free to come and ask any questions you might have on https://chat.tangled.sh! diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 00000000..7286491c --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,11 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: . + opt: + - paths=import + - module=tangled.org/core + - local: protoc-gen-prost + out: shuttle/src/gen + opt: + - bytes=. diff --git a/buf.lock b/buf.lock new file mode 100644 index 00000000..709ae023 --- /dev/null +++ b/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/bufbuild/protovalidate + commit: 50325440f8f24053b047484a6bf60b76 + digest: b5:74cb6f5c0853c3c10aafc701614194bbd63326bdb8ef4068214454b8894b03ba4113e04b3a33a8321cdf05336e37db4dc14a5e2495db8462566914f36086ba31 diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 00000000..3c20343f --- /dev/null +++ b/buf.yaml @@ -0,0 +1,5 @@ +version: v2 +modules: + - path: spindle/agentproto +deps: + - buf.build/bufbuild/protovalidate diff --git a/cmd/spindle-microvm-run/main.go b/cmd/spindle-microvm-run/main.go new file mode 100644 index 00000000..5ab4181b --- /dev/null +++ b/cmd/spindle-microvm-run/main.go @@ -0,0 +1,26 @@ +//go:build linux + +package main + +import ( + "context" + "log/slog" + "os" + + tlog "tangled.org/core/log" +) + +func main() { + cmd := SpindleMicroVMRunCommand() + + logger := tlog.New("spindle-microvm-run") + slog.SetDefault(logger) + + ctx := context.Background() + ctx = tlog.IntoContext(ctx, logger) + + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) + os.Exit(-1) + } +} diff --git a/cmd/spindle-microvm-run/main_linux.go b/cmd/spindle-microvm-run/main_linux.go new file mode 100644 index 00000000..ba835385 --- /dev/null +++ b/cmd/spindle-microvm-run/main_linux.go @@ -0,0 +1,357 @@ +//go:build linux + +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "net" + "os" + "time" + + "github.com/mdlayher/vsock" + "github.com/urfave/cli/v3" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engines/microvm" +) + +func SpindleMicroVMRunCommand() *cli.Command { + return &cli.Command{ + Name: "spindle-microvm-run", + Usage: "launch the Spindle base microVM and run one command over vsock", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "image-spec", + Sources: cli.EnvVars("SPINDLE_MICROVM_IMAGE_SPEC"), + Usage: "path to microVM image spec JSON", + }, + &cli.StringFlag{ + Name: "mkfs-ext4", + Usage: "override mkfs.ext4 binary", + }, + &cli.StringFlag{ + Name: "work-dir", + Usage: "directory for per-run disks and sockets", + }, + &cli.UintFlag{ + Name: "cid", + Usage: "guest vsock CID; defaults to a random high CID", + }, + &cli.UintFlag{ + Name: "port", + Value: 10240, + Usage: "host vsock port to listen on", + }, + &cli.UintFlag{ + Name: "memory-mib", + Usage: "override the guest memory size in MiB (defaults to the image spec)", + }, + &cli.BoolFlag{ + Name: "disable-kvm", + Usage: "run without -enable-kvm even if /dev/kvm is available", + }, + &cli.BoolFlag{ + Name: "dev", + Usage: "enable dev mode (allows host network access, disables SSL verification)", + }, + &cli.DurationFlag{ + Name: "qmp-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qmp to become ready", + }, + &cli.DurationFlag{ + Name: "accept-timeout", + Value: 15 * time.Second, + Usage: "how long to wait for the guest agent after qemu starts", + }, + &cli.DurationFlag{ + Name: "exec-timeout", + Value: 30 * time.Second, + Usage: "timeout for the guest command", + }, + &cli.DurationFlag{ + Name: "cache-drain-timeout", + Value: 5 * time.Minute, + Usage: "how long to wait for queued cache uploads after the guest command exits", + }, + &cli.DurationFlag{ + Name: "shutdown-timeout", + Value: 10 * time.Second, + Usage: "how long to wait for qemu to exit after guest powerdown", + }, + &cli.StringFlag{ + Name: "cwd", + Usage: "guest working directory", + }, + &cli.StringSliceFlag{ + Name: "cache-read-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_READ_URLS"), + Usage: "Nix binary cache URL to pass to the guest; repeatable", + }, + &cli.StringSliceFlag{ + Name: "cache-trusted-public-key", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS"), + Usage: "Nix binary cache public key to trust in the guest; repeatable", + }, + &cli.StringFlag{ + Name: "cache-upload-url", + Sources: cli.EnvVars("SPINDLE_NIX_CACHE_UPLOAD_URL"), + Usage: "optional cache upload URL for guest-built store paths", + }, + &cli.StringFlag{ + Name: "activate-config", + Usage: "JSON user config to activate before exec (e.g. '{\"services\":{\"openssh\":{\"enable\":true}}}')", + }, + &cli.StringFlag{ + Name: "db", + Usage: "path to sqlite database for config cache", + }, + }, + Action: runMicroVMRunDev, + } +} + +func runMicroVMRunDev(ctx context.Context, cmd *cli.Command) error { + imageSpecPath := cmd.String("image-spec") + if imageSpecPath == "" { + return fmt.Errorf("--image-spec or SPINDLE_MICROVM_IMAGE_SPEC is required") + } + + imageSpec, err := microvm.LoadImageSpec(imageSpecPath) + if err != nil { + return err + } + + port := uint32(cmd.Uint("port")) + // tell the guest which host vsock port to dial back on. shuttle reads the + // cmdline params this is so we can run multiple of this process + // concurrently, because otherwise it listens on a specific vsock port, and + // we cant bind to the same port twice... + imageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", imageSpec.BootArgs, port) + if mib := cmd.Uint("memory-mib"); mib > 0 { + imageSpec.MemoryMiB = int(mib) + } + ln, err := vsock.Listen(port, nil) + if err != nil { + return fmt.Errorf("listen on vsock port %d: %w", port, err) + } + defer ln.Close() + + vm, err := microvm.StartVM(ctx, microvm.VMConfig{ + Image: imageSpec, + BootTimeout: cmd.Duration("qmp-timeout"), + CID: uint32(cmd.Uint("cid")), + EnableKVM: !cmd.Bool("disable-kvm"), + MkfsExt4: cmd.String("mkfs-ext4"), + WorkDir: cmd.String("work-dir"), + Dev: cmd.Bool("dev"), + }, slog.Default()) + if err != nil { + return err + } + defer vm.Close() + + logs := vm.Logs() + fmt.Fprintf(os.Stderr, "microvm started: cid=%d work-dir=%s serial-log=%s qemu-log=%s\n", + vm.CID(), + vm.WorkDir(), + logs.Serial, + logs.Extra["qemu"], + ) + + logger := slog.Default() + + if cmd.Duration("accept-timeout") > 0 { + if err := ln.SetDeadline(time.Now().Add(cmd.Duration("accept-timeout"))); err != nil { + return fmt.Errorf("set accept deadline: %w", err) + } + } + + argv := cmd.Args().Slice() + if len(argv) == 0 { + argv = []string{"/run/current-system/sw/bin/echo", "hello-from-spindle"} + } + jobID := "spindle-microvm-run" + execID := "dev-1" + + fmt.Fprintf(os.Stderr, "listening for agent on %s\n", ln.Addr()) + conn, err := acceptExpectedVsockConn(ln, vm.CID(), logger) + if err != nil { + return fmt.Errorf("accept agent connection: %w", err) + } + defer conn.Close() + + upstreams, err := microvm.BuildCacheUpstreams(cmd.StringSlice("cache-read-url"), nil) + if err != nil { + return fmt.Errorf("build cache upstreams: %w", err) + } + + var readCache *microvm.ReadCacheProxy + if len(cmd.StringSlice("cache-read-url")) > 0 { + var err error + readCache, err = microvm.StartReadCacheProxy(ctx, vm.CID(), upstreams, logger) + if err != nil { + return fmt.Errorf("start read cache proxy: %w", err) + } + defer readCache.Close() + } + + var uploadCache *microvm.UploadCacheProxy + if cmd.String("cache-upload-url") != "" { + var err error + uploadCache, err = microvm.StartUploadCacheProxy(ctx, vm.CID(), cmd.String("cache-upload-url"), upstreams, logger) + if err != nil { + return fmt.Errorf("start upload cache proxy: %w", err) + } + defer uploadCache.Close() + } + dnsProxy, err := microvm.StartDNSProxy(ctx, vm.CID(), logger) + if err != nil { + return fmt.Errorf("start dns proxy: %w", err) + } + defer dnsProxy.Close() + + session := microvm.NewAgentSession(conn, logger) + + initCtx, cancelInit := context.WithTimeout(ctx, 30*time.Second) + defer cancelInit() + if err := session.Init(initCtx, &agentv1.Init{ + JobId: jobID, + CacheTrustedPublicKeys: cmd.StringSlice("cache-trusted-public-key"), + CacheReadProxyPort: readCache.Port(), + CacheUploadProxyPort: uploadCache.Port(), + DnsProxyPort: dnsProxy.Port(), + }); err != nil { + return fmt.Errorf("init agent: %w", err) + } + + execCtx := ctx + if cmd.Duration("exec-timeout") > 0 { + var cancel context.CancelFunc + execCtx, cancel = context.WithTimeout(ctx, cmd.Duration("exec-timeout")) + defer cancel() + } + + if cmd.String("activate-config") != "" { + actCtx := execCtx + baseHash, err := microvm.BaseConfigHash(imageSpec) + if err != nil { + return fmt.Errorf("calculate base config hash: %w", err) + } + + var d *db.DB + var configKey string + var cachedToplevel string + if cmd.String("db") != "" { + d, err = db.Make(ctx, cmd.String("db")) + if err != nil { + return fmt.Errorf("failed to open database: %w", err) + } + defer d.Close() + + configKey, err = microvm.BuildConfigKey(imageSpec, cmd.String("activate-config")) + if err != nil { + return fmt.Errorf("calculate config key: %w", err) + } + + record, err := d.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("lookup config cache: %w", err) + } + } else { + cachedToplevel = record.Toplevel + fmt.Printf("realizing cached NixOS config %s\n", cachedToplevel) + } + } + + result, err := session.ActivateConfig(actCtx, "dev-activate", &agentv1.ActivateConfig{ + ConfigKey: configKey, + BaseConfigHash: baseHash, + UserConfig: cmd.String("activate-config"), + Toplevel: cachedToplevel, + }) + if err != nil { + return fmt.Errorf("activate config: %w", err) + } + fmt.Fprintf(os.Stderr, "activated config toplevel: %s\n", result.Toplevel) + + if d != nil && cachedToplevel == "" && result.Toplevel != "" && configKey != "" { + err = d.SaveNixOSToplevelCacheRecord(configKey, result.Toplevel) + if err != nil { + return fmt.Errorf("save config cache: %w", err) + } + } + } + + exitCode, err := session.Exec(execCtx, microvm.AgentExec{ + ID: execID, + ExecStart: &agentv1.ExecStart{ + Argv: argv, + Cwd: cmd.String("cwd"), + }, + Stdout: os.Stdout, + Stderr: os.Stderr, + }) + if err != nil { + return err + } + + if uploadCache != nil { + drainCtx := ctx + if cmd.Duration("cache-drain-timeout") > 0 { + var cancel context.CancelFunc + drainCtx, cancel = context.WithTimeout(ctx, cmd.Duration("cache-drain-timeout")) + defer cancel() + } + uploaded, err := session.Drain(drainCtx) + if err != nil { + return err + } + fmt.Printf("cache uploaded: %d\n", uploaded) + } + + // mirror the engine shutdown order: ask the agent to power off first, + // then fall back to qemu powerdown / kill + shutdownCtx, cancel := context.WithTimeout(context.Background(), cmd.Duration("shutdown-timeout")) + defer cancel() + poweredOff := false + if err := session.Poweroff(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "agent poweroff: %s\n", err) + } else if err := vm.WaitContext(shutdownCtx); err == nil { + poweredOff = true + } + if !poweredOff { + if err := vm.Shutdown(shutdownCtx); err != nil { + fmt.Fprintf(os.Stderr, "microvm shutdown fallback: %s\n", err) + } + } + + if exitCode != 0 { + return fmt.Errorf("guest command exited with code %d", exitCode) + } + return nil +} + +func acceptExpectedVsockConn(ln *vsock.Listener, allowedCID uint32, logger *slog.Logger) (net.Conn, error) { + for { + conn, err := ln.Accept() + if err != nil { + return nil, err + } + if allowedCID == 0 { + return conn, nil + } + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == allowedCID { + return conn, nil + } + remote := conn.RemoteAddr() + _ = conn.Close() + logger.Warn("dropped agent connection from unexpected cid", "remote", remote, "expected", allowedCID) + } +} diff --git a/cmd/spindle-microvm-run/unsupported.go b/cmd/spindle-microvm-run/unsupported.go new file mode 100644 index 00000000..cca69a7d --- /dev/null +++ b/cmd/spindle-microvm-run/unsupported.go @@ -0,0 +1,13 @@ +//go:build !linux + +package main + +import ( + "fmt" + "os" +) + +func main() { + fmt.Fprintf(os.Stderr, "spindle-microvm-run is only supported on Linux\n") + os.Exit(-1) +} diff --git a/cmd/spindle/main.go b/cmd/spindle/main.go index 083f29f6..c377a7ca 100644 --- a/cmd/spindle/main.go +++ b/cmd/spindle/main.go @@ -5,20 +5,39 @@ import ( "log/slog" "os" + "github.com/urfave/cli/v3" tlog "tangled.org/core/log" "tangled.org/core/spindle" ) func main() { + cmd := &cli.Command{ + Name: "spindle", + Usage: "spindle continuous integration runner", + Commands: []*cli.Command{ + Command(), + }, + DefaultCommand: "run", + } + logger := tlog.New("spindle") slog.SetDefault(logger) ctx := context.Background() ctx = tlog.IntoContext(ctx, logger) - err := spindle.Run(ctx) - if err != nil { - logger.Error("error running spindle", "error", err) + if err := cmd.Run(ctx, os.Args); err != nil { + logger.Error(err.Error()) os.Exit(-1) } } + +func Command() *cli.Command { + return &cli.Command{ + Name: "run", + Usage: "run the spindle server", + Action: func(ctx context.Context, cmd *cli.Command) error { + return spindle.Run(ctx) + }, + } +} diff --git a/docker-compose.yml b/docker-compose.yml index dcc0be8f..ad99a93c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,6 +50,7 @@ services: PDS_URL: http://pds:3000 OWNER_USER: alice KNOT_HOSTNAME: knot.tngl.boltless.dev + SPINDLE_HOSTNAME: spindle.tngl.boltless.dev volumes: - ./localinfra/scripts/init-accounts.sh:/init.sh:ro - init-state:/shared @@ -85,7 +86,7 @@ services: JETSTREAM_WS_URL: wss://pds.tngl.boltless.dev/xrpc/com.atproto.sync.subscribeRepos volumes: - jetstream-data:/data - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: pds: condition: service_healthy @@ -113,7 +114,7 @@ services: - knot-data:/home/git - knot-ssh-keys:/etc/ssh/keys - init-state:/shared:ro - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:5555/"] interval: 2s @@ -131,6 +132,57 @@ services: condition: service_completed_successfully networks: [tngl] + spindle: + build: + context: . + dockerfile: localinfra/spindle.Dockerfile + restart: unless-stopped + environment: + SPINDLE_SERVER_HOSTNAME: spindle.tngl.boltless.dev + SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 + SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db + SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev + SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe + SPINDLE_SERVER_DEV: "true" + SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev + SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db + SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev + SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images + SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays + SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" + SPINDLE_S3_LOG_BUCKET: "" + devices: + - /dev/vsock:/dev/vsock + - /dev/kvm:/dev/kvm + - /dev/vhost-vsock:/dev/vhost-vsock + - /dev/net/tun:/dev/net/tun + cap_add: + - NET_ADMIN + - SYS_ADMIN + security_opt: + - label=disable + - seccomp=unconfined + volumes: + - spindle-data:/var/lib/spindle + - spindle-logs:/var/log/spindle + - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro + - init-state:/shared:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] + interval: 2s + timeout: 2s + retries: 30 + start_period: 5s + depends_on: + plc: + condition: service_started + jetstream: + condition: service_started + init-accounts: + condition: service_completed_successfully + networks: [tngl] + knotmirror-tap: image: ghcr.io/bluesky-social/indigo/tap:sha-4f47add43060c27e8a37d9d76482ecddf001fcd8 # 0.1.10 restart: unless-stopped @@ -144,7 +196,7 @@ services: TAP_RESYNC_PARALLELISM: "10" TAP_RETRY_TIMEOUT: 60s volumes: - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: postgres: condition: service_started @@ -171,7 +223,7 @@ services: MIRROR_RESYNC_PARALLELISM: "4" volumes: - knotmirror-data:/data - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:7000/"] interval: 2s @@ -202,7 +254,8 @@ services: - ./blog/templates:/build/blog/templates:ro - ./blog/posts:/build/blog/posts:ro - ./appview/pages/static:/build/appview/pages/static - command: ["-i", "input.css", "-o", "appview/pages/static/tw.css", "--watch=always"] + command: + ["-i", "input.css", "-o", "appview/pages/static/tw.css", "--watch=always"] network_mode: none appview: @@ -226,7 +279,7 @@ services: - go-mod-cache:/go/mod - appview-data:/var/lib/appview - init-state:/shared:ro - - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro + - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro depends_on: redis: condition: service_started @@ -267,6 +320,8 @@ volumes: knot-data: knot-ssh-keys: knotmirror-data: + spindle-data: + spindle-logs: init-state: go-cache: go-mod-cache: diff --git a/docs/DOCS.md b/docs/DOCS.md index b63ec9c3..2fcf9aa1 100644 --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -760,7 +760,8 @@ simple format. They're located in the `.tangled/workflows` directory at the root of your repository, and are defined using YAML. -The fields are: +A workflow has a set of common fields that apply no matter +which engine you pick: - [Trigger](#trigger): A **required** field that defines when a workflow should be triggered. @@ -768,13 +769,15 @@ The fields are: engine a workflow should run on. - [Clone options](#clone-options): An **optional** field that defines how the repository should be cloned. -- [Dependencies](#dependencies): An **optional** field that - allows you to list dependencies you may need. - [Environment](#environment): An **optional** field that allows you to define environment variables. - [Steps](#steps): An **optional** field that allows you to define what steps should run in the workflow. +On top of these, each engine has its own options for things +like dependencies and images. See [Engines](#engines) for +the per-engine fields. + ### Trigger The first thing to add to a workflow is the trigger, which @@ -853,6 +856,11 @@ supported engines are: search for packages on https://search.nixos.org, and there's a pretty good chance the package(s) you're looking for will be there. + See [Nixery engine](#nixery-engine). +- `microvm`: Runs the whole workflow inside its own + microVM. Has configuration features for NixOS images + that will let you enable services, do Docker-in-VM, etc. + See [microVM engine](#microvm-engine). Example: @@ -860,6 +868,10 @@ Example: engine: "nixery" ``` +Each engine also adds its own workflow fields (dependencies, +images, services, and so on). These are documented under +[Engines](#engines). + ### Clone options When a workflow starts, the first step is to clone the @@ -891,41 +903,6 @@ clone: submodules: false ``` -### Dependencies - -Usually when you're running a workflow, you'll need -additional dependencies. The `dependencies` field lets you -define which dependencies to get, and from where. It's a -key-value map, with the key being the registry to fetch -dependencies from, and the value being the list of -dependencies to fetch. - -The registry URL syntax can be found [on the nix -manual](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-registry-add). - -Say you want to fetch Node.js and Go from `nixpkgs`, and a -package called `my_pkg` you've made from your own registry -at your repository at -`https://tangled.org/@example.com/my_pkg`. You can define -those dependencies like so: - -```yaml -dependencies: - # nixpkgs - nixpkgs: - - nodejs - - go - # unstable - nixpkgs/nixpkgs-unstable: - - bun - # custom registry - git+https://tangled.org/@example.com/my_pkg: - - my_pkg -``` - -Now these dependencies are available to use in your -workflow! - ### Environment The `environment` field allows you define environment @@ -992,9 +969,9 @@ following fields: - `command`: This field allows you to define a command to run in that step. The step is run in a Bash shell, and the logs from the command will be visible in the pipelines - page on the Tangled website. The - [dependencies](#dependencies) you added will be available - to use here. + page on the Tangled website. Any dependencies you added in + your engine's section (see [Engines](#engines)) will be + available to use here. - `environment`: Similar to the global [environment](#environment) config, this **optional** field is a key-value map that allows you to set @@ -1018,7 +995,49 @@ steps: NODE_ENV: "production" ``` -### Complete workflow +## Engines + +The common fields above apply to every workflow. Each engine +then adds its own fields on top. Pick an engine with the +[`engine`](#engine) field and use the matching section below. + +### Nixery engine + +#### Dependencies + +When you're running a workflow you'll usually need additional +dependencies. The `dependencies` field lets you define which +dependencies to get, and from where. It's a key-value map, +with the key being the registry to fetch dependencies from, +and the value being the list of dependencies to fetch. + +The registry URL syntax can be found [on the nix +manual](https://nix.dev/manual/nix/2.18/command-ref/new-cli/nix3-registry-add). + +Say you want to fetch Node.js and Go from `nixpkgs`, and a +package called `my_pkg` you've made from your own registry +at your repository at +`https://tangled.org/@example.com/my_pkg`. You can define +those dependencies like so: + +```yaml +dependencies: + # nixpkgs + nixpkgs: + - nodejs + - go + # unstable + nixpkgs/nixpkgs-unstable: + - bun + # custom registry + git+https://tangled.org/@example.com/my_pkg: + - my_pkg +``` + +Now these dependencies are available to use in your +workflow! + +#### Complete nixery workflow ```yaml # .tangled/workflows/build.yml @@ -1068,12 +1087,116 @@ If you want another example of a workflow, you can look at the one [Tangled uses to build the project](https://tangled.org/@tangled.org/core/blob/master/.tangled/workflows/build.yml). +### microVM engine + +#### Image + +A workflow picks the image to boot with the top-level `image` +field: + +```yaml +engine: microvm +image: nixos +``` + +There are two flavours of images: + +- **NixOS images** (e.g. `nixos`): the whole guest is built + with Nix, so you can configure it from the workflow file + itself. The `dependencies`, `services`, `virtualisation`, + `registry` and `caches` fields below are all understood + here, and the guest builds and activates that configuration + before any of your steps run. +- **Non-NixOS images** (e.g. `alpine`): there's no NixOS to + configure, so the workflow-level config fields above have + no effect. You still get a full machine to run steps in. + +The available image names depend on what the spindle operator +has installed. `nixos` and `alpine` are examples. If `image` +is omitted, the spindle's configured default image is used. + +#### Dependencies + +On the microVM engine, `dependencies` is a flat list of +packages that get added to the guest's `PATH` (via +`environment.systemPackages`). This field only applies to +**NixOS images**, for other images you can use the package +manager included in a step. + +A bare name like `go` is looked up in nixpkgs. You can also +point at any flake with the `flakeref#attr` syntax, so +`github:nixos/nixpkgs#hello` pulls `hello` straight out of +that flake. + +```yaml +dependencies: + - go + - github:nixos/nixpkgs#hello +``` + +#### Registry + +The `registry` field remaps flake references, the same way +`nix registry` does. This lets you pin or alias the flakes +used by `dependencies`. + +For example, pin `nixpkgs` to `nixos-unstable` so that the +bare `go` above resolves from unstable, and alias your own +flake so you can use `myflake#tool` in `dependencies`: + +```yaml +registry: + nixpkgs: github:nixos/nixpkgs/nixos-unstable + myflake: github:me/x +``` + +#### Caches + +The `caches` field is a map of Nix binary cache URL to its +trusted public key. These are fed into the spindle's read +proxy, so the guest can substitute prebuilt paths from them +instead of building everything from scratch. + +```yaml +caches: + https://nix-community.cachix.org: "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" +``` + +#### Services and virtualisation + +The `services` and `virtualisation` fields are passed straight +through to NixOS. Anything you could write under +`services.*` or `virtualisation.*` in a NixOS configuration, +you can write here, and it's brought up before any of your +steps run. + +As a convenience, `true` works as shorthand for +`.enable = true` anywhere an `enable` option exists (e.g. +`virtualisation.docker: true`). + +```yaml +services: + postgresql: + enable: true + ensureDatabases: ["spindle-workflow"] + ensureUsers: + - name: spindle-workflow + ensureDBOwnership: true + +virtualisation: + docker: true +``` + ## Self-hosting guide ### Prerequisites - Go -- Docker (the only supported backend currently) +- For the **nixery** engine: Docker (or Podman with Docker + compatibility enabled). +- For the **microVM** engine: a Linux host with KVM, plus the + microVM host dependencies described in [Running microVM + workflows](#running-microvm-workflows). ### Configuration @@ -1090,6 +1213,57 @@ Spindle is configured using environment variables. The following environment var - `SPINDLE_PIPELINES_NIXERY`: The Nixery URL (default: `"nixery.tangled.sh"`). - `SPINDLE_PIPELINES_WORKFLOW_TIMEOUT`: The default workflow timeout (default: `"5m"`). +For the microVM engine, the following are also available +(prefix `SPINDLE_MICROVM_PIPELINES_`): + +- `SPINDLE_MICROVM_PIPELINES_IMAGE_DIR`: Directory containing + microVM images (**required** to use the engine). See + [Running microVM workflows](#running-microvm-workflows). +- `SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE`: Image used when a + workflow doesn't set `image` (default: `"nixos-x86_64"`). +- `SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR`: Where per-workflow + temporary disks are created (default: the system temp dir). +- `SPINDLE_MICROVM_PIPELINES_ENABLE_KVM`: Use KVM hardware + acceleration (default: `true`). Without KVM, guests fall + back to slow software emulation. +- `SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT`: Default + workflow timeout (default: `"5m"`). + +Optional resource limits (a value of `0` disables that +limit). The limits cap usage across all running microVM +workflows: + +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB` +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS` +- `SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB` + +Optional cgroup enforcement: + +- `SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS`: Place each + workflow's QEMU and slirp4netns in a per-workflow cgroup= + (default: `false`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT`: Parent cgroup; + `self` resolves the spindle service's own cgroup (default: + `"self"`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX`: Max processes + per workflow cgroup (default: `4096`). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB`: Max swap + per workflow cgroup (default: `0`, no swap). +- `SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB`: + Memory protected for spindle itself so it isn't OOM-killed + before the workflows (default: `512`). + +To push paths built inside microVMs back to a shared Nix +cache (and read from it), configure the cache (prefix +`SPINDLE_NIX_CACHE_`): + +- `SPINDLE_NIX_CACHE_READ_URLS`: Comma-separated binary cache + URLs the guest reads from. +- `SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS`: Comma-separated + trusted public keys for those caches. +- `SPINDLE_NIX_CACHE_UPLOAD_URL`: Cache URL that paths built + in the guest are uploaded to. + ### Running spindle 1. **Set the environment variables.** For example: @@ -1122,6 +1296,70 @@ Spindle is configured using environment variables. The following environment var Spindle will now start, connect to the Jetstream server, and begin processing pipelines. +### Running microVM workflows + +The microVM engine needs a few extra things on the host, and +it needs images to boot. + +#### Host dependencies + +microVM workflows depend on a handful of host tools and +devices. spindle checks for the ones an image needs right +before it launches, so a missing dependency surfaces as a +clear error. You'll need: + +- `qemu`: the runner. The QEMU binary for the image's arch + must be present (e.g. `qemu-system-x86_64`). +- `mkfs.ext4` (from `e2fsprogs`): to format the per-workflow + writable volumes. +- [`slirp4netns`](https://github.com/rootless-containers/slirp4netns#install), + `ip` (from `iproute2`), `mount` and `unshare` (from `util-linux`): + used to sandbox guest networking. +- `/dev/kvm`: for hardware acceleration (unless you disable + KVM with `SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=false`). +- `/dev/vhost-vsock`: the guest agent talks to spindle over + vsock. + +On NixOS, the [spindle +module](https://tangled.org/tangled.org/core/blob/master/nix/modules/spindle.nix) +puts `qemu`, `e2fsprogs`, `slirp4netns`, `iproute2` and +`util-linux` on the service's `PATH` for you. + +#### Building images + +Images are built with Nix. The flake exposes packages for the +two stock images (use the `-tarball` prefixed ones for a gzipped +tarball you can copy to another host): + +```shell +# a NixOS image +nix build .#spindle-nixos-image +# an Alpine image +nix build .#spindle-alpine-image +``` + +#### Installing images + +Spindle looks for images in +`SPINDLE_MICROVM_PIPELINES_IMAGE_DIR`. An image is resolved by +the name a workflow puts in its `image` field, matched +literally against what's on disk: + +1. a directory `/` containing a `spec.json` (next to the + kernel/initrd/store-disk), or +2. a flat `.json` self-contained spec. + +Resolution depends only on the name and what's on disk, never +on the host doing the resolving, so the same workflow resolves +to the same image on every spindle. If you keep multiple +arches side by side, you can name them `-` (e.g. +`nixos-x86_64`, `alpine-aarch64`); the suffix is just part of +the name. To make a name like `nixos` work if you are hosting +multiple arches, you can use symlinks. + +On NixOS, you'll most likely want to use `systemd.tmpfiles.rules` +to set these up declaratively. + ## Architecture Spindle is a small CI runner service. Here's a high-level overview of how it operates: @@ -1135,16 +1373,24 @@ Spindle is a small CI runner service. Here's a high-level overview of how it ope - The spindle engine then handles execution of the pipeline, with results and logs beamed on the spindle event stream over WebSocket -### The engine - -At present, the only supported backend is Docker (and Podman, if Docker -compatibility is enabled, so that `/run/docker.sock` is created). spindle -executes each step in the pipeline in a fresh container, with state persisted -across steps within the `/tangled/workspace` directory. - -The base image for the container is constructed on the fly using -[Nixery](https://nixery.dev), which is handy for caching layers for frequently -used packages. +### The engines + +Spindle has two execution backends, picked per-workflow with +the [`engine`](#engine) field: + +- **nixery**: executes each step in a fresh Docker container + (Podman works too, if Docker compatibility is enabled so + that `/run/docker.sock` is created), with state persisted + across steps within the `/tangled/workspace` directory. The + base image for the container is constructed on the fly using + [Nixery](https://nixery.dev), which is/rhandy for caching + layers for frequently used packages. +- **microvm**: runs the whole workflow inside its own + microVM, supporting different images, with extra + configuration for NixOS images (e.g. services in workflow file) + See the [engine + README](https://tangled.org/tangled.org/core/blob/master/spindle/engines/microvm/README.md) + for the architecture in depth. The pipeline manifest is [specified here](https://docs.tangled.org/spindles.html#pipelines). diff --git a/eventconsumer/consumer.go b/eventconsumer/consumer.go index b540f989..08182f5a 100644 --- a/eventconsumer/consumer.go +++ b/eventconsumer/consumer.go @@ -5,7 +5,6 @@ import ( "encoding/json" "log/slog" "net/http" - "net/url" "sync" "time" @@ -29,7 +28,6 @@ type ConsumerConfig struct { QueueSize int Logger *slog.Logger CursorStore cursor.Store - URLFunc func(Source, int64) (*url.URL, error) Dialer *websocket.Dialer RequestHeader http.Header @@ -93,9 +91,6 @@ func NewConsumer(cfg ConsumerConfig) *Consumer { if cfg.CursorStore == nil { cfg.CursorStore = &cursor.MemoryStore{} } - if cfg.URLFunc == nil { - cfg.URLFunc = DefaultURL(false) - } dialer := cfg.Dialer if dialer == nil { dialer = websocket.DefaultDialer @@ -263,7 +258,7 @@ func (c *Consumer) startConnectionLoop(ctx context.Context, source Source) { func (c *Consumer) runConnection(ctx context.Context, source Source) error { cursor := c.cfg.CursorStore.Get(source.Key()) - u, err := c.cfg.URLFunc(source, cursor) + u, err := source.URL(cursor) if err != nil { return err } diff --git a/eventconsumer/consumer_test.go b/eventconsumer/consumer_test.go index d4255234..2cd19082 100644 --- a/eventconsumer/consumer_test.go +++ b/eventconsumer/consumer_test.go @@ -69,7 +69,7 @@ func startEventServer(t *testing.T, src *memSrc) (Source, *notifier.Notifier) { srv := httptest.NewServer(mux) t.Cleanup(srv.Close) addr := strings.TrimPrefix(srv.URL, "http://") - return Source{Kind: "test", Host: addr}, &n + return Source{Kind: "test", Host: addr, NoTLS: true}, &n } func TestConsumer_DrainAdvancesCursor(t *testing.T) { @@ -95,7 +95,6 @@ func TestConsumer_DrainAdvancesCursor(t *testing.T) { QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -158,7 +157,6 @@ func TestConsumer_CursorMonotonic_OutOfOrderWorkers(t *testing.T) { QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -203,7 +201,6 @@ func TestConsumer_StopTerminatesWithoutCtxCancel(t *testing.T) { QueueSize: 8, ConnectionTimeout: 2 * time.Second, CursorStore: &cursor.MemoryStore{}, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) @@ -249,7 +246,6 @@ func TestConsumer_ResumesFromStoredCursor(t *testing.T) { QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), } c := NewConsumer(cfg) diff --git a/eventconsumer/source.go b/eventconsumer/source.go index f0b768cb..010d4267 100644 --- a/eventconsumer/source.go +++ b/eventconsumer/source.go @@ -5,6 +5,7 @@ import ( "strconv" "tangled.org/core/eventconsumer/cursor" + "tangled.org/core/hostutil" ) type Kind string @@ -15,12 +16,19 @@ const ( ) type Source struct { - Kind Kind - Host string + Kind Kind + Host string + NoTLS bool // use TLS by default } -func NewKnotSource(host string) Source { return Source{Kind: KindKnot, Host: host} } -func NewSpindleSource(host string) Source { return Source{Kind: KindSpindle, Host: host} } +func NewKnotSource(host string) Source { + host, noTLS, _ := hostutil.ParseHostname(host) + return Source{Kind: KindKnot, Host: host, NoTLS: noTLS} +} +func NewSpindleSource(host string) Source { + host, noTLS, _ := hostutil.ParseHostname(host) + return Source{Kind: KindSpindle, Host: host, NoTLS: noTLS} +} func (s Source) Key() string { return string(s.Kind) + ":" + s.Host } @@ -33,21 +41,19 @@ func MigrateLegacyCursor(store cursor.Store, s Source) { } } -func DefaultURL(dev bool) func(Source, int64) (*url.URL, error) { +func (s Source) URL(cursor int64) (*url.URL, error) { scheme := "wss" - if dev { + if s.NoTLS { scheme = "ws" } - return func(s Source, cursor int64) (*url.URL, error) { - u, err := url.Parse(scheme + "://" + s.Host + "/events") - if err != nil { - return nil, err - } - if cursor != 0 { - q := url.Values{} - q.Add("cursor", strconv.FormatInt(cursor, 10)) - u.RawQuery = q.Encode() - } - return u, nil + u, err := url.Parse(scheme + "://" + s.Host + "/events") + if err != nil { + return nil, err + } + if cursor != 0 { + q := url.Values{} + q.Add("cursor", strconv.FormatInt(cursor, 10)) + u.RawQuery = q.Encode() } + return u, nil } diff --git a/eventconsumer/upgrade_test.go b/eventconsumer/upgrade_test.go index 7e92425e..3c90f34b 100644 --- a/eventconsumer/upgrade_test.go +++ b/eventconsumer/upgrade_test.go @@ -39,7 +39,6 @@ func drainProcessed(t *testing.T, store cursor.Store, source Source) []int64 { QueueSize: 16, ConnectionTimeout: 2 * time.Second, CursorStore: store, - URLFunc: DefaultURL(true), Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), }) diff --git a/flake.lock b/flake.lock index 1c5f8cb9..45dc1eef 100644 --- a/flake.lock +++ b/flake.lock @@ -117,14 +117,14 @@ "ibm-plex-mono-src": { "flake": false, "locked": { - "lastModified": 1731402384, + "lastModified": 1731402378, "narHash": "sha256-OwUmrPfEehLDz0fl2ChYLK8FQM2p0G1+EMrGsYEq+6g=", "type": "tarball", - "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + "url": "https://github.com/IBM/plex/releases/download/@ibm/plex-mono@1.1.0/ibm-plex-mono.zip" }, "original": { "type": "tarball", - "url": "https://github.com/IBM/plex/releases/download/@ibm%2Fplex-mono@1.1.0/ibm-plex-mono.zip" + "url": "https://github.com/IBM/plex/releases/download/@ibm/plex-mono@1.1.0/ibm-plex-mono.zip" } }, "indigo": { @@ -181,6 +181,27 @@ "url": "https://cdn.jsdelivr.net/npm/mermaid@11.12.3/dist/mermaid.min.js" } }, + "microvm": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "spectrum": "spectrum" + }, + "locked": { + "lastModified": 1779970379, + "narHash": "sha256-ZHsxoYXXnfJtMVh1/yY+1Eh9hHcPBhE28Qvinauh+BQ=", + "owner": "microvm-nix", + "repo": "microvm.nix", + "rev": "0d49083ba2d7419b22908ac392777c16df9a032e", + "type": "github" + }, + "original": { + "owner": "microvm-nix", + "repo": "microvm.nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1771848320, @@ -210,6 +231,7 @@ "inter-fonts-src": "inter-fonts-src", "lucide-src": "lucide-src", "mermaid-src": "mermaid-src", + "microvm": "microvm", "nixpkgs": "nixpkgs", "sqlite-lib-src": "sqlite-lib-src" } @@ -231,6 +253,22 @@ "type": "github" } }, + "spectrum": { + "flake": false, + "locked": { + "lastModified": 1778940603, + "narHash": "sha256-voSM8dZNlaOWN3kbYFky+FNY6fFQOEw0xF+ZMpZKkCQ=", + "ref": "refs/heads/main", + "rev": "367dd227f539267eae2b62770b4c17b88ac8c1f1", + "revCount": 1265, + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + }, + "original": { + "type": "git", + "url": "https://spectrum-os.org/git/spectrum" + } + }, "sqlite-lib-src": { "flake": false, "locked": { diff --git a/flake.nix b/flake.nix index 12aa773b..2bad35f3 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,10 @@ inputs = { nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable"; + microvm = { + url = "github:microvm-nix/microvm.nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; fenix = { url = "github:nix-community/fenix"; inputs.nixpkgs.follows = "nixpkgs"; @@ -69,6 +73,7 @@ ibm-plex-mono-src, actor-typeahead-src, mermaid-src, + microvm, ... }: let supportedSystems = ["x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarch64-darwin"]; @@ -84,13 +89,41 @@ root = ./.; fileset = fs.difference (fs.intersection (fs.gitTracked ./.) (fs.fileFilter (file: !(file.hasExt "nix")) ./.)) (fs.maybeMissing ./.jj); }; + rustSrc = let + fs = pkgs.lib.fileset; + in + fs.toSource { + root = ./.; + fileset = + fs.intersection + (fs.fromSource self.src) + (fs.unions [ + ./Cargo.toml + ./Cargo.lock + ./shuttle + ./bobbin + ]); + }; buildGoApplication = (self.callPackage "${gomod2nix}/builder" { gomod2nix = gomod2nix.legacyPackages.${pkgs.stdenv.hostPlatform.system}.gomod2nix; }).buildGoApplication; rustPlatform = pkgs.makeRustPlatform { - inherit (fenix.packages.${pkgs.system}.stable) rustc cargo; + inherit (fenix.packages.${pkgs.stdenv.hostPlatform.system}.stable) rustc cargo; }; + rustPlatformStatic = let + system = pkgs.stdenv.hostPlatform.system; + muslTarget = pkgs.pkgsStatic.stdenv.hostPlatform.rust.rustcTarget; + toolchain = fenix.packages.${system}.combine [ + fenix.packages.${system}.stable.cargo + fenix.packages.${system}.stable.rustc + fenix.packages.${system}.targets.${muslTarget}.stable.rust-std + ]; + in + pkgs.pkgsStatic.makeRustPlatform { + cargo = toolchain; + rustc = toolchain; + }; modules = ./nix/gomod2nix.toml; sqlite-lib = self.callPackage ./nix/pkgs/sqlite-lib.nix { inherit sqlite-lib-src; @@ -107,6 +140,13 @@ inherit (pkgs) pagefind; }; spindle = self.callPackage ./nix/pkgs/spindle.nix {}; + shuttle = self.callPackage ./nix/pkgs/shuttle.nix { + src = self.rustSrc; + }; + shuttle-static = self.callPackage ./nix/pkgs/shuttle.nix { + src = self.rustSrc; + rustPlatform = self.rustPlatformStatic; + }; knot-unwrapped = self.callPackage ./nix/pkgs/knot-unwrapped.nix {}; knot = self.callPackage ./nix/pkgs/knot.nix {}; dolly = self.callPackage ./nix/pkgs/dolly.nix {}; @@ -116,11 +156,12 @@ }); in { overlays.default = final: prev: { - inherit (mkPackageSet final) lexgen goat sqlite-lib spindle knot-unwrapped knot appview docs dolly tap knotmirror bobbin; + inherit (mkPackageSet final) lexgen goat sqlite-lib spindle shuttle knot-unwrapped knot appview docs dolly tap knotmirror bobbin; }; packages = forAllSystems (system: let pkgs = nixpkgsFor.${system}; + linuxPkgs = nixpkgsFor."x86_64-linux"; packages = mkPackageSet pkgs; staticPackages = mkPackageSet pkgs.pkgsStatic; crossPackages = mkPackageSet pkgs.pkgsCross.gnu64.pkgsStatic; @@ -137,6 +178,8 @@ knot-unwrapped sqlite-lib docs + shuttle + shuttle-static dolly tap knotmirror @@ -189,6 +232,48 @@ # }; }; }; + + spindle-nixos-image = linuxPkgs.callPackage ./nix/pkgs/spindle-nixos-image.nix { + nixosSystem = self.nixosConfigurations.spindle-nixos; + }; + spindle-nixos-image-tarball = linuxPkgs.runCommand "spindle-nixos-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-nixos-image} -h -czf $out . + ''; + + spindle-alpine-image = let + branch = "3.24"; + version = "${branch}.0"; + arch = "x86_64"; + cdn = "https://dl-cdn.alpinelinux.org/alpine/v${branch}/releases/${arch}"; + + shuttle = (mkPackageSet linuxPkgs).shuttle-static; + in + linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix { + inherit arch shuttle; + repositories = [ + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/main" + "https://dl-cdn.alpinelinux.org/alpine/v${branch}/community" + ]; + rootfs = linuxPkgs.fetchurl { + url = "${cdn}/alpine-minirootfs-${version}-${arch}.tar.gz"; + hash = "sha256-3poRwODn6clNs+2K97RQ6vwLE2h71+kZnVUFDyCqCok="; + }; + kernel = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/vmlinuz-virt"; + hash = "sha256-Hmv5Ancgx1w+0NeRcfIbV5HuQMqXldB8fG4E3F6irpA="; + }; + initramfs = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/initramfs-virt"; + hash = "sha256-ZCWGSaVMOYOmLz1Gwsf2RhYarMqk+tFVA6MMDWiHVJQ="; + }; + modloop = linuxPkgs.fetchurl { + url = "${cdn}/netboot-${version}/modloop-virt"; + hash = "sha256-p3yO7yU28k04iT01sOzhDmEYi+Yl7VZs5r3RYsWCBX0="; + }; + }; + spindle-alpine-image-tarball = linuxPkgs.runCommand "spindle-alpine-image-tarball.tar.gz" {} '' + tar -S -C ${self.packages.${system}.spindle-alpine-image} -h -czf $out . + ''; }); defaultPackage = forAllSystems (system: self.packages.${system}.appview); devShells = forAllSystems (system: let @@ -219,6 +304,15 @@ pkgs.redis pkgs.worker-build pkgs.cargo-generate + pkgs.qemu + pkgs.cdrkit + pkgs.parted + pkgs.buf + pkgs.protobuf + pkgs.protoc-gen-prost + pkgs.protoc-gen-prost-crate + pkgs.protoc-gen-prost-serde + pkgs.protoc-gen-go (fenix.packages.${system}.combine [ fenix.packages.${system}.stable.cargo fenix.packages.${system}.stable.rustc @@ -231,6 +325,10 @@ packages'.lexgen packages'.treefmt-wrapper packages'.tap + pkgs.e2fsprogs + pkgs.slirp4netns + pkgs.iproute2 + pkgs.util-linux ]; shellHook = '' mkdir -p appview/pages/static @@ -306,6 +404,24 @@ exec ${pkgs.python3}/bin/python3 -m http.server 1414 ''); }; + regenerate-proto = { + type = "app"; + program = + (pkgs.writeShellApplication { + name = "regenerate-proto"; + runtimeInputs = with pkgs; [git buf coreutils]; + text = '' + rootDir=$(git rev-parse --show-toplevel 2>/dev/null || pwd) + cd "$rootDir" + echo ">>> regenerating protobuf files.." + buf generate + echo ">>> generating file descriptor set for shuttle..." + buf build -o shuttle/src/gen/file_descriptor_set.bin + echo ">>> done" + ''; + }) + + "/bin/regenerate-proto"; + }; vm = let guestSystem = if pkgs.stdenv.hostPlatform.isAarch64 @@ -409,7 +525,27 @@ services.tangled.spindle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.spindle; }; + nixosModules.shuttle = { + lib, + pkgs, + ... + }: { + imports = [./nix/modules/shuttle.nix]; + + services.tangled.shuttle.package = lib.mkDefault self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle; + }; + + nixosModules.spindle-nixos = import ./nix/microvm/spindle-vm.nix {inherit self microvm;} ./nix/microvm/qemu.nix; formatter = forAllSystems (system: self.packages.${system}.treefmt-wrapper); + + nixosConfigurations = let + spindleNixosBase = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [self.nixosModules.spindle-nixos]; + }; + in { + spindle-nixos = spindleNixosBase; + }; }; } diff --git a/go.mod b/go.mod index 2d45e87d..6a40856d 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module tangled.org/core go 1.25.0 require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 + buf.build/go/protovalidate v1.2.0 github.com/Blank-Xu/sql-adapter v1.1.1 github.com/adrg/frontmatter v0.2.0 github.com/alecthomas/assert/v2 v2.11.0 @@ -27,9 +29,11 @@ require ( github.com/charmbracelet/ssh v0.0.0-20250128164007-98fd5ae11894 github.com/charmbracelet/wish v1.4.7 github.com/cloudflare/cloudflare-go/v6 v6.7.0 + github.com/containerd/cgroups/v3 v3.1.3 github.com/cyphar/filepath-securejoin v0.4.1 github.com/dgraph-io/ristretto v0.2.0 github.com/did-method-plc/go-didplc v0.2.2 + github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 github.com/docker/docker v28.2.2+incompatible @@ -49,11 +53,14 @@ require ( github.com/jackc/pgx/v5 v5.8.0 github.com/landlock-lsm/go-landlock v0.8.1 github.com/mattn/go-sqlite3 v1.14.34 + github.com/mdlayher/vsock v1.3.0 + github.com/miekg/dns v1.1.72 github.com/microcosm-cc/bluemonday v1.0.27 github.com/multiformats/go-multihash v0.2.3 github.com/openbao/openbao/api/v2 v2.3.0 github.com/posthog/posthog-go v1.5.5 github.com/prometheus/client_golang v1.23.2 + github.com/prometheus/procfs v0.19.2 github.com/redis/go-redis/v9 v9.7.3 github.com/resend/resend-go/v3 v3.5.0 github.com/sethvargo/go-envconfig v1.1.0 @@ -67,16 +74,18 @@ require ( github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab go.abhg.dev/goldmark/mermaid v0.6.0 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.51.0 golang.org/x/image v0.31.0 - golang.org/x/net v0.50.0 - golang.org/x/sync v0.19.0 - golang.org/x/sys v0.41.0 + golang.org/x/net v0.55.0 + golang.org/x/sync v0.20.0 + golang.org/x/sys v0.45.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) require ( + cel.dev/expr v0.25.1 // indirect dario.cat/mergo v1.0.1 // indirect github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -85,6 +94,7 @@ require ( github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b // indirect github.com/alecthomas/repr v0.5.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect @@ -134,6 +144,7 @@ require ( github.com/charmbracelet/x/term v0.2.2 // indirect github.com/charmbracelet/x/termios v0.1.0 // indirect github.com/charmbracelet/x/windows v0.2.0 // indirect + github.com/cilium/ebpf v0.16.0 // indirect github.com/clipperhouse/displaywidth v0.9.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.5.0 // indirect @@ -141,9 +152,11 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/creack/pty v1.1.21 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -163,6 +176,7 @@ require ( github.com/go-redis/cache/v9 v9.0.0 // indirect github.com/go-test/deep v1.1.1 // indirect github.com/goccy/go-json v0.10.5 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect @@ -170,6 +184,7 @@ require ( github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/cel-go v0.28.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect @@ -219,10 +234,12 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/mdlayher/socket v0.6.0 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -240,6 +257,7 @@ require ( github.com/onsi/gomega v1.37.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/opencontainers/runtime-spec v1.3.0 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/pjbgf/sha1cd v0.3.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -247,11 +265,11 @@ require ( github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect - github.com/prometheus/procfs v0.19.2 // indirect github.com/puzpuzpuz/xsync/v4 v4.2.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect @@ -276,9 +294,12 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/mod v0.35.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.12.0 // indirect - google.golang.org/protobuf v1.36.11 // indirect + golang.org/x/tools v0.44.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 5aa7a19d..e0c5a9c7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,9 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1 h1:s6hzCXtND/ICdGPTMGk7C+/BFlr2Jg5GyH0NKf4XGXg= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260415201107-50325440f8f2.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= @@ -23,6 +29,8 @@ github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= @@ -120,6 +128,8 @@ github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/ github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2 h1:63+EsT7kltod8g1eA0eNuvq1q9ANJWRdxlLeJjJDVYY= github.com/boltlessengineer/indigo v0.0.0-20260315101958-fb1dfa36fed2/go.mod h1:VG/LeqLGNI3Ew7lsYixajnZGFfWPv144qbUddh+Oyag= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -183,6 +193,8 @@ github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHG github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA= github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA= github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= @@ -193,12 +205,16 @@ github.com/cloudflare/circl v1.6.2-0.20250618153321-aa837fd1539d h1:IiIprFGH6Sqs github.com/cloudflare/circl v1.6.2-0.20250618153321-aa837fd1539d/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cloudflare/cloudflare-go/v6 v6.7.0 h1:MP6Xy5WmsyrxgTxoLeq/vraqR0nbTtXoHhW4vAYc4SY= github.com/cloudflare/cloudflare-go/v6 v6.7.0/go.mod h1:Lj3MUqjvKctXRpdRhLQxZYRrNZHuRs0XYuH8JtQGyoI= +github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6dyLYXQ= +github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0= @@ -221,6 +237,10 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/did-method-plc/go-didplc v0.2.2 h1:53HFhTT8NCAeFmZ6fdIZCf3PGDvj7A3cDjzOOEqn5XM= github.com/did-method-plc/go-didplc v0.2.2/go.mod h1:bKdJ21irnwNHgVLWWL32zUWqZueXYbJRUcxplZghByo= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e h1:SCnqm8SjSa0QqRxXbo5YY//S+OryeJioe17nK+iDZpg= +github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e/go.mod h1:o129ljs6alsIQTc8d6eweihqpmmrbxZ2g1jhgjhPykI= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242 h1:rh6rt8pF5U4iyQ86h6lRDenJoX4ht2wFnZXB9ogIrIM= +github.com/digitalocean/go-qemu v0.0.0-20250212194115-ee9b0668d242/go.mod h1:LGHUtlhsY4vRGM6AHejEQKVI5e3eHbSylMHwTSpQtVw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= @@ -285,6 +305,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= @@ -301,6 +323,9 @@ github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= @@ -325,6 +350,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/cel-go v0.28.0 h1:KjSWstCpz/MN5t4a8gnGJNIYUsJRpdi/r97xWDphIQc= +github.com/google/cel-go v0.28.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -465,6 +492,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink/v2 v2.0.1 h1:xda7qaHDSVOsADNouv7ukSuicKZO7GgVUCXxpaIEIlM= +github.com/jsimonetti/rtnetlink/v2 v2.0.1/go.mod h1:7MoNYNbb3UaDHtF8udiJo/RH6VsTKP1pqKLUTVCvToE= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= @@ -522,8 +553,16 @@ github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byF github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= +github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= +github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI= +github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI= +github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -534,6 +573,8 @@ github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -606,6 +647,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= +github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= @@ -642,6 +685,8 @@ github.com/resend/resend-go/v3 v3.5.0 h1:yScYxHinY352Mj7Cn9rbWsR2gDqD2mtFPWwh2Uy github.com/resend/resend-go/v3 v3.5.0/go.mod h1:iI7VA0NoGjWvsNii5iNC5Dy0llsI3HncXPejhniYzwE= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -770,6 +815,8 @@ go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -777,8 +824,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/image v0.31.0 h1:mLChjE2MV6g1S7oqbXC0/UcKijjm5fnJLUYKIYrLESA= @@ -793,6 +840,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -813,8 +862,8 @@ golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -822,8 +871,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -845,6 +894,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -855,8 +905,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -866,8 +916,8 @@ golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -878,8 +928,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -899,6 +949,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/knotmirror/hostutil/hostutil.go b/hostutil/hostutil.go similarity index 74% rename from knotmirror/hostutil/hostutil.go rename to hostutil/hostutil.go index 26c9b2b7..2038a06b 100644 --- a/knotmirror/hostutil/hostutil.go +++ b/hostutil/hostutil.go @@ -54,3 +54,27 @@ func ParseHostname(raw string) (hostname string, noSSL bool, err error) { // lower-case in response return h.Normalize().String(), noSSL, nil } + +func EnsureHttpScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "http://" + hostname, nil + } else { + return "https://" + hostname, nil + } +} + +func EnsureWsScheme(host string) (string, error) { + hostname, noSSL, err := ParseHostname(host) + if err != nil { + return "", err + } + if noSSL { + return "ws://" + hostname, nil + } else { + return "wss://" + hostname, nil + } +} diff --git a/knotmirror/xrpc/sync_request_crawl.go b/knotmirror/xrpc/sync_request_crawl.go index 900b8beb..5858ec38 100644 --- a/knotmirror/xrpc/sync_request_crawl.go +++ b/knotmirror/xrpc/sync_request_crawl.go @@ -11,8 +11,8 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/xrpc" "tangled.org/core/api/tangled" + "tangled.org/core/hostutil" "tangled.org/core/knotmirror/db" - "tangled.org/core/knotmirror/hostutil" "tangled.org/core/knotmirror/models" ) diff --git a/localinfra/Caddyfile b/localinfra/Caddyfile index f568ffac..42697d97 100644 --- a/localinfra/Caddyfile +++ b/localinfra/Caddyfile @@ -37,6 +37,16 @@ knot.tngl.boltless.dev { reverse_proxy knot:5555 } +# spindle +http://spindle.tngl.boltless.dev { + reverse_proxy spindle:6555 +} + +spindle.tngl.boltless.dev { + tls internal + reverse_proxy spindle:6555 +} + # knotmirror mirror.tngl.boltless.dev { tls internal diff --git a/localinfra/appview.Dockerfile b/localinfra/appview.Dockerfile index ee360aeb..d199ecc3 100644 --- a/localinfra/appview.Dockerfile +++ b/localinfra/appview.Dockerfile @@ -41,6 +41,10 @@ export TANGLED_OAUTH_CLIENT_KID="$(cat $KID)" [ -r /shared/label-defaults ] && export TANGLED_LABEL_DEFAULTS="$(cat /shared/label-defaults)" [ -r /shared/label-gfi ] && export TANGLED_LABEL_GFI="$(cat /shared/label-gfi)" +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + exec air -c /src/.air/appview.toml EOF RUN chmod +x /usr/local/bin/appview-entrypoint.sh diff --git a/localinfra/knot.Dockerfile b/localinfra/knot.Dockerfile index 3ae41a5c..aab62bf5 100644 --- a/localinfra/knot.Dockerfile +++ b/localinfra/knot.Dockerfile @@ -84,6 +84,10 @@ mkdir -p /etc/ssh/keys [ -f /etc/ssh/keys/ssh_host_ecdsa_key ] || ssh-keygen -t ecdsa -f /etc/ssh/keys/ssh_host_ecdsa_key -q -N "" [ -f /etc/ssh/keys/ssh_host_ed25519_key ] || ssh-keygen -t ed25519 -f /etc/ssh/keys/ssh_host_ed25519_key -q -N "" +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + /usr/sbin/sshd -D -e & exec su-exec git /usr/local/bin/knot server EOF diff --git a/localinfra/knotmirror.Dockerfile b/localinfra/knotmirror.Dockerfile index 37b67eb1..807f56e2 100644 --- a/localinfra/knotmirror.Dockerfile +++ b/localinfra/knotmirror.Dockerfile @@ -14,13 +14,10 @@ FROM alpine:3.22 RUN apk add --no-cache git tini ca-certificates -# Trust dev CA in the system bundle so git/curl/openssl all accept caddy certs. -COPY localinfra/certs/root.crt /usr/local/share/ca-certificates/caddy.crt -RUN update-ca-certificates COPY --from=build /knotmirror /usr/local/bin/knotmirror EXPOSE 7000 ENTRYPOINT ["/sbin/tini", "--"] -CMD ["/usr/local/bin/knotmirror", "serve"] +CMD ["sh", "-c", "if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then update-ca-certificates; fi && exec /usr/local/bin/knotmirror serve"] diff --git a/localinfra/readme.md b/localinfra/readme.md index 3ee6ab8a..bc062c6d 100644 --- a/localinfra/readme.md +++ b/localinfra/readme.md @@ -14,13 +14,11 @@ To make that work: - atproto_pds () - jetstream () - knot () +- spindle () - knotmirror () - appview () (live reloading) - caddy reverse proxy -> [!NOTE] -> Spindle is not included yet. - ## Setup 1. Generate the dev CA from the repo root: @@ -44,5 +42,10 @@ To make that work: ``` - Depending on your browser you may have to import the certificate into your browser profiles too as some have their own certs do not use your system ones 3. run `./localinfra/scripts/appview-static-files.sh` -4. `docker compose up` -5. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. +4. Prepare the spindle microVM images: + ```bash + ./localinfra/scripts/prepare-spindle-images.sh + ``` + This writes the image directory under `out/localinfra-spindle-images`. +5. `docker compose up` +6. AppView will be running on `127.0.0.1:3000` with two test users: `alice.pds.tngl.boltless.dev` and `bob.pds.tngl.boltless.dev`. Both with password `password`. diff --git a/localinfra/scripts/init-accounts.sh b/localinfra/scripts/init-accounts.sh index e866c0dc..fda02c5c 100644 --- a/localinfra/scripts/init-accounts.sh +++ b/localinfra/scripts/init-accounts.sh @@ -187,5 +187,6 @@ printf '[env] wrote label-defaults, label-gfi\n' >&2 JWT=$(login "$OWNER_DID") put_record "$JWT" "$OWNER_DID" "sh.tangled.knot" $KNOT_HOSTNAME "{\"createdAt\": \"${CREATED_AT}\"}" +put_record "$JWT" "$OWNER_DID" "sh.tangled.spindle" "$SPINDLE_HOSTNAME" "{\"createdAt\": \"${CREATED_AT}\"}" printf 'done.\n' >&2 diff --git a/localinfra/scripts/prepare-spindle-images.sh b/localinfra/scripts/prepare-spindle-images.sh new file mode 100755 index 00000000..93f31cf4 --- /dev/null +++ b/localinfra/scripts/prepare-spindle-images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "$0")/../.." && pwd) +image_root="${1:-$repo/out/localinfra-spindle-images}" + +mkdir -p "$image_root" + +extract_image() { + local package="$1" + local name="$2" + shift 2 + + local tarball + tarball=$(nix build "$repo#$package" --no-link --print-out-paths) + + [ -d "$image_root/$name" ] && chmod -R +w "$image_root/$name" || true + rm -rf "$image_root/$name" + mkdir -p "$image_root/$name" + tar -C "$image_root/$name" -xzf "$tarball" + + local alias + for alias in "$@"; do + rm -rf "$image_root/$alias" + ln -s "$name" "$image_root/$alias" + done +} + +extract_image spindle-nixos-image-tarball nixos-x86_64 nixos +extract_image spindle-alpine-image-tarball alpine-x86_64 alpine + +echo "prepared spindle microVM images in $image_root" diff --git a/localinfra/spindle.Dockerfile b/localinfra/spindle.Dockerfile new file mode 100644 index 00000000..33e97334 --- /dev/null +++ b/localinfra/spindle.Dockerfile @@ -0,0 +1,66 @@ +# Development only. Not for production use. + +FROM golang:1.25-alpine AS builder + +RUN apk add --no-cache git build-base sqlite-dev + +ENV CGO_ENABLED=1 +ENV GOCACHE=/go/cache +ENV GOMODCACHE=/go/mod + +WORKDIR /src + +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/cache \ + --mount=type=cache,target=/go/mod \ + go mod download + +COPY . . +RUN --mount=type=cache,target=/go/cache \ + --mount=type=cache,target=/go/mod \ + go build -tags libsqlite3 -o /out/spindle ./cmd/spindle && \ + go build -tags libsqlite3 -o /out/spindle-microvm-run ./cmd/spindle-microvm-run + +FROM alpine:3.20 + +RUN apk add --no-cache \ + bash \ + ca-certificates \ + e2fsprogs \ + git \ + iproute2 \ + qemu-system-x86_64 \ + shadow \ + slirp4netns \ + sqlite-libs \ + tini \ + util-linux + + +COPY --from=builder /out/spindle /usr/local/bin/spindle +COPY --from=builder /out/spindle-microvm-run /usr/local/bin/spindle-microvm-run +RUN chmod 0755 /usr/local/bin/spindle /usr/local/bin/spindle-microvm-run + +COPY <<'EOF' /usr/local/bin/spindle-entrypoint.sh +#!/bin/sh +set -eu + +[ -z "${SPINDLE_SERVER_OWNER:-}" ] && [ -r /shared/owner-did ] && \ + export SPINDLE_SERVER_OWNER="$(cat /shared/owner-did)" +: "${SPINDLE_SERVER_OWNER:?set via env or /shared/owner-did}" + +mkdir -p /var/lib/spindle /var/lib/spindle/overlays /var/log/spindle + +if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then + update-ca-certificates +fi + +exec /usr/local/bin/spindle run +EOF +RUN chmod +x /usr/local/bin/spindle-entrypoint.sh + +VOLUME /var/lib/spindle +EXPOSE 6555 + +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/usr/local/bin/spindle-entrypoint.sh"] diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml index 316303a4..83a61bea 100644 --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -1,6 +1,15 @@ schema = 3 [mod] + [mod."buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go"] + version = "v1.36.11-20260415201107-50325440f8f2.1" + hash = "sha256-oyhP92KT9a++f5riA9sI1myID7MR39AzHuT3cKx5hgg=" + [mod."buf.build/go/protovalidate"] + version = "v1.2.0" + hash = "sha256-4+42DSj7LAdJyPPxif3b5QRH5MCjedudgylTHBibZlQ=" + [mod."cel.dev/expr"] + version = "v0.25.1" + hash = "sha256-TEdMxFUPK7IZuCXMufwCkbN+ZZIXSQclljIybFZcByo=" [mod."dario.cat/mergo"] version = "v1.0.1" hash = "sha256-wcG6+x0k6KzOSlaPA+1RFxa06/RIAePJTAjjuhLbImw=" @@ -38,6 +47,9 @@ schema = 3 [mod."github.com/anmitsu/go-shlex"] version = "v0.0.0-20200514113438-38f4b401e2be" hash = "sha256-L3Ak4X2z7WXq7vMKuiHCOJ29nlpajUQ08Sfb9T0yP54=" + [mod."github.com/antlr4-go/antlr/v4"] + version = "v4.13.1" + hash = "sha256-beAuxHNRUuhzcSJUh/8ztVf1zCUiaT72fg2Jvx0AuNQ=" [mod."github.com/avast/retry-go/v4"] version = "v4.6.1" hash = "sha256-PeZc8k4rDV64+k8nZt/oy1YNVbLevltXP3ZD1jf6Z6k=" @@ -244,6 +256,9 @@ schema = 3 [mod."github.com/charmbracelet/x/windows"] version = "v0.2.0" hash = "sha256-pDAd1E5w66E/d3vuTyzgnW+W/KegZ2sxQQMfoEn7S1A=" + [mod."github.com/cilium/ebpf"] + version = "v0.16.0" + hash = "sha256-xACuieGmiUUjoTT/9MpvPBNexp98S/AZbLxm5f9nqDk=" [mod."github.com/clipperhouse/displaywidth"] version = "v0.9.0" hash = "sha256-9CNyTZPSncKQ7Y0my9DR4WYXDjtDHYNL512D691WDAM=" @@ -259,6 +274,9 @@ schema = 3 [mod."github.com/cloudflare/cloudflare-go/v6"] version = "v6.7.0" hash = "sha256-ycQpx1II/JgBgrCRwY5qiVKStGv5wuCANy1091sJ5Zw=" + [mod."github.com/containerd/cgroups/v3"] + version = "v3.1.3" + hash = "sha256-1a5heWXIzME7iMu2L35OBiAOi2Z/gnpg2fjvP6On9sM=" [mod."github.com/containerd/errdefs"] version = "v1.0.0" hash = "sha256-wMZGoeqvRhuovYCJx0Js4P3qFCNTZ/6Atea/kNYoPMI=" @@ -268,6 +286,9 @@ schema = 3 [mod."github.com/containerd/log"] version = "v0.1.0" hash = "sha256-vuE6Mie2gSxiN3jTKTZovjcbdBd1YEExb7IBe3GM+9s=" + [mod."github.com/coreos/go-systemd/v22"] + version = "v22.5.0" + hash = "sha256-E2zXikbmIQImghstLUWuey1YgA0Folu3F+fi5k4hCxA=" [mod."github.com/creack/pty"] version = "v1.1.21" hash = "sha256-pjGw6wQlrVhN65XaIxZueNJqnXThGu00u24rKOLzxS0=" @@ -286,6 +307,12 @@ schema = 3 [mod."github.com/did-method-plc/go-didplc"] version = "v0.2.2" hash = "sha256-TF5vdW1U2q5F23ELmbqhdvpNgDQjBs9i/CWJlzqWNKs=" + [mod."github.com/digitalocean/go-libvirt"] + version = "v0.0.0-20220804181439-8648fbde413e" + hash = "sha256-xgRZCefeUNM76M9ht5FCgCf4dnwAdeP/r+FZlJ2gmeY=" + [mod."github.com/digitalocean/go-qemu"] + version = "v0.0.0-20250212194115-ee9b0668d242" + hash = "sha256-AzY84aq9CqGqYwmGkKoi4PX08E/hRQC46mrZwKSMwXE=" [mod."github.com/distribution/reference"] version = "v0.6.0" hash = "sha256-gr4tL+qz4jKyAtl8LINcxMSanztdt+pybj1T+2ulQv4=" @@ -368,6 +395,9 @@ schema = 3 [mod."github.com/goccy/go-json"] version = "v0.10.5" hash = "sha256-/EtlGihP0/7oInzMC5E0InZ4b5Ad3s4xOpqotloi3xw=" + [mod."github.com/godbus/dbus/v5"] + version = "v5.1.0" + hash = "sha256-xOCMJpQK3KTmHTPn/CdqI4j0eENCtMmJDgAIoYqYOEY=" [mod."github.com/gogo/protobuf"] version = "v1.3.2" hash = "sha256-pogILFrrk+cAtb0ulqn9+gRZJ7sGnnLLdtqITvxvG6c=" @@ -389,6 +419,9 @@ schema = 3 [mod."github.com/golang/snappy"] version = "v0.0.4" hash = "sha256-Umx+5xHAQCN/Gi4HbtMhnDCSPFAXSsjVbXd8n5LhjAA=" + [mod."github.com/google/cel-go"] + version = "v0.28.0" + hash = "sha256-86MSoJX3fovcAWir41Z34nM0HcaDh0yRA+lIiVA1gEM=" [mod."github.com/google/go-querystring"] version = "v1.1.0" hash = "sha256-itsKgKghuX26czU79cK6C2n+lc27jm5Dw1XbIRgwZJY=" @@ -569,9 +602,18 @@ schema = 3 [mod."github.com/mattn/go-sqlite3"] version = "v1.14.34" hash = "sha256-PGOevTQb3gNdGu4H2bhvXJlntkMkDt+UhuiK1IpuorQ=" + [mod."github.com/mdlayher/socket"] + version = "v0.6.0" + hash = "sha256-Qpwu6m0PqySAGxSkHfME00YLO+lhBc825EGqASDpob8=" + [mod."github.com/mdlayher/vsock"] + version = "v1.3.0" + hash = "sha256-rkN/QRzs5hZ3idZ131yrNEkwfe+cyRkAizumJMQmFLQ=" [mod."github.com/microcosm-cc/bluemonday"] version = "v1.0.27" hash = "sha256-EZSya9FLPQ83CL7N2cZy21fdS35hViTkiMK5f3op8Es=" + [mod."github.com/miekg/dns"] + version = "v1.1.72" + hash = "sha256-i5TwAlGQjz6lF2SNc7MWuTLmxR/7f2Uomco3V0rKaRU=" [mod."github.com/minio/sha256-simd"] version = "v1.0.1" hash = "sha256-4hfGDIQaWq8fvtGzHDhoK9v2IocXnJY7OAL6saMJbmA=" @@ -584,6 +626,9 @@ schema = 3 [mod."github.com/moby/sys/atomicwriter"] version = "v0.1.0" hash = "sha256-i46GNrsICnJ0AYkN+ocbVZ2GNTQVEsrVX5WcjKzjtBM=" + [mod."github.com/moby/sys/userns"] + version = "v0.1.0" + hash = "sha256-zwXKyEZIH/FZjSVuSGmtwThDxPutj1pY+N6Ae6oVPuc=" [mod."github.com/moby/term"] version = "v0.5.2" hash = "sha256-/G20jUZKx36ktmPU/nEw/gX7kRTl1Dbu7zvNBYNt4xU=" @@ -641,6 +686,9 @@ schema = 3 [mod."github.com/opencontainers/image-spec"] version = "v1.1.1" hash = "sha256-bxBjtl+6846Ed3QHwdssOrNvlHV6b+Dn17zPISSQGP8=" + [mod."github.com/opencontainers/runtime-spec"] + version = "v1.3.0" + hash = "sha256-B2QF7FlUYZDL9eNA0+JD7WasnBryMXNIDbdSGS4MMG4=" [mod."github.com/opentracing/opentracing-go"] version = "v1.2.1-0.20220228012449-10b1cf09e00b" hash = "sha256-77oWcDviIoGWHVAotbgmGRpLGpH5AUy+pM15pl3vRrw=" @@ -693,6 +741,9 @@ schema = 3 [mod."github.com/sethvargo/go-envconfig"] version = "v1.1.0" hash = "sha256-WelRHfyZG9hrA4fbQcfBawb2ZXBQNT1ourEYHzQdZ4w=" + [mod."github.com/sirupsen/logrus"] + version = "v1.9.3" + hash = "sha256-EnxsWdEUPYid+aZ9H4/iMTs1XMvCLbXZRDyvj89Ebms=" [mod."github.com/spaolacci/murmur3"] version = "v1.1.0" hash = "sha256-RWD4PPrlAsZZ8Xy356MBxpj+/NZI7w2XOU14Ob7/Y9M=" @@ -793,32 +844,44 @@ schema = 3 version = "v2.4.3" hash = "sha256-WqfrOUQFvfuORgl1yyVOcsEXU/vwWQHkcVWx3vCxvaw=" [mod."golang.org/x/crypto"] - version = "v0.48.0" - hash = "sha256-uBIGGSGmWWklRxX6XTOqUECzz165UFY9Y99Ka3pLKAw=" + version = "v0.51.0" + hash = "sha256-/R74sc1mcOaOuBeXRQzrXrHAgA5VhNWc6SfQJaxb17U=" [mod."golang.org/x/exp"] version = "v0.0.0-20260112195511-716be5621a96" hash = "sha256-rWqwXzLvvhcI/ZkOQMqCXMKI5FAuHd9YNoKTXujmboA=" [mod."golang.org/x/image"] version = "v0.31.0" hash = "sha256-ZFTlu9+4QToPPLA8C5UcG2eq/lQylq81RoG/WtYo9rg=" + [mod."golang.org/x/mod"] + version = "v0.35.0" + hash = "sha256-ICEQxokHywOFInDPqoP+go9l1tZSz3roknF5SXPtNV4=" [mod."golang.org/x/net"] - version = "v0.50.0" - hash = "sha256-A3tvRuVotO4d8S1FX9ri9CpMJacrFJmHebLJ5m9b+Ss=" + version = "v0.55.0" + hash = "sha256-Phi2mSmBGOJcvqPPAit3uqF3UP8SKRI9dHj6yTM3s5s=" [mod."golang.org/x/sync"] - version = "v0.19.0" - hash = "sha256-RbRZ+sKZUurOczGhhzOoY/sojTlta3H9XjL4PXX/cno=" + version = "v0.20.0" + hash = "sha256-ybcjhCfK6lroUM0yswUvWooW8MOQZBXyiSqoxG6Uy0Y=" [mod."golang.org/x/sys"] - version = "v0.41.0" - hash = "sha256-owjs3/IzAKfFlIz1U1fiHSfl2+bTUhaXTyWEjL5SWHk=" + version = "v0.45.0" + hash = "sha256-hkBoNazrDA67ER6sWhb+EKxx9nJ24+nz3zGy+zT5Hvw=" [mod."golang.org/x/text"] - version = "v0.34.0" - hash = "sha256-wGKd1JkeiFROibvo2kkAuQ7JajSIfV4utGaoGbTQhQM=" + version = "v0.37.0" + hash = "sha256-8XDOnlPIybcDRy89fkjG5VqtIt5Ku+LmaqYhgKl7i1E=" [mod."golang.org/x/time"] version = "v0.12.0" hash = "sha256-Cp3oxrCMH2wyxjzr5SHVmyhgaoUuSl56Uy00Q7DYEpw=" + [mod."golang.org/x/tools"] + version = "v0.44.0" + hash = "sha256-xuj5FLtSJsAojLLTLXtPdLAIFNTKoVFbDMuqRXmj2W4=" [mod."golang.org/x/xerrors"] version = "v0.0.0-20240903120638-7835f813f4da" hash = "sha256-bE7CcrnAvryNvM26ieJGXqbAtuLwHaGcmtVMsVnksqo=" + [mod."google.golang.org/genproto/googleapis/api"] + version = "v0.0.0-20260209200024-4cfbd4190f57" + hash = "sha256-2C7DZwLpDDdmUhVUcRDaotbtkhQFOQ9a1SsdVC8lOqc=" + [mod."google.golang.org/genproto/googleapis/rpc"] + version = "v0.0.0-20260209200024-4cfbd4190f57" + hash = "sha256-gdgUw1LzgVOrarF1cGBUI9uoaR/d6lur2RwxUDKnOZA=" [mod."google.golang.org/protobuf"] version = "v1.36.11" hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE=" diff --git a/nix/microvm/base.nix b/nix/microvm/base.nix new file mode 100644 index 00000000..677143c0 --- /dev/null +++ b/nix/microvm/base.nix @@ -0,0 +1,309 @@ +{ + config, + pkgs, + lib, + ... +}: let + # these are modules / module trees we keep. everything else is pruned. + # this is a "cheap" way to save on what we ship, we don't have to recompile anything. + # this saves about 118mb! + keepTrees = [ + "crypto" + "lib" + "arch" + "drivers/virtio" + # net: the firewall modprobes across netfilter/ipv4/ipv6; docker adds the + # bridge/llc/802(stp)/xfrm machinery + NAT targets in netfilter. + "net/core" + "net/netfilter" + "net/ipv4" + "net/ipv6" + "net/packet" + "net/sched" + "net/vmw_vsock" + "net/bridge" + "net/llc" + "net/802" + "net/xfrm" + "fs/configfs" + "fs/autofs" + "fs/nls" + "fs/unicode" + ]; + keepMods = [ + # boot + storage + common workflow filesystems + "erofs" + "ext4" + "jbd2" + "mbcache" + "overlay" + "fuse" + "loop" + # "btrfs" + # "xfs" + # "f2fs" + # "vfat" + # "exfat" + "squashfs" + "isofs" + "dm-mod" + "zram" + # virtio devices the runner exposes + "virtio" + "virtio_mmio" + "virtio_pci" + "virtio_blk" + "virtio_net" + "virtio_rng" + "virtio_console" + "vsock_loopback" + "vmw_vsock_virtio_transport" + "vmw_vsock_virtio_transport_common" + # container networking (docker default bridge + common custom networks) + "veth" + "tun" + "tap" + "bridge" + "br_netfilter" + "macvlan" + "ipvlan" + "vxlan" + "geneve" + "dummy" + "wireguard" + ]; + keepTreesFile = pkgs.writeText "keep-trees" (lib.concatStringsSep "\n" keepTrees); + keepModsFile = pkgs.writeText "keep-mods" (lib.concatStringsSep "\n" keepMods); + slimModulesScript = pkgs.writeText "slim-modules.py" '' + import os, shutil, sys + + src, dst, trees_file, mods_file = sys.argv[1:5] + KEEP_TREES = open(trees_file).read().split() + KEEP_MODS = open(mods_file).read().split() + + def norm(name): + return name.replace("-", "_") + + kerneldir = os.path.join(src, "kernel") + + bypath, byname = {}, {} + for root, _, files in os.walk(kerneldir): + for f in files: + if ".ko" not in f: + continue + ap = os.path.join(root, f) + rel = os.path.relpath(ap, src) + bypath[rel] = ap + byname[norm(f.split(".ko")[0])] = rel + + deps = {} + with open(os.path.join(src, "modules.dep")) as fh: + for line in fh: + if ":" in line: + mod, rest = line.split(":", 1) + deps[mod.strip()] = rest.split() + + keep = set() + def add(rel): + if rel in keep or rel not in bypath: + return + keep.add(rel) + for dep in deps.get(rel, []): + add(dep) + + for tree in KEEP_TREES: + for root, _, files in os.walk(os.path.join(kerneldir, tree)): + for f in files: + if ".ko" in f: + add(os.path.relpath(os.path.join(root, f), src)) + for mod in KEEP_MODS: + rel = byname.get(norm(mod)) + if rel: + add(rel) + + for rel in keep: + target = os.path.join(dst, rel) + os.makedirs(os.path.dirname(target), exist_ok=True) + shutil.copy2(bypath[rel], target) + print(f"kept {len(keep)} of {len(bypath)} modules") + ''; + slimKernelModules = + pkgs.runCommand "${config.boot.kernelPackages.kernel.name}-modules-microvm" + {nativeBuildInputs = [pkgs.python3 pkgs.kmod];} + '' + src=${lib.getOutput "modules" config.boot.kernelPackages.kernel}/lib/modules + ver=$(ls "$src") + mkdir -p "$out/lib/modules/$ver" + for f in "$src/$ver"/modules.builtin* "$src/$ver"/modules.order; do + [ -e "$f" ] && cp "$f" "$out/lib/modules/$ver/" + done + python3 ${slimModulesScript} "$src/$ver" "$out/lib/modules/$ver" ${keepTreesFile} ${keepModsFile} + # regen modules.dep + depmod -b "$out" "$ver" + ''; +in { + system.stateVersion = "26.05"; + + # actually use our slimmed down modules set + system.modulesTree = lib.mkForce ([slimKernelModules] ++ config.boot.extraModulePackages); + + boot.initrd.includeDefaultModules = lib.mkForce false; + boot.initrd.availableKernelModules = lib.mkForce []; + boot.initrd.kernelModules = lib.mkForce [ + "virtio_pci" + "virtio_mmio" + "virtio_blk" + "virtio_console" + "erofs" + "ext4" + "overlay" + ]; + boot.kernelModules = ["loop"]; + + # some zram to help situations where burst memory usage causes OOM + zramSwap = { + enable = true; + algorithm = "zstd"; + memoryPercent = 50; + }; + + programs.nano.enable = false; + # we are on a microvm we don't need the hardware map + environment.etc."udev/hwdb.bin".enable = lib.mkForce false; + + networking.hostName = "spindle-microvm"; + networking.useDHCP = false; + systemd.network.networks."40-eth0" = { + matchConfig.Name = "eth0"; + address = ["10.0.3.15/24"]; + gateway = ["10.0.3.2"]; + dns = ["127.0.0.1"]; + }; + networking.nameservers = ["127.0.0.1"]; + + # this is disabled by microvm optimizations but we do need it + system.switch.enable = lib.mkForce true; + + # don't install docs or any xdg things, not necessary + documentation.enable = false; + xdg.mime.enable = false; + xdg.icons.enable = false; + xdg.sounds.enable = false; + + users.groups.spindle-workflow = { + gid = 970; + }; + users.users.spindle-workflow = { + isSystemUser = true; + uid = 970; + group = "spindle-workflow"; + home = "/workspace"; + createHome = false; + }; + users.users.spindle-workflow.extraGroups = lib.mkIf config.virtualisation.docker.enable [ + "docker" + ]; + virtualisation.docker.listenOptions = [ + "/run/docker.sock" + "/var/run/docker.sock" + ]; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + trusted-users = ["root"]; + allowed-users = ["spindle-workflow"]; + }; + registry.nixpkgs.to = { + type = "path"; + path = pkgs.path; + }; + extraOptions = '' + extra-experimental-features = nix-command flakes + !include /run/spindle/nix.conf + ''; + nixPath = ["nixpkgs=${config.nix.registry.nixpkgs.to.path}"]; + }; + + systemd.tmpfiles.rules = [ + "d /run/spindle 0755 root root -" + "d /workspace 0755 spindle-workflow spindle-workflow -" + "d /workspace/repo 0755 spindle-workflow spindle-workflow -" + ]; + + # add any common packages / services here + environment.systemPackages = [pkgs.gitMinimal]; + # disable default nixos packages ([perl rsync strace]) + environment.defaultPackages = []; + # this removed nixos-rebuild-ng and nixos-generate-config, which lets us + # remove python3 closure (~107MB) + system.disableInstallerTools = true; + + # a single volume that will back /workspace, /var, and the nix store + microvm.storeOnDisk = true; + microvm.storeDiskType = "erofs"; + # lz4hc, not zstd: the stock nixpkgs kernel builds erofs without + # CONFIG_EROFS_FS_ZIP_ZSTD, so a zstd image fails to mount at boot ("algorithm + # 3 isn't enabled on this kernel"); only lz4 is guaranteed. -Efragments and + # -Ededupe are omitted because microvm.nix falls back to single-threaded + # erofs-utils when either is present, which makes image builds really slow. + # for now, we take the compression hit, which isn't too much anyway. + # todo(dawn): the remaining big save needs a custom guest kernel (we'd want a + # binary cache first so downstream users don't rebuild it every time): enable + # EROFS_FS_ZIP_ZSTD for a better-compressing store-disk, build the essentials + # (virtio/erofs/ext4/overlay/netfilter) in as =y, and strip the kernel image + # itself. the modules tree is already pruned without a recompile, see + # slimKernelModules above. + microvm.storeDiskErofsFlags = [ + "-zlz4hc" + "-Eztailpacking" + "-C131072" # bigger compression window lets lz4hc compress better (~47mb) + ]; + microvm.writableStoreOverlay = "/persist/rw-store"; + microvm.volumes = [ + { + image = "persist.img"; + mountPoint = "/persist"; + size = 1024 * 16; + fsType = "ext4"; + } + ]; + + # /persist must be mounted before the writable store overlay activates + fileSystems."/persist".neededForBoot = true; + + fileSystems."/workspace" = { + device = "/persist/workspace"; + fsType = "none"; + options = ["bind"]; + depends = ["/persist"]; + }; + # bind mounting /var is important since docker etc. can't use overlayfs + # (overlayfs on overlayfs does not work) + fileSystems."/var" = { + device = "/persist/var"; + fsType = "none"; + options = ["bind"]; + depends = ["/persist"]; + }; + + # create bind sources before local-fs.target, which means we have to do this + # at initrd time + boot.initrd.systemd.enable = true; + boot.initrd.systemd.tmpfiles.settings."00-persist-layout" = { + "/sysroot/persist/rw-store".d = { + mode = "0755"; + }; + "/sysroot/persist/workspace".d = { + mode = "0755"; + user = "spindle-workflow"; + group = "spindle-workflow"; + }; + "/sysroot/persist/var".d = { + mode = "0755"; + }; + }; +} diff --git a/nix/microvm/qemu.nix b/nix/microvm/qemu.nix new file mode 100644 index 00000000..ceb4b34a --- /dev/null +++ b/nix/microvm/qemu.nix @@ -0,0 +1,25 @@ +{...}: { + microvm = { + hypervisor = "qemu"; + qemu.machine = "microvm"; + + optimize.enable = true; + + vcpu = 2; + # don't set to 2048, https://github.com/microvm-nix/microvm.nix/issues/171 + mem = 2049; + + interfaces = [ + { + type = "user"; + id = "net0"; + mac = "02:00:00:00:10:01"; + } + ]; + vsock.cid = 3; + + socket = "control.socket"; + }; + + boot.kernelModules = ["vsock_loopback"]; +} diff --git a/nix/microvm/spindle-vm.nix b/nix/microvm/spindle-vm.nix new file mode 100644 index 00000000..2b5d89c4 --- /dev/null +++ b/nix/microvm/spindle-vm.nix @@ -0,0 +1,48 @@ +{ + self, + microvm, +}: runnerModule: { + imports = [ + microvm.nixosModules.microvm + ./base.nix + runnerModule + self.nixosModules.shuttle + ({pkgs, ...}: { + services.tangled.shuttle.enable = true; + + environment.etc = { + "spindle/nixos/base.nix".source = ./base.nix; + "spindle/nixos/runner.nix".source = runnerModule; + "spindle/nixos/shuttle.nix".text = '' + { config, lib, pkgs, ... }: + { + imports = [${../modules/shuttle.nix}]; + services.tangled.shuttle.package = lib.mkDefault ${self.packages.${pkgs.stdenv.hostPlatform.system}.shuttle}; + } + ''; + "spindle/nixos/user-config.nix".source = ./user-config.nix; + "spindle/nixos/microvm".source = microvm; + # pkgs.path is fine here because we pass the nixpkgs source into the vm in ./base.nix + "spindle/nixos/default.nix".text = '' + let + nixpkgs = ${pkgs.path}; + nixos = import (nixpkgs + "/nixos") { + system = "${pkgs.stdenv.hostPlatform.system}"; + configuration = { + imports = [ + /etc/spindle/nixos/microvm/nixos-modules/microvm/default.nix + /etc/spindle/nixos/base.nix + /etc/spindle/nixos/runner.nix + /etc/spindle/nixos/shuttle.nix + /etc/spindle/nixos/user-config.nix + ]; + services.tangled.shuttle.enable = true; + }; + }; + in + nixos.system + ''; + }; + }) + ]; +} diff --git a/nix/microvm/user-config.nix b/nix/microvm/user-config.nix new file mode 100644 index 00000000..989b33a2 --- /dev/null +++ b/nix/microvm/user-config.nix @@ -0,0 +1,110 @@ +{ + pkgs, + lib, + options, + ... +} @ args: let + configPath = /run/spindle/user-config/config.json; + userConfig = + args.userConfig + or ( + if builtins.pathExists configPath + then lib.importJSON configPath + else {} + ); + + registry = userConfig.registry or {}; + + # registry targets may be structured attrs or flake ref strings; strings are + # parsed by nix itself in getFlake. flakeRefToString rejects unforced attr + # values, hence the toJSON round-trip + toRefString = target: + if builtins.isAttrs target + then builtins.flakeRefToString (builtins.fromJSON (builtins.toJSON target)) + else target; + + # user registry entries shadow the system registry (which pins nixpkgs) + getFlake = ref: builtins.getFlake (toRefString (registry.${ref} or ref)); + + # "flakeref#attr" or a bare attr looked up in nixpkgs. nixpkgs refs use the + # already-evaluated pkgs directly instead of re-evaluating via getFlake, + # unless the user remapped nixpkgs in their registry + resolvePackage = ref: let + parts = lib.splitString "#" ref; + hasAttr = lib.length parts > 1; + flakeRef = + if hasAttr + then lib.head parts + else "nixpkgs"; + pkgName = + if hasAttr + then lib.elemAt parts 1 + else ref; + system = pkgs.stdenv.hostPlatform.system; + flake = getFlake flakeRef; + notFound = throw "Package ${pkgName} not found in ${flakeRef}"; + in + if flakeRef == "nixpkgs" && !(registry ? nixpkgs) + then pkgs.${pkgName} or notFound + else flake.legacyPackages.${system}.${pkgName} or flake.packages.${system}.${pkgName} or notFound; + + # strings are resolved as package references only where the option type + # actually expects packages; everything else passes through untouched + resolveForType = type: v: + if type.name == "package" && builtins.isString v + then resolvePackage v + # path-typed options (e.g. services.udev.packages) accept derivations via + # coercion; "#" disambiguates flake refs from actual paths, which are + # always absolute + else if type.name == "path" && builtins.isString v && lib.hasInfix "#" v && !lib.hasPrefix "/" v + then resolvePackage v + else if type.name == "nullOr" && v != null + then resolveForType type.nestedTypes.elemType v + else if type.name == "listOf" && builtins.isList v + then map (resolveForType type.nestedTypes.elemType) v + else if (type.name == "attrsOf" || type.name == "lazyAttrsOf") && builtins.isAttrs v + then builtins.mapAttrs (_: resolveForType type.nestedTypes.elemType) v + else if type.name == "submodule" && builtins.isAttrs v + then resolveOptions (type.getSubOptions []) v + else v; + + resolveOptions = opts: builtins.mapAttrs (name: resolveValue (opts.${name} or null)); + + resolveValue = opt: v: + if !builtins.isAttrs opt + then v + else if lib.isOption opt + then resolveForType opt.type v + else if builtins.isAttrs v + then resolveOptions opt v + else v; + + # `foo = true` is shorthand for `foo.enable = true`, but only when an + # enable option actually exists under foo + hasEnableOption = opt: + builtins.isAttrs opt + && ( + if lib.isOption opt + then (opt.type.getSubOptions opt.loc) ? enable + else opt ? enable && lib.isOption opt.enable + ); + + normalize = opts: name: v: let + opt = opts.${name} or null; + in + if builtins.isBool v && hasEnableOption opt + then {enable = v;} + else resolveValue opt v; +in { + nix.registry = builtins.mapAttrs (name: _: + lib.mkForce { + to = { + type = "path"; + path = (getFlake name).outPath; + }; + }) + registry; + environment.systemPackages = map resolvePackage (userConfig.dependencies or []); + services = builtins.mapAttrs (normalize (options.services or {})) (userConfig.services or {}); + virtualisation = builtins.mapAttrs (normalize (options.virtualisation or {})) (userConfig.virtualisation or {}); +} diff --git a/nix/modules/shuttle.nix b/nix/modules/shuttle.nix new file mode 100644 index 00000000..c209b7dc --- /dev/null +++ b/nix/modules/shuttle.nix @@ -0,0 +1,57 @@ +{ + config, + lib, + pkgs, + ... +}: let + cfg = config.services.tangled.shuttle; + + postBuildHook = pkgs.writeShellApplication { + name = "spindle-post-build-hook"; + text = '' + set -f + + if [ -z "''${OUT_PATHS:-}" ]; then + exit 0 + fi + + # OUT_PATHS is intentionally split into individual store paths for the agent + # shellcheck disable=SC2086 + exec ${cfg.package}/bin/shuttle enqueue-built-paths $OUT_PATHS + ''; + }; +in { + options.services.tangled.shuttle = { + enable = lib.mkEnableOption "the shuttle guest agent"; + + package = lib.mkOption { + type = lib.types.package; + description = "package providing the shuttle executable."; + }; + }; + + config = lib.mkIf cfg.enable { + nix.settings.post-build-hook = "${postBuildHook}/bin/spindle-post-build-hook"; + + systemd.services.shuttle = { + description = "shuttle guest agent"; + wantedBy = ["multi-user.target"]; + wants = ["network-online.target"]; + after = [ + "local-fs.target" + "network-online.target" + ]; + before = ["nix-daemon.service"]; + restartIfChanged = false; + environment = { + NIX_PATH = lib.concatStringsSep ":" config.nix.nixPath; + }; + serviceConfig = { + Type = "simple"; + ExecStart = "${cfg.package}/bin/shuttle"; + Restart = "always"; + RestartSec = "1s"; + }; + }; + }; +} diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix index 502a681b..3edbfe22 100644 --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -1,6 +1,7 @@ { config, lib, + pkgs, ... }: let cfg = config.services.tangled.spindle; @@ -75,12 +76,6 @@ in description = "Maximum number of jobs queue up"; }; - maxConcurrentWorkflows = mkOption { - type = types.int; - default = 8; - description = "Maximum number of workflow containers running simultaneously (controls total memory usage)"; - }; - secrets = { provider = mkOption { type = types.str; @@ -136,28 +131,155 @@ in }; pipelines = { - nixery = mkOption { + logBucket = mkOption { type = types.str; - default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet - description = "Nixery instance to use"; + default = "tangled-logs"; + description = "S3 bucket for workflow logs"; }; - workflowTimeout = mkOption { type = types.str; default = "5m"; - description = "Timeout for each step of a pipeline"; + description = "Timeout for each workflow step"; }; - maxJobMemoryMb = mkOption { - type = types.int; - default = 6144; - description = "Memory limit per workflow container in MiB (default 6 GiB)"; + nixery = { + nixery = mkOption { + type = types.str; + default = "nixery.tangled.sh"; # note: this is *not* on tangled.org yet + description = "Nixery instance to use"; + }; + + maxJobMemoryMb = mkOption { + type = types.int; + default = 6144; + description = "Memory limit per nixery workflow container in MiB (default 6 GiB)"; + }; + maxConcurrentWorkflows = mkOption { + type = types.int; + default = 8; + description = "Maximum number of nixery workflows running simultaneously. Zero disables this limit."; + }; }; - logBucket = mkOption { + microvm = { + enableKVM = mkOption { + type = types.bool; + default = true; + description = "Enable KVM hardware acceleration"; + }; + + imageDir = mkOption { + type = types.str; + default = "/var/lib/spindle/images"; + description = "Directory containing microVM image spec JSONs or image spec directories"; + }; + overlayDir = mkOption { + type = types.str; + default = "/tmp"; + description = "Directory to store microVM temporary overlay files"; + }; + defaultImage = mkOption { + type = types.str; + default = "nixos"; + description = "Default microVM image spec to use if none is specified in workflow"; + }; + agentPort = mkOption { + type = types.port; + default = 10240; + description = "Host vsock port the microVM agent connects back to"; + }; + + limits = { + total = { + memoryMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared guest memory in MiB allowed across all running microVM workflows. Zero disables this limit."; + }; + vcpus = mkOption { + type = types.int; + default = 0; + description = "Maximum declared vCPUs allowed across all running microVM workflows. Zero disables this limit."; + }; + diskMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared disk in MiB allowed across all running microVM workflows. Zero disables this limit."; + }; + }; + + workflow = { + memoryMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared guest memory in MiB allowed for a single microVM workflow. Zero disables this limit."; + }; + vcpus = mkOption { + type = types.int; + default = 0; + description = "Maximum declared vCPUs allowed for a single microVM workflow. Zero disables this limit."; + }; + diskMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum declared disk in MiB allowed for a single microVM workflow. Zero disables this limit."; + }; + }; + }; + + cgroup = { + enable = mkOption { + type = types.bool; + default = false; + description = "Enable cgroup v2 containment for microVM processes."; + }; + parent = mkOption { + type = types.str; + default = "self"; + description = "Parent cgroup for microVM workflow cgroups. Use 'self' to resolve the spindle service cgroup."; + }; + pidsMax = mkOption { + type = types.int; + default = 4096; + description = "Maximum number of processes allowed in each microVM workflow cgroup."; + }; + swapMaxMiB = mkOption { + type = types.int; + default = 0; + description = "Maximum swap in MiB allowed in each microVM workflow cgroup. Zero disables swap."; + }; + supervisorMinMiB = mkOption { + type = types.int; + default = 512; + description = '' + Amount of memory in MiB that will be protected by the cgroup for the spindle + (allowing it to not get OOMed first.) + ''; + }; + }; + }; + }; + + cache = { + readUrls = mkOption { + type = types.listOf types.str; + default = []; + example = ["http://ncps.internal:8501"]; + description = "Nix binary cache URLs the Spindle guest should read from."; + }; + + trustedPublicKeys = mkOption { + type = types.listOf types.str; + default = []; + example = ["ncps.internal-1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="]; + description = "Public keys trusted for the configured Nix binary caches."; + }; + + uploadUrl = mkOption { type = types.str; - default = "tangled-logs"; - description = "S3 bucket for workflow logs"; + default = ""; + example = "http://ncps.internal:8501/upload"; + description = "Optional cache upload URL used by live cache import paths."; }; }; @@ -177,45 +299,91 @@ in }; }; - config = mkIf cfg.enable { - virtualisation.docker.enable = true; - - systemd.services.spindle = { - description = "spindle service"; - after = ["network.target" "docker.service"]; - wantedBy = ["multi-user.target"]; - serviceConfig = { - LogsDirectory = "spindle"; - StateDirectory = "spindle"; - EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; - - Environment = [ - "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}" - "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}" - "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}" - "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}" - "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}" - "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" - "SPINDLE_SERVER_OWNER=${cfg.server.owner}" - "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}" - "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}" - "SPINDLE_SERVER_MAX_CONCURRENT_WORKFLOWS=${toString cfg.server.maxConcurrentWorkflows}" - "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" - "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" - "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" - "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}" - "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}" - "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}" - "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}" - "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}" - "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" - "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" - "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.maxJobMemoryMb}" - "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" + config = let + deps = [ + pkgs.qemu + pkgs.e2fsprogs + pkgs.slirp4netns + pkgs.iproute2 + pkgs.util-linux + ]; + in + mkIf cfg.enable { + environment.systemPackages = [ + (pkgs.writeShellScriptBin "spindle" '' + export PATH="${lib.makeBinPath deps}:$PATH" + ${lib.optionalString (cfg.environmentFile != null) "set -a; source ${cfg.environmentFile}; set +a"} + ${lib.concatMapStringsSep "\n" ( + e: "export ${e}" + ) + config.systemd.services.spindle.serviceConfig.Environment} + exec ${cfg.package}/bin/spindle "$@" + '') + ]; + + virtualisation.docker.enable = true; + + systemd.services.spindle = { + description = "spindle service"; + after = [ + "network.target" + "docker.service" ]; - ExecStart = "${cfg.package}/bin/spindle"; - Restart = "always"; + wantedBy = ["multi-user.target"]; + path = deps; + serviceConfig = { + LogsDirectory = "spindle"; + StateDirectory = "spindle"; + Delegate = cfg.pipelines.microvm.cgroup.enable; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; + + Environment = [ + "SPINDLE_SERVER_LISTEN_ADDR=${cfg.server.listenAddr}" + "SPINDLE_SERVER_DB_PATH=${cfg.server.dbPath}" + "SPINDLE_SERVER_HOSTNAME=${cfg.server.hostname}" + "SPINDLE_SERVER_PLC_URL=${cfg.server.plcUrl}" + "SPINDLE_SERVER_JETSTREAM_ENDPOINT=${cfg.server.jetstreamEndpoint}" + "SPINDLE_SERVER_DEV=${lib.boolToString cfg.server.dev}" + "SPINDLE_SERVER_OWNER=${cfg.server.owner}" + "SPINDLE_SERVER_MAX_JOB_COUNT=${toString cfg.server.maxJobCount}" + "SPINDLE_SERVER_QUEUE_SIZE=${toString cfg.server.queueSize}" + "SPINDLE_SERVER_SECRETS_PROVIDER=${cfg.server.secrets.provider}" + "SPINDLE_SERVER_SECRETS_OPENBAO_PROXY_ADDR=${cfg.server.secrets.openbao.proxyAddr}" + "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" + "SPINDLE_SERVER_TAP_EMBED=${lib.boolToString cfg.server.tap.embed}" + "SPINDLE_SERVER_TAP_URL=${cfg.server.tap.url}" + "SPINDLE_SERVER_TAP_BIND=${cfg.server.tap.bind}" + "SPINDLE_SERVER_TAP_DB_PATH=${cfg.server.tap.dbPath}" + "SPINDLE_SERVER_TAP_RELAY_URL=${cfg.server.tap.relayUrl}" + "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery.nixery}" + "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" + "SPINDLE_NIXERY_PIPELINES_MAX_JOB_MEMORY_MB=${toString cfg.pipelines.nixery.maxJobMemoryMb}" + "SPINDLE_NIXERY_PIPELINES_MAX_CONCURRENT_WORKFLOWS=${toString cfg.pipelines.nixery.maxConcurrentWorkflows}" + "SPINDLE_MICROVM_PIPELINES_IMAGE_DIR=${cfg.pipelines.microvm.imageDir}" + "SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR=${cfg.pipelines.microvm.overlayDir}" + "SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE=${cfg.pipelines.microvm.defaultImage}" + "SPINDLE_MICROVM_PIPELINES_AGENT_PORT=${toString cfg.pipelines.microvm.agentPort}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_KVM=${lib.boolToString cfg.pipelines.microvm.enableKVM}" + "SPINDLE_MICROVM_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.total.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_VCPUS=${toString cfg.pipelines.microvm.limits.total.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_TOTAL_DISK_MIB=${toString cfg.pipelines.microvm.limits.total.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_MEMORY_MIB=${toString cfg.pipelines.microvm.limits.workflow.memoryMiB}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_VCPUS=${toString cfg.pipelines.microvm.limits.workflow.vcpus}" + "SPINDLE_MICROVM_PIPELINES_MAX_WORKFLOW_DISK_MIB=${toString cfg.pipelines.microvm.limits.workflow.diskMiB}" + "SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS=${lib.boolToString cfg.pipelines.microvm.cgroup.enable}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PARENT=${cfg.pipelines.microvm.cgroup.parent}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_PIDS_MAX=${toString cfg.pipelines.microvm.cgroup.pidsMax}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SWAP_MAX_MIB=${toString cfg.pipelines.microvm.cgroup.swapMaxMiB}" + "SPINDLE_MICROVM_PIPELINES_CGROUP_SUPERVISOR_MEMORY_MIN_MIB=${toString cfg.pipelines.microvm.cgroup.supervisorMinMiB}" + "SPINDLE_NIX_CACHE_READ_URLS=${concatStringsSep "," cfg.cache.readUrls}" + "SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS=${concatStringsSep "," cfg.cache.trustedPublicKeys}" + "SPINDLE_NIX_CACHE_UPLOAD_URL=${cfg.cache.uploadUrl}" + "SPINDLE_S3_LOG_BUCKET=${cfg.pipelines.logBucket}" + ]; + ExecStart = "${cfg.package}/bin/spindle"; + Restart = "always"; + }; }; }; - }; } diff --git a/nix/pkgs/shuttle.nix b/nix/pkgs/shuttle.nix new file mode 100644 index 00000000..8ff32428 --- /dev/null +++ b/nix/pkgs/shuttle.nix @@ -0,0 +1,23 @@ +{ + rustPlatform, + src, + protobuf, + ... +}: let + flags = ["--bin" "shuttle" "--package" "shuttle"]; +in + rustPlatform.buildRustPackage { + pname = "shuttle"; + version = "0.1.0"; + + inherit src; + + cargoLock.lockFile = "${src}/Cargo.lock"; + + nativeBuildInputs = [ + protobuf + ]; + + cargoBuildFlags = flags; + cargoTestFlags = flags; + } diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix new file mode 100644 index 00000000..c7ca1290 --- /dev/null +++ b/nix/pkgs/spindle-alpine-image.nix @@ -0,0 +1,222 @@ +{ + pkgsStatic, + runCommand, + writeText, + squashfsTools, + shuttle, + binutils, + publicsuffix-list, + rootfs, + kernel, + initramfs, + modloop, + repositories, + arch ? "x86_64", +}: let + nix = pkgsStatic.nixStatic; + bash = pkgsStatic.bashNonInteractive; + git = + (pkgsStatic.gitMinimal.override { + curl = pkgsStatic.curlMinimal; + pythonSupport = false; + withManual = false; + nlsSupport = false; + }).overrideAttrs (old: { + doCheck = false; + doInstallCheck = false; + configureFlags = (old.configureFlags or []) ++ ["ac_cv_lib_curl_curl_global_init=yes"]; + }); + guestTools = [nix bash git]; + + # run by busybox at sysinit + setupScript = writeText "spindle-setup" '' + #!/bin/sh + + mountpoint -q /proc || mount -t proc proc /proc + mountpoint -q /sys || mount -t sysfs sys /sys + mountpoint -q /dev || mount -t devtmpfs dev /dev + mountpoint -q /dev/pts || { + mkdir -p /dev/pts + mount -t devpts devpts /dev/pts + } + mountpoint -q /dev/shm || { + mkdir -p /dev/shm + mount -t tmpfs -o mode=1777 shm /dev/shm + } + mountpoint -q /run || mount -t tmpfs -o mode=0755 run /run + mountpoint -q /tmp || mount -t tmpfs -o mode=1777 tmp /tmp + + # the initramfs mdev leaves these 0660, which breaks non-root workflows + chmod 666 /dev/null /dev/zero /dev/full /dev/random /dev/urandom /dev/tty /dev/ptmx 2>/dev/null + + modprobe vmw_vsock_virtio_transport + # shuttle's cache enqueue listener binds a guest-local (CID 1) vsock + modprobe vsock_loopback + modprobe ext4 + + # /dev/vda is the squashfs root; the first spindle volume backs /workspace + if [ -b /dev/vdb ]; then + mount -t ext4 /dev/vdb /workspace + mkdir -p /workspace/repo + chown spindle-workflow:spindle-workflow /workspace /workspace/repo + fi + + ip link set lo up + ip link set eth0 up + ip addr add 10.0.3.15/24 dev eth0 + ip route add default via 10.0.3.2 + hostname -F /etc/hostname + ''; + + inittab = writeText "inittab" '' + ::sysinit:/sbin/spindle-setup + ::respawn:/usr/local/bin/nix-daemon + ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle + ::ctrlaltdel:/sbin/reboot + ''; + + profileScript = writeText "spindle-profile" '' + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt + export NIX_REMOTE=daemon + ''; + + # mirror nix/microvm/base.nix and nix/modules/shuttle.nix + nixConf = writeText "nix.conf" '' + experimental-features = nix-command flakes + trusted-users = root + allowed-users = spindle-workflow + post-build-hook = /usr/libexec/spindle-post-build-hook + !include /run/spindle/nix.conf + ''; + + apkRepositories = writeText "apk-repositories" (builtins.concatStringsSep "\n" repositories + "\n"); + + postBuildHook = writeText "spindle-post-build-hook" '' + #!/bin/sh + set -f + + if [ -z "''${OUT_PATHS:-}" ]; then + exit 0 + fi + + # OUT_PATHS is intentionally split into individual store paths + exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS + ''; + + imageSpecJSON = writeText "spec.json" ( + builtins.toJSON { + inherit arch; + bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 root=/dev/vda rootfstype=squashfs modules=virtio_blk,virtio_net,virtio_console overlaytmpfs=yes init=/sbin/init"; + kernel = "kernel"; + initrd = "initrd"; + runnerType = "qemu"; + runnerConfig = { + cpu = "host,+x2apic,-sgx"; + machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; + console = "hvc0"; + extraArgs = []; + }; + memoryMiB = 2048; + storeDisk = "store-disk"; + storeDiskType = "squashfs"; + vcpus = 2; + shell = "/usr/local/bin/bash"; + networkInterfaces = [ + { + type = "slirp4netns"; + id = "net0"; + mac = "02:00:00:00:10:01"; + } + ]; + volumes = [ + { + fsType = "ext4"; + image = "workspace.img"; + imageType = "raw"; + mountPoint = "/workspace"; + readOnly = false; + sizeMiB = 1024 * 10; + } + ]; + } + ); +in + runCommand "spindle-alpine-image-${arch}" { + nativeBuildInputs = [squashfsTools binutils]; + } '' + mkdir -p rootfs + tar -xzpf ${rootfs} -C rootfs + + # kernel modules from modloop (ships its own modules.dep, no depmod needed) + unsquashfs -q -d modloop ${modloop} + mkdir -p rootfs/lib/modules + cp -a modloop/modules/* rootfs/lib/modules/ + + install -D -m 0755 ${shuttle}/bin/shuttle rootfs/usr/bin/shuttle + install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup + install -D -m 0644 ${inittab} rootfs/etc/inittab + install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh + install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf + install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook + + # install dependencies + # we only copy binaries + libexec for minimal deps so the image size doesn't + # increase so much (if we copy the whole guestTools closure for example, it + # doubles the disk size) + mkdir -p rootfs/nix/store rootfs/usr/local/bin + for pkg in ${toString guestTools}; do + for bin in "$pkg/bin/"*; do + [[ -e "$bin" ]] || continue + name=$(basename "$bin") + # we resolve symlinks as to copy the actual binaries + if [[ -L "$bin" ]]; then + real=$(readlink "$bin") + else + real="$bin" + fi + # handle symlinks properly + if [[ "$real" != /nix/store* ]]; then + ln -vsf "$real" "rootfs/usr/local/bin/$name" + else + cp -v "$real" "rootfs/usr/local/bin/$name" + fi + done + # libexec has binaries used by packages even if statically compiled + if [[ -d "$pkg/libexec" ]]; then + mkdir -p "rootfs$pkg" + cp -av "$pkg/libexec" "rootfs$pkg/" + fi + done + # this is necessary for nix to work, it is not a library but nix hardcodes + # it in it's binary + cp -rv ${publicsuffix-list} rootfs/nix/store/ + + # scripts commonly hardcode #!/bin/bash + ln -sf ${bash}/bin/bash rootfs/bin/bash + + echo "spindle-microvm" > rootfs/etc/hostname + printf 'nameserver 127.0.0.1\n' > rootfs/etc/resolv.conf + install -D -m 0644 ${apkRepositories} rootfs/etc/apk/repositories + + echo "spindle-workflow:x:970:970:spindle workflow:/workspace:/bin/sh" >> rootfs/etc/passwd + echo "spindle-workflow:x:970:" >> rootfs/etc/group + echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow + mkdir -p rootfs/workspace + + # setup nix build users for the daemon + members="" + for i in $(seq 1 8); do + echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd + echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow + members="$members''${members:+,}nixbld$i" + done + echo "nixbld:x:30000:$members" >> rootfs/etc/group + + mkdir -p "$out" + mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet \ + -p '/sbin/apk m 4755 0 0' # suid apk so spindle-workflow can use it without having to doas or smth + cp ${kernel} "$out/kernel" + cp ${initramfs} "$out/initrd" + cp ${imageSpecJSON} "$out/spec.json" + '' diff --git a/nix/pkgs/spindle-nixos-image.nix b/nix/pkgs/spindle-nixos-image.nix new file mode 100644 index 00000000..af04f8f0 --- /dev/null +++ b/nix/pkgs/spindle-nixos-image.nix @@ -0,0 +1,58 @@ +{ + pkgs, + lib, + nixosSystem, +}: let + system = nixosSystem.pkgs.stdenv.hostPlatform.qemuArch; + microvm = nixosSystem.config.microvm; + baseConfigHash = lib.pipe nixosSystem.config.system.build.toplevel.outPath [ + (lib.strings.removePrefix "/nix/store/") + (lib.strings.splitString "-") + lib.head + ]; + imageSpecJSON = pkgs.writeText "spec.json" ( + builtins.toJSON { + arch = system; + bootArgs = "earlyprintk=ttyS0 console=hvc0 reboot=t panic=-1 ${lib.concatStringsSep " " microvm.kernelParams}"; + kernel = "kernel"; + initrd = "initrd"; + runnerType = "qemu"; + runnerConfig = { + cpu = "host,+x2apic,-sgx"; + machine = "microvm,accel=kvm:tcg,acpi=on,mem-merge=on,pcie=off,pic=off,pit=off,rtc=on,usb=off"; + console = "hvc0"; + extraArgs = []; + }; + memoryMiB = microvm.mem; + storeDisk = "store-disk"; + storeDiskType = microvm.storeDiskType; + vcpus = microvm.vcpu; + shell = "/run/current-system/sw/bin/bash"; + baseConfigHash = baseConfigHash; + networkInterfaces = + map (interface: { + type = "slirp4netns"; + id = interface.id; + mac = interface.mac; + }) + microvm.interfaces; + volumes = + map (volume: { + fsType = volume.fsType; + image = volume.image; + imageType = volume.imageType; + mountPoint = volume.mountPoint; + readOnly = volume.readOnly; + sizeMiB = volume.size; + }) + microvm.volumes; + } + ); +in + pkgs.runCommand "spindle-nixos-image-${system}" {} '' + mkdir -p "$out" + cp ${imageSpecJSON} "$out/spec.json" + ln -s ${microvm.kernel}/bzImage "$out/kernel" + ln -s ${microvm.initrdPath} "$out/initrd" + ln -s ${microvm.storeDisk} "$out/store-disk" + '' diff --git a/nix/vm.nix b/nix/vm.nix index bef5a80b..764a462d 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -4,6 +4,8 @@ hostSystem, self, }: let + lib = nixpkgs.lib; + envVar = name: let var = builtins.getEnv name; in @@ -19,8 +21,21 @@ plcUrl = envVarOr "TANGLED_VM_PLC_URL" "https://plc.directory"; jetstream = envVarOr "TANGLED_VM_JETSTREAM_ENDPOINT" "wss://jetstream1.us-west.bsky.network/subscribe"; + + checkFile = value: path: + if builtins.pathExists path + then lib.hasPrefix value (builtins.readFile path) + else false; + _nestedVirt = + (checkFile "1" /sys/module/kvm_amd/parameters/nested) + || (checkFile "Y" /sys/module/kvm_intel/parameters/nested); + nestedVirtWarning = '' + KVM nested virtualisation is not enabled on this host. + You should enable it if you can for better performance when testing the QEMU spindle engine! + ''; + nestedVirt = lib.warnIf (!_nestedVirt) nestedVirtWarning _nestedVirt; in - nixpkgs.lib.nixosSystem { + lib.nixosSystem { inherit system; modules = [ self.nixosModules.knot @@ -36,9 +51,11 @@ in host.pkgs = import nixpkgs {system = hostSystem;}; graphics = false; - memorySize = 2048; - diskSize = 10 * 1024; + memorySize = 3072; + diskSize = 20 * 1024; cores = 2; + qemu.options = lib.optionals nestedVirt ["-enable-kvm" "-cpu host"]; + forwardPorts = [ # ssh { @@ -101,6 +118,12 @@ in }; }; }; + systemd.tmpfiles.rules = [ + "L+ /var/lib/spindle/images/nixos-x86_64 - - - - ${self.packages.${system}.spindle-nixos-image}" + "L+ /var/lib/spindle/images/nixos - - - - /var/lib/spindle/images/nixos-x86_64" + "L+ /var/lib/spindle/images/alpine-x86_64 - - - - ${self.packages.${system}.spindle-alpine-image}" + "L+ /var/lib/spindle/images/alpine - - - - /var/lib/spindle/images/alpine-x86_64" + ]; # This is fine because any and all ports that are forwarded to host are explicitly marked above, we don't need a separate guest firewall networking.firewall.enable = false; services.timesyncd.enable = lib.mkForce true; @@ -141,7 +164,30 @@ in pipelines = { logBucket = envVarOr "SPINDLE_S3_LOG_BUCKET" ""; + microvm = { + enableKVM = nestedVirt; + }; + }; + + cache = { + readUrls = ["http://127.0.0.1:8501"]; + trustedPublicKeys = ["cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU="]; + uploadUrl = "http://127.0.0.1:8501/upload"; + }; + }; + services.ncps = { + enable = true; + cache = { + allowPutVerb = true; + allowDeleteVerb = true; + hostName = "cache.local"; + secretKeyPath = pkgs.writeText "ncps-secret-key" "cache.local:hay0+jvBNguou2tNt19FvrBCogHwHc+mqQe3bww5ZX4XtiqkzO4F0gth3+/7Axk3ZgrELOWJdDIWZ57M7HDutQ=="; + upstream = { + urls = ["https://cache.nixos.org"]; + publicKeys = ["cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="]; + }; }; + server.addr = "127.0.0.1:8501"; }; services.postgresql = { enable = true; @@ -191,7 +237,7 @@ in }; in { knot = mkDataSyncScripts "/mnt/knot-data" config.services.tangled.knot.stateDir; - spindle = mkDataSyncScripts "/mnt/spindle-data" (builtins.dirOf config.services.tangled.spindle.server.dbPath); + spindle = mkDataSyncScripts "/mnt/spindle-data" (dirOf config.services.tangled.spindle.server.dbPath); knotmirror.after = ["postgresql.target"]; tap-knotmirror.after = ["postgresql.target"]; }; diff --git a/shuttle/Cargo.toml b/shuttle/Cargo.toml new file mode 100644 index 00000000..86e3d99d --- /dev/null +++ b/shuttle/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "shuttle" +version = "0.1.0" +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +anyhow = "1" +base64 = "0.22" +nix = { version = "0.31", features = ["fs", "process", "reboot", "signal", "user"] } +prost = "0.14" +prost-reflect = "0.16" +prost-protovalidate = "0.3" +once_cell = "1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tempfile = "3" +tokio = { workspace = true, features = ["fs", "net", "process"] } +tokio-vsock = "0.7.2" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/shuttle/README.md b/shuttle/README.md new file mode 100644 index 00000000..8effc66f --- /dev/null +++ b/shuttle/README.md @@ -0,0 +1 @@ +shuttle is a tiny agent for micro VM guests that handles the communication between host <-> guest. diff --git a/shuttle/src/activation.rs b/shuttle/src/activation.rs new file mode 100644 index 00000000..bcd41449 --- /dev/null +++ b/shuttle/src/activation.rs @@ -0,0 +1,146 @@ +use crate::command::{self, Spec, run_capture}; +use crate::nix_config::{SPINDLE_RUN_DIR, nix_executable}; +use crate::protocol::{self, Message, v1}; +use anyhow::{Context, Result}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tokio::sync::mpsc::Sender; +use tracing::info; + +const USER_CONFIG_DIR: &str = "/run/spindle/user-config"; + +pub async fn run(id: String, req: v1::ActivateConfig, out: Sender) { + let config_key = req.config_key.clone(); + let result = activate(&req).await; + let msg = Message { + id, + activate_config_result: Some(v1::ActivateConfigResult { + config_key, + toplevel: (result.as_ref()) + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(), + error: protocol::error_or_empty(result.err().map(|e| format!("{e:#}"))), + }), + ..Default::default() + }; + let _ = out.send(msg).await; +} + +async fn activate(req: &v1::ActivateConfig) -> Result { + let need_build = req.toplevel.is_empty(); + let timeout = (req.timeout_seconds > 0) + .then(|| Duration::from_secs(u64::from(req.timeout_seconds))) + .or_else(|| need_build.then_some(Duration::from_secs(10 * 60))) + .unwrap_or(Duration::from_secs(2 * 60)); + + let toplevel = if need_build { + build_toplevel(req, timeout).await? + } else { + realise_toplevel(&req.toplevel, timeout).await? + }; + + if !toplevel.starts_with("/nix/store/") { + anyhow::bail!("config toplevel {toplevel:?} is not a nix store path"); + } + + switch_to_configuration(&toplevel, timeout).await?; + info!( + config_key = %req.config_key, + base_config_hash = %req.base_config_hash, + ?toplevel, + "activated NixOS config" + ); + Ok(toplevel) +} + +async fn build_toplevel(req: &v1::ActivateConfig, timeout: Duration) -> Result { + let user_config = (req.user_config.is_empty()) + .then_some("{}") + .unwrap_or_else(|| &req.user_config); + + info!("writing user config to {USER_CONFIG_DIR}/config.json"); + write_user_config(user_config).context("write user config")?; + + info!("running nix build command for user config toplevel..."); + let output = run_capture( + Spec::new(nix_executable()) + .args([ + "build", + "--no-link", + "--show-trace", + "--json", + "--file", + "/etc/spindle/nixos/default.nix", + ]) + .cwd(SPINDLE_RUN_DIR) + .timeout(timeout), + ) + .await?; + + if !output.success() { + anyhow::bail!( + "nix config build failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + #[derive(Debug, serde::Deserialize)] + struct NixBuildResult { + outputs: NixBuildOutputs, + } + #[derive(Debug, serde::Deserialize)] + struct NixBuildOutputs { + out: PathBuf, + } + let [result] = serde_json::from_slice::<[NixBuildResult; 1]>(&output.stdout) + .context("parse nix build --json output")?; + Ok(result.outputs.out) +} + +fn write_user_config(user_config: &str) -> Result<()> { + fs::create_dir_all(USER_CONFIG_DIR).with_context(|| format!("create {USER_CONFIG_DIR}"))?; + + let config_path = format!("{USER_CONFIG_DIR}/config.json"); + fs::write(&config_path, user_config).with_context(|| format!("write {config_path}"))?; + Ok(()) +} + +async fn realise_toplevel(toplevel: &str, timeout: Duration) -> Result { + if !toplevel.starts_with("/nix/store/") { + anyhow::bail!("cached config toplevel {toplevel:?} is not a nix store path"); + } + let output = command::run_capture( + Spec::new(nix_executable()) + .args(["build", "--no-link", "--show-trace", toplevel]) + .timeout(timeout), + ) + .await?; + if !output.success() { + anyhow::bail!( + "realise cached config failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + Ok(PathBuf::from(toplevel)) +} + +async fn switch_to_configuration(toplevel: &Path, timeout: Duration) -> Result<()> { + info!("switching to new configuration: {:?}", toplevel); + let switch = toplevel.join("bin/switch-to-configuration"); + let output = run_capture(Spec::new(switch).args(["test"]).timeout(timeout)).await?; + if !output.success() { + anyhow::bail!( + "switch-to-configuration test failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + Ok(()) +} diff --git a/shuttle/src/cache/mod.rs b/shuttle/src/cache/mod.rs new file mode 100644 index 00000000..1a718424 --- /dev/null +++ b/shuttle/src/cache/mod.rs @@ -0,0 +1,724 @@ +use crate::command::{self, Spec}; +use crate::nix_config::{SPINDLE_RUN_DIR, clean_store_paths, nix_executable}; +use crate::protocol::{Message, v1}; +use anyhow::{Context, Result}; +use nix::unistd::{Group, chown}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use std::fmt::{self, Write as FmtWrite}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read}; +use std::net::Shutdown; +use std::os::unix::fs::OpenOptionsExt; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::sync::{Semaphore, mpsc, oneshot, watch}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VMADDR_CID_LOCAL, VsockAddr, VsockListener, VsockStream}; +use tracing::{info, warn}; + +mod read_proxy; +mod write_proxy; + +pub use read_proxy::ReadCacheProxy; +pub use write_proxy::WriteCacheProxy; + +const UPLOAD_QUEUE_CAPACITY: usize = 128; +const CONNECTION_WORKERS: usize = 4; +const CACHE_ENQUEUE_IO_TIMEOUT: Duration = Duration::from_secs(2); +const DEFAULT_CACHE_ENQUEUE_PORT: u32 = 10241; +const SHUTTLE_CACHE_ENQUEUE_PORT_ENV: &str = "SHUTTLE_CACHE_VSOCK_PORT"; +const NIX_BUILD_GROUP: &str = "nixbld"; +const SPINDLE_HOOK_TOKEN: &str = "/run/spindle/hook-token"; + +#[derive(Clone, Debug, Default)] +pub struct CacheStats { + pub pending: u32, + pub active: u32, + pub uploaded: u32, + pub failed: u32, + pub last_error: Option, +} + +#[derive(Debug, Default)] +struct CacheState { + stats: CacheStats, + enqueue_active: u32, + stopped: bool, +} + +impl CacheState { + fn snapshot(&self) -> CacheSnapshot { + CacheSnapshot { + stats: self.stats.clone(), + enqueue_active: self.enqueue_active, + stopped: self.stopped, + } + } +} + +#[derive(Clone, Debug, Default)] +struct CacheSnapshot { + stats: CacheStats, + enqueue_active: u32, + stopped: bool, +} + +impl CacheSnapshot { + fn is_idle(&self) -> bool { + self.stats.pending == 0 && self.stats.active == 0 && self.enqueue_active == 0 + } +} + +#[derive(Clone)] +pub struct CacheUploadManager { + inner: Arc, +} + +struct CacheUploadInner { + cmd_tx: mpsc::Sender, + stats_rx: watch::Receiver, + handles: Mutex>>, +} + +struct UploadJob { + paths: Vec, + count: u32, +} + +enum Cmd { + EnqueueStarted, + EnqueueFinished, + Enqueue { + paths: Vec, + reply: oneshot::Sender>, + }, + UploadStarted { + count: u32, + }, + UploadFinished { + count: u32, + error: Option, + }, + UploadWorkerStopped, + Stop, +} + +impl CacheUploadManager { + pub async fn start(upload_url: &str, event_tx: mpsc::Sender) -> Result> { + if upload_url.is_empty() { + // nothing to upload to, so don't require the guest-local vsock + // listener (vsock_loopback) or the nix post-build hook + info!("no cache upload url configured, cache uploads disabled"); + return Ok(None); + } + let token = create_hook_token().context("create cache hook token")?; + let port = cache_enqueue_port(); + let listener = VsockListener::bind(VsockAddr::new(VMADDR_CID_LOCAL, port)) + .with_context(|| format!("listen on guest-local vsock port {port}"))?; + + let (cmd_tx, cmd_rx) = mpsc::channel::(UPLOAD_QUEUE_CAPACITY); + let (upload_tx, upload_rx) = mpsc::channel::(UPLOAD_QUEUE_CAPACITY); + let (stats_tx, stats_rx) = watch::channel(CacheSnapshot::default()); + + let mut handles = Vec::with_capacity(3); + + handles.push(tokio::spawn(async move { + cache_manager_loop(cmd_rx, upload_tx, stats_tx).await; + })); + + let upload_cmd_tx = cmd_tx.clone(); + let upload_url = upload_url.to_owned(); + handles.push(tokio::spawn(async move { + upload_loop(upload_rx, upload_cmd_tx, upload_url).await; + })); + + let accept_cmd_tx = cmd_tx.clone(); + handles.push(tokio::spawn(async move { + accept_loop(listener, token, event_tx, accept_cmd_tx).await; + })); + + info!( + port, + workers = CONNECTION_WORKERS, + "cache upload queue ready" + ); + let inner = Arc::new(CacheUploadInner { + cmd_tx, + stats_rx, + handles: Mutex::new(handles), + }); + Ok(Some(Self { inner })) + } + + pub async fn drain(&self, timeout: Option) -> CacheStats { + let mut stats_rx = self.inner.stats_rx.clone(); + let wait = async { + loop { + let snapshot = stats_rx.borrow_and_update().clone(); + if snapshot.is_idle() || snapshot.stopped { + return snapshot.stats; + } + if stats_rx.changed().await.is_err() { + let mut stats = stats_rx.borrow().stats.clone(); + stats.last_error = Some("cache manager stopped".to_owned()); + return stats; + } + } + }; + + match timeout { + Some(timeout) => match tokio::time::timeout(timeout, wait).await { + Ok(stats) => stats, + Err(_) => { + let mut stats = self.inner.stats_rx.borrow().stats.clone(); + stats.last_error = Some("cache drain timed out".to_owned()); + stats + } + }, + None => wait.await, + } + } +} + +impl Drop for CacheUploadInner { + fn drop(&mut self) { + let _ = self.cmd_tx.try_send(Cmd::Stop); + if let Ok(mut handles) = self.handles.lock() { + for handle in handles.drain(..) { + handle.abort(); + } + } + let _ = fs::remove_file(SPINDLE_HOOK_TOKEN); + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct EnqueueBuiltPathsRequest { + token: String, + paths: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct EnqueueBuiltPathsResponse { + queued: usize, + #[serde(default, skip_serializing_if = "String::is_empty")] + error: String, +} + +#[derive(Debug)] +enum JsonLineError { + Empty, + TimedOut, + Io(io::Error), + Json(serde_json::Error), +} + +impl JsonLineError { + fn enqueue_request_message(self) -> String { + match self { + Self::Empty => "empty cache enqueue request".to_owned(), + Self::TimedOut => "cache enqueue read timed out".to_owned(), + Self::Io(error) => error.to_string(), + Self::Json(error) => error.to_string(), + } + } +} + +impl fmt::Display for JsonLineError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => f.write_str("empty message"), + Self::TimedOut => f.write_str("timed out"), + Self::Io(error) => error.fmt(f), + Self::Json(error) => error.fmt(f), + } + } +} + +async fn cache_manager_loop( + mut cmd_rx: mpsc::Receiver, + upload_tx: mpsc::Sender, + stats_tx: watch::Sender, +) { + let mut state = CacheState::default(); + + let publish_state = |state: &CacheState| { + let _ = stats_tx.send(state.snapshot()); + }; + + while let Some(command) = cmd_rx.recv().await { + match command { + Cmd::EnqueueStarted => { + state.enqueue_active += 1; + publish_state(&state); + } + Cmd::EnqueueFinished => { + decrement_counter(&mut state.enqueue_active, 1, "cache enqueue active"); + publish_state(&state); + } + Cmd::Enqueue { paths, reply } => { + let result = enqueue_upload_job(&mut state, &upload_tx, paths); + publish_state(&state); + let _ = reply.send(result); + } + Cmd::UploadStarted { count } => { + decrement_counter(&mut state.stats.pending, count, "cache pending uploads"); + state.stats.active += count; + publish_state(&state); + } + Cmd::UploadFinished { count, error } => { + decrement_counter(&mut state.stats.active, count, "cache active uploads"); + if let Some(error) = error { + state.stats.failed += count; + state.stats.last_error = Some(error); + } else { + state.stats.uploaded += count; + } + publish_state(&state); + } + Cmd::UploadWorkerStopped => { + state.stopped = true; + publish_state(&state); + } + Cmd::Stop => { + state.stopped = true; + publish_state(&state); + break; + } + } + } + + state.stopped = true; + publish_state(&state); +} + +fn enqueue_upload_job( + state: &mut CacheState, + upload_tx: &mpsc::Sender, + paths: Vec, +) -> Result { + if state.stopped { + return Err("cache manager stopped".to_owned()); + } + + let count = paths.len() as u32; + if count == 0 { + return Ok(0); + } + + match upload_tx.try_send(UploadJob { paths, count }) { + Ok(()) => { + state.stats.pending += count; + Ok(count as usize) + } + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => { + state.stopped = true; + Err("cache upload worker stopped".to_owned()) + } + } +} + +fn decrement_counter(counter: &mut u32, count: u32, name: &'static str) { + match counter.checked_sub(count) { + Some(value) => *counter = value, + None => { + warn!(name, current = *counter, count, "cache counter underflow"); + *counter = 0; + } + } +} + +async fn upload_loop( + mut upload_rx: mpsc::Receiver, + cmd_tx: mpsc::Sender, + upload_url: String, +) { + while let Some(job) = upload_rx.recv().await { + let cmd = Cmd::UploadStarted { count: job.count }; + if cmd_tx.send(cmd).await.is_err() { + break; + } + + let error = upload_paths(&upload_url, &job.paths) + .await + .err() + .map(|error| error.to_string()); + + let cmd = Cmd::UploadFinished { + count: job.count, + error, + }; + if cmd_tx.send(cmd).await.is_err() { + break; + } + } + + let _ = cmd_tx.send(Cmd::UploadWorkerStopped).await; +} + +// runs nix copy against the write cache proxy, which goes to the spindle +// and spindle will then forward the request to the actual binary cache +async fn upload_paths(upload_url: &str, paths: &[String]) -> Result<()> { + fn add_query_param(url: &str, key: &str, value: &str) -> String { + let separator = url.contains('?').then_some('&').unwrap_or('?'); + format!("{}{}{}={}", url, separator, key, value) + } + + if paths.is_empty() || upload_url.is_empty() { + return Ok(()); + } + + // we use zstd 3 because its the best usually. it is faster than no + // compression also because of IO savings + let dest_url = add_query_param(upload_url, "compression", "zstd"); + let dest_url = add_query_param(&dest_url, "compression-level", "3"); + let dest_url = add_query_param(&dest_url, "parallel-compression", "true"); + + let spec = Spec::new(nix_executable()) + .args(["copy", "--to", &dest_url]) + .args(paths.iter().cloned()) + .timeout(Duration::from_secs(10 * 60)); + + let output = command::run_capture(spec).await.context("run nix copy")?; + if !output.success() { + anyhow::bail!( + "nix copy failed: exit={} error={:?} output={}", + output.exit.exit_code, + output.exit.error, + output.combined_lossy(), + ); + } + + info!(paths = paths.len(), %upload_url, "uploaded cache paths"); + Ok(()) +} + +async fn accept_loop( + listener: VsockListener, + token: String, + event_tx: mpsc::Sender, + cmd_tx: mpsc::Sender, +) { + let permits = Arc::new(Semaphore::new(CONNECTION_WORKERS)); + let mut tasks = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, _addr)) => { + let Ok(permit) = permits.clone().try_acquire_owned() else { + tasks.spawn(async move { + let mut conn = conn; + write_enqueue_response( + &mut conn, + 0, + Some("cache enqueue workers are busy".to_owned()), + ) + .await; + }); + warn!("cache enqueue dropped because workers are busy"); + continue; + }; + + let worker_cmd_tx = cmd_tx.clone(); + if let Err(error) = start_enqueue_request(&worker_cmd_tx) { + tasks.spawn(async move { + let mut conn = conn; + write_enqueue_response(&mut conn, 0, Some(error)).await; + }); + continue; + } + + let worker_token = token.clone(); + let worker_event_tx = event_tx.clone(); + tasks.spawn(async move { + let _permit = permit; + handle_enqueue_conn( + conn, + &worker_token, + &worker_event_tx, + &worker_cmd_tx, + ) + .await; + let _ = worker_cmd_tx.send(Cmd::EnqueueFinished).await; + }); + } + Err(error) => { + if error.kind() != io::ErrorKind::Interrupted { + warn!(%error, "cache enqueue accept failed"); + } + } + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_enqueue_task_result(result); + } + } + } +} + +fn log_enqueue_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "cache enqueue task failed"); + } +} + +async fn handle_enqueue_conn( + mut conn: VsockStream, + expected_token: &str, + event_tx: &mpsc::Sender, + cmd_tx: &mpsc::Sender, +) { + let req: EnqueueBuiltPathsRequest = match read_enqueue_request(&mut conn).await { + Ok(req) => req, + Err(error) => { + write_enqueue_response(&mut conn, 0, Some(error)).await; + return; + } + }; + + if req.token != expected_token { + write_enqueue_response(&mut conn, 0, Some("invalid cache enqueue token".to_owned())).await; + return; + } + + match enqueue_paths(cmd_tx, req.paths).await { + Ok(queued) => { + send_built_paths_event(event_tx, queued.event_paths).await; + write_enqueue_response(&mut conn, queued.count, None).await; + } + Err(error) => write_enqueue_response(&mut conn, 0, Some(error)).await, + } +} + +fn start_enqueue_request(cmd_tx: &mpsc::Sender) -> Result<(), String> { + match cmd_tx.try_send(Cmd::EnqueueStarted) { + Ok(()) => Ok(()), + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => Err("cache upload worker stopped".to_owned()), + } +} + +async fn read_enqueue_request(conn: &mut VsockStream) -> Result { + read_json_line(conn) + .await + .map_err(JsonLineError::enqueue_request_message) +} + +async fn read_json_line(conn: &mut VsockStream) -> Result +where + T: DeserializeOwned, +{ + let mut data = Vec::new(); + let mut reader = BufReader::new(conn); + let bytes_read = tokio::time::timeout( + CACHE_ENQUEUE_IO_TIMEOUT, + reader.read_until(b'\n', &mut data), + ) + .await + .map_err(|_| JsonLineError::TimedOut)? + .map_err(JsonLineError::Io)?; + + if bytes_read == 0 { + return Err(JsonLineError::Empty); + } + + serde_json::from_slice(&data).map_err(JsonLineError::Json) +} + +struct QueuedPaths { + count: usize, + event_paths: Vec, +} + +async fn enqueue_paths( + cmd_tx: &mpsc::Sender, + paths: Vec, +) -> Result { + let paths = clean_store_paths(&paths); + let event_paths = paths.clone(); + if paths.is_empty() { + return Ok(QueuedPaths { + count: 0, + event_paths, + }); + } + + let (reply, queued) = oneshot::channel(); + match cmd_tx.try_send(Cmd::Enqueue { paths, reply }) { + Ok(()) => match queued.await { + Ok(Ok(count)) => Ok(QueuedPaths { count, event_paths }), + Ok(Err(error)) => Err(error), + Err(_) => Err("cache upload worker stopped".to_owned()), + }, + Err(mpsc::error::TrySendError::Full(_)) => Err("cache upload queue is full".to_owned()), + Err(mpsc::error::TrySendError::Closed(_)) => Err("cache upload worker stopped".to_owned()), + } +} + +async fn send_built_paths_event(event_tx: &mpsc::Sender, paths: Vec) { + if paths.is_empty() { + return; + } + + let msg = Message { + id: "built-paths".to_owned(), + built_paths: Some(v1::BuiltPaths { + paths, + reason: "post_build_hook".to_owned(), + }), + ..Default::default() + }; + let _ = event_tx.send(msg).await; +} + +async fn write_enqueue_response(conn: &mut VsockStream, queued: usize, error: Option) { + let response = EnqueueBuiltPathsResponse { + queued, + error: error.unwrap_or_default(), + }; + let _ = write_json_line(conn, &response).await; +} + +async fn write_json_line(conn: &mut VsockStream, value: &T) -> Result<(), JsonLineError> +where + T: Serialize + ?Sized, +{ + let data = serde_json::to_vec(value).map_err(JsonLineError::Json)?; + tokio::time::timeout(CACHE_ENQUEUE_IO_TIMEOUT, async { + conn.write_all(&data).await?; + conn.write_all(b"\n").await?; + VsockStream::shutdown(conn, Shutdown::Write) + }) + .await + .map_err(|_| JsonLineError::TimedOut)? + .map_err(JsonLineError::Io) +} + +// we use a loopback vsock here since its better than having to do the whole http song and dance! +pub async fn enqueue_built_paths(paths: &[String]) { + let paths = clean_store_paths(paths); + if paths.is_empty() { + return; + } + + let token = match read_hook_token() { + Ok(token) => token, + Err(_) => return, + }; + + if token.is_empty() { + return; + } + + let mut conn = + match VsockStream::connect(VsockAddr::new(VMADDR_CID_LOCAL, cache_enqueue_port())).await { + Ok(conn) => conn, + Err(error) => { + warn!(paths = paths.len(), %error, "cache enqueue unavailable"); + return; + } + }; + + let request = EnqueueBuiltPathsRequest { token, paths }; + match write_json_line(&mut conn, &request).await { + Ok(()) => {} + Err(JsonLineError::Json(error)) => { + warn!(%error, "cache enqueue encode failed"); + return; + } + Err(JsonLineError::TimedOut) => { + warn!("cache enqueue write timed out"); + return; + } + Err(error) => { + warn!(%error, "cache enqueue write failed"); + return; + } + } + + let response: EnqueueBuiltPathsResponse = match read_json_line(&mut conn).await { + Ok(response) => response, + Err(JsonLineError::Empty) => { + warn!("cache enqueue ack was empty"); + return; + } + Err(JsonLineError::TimedOut) => { + warn!("cache enqueue ack timed out"); + return; + } + Err(error) => { + warn!(%error, "cache enqueue ack failed"); + return; + } + }; + + if !response.error.is_empty() { + warn!(error = %response.error, "cache enqueue rejected"); + return; + } + + info!(queued = response.queued, "cache paths enqueued"); +} + +fn cache_enqueue_port() -> u32 { + std::env::var(SHUTTLE_CACHE_ENQUEUE_PORT_ENV) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_CACHE_ENQUEUE_PORT) +} + +fn create_hook_token() -> Result { + use std::io::Write; + + fs::create_dir_all(SPINDLE_RUN_DIR).with_context(|| format!("create {SPINDLE_RUN_DIR}"))?; + let token = random_token().context("generate hook token")?; + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o640) + .open(SPINDLE_HOOK_TOKEN) + .with_context(|| format!("create {SPINDLE_HOOK_TOKEN}"))?; + allow_nix_build_group(SPINDLE_HOOK_TOKEN)?; + file.write_all(token.as_bytes()) + .with_context(|| format!("write {SPINDLE_HOOK_TOKEN}"))?; + file.write_all(b"\n") + .with_context(|| format!("write {SPINDLE_HOOK_TOKEN}"))?; + Ok(token) +} + +fn allow_nix_build_group(path: &str) -> Result<()> { + let Some(group) = + Group::from_name(NIX_BUILD_GROUP).with_context(|| format!("lookup {NIX_BUILD_GROUP}"))? + else { + warn!( + group = NIX_BUILD_GROUP, + "nix build group not found; cache hook token remains root-only" + ); + return Ok(()); + }; + + chown(path, None, Some(group.gid)).with_context(|| format!("chown {path} to {NIX_BUILD_GROUP}")) +} + +fn read_hook_token() -> Result { + fs::read_to_string(SPINDLE_HOOK_TOKEN) + .map(|token| token.trim().to_owned()) + .with_context(|| format!("read {SPINDLE_HOOK_TOKEN}")) +} + +fn random_token() -> Result { + let mut bytes = [0_u8; 32]; + File::open("/dev/urandom") + .context("open /dev/urandom")? + .read_exact(&mut bytes) + .context("read /dev/urandom")?; + + let mut token = String::with_capacity(bytes.len() * 2); + for byte in bytes { + write!(&mut token, "{byte:02x}").unwrap(); + } + Ok(token) +} diff --git a/shuttle/src/cache/read_proxy.rs b/shuttle/src/cache/read_proxy.rs new file mode 100644 index 00000000..8019358c --- /dev/null +++ b/shuttle/src/cache/read_proxy.rs @@ -0,0 +1,27 @@ +use crate::host_proxy::VsockTcpProxy; +use anyhow::Result; + +const DEFAULT_CACHE_READ_PROXY_ADDR: &str = "127.0.0.1:10500"; +const SHUTTLE_CACHE_READ_PROXY_ADDR_ENV: &str = "SHUTTLE_CACHE_READ_PROXY_ADDR"; + +pub struct ReadCacheProxy { + inner: VsockTcpProxy, +} + +impl ReadCacheProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let addr = std::env::var(SHUTTLE_CACHE_READ_PROXY_ADDR_ENV) + .unwrap_or_else(|_| DEFAULT_CACHE_READ_PROXY_ADDR.to_owned()); + + let inner = VsockTcpProxy::start("read cache proxy", &addr, host_cid, host_port).await?; + Ok(Some(Self { inner })) + } + + pub fn url(&self) -> &str { + self.inner.url() + } +} diff --git a/shuttle/src/cache/write_proxy.rs b/shuttle/src/cache/write_proxy.rs new file mode 100644 index 00000000..8d88fd5b --- /dev/null +++ b/shuttle/src/cache/write_proxy.rs @@ -0,0 +1,22 @@ +use crate::host_proxy::VsockTcpProxy; +use anyhow::Result; + +pub struct WriteCacheProxy { + inner: VsockTcpProxy, +} + +impl WriteCacheProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let inner = + VsockTcpProxy::start("write cache proxy", "127.0.0.1:0", host_cid, host_port).await?; + Ok(Some(Self { inner })) + } + + pub fn url(&self) -> &str { + self.inner.url() + } +} diff --git a/shuttle/src/command.rs b/shuttle/src/command.rs new file mode 100644 index 00000000..01a7ab2d --- /dev/null +++ b/shuttle/src/command.rs @@ -0,0 +1,298 @@ +use anyhow::{Context, Result}; +use nix::sys::signal::{Signal, kill}; +use nix::unistd::{Gid, Pid, Uid, User, getgrouplist, setgid, setgroups, setuid}; +use std::ffi::{CString, OsStr, OsString}; +use std::io; +use std::os::unix::process::ExitStatusExt; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt}; +use tokio::process::{Child, Command}; +use tokio::sync::mpsc::{self, Receiver, Sender}; +use tokio::task::JoinHandle; +use tracing::warn; + +#[derive(Clone, Debug)] +pub struct Spec { + pub program: OsString, + pub args: Vec, + pub env: Vec<(OsString, OsString)>, + pub cwd: Option, + pub timeout: Option, + pub uid: Option, + pub gid: Option, +} + +impl Spec { + pub fn new(program: impl Into) -> Self { + Self { + program: program.into(), + args: Vec::new(), + env: Vec::new(), + cwd: None, + timeout: None, + uid: None, + gid: None, + } + } + + pub fn arg(mut self, arg: impl Into) -> Self { + self.args.push(arg.into()); + self + } + + pub fn args(mut self, args: I) -> Self + where + I: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(Into::into)); + self + } + + pub fn envs(mut self, env: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.env.extend( + env.into_iter() + .map(|(key, value)| (key.into(), value.into())), + ); + self + } + + pub fn cwd(mut self, cwd: impl Into) -> Self { + self.cwd = Some(cwd.into()); + self + } + + pub fn timeout(mut self, timeout: Duration) -> Self { + self.timeout = Some(timeout); + self + } + + pub fn run_as(mut self, uid: u32, gid: u32) -> Self { + self.uid = Some(uid); + self.gid = Some(gid); + self + } +} + +#[derive(Clone, Debug)] +pub struct ExitResult { + pub exit_code: i32, + pub error: Option, + pub timed_out: bool, +} + +#[derive(Clone, Debug)] +pub struct CaptureOutput { + pub exit: ExitResult, + pub stdout: Vec, + pub stderr: Vec, +} + +impl CaptureOutput { + pub fn success(&self) -> bool { + self.exit.exit_code == 0 && self.exit.error.is_none() + } + + pub fn combined_lossy(&self) -> String { + let mut data = self.stdout.clone(); + data.extend_from_slice(&self.stderr); + String::from_utf8_lossy(&data).trim().to_owned() + } +} + +#[derive(Clone, Copy, Debug)] +pub enum OutKind { + Stdout, + Stderr, +} + +#[derive(Clone, Debug)] +pub struct OutData { + pub data: Vec, + pub kind: OutKind, +} + +pub struct StreamingCommand { + events: Receiver, + exit: JoinHandle>, +} + +impl StreamingCommand { + pub fn into_parts(self) -> (Receiver, JoinHandle>) { + (self.events, self.exit) + } +} + +pub async fn run_capture(spec: Spec) -> Result { + let running = spawn_streaming(spec)?; + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let (mut events, exit_task) = running.into_parts(); + + while let Some(event) = events.recv().await { + match event.kind { + OutKind::Stdout => stdout.extend_from_slice(&event.data), + OutKind::Stderr => stderr.extend_from_slice(&event.data), + } + } + + let exit = exit_task + .await + .unwrap_or_else(|error| Err(anyhow::anyhow!("command supervisor failed: {error}")))?; + Ok(CaptureOutput { + exit, + stdout, + stderr, + }) +} + +pub fn spawn_streaming(mut spec: Spec) -> Result { + let mut child = spawn(&mut spec)?; + let stdout = child.stdout.take().context("stdout pipe missing")?; + let stderr = child.stderr.take().context("stderr pipe missing")?; + + let (events_tx, events_rx) = mpsc::channel(64); + let stdout_thread = spawn_reader(stdout, events_tx.clone(), OutKind::Stdout); + let stderr_thread = spawn_reader(stderr, events_tx.clone(), OutKind::Stderr); + drop(events_tx); + + let exit = tokio::spawn(async move { + let exit = wait_child(&mut child, spec.timeout).await; + + // ensure all output is observed before exiting + // this assumes children dont daemonize and hold onto the stdout/err + stdout_thread.await.context("stdout reader task failed")?; + stderr_thread.await.context("stderr reader task failed")?; + + Ok(exit) + }); + + Ok(StreamingCommand { + events: events_rx, + exit, + }) +} + +fn spawn(spec: &mut Spec) -> Result { + let mut cmd = Command::new(&spec.program); + cmd.args(&spec.args) + .envs(spec.env.iter().map(|(key, value)| (key, value))) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + if let Some(cwd) = &spec.cwd { + cmd.current_dir(cwd); + } + + // don't use rust's .uid() / .gid() methods here because they clear + // supplemantary groups, which means for example adding a user to "docker" + // group won't actually let it access the sock. + // https://github.com/rust-lang/rust/issues/90747 + if let (Some(uid), Some(gid)) = (spec.uid, spec.gid) { + let username = User::from_uid(Uid::from_raw(uid)) + .ok() + .flatten() + .map(|u| u.name) + .with_context(|| format!("lookup passwd entry for uid {uid}"))?; + let cname = CString::new(username) + .with_context(|| format!("username for uid {uid} contained a null byte"))?; + // resolve groups beforehand so we don't have to read /etc/group in the pre_exec + let groups = + getgrouplist(&cname, Gid::from_raw(gid)).context("resolve supplementary groups")?; + // SAFETY: pre_exec runs between fork and execve in the child. + // we only call async-signal-safe syscalls and we don't touch any + // shared state, no allocator, no mutexes, no globals. + unsafe { + cmd.pre_exec(move || { + setgroups(&groups).map_err(io::Error::from)?; + setgid(Gid::from_raw(gid)).map_err(io::Error::from)?; + setuid(Uid::from_raw(uid)).map_err(io::Error::from)?; + Ok(()) + }); + } + } + + // allow us to kill this whole process tree on deadline + cmd.process_group(0); + + cmd.spawn() + .with_context(|| format!("spawn {}", display_os(&spec.program))) +} + +async fn wait_child(child: &mut Child, timeout: Option) -> ExitResult { + let wait = child.wait(); + let status = match timeout { + Some(timeout) => match tokio::time::timeout(timeout, wait).await { + Ok(status) => status, + Err(_) => { + if let Some(pid) = child.id() + && let Err(error) = kill(Pid::from_raw(-(pid as i32)), Signal::SIGKILL) + { + warn!(pid, %error, "failed to kill process group"); + } + let _ = child.wait().await; + return ExitResult { + exit_code: 124, + error: Some("command timed out".to_owned()), + timed_out: true, + }; + } + }, + None => wait.await, + }; + + match status { + Ok(status) => ExitResult { + exit_code: status + .code() + .or_else(|| status.signal().map(|signal| 128 + signal)) + .unwrap_or(1), + error: None, + timed_out: false, + }, + Err(error) => ExitResult { + exit_code: 1, + error: Some(error.to_string()), + timed_out: false, + }, + } +} + +fn spawn_reader( + mut reader: impl AsyncRead + Unpin + Send + 'static, + events: Sender, + kind: OutKind, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut buf = [0_u8; 32 * 1024]; + loop { + match reader.read(&mut buf).await { + Ok(0) => return, + Ok(n) => { + let event = OutData { + data: buf[..n].to_vec(), + kind, + }; + if events.send(event).await.is_err() { + return; + } + } + Err(error) => { + warn!(%error, "failed to read command stream"); + return; + } + } + } + }) +} + +fn display_os(value: &OsStr) -> String { + value.to_string_lossy().into_owned() +} diff --git a/shuttle/src/dns_proxy.rs b/shuttle/src/dns_proxy.rs new file mode 100644 index 00000000..02bcb439 --- /dev/null +++ b/shuttle/src/dns_proxy.rs @@ -0,0 +1,255 @@ +use anyhow::{Context, Result}; +use std::io; +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream, UdpSocket}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +const DEFAULT_DNS_PROXY_ADDR: &str = "127.0.0.1:53"; +const SHUTTLE_DNS_PROXY_ADDR_ENV: &str = "SHUTTLE_DNS_PROXY_ADDR"; + +const MAX_DNS_MESSAGE_BYTES: usize = u16::MAX as usize; +const DNS_IO_TIMEOUT: Duration = Duration::from_secs(10); +const DNS_TCP_IDLE_TIMEOUT: Duration = Duration::from_secs(120); + +// implements a proxy that sends dns requests to the spindle and +// lets the spindle resolve any queries, and streams the response back. +// +// we use this because the way spindle isolates QEMU VMs is unreliable +// to rely on. if we unblock the blackholed-routes for private nameservers +// like slirp one, we risk leaking internal DNS zones, even if the guest +// can't connect to them. there is also potentially DNS rebinding issues. +// and other slirp4netns quirks... +// +// this way we also get to filter the DNS queries very easily, so we can +// make sure we remove everything that would leak a host information. +pub struct DnsProxy { + handles: Vec>, +} + +#[derive(Clone)] +struct HostDnsClient { + host_cid: u32, + host_port: u32, +} + +impl DnsProxy { + pub async fn start(host_cid: u32, host_port: u32) -> Result> { + if host_port == 0 { + return Ok(None); + } + + let addr = std::env::var(SHUTTLE_DNS_PROXY_ADDR_ENV) + .unwrap_or_else(|_| DEFAULT_DNS_PROXY_ADDR.to_owned()); + + let udp = Arc::new( + UdpSocket::bind(&addr) + .await + .with_context(|| format!("bind dns udp listener {addr}"))?, + ); + + let tcp = TcpListener::bind(&addr) + .await + .with_context(|| format!("bind dns tcp listener {addr}"))?; + + let host = HostDnsClient { + host_cid, + host_port, + }; + + let handles = vec![ + tokio::spawn(udp_loop(udp, host.clone())), + tokio::spawn(tcp_loop(tcp, host)), + ]; + + info!(%addr, host_cid, host_port, "dns proxy ready"); + Ok(Some(Self { handles })) + } +} + +impl Drop for DnsProxy { + fn drop(&mut self) { + for handle in self.handles.drain(..) { + handle.abort(); + } + } +} + +impl HostDnsClient { + async fn query(&self, query: Vec) -> Result> { + match self.query_once(&query).await { + Ok(response) => Ok(response), + Err(first_error) => self.query_once(&query).await.with_context(|| { + format!("dns host query failed after retry; first error: {first_error:#}") + }), + } + } + + async fn query_once(&self, query: &[u8]) -> Result> { + let addr = VsockAddr::new(self.host_cid, self.host_port); + + let mut host = tokio::time::timeout(DNS_IO_TIMEOUT, VsockStream::connect(addr)) + .await + .context("dns host connect timed out")? + .with_context(|| { + format!( + "dial host dns proxy cid={} port={}", + self.host_cid, self.host_port + ) + })?; + + tokio::time::timeout(DNS_IO_TIMEOUT, async { + write_dns_packet(&mut host, query) + .await + .context("write dns query to host")?; + + read_dns_packet(&mut host) + .await + .context("read dns response from host")? + .context("host dns proxy closed without response") + }) + .await + .context("dns host query timed out")? + } +} + +async fn udp_loop(socket: Arc, host: HostDnsClient) { + let mut buf = vec![0; MAX_DNS_MESSAGE_BYTES]; + let mut tasks = JoinSet::new(); + + loop { + tokio::select! { + received = socket.recv_from(&mut buf) => match received { + Ok((len, peer)) => { + let query = buf[..len].to_vec(); + let socket = socket.clone(); + let host = host.clone(); + + tasks.spawn(async move { + if let Err(error) = handle_udp_query(socket, peer, query, host).await { + warn!(%peer, %error, "dns udp query failed"); + } + }); + } + Err(error) => warn!(%error, "dns udp recv failed"), + }, + + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_dns_task_result(result); + } + } + } +} + +async fn handle_udp_query( + socket: Arc, + peer: SocketAddr, + query: Vec, + host: HostDnsClient, +) -> Result<()> { + let response = host.query(query).await?; + + socket + .send_to(&response, peer) + .await + .context("send dns udp response")?; + + Ok(()) +} + +async fn tcp_loop(listener: TcpListener, host: HostDnsClient) { + let mut tasks = JoinSet::new(); + + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, peer)) => { + let host = host.clone(); + + tasks.spawn(async move { + if let Err(error) = handle_tcp_conn(conn, host).await { + warn!(%peer, %error, "dns tcp connection failed"); + } + }); + } + Err(error) => warn!(%error, "dns tcp accept failed"), + }, + + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_dns_task_result(result); + } + } + } +} + +async fn handle_tcp_conn(mut tcp: TcpStream, host: HostDnsClient) -> Result<()> { + loop { + let query = tokio::time::timeout(DNS_TCP_IDLE_TIMEOUT, read_dns_packet(&mut tcp)) + .await + .context("dns tcp idle timeout")? + .context("read dns tcp query")?; + + let Some(query) = query else { + return Ok(()); + }; + + let response = host.query(query).await?; + + write_dns_packet(&mut tcp, &response) + .await + .context("write dns tcp response")?; + } +} + +async fn read_dns_packet(reader: &mut R) -> io::Result>> +where + R: AsyncRead + Unpin, +{ + let mut len_buf = [0; 2]; + + match reader.read_exact(&mut len_buf).await { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(error) => return Err(error), + } + + let len = u16::from_be_bytes(len_buf) as usize; + if len == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "empty dns packet", + )); + } + + let mut packet = vec![0; len]; + reader.read_exact(&mut packet).await?; + Ok(Some(packet)) +} + +async fn write_dns_packet(writer: &mut W, packet: &[u8]) -> io::Result<()> +where + W: AsyncWrite + Unpin, +{ + if packet.is_empty() || packet.len() > MAX_DNS_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid dns packet size {}", packet.len()), + )); + } + + writer + .write_all(&(packet.len() as u16).to_be_bytes()) + .await?; + writer.write_all(packet).await?; + writer.flush().await +} + +fn log_dns_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "dns proxy task failed"); + } +} diff --git a/shuttle/src/exec.rs b/shuttle/src/exec.rs new file mode 100644 index 00000000..da460785 --- /dev/null +++ b/shuttle/src/exec.rs @@ -0,0 +1,188 @@ +use crate::command::{self, OutKind, Spec}; +use crate::protocol::{self, Message, v1}; +use nix::unistd::{Group, User}; +use std::ffi::OsString; +use std::time::Duration; +use tokio::sync::mpsc::Sender; +use tracing::info; + +const DEFAULT_USER: &str = "spindle-workflow"; + +pub async fn run(id: String, req: v1::ExecStart, out: Sender) { + let send_exit = async |exit_code: i32, error: Option, timed_out: bool| { + let msg = Message { + id: id.clone(), + exec_exit: Some(v1::ExecExit { + exit_code, + error: protocol::error_or_empty(error), + timed_out, + }), + ..Default::default() + }; + let _ = out.send(msg).await; + }; + + if req.argv.is_empty() { + send_exit(127, Some("missing argv".to_owned()), false).await; + return; + } + + let user = if req.user.is_empty() { + DEFAULT_USER + } else { + req.user.as_str() + }; + let run_as = match resolve_user(user) { + Ok(run_as) => run_as, + Err(err) => { + send_exit(127, Some(err), false).await; + return; + } + }; + + let mut spec = Spec::new(req.argv[0].clone()) + .args(req.argv[1..].iter().cloned()) + .envs(parse_env(&req.env)) + .run_as(run_as.uid, run_as.gid); + if !req.cwd.is_empty() { + spec = spec.cwd(req.cwd.clone()); + } + let timeout = + (req.timeout_seconds > 0).then(|| Duration::from_secs(u64::from(req.timeout_seconds))); + if let Some(timeout) = timeout { + spec = spec.timeout(timeout); + } + + info!( + %id, + user = %run_as.name, + uid = run_as.uid, + gid = run_as.gid, + argv = ?req.argv, + cwd = ?req.cwd, + "starting exec" + ); + + let cmd = match command::spawn_streaming(spec) { + Ok(cmd) => cmd, + Err(err) => { + send_exit(127, Some(err.to_string()), false).await; + return; + } + }; + let (mut events, exit_task) = cmd.into_parts(); + while let Some(event) = events.recv().await { + let data = String::from_utf8_lossy(&event.data).into_owned(); + let output = match event.kind { + OutKind::Stdout => Message { + id: id.clone(), + exec_stdout: Some(v1::ExecStdout { data }), + ..Default::default() + }, + OutKind::Stderr => Message { + id: id.clone(), + exec_stderr: Some(v1::ExecStderr { data }), + ..Default::default() + }, + }; + let _ = out.send(output).await; + } + let exit = match exit_task + .await + .unwrap_or_else(|error| Err(anyhow::anyhow!("command supervisor failed: {error}"))) + { + Ok(exit) => exit, + Err(err) => { + send_exit(127, Some(err.to_string()), false).await; + return; + } + }; + + send_exit(exit.exit_code, exit.error, exit.timed_out).await +} + +#[derive(Clone, Debug)] +struct ResolvedUser { + name: String, + uid: u32, + gid: u32, +} + +fn resolve_user(spec: &str) -> Result { + let spec = spec.trim(); + if spec.is_empty() { + return resolve_user(DEFAULT_USER); + } + + let (user_part, group_part) = spec + .split_once(':') + .map(|(user, group)| (user, Some(group))) + .unwrap_or((spec, None)); + + let mut user = lookup_user(user_part)?; + if let Some(group) = group_part.filter(|group| !group.is_empty()) { + user.gid = lookup_group(group)?; + } + + if user.uid == 0 || user.gid == 0 { + return Err(format!("refusing to run exec as privileged user {spec:?}")); + } + + Ok(user) +} + +fn lookup_user(name: &str) -> Result { + match User::from_name(name) { + Ok(Some(user)) => Ok(ResolvedUser { + name: name.to_owned(), + uid: user.uid.as_raw(), + gid: user.gid.as_raw(), + }), + Ok(None) => { + let uid = name + .parse::() + .map_err(|_| format!("workflow user {name:?} was not found"))?; + Ok(ResolvedUser { + name: name.to_owned(), + uid, + gid: uid, + }) + } + Err(error) => Err(format!("lookup workflow user {name:?}: {error}")), + } +} + +fn lookup_group(name: &str) -> Result { + match Group::from_name(name) { + Ok(Some(group)) => Ok(group.gid.as_raw()), + Ok(None) => name + .parse::() + .map_err(|_| format!("workflow group {name:?} was not found")), + Err(error) => Err(format!("lookup workflow group {name:?}: {error}")), + } +} + +fn parse_env(values: &[String]) -> Vec<(OsString, OsString)> { + values + .iter() + .filter_map(|value| value.split_once('=')) + .map(|(key, value)| (OsString::from(key), OsString::from(value))) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refuses_root_exec_user() { + let err = resolve_user("root").unwrap_err(); + assert!(err.contains("refusing to run exec as privileged user")); + } + + #[test] + fn refuses_root_exec_group() { + let err = resolve_user("65534:0").unwrap_err(); + assert!(err.contains("refusing to run exec as privileged user")); + } +} diff --git a/shuttle/src/gen/file_descriptor_set.bin b/shuttle/src/gen/file_descriptor_set.bin new file mode 100644 index 0000000000000000000000000000000000000000..e89630f2b2a2a231d14694f57becb7075029bda4 GIT binary patch literal 314060 zcmd;@c#DNgAw54oJttMapeVm2KdCfLKP9y|xhS)sB)>=xEGEPcQ32s4m8L13V&oEZ z%gjl2fvXHoEfKQdVoA%)NtIw!VAkL^Qqe=xrH85`0AyVdScg_MN5h*_{}}}uL8Mf! zP&R}wWTKZ;nx=>3o-#v_NIb}zc!Y3#acW6?YDGzEUU6oAo|G;~4Vo@gyMt3pTw%(U zjD5u!2p%jA_@r(p+>?I6&H4=r9W(3{Aj+#nue{pSkpiu zBf&&UxIy%w20SG8NDp{NF7CY4;*!)9NGbwl7H%WL>5vPe37UqGk|87^lbQ@8xcK2| zD848$FFjQPlAiT3!Wu1D_27j^5J+1PAKV-;U&w`vD?cwaKP^5bH91Fu14l6XgOs3| z9K;FM=#rY8BV>t4p6obMmoFqcW2DefF215vP##N7f%=RS>EBLC^IcH8R95VJn$J=U}(k~R(fGbT7ydxOH!@4LjA&A zgFM|lU0oo~L@{1!3L_Ucl21S-YF=q>Qfd)2sjxwKLegBEX{9+ikaP>FyE#At;4~}5 z#R9GoL2l&?0<(prxHw?xT7p%9QG+80&WFeqq$ZZ6rbw{CP-od z4_JesJRwIe)||wo)Eo&Ga5mJ&hy-jl==p%O1VMBP*+GK_6a;)mTEv+EDx#smD9yzQ z3R+0GgTjh4hyx@54tFUo4sg+#k_u~2!TF%VCM7j3u{5V7zAQ1PG*tpr2yzDTKn24< zVnXs<+~5K;J~J;RwE|oVvj#y5Ur&$-DB81%^Yb9qbAjtv4v+xYdUHf+0cilpV)q^- zw;{Cv)VcV&UNPtyHfQtuWfM)!0kpR08 zFBd0Rjg!B>j{pbU%;1n9Pd|47PPm!wLH?lu0$gwtd|iWs9o=08xZx_CB12q*1$f{_ zfShG)B)|)|!qqR-SAb7Qkc$^=W3U^@S;j^J{AfaECISL*Q-VGHLX3?B1W^RcOaz3K zoVi%R$xuj$i_gc=$<-&`KLC_z9DM{B5n@5E0j`cAt}X)1$YOz^o619l5ptOvcOBhRg7po=qT&$3^3@NEKF_Ig|AaKe?Zx=xH zAvak;IUQS*6=VduBZXoy8U&CwlNnYQpjoa5G7_ayB-k(jHKg!23_wKzpAm@-18Xip z47U(nK{qpUNq}@?3tz0Ug(y`)DFRWdS|f@LNF-qT0&Wj_Oeig9Wz8V|A!37guIpL1_u3 zNe>Dz&LB>(2)O(M71X6AaCwkhID@$I!9rjK7KlIvRaV?aau`Vy;uv&y3K?(-B@5*0jW&;?y91h;%Sa7Ni5*I4(lc0ZNY?a2*h7xDKVYoLtZ-N60StGdqM)4)O zR#PtFf=Wyqg-~oPs6^MM3LY3LN=*j07R8ZOgT{xT5<+@h0?EnosdIjQl9MX7m- z#S-#h#VjR7rKv%D$;qx@Nk_1pkQSFiT5OGjz2rLXr*u~|UX(ccd)xpZR!D3L;G`Iu`G73;F z)&MKxE66B7wpW>p7o-xD?7^l>Xo8jT6l4@2Y&78#11ZW)ElEsCEJ;Liq7GQKNI^z{ zFH{ntOOcDGs5Ge(o<4NJYPmsTuoOae4<>?E{WT>kE*|7`FT}|ON%;bdpmvnAzl&>p zuxFI3028Ps=HnUS8Xpwu7vkybD!`1?H}GQN&=S!0V&PB|P!9qH4@RO?`ohe`hdMYQ zWWyy69mfL?mzHN1fir7vBBWCdFJVNWs)AEX!ZV9f-SUfa6HA1Axa9NluG*;M<+zQA?D*0w*`P)}ivX(^3kRzJn->cQn*h5P3kQb) zrxyzcrvR7ITwX37uy;VIEI+Rp)B#Ke^|rvBVLl^ej5-xrwVpGmCbnXObPs}9!8$=5 zYEUhg3TlCXdckZEo{&2iTUIgHY*5w7hjqL^H75n?5co46QMObA41kAv8s~TbS25f0vbn0ty#fB zLK<8=DXB@N>G4IWDT&D?sPzdqSSkoCBjnA+2^!7G1CP9bY7agmUHrid35uZ9l2ov! zAcTG)4=(nS#G>@nl41#A1!fICBg~OHl$e4T6#~`*3fCZ*cA;=C0qDqaJhZ1=EFl6O z{L{hjI_P+^3#88-#1AzE$}dLA7owyL2!(R-Lr0S0i%Sa%@{39&#K8IqI0!bn1kn@3 z2Q?%Z!l&J+np6}QA8KGA566QFD`JM@LEREqUk@}<1nIy@O=IHXL-m-DJC_hNtm8BD zN{aGRN|RGlpcxkTu)iQwk0)FUvO$P)4qPnbb1=*x7kFVRq|YS`GXONTRGwIr2WnJ< zyv`XUgd`OXkr8s_f+d&uqSV~{veXplh!bx2^FiGo1W~V~&&3KZeuTKVpaqZsgAfN7 zi?e@_ivT02K7>nBgc1;fxmjJ|E zM5YuH=MoNa403l3i3g8~AdD8_5E`L`<08OhC7N3i1K6YVgPnn~)Tj7`kY@t6xY^qyRg*M}l30 z!aSW_1vrGnxP*{3`MQR9_`3*jVym%)1ca5=FmmxAkD{OyDxf~ppsH+@E;4a(gL?9i zh%E+Vs z#3cXQ@YB? z#Ru)^!U7gK#FW65E|HB{)EH3&B|})}7`cT`k2bH;OlB@#l(8SQe1_9?g+z_yWkxHZNfTImkOTx4Nfi51NS=upc=`1_!Vczp`a=27A$+*}3-(Qn%p5P+gLuI%fSD(hgS?yw zRHkF?NFrPVsyaX&9w$w(L)gF$;TB?Z_VEvPbpbnrQ;5Ysz|~LUIm}AL%A*;KT;fHk zkcnF8FhOcwa()V|b<1aDh}lU(SRMq`3Jwxis75DUuuG7G#L>?M>K1Mxwg5+GZ&w$D zYeZoVKruzAj*A~O+6qb=nJJK&b5Q!lI^~LR5oq`|EHMYF0~#n0r*I0f`1$*}f`f!x zhz&f49*J-UAILU@R-rmB0qDR_JUmE1S%S|9%d9cN0nkAvcwnd@1%`+azmKb*dx%GT zK#(hVK2V|s7AO2r%U~LXni#pbA)`0o0VnXZI-e2dbO$0m>wyPvz^a`r!PYX*V1y(H zaUmfeS9eF}$ap8$;1Ezr<{tz}qg+C)jy^vA;fR3X0@(^3Ejv?aVlNrP(ix~NZxLhLw{DVT`9esSj?iCdhfQp3p2gLih zhPnD6oGlJ@AG(233mLftF|1c=WaJXZ;RGR7E=3ntH^)$)koe$8UnhT`_%P34PbW_w z&k%Tp3|UeEb&jJCXs%d<3&VK=OrSoZkH52{56Bk+%&ZZXyR4xFi!`1UV2=!U-@bg>i{uEr^7ax#VFJ;_(o7y81c$yLkG!Be|1{ ziw#mDL0st?5dbc&1elaOx%g0v6Cr6X@z4-Ai})}{A5RxYwC*t%7qm_TiH;q-hdJ`nAm50_Ntc+D=v1rjW%EXl~v zlM({41Hfz~MQwgiMWD6ThEPRdHcXKg8%MXFzyv=nHjW8?0u%kb*f^fE3%p=gde6)y z1XUEAS^~|K#X=EW9MCLX3|;4KgQtXnY12cPggUYo1UC&dhLoF`mzi6d8xI{CB0Po! z)c`dUY(!!OiV=`149_rBE<^*=2&o^8ToSk(OV%);AD2jeS!z*HW(s&_4?2ih4E7s= z{;M#OMyQ*Lg`7ZbxQf)2c(_R*FA^|`H!ZUwH3epd7buf5d9iRX2{0oT?kUY<=Hd%R zS*|SP!o`u34_(IxS=FzB5!mNx&6baY;h;$ET#0BqnF1rXVZ;Eo|lt5{D{qfy%@5YL!B^CqTC=qzk1& z_(I^#31}M+l1kI!A&b5dtHlwmb}20^dlgVa6w#c&z{Dj8o?QmbB*T}G`*Lw5=H=y= zK-!FuSi(` z#Q_tAS4|8;>|D&jt|0=9pjlT(A5X_%0Vb{GOkC{HtPiTCU0poUYB^ypKA0e+DiLU5 z7eW#Y2=Wi{HxhV;DQGP4oLxweiwCCN$iT>0;03yniNH(n@G4A|VSEUv#p>_p6Dhz5 zs^nl2Mo1D&Lb6=qFbPXbOG|Sk$+^93=+YKQ((^E-Es><>_p%8Ib8*8A_jQaA`2YX^ ze|D`&>da(j%rGf%@Nq2GvXbXTs8n_?ArQn=jP+63jo>8J;WME(c)u7;+m#n9t;Fy!6 z0G2CO0ClH}%2HGGxD+xv?ODw7cyQ^47p**uBLm|HiOy`%Da4F>G zr@)tJ>nJ1^r79Gp7UgD^l%%F86cpu`Wu~O2D3oL*mMD~Dq$;H4gND5`^U@WP^YbA4 zPKrUAxD;|zORTuK6hJ@=&0)m~`Drj0fi}u0Em0^gNKFQ%AccaWOi;oq0;QciNGd7@ZNt;!;!+6l@C;T6_IC>j zcMNh>@C;T62=Wi}ba8c2aEeq2@o-ge_78{*@^tqIQSk8had8a_R&exlQE>M63kmXc z3JviO3g%K!atv1R3|0cmIQm5@xJCp7xdsO-_y;L~`&h0n3gM1HL5_YQp02?<3Z8z> zKB1sakB)*tqhE+4q*DpD(9JW%4`i8} ze-M{~qe6gVP>82o5cLf4X=ExjKh{JcM95d%C## zg*f`?a47@_xH@|}`sgUQM!5P0_&5ed>Oc(*b`1;#&1O3KD7ZNKI=Z_CD`*f3m4G0B z=g=TmUy!T)-4udDoq|I=LqbDb72N&(UBH12o%IX0R`Br$1;3j@Xt1jemx7C9h$GnQ zfFOT2&yZkikTaY@gFV3k=jj*X8Wa=?Uc9ZT;Nc(c8s-|L;OrP0?CJszVShi6Z$W9r z)juc_6c3ZZp24n~3XVaZ!JdBZATvS19PSvY;2#xQq$5DTv8MBbU=j-q@*bZXTpqB1?Xs7F{E(HELH%u+7uu= zXp>UGMMp|%T4o-2%VDuXT7GF>3P>$v%P&}Uu^uP`lz|2t6_A7#k`wblDrOQDK&?Tv+MEkk4=E((RYF{<0NOM$6B1v!M6K}?9CE`vE(P*sQ-#DoayF(`rsRfL#9Oo*U9gFIMJUWggQga{fl z$Z*wjF>!DTF&i_;NR>dt7O{yMQt>3ED!|sbrhw{ZP%)aCqL7rDoCvDIi{K3t-6T+Q z;8IA;NzX6JEXl|%R!GUuQ!h~{&o9bWNGSy^z*R`f&o3!1DM~CT$jnREFY7scPAj_^564Bx&IJE>+zk}*$P*o4AYakKL z1xWx3nI#GZiABY!pk&3x#mvIMD8y{eA<4zf#lphC#300M!KA^(!^H~Xuz+}6Fdmx_ zvjw*tjK?U%Y$2}<<1q^{Td1jXseue+6=JbwvS*c&RDgGTK&i1vp(H;a)UVUyf>_4N zAjE9VD+V!w732;i9uq{47vvCD24*1^YjJxPE`Ba{76w)ZW+nz97HbK6W-fj%4yXX5 z5VH-7D3SoP5VI|dC6WM>5VIYNE0-=86Dxxdi!(zCqm&{zfI(Zx6pB;9EtTxN{PH{n z=->r3jKFSlX650s;$mS1hmk9jn3S%9qe4MWVsdImeojhi5jZ6z<|&lsp%|e6o^a9w z2O>x>qY$$zlOV{?ARdztv#W?G*GxvR!Hh!8o=oCW-Hd1^LtO*50a9gGg5nlBhNF;_ znvqzR32Gi_C8eh27o}<`Bq~5h7Pu736F~(^VsS9Z^rRXUrgn*i7nZ*jJX=$KV zbXjUnr9xt!f>LH4xCBX6C`c?WQG&ah5$rYrsJlU#)>BxFtBDEhZYCjSe@I z7^DicaSfbwxfGHTle5bci&BaeK)Eon1iCjC9zaZtpg@2H5ECd!gvCJVoRxuzS%^7+ zNgNt>%#0u&Od&Iq5OaXA7}t783Sbdp4rLOPTF9sX?a&rO+V2^OWvL2Dsi}DiMX5Qd zpi%`A9H62H(t-mgGjMxeA+abav!n>raY)Qd;Zi8h$S=)F0kwxgMLhK20#GFd36zqI z)Jlc&%$yvB#GLZP%3_eBlFZyx-Tb1I)FMy=GQUV6HLO7gH6EJ`iT&jI(GAmLD)it0@+NE11~2ozqK&?>pKq&PDr6_hkU z#)0xS4;L#d0}~q~$bM0%AJ~|Ln4=`XqE94Mj zj$u-QD&$}U@g!k9CL!h+8AWKQa0)TU!WD8df_N~6oJ>N@v2cZ4LdMbP8{E-gy)l0XfD)D&2bQUIm;{Gt+Y2xb<; z0x&->r;!OTduVUFq$o8NTt{Ap}m;ARe<2 zbB3f8mloJcMj_@*COIWp1#rtpDJ?%=Nk;)h=_Mr=Dd{MrmLx+w&cevV$O2LfvxAXU zh&dB(2goCtLg2Utsbm&n&XkemG6CDcB*dJfRG?N!C8Vzh z5{BEx!~)U=vyF*Wh&cyt8>ryQ5t0E#0ShA&sI1D7SKtZ++r})!oX4ag<)Q%Ihnotv z57eT7oOJ?i$bpIoh#Mdy0?-mBFS!!Z{RfW_z@5U(0x}ck6lPW-<~(k2VaUSB1S%Ty z1i_gI!~@x`s?Jpfb_$CSb0L$OR4%$xKx@E>bBG?7f?IwObTmg7JnjQ(pyy_$7eTuB zpp>tt;F1q6Docw~;cjLDTMlzG3#$-wA>7TN08<{09CD^ zj(uuHa!x5|)Cbh-2Um#rV}q3i8c0#%vt?gw~$2--=3ha?*dNF&TXHjpvg;L??a5nK#cNUK9b zl8sr2xk5`DRA{m=f{NivCS9l$JBtu=B|Lw!vk5U*LYt+GOrRpVQXbNPWn^M!7Gkc{ z)`1$rA;et8qz5&Gg9XHcxrBpFh`EYe5hlkd#9XDM3F9#fF<0s6LJi>*Vy=c8!pQ>S z!3^PK6JoCBmWOHs746juFhe+*g_x`1hHwcn*D}FEmWu_%gBilbCd6FJ4J%+k#eS_g zq|MF9#KkPcT&t+W^%au3xP@4n8JYB@o-smNbKpfeu*QEeG@&Hsp!CWTlR)D>pwtEL zp+ZYQ(CD5XmqKKIDR`8rI57>B7K^|uyCFpiXt`EVssh9s*bq`iVi9Pwrb1C^9;ns8 zrI1#d2OgD81RWg$u52?>kvdYv3c9-Bgo)f&%q&*OOHBo(TTmVZ_l=<=si46(aK8am zU4uFUj7;1tV1ILQLAh)oF1I9%%Lw91X~DS6Ag-<+*H0#J!sQWS>0xA&lX}Gj$~CZt zYJM8nN#GVExWdrmf=HKww>qSzKw9yT=9LmynGvXL2eXWoAf0uj)-D&9f@hjS0<;N} zppXd~Bup(%%`1V9i-KEg@ZKb{PS7xzpQoR@6>^+Hn{bf8fe!zHDrL|ta8hX+ymp4w z_=!1O3dxDZsh|PaY{)K7;JuNQDS;(3TWhkO92}G zpdoXp?clO0A5zAHO@nrH6&wRRp{=xx)XX9U$VOcS(D;67Vmhc4g+vq&3pngy@y5dn z;&KZ@BZ`L+#1)c&ahX9}X<2X=gN2caSBPa2Ba?#EXHZ&)_a0!$3+xoKGlRI2pyC~55~C2yOhzV1zGMQme?VNANsMeDS#Ei#Rz|R51yvZA86>L#Doa5o zF$uBEVr0^TN;0v4xiFKM*g#ykNg(gcVpPzCDP{)A>VRqzkV(u!EOQu{w4svBEMP9o zBxW`cmm3nGOrXXI%N#~&HJDaq!bq))DKKTEQ=Xg^x+CXk+_(V z1z{#A8W%J2Dj`%bg0v`WAq1E~0=l63925bpLM%%WiGq~{%!Ne&D;tOlts|H~eNmRB zj8d>f!O9Ghg(nI&A(rKgO!|=EXJTUkb73a2fo0*zi;WQ^s{l*YY|J1o+)eC4EGrqA zU@3;31Q{dQjqHTgS+>k&#(yHlqUgATLna z2b+OZC`n9LfSi$3tfv4wD7haTeGJ?25u=1Fb8N^jm;c^BQQA}JyEISyPbfiob zp!+z%Ba;cK6`*nb1W@}yApx}HC?`Q7JGHVrA2ezP>aygO=E9nBOiWxXU{hcY;9>)D zxnVU87bA!(1Is&H%pk6&HWv>U6AQBt+ipgN_l%5O++55o%#1=TyBRrTx%|0USil3r zdl{Lmq#VHwIPge!kwS4vei5km3ONK4Gz6ZKpIQtWMl4EA&QH(FtV&hLNG(d$QwUB? zRe*I^K>@`KPMmvTbsjSl1Dg=bUPfqH&J0eNdl}`8!BNc&PL_Ka%`CxXIWwp%-_OW& z4O*5ngUj;$j9j8z-dt=f%#gDDAR~*Mlr`A3h+GKC%L*x}$vKIjF*s0b8{DBvRX`{L z=R!R$aES;uj75m$AS{uC1z14>&};-2U<3&WOCq!|g9Kz`xvHQJViaOI%E+Q5l?8SX zXkffj7j(o3X#YkEByB+(%LsRX6EC3{y405l6BG*Djs0WyYSWYpr$VyFs zG!#MhAo~E+(gE8GDn3CD!SF_C9%KL-c3g!XNQ;gF((!a)5lG`5JT43OD%hu>-aE8i zRm=s>3!pPoLFpb;MT24iRN0+E_9qi7$k9mt1eIo|7)2meG*}BWNQJZv*EyI!nT1%+ zGqT7_?PUaYHA;#~Q*{(F^HMTFCzL>k=|J9tCZtTnY8B9oAV>z`ad0})fw&bkmxqu6 z$1Hdb58OY38U`LLPD{+p0ga=9M#D36z&QuB%p|jzOQAd?HBTWCk$BO~%LFY>01e`$ zAVof?@pT?KP?%XkKI9dI$2BOUpJx=3LI{AvKvs^c3l=CWLM)d-fl`GxOc3q?5B?{C zx~FNGd7z#TD3~BSOZ2!PU8M5NVn_!HuP)GhV18a=4w8pJZNSUO9%5kyxem!gEQ}xl zBoDDLg9K#dxKy}ULB#{75X)6Y7JemB1xPnMMIo~|9=7v@3*4AuW@g}I04d|;gM|wx zsKt1d5m|rXF!6RZIpA~_I!b*!%i;ArjT>L=`ESX{MkkA9PL#DDUJKq(YY}W#*-GA(cvyG7B=I23pmWT9KMu3LdpezpQK(ZuVSSHL4AS&x1`V&W++$Q!=5hd8 z&B7qW_K=b3DI>F#0l2({oRX0Y3A|#^$(f*_#*$lM;R2eyWqHWRF2UstD*r%pwn8kA z8CgW6j3GmF8AufdeE1fe;UNVWIPpU&8Cco{8NvuQg&&fnKmtr40U=>X5VA0Uy!C{U z1zIwL_>4jIc6cYSBy-r8Lg%22zV9jI?w=&zl*<>LRwB{ z0l4si&32aLDGULm3l z)Y*H*$SnhnHc(mficuEQLttiNW)@<3#i#@6-Z3*Va|p4#W@NL1DP{+8*`*#dh2+xWlKfogP#w5s2TJ>Spq4l^ zwla%h>poB$C!lU8GiV#(2Gc2Iw`k9ewG83e0$^uS&pBdS?xq`Xa zSRkthSiXT0pRjPm@Jlljh0HWA1=yxj?nU{f1>nInQ1XTyu>vZB@*wl?&>2^dNLop1 z5%kgq$od@6nq~!${BqE2st#n?m@$_^W^O@FDrgWKTqnRMX^Trh6ZT1g#URJrr6`nTCW0+afR-Kbt)~gF$`o3&fFcLfvFu=i))Qa>CQ!%<3Ulc~ zLz+v7rIU$8Oi5P3Cp9m5J2R8J|UKVCKhqH=lB@G0+2Q^$Ot|rkbsC7JSp=Fu}nbL z!OsX5Kvaqm#M*ZhAK}Y>rz#|~@nV_ToEa1Vp`ApEE zA{Oun$b2RtNK>2zJOVPG2|8>Bo=sT5!~|(pv4C6g3lJuO`u7W%xCOxNC>C%uEMO88 zhjBr}(o9m);1PBfkP8DTwr^2S_KogSSl>kYpptXUZxJXk-$}h=)OooEGHlUFKuue#O z6QLK@uguQ_FE;|W6hM1}bs&uf=pI_oz$;h^d~!FmWy%6>nJz_m4b(DS%EZkN4HZz! zbSaa72#gCFyJixX;MxwF-(&#=%Niyo1*sKG;I19CNCu}k&@cqJaw-Ou?3u+N?}EA( zptUiuZU)Fl5TC+ixD>#5HWWif0TLC|_0)Ah9X8O+erZk)(%6cg0;0#Pqu{KqtqW@X zC=@5>gI2b3DHN507OH?I8(~Q_EfcyI1Z1s(My6h>oY5Ep}O2lvE4 z0}OCq6@j8Au?V#959FuRJcK&X0>&cnglm4D0;pEb2Uk*{Wj|ct0U+>Url2**pl%!s zsDaF~1`+C@&iWceA_ui})-VY{MkiT7Ly#bOAJg;j`U0~3?HQ~@~55iNL- zn{bD-LQ*PdNgG&s3b@(nS)u^)9(ZvZXzK>p!>~d4#Js%JB2b+T3M_D{gQpnKY{v$K z+gZWxfoD)wMv(J_q@XDVG~2O(Nmh<)A|%DI39)QrVp5T6f~1(d(%jUd%w$Ng>mXJW zfMNu`WS}Iq0<;MTasmlt(;_SdfoHa%OKp;DZoE2c|A_RNNn3xo$%0OdK;35GuE)N=#1MTigNv!~dRH6d-#sg4m1Qf6!KPF~^&gTP%yLvH1 z8hrH(7kCvFY>^(g^athKM1|Z`&p|BJfk%5G9UYKkKx?OA>k*I_ zs_TKm0K6dzG_<6Ukys2WN)-|nltAZofD^G2wDSwvG*OZPGar<55=)>`>ctS7Ksyd{ za$u_xp-UJ*<8z<_3ACRt6@2O=~SnURC2p?jSfXuM4Fo9-R4uMk922OC1!X?CVlZi=HY9S{$2*Ja<;NZyw zt%8KMBa~pnj!Jq8;gCUBs0`TWpqPQF00j}K6Ofpk4O@$noT>mi%BUzMC$$(n*Z|Q4 zDqcW4PQV62jDug00WuMklR?n~iz?VSQciweIw(kCNxLLp0kqKtv=pTzUm-sSyd|+D z-x#_+1e75XOB5hQNOCH~k~oYpafcIo{?IT znXClKF_4uw2+c6Zf%lyh=jVVXGeNnAOF<(wF*yUYt`6iRSSUee*g#FEVz3U-Rsgsa z5F0f06oM=BO2C#vM2bP9pP+q?(Bc}hAqTW+L;=(&gg7Y$vZ@e#VmWB}JE-t-4DiHe z7`*WS8Yh8w6+uH>N=2!OIZ&U1Mo7Tb4XiQ*2Uf8T)JupcgUxg$7lE$20k6-6?bTL* zEY#KEQYg;MOHNhD%g@t=4KKq+oghI{3|kytQk0mS4UaWFScetVA^`VEK|3cAlMtZl z8L};0AqBK$GzV0QlxL)Z*B*fiK5(&6UIf~>h)B$!Y9cSc1T+K)Di9z;!=RDjw9*_< zpH`s=w2)c1q$m@-X9k?rA$@4b5+HD}Bwv|Ex>P0RI@EQj+pL-Kg+;OpjN-%h{71T|?$s_-;b72H9&k1z=|&2XOyT4u9?J1F;IWi~5#g}{AK znXS&n#KI=T_LzyGftgWC64?`Z3h-t(bW9c0e0a>n0qNGTfqe1=<`XusPoBW+X9N4> z3Ct&KV4pmJ`GgJZlP55ruz`H?j0w6diVf_OX9$x(K6%E(tqiW4*uXw{#-sun-(~~* zUgtM!7338og*r+9%d3Dnhk!33RpVFPz1UoeS7COFu@6C5v?6qUHvgIbqt zpc(KFOiZ7dn5E`1VOh5VYi)v?XUI!eAOhfZ44`HoY_|_+)e7h&`OH-CU^ys5ftRR& z0vYT26mSm?u{;Gd+zXn{18<3kG|fR*`aADOT&_X5kZ2(f%*f-al_3$TI&pmY9U0Y;Dj zbm1uV+~mWC|N0t>K$1dwb1jn;f(f-d(0 zt6&DHkd)$D1!_;Sf%@A&n3xQ(FI~c!1W+Z=lLD7QQY9pBL6ZtNZlIkdP;!9IKZD0j zaV8Bew53aI;NJHSL}~~1zJD-5=Y`ln>#$gUFv&tz46=deg?=#U>2u|9v9Pd#GRhxN zS1Jy%um|iCunWPhP;9Gvic&%SVLb&$a28|CyjGc-X*;?EW)B zSMacb_7AZ9XVTQ-GUQ@nVHaX+W@hMMM&I272`K1d7iirFEjB@2mS$!S1@LGPJE+)f zVTO)Sv4e}v7G~%O6+5`tY+;5r+S$RyW(zZPgo+(pY_>2%N2u6A#bzrrbcBi>Tx_-? zOahHiwK7BdP3+)evy~Y-Ld6a)Hd~pYBUJ35VzZr@39_t{oe7j0+7Tv!ip_RrXy~wm zi_Las8Aut+4lXv^nbp*}RJfQ}IE2``nHeT9GfIgeS0xJ2`*ENF0_uu%Gjo9U2C%Sj zfC8ik79bqp0O^5Q&H)aP9$0{IfCHol79bqp0O^4R2nQ%YdYPds-8jGj(u*(&RFCvB zb1Q)B5e{&G^fD_#R=RP31EiN(3$#oN+QAo6O9xkeN|R2_--vmui99l!<781L|E(X6BFurxs37;7ox9 z4ktKprobG+2@aeou)yI22hJ2&;BbNiX9_5Ari02dPEhE~U}jR3>R?px1YMN{9$JI$ z#RheqAlV$;U59PT!8)E(o>-}nSj?pWKAaG})&Us=?j1w6Xo8!4MX712pe@u2hz24^ z8MHXy1V_{igs(s;cLu^&pfY*}vk;`m&k2sG8O-vK+{6h=xiguWAj>B>!4WkRVG?L{ z(oBR&pb(hJEF=p~0i587n#rsH+B(C+!U-zsXEQTF7H)Hb3)9&MlR$;(Y-Z>(NltKK zI-6M%l9o8Zh3RZ&Wfd-0E|5tqLM-!`narik6+B@F1s8*w%FyivkZZz<6=3&lLA#Z( zCVnwwT_Gp9l$?if0;rUn#|&+ja)P!Qvdm)^fH?uQ0B0V^381k6PEbx{na8YZ2K6cn zJIG0Drceih)>UJ3ASj!rg4P^@?lD7hAXFUMUxPc46|`U*;Xu&f-2#LILG9iJ2nT`& z?-n2&$jTwavH;;gR(23q&6I0B9~Tn~mk`@wW`;v7j7EM6wv+=d1?YL?Tt*jo6li<1KLyRQ)^pA{>V!`Hq-x=f&5 zNud5#QmO*1@0_AgoC7+XDW_5)B@=Y0Q)*rbXv=DT5!hbnSeimcW?l(oTMEPs&@o84 ziFug?rJw^YK!YIpkW&?45d!Kr!Io%%&e;Kb3uB2!B51A#w)GBF^nlu+TF{_@r#^7s z6_o$Mqr0HlM#$McptfvfDyT=O03F7FoW=ke2nKDR0(bO4CL+3tdHE%vG5Guhp?AQWH}^-9ykKid4|RN@^}>WHAxa?E^7EM#3BkI`0Y6!Awrf$w^I7 zK)I9{I&c6Tbx8yFO2Oq1I6gtknwgUVUPD-%T$BlFj_W8SmOxWSL4Ial z33$y9czCK<0qiu;?szW9(H7t>OrS0^XwU>H20(KU;9v(uwQB{`R`?>m&d4tYt!V_MC(t2U#a3JjTHs*@$jA{a-GWy}f(Cs+tu|1r4%}x3FRUrf%uCNn zRY)u-$N{a<1f7tQUj#btD!(W_F)y+_m>L)u7=TPE$tVI3S6G4oC{=)_5}@-QmHDNhER>p8TnbJ^i6sgkA?RQLBz96k zhCpJr6f{5v9%q4|rSqelbIj0iTScjPgvI%t4Z$WAv zs9*ZS zvAlqe+W9GtD+zWbXZzOemTf) z&?y|LdEnE#pdp{Drx03P3LY{7MGH9jfHmjmfnx>keP}|}1MksHPRxV$a6oIWA$Lu} zQVKZLfd?N!Bb10R0Xqv4y`a(h(h|^6I4BQ7i;LpYBwc7}2w7|f%I?mPC2Ft|t{8M$ z2~Z^1u6N-pnOrJkXn%lTCD=g915hKhzBwRa`1*SG_iq<_XO?B%q&(YDJ{w? zRsio@%?D*INcmBuqmZajo|*(}`$0~zfK1maloo&tD9KkyEG@~;O)LQ|E(0BLQVbfD zh6WTgKtV&PpfM)U7}a5B4q>h@pr#%dsMUCqnMqmdDWgJgY6-YpRREpts{pz@EipY6 zw9vUIRRMe}XlgN1{RAFK0GD{sVM%bg09xJ(J}(Be!px~s0XoYC8cPSAi;@p27a`Y< zf`$P>r3a+-2Q~;a0-jcqng<$RO+<_>z{l>uc?RY%&@n^cfsZ`Ufrg;5NbnpkBw4@@ zGJy>}KwGSk1``*!b$k-m9^nGFj!(i`e_Y_!@kwSO$Z!uAxOIGzSy74WBWSpX3)D8e zz|5pA^@vHqIUjVXRSqQB!1)Z?Tqp;fb(C5J2}*F_AqO4w3_NJSK~6DH2mw{z5T)Rt zgbjh_=Yh^S0;OQ&^B~kgaZ!S3Q)_5~PB_fSPXRZ$%Rxh~pxJ{w(Aie7o$K&ShOzJ$ z)g_*33ZSE&(XMv|MOaE^F{mU+0mo+^xOxKR4#;7FV7I3h!>6XWklhPUw|S5m@60^# zIVB+7&{_i)WZ-LPAqfuJ?&kux`!68k3)JqvfQTL0ey-QvyLt&!F5?1yIkcAQ5zBt%4pX zWFQCbfU6Ji3=86L0BFgg10J=3%!+`T#F@#6_4m**97vEuOaiwTOTkMPKK zAty&cD=)u93sf3l1{*BAz`+ig?oR<#K_CaB-{q^P0Gq@B`&$7;FC=QfGaCw^{0s9o zxcr8-G(iCcN~rkKodQy_15F?hPIn*^;gJj)Ot^uFWKfsv2CS371@1fFU>1N3CUAkf zWH*@AAOjIxOiUa?EH{`nA*+?Sn3&i>TrEp3O>hce7Gk;0%;YL14cTuFDSQ<`lSZHt zBm=S}4AdRs0{7-_BOC$Rpm7`F2++cm+sp!xHPT$*-rQ|w10!(Xi3_}B`Zlw%3wYrg zhzqvY#F^_nBq6d0u{>vHvXeT%tl$W$olEkKz>_)P!BqHsMM=H_X!;S#Ukh?|F{r_snw(t>s>I=SJgm-3tpK$^!7I%`=ktJ1sMVnQI6n;%{#=j<09B=s!~)v>@*I&^K>hCL zh{OUKoqWzL04tV2{qEob7mb|XxahwyPq@b+Hm=RQY>f{`YUFp_sq;v zHVW`Vr6Gl4Dr9jzjsyDgixj{`S#f3+Y#$^Scn$k2W_DpN7cMpyF3?s7A(q$7EPP5P zaKpi;CV=)UL9~Ms7^r)enU}5uYI4Vl|Dj-fZ_AdY(prin5yn+`1Lt_>)<^>98&^k?!$)F>UIu`&k z8C1W&V}>rq1`B|V78U`IvvGlnr4P(Z*3ftZjcj~CBmvO)@CRn-WEvNEeE0*i1Z1@Y z7kGU51G74;&;pHYd_WXhpplIa%vzRQk09*~HX)Yp%uGsBmzfYvGxvO0`v=-ED=sN2 z1)YS8D6|PT%skT+@^e9_;6mzg1!O}(^&WV?4B}v2P^2IZR<;5)Id#F?TaprsQ;{1z zuqGqq>`MjEaqK#v5(suoGx(5TUE~FHpjri{3%nHw)Ek8L+1m@?2U@1UXuECxPqG&4^D++2X28e*@Ypajm`prswqW&&tC zWf5qV0@A_7pay6yU5-Tv=R_3K|><0Ud4!>h>dSN&+2_mCB_6 znstL@Xs8_M07Z0XC}_aKCov@jJj9il1MM1u&M$@JCuC2Rr-HlKsgT|msPG4k5kWlw zYv+PG|A?&s;5p62qMS+v=)!bxb%DNqn2QV3GLXme^2}D_yw?_q>`3c z20E`gO`#Ojkc2rLbQ*MG3CuB|!@E;bp(n+G7XIafx~WC@g0u*H%A^j+98jAK+K~a( z5#ZVcv=B5W1roN9@jS>ORG>vX5L=N$405^*XcP!uFG1@2)Lf8>p!y!3Btfg%npvRp zJX}nmRc*~I(7{|T(AiHc%`EU{5NK6fGYh;K#Lgka(#!&H2C=h)xLTIbl8Zx#rGsUkW z;S^$NWr3HFpe16h2zx-A@LLh~fF?0o5&i)!5o<;Ghm!+j6x=_Y>>#d|C6_iAGYc1| z5K9{i6Q7bCeCKFEMghhR^Pn@QSXj6~8%Wz&xOpKi25%ol<}!iI;^yT7cagb-SlU^@ zrxLQTaB(q!c-)FypCKJ)E+Lj)7A7O9CyWZ7&_)iZ6{~JxE#01#~ z9!1M6fwbL0qj&kBVF~a=d}exHeo<;P53kcJLu-AgYb4ngZ)dJ%C58dK-Kc2Rhjm+Fu4Y%;00M5buFk z=z+FogBK@(M;=p)xgaGIc!NM*DyZL<0&hEkS~kww+VJcKI%#Vn!fV{DAP2#-8)(bS zL>B1W2^SMLGe}k$mMge9KwLG*+zA&GH#>-{Zpt+Yl2mwvSf;WtX-YLCjjiOvhTjyD zAx;9VGfT^?fDZeDn&+U^X?dV?!a(JIPL2YS7A{E90Qy ziZ0jy4BWe*1FEJXybC&@YAOqK7MY6)wBKhcix8}n#lsBpftm)FCpfNng;-{=Flk6x zLr(fDNK6LZa8QKlCTp0r;MN@|$wDi6xHCY-vyu?TSHXih;LO zLl>)p$F(5+P4Hd>SW^g80Kkrz|*37P8vb)a!t0}5ZzW`1~h@_}X&5#ha_=Q*&fugkl(abE!C?GaX^CLqMJoP|kM zYCR*kYk?Hrpo$f;_7mKI1vON`+aW-06|9cdQ-C@VyzU9&M#z$5WK%)MNrQG_gJTe$ zoO zb1?~kj$UF>QsFuU@w}iA%NiCYJ*nLkcs@6^BoTZ`475Da1Jzz>phKY%F-0&GzpV%Z2P z5jGPF2xvJ3N&`iuNtKvI7tx6TG|LIz`2Z>dV8#-e48T@H^}tht5DVB7umBKZ1##g4 zAjAmb!n3dtGl;9G3@Z|ag;=(O0-%YB!~-6N>rsqcOu{VSFoA`qFdK*qJ+p#~NthAD zg`Qc##U#uO;wmb0)qra{5h0e{EKF8Xc?#gW2|xp8pv4YE9N%50c8Y` zs1kIZNl5`V(F9&m08a@bEMV(lE*D_~aiOO%aWRR2&Vpirp2oz*B*F}mHM8VuX9P8h zS%p~kvoJ|YRZ)Br1NpuHEjs1`9ow*<1$t})7x>tQ{VdR98@Ry7Htc5+m4Gz3IfYmb zvM_+r;o81JJ1s;5H_> zi3YuQ3KXO8xh7rEF+uP{^C3+LP;{f24sO-wCW4M62c1I;=^BE@p}|w%;Hm~>ENJrt zWL^bwT7;E?f+uJbEodVP7ibVOGdUBy^$)b+C@r-dbU6d$j^AR)5G&|V4N%hs;Vl|gYGaxh=2-w@C-MowFdV$XutqGo&xnpu{GpabMPhs$lx$|Z7O&b zGidG}bhsP1=LKq`gInP+oggcnpqHb7;}F!x%7cazq^$tX#*(^H+cH;I*R~gy&^Yw`tmx9 z7G&I+n~8x#h~+v9bljPniGdx&)wSVz4XR1GL4%)nSeV?T?l3C2KqmVjJu+w$4>YO+ zo#HGmNz6-0EJ{&ON=bz+VA0T20uAJ+78mCi=|HA%YS#O8y8Dl%Qnw2sT2%4W7Jv#3BosVCM!; z-aTTGH-H8uXk7IXi-JD5Zs6t=VtK;CBn%!};s%Z4JYnJHfv5zJ;vjQD$1#CqK@|`; zXk_UL3m;@pIX5RLHj%ktwS4?gbHO}8$T7p*;03TxSP)#8>;g7$#R59LhJ{H^Y9bq` zVGQ5r1q~BWvpF#jZAk_+`=q8QB+4gJy41Q;-fNSIEiF z&xRb94IaZslhCBu6ZRzm9P>XbWtLt;|ew&v>~NZ0i|3=s7TCFfI16w7ADjY zAZZYW7?PNSa4=+jpMtG|5x5>uP^g7e6!1b7)j_$51<1bRQt$)~b%ACIAj4#k6_FYF zplbv`1u4?74=896&bI}Pv%+dK@VXOl4+V5>2jt2OP~1SL=b;03u%UWTy#=}g32b0? zYGpBa%@lYkKcry*+Gh&8s}piqIOvv3(2g1Spo+%NKg87vc8LRYAQ@WKfJViT=01w`6dWNp-oTnY;8X?L@0nNu zT4w`ZR|y_6f*Am9kbswHC8dJr=AegmfNX-yEI{fe@SGv&q;=?$M9}FQkkSxz@Yrue zX#ko<|BWaOK(pw-Sp*~@xfnc){+mTcmP>_;nFW;J|FSUgDv5*Fpn&G+;)}s|P-a$v z#s?w8x!j;};=iD*4z`aIWFj(`Nr(l^m4}-64`iYc*u=cVy!`mY;&_nBFvCEHOCbyc z&D;J%;)2e50&(wxX7RZ}s}#CenF6HFF++ze!D%8VH9ax8QXwNTF9me{7pT1f-c}Bd zGf1yEIUiC7g3i_gw z;M&j_Jm#qf$^4*eXka&HfzS0SR!D&FMo$1O&&vnZ)A^;KHKutfNYnF>m^Fg0A_65h zuw&7Z1$c}Dw8BWCBo%bkU1CwCLT+jzc=cssdLn2LOChN!KRY!Kvd*=NVDdZSQ$eDR5`Q>?# zMf9L;sEN7IWwaTIDGDW-x!^;kAWaL{Tsb#*Eqxa&bozxGv~iQAixoN^%*_N!8(pjd zkP%;Q@Q$M{R!hiaFgJKDeHW{ZKX?HMh|4U*(#2})2kE7AgDS-dtdN0A(vD#quDx__ z@cya^h~NgzpHE+B!o5(}>6p#q@0T>ac*6*NE&gU$wPf)4aRIZy!XV$j?+7o>To1U`pF2~scWu zU-Gbkj@Vtq%Fe?j&&9^V16eh+gq4L~iXS#m4!WEhRC0n=>3|h53b8CjpE4&9Fcnacw&LQF(GwE-N+;51TMkjkY1 z4P!k8(9uGur$cAvfzD`xbicua+|XJKw0;_NTsrs^vqS~w{Gt?iV+dq$q5>qf>nVU+ z;-DE+g&fcnE>s-6R1-A7l31xw1U``kbW%-zQ6;EENH2QLSjy4B53SF0lt_u5z&j=QP99jt$0ILud;kpbdZ9pTetW45UM<7uR8t(>OQV3oV2ww7-1llGJ zYScjvNQJj|3yMJJPJ)gm1;;da5)tB4F3`q6(4baoT3RM}{0pPx03B+PssI{{(o+aZ zgm%snL3c_*&b$T<7nK)f!gd9K?~ee@?t#iRa3ch~L?;z;ASEQ_K=-mI78NB{f_it* zGgQHCq(soMBA`t#Ip_yDCgtQO=^+lp(*d{76ksb8L1zSk-H-{oUyw@yyr={+aRM2= zfUahNoE8JN5wuJddZ#Hg+dzgQp$F+HWae?fPuv6dZom!G1X!>_`cRk$SHhwK+@dSV z2Q9QIDay|QjRmA4pQ{Hcen98fZ$}hApdpj(u;vjDc&qGoRsk_+0RuXtdON82nFSh= z6@9X3OS%nX8E97@KTdAL7VA7^ z{5;TsPauDQmhGnHCxdwsLJg9Dh zdLlUyd|wtw1X^XMrXV{WJkADMR151+B5Y7lg6v090>>CMPT);+=wKhT>V)<-xfDPS z@elHK^a1$=wXX~sDTC~vg7-q8X$90j&I7Np2etSik*$EVdJUAWz{LjW^w^Y?)D#`? zz%Cc~Fj~Fb#F7j>&%Bb<^i)`vl?Ob4bp(;HL5<5JtkCH<9`Ne(BdpMyyLi9@SVvf; zAv=wEzynxESY;vWzZkWrH0*>I$&fuy-& zB*Q??IE8E&=s?g@tVo7239+2TY8WJ%kqrY4@SR0A43yW;f()C<#l^zH1KKnFjFm}S zs*4M@RuVi)4$i%xRg%z}D<`okGpEvCLBS`nsuJEU1?OSVSa2$Mf=jDo>Wp8~hExfD_p z(^HE;+b(lJp$hUmsG)&+Fd8VtAY}?Tqr%tjBW)ewf>&DLxP#_J@Ss|8X?l8UF?jzN zXn_p!h3<)^B^mjkdvqn+xwxA2)-H|oaw+@JCi|! zBdJAcptVi;Y2cH@5XB&+yu!V$E`6x4JqEdsSa za`TH)K}ij?hd~!~K>#!ZXuy;4XT!lLFv3CBR>Ung9q3$ zkZIr>h(WuYA!}}OOQHKDOEN$ff|GM{X%Xm{(40zeos$DvUYw^3Iy?+vAZVXYW*&5i z272sHVuH>=EKj;f+&M$n8Yx>bnc#4Rxsv|$|OnhJ=CMXAN5IiMB|Ia-YV*QQW#s{HIeLbuc|e1u&k#*2 z(9q^HR_Ix{JmA67XRP9o+YorbTaKQwN*?NlnZJ4G_a`&jy7$ zXty(XnPv*e&qb*UU^Bs;6O@2MYN3ML^Z7*zNF5(gzaMrJ0;G8f+EMcw5f-2wHLnq2 z0UCXK%?dqWjR!pX_L@}*vb%x@Jo@&U6?&2d59k0Mme;Imrd%&TGjcqj!MBgBOomc7 z5i@e&Ljz8O-3OB%#?jus=Y#6B3A^Jt-d%fe1Qn z<0B#vL8oneWQDHQ8Z<&TTjg19t0%RhahY7Ub;tRrQp#2tK5NREB>g*R*amYLY4-*>) zNUs!RBAbVajUB|5*5U#mTEhdHHT=QKWCA&~268+jxCsqT%is`#&O1TJ0ALk+5vUAK z0dKqp*T#^JS4wJ9X*#4rPb>jV{p+Rcfs#6;dkHZPRAz!kx)q93^NKS|KnXm#C^ZGt zT};eDLJS=F<;W0a#NKw4nb7t32e2L>?y4g8m<@N{~rs9wyL& z{vWK$MqH7g=wM+IV*Aa?^q-Yk%2UBJ4OBaWn!`b^A+CNQp8kIE!TzB^&aR-k2i`XV zw`o9o#~>SkAaw?)&PvM6ODuxjMZ^O>)ciLqyBM_l556Sz4=W4gVk#ccrKyZUEPoK= z0ZiZ%=>M=vNWyh63bFh})&Xj8{zcXSYH7Wpdbgd;Tn=q zKs{D2&|%hzuyr7Mpjd*m_Db_$iwBWY6v&XIR0U9TE~gSSBnjHitpHj;0ci$+#v4H^ zVqkl2KuN6x5ow^hqK%DR92y1Sivru(SY)7sI*^n+=}$m_em|53&wWsn>(7 z19T}%53&w6A(mcbpMwgcUSywxE@kOO_Bp!{OCPdFL0P#E*;3G@EPZUUa&Vt>2(k1d z`y8~(rXShopi5c$k$ui7#4>@6MH(KMpj8bM*jOOf&VnM06Lj|l8}y_%9?-p;li8Rc zXI}DvFTk9P$P%FAnJ2SB&u8Od0+}_L4SGHs5BPZI$!uyGkZvpJRLiMsOf%S^!@=M~ z8>g~yK_eb=XyY_C7Db5DKztS70oOB+wBl~M=BBN3LbDP z1AH~_B2ZGj0=n9ShZ%gg$}%<<9jU{h!X0t7I`}kGaAy`e-=$CvTKfpTo*@x-?u|lb z9;n-$2fG!o7}TLI%E<(sRF{zmD!xITU2;?wQrmQ|pTNe2(frb0G=f}5VOU{?PK!J!mMM8W68nQ=DiaCkJC7_)`sOb^b0tAHs z_QVId?;138ngqFc3w!DVE%Mon90;JJ4>u!6AgE&B%q9wL8!$71d#{_>)HT4X?s%AZ zgjlw+L2n`FVFGPU+lnYRc-TO)@NyV5m%Wt@dY?HD6KHGNRyG|y@KOLCCSDA_{58oe~1!~1Y7CrHRmQwCz zgD$1y;RH?gB6FEQnz?znETJatV`Jh|(t#gA23a1bqX41f^OHdL4wR(Er{u@y<(I^l z7N^3jVH9H7#|GVZ%L878iOgk!%7U)E<`rT)$i{FMx^@8+#0S|pL?9I{FKF$;F*YV; z&=oGDYiB^!94~k&z%f`FB<|knph8Nt8I0?%zyx_IeCt(=|wE6Q4jWP@`D8rnAWf)#?hB*VvFudRl za|V=QKk1MQhS&&DClmB7Wq!Usxzm)V#Uq=F!ONkDftK)Yl}=kJ8 z3PGv4`DLIJT(K$845X;N=H=;Az>%Y}S6@?l*|bEX4Ad&Bhl}rSO4TWG_dnlu|_BjSoB*@e&c- zpcdUrSe3#DKFsSZMaFy~NR;BQPtCaV!DurK&?GqbA4?AQzCTJ`B zCpHcVuJ52in;%rCdsm7P0)`s_r{1Q}ctXy6>=JoF81(eTNm}{NOc=-(kf#KX?t}cQys+cnm+NK=?bF zr;+kBKe%P~7ZKW^mf2rep5_O)%>KgiG(Wh~{0qv{>mX?qbV)Qjld;r%&{Y7C0ddI4 zH+TvJv<)UNzXZG!54^Bb7A^(M$rgTaebK}Yodn=# z0@W8y?9f&eKX@5W6FYRWg&$mBG_lJ=PR8d4*B4FfijZYI{NQCgP3%gBTpnC3Ec~F_ zqLrP=TFQ#_fhkD(hwks@2N$xf2seTX*;aOL$T@-h;6k>QT^+K;fFE4Qwz6AVf!po; zphC8Toq8QzesCe%fp83{knLcHc69l{g=_~qw4=)pE@V5{p&eb&K^@)fwCU*bg9_Ph zc4$YJA6&?GvqL+&{NO^in;p95T|kJfkDXx|w2%cInbF72AqJ`B1we)D6!y^pT0|u; z04{!~z@l6L+;5!14(&GzfQ#QL?9c&P0dVmn3PbuK=ikozKo>guJm8cJdod zYb>ZGw63n8))eb;8NFZxFLtp~fZSgKJ4jo>Pyuw%D(K!kVnVV44M;4=&&f~Etb!b&l#!nZKKv9k zSzVk8w+eKrWNA@xYEBvG==h}45|BaVsS4l|_+Yo#K_;wAGE#HF1BD7n;FIeV%0Xk% z$(hiDq>56(yA2@cswRV$_bMnjm4a@TfK|%^;EG~CB3*!bSMy2rhB>*n9mcmLcK_Rvk>TA>>e0+O1Xr_2?mX&^nJnY#D)BOb}f3Zib~>L2%1= zGc1bWM_th30tJoyA_ef`!OT1bkC2c6(2BMk&|VZxNaUx0Hg%Sy=9TDhf!2-Y zrDlRwGTKAbg3ek^0TpYAy;P8GTcG6&&?Ttut|6eKK9fMLI7a?;F0ij6p znQ-f%>ncH|CwPMyXnqfrFkuU#1N?(SpzSC@@CKzF?9k;7f{+bLJK3Sj9l(4>A(ox& z(B%$b0Va@ulr)zL7aI#mfKiBL7dwlDk_gx<;2Rg=Zh>rL0;_^369&(sfd!aA`ozS! z%%SEm39;;EXAxFXS3vj$vIhecCg9^qKyd&%wQ`qxuz!Qj_!3^FU{tLsrLwmd1lmmrqSB0u>riz0jMlia}jOaOV_y zYyoKBJm|mzE2{v|dj28>@S%6$Bky2G*aYO~=YXcU@U3F8xV9`l$DipenDkOzJ{hIWJy6@YFTO#QZ zLN8YY9VvI2okN=IG-%LT2(;kmCOgv|c4nzvOyK!+(3AzN07kh#3mVFL3Xm((AbU#? zu?V`gyf_uS3lh93Em0vQzeKk<6>^0}3ivbxJq3l#;&}K0O`s`x@J52nG|)xSFx~Jy zRG@(d(DBKz>#Q_VQuFdb+q=M{4h5itV!?L_prirNzF@Eb@_`tddRz*wdg*#P)e6x{ zY5Dm|I*`40u{sLTN=b=DdP#{@N;=^4tH46JNJGn}oot zrEaowfr@c979r45Cn1(w>?}&)avscQ0SiE`oCOQ8f&`$en7{&zAOR6sgcfFyfPx}; zJR78fQHbR>JBu=00cd*UHam0$7FYo*NCmGTLIqfhkQ_n)bVmidq7ryUUI^6gyvxo6 zx%W^A+*-K{D{+Lt9mTus+`{1Dbs=zTdrMNmE96U3p~(}Pnsh*$q(Rq7fKxGOrW%x36-q#7NMu%}7IA^9*1Qt% z5_Zsb@I=s&5=Ed43Ej}5r{G$V2)>jeKM#C-FgQghK!>1`K=-tR_JM$AgM`5K=tG3p zKz*Hu?9lCALg0GzAv^Q{S|M;f`jA~!4H`(Gvz^$XYwU!;b;KitNuUv@M+lQZ!SIM( z2(s){2wX=zVpoSOfDi)J5s%rSx2Xz&3;4$flRz5*9 zK&vgEu?s=gJqv*kS9r#*qRMp}RBZ?gv3+7^Si#9C^_vMY)#M5~i4BxsK<5jjL-t=O zXn^+d7UYAPt(l3S8=)ZQ=fTeKDb@t_)**{qz&F(tC+8QWf-b58A8-OXaSXK9H?ahA zNs#<8IX3xIdbf@+QoF3A2*@Rnh)WzfxE#TofUCE#+dSfM;sy$p0&3HWxs z+{B8^Tu`SUv|&dVbjKQa4@630Qf5wONhPGon3I}V2HIQ!YCu;ifX@s7O)`V#oQsoF z^Ad|P^C3G>OF=u2;LEf?3wyxF)up6@wk3hKo#v&0+qFo?9e@w0fy~kAC}iXpq^6aE zLJBcv0M30GnI#H|NT;@Cq!y*>aT)nj=Z;OcRLGVoF7OFopp!U1v2%b*N){GjP&ei? zJJSz#$g)#ma3lXSJG%(D&=ZC<^1rY{mluQij6y75*rAJq!2(Pm0qCk*kN~3)%U5<5 zNFfO3Lj)j2FIa#Hq=H`nZU>VP%Qs{lpj`Y7RXORGxs~`oS zrBC14Ss+~oumBTCg&1gUHVca|6N3=jPj;qe4i<1>Dhw`af5LKyFu17w$qv0mRv28= z{$!Vg9O)+vE^2?WYiV=wbFqTV;s6<@)6BsP@c#K;+;iBX895k(UtGe`tk6C)Ey1X&Yk+JS?`8DTD{sBGep)I$;h zt1)l{ca(%#7=_qcIhZ;*VCe?57@(DdogWe;kX^QI9PnfUSsK~K0bN5b3{n8P1b_p& zh8!foD8$mv0o`p2=0gM^XC#0HK&$6Dpl2k21ekem93qg?Rv26>b#W+w7GQ%? z0V9VHTNj7YWOyoI1f_y*4tOeH1gC-?4i-r5FU-Wq0m^&5AOT1!VB`=JV(a5znZN-{ z1&rWS(9Z!+1&kb^R4@?{Hy{DfsrVf52nC5S39(G%Fmi+h1QR0%=+N3p94wB|=mQBb z3$aW>(F95blQ@j*Az=gB+%lDeX(l{EL3wa02fGY3Lcu#yrg5-nK8 z1ek z9-*Kjaw!M9C^UM&yH%EPuqZ$y6tY`o83%OpgD`Zr$}$c)dAJTnA(rKc2!-rcS&oQM z$ZnP8hzMl{mD?*2d72p<1uHn9=bnJnwFbJ`%1Q`fg zgAI}fEu2IZV1lXu`en3A3<(x+B}+z60?=XUcFuNBF=Bn1ooibI2<|Qv`V1+71qQJog6Ge5Pz_+fK$XyL=s_Pf~XJ>gp^B6EJAF%IhYQ^ zBNWuu-i?S*&;;sk4(PTsVeka%ZVqY4rbS_JTYEQ$o<1}}SvWw3861X3C<`b;_rSf( z!~%}cy@&{90Y~UQM1-<{B6L3o%RzXAvVbG>03t$Jz!7?ggT)70{DK6Sg;)+D;szwb z1QIdwf_RgKRfz2<2h&M-+<=<;M>*I*p#TauR#4m=Lqr8DxQ%j*13K6$%)$!Z5O|CO zy0;7@z$nCWoC7-43FbotAos_C1whAHaPaU#W0MsW@+S}l04q3dPH;eXKY?^WR7gre z9L>Zk#CD2<=?Xksyfy4)!a=uD?D_;k5b9Ac0@hcI|2>?sZvRcPF>a)1m| zy8@3JR#4oWM#K#(IBw3M2rvqUr4HDVg={E%N#6D2u+}7)@2k;pxk$v z!^i;=SS)NpY}YuLZoy+1)CIf7!4BELDa^tKis9=VETHgUX94pWg;=g53TQTP3}5F^ zR)NMB8>ouDfyhg2;M{iuk(b!Ox$g!CI4^-x5gRD}ZXz0CY~UEa$pO864`eAsg)pf7 z52|312904h4v=By zui-Jw28!W3a33yBEzzA2(t*W zJVX&-6=HdWVknys%VQ4c9!g=5wd_JHPf!fy5Mp@>au_5Xv4PUjGY*y)@N~omPDjrX z>4*)Sj$U%GctVo|NPt<0R=uQ*s-5Sl>g=oO;n&Bh@DN=L6a z44fe8h?zl%?JWmW11GfQ4IcV>%fTfL=~^%|fVy<=I9UF{+zJw56k>VD!KnbT1tb7U zZ0|YPbhre$I9Y@tB8);T?>X4ik%gE*LR#8fexM!*SQVoX%O?&tQ7Ievp>zdBsk*Qo zS4IX#COV*l*AznifF0_@a~YRFo#YS6eUXyrv}3TSl&=+O5HYlT|S;gX<@*q~9F zRPfM$ZZ7CNs1(pnckmfkpamD8vu?rL=RgY?5b+3VOZ??v<3aK<3+Py24t@bhbDNog zQ;6k1s4@XJ?U)%rOT7Pc@QOn60*KEDQoswDn-c~JFo6VwKrL2KS;Q{H*2u}!#mNCN zkO{QWzmXH(dt+x4Vrk^$Hi5~4I>MZ$elRYx5K9v$8#}ZtV&@QIYvSbS;$(*urcCUh zvZ$Gp1G0Han28-+7PWA4C_(ZS6FaC3XyxQ+=Y&?EAQ46(mNrf93=K`wT%0BBgHpOa-WCo9CItPGG| z!~{+@AtVt{@062G3P}Xi8RcY?MHs{=#5RSKWd^cApps!Kib0@2n8wKlD(N^N_Jaay zIwu>nhYhlxNr+`8C#*6D3owF2M4)9KSOly_$rRxpP)~3sr@1e}J0 z3{vI-M;psBP9`ZOG4MGc;Hf6iN~e<4VoX$h3kQ zxSW$oSVLZT4pk29RsLXL0WYH zKKB812Yd-=HVir-3_5r$1ay5hXki2RSTe{7KZw(!zz2Ck$2vh*%fKhQK$BT1;6dah z&@eCLw5sA%@Bw0wp;5>LWNAreeja#k5s42OaR#rmg-<}IPiV}1-m_l)CVo`Di z`0x_&6iRYVesL=3@TkO+Wbi$b&|SVd&|7^W^N-+wRme>(N>2rqq@bHHL5+cRoE%bI z8$dIJBA}XnGbhtNPG+h3pwI&ymk;(NXw0}$Av-U>9KJ~!;><+QOiy7cce(bFG@)*0*#NCfEKqv$G$U5pi@QQF?7)GSnyyzXftjm=s>dE(&7@( zNp+xe{lT}lC?tYL<3U%JLkt8}kWkk{rvXL4ZK2JaTuR_VT?8@>xP=osBnRfR2(fJ8 zgbpZy1=v6W&@J3x0Y;Dj^w0#b05eEHUV+Pxi;V>&0O}NTvKUI)!J0te1Pe>2kmcIR ziFqK$LwE9~q~?JpPr#`N;bc%77i=Er<}744fyN=XbMlIVYY4Cku$2-z2m#RjS)BR? zTmeuwffmScvS>>=;CGWcI7o?d6=*1J7qY8BgK4{vT?NYdyEr8v1J)pCf{M{yoSKl+ zu|&WX{vJ*i&`bk3;6N4C9!_Ze4w43q<)8{M39*0$KurXY3PuoL05X*T=3@webqEN8 zD?brt1}-6%y__tNLo+~J1`wZD99-RrfEvB~Ihi0=ql$p%0QSS$4fc>1@kU2dO zaQ}Kgr-}~5Wbhooeo%-vfu>tUKz={W$)qk-0-n0h&CJWpEzMPco|cfB0^gOdk(rmA zQwloNO%rlV4LA{$CxQ=(gWM(#Zoj!gP67Z;d8L6)+=VCuT@Maw>w#A^!E!S!wSy;B z4wdKga_W5D$PR%1U$53KilU z4?hK$3$iaBa+(2XdJKHc7TE1YsX3Y8!xb<*0ZM1b5uO00vf~I(fWrAW!V{nrb{yo1 zTZ~*xETTee=Q$a!b1_O~DY)h(Cl(Z!<|LM+g4Q5lJIVrn8gp@eX%Xly)yy2w*+2QY z(3_JIxfI+ob5aotLju6}EP~<+Ud!ll86{9>Rt~aCBr!(;>Kf2thYFc_Y59<8I#EU; zmh+q(Vq8svTr4c2prreolSxtPvw(u5f=@oQ=F9}2ew7AVNSCNkkeQkco`^>^Q6Uj} zQ!V&vcZgp>bB#qssl^5Pc`4x9ArZ9n2X@^#_yRES&E&8m8RV{_Tu5yV>LP$n>`Vbw zpe6aBVy?6pG_R>plCR+D;##Z&Ie|ViFD11iwWwG}AtgV#6n;iG><-9$kPK)Bxi~X7 zGbgbKbXi3{_!uv6BPX>Y5mdSAC=@4Fg4R}pQ&A$s3pttDso*pCOfJw$bg)mrmga-*d;_hh0Co64jStAh zL&XJ&c?t>&1&JjYVEqUJu|5Z6O?0A8rlx{Iw1SC_f`N{Lk&c3aLM&J@=oZEDjC@e? z2G^xX5en6qgr-qP!33liq%kKkDK$qy1Lk|s0v-j0=wzJ~3@xTGEuem>f(B$=5>$Pv zP8x=KLzsH-N+*r9{Cue5be#+g#l~>OrMXF|MG6{*pd%bX?UqzfFzG5dD!_sb+;%BS zO#__}13Hr#b_6%{%;?OL;?$fp1&vI-R6THsRFV%mXcQC@1&KwlL=R4`;AT`N==uZD zq0``X3Yo=_gTX+D)qxsb3hP1U>E^bZkpbDkM`t zS6b;o+EwsGo(MW!xg@hF6=V*$;hLh5l#`#Ft)NkmUtFA-lv4>hkrdQ21Em-Ep`&1> z;M1!SQCtjJo(3`roYX+ap_P`T79oPGJOgxh6F7Z=Lk(O4fKsFq+`M8X(9)|^&|*H2 zL6H0bTKok%7bp?bnn_8m(Bo3jRq%x!fC`HZNK+DQHONfR;jG}a3@b|@r)w8OYC}*X zIxjs3>~CoJDWs$(=Oh-v3!CBs(7~$UJ#2}|Mft_W3UKci0>p1AO3-ixUz($!s|#we7o~zzP*N(~WRRc1VF8T>P?Z8w zsQ~s3cp-}peA!BQMrLvbWJw?vUxUI3kyO=-6~NgWcKmEkc_Qd?D~00HB+zwF`DqH7 zCB>k!I5iJcP=VA!(h9^y&@vGeu2BD^rYeD2`k(|<0b8wy9Dtys%t324A#HS!DM_j6 znR%eofk66@OoE3DXcbGbLUBeSsLh^K3F#)l3VQr~1RaYEb}zWMg2=4Npxa$Qh9yD9 z{E9$3n4q_^rGZyGLED?6;0E(=SW`_D++hCA30*EB3T`m}<`jXPtSJg^F#qNRPZhGT zh=TgEe>j=Wb3sO}MZrC_Kb%}bTwUy3Y#@113ug)!i;mP!HU&>avj&p6K;<+nx#DW0 zflEM0FB8;w=7NVL=rC&6#N-SGXn_qX5n#yxa+5(!ka6@eCgfg&so+N1!Tt`1r?t;dDhz5%%vTt0%^9N@~_ z7+jf~>nIp1#BwRXi)gR{(3xd=&~{HeDEVm^Ybq${>w_8r#-L~fH30O$$x*{x6T|^y zeSJ{-z#OgsR0C@uq`@W_A_jnA*%(V$qgbv0I@=j+8FUYwf(h7_@ERDsF^%kU1zQCZ z(DB}RDM+iV!EJ7c`!NkSLNy$Giz2!W5Z6F1I|SuNY>Md4mLe!=mqO#Ngj}C!~ba)xh1#${V1k$R5rqYtsqTI|p$h{TCskxcS z`8oM{pym~5NJ11eY{DYMGKC8|Oac~Q1qtv%rtQH3j35DF$QT7!fEgqp1vxqsEWiO0 zkkJMYvqA)zKq9hQT$h-jd7e>-Wi}|!A7p~vn+Qr!snB)Bpd-~&L3uMRzqAOJdyB0U zic1oUN)&Q3L1zF$*va`hrMY=JTnedqDPRdi`&9>0%amsprz#{C7nkOyra)@(#9}0a zGdMl%KTDDGol=N8E;}r3KwLC2E61II&zkhSzM5l2)m!ecii#$0WH04xM3@t1{*$?VzP*z5qABfz-fz%L?b`Gcu3OeaAv7{t1IRjiz zf^!U#dQffPr~ta-G8Ix2X6Av~<{0+o=Yd+>#i_}q;FG375n8MR?jI=RroCO9OZcN4&J zlOAMC323Vb*lZopr4t|?NKaCwLTYY7NhLT(fjtlL3~0SS)P9&bkl{^G3j%WO6ZCw4 zkl+0CK!-GEzylc+te`dHplZ0F2y(MK=)MJT3@Ct4f-EYAZrIV+hb|%q^(_kuK=)ad z!rFb{mM1I_i*-PEDucohG{8}kT3i6SQ39N3@)Qz_OHzw6i?cz?$00s|8;4n9f(lG< zZUe<2B%VQ88g%X`=(^aV;u1YB1;5mCkTfJagGcy~T21+-B?YCRS#Hn<2~ZcL*oq5O z@xm(>@crfru#pLH-RlfV_zK7gH!VM352R0DUjdsmXd?_r8qL_G#3IO5zbJAb59g&T zBqdfU*eV!2)sCfpuOCdcKG+_f8k}3u#cxXlfRT?EliJ9PAETBmh z+=|!N2Ne+@)AK=%^rCdoom=2Lm_WG$R16iRDkSA3=4C??y#jO_6{sZ2EP*xz!Ttty zEZ~8F+j)p=t6&6C4)-cFPLlIMuCP@wg~W=Mf)lvo4|O#>MSwz5K?~%1um(K^1s_PE z0Z-mw4=QLust|B%9&|ofik^a&K1j1ZE|bAGnkXnJXep!@fo6wb8sH9vq*sWff~|rX zJXRq6574dGkYWoKJD>?ISak#%-#vp|9f5Z2oZ*6QyaKfgKx4dTxI`c`?_dFDP}QOi zIvE?%HDDHEIm^WYnPvwm04?e~i)bg`z-DM@f@g+60xUu- z=eSsmz*8zQHuYj|DVIdydP%kV}J$nMIUUi0u*= z(={$uDMlDNjj3~5_|W}ZQYT;^gCMG|2aV!6V_VumEbB*b!+i^YlS z9cc7iOo;6c7sFpJM$iU!NGm!Cd|wc#6wd(O4^1zja62JS}R=MsY)k|YN1M&IWGulZwP5d(FjA8;}K zfp(+Cz}@HvTwEet{h(ux#XwDWA(qFWCUre>FB;SXfc7+CH8oniAA@x?^gx{==;6&^2eE)%1j$%n0alOzv^@$IU<3(3+oNCs z(8+*c?lyzY*C_+_lEAknX%y#!Lk)5% zC^)}nq(aUjw*~j>bQQooVpta!}J zlIB1wEkARy{NiGjickoG-9HH#3c!dQ^omyAGuI+BIM)>G0@UWMj@8Z zT%3YjZd@Fo1P|Go_mzu{U&;g&bVaG4ay2g>Zc7n3NFb+>fTTbZ9H4#iput4YEyAF+ zdtigXMb1|)Hpt3Is1RrwBo{9qmkr!DMj@8(Tx@(&S|HnCWgf_2Q2Na;f;5%E-R#^_ z@DX*;rIru_L4D5eTx^`+^?^_!&=x%|9$qdBxE)MFEI+x}1fS#PXX9x?u^-hX^1_CeUVr-(1ioiC`6s zAT45$^<`iIW{`lQ64wnTF3^H@wia%N4sJ%NlT6StELat)UIt(Kst!t!i3*T~qmb$j zH1(RA47!RI97&M1K$*Fq)tp=kkitq&A-FO(DL)6i-V<_}ZBi;|dntIuCuoQf(j`II zqfk_u1G+XKAKVfyC`v6$%`4#o4V-JFR#<_DLvx_602>NgSb$_CXvQlH?r4SF{1njM z?;@~Li2cQ>C7{d56cV5nc>?%S*96d3-2_n2wj?995=EGFX4hOi_=djh0IhzDtaM)h&+UsRGOw_#l_(QRU#z8#a^76 z44QJ6U{qk#U=D(Dg+#bm^Ahv&izS#87&TafARMP=MlRvx{9HXWGo5&0CV~e`0~n(W zp(@hzL7T7j@{7_D4g^PmK6tSh+_42oKNuMoF)_ObIIUumgB+oz2R1CJG)*r&H7Ca# zw74t;bWyL@O=Zv(EpiM+j9h1UxEOhanB^FRj1Ka^&o+Y{Y^I=#IE@W_`!{%@Vsb`e zUOM>HwOr6el%O*-iZatPN)(I?3@o4;6g=~iL0hbIaumRFpgay)pRC8F09&1yl3JFU zlV6ZpR16Cc&_YFU)=AFK(S=;GU98W=r4W>wk_qZuC4pD;ffFg{_FT|PGVlNuSOjtb z7pNbZ3m)1i$pCd4p;Me((BuzUW}yR4)R1d$L5(2Ts0FxY183kg_*6b9dcjL4z+(fT z#fl}6-an{+h3YWyI1aSE3SRvIUh-Cy3Yz5wn+h3YPR=i=1cwTED}81%_~H}LkTnfl>3m&Neg{&Xs#4L~s z@HqgWA{UabA!#cwA5tBH2J^w;4qgrk>P!@6CY6@t7eQvNkrs%i=B4Bp6@!PL3ySh{ z^Gi|{AYoDhx|yXY6MO(+QGPC{Z3eolF+Z)OJh3PhT)u-Q$w8-~WTs^%D-;xE<`*fH z7iE@|q~Z0HjsSx7fs^IJ& z5EurJ&>(tl$}}1eS61i&Sup2nccw z4p#6FQtyq$1@}nY@wTHh#$x@H~%0m1xJMd$Dj~T=TINV zAccU?paB11R|SyITs(uFeH=Y~U0w7PJpB~>{1sfoT>U~6f;}94e84usJ<6rvAMWQG z1advHHx-;*6?{A$oqSvs-28(SK>l>`403f20eJ|)a`tp_^$T(I(cw}E4sdn$bo9|t zaE);F4e)UciqwG`8tfVv3c9k-(MQ3>(bv)4HCRD|P^bh1`8$UOx%z@!?eC@#9O@Jt z;u#Ve;;P{8@9zQ*>|ocRFi&UKU~2^*e^BtdDTD^Q>ToHzIEFZatqutCck>Jhwgx%F zDKyv<9B`g~A+A9|p#h*9i!>EH{KH+tT!R#x9Ycd%UBDsi?+5ZN7bukcgCark015|i z4Cp9?d$@*pxCVg&6&zZQpa22gitZc&4lbxN{~(1B|DX^qB!4RSxw`v!y1V*0yMo-~ z4>Bg)GuTyA!7<1)*wfD)WF{z>!yO|P{6j-P7K2h`Xs|06gpHJRb-?kd;OVB|=o01$ za=E`7$iWH${=vbX(BuLRHfIlLnCo$Iad0s*2{9`$NPt)zLd;4G@?4x;OdRqMCI=TI zhY+(eg9liU8^VMLsxat)1sR2yK}?9CDuW(akVA+W#DoZ{F(`orxrCTOOo*U5gCbZ^ zMTi;1ga~RdSc3&sg_uFixdL2Fpxwu*3`LAm6F{r(V6`slyawd#4;=;YW%aty9teEc zATt*9h2QIsz%>i(Qlvb3O3|<5VD!bqkpkqz)@*!MMC#pC< z2VAS^DR`EE8sDH+eF12f4#YsvZH38+IiLj+MPO5tK{o*uloTs~7R@Orr6g7=fsYte z%FWL!$xwnB1L<0(fLhxjnYpROC5gEOkdXq&!V&lgOEPG{v;;ID#HCPDp09vV4L)(G z804lz1$a`-L>Q!}5Cm>9fL41H6cpuGWP+-L zoJs|4T|)yM0|NtvO2|Aqmx3~Ugq@2E##Au00u7fJl$N9-oB_52#kt^mub?=!G$kKA zo&_q6K@}vzKj1F9f~|s{o*sB)RsrEeaGw%c2JT;2a}cHvH1MjR029@N#AC4nxK|42 z=|UzVVSK1;PGfF9wYfF$fQ9a0I3#mZX+s=BDa_h?LZvl0;Au6;N4{k)H?4 zza=ROwhEXEHBu5Qi*1c{6mm23N=s6UZ4C`HA+Z9>G77fvL;E zQ6lKDS5Rjn4|4ZxQdVkm3Fz+lROl)qP-iAD9TbG%Vhhy0E-p<=%dAjPDppd^$jk%Z z6aX4dga*APsO$$1-GZ(VOa)C&LXTvGEqDT5UkvIw!1_MmsDX4&LEChTQj3d0YpII4 z6kzodXdyI2fgb7#CSwJtt>Ct=f`J0ES;ZNk^<@g6NqLyxGf@HxG;p9~T&x80FK9tN z!Zc9bVTfcHbU_?wmK({5*i6%dB`-sexgc}UOyyF@%>=C_1^WkTE@+xQwE|RCgM5zL zSi@o^JuVPn1|62goXR1|Mf|&s*a`baE^Axl|oZQSj{fr_7Ed?VJg^VKol*CE}Ed^sUQ#~V71yB~z z2i2_#T97gxQqh7Ytyn>$jjSN=v2*cofq9HV%$XdZEvR4~vk-HpkT6#>=s0cAwX)2G zOhQs$nP33{OGDtU7ia-BDCvP#;3JX>@}4`S@B&Y&fbzQ@JTnz5ZfJ-3(8V?E31>v9sk*H9R54}V@zepi3H9Zl$%p(EpAgDq}I){}4 z#bBp^#-bs{f!Z;kMU|k1r+N8#x}a4(keeq!g9@OiQ9v;b(z6C-1eo^{b5j*S%|}oc z0Oe^kccLc|OG`@~D25~tC*BV3bgxfCE{Nr(X?@H9|-ZenpZWK;`tj7yP=6THgKH?cSyG?WFJ04kPXRAAQN z3}OXyoLX@VW%0o^f`^|17^93J_UjQE%K|$Ki-^z}%2ioDVroK2agLw74Wc7t)^ucbhjo+MDK3miBB zMfsqPV`hF{F&CFYct&a-#81%8JfO*r)CzFiCW59iplU(QSI~5HeqM1Z{Dy%1A}-Kg zEnQexgHL71DFL720PRRR26%!-3&GR8Afpq()_|5iLjx-!bi9g6|5ADj39|qK_Lk-^rfJXqF|+92$NAzr~s*iacg0WN{}Mx?i#3Im6d`8 z7idFBD#$IRIVF%~m!LUySZKg!fS@Czpp}5ou&L0g1ohArs<;$1ARETOY9MAnj!p*! zE+{OZ8|EP?JRjO-1C3gNH>ZQvrGfAFD=00|wBiCUmnF;VP;Y@o(i}nM0c5#4*twv< z2l)$p-7@HCgNo$Tf)Y^g4dg!XavVrsDnAc2be+frUS|c`T>;vG4a#bspi2h8`3;m< zp{tc(#)5j>sd*)!xJS!F;4u^~M1}$F3dn(Om`lvbfs{Ot1eL(Br_MDPYW&=jXaUTSh`adBc%W=Pl#w zfO=mAx;dcvXi(&%Bt$*XI1u>!t$dKDgIt3{PzwgAr@&5w*au12#R@5zMXBKP^r7p! zV7U^q_%SmNa;z8JST2Pe@C-1l!~q)&F)=f*xFj_(1?*^Oc!8I+g9eJqGE+(ub6`0g zJ{wyEE6R#ep;OWz)3NymS~fwhIs#po0Uo^q@4kahe1q0$LT6I-K;;m06bqcgVV(u= zYXEhIlXFrNi*hRA)4||P8|kTFL&1~2pcR!!`!_)81eER^13W?7s`5a)Y%0M)0|@}= zf+z4sj8w#pA24s}!x-S1ilWrQ(##^zEJRXjMq(Lgq85}(9RoZ;3sEqO*ic9j?hgxd zP(ka6oOUuXi+RwF-_&H#S{l%vXV9)saNL83Lco<%Nn%kt=-v=eT7s@mC`wgG&j*$H zkOec4nj;0YE;CgDRPbhkR!G2{4!X4tw8sLv(gGAX;L-$U1H^sBpiu^R`iF%gC{92j zm;%~90lrKcx+}L>p){`~zZA5IMo$5pe_;U!%C`lH;Nd|~k)O{6+LfAIl$iwC@d$M* zWF!Q-krwP^@Lp9=9hwT20~Z!KIq>=&RK_HO&ZdFXm>_?EoChyJ;F}7;;RkXC!aJZ? z!&1e9jsYx(SqDmT;K4)CZkasLm?$)(rB-Bu57NRY_aG~*An^l_9mv=lXt=h7OCb?M zPcdYz6s)iW1yM$-0%j?xkN|EbB|t}k;l3;`O+v1A5njY)5-6M?Ap#05a0@Llj|-!< z0nQzH3ZUcrz^MaX*MN_p0v%O=9vVn4ffp0#VZnvAn-#Pd5+&52EeKd+F%41oqE^GG zwXl@}ytal`ObQCgRtnLEv5+bmTutSrC?KMo1bYl1W+;G8JOmB%LyR^;GP(rRcFIfx zw^)#ZHWSje0LPmZ=sFv{BqbeC!#7zemJ8H#PK9{AD76^gAV^9j#_Mn=DJWFn^t+A% zvKt-qK#Ku$z;{i67wd!1Ferd*sDj?{0zTLQl%t9=Q$V>h6SQKv%Rt%Q>)(kit7TCll0MgCEfVT3QGW zIgsU`Y8!GNDyTG1%go6ExBWpLfrNBnDd=cCP*Dt;Zvm}~fUSbfEP}-lxWWN%CD+D-(ID06-TBrGG3ZQE#ku!5~Vo7FkS|!8<#i_Z8c_o?2 z#as#+Acf$SiQxVwv~dZp>p&OkKr6dqaI0P!(rtFlOU{RGQh;;}6hJ)#P=n1AGXDrl zMo4bWECwChlMiYZfY<0iPwjzDy_SPcLxlEg5<$oGK=%bgnkwMk&AEws5Q~Z-D-bjD zASc9tstfP}(&YTSvQ*GkU=nhQ0!S2E3&ZRIYlU~Op2q?-2xi|q_v*m!#>?&3$E=|sWRjJ^j7w!yDKo?snD1p4BhwK>W zuoUD-EszSZ@1UYkH-M(@K}BCG3Htq^tx@sV0IB=gY}2huIFd7^E1@UcbcL zR2_6{P)BS)b)bJTY2tbJCGb59XlpJ^y z5U5zsPlKdEaE%HnA)z}R8NfRoKQlsiIx>K-U;fMpz4ea)yzS^SqY!vk2gqn=AeAKV zy|^U{nXPVNWawvPlsYY|;E1&N9&rjT=zPBXOvv03sD@R*vKs;{UIkjVmtO?g1IwiV z+aUou^b54}4!Rzflx-PY$O}_J14yuK8KCAZ#2|3@0d$ZCXj~n1xi!2=Ck*i3X08W(q*dN5Mj%)ul?sxv8L2L6sE18}1W9hfIMc4hlf0MuBQyg%VKH5!8^( zO9ihw293*uw}FDrwgfkVbrecLU2BCzg=ixa-3-vlZlI%-KnFmBO^?;Uzu^-UpZZ{x z;NCtcxQk%>MZkS(0|Ns?T`&qUFt7rnD9|8?rKP2%uAz~xv0;djv6ZQ%m8qp3NDPdl z^c0*bL9J|1Uo{zYs|H95xQzieT?ce}Uus@)DR?d`u>`~g^<0u6eR_~$P=5~WfaoAM zXD$U}V`EE%U&_nN^)gdS(!hIji_((yi_(%oD)dS!N;JXi)uH_>=)99F?-f8&Wr;ZoC1p?> zQIxBdl<6pd9R#wHEH9Pmm6U-zh|fnQU`3_HsmaiF&nP}b+Khk*P;fMdXXY6jDI~b3 zmIQ-O7Xe8-7K2XA1hJ4j>*nd>8sh2es*qL!^(HPoYH1}fmxg-!g_xNrlw>AngSHE4 zXlOu0G}B7-Qp!Ct(=%KWOHx5LX)4&*C>R@QD%2=|<$UtXVN$Sg1D(L75T2Qrl3!j7 zHWHLW4GmCM>g&N3KuRe?GjJZ#H84bEBgn*wzCKvHp_z$^iMg?v0eFcyxQu|F)fbwV zSphBUK!q84W`u6q1-TuZD#6JB%z_3FC|Q8{DCtuJ93q+u8XAy5(T6NjH!v{JRM1s0 zL^y{)3I>@23NDyAsz~O57rukiKVmzX31T-2!o}bO;FVaGsE`1Od_Cy7dLSjfnK?O` z#YhPNRGKQ}f<+Z<6>w{SMHk2ru>TPuqhO1u0QK@x%bh@%^Q0DOXo7+$7`%<4SOaPW zBw(Q1TR_*98HTA&P{%D}&0aF4&;F$-SPbtyM z%P&XrHB?j~FTY&DRsqU`sKM(|NFoEd8y*fgeVmtHu9u!#;tHx)A)%n5iR5vRAqdYy z6@x-TLldh<&2V}Y8ol5M0SyqLZ90N)nM8^v*e)VOYIaX8aV{+?O3f<)`Amc8T|^+& z(DYDQ06yjgye161KL@l-4RX@~Y@`TO{g71qKsGFa78ihy*@jGTfQAr37e0V;x>7Z$ z)ljQj4c;h+<^8wjho0tP@s(>~=fe&s6RpaUT3Z(_0O59Qb z>JZ3n0T4lyO-P8go~8~L*nZeKc@UeFqLdV%o3lX6ZITl~r#gX}a-ehoZii|pg@!mQ zXjf)?n2TXdlD z`DF02FVMM%3LrZ{dLidS!u$YTiiPA3&;)&c8t8;8aAOVSxOOEY14DCA-C$@MVqj=x zY+z+*s%Kyrr39UaDOLdU;SEs82~c2_VEgm)5UplY9R-kSpjIttb|v!Fx= ze5M>IhG7l0L%N-i0o#f|yk#12W1;6*v05-laMD1}QQx+LE-*gqJY$uu+( zO*;5_Bzn2|Rhc*W_6>^IgVyX#p z9}gs#LmJ;)3egG0B}Hi^?1QIf$0PspHYo7}__@Ul+@b=62+)m&ZGln_-9D^*`B)nGlC``v6Tk&Fc;*H?lMS&7v~du0(;H}1y#+L?J{Pugkfno> zNkpoTQ32^3F=)#iWH>0pflLJ*K%JbbfM{WHf%}V~Y=qoU(?i`|h@(ryr2y*tp!JEO zAQKAUU51b&3Lyjc9gIxupo5V>BlV0zEFFv-{4g%)ic3ZzVXmhT&oBzH^f59CNnK-9 z@XOEBh3~BcRSw{JO^}BOZUO{ddJ5T;2P?55%R``(@(9Dg)566ipsTXL*KdIK;=%0% zcUqttK`AS-L;=z{Ps=X?b&5fCIVfDw0}K@1NZac`1EW7_n@zTlI~HOFF4EO}{KLCli#!;0)+g1`p^ACYYoY#&$f&GLwRm{33{Y zDFR1QVVVG*CqmlLCq-y>MJa%bC8#tfRfvy^DKiCpb`@t3Q)UWiUIjYY2R^upGl(6^ z6_VrPf>!gH`FRq|3XB?@K}aG>3mLh1pmT;GlZ%B^x%iS(bK;RSv4Zyx@+PO|xI!8X znfZA_DqPITsW}qN3d|bZMnZbXL8S*WCMCWJ*`O%R)kAvUpN(-e$q6GSPO)uu(Pw5NWXZ)*46~3MtWg@( z!eBHT;mQ$O%}gYCaBD?a3e{?6BBaa3l9ZpHBf$&SCXQ;OQ+|F9%zt2I;E)8j;3W9K z8qgvd(z=6bVFL{U=B0zuQxfPRCX_K>mjzG#wSDTnu333u_!@GniSOc1qU`KMmbaB9x3)ymUfy+j4)gmDS)+~eSGH~Gr zT89PG%?Z{45)rcD;sn*i;C7RQ7&hIY#u!){t{bkwiH(Z|v}(eOg+oF>(u;*dNb0qVu-zZ=_RR}3TkQ!C_-Qn4UkLqGK>8Z^E5OSYHA=I3p-oL!J~RP zC7LjoCZ!^s(SYn=u-Ei*N}vw(U%a`!^b#F-+(5&a>o_Bq zC?w*+?vGEcfHuAH`osom4i1k%6L(IDf{iUC-KwcUXk?!trw+961|9c7s!z@`a`7Pf zBvon;N>o4+0W2<3iHr=WKXXb_DT@sJX&yOZV5y($IwO}DvVV}G2EWgUjhs{k8%U&3 z6gi~(%!4tADZNB!AqN+8dPyp{G05c9!*R%?gM(`-yc|j|NtJ3xE{CAGiTIoijXO{g zi&Rviq%=Z~heyU4q{Nj9%ICXL5*N74fu$|vq?JlyMkm?LL}dC@%v^sBDQ$tggOs-L zc?^-Vu=@*7zDM#H&U}v=8-&XMEFQbV$i)lGG$4rsjU zs2K6d3oWWpeT!&jK$1~<36k&N$?zE?mpF>|kRlGhM~RL)%u)f#qhv)L86Ndu3<8x6 zLbY5>nRyay3d|ZzPPvCXvbZu}?WoK=sTf6Y4GJ1n!f2Xh=BaBU+H#LTffl!5zS044E^egp2ccG!7CWO;{vnSXt}HGd z$oK{Qn(XrsO?F17ZpK3%?TlP4j9i>hcMN=+eLYf}ozZC);~|e}j9il$xwzmi9BhsD zn}|j`qth|=Lmr3Nx!MsyI>4>=eT?u#25$D^ZS2BZwzF@0j%v3JoPtHHwkAYpju40&c&Z@AxJV{+Uy$;@dXM~N$!rD}HX}?#XwBH$>5)XOAamB!^8%WceCJp!s z(9~sKW(lkT5A8X?DmqAOnpe1ikNh((7pccC5V+c7V zuz>>)h9K}z`d>EqP&%vAE4D)(&)B$1U?G)WB9(#U4&38y;E+NZ$whJ+w7Z8Zun^Ozr%J6saV{kF!Q6#?R2=*8 z7db(VKe=KZ1wYEjC5G&Nq<~+A*C#lF9&>n$IFEq#Ga(K_FH(R^4`Q9c!sZX?nD;4m z#F#g$(^mFF9vj)YniyfF1E}F#N8z%(<_8;Axb*jmtp?GR~T!rY5g zQVxj*Cd}PKwSftDKT^OC@dhR{S_2d8#Z;+*ZD2wi0m_g=vVjSAK2kCnf(=Z#H-<+8 z6X6}CXcFOT)XsRwqnVMbfsu;~?w$c{ zWi}xe7w|g0XFKHanvJUz9$0j3X0Bu8;(-PVcwlhQ4`gm*n zQb6uUN-0CBwRx72iw~aSLH-!x&CRQfToS1ML5i56-`+$_*Ym>0*XZr|Ws) z)Ai6Gr(uf|Y0?}tq{$1PG>2MEr6wnAP!o0CD+@}Klb)lRNKH-_lqM&5Ad$ZF=EzM> z7W5`3*gb=#$;pD&oI`YjU!nG&!j=%87K&3EC*9AiT*5I>*G=Xh23e1>sFjs3Qh) zlv5D7$q9GQU}Zm=tEn)`32Jg;ALSH6X>tzGD5nrglM_6uIB=t!Lg-CSuzLnelT!$- z$qDX&(s`56Yc?|pdaE*P9gLrC)kez-sFV1 z0+cIIa%fyY8^C?`s>I~5%A$mPoB;)R}>Ouuo?F2rh1cBhI%9;IBxpc8w+r$N(w%g|h; z<(%wJ9gK%OS{S(+K_{0&j;GekDH+gaXFFm&C%e-ZwnH8t*tp6N!9~}h&h?-Z1fjt) zG*)zO2OSEC>|&&{b8xpj_cLSTA?&wj{b zH#_JkJZ^+12dweA4<)sLmJ<%KWu3=R5(~%!NQq@AH9yZWa`7W2hW1sLgDA;1qFw0;c{DikbVJ=20K8Hvf6z1}w+XjXE04eZ?cpH=rqYVo7 zv@Ul=5-V$)a4mC^x?ewWAD#@;enC^2p=L;o^hc9!I~C&~C&?D8EzX zA&)Yy5-x7IGw8kMa~{%AD8ExD;~|e$M$id-JV-7Y&~|7KVl0&3=_lJEkFRW8X@HNk*J6o=T5=yBBa0`dV;dKpUX(esR40!S$z zEHY?KbsGNH@3Xcfx1D8EzkA&&&EIHYQTh6AJK zr~{+?PO^tQq_`xIk}-uNqli{1Y-kj4Dy>qO(}!@Y6z&D2;2+|x zQUT0XDcF|-Hb#nQl|tMB%AZ4PkQC7>h5G<0y$sD(DcmQ+rd5jY5mMv~t5&H1TC0>^ zBc-UVQUSD9DQ$;J(ORVfXsuGp$4W0Ua&bE4=jQ~K=A;%2HKUA^a=^}6U;{ge~xmK6=HORjC@9nhJ`r8(DHv^WgXBCZ#H<7AquYBo-wm zm!uXIBOC&~vm-SRW;4h_p^1!K?75kF@gU2Yz?MT7;qpN(he}E%VmbhJ`237Bqz-RY4FQG({C z+2J=g7rzTfZf%=c*vuctC|buFu1%_I!=cn zMWBCS}F~v`;em* z>;yFT!6Ftchv7V!1qkOQRzSiJl++3mOG;9UK-Xt8YH-3Xt$-ypsH9Z9BB&J5%S|jv z&PXkW-T?`f1Q!4)`KiSUpj)iLN)(Dx(^D%Hpb8XFiv&ncgv!C(#17?ymgch+6s4wR zR!FcbfLxS!$RnF8lZzLc&LDD95fBFzmn0UI6o+S)WWd}6){5$+jKs231&BWEP66wI zxdduE=t>c`;?gvbLpZ<=$vNba$(0Uw2t-aQ4C0W~ycA?-KqRp^10sRd84xMBGay`{ zDU4hk$@zIDiJ5uD5}aU%Kyw~HG(=#sQV9^J!1&M*fpJkC0~J*$E=__MgVjk;RmCtz z!CeB%c|4#&CzuKeaj?^>4|!B@m2n9}%mknQ4O1Z%4^BcbF4T3%W`JD?lPBOZWPLE# zap&ciI75R<2y#j#mjbf}Ct@=Vq+X-TkR9U4Aa1Z5V3$fFg-m81O)iyuz{tg#nwyxJ zBf+D9~Vo$-=RzY1|0m{?MEOrIiqoK)_!X<=kczkMZK}n^MQCVV6X{thIu>zQ{ zqfnlanVbRQgGvhQ_DW&3R}o~d2SX6VV?wVPxi~WNi%aqnb5kXF!M;4hc*x@fBUd~Z zEL*_jg(T2D3X_pCg4zWWfO!nVn;uZRG&H$VxkRBph3SHNR+lKx!ZbOL)ESutLQ?2H^bCNGQW){GchXd3BP>nDL;3=qJ zO7t>|!TGz0O9Yx7pk_b=+lHvX#^o*<9PWZe0xTeeW-xKFl$K_uNQi+$q><^6$6rRS z`Hb+)50VrTK@X46P)`@BDyStOB`|NH`5fHpOfA+>kBc_YwM^7abJTT8eI{+G*Toqh$xa@=mh#pZwd>DqoU5qmTO*FYCF>)!ugAl_AXdt=}8Ho6u3aezWgc01Su)q}r zDZ(1OW@tesA%i=3(bZxNUU&sdAb8OYgS#1L@S33qFS-$=1}|QxA_XtHJ#eQY2d^1w z@O}ebS6l$i+!C_jK)uFz$m0?tR~o#|&x9$!m!r+$NfK%r+)%s>zkXB?eE5 zP_57qA~hGw;dBBl1UNDaAURj4lZlHPl-i&+O2~n|{*>{M$3sT0e0V8~q(n#>Yc7S9 zi&8dlyUR?VW)cnn6CDLe7~~<9qHz7tP%t1q8$#WpfX6Mckl+IO3^gpwpzfBC#~Btd zC0N743|dGbhXqUn;jl2n8Wu49q=p6DEl6PjGYjq(5ch%T5W_||%m=?VpjnMLs8 z9HbgA)Xm7s&dn>(Eh*Ejimf&>MQ6;#$uLA8JEZB&3U; zX3|P?aw>HTOA~W4(=t<26at_|Du9xZR3!Hrsfv${hB6FvEn~GbVq)}Q z0;n>YTA=oH6KEwTq`iV{8#L^kh)PiSJp=2jAsY|(3_NMUWZeq$gcdS$3BdUADfuPf za8?6{^B1N=9v_&v&N6a|L-PT;LLmo|!Wz;jNCZs}rDW!%E2QL?NEM(5KhzCySEJ@r zL|Vle)`VhVE+dyL4*x)7B9!u&z~)s|LSBVM5Fad@UGhsnakPe&OQ1Y8Cnr8TFTXr5 z9z4GwA)&yi!Dl3)msFYts?jr35=&C`yg>?rK=ThyPgxFmJY?Z|#mJ=$^)0#)dKsyS zDXB&Apll$dfF6?`At3<@5EYP&vVajXD2i?jDAO7eh%Uv9MA*a$$T7(!njj4j^B@8W z_Uds~_MoySQMbxbH%d`iMO7_ET}xY6FD|~OE~diFAXZyNUBOB{E;=rz)WE>N08Bc9 zNf(fuCa6Q82{JA(T@T%r$ex8pvnf&0j1dl+5ND%XfEoK05X*i z8kJ{piaX>H%@xVT32^{uCf^4<$q8};)@flB%W+Q#!|Vi4;&+24%uy!siw=3@bLDdJ zLaYVN&w{Lu!L%BA;umcl0a4SwIf(THu<83QkZT~*=n|0Wd+4S*m}?+%Qqh>ML7wKt zbPd5tUKGbbChjLQa=~U+!PDcMutm&J$3SOYrIHmv$u%z>I$;m1Vj$TZst7zi2hOBm zX$t0en;E&Fv#j7Ladn03P3gr5UKzLXC`EunAr^1x5`X zr_4hhXP**_aU!_76!D*Gm`CUvmK#YUC0m23Czl6`JN`U6O;bFlC2@BYCD9jx& zK6LsN#s$X>d=4Hes(|SVs0@|~-KC&OraTGI*(W?sQy33L;y1Rcr9K;<{dGeqG+pN;K2|CnyGsNo2lal`}Qp3A&--w zxm7MmD1&g5fgPte!=Qz(7+JF0%!|`xDjX+yXEoN4fP~s-tH;H zn`Y37@^g%bJWerkC2%3UNod;61Ys3?4vl~}4b4#eNx=jjl0PvlNAjnc&{a_188rC< znRwmDc*tWHBUc2>pP;!1EL~^N)C1GEQR?C zq#NB*n7=@BLMDjykbDZv8a(hD;32k?+L(I9%*E-NSDFiHOQA0LWQ8yJOwB9Jg*K(4 zKve@#O)1FINL8fuh!}+_cp;>^CakQ3wx$%hcv4c+GV@YX;`8%zDkVUR6gh&pp<@1d zIh8^+;O%M53d|a;PS7<<>|jUIWt~eIygX(ByCD9MM+{dK7dO}iG%Svh*21!aErzXy z1zSuo>Xjz2aPbDE7NjPYq^3Y3d;w^!US?i=W=U#pu>>P{F<}$qA&+{lS}vg?s8T&p zV-l)JuX&|bBn>vU2gQj0k_Q597uErL6m86a=c&{XlQ6?U{YGc%*ElG zSO7^4GeJR}nHLWlMunw@nnNCyT;*K6xrqhfbO4rvF5ASJ4#0}QL7Z3s@gHc-6?}nk zib85$Nl|7hQqh1&24KC2(9O&XNX#rM7McNaTVe&&ZIEP84Ow{yaT`odDiwe9gU!d{ zHb_?kx6@$eBb=640dblR7fW_(CE?Hqs}eHcVgvW=i^2T>e6a;pha6poTpW&hm5|ut z1DAoIMe&?LpjM(Nyi@{}i=08MdHE%tc}j1@xOiPkixNvR^Yb7Y7r}eXpkU-SlGIDj z&ri=u)q{;c>A}>)R>$+DKnfL3DTbt%$` zDoBnU(7mh6At$P^Dllr`bvoj(6}Zdke%K0<&7f^_Lt$Ghs*{nnVh#Rnt*A~PlG|F* zynqz^LvUN`7wBOtpivvVSrzFV7Nh_e&>gSH?f_-op||4|%?C(HX{haZMf1rp*zt<) zBc#X~`a51{g9hO8B-j<0@mJ8$%RIQ@fkBsk@Ki>4lM^(Cinm;X9p3_XAr1S)DI@sJsQW))RxCWUy?GS?yL5Zu{tI>va&;}9d)e(-f5sW~YSH)iHZt!0Ev z9Dps;OU%j9D9}+T0F8NLjQoIxxIyDwSS9pQD>92qiZ#-76bkf^=0ei6)%DahH4y{q zpy43}Sl1bBE66guz64ne^Bkzx^aYWI!QCdOJB)`sZZL9PW#ke>iUDY9-H*+OiW=Dc z2+^Ysp7X}=Crpx%UlE=K?H_{qH8HOew9_!7n4o`Q_Q3oM=@klvBC-f*3=4mvlt1Jl z!zINfgcKeGbBWRmaW1})%-q!ClEhr-_`(u+Mu2oYF*5>OgA;T*lfMLJ1T+)SVf=9g za@P}|8>&HK8Ea07Uw*mJTIAj?zOeeocF5xw8&?H#%+j^LyOEI#DdB^U>Kyc^i|%CPl0-xt z*v&{yvBBNrJ;;ch=)i6ta$Vk&j9gM^;h%~W0Yk9Q`wOMdi!Z~TWIyC_lpWM+LMq$_ z@bu9GXz2-boUG{hn_D z7h5`{Z2&rng2O2svWXR`Z2*y$vV*n_AY6Fczy_mj0PUusHxC$tSkuAHgPn|AY@pp? znR(zdC-C+fTNw{|G%|A4BUc*`y;9lWrU}@3UC68!L;}>?0ZV~fE!rqju)bPSDtG~L zCU^!iKMmvs^qEqKVX)Rb)WJd_h~_-#_y=swc|_A%?vRHxmn3qYAlRD6-{*DZ;tI>m z&jC*{6$_bjah1V&5{%%X3vtx-%5c>|NJ^ECGIDXkWrTFOSi$`b=rkFc3E&=cKw?Qo z5JZ)bHWv#hNF^Wxq&QWB6$y!Rv4a%HXQqG`7jOo#fw-P2LK0l;xv9m)iRq9@T+SdE z7o;UEzbHOCwGuqb$`QntmS5zZTIt2Y!6?8K#07S1Jg5(<jvM8WwP+t`KEGaP>-On7IVdrG*r^cp!G=mF6a;7J(=2S%bL2Vt!yDAz9D` zp`eaAXo)w}hnx@viMgplo-pZ>$^wW#`HZx%k4d=!H}kqy|BZ z0x^YLxj3>@E91cifhNF6Gl)GqwGwQMCl?oZP84DgXgZ2CgE+yu!A6O3v1aC_q*j0v z0t@K)3@}$nnu`N;Vi-6A!MzO+h9Gv3h<9qGkR%s-W?l(c4piDOgLbxO=9Pe?q`5di z^T}X2PSE-YkUo$INKzhlEC5&WAqg%KsJN$Fe3WaDe|(suPpGQ^qmUpMFI2|S zC)_bISb$lHg@Z|eNs9+6?CKj35~;-p<+-@JIfnX#Xo*Alo^J6_A&7}uq6mplzX1P$ zP#?z-R~M~3E-s`c-O^m*-hTe!e(^!B?yeDNZdKrtL6!t9n2+~xb#!qJiuZH$1-V&~ zOBR<3h_{%m!r&QBNFE~z`a-gze*tLjx>%we6#k&14Q1~GR1tWn4W3=C{J2>2^HTHE zgk&*H^#_TgnTfWa0;~jLBE%SwdEg9$#XN8h#cv*13Di8WF;=l$Tm?m`&?QZT4DEn; zTo$$92!QK{n2cngQy!O4QfX#R3W^RV-Z0RiH;E;w;LIGr808KMNw7lw^wd24^nCrK z(liJkycPg_NKRr&sy>*PoS&0anwFMYR18%KIxGde0mZB9C?^-Y5Gx-;rv@X}ITkKP z9wBBv1|g%vEDFx~1(ija=@}&oMg~U4x<&>@W(rQFX$m2!$r*Y1Ir-_Csl_@9o_Wc7 zTwDr1naQbn#i=O@rFki-MG7SusS1t-iOCtMFc}?%u+*aB%=|nBBRvBJ4UhsQs3s*% zYc7S#{8EM7#7gk7T&2aJ?^%*4E8NNo$Y1!{&K zmqH}eg#0AX`BMss3d#8el?wT3$jTHFOSrfcz#yZfq`*pFzr4I$FA?lgz5Jqd{Tzrt ziuHXwon8HcU3HE047j)yLi4}_6h*0prI|&kDGEuI3W)^;Iho0cNja$sIf><nIfGr4PVEBL!9 zIQm5@czgP}=qRLSmSm(BDWp~ufKEwL$S+dJ1O;GfiXN9ja4IN_;nAazmJf-w;)2xV z%(Tp8g`C8^^wPxiRE6~XvecqH(EhK2)S}$XVo)R&gZ4;rDdc44W|lx&4@eOY^^qPI z*K2MrMkXO<9R^XU2iywL;M5CCnz0(77$~;Vhon)MuwHV0uD&KHG(btDI5jshuOu@W zwBibsU_t38Co`!iv8WR4Vz7yzyuhX4S)x!-lwX#al3J`#T#}fVl30|YkeHX34|8Bi zKFpLRY)!^F3Hb@SfK}Uo_aAjCFZ48lqjU9=A}Yb_J9&- zG$fbmrl;nC4rT&*G!* z#X4}W>VTq90ki-S6hbmmi&G)#T_Le3RUtJmEx#x^H3hVc zw5T)>)awP6o3L_47g5rHGjwKZF;cMI_jQl)Ov=)GM zfHdb9DMUMKYm=fmIT=*?aB?v*GYB#3F=%mtn2bWq`V1OiCX*1e0aTP(h}n=qlZ%6k zkwb{tm>~`%&%q60LIh11l)!>qLd+m0M9`E$4=l(b#0+9W1kD(<;7g!k~=Xa zB@@)31r^!Q7>5^PphONzQ_vi(kONA-dMJ4!IWbQmDU}P-aDcTFlJj#xH7isbxQtBA zD=r1EXHKnD$W2WxQAkcL0=Hh zVrLLywr5h8s#X9uFBKs9tP<=6gtNg`C8a79q!y*+7lDpu%Flxwn+SD_o`O@QLP}~{ zVrfo^4wph+K61egs%79B6VpNGeU>OB!U`lkE-o%EW_AW9A!ZwPRS25}U(cQHa@ILLS0q7GkznQser>2#!ZL1}{dbXN(HqHa!Wkh^(HpI0Uu1Ahj6%W3Gvj zi;104h}n}#S*o1Om;+b9z(D0YaO^P&F$Yr+dteWc z6MGPikXT}35@HU3#vT(3$U#tXHX-Iehh&fnJiR%b67ZW?P5OXS% zuG9`@g4iOH4ABNgPL=&GC&7ZW@Z+H4g~}g z;F2>jN5R?E2i76U%+G^1Jwbh`N`-Qe!JtAEQAlSNE0jV?Hl!vZSUN=kr5)&*S7K}g zDeu6IM$iC}LS}J%RccW_dRqPQsrXvCV1;@bP`7C!wRB;PFi^3}jFbeJ zSwK+=NdnAlLd>b$Qs7L>%qYa1Dh(-`nL*L6p~Kb4!o|eGAjDeE(9Xyx^_f{A0TiAI z3Pq{VCSx(|^j&a|51bqnKz*H()I3m+7Zh{Okbadbq{o?=pU0)(1M5d>IJ^32Dil}d zl_XZ^DYzykXMl@2h0MIqYz!3nyLWmrKIF1mx2a)!QzNE z4=8a###2)AlZ#QisQS2Ud5zocN%)-DZ#9Yq7!?lu; zi-m=Oi9v|Dib+yxHlsp9W=aCM83jEw4>TcDq5w{{#h^}dqC#$BazBf?x=&@0gi4xP*+%?I3vF_Ck1q%Gc?K}vm@YsW^rWdXSr&SMo~uHqJi$uSBsR|$#0c+5h~RpJs{%}iiFG77OYFfz%3 zT8#VK4NLQ&;D_3t0?Hee(5NT|6(6NJpb;m9w4(f6aE<8f>cfRp*23%o zwOK*^vrO=ac8VSs7cV&U7+Juf0OzuTxZHv;S#TH$Ny50yAXyn%u0@a#WD;U&V`Ng4 zn#`n-fJlJgKm_H`#N?7vjKUuhnB|!{IiQ393xyO=sBs}RHbHH%OwgDP=q?aQ(l1Ia z&d(`J1?P0g7+0bKsKifA%!8y0$anzQlu}4&f?R=c7%VJO5n&F|n3oR=aflH*ppXQc z0~(b91zlz_XyIaUW=bk(ATTX4GY8beOw3CGsYH&aqSUO^5ORHgUe=E z-CC52+~oj=LtaT{Q7X)>kQ@mwg;O$%6O(d49ScZ_1sM}dQ~;lTke>$5#4xQ$)`K)) zl>X?2Ke`L`xVX5vm{}MZg;=^7IYhXwuyL`lFfuU+vCLy+Qk6Q&rjU@FniG$RH*kVV z1XW3(mAfgB{EyzIfL1ODUC=I2W-4k8#sv;0P$B|tas&k(E-Y>T%hb)oS2&mD)hl_M+9{OXcQ7w=PC?LmFrsx?Z`u#SNi%2AsMOAq>_-9 zpPv9JM3Pb!Q0h+;BRx=K50dKf4AdbdM+I92B~WV7OUut!uv0KGQUZ@U;;|7tjR0DI zu3%w8nRPY_7A6quATCbKf!69Ex1iSQpyG#-iGc;2tsun@BNGEFhzl)#7?~ItL0lmP zaPh;)#J~*VDywkqU;$@IMj@8)b&9zh(q=M5KG!udfDID34nB0ds zsz^rmgSP=u(jym*+JNBUQ1JL9qNz{g6p1!;oLd%^d?f0#l< zcD6!K-{Afbqzy@AvI8|M)-y7}k{u%(C?#?8L(>_k=wHt$APnO&gJi|TxlZzdlO2-~ z%VkC;HK|>E3JKsr3j7HVn(x6QK;V>Dl95<~lKfz`7`Tir<^q+zkb6E7OBBEx$@BA4 z6(A#ZpssImYF-JrF_r)wgon0(bBgof#-@PV2%s);B6xH;KQ9$*E~sRLjJ<&lv4mKt z5DyCacu)Zft+qksC3w~bRIg9koSQy?9?%wm-Gm}j0sB4nZ{sWc5@J~&+{C_t*K;^f4f#3C%FflE9iJ;>k? z)H~qnFgX)6dzV@ay1)S0dOZbfb}1z0zOhe4?NLvkH89QfdWgViw6jXjPe; zng|_l&;iw#;O1IEX;DFbaVjVoq@YvFHnmDG+_oBA^?S3Dd<)isF|s0X{pI2nPsVn;{ zPEIA%(BjOT)I3lqXF|FvU<(xxege<0gL;mTWCzM{pb08SYEXcTju)qvfVDx5M9~bI z1}?}+Edk9uaVhxamw=|k6H7p2R-h&mw0jAP&O~_e1C0hy{!fJrJbFUb1EH+@0nI6c zoC6NOVrUG46CfADtzbz=(1ZFnAcsQxLeO?HXzmQ!uPDs}c?7Hj61boQgQP4uIll-z z99NPLPS{|T;2IV*s|GG$l2S7g%QEweV8t6~TE8eWB_%ZvG%W{9dVwS! zl~b6rw3rne=#(E&kpRk431FRu381bOgl&|d5TBfwhg<+cl1FAStWN<>^pLs*S0fBu z6M!l`NK6-7DMTwltWwee&77v@q!=m1DnP0!eSHPj3V6wkaF#KWvrG~&3ubD%%NXu1 z6Qx)k1z25T1)iyfx)ajhg!sr1G}R4mN`eoEf(ROc=B^=v`T02zF=GW=1!J%nWJCuX z3MLA+3MNQ{I*{fcsP}mpQ4ce*fvQ$+X=ud@>U~~jlu?9nnL)BDs*vs(rx43kMkZb^ z9#Eak$pGSU^FUN`g4&H&84+A2s4Tc&$0fvajgd(VqLGUM#N!s_;^AUqVG?4y$;j}4 zkr8YI$o89z91>hRL1Ru#pbp?&MkZOQRg4Ob@Qx?gZ{VYC!JT$!hZPb{png(newv_>V%mi*_-(`fhvYEiG?7NIY65zmN0=Kg7GD^#EIdieFFo8nhKFHgq zpy40TndUmMoP;P#5X}yxBn#>fLK;|1OrZAGeS`xTS%p~cGeX;2OiZBm)_q0+m;)G@ zL9)^^TxDELEX+b|PZ$}#F)~VJD1drQ3GjAQaRMxNDu7muDx~ItM)!)ri?&MgL6szE zy$NXG8(aZ_%DB{g(87b%oD{@@5OAXy)M9|PPK%-4e`ZD@mM4rHLR@!2eGO(%?0*FX z!#NgEIH0z?Afb|ql%XNX0@U@$fTjXO1cJKBjwquy;93&Y5id$DDJ{waB?!=JZAfkc zI~5evkOBr;-|H#3l@@^)Qh`0lg*13skdp{n-~gUEg;#wg8L7F&sX1kku_ff$T#$j_ zG82N7P+6~1D37~_ta#Hgkg;IJ6xF1chyoINtIU1^7sqHK_h9N=+;VttClFLeU9oDHoSO zYc0_9I_N4r=$It9AXk76prRG!V25k!!3sj?Oa{0hQ-Eq$NCXvv=yNyV*oLq7fd(2# zA!v35g;RN+EiWO4xk`oJxOLM?w11JH3Qwq2(mI|8Z0B0IlhR{=RNi9gt z1J~A|2@;Tcu(!bd2!-VQBG57w1yFsDG)xX!TUnB?08LH>Mc}0|;KT%;xq#&*7DgeK zU!c5niMq)xMf`|U_$7vOjt9ZCad32^%e+~a|kU!y~5;Jp+InC&}I z#Q;vN3JKtH2T{174JLrIKcpQ8ADRN|0OeL3*&21Q7dE;E8Ce5maM&~_C?m6gGjbOb zbYy`AoRPbjpj9mkI3ss4K?k&0z!|xVNm-Ta4Kp}nFbc8EWMYz(y2}jC7}#?IXzm%* z{D)70DkLhDq*j3EG7y6#T(C9>vE#w?9gxB@DM|lm1+~^8T`=(KM(PhyK@tgg7y>%e z4)QmoQN#jn6wO2=70@8mOeScfhy~mzn#lwmgkoU=HHv03iAiuh1Pwy5Ffj?STw-F9 zle)wQt~+6s7C6a3+9U8H98#4gfjYY-dJ2x9dLE^Jj@){ zRpht?R%#(SC-8iTY?7RcvJ?n1+y<@hQ@Io}!6PEYkRCo{ zv;j1Z1r8E$Lk+TC4?Oe)YD(xSAhn|*jR?s6AZVEmyy*&Rpn}>Rpz%PkU7%h+cx4tu zu|i6Io_dKw35q+R<8|PHDX=EcDYc*>1@H(YOhktZ)Qki-qf0VUE5VvTR^=CgwtRqB zd_i+SUMgrgBWQ(xNj_*O2Fc|hWn8fFi^P-^$b4O9UP@+iB5d#ml%T=E08$6)qa}ex zby5`&qj4ZlA)3}i<*)qG5{xXC0LdJB;7g|y6re+MpruUUjWMZZsd=CTj;&P;(Mepk zf~bYq0&ORQG6OtoCBROl1)uT?K2DYh$AHGc!D$?(8`kR1NlZ%3DF(Hk2_$vMX~YOi zLFEIw!yvn55+LKP;7GxfjzIYl5|PCqbM*DW16hzJ1GIYwHUbo{>fp=3K~aqytqQP( zEg->ou+DhUF*@Lu9e8DxF#%6O9E=vs@cbK+54sAeC=YryIM_ACXt_a00kpEHBqNne zAsU=>6yOObIU^M`8a?9xQ8b|F16QALW>-bLm``KK=~0=;J|VM zr2WmzBE)hZ5oye9ATGT94I0(C&jcOnVqszijWmHG4cw;UU=U(^$i(!Qi5WZ+#=-#_ zMtsP`!Na8lZk@9Tu{>sCk(3fq2!_lhL9$XvWkG5&tWC$l23EueF@Xux^L@<3F9PE- zgJi`ez+;arpak`Vi3Kv+$->0SCdBfDi3KtV$-=}68h~OF5QA!EWd_MgN)5^^V;=;9ZGlRHrtvo_3ubEh6p~26?2IdOD zxQrmKAY{ang^7n5#FdtTMh>qK%Nr&ZIjCA*HZT{am6s93g=^(y263fjp<4NbSl%+R zKo&f(F!8a0xsc^RjmAg-h|RI88>%SR>_m~VvGz+6~D6k-H%;R#WQ8N?Nr1aaY^C&CQk z!b49~h~*m-3oN6GvVpmfsR$M(QAQA#7Z&@X%pk6?D3==8A7Vl*Kbcs>q$Cu=Q*&~3 z!JFl=c}0v3tPp0X7$b-aH&l!n#1#fjVt|I%#D!RXA-p2a2Ij)NA`TjhV}g4{oEanw z_lkrN%WozYWoT4Ouz|U-n3rG#afM-7LV_8@RZxP)rlb(dA0`$hs8&ffFc+p(k`cre zh9wh8W)N3i5j@Yr!YRb^pNUBbVh(5~=RXrSG}6E`ImlclkXmlYtPcyQ*!a)H5A_-+ zXc_^T3s%c72(g4qh^2vmaXj9M7Nd-`u-pb4&&h-^E-opwS z2%W;rBq8-|xQ+K9b~E5#h)%CDAB20b?XIBjzz-{UjBpAwbc~RdiGfXsWePKN;D;4F zMmU8TI`G2^9wVH>EG7<4#;lw|EYp~wd4&~}SEeyT^9n0CuOM@oKx(0Rg%y-nrZGeF z3M)9TAalWLp?QUsONeDUEU&PF^2&5rUSS1Qx6?s+1+=+{l|hJY4l~0SW=5$Jq}@57 zB`OKvwh=hn=s}u%2@2r3eo%Wp1!<}Q+@AtBFxA29c}i2k9RQS$IbXoX zyl59Z#*ql>gTO~$KqFpx`6Z}3-SG@OfFl#K^Bz2)2WgNifJYiY{cOC=eu!cm0|t;~ z0-)Ji@J4u0!yZQ`0IX6ky+pxQ!2qNT)c#jMth|QU0tr9RZ~>v7HNN2jShoVS*b8(} zZc$=!31l)X4K3KwVhR$!pt=IQOd1{!3YlqKkj+Z)C?F&3J=20ypFFtG>4OacxegxXA@Dg(a19Tc4~Jw~h0J2m(4`J|5D=Vi z5kv2wf(BGP45}d+UM^NpsmLe`zyG#Q0h9xyXWOYLICQ)t2p zNbu1GpkY$zvM$iT7NleWdp-ksSW^K~)`QP80fiR0paQSgu;NnC1irxBTBFt z3UV^|pbAjU0lIY#GBg7o*MaMXk9~oI1KBpv?uk4Fh%cbSp%5Q~hR5)#23@HI^D%V3 z0Fq)shsGpF!{kjD}63))8k zaR_9UAh;@pS5pcase0*ppv(cPmOz$a)N7fr$`f?118AcH$nt#M`~q+-3~nT3f=&g3 z)W5K`s34=D8~+s&pkqf!r=5Y8ra)3Us2A{nnF-t-V`X4sWCA5^Nhz+G%;1#HB*gN8 znMp*di&?=DvTha9Jb|?6LBqYEjh}GGW69j0(`-Olp*S_IGzUE91~vhv1yl$?YkqjC znF3p>1kO4o`CJMmsksIDMTte3IhD|96Nm#8QcA%)%AhBWfHr}GG8JNG02X(6TF&aQ z85roYQ1B!O=(bq!^h^q}A3+O=p%oNp<1?r<)z=5j+W+w390V@L&Xo~X#vyd>T;mZozC-R9I zvfPiAfrUYc&HgoczQRaBD~*B^7j{VG?wx783Q4!Bg;& zTM20(r3sMC172~PTAT{m;Z+PiSPZ(p34JeI0{S5!ZXk2Q5_3vZ6OcvOZZYlh$7II1Eh&Lh@b-K%4E>=8(15t>4Ul-8dQoy&y@miI>a(4?OO@<9i+vf zuMagByhajo+k^?|xHHgtdQc!hrE#CS1P*e@@&2H>Q6oJAw3Noe2JV10u|PXuY~V#2 zO)T7yF&Z{-2ds%j5L|+Q6f+C4G_goZaq)Apg18)@kddB8no$EX2~n!ji)U8gga>4LP^7Fs)>P3^}ubhn(A4I0U&)K%$XR zh;0%J%XAiYsqIV(2{{Df5arBzP>G2ag$f#wNVkE`t!RSc6dczOaY&xdELMO7d`Uj2 zkASj-28*FOT%beYAZr+5rBfziI{;`eFA=d0j&BtA8ye_=+xXxJB{JG`Ks^JHM$9+| ztvQ{0khr}b35ZhuFmK7}QQpdpYn2J9$q69>K z5h6ZN!XFxo@L08lgc8nJ#T}DSlcBNjy&?L%YM6`nH zki{%qkl{l%@T}Zo76CzMv@&skqCsc{JX)Ec(YlyLcm+IKA%cuTEK69}M4{0N5n~o& zTgt+=oW&iQdl`k;RC^0V`v=>OBD77rLs2F+QIC7x?i`D$n63{ktP@%zvwCo#f3f9sDUO|9PjRGZK zSd$90V;ypKRWWso5R^bP)iY3t1}8SX^paRoQ(ii#JTV1VAt0^#`tY?>sFr}6u83{4 zprnehH9iN~;(Sme3bGCWvY-L$x+SoS6>Jr%71B$r6ioFDbQE$*tQ5cogO-xvbB&<^ zsMtoh1|M)RdAT9%00&*Dq=snPMEE^Lus}RdRMD_&D zzwSenx}f>jeTY(*nFEwx1n;u2LQ7p{X!hLC!ghp(9a`!_1Q~@`4zO^DaY0I5h#0dF z+d&qN!z}j5+8Kq|j5{0atNr>$<3)=-$?VwWT z3=4-cvUX5rKg+^#o+T1lJF^hmMHaSesM57LKbd>B!nygxIdL zu-!q`4w_WH!NQ?}tR1vZ;wB5nZI(2yJ&>}8MTqSq3(EvnHmTKYpn`#Ljf0Y;k;`mQ zu7wvO(5ix9L6iz!Re`T0;sPbpgq)HQQW7DDC@8nUOQlq#YzQgJ3AJw1OF$JO;c^Mx zO3)HIM7aRUyogc?bdLp+Lr_ZtT*s3lmjC1G^=^2i-h~K)x|}Q=;?TSc z5n~o&`@+KUjl~C=^}(WyLTo=+*#4qw2bEht5hVviA+r$MFBXnJ@R9>8$|S`0kA?aV@KEv#&vsM=2l zSy}S9?n0uKO^9s`E7N*bR;e>g3JIVEkc6TZx=a=xy`Zrw^mu|+M#WfLo|&LgXPij_ z5x0o-ain)!AyEd7VW@kl7|W1QA}XfYScF*CAYz)06~u+dG-z;W4XdCywC@XA5VnR@ zN*Wr|Y#gAtky#IqX*Os~uVIy4507bxAfph=T2>Y@XiP)In1xu@v9iQ+b%6HCvN5p> zv2A8$+Q!N%RiTiOmYJ7XLb&MxQHCQHL0efM3P248zeGPo)WOD8(Q_rF@yX62#IhL? zDD12tE_9xqjR~~WZ8Ix$aS|I7XsO#~R`@UvI|s-c(%ayH!VV3T&8#xp;DG`WWE5iA zf(R6d7_$(|R#ui6t~St04>nN8^9U=`aaLBTui(ST5_1c3QWM}EDCpETXkZTIlO2p)%q*ZBdWMxr611J%6@LCRI4Gg102J@w zP)6+_A}wWsZDTD?EdebgfE-y@0=_>u1#}D+LMdb>2%0<~_blMP=m2(-BtkoM5fA7P zLlzb`&>;F5R$j<*K{oILj5DmfkX2Z0tPD&-EN56nCAb!YHm9-+v0Y?kSj^5SHHA^Z zCBHN&CzZI7^AxaJDh!=NEX6x^4k`&QvT_J>%>&g{?4Ysp+pJ9YSXrfd2mRPN#CwDX z&mrLn%J`+Az5}j13n0qy4xh7whtF@rG7&p?`2032bd5DTc=-G_s}Q8JVF#~Yx(%yr z*qInOgxGGg%G`ryB9H)NWc4^*2Ef(bGTvD^V=7|@V9OpIBGC%34#_=vqFbj zVPeceY;RcE-mzM7-GW3VlMvf)R+a`fcBzx#s3hE|K*S|-PZLYLa={~({1J3W_>w!2 zNiZ^zeL!x+g7durcrX)Xl#ZPVG>HBi5woD-t>3KNkfqq{;6e1?tb#(|YLJ}?)Ry_p zD%`*Zi&@CnI4F8V8rWbl3lRh@`(_#4y+C3z4X-ULkHQUIHR zwGm9zAUb>+jN7JFoVQvHEY&^qK?BL<_sci5f7&IU= zl?`45gNDctZ^b+`(Iz)t=0*e-->JZmJ-ABa& zjPSrZJ7|Lq%LzmY2FiRV*x*eK(7^f$Hh5D5G_ZbxP53#y1Oty)GYGMrWMexA&&3cy zMj@6{Y#b6?pb0^CCWsib5Zh@sjk@1M5)RnT6PHv#~uu)eb7b?yzyl zAUhs3uzr_~<31Z?U>&NRMTqSo8{0Ef?Vvz>#Kxh9tQ|D4{+Ny9DO)kuSx5t#Rfz2~ z8`C#7R;fLMa9~{_vzSYP;7TzZz~;qzp+8a*4ddLL*}3elKRF58C!=5f_5#k zu?T}ll3-$>{fBHUd0Y=5G0P^z*2K=#%FZfverS)aqeU;I=mz&}L1UMoC9JroP)Ltx zqKDVnnLxwqP3-W<1`XadvBM)9G`!x#4&7DG&IB4>Z(@fI8?rNjhS!_eWn0-{kqsF& z2gR9OD?2Q*A%dUb#^Au@OmdAP(Z`$ zo$S!zb$0OZdM7({c%2$>%w}hifEJ7pF=io_IqWQAT;L&gP=Z{i`bz{PfDWvq#yGh3fi|vAYLr9A)G^~5G@^ch9l9o+13aRBfE~IfodZ0get;di zCY=L3qJDr~@F+WE+!Z7MiQoh5LPyyl$tJIgh8cBw<)IHbymI&@Nl3M1-JlgOMNAryz(=Q2GsarhOc+Tnp;e-eBht;rakNc$Wh-l>VBX zrHg}2>i%FFN*^}s(Xl5w)RAwq3PN0K;xSgCqc{U?)Cpn(60g%tCDM*f~D1Lk7&DqKrappV-;HqiP3b%+Ku5_2)2!pk<)!9N*X>1Ljas zCLy*T>}-EgwS!8PpX|`}=P-q!WuWXFf7l@d=1@^)A+~?)Y|R`fjt7Ope|G3_Jxn2J z87K!w69;6#94g8p#MZ*W)`_YeG~Cq60UfS~DP$I6YvbVP;7H?I0U9vpU}6zso5#Vj zmxE1eI#mbEA$f)HMi|2D&xg~1IT0lfa*bz-$c&J|a!8hf47lK4BLG$f8aGz3RWKz` z4xw6yyG(+Qk%J8wnkR1}op7RkS z;yMaOIts?%RpyXfLsTY58VQFQfUR}~je(!yfDVgtfXBd3alrelpfT`M9MJXU9N;nV zQykFs6!8@`GI#uvH z2W(`Q6Ew2h$jQ{gIYdTw%}j_G*)=nPbUE-33YnSU8QJ9okL)&bLTgG+@W^f>C$y&I z1dr@CazaOTIl&{ljhxWAGEVTwZX>633n!$e1POq0gb-UJr%Vecq^5)kG77OYaY9FS zVPeceEX|zIkzG#E$Zjhq({fJ87!fCUWVe+QHnPjf#3;lzfs~eB23bC!=Wa{IBL?l61>1-;vTJ5S z{2+=M=vr3j86%Jp3bIFbVa~uDHZe0HdOb8G%pl`Bct=e*!6Uo75!n$mUa%Wc!h%M2 zcOyzz(8%s?PQhE8kda+Z_{i=aPPT)bkik)yAfph=UQX!9E=&w`kPj!v0Zz!sE>x6J zi0u$3+i_Iwpp1E#6FRaBQ^+jDc7&7T7$;<87b?mm#CC#{?JTNxP^of~6FRaBQ^+jD zc8ZhZ3@2n{7b?mu#CDF8?J}x%P$-<|gpS$36fz63UEt)n#0eSMg^IEWv0dS0yNRkD zG-`5{lLKGukL*Gx{61&0HL0W^!Uib~(YTn`d#s`>UYU z&9k`R{Z-J&?kq0o$Sx;%WOo)9bYzzkJhD5BOJXh;tiK9d-8_p+axNFFzX}lqE#l*X zj_iVTLfXD8bGTq5yPQmHLToF!nAUI&!jWA>IZe#TA*dyJ8R#@u$Z>K6Ms`VxSt3Su zIl;@BS0Z8;G_t!A5xby~-IZL>kzG#k$nHum=*TW7cw~1am&_V?>_P@{L6Ic81|GW* zK}I2#Rb0@KU64*_?5^g5jqGxQMs{~`G416Vi6gtP+zn~D98N?N0 z+r=ff7oH#>f{a2eySbnvyC9v=1hEH{NWdezoUB4Dhq#8~$Sx;nWcLskbYzzkJhFR; z3p%pP2_D%!#05JRgG-3*I2XfY9`Lakq2NJ&;)Zrh!GrZw7~6$dig$1qbl1yqE)HR? zS)eI4F3{laMJ}c*T&z-^gMQgA#C!0uK1kjIj}<}AlQJ;^^A};Xw?o}=hL9XMFSY#Apd%(r= zl#5+z8+g?xRaWi}j`3ZzNC!tZmH}Q!bP^luIEHw+z(c$bxS)Hnxxhob54fOvvAMuQ zybrjbIdELQlCML%bjXNE|=l5`M}B8RCTrG77OgjzA zx(10xCLy+uTrA(X*rkqwqj6X*-X$23$UR=jY$d6Y3fk?9xqg=mJj(kK5v!o_n2%iC zkfSiTz@xk$xuE+~xxk~mAGw6T!DAINpbLr`k#F!=g$RP~DByw~g#pqDihgDxw$EH_ zU%4Qoyj-AB-XB~{_qZXWyj&chj@l0{4iTp`xI>7`WM{qiP3b(aGG<5m%T((DDD=9MiZV zQE~gF<0GH*_oo zrjS{PZ2>pOB5ufdFI1F8h;0cs+e%dJps|#t+|aQUm_lYDwq@KLE4b6RHi1_1a)HKs z_i?k_7pX@MZ*P z+o*Odq3$-0| z?mai#HB{}O(&`d7^q32nLeL&2ZjP(mkda@gDCpdKZnit9+ChPMgByCx1xz8c5Zg^| zj@#VDTn8ZyV^$%ySKLf*xml$)4B0ikILZL5L%+~eOk8d!X7HB_Jox*H8#?&Q1zy(s ziW}aU1r7eb;)Wh=!37@teZ>tu+JXx_`1^`m@+~)H@E0TiYUzR^Md~d#WbhXz2wLaI z4L#ZdCI-3=otq_(>k1?`*@W1Bax?wmW|cZT*p~H@Hu#GcrI6^#EiEolz&H9!dgKx@ z{>ud(|NV)GV9@yQPecTR#(#fuLyxoI0+0XxEwe{#$Ifk!Z8s23DZa)00v z3=ss~HO>tk?gQxrr3BEe;oK~-T}; za=kpTOb;3D6=Lh+k?-Y!WqODpqYz6s5A@gukWOft=;2|39^1ghD#S90XDAK|2OL%>|%A>c_oupwY>A-3r}43Bu>L%_tY?ghmtRhIXHQXql#z1*O4 z0eN6Uz}%oA;DtO)OL&IL5HM_p1^cyvpiBk{O`>}5c$fHcgNJ|@!ZHvycnEkQECX?a zhkzIIK!}A+|?6Y|l}(gRLAljkD6SaqcAwh$E2$9 zFw}NtA-09QY|Bu!gG#GKywD+Fm_pEICti-FypSPas3?mN+j3sEHK^J_fw+PfIs^<; z$SlORl9yvOFLVf)8?^X$Coj_;-l0AOjHUgHa|oE|<_9>3LGBO4x%ZzNJOsRx7djfm z4ITpC$qO9~;${L30q^964gqt6hk$qTLWh94!9&11c_sJoLPmo?0-%iftW0){0BX!Qq46yQl)(C8GN zAz+k+y@vSCcYqj;G+>Nl88A0^4EQD@QGmvPZz2)}Xbku!FLVr;8$1SllNUM$%ncp` zzR4?h2c9S(!@WXmH+kjnz!L>T5Om@PFLVqTq!XGbZu7#%fVo+PSRV2Y#W7%R&=~MT zUg#JwH+T&AAun_cm>WC>{E$~vg3FtWiG@dq?I|zAMt(*qTZLc}76BKdE&}G_;^tyz z;b9bFdCChLs^tL<)xPIt`oudFhH4?6hj+RmnE>4NMqaH8@iw&m4bCE1*EK+t;aRQA z10Je<4@>7f;Gx?0uyoD?9;$uM3mvNE0T0!_=YhG-}B0Rf~Iqj04VtivAyS& z{RBU4b{TLKvzfeLWgR3Ktr{kd6}m2L56#Hz(cj4d0|7fJWPy2Z2x#! zn)#?YR697%*+q+VaCBoCs)a-+v9XS0wJr~MsP-SMHO>Pbs{O|c9jfI457qwTg$~v7 zfQM@T@d`EbL56BU0+2ZV$1B{-2N|k`34*TF=7kQ`!o)ylDe|#3@j-`bc|b$8{d_D_ z_=e?BEy0LH?yEwEYDtY$Q2!KjwJr~MsJ5RE+Q;Hy0(DOM`JjC)9`I0YKOeM@#RDFy z?dKDo0*_V5sH_lMKcC1Hc&tJML47Pf=uj<4Cn)-vh1e$Yu}$WK4At_0hH9trG5vw( zT}DPBmT7#jp;{i$Q0+=SmJ@uVL$$OSs>PA+(1&UvX$`ho7s4a9I3ate7Ge!#jVyR9 z1R_A>ayY1B*t%GXR_j8{g17{2s1_nc^a41ja+s^|4At_0hiX?MGA5|ozmgAL+=7N` zSMtG&ThLJLN{DN%W6L8P%TW1S%_^7AICaA$WSd* zlu?Ln10UN~RPCTFx{(h$R0~rGnporG*un=Hs)dR&39)VCW800Y9aO$-=i`8!j{sB1 zEX1~hk7E}fWT+M@$}GgThmY+bs&-H)?B#^zj z9W?lHm=8Mi0#gV&T7r+`7+)IK8qjK89?($j9X^)dd~8y)M_22D_t?^Ms1~UV!gUZf za_wiT5Dm#ipkY!_xd7o2UmSrdLg+v(arR*zybG}sd>jw?AcM3}QAQ!Q$9!xrP_=`~sV9h%0&FB`koGAb z$8&f|0TyKvVtdKQ_6}7$sGNF*C@CNcnT6P1^Krc8gACF_ZD$r@d(X%A1ywt!och2A z9i)XRWENuk$j9-S4>Cv#6=e}(`^v}m3spNP5Wn$32WepnnT6QC^Kty-gAUU2fCgzh z_?f!-hw>mTQtnqkO7~a?X`!hY(zhkldnIO&mIpjY+rbYVq~!q*(suAe2Wff0gR~v| z&_P-r@E~mmKXj0m2RumI!7tg(4;iEd34q$TLTnxUQr-NJL0Xs~qYz6cKXi~5CdMqp z(!~!Oq~!q((oW}Rn#Dil2Wiow6cSzFzA3(Sx}--g5recm;6d8yhzJG^(oRQ2Fldl= zIzM!fmIpjYJDndoNXr8rq@B($I}0Acknva{w(0zGv)~a75o8o%nZXYoqy^~&r37Xn zmYMvpL0TTrAngi%rq%o-Zjcr^L10ERmjVdj8;44*NX#wB0oB`h2WcTj6CR}H0T0ry zKqLy#Angi7q5uujuHc6b((-@@X;<(=2Wff0gS0F7{DN%YIOr1`X1}#F&Lx4)8+9Fv;5Fe177eT?OA@IOZ<>QT95!Fj?eN7U*d-h(!vB8g;>tJ^AFoWTD%d7+)YIc(vlLX;5rWDN;F>ZAniRwtb#fx_xPcMw7lR! z+I#%aL0Vq$AniSV;m7b;g^b4vvEAbrc?^$Lh#;d7%YA<6AT3BIDEgU&*dFk+J>rK9 z((-}^X`k{lofd%QT~N>JDL-tGmKQWg`-7ily1?ilE$s$rv7|d}gS6m9TA)D@1xPtT zY$Za`AT8KZ=s*a}Ug8$4K^q6BIDqq?J zpo6q9h0H>1?E)N~0+2yks3@}#Tekq)1XS&yQ0NhW4${IDG7GWw3UKrbKn7`{qAWse z69w3&p=t*WeoPX84${IDG7GUy7T}mF09{YZ3mT+dEx>Y0V01k#J=W8Lx^RTbAmr)~ zI!X&J8X)Uwp}7h&8i0TB2BM1!M=C(ghieu!Uz%fz_@g(MY zT8NuT8mfgKL5_c@mKQu!yIKI=I0R+1)dKKl1Zb#swE(;s0UD}ZEg*afURr_2KpBMC z)(Eg|gy&d@Afph=S^?-#EiYoIcAWsn1_8)WEmV|Ih;5Sq+jdm#pwenHqPzeb2^y;1 zBEYc?US5DjnS|JO2(ax%)eb7Hb|T6Ph(cx|wp{`odjuduwNTqZ6GZ}Shf%eIN~`?> z(4ktGLS`Yh0|Fd}1Rz7TP*D~kwj%;;CsDP70`aH-bf^}lkXeZBm;lEK0q9UIFKDRt zx&YHHfx$UcOWd_uILZK=L$yRVe8E*Mf%UY!;Gx><0??saUhq)ubphy5EiZVe_PPLc zsFoKzRC`?jI#kOG9;&@AAbCpwGE@r^0JU_5*scpm-4cKd)xrcBg;;I~K!<8!V$4D; zHw9orwY;FA+GhexF9n9hP%Toi4k?_$Jyb$NwWLKZ5ks}S;Gx=QhzJG^)jmT+Fleau znE-UCmKQu!`%C~jRLcth`3XA(;$lB$Lc}`Yxx+3SegYnM7ZXH&Z6N14c7JwGEER< zmFl6^`L|pOo@ojt8L0|snW;G#P2t3n}f;Q;aOD62Oh8O6@*rCeBklgUO{dl@R%tdc)Ye(P*?)SWfo%T6@-?6eBklgUP0Lj zf{-c>BmhdoLTtT)auWn0RUAx^QHZ5a5LyDl#F&Lx`URomwS1uQ+KGZpn*||bK78Qu z+KGZ3f?TH{5y>dTHb;J? z4UJb&Yj}p&(?)79;?P<~f2Q3k4xVwlG0PA(pv< z(7`>J80ZW{LAC{gR$MnBG07yvwpNg3lOVg)32;mft5a|Z$0ce{4OGFA9;#k@Tl!tL6J@Hh=mN$f?`K>6Fg!ef{a2e z>jc@vxkR|Qz&b%GfLVxby&&60LCB~rA86Eeiy+f1Ay^IuC95rOX#Dm#qU;5Y z-yRnfekKSRzvY9E-<}X;J1YnofQ1QyHth*QM?_#^pi9UEInD?|#&4mbj6!VZ1lca5 zY6oT3^McS3RG30$A+`&G9G3(kw+9l1k0EB=;Ah+yZ5} zUx=m!Xe9R+qG(YZC1fJAJu!p1eBi;{WkT?tF6eT!WkT?tF6eNCWkSNDT%g55eBi;{WkM2?(3UbQ z2Pg`pRtmv-x{!O&K!VaMg7)>=)X$N1J_1Ubz!KA-4lj-q5}>M)25IrGzQt4yz8?zx#792xi0&~& zvH*?f9z!Gx(1`9aAz>+KvS4QxVmT%xD+kT`>>NUD$Asih!jlDL@D?Pfa1x#@AcBlS zEXRddB%xU!BE~GlazY3;qRYoB#Bxq(D30jzfkt%C2|-76`M@K(=Y*gmx_saf-E%_F z5nX;Ewo5_`^F-hyy2P%&1xF~x@>_n;^;tr&5nX;J1|haPLQMCCs5qjl;F-onyAfU3 z>;~@jw@{B0*9FJB0+$~=qI(CH*!jUDx_4lSogX}+dq)U5qRS6nfqO>?I-<)D9?`uc zBzqs4*g*oIq74+_a`&N$9VQ4meoqKGq6-sa7Gk+41Rc@k2aV`H5MpW)hKvXCgGY29 z2*E~l`9UjiUkkB(5E`x{x=8U3t`c#L=n^rp2~8ll7UA-PM|58cK}U4?!HaNT3qeP8 z`N1Q)uZ5r^y8Pe~-Pc0GAA}$yx*!2aG`|)S`5*)t(S->z3bDKqf{y6I#F&NH-U_k3 z7lMxH@`D!P{t;qn6dt}Kx_ILfxkCwAg$s=@qBC?(3HA|Pe(;FyA4J50`YV5gpd-5c z;B~lvgrFn3{NNGYKSCmn!mx;i4BiT{{Sgvv6oy4CM37O4<*yKQL>HtJlmeKA*!~Hz zH3&mSbooIey3N8&FX1^D)Y)nl<`CfmkLdD)Ms%kMvuqV+8y(SA$febYE|#Q+bwn4u zybCd`NNnk}TKX^oU znlQZV1&s+z6NZ<)pb_0^!tk;eG@?6ASa_>2WMqUNKB7Bam~D%SoQ5GS#^}=jhP_=_bMm7jT$7Ns&nT6Ok3Uh20hK}g+gI4686lQrLJUXI_u?%6L zN+aaD5H_Lha3_@(DgxSu+^Dab?QHbTVFmyzh zA2FhPMwsKAFl0m*D#|Ftc0ri!Dynu+xpfgya)6Bljp$wy=C}ecIl!V!LTuNB*>0n1 z2bEjb5hVviA+r$M4PlO3!jKVNsO_NnC1JLQsMR$Q3lnJ{!jmmjn?_p30|58=T%qN|Wu%%zZ?nwMIX zSdyBe$Hk?9?aD*&I1iS#3GNYH;H4%W{8Z;5nWiCfVGSv34_oYU8ri3Ms!iu^kN;+Ivh|0I-(2G z2~8H$MPMVk{H#JOb47;Yh%P^9M0c(TbVQdQJfb^S1UjP24<6B-DsK+&|3yV_X;~vLST>m?iYcK|G@+qg;;ir zK*#@JVxR*MMcDRtjfh)N&*iiTuK+ZvL92C7iwKK=Cn5#FW4otCL@&VO7BY-0#CBRl>;gP)A%cuT zEN4X6B)CAUQb0OEi2!utvk2RH5y;rC0BCIYk_gidQCL0(rL0RL93otOpw)5$pt0Sj zA}lSUY*K$mSL@PXGzDksLm%9Qr8wAnT^OJE!X=&D@m-izkTF&8*fLCr$Pr$+a@g8f ziiUV$ra|0;Hdq3aBzpZDTm#H)c-HL-fJb?sA~GvzoZu;KqfsU5I6fz63{Sx8$BLW%ag^IEWvHcTaYZgWEJ7~1zzX)`+1f~#l z4!bBvlV}>(F-S?oBE&XBlx4dpo7C3Pb-OfQw@X~Pgj_L#PF{dEc0mhMQHu$rej?VT zyRb|M(n*E2yU51DhI(OQ)6tAZ`-n&_LD>TEV+nlw+%CG1qZOGn!S1?T9GTaZy&OZG&(HFLC3&ILigB z1HSOIOlU&vK1hQaL06gk@ zSrj_FCjcJxy(}t!4Ia&q0bWpSDO`g`GenS4h~6Vgv&xhl#M%hd;3$14CD z^?f4B^jwrx>hH)}$BUXQutheWmAo+J#18yoQIB=xR{%Wn`vj3RKqJ3T5J>|x^7}+o zL>gKGfY$Op5tWmNmH?o&yiY_Gp2L#{WRO>g?TM)3b9mB#2!bx=5@nHsmH-ejW+9em zqOg%)0ahWFH=;vv5rvNY3JS4(5M|gSju`o+ zY%#AO=yokp*vPLSXyo^oDAQljp)>Len+U{RK<#g5Z(g zU$6u(2p;+UB?=w+6$Fp`{t|`GfC+*}et(I|{e>oYkN~KN6Jq-%D*qRn;9-J{LM*>U zp)+7GF=io_Kcdi)UqR5w?>|wd1!9m9BSG-U?>|x4$gd!1J#VKNOP|=V9r;CydT{NC zWkD}2N{JktF*Cunq*o9;^4lo}ZLSM~m-KduK}UWC!6Uz&V$hLaLGZ|Lr76CUGGA;MkNo0|PUH?Q z^2jeK*&5fXUP184?<_>zg8DGC#GqX_uMcb1swe0bbK26=_pW{HW- zhsP~MkWq+bwitBe7o-!E2$+S~=7_P)6N8NW3W7#{7m6{pi^K9Ur~|f83^wvB2paj_ zBF1uAY;@$egeI$cAuTH`sSo?eFL-en(x3_{DKMR~kzcUYsKY2kukD4Ggs{<7DvbQX z+=F%GmxOh47;eKe@+$})`Q3uZte_EtEr@~`H1fLzQ4oVhez%B;To!|j{0hQHez%IT z?Gl5Ge!>Jn+X%&=Bfl^)W+AriVjMfgAS1s}QAQ!Q-C}G9P_=`y?H)1c$S+JGvk==} zF^>IWkda@gD3cJ|K{2*tsM1F^;2Rkda@gD6%0$vH9Ts0~aZ=x}#2j*DW@kzYa3 z`rhYaEN$YWBfs=m-%DJ%gj_MgM}EN>6SBe=p0^;w2;db$ct(C9I;l4D3o(v5MHcl( ze&Md8ash^aFYMt(ntaeRW8 zAz)D^A+|4KY(G)8gG#Wkh%yACkXeZBn;6FrG04a-)OOH}lo;E8RPCS=?6(+niqd0Wr7p9O|h^UJdmcpYMBFHGjvRE8C@(a=lN)OCJEK9^;Bfo;6 zk>AbYOxwgq=*TZDSzxQr36A`ttl`Cbhy%=cq>*B*Bfo;+k>AaTqyZZF-Hb>YppoCr z;?R*_LGYU2&En9JUqSGi-_7C*+u%t9GN3EOwpmyaxzm> z6iV{J2QMU~=jW&Ar0NwE<(K3qm8R)|EDQrJc1TbNPtD2E_0G#L&(jU5EJ)RZ9P}&1 zD8zC=oI?<@hFAzRl6*>>>8vj_uxr^kpq$xZi~y_gC_-uAfph=9Z z6Qqz?h~+LwYzHG3Gm8+b5X<8sS(XZc%F@TMvQ!9MmOh4+r9$Ac^f9O`T?8sig@xE& zh%>B_W0acAs1RIIl$n=KTxnViRtqanA;l>dq&y|5IEC1Ww?GwU6k>TH&LPaT1e8aG zK?Uk3ai*{0tWuK)|E3IZFySsr6EaeBa`F>Ec?=Sw$T<)D=mgkAc!{b6Hb4ozP!$Fj zs-IvvNf=zHeiDb?5h4sOR6mJBFT4~67pk9NIY}5?sD2Wc`3lWRAOTPY5n}r!F8dXl zlVF03LM)#_xdxPzU}DTdEMGwVlG&i83c{d5wMl}hRf1Kji&-HdCp9lYAtf`pB(Ws5 zSfL~%u>@SvBA;0gZs=zumZgGaiVIScGt)9tQ@9lJN^_G^ixl$H6p}L%ixQJdQj3Ze zG(z(-lk-zj6~IJ6er8@tv8Ikfc}8Y(1~@IIWTvI17Ad3^<>!J8gqXqwGA5}KH?wzKuiGjno4Z3=>WRw20nEtx=!RR9M@QmTR} zlAlRWFFC1s(6KCqTJXMEE^r|#3@&k-B%u9QVesT$lLS1SfJ)pZ2_Z=3D-14indL1Bz!E9t2Q8V3rt3dWcv3=@kG%Opg~VPXYw;VB0+2sTLqdLM}} zc4l7!q;c*=py5`q$d{8V_#fe11Ru}qeLE&u@O1SK(MA(kl;EX7b2seY+LwflD4=V**1vAVN zf`vthWg{YvSy(|_cpQTUsy9jqLDs|xGl53?H%iDUfY-zcGl53?H%cgOfyXg;aGODh zZKH(J7I+*(1Q~@`Hc3E7MnF2DCB$Y4mO`$}kVYe`5ZgfsrXv!pQiqrnz;z2aZW0uz zSi68t#2VZ1tPH6LzzPv{0IUk&QlNG=hE#_p{5Ycu3t$IF)h+ojE=YtZ^kuPEJ*!MvRg(L9zg`7wXiYCP)@c4xYG77OALc}jb zj9G}~umnpb*A+?B_*jvdBkPkT+AYtOQ9$=y)-AWNFlYNpeVJtI5R&_K_kf3NTIkg zuOzVo)PKk<$pDwjNGb}7^2;()Qd1PlGfOh^OG^|I^D4O%Qc`m=b2CAG5v&O#B|o(| zPrXC|;S5Z>sh?6nwkm*oEMVUkrKYD=C_sZvAt5e0QMbxbH!4~AW{%$aQp!x1%U>~A0VoE(6Z z=`SikJFrQq>6v+XnR)4;ekJG@Hi$H~B*CQsiXBjzC@x7XDgmb^a8iR6Rw)V)=Yf(1 z_VR;>gOIRxDnS;5lLgpM5L>{3K~5?F2cd$kf>J?IDweqCU=d>ZfQWkzRuC6n{d0g; zTuVT2u@q+FU}Pn~b3o(%gM{2?c-%t-8HHFrN5Xkp@Nh(Z)rvZo{qYz8KBy_SIq!XI)CrGjsaeaZL0d66-xspr^Bw3{%LwCc$ zd_Y_}0M(%2qOv3-6-V(;d|H564LQ>tQP{&{hU$p`W}^b!3Iz%xAGA^eZlbM%63zs` z%_77y7m)zCL5ERE!V>^D=r%@4Avx#>BsViiv7$1xol;bHiiCneAfcB+z& zf>KgqUSeLNQY`7YJQFln3^xehJOo4~%qD9tw9#cA79o~3h=jny3gW^O0uSg^CrRi{ z<-$xn%ph5D$mKS|OgtQ*sFPX`PY68Fgs?_ZdObWLKm-|uSk_8HZz>1rgq8&BBw6yf zCO|q6yh3cdB$@U~vP%7jmIRrE`wz*9d7u&iIXQrv{P~cJc40{gx(XIoa!|mR7Kj_o z1NF+lYY-4A#_^;K)D|h-EhP3r3{2}sErtj zgO^2!WfvlGfUcF^g(xL>89}n}rUEZBh$}4%EhTt4K+!3;7oIqHp^0Ocr2JlZ;(!Q( zPLx746(C~FLM(eESz@`OLF31u9b^Y3ncC%;rMwjqK+8J7t2z>(^O?|IpF(0vSz=yt zY6?_1F vU!stlpPQSXrvPd0bHVyp#d^?AGrw@eODcYwEQA2h180~+=84`1*Nq7dy(OA_-^ z5{pt|H8M&{3W}}t^)pIxbM!!&K(9O_v7|g*FTW^VKesfeB(oqfJykz5ub{L<52Q%B zEHNiDMK@I!O4=Ej z$%#3TUK%K*vCJMqDg>-yjHn7=qkv$qlQU}w&iJ^}2SREX>3Q8oZGdgk0z=%*4k5O7bFiB_S)fK?0xxT8QnkB-?dK$m(U7 zAZY8mBnM=FGfa$0h~=szhXJ&j=L6?-W+9eqk{nK4!d#pzP<4z#Y&Rs??w}e7n!&s& z$)SL(nUP6|<(4FeD~cGi5X)_lCHk`X%&=V!AsQ?l2UW>%dNN+bU`aF6hJNl^(%@qi%Y<1 z7hKO2LFTe@5|dJMiggrVg(H)T_sH&g{0CF z$jmjUCW4G|fHr94L59v05~1qB#)V|0Dimd=XO!rI2Qwkg0jFk==A=}Gl+5%@kO%W} zD&ZO+eoxF(0JqK)6-tT{Gjl+rFDdyYj~b?c3N>gMr~@t$kwaTg3DcOQR0Shb%+CSJKH`7jIi4S!X@%H6OR{~3XHbYBqY%p%Ne);e z9wNph#PU^=LmQewA!5uzEZ-zKJfJxqEXF9r_Cu2G52}Hn9RE|2Lm62!D98VjUUW^Q4Wo4+O!v%7>l97Rdp_PG)lZBO|le3GJfq|ism6ZXw z5#f|yk^yyEu|i@|szNzv1uVEJk(vS<3(3p_jm)Q(AnI~(oo#GsVMaoo4N?v&qM|@W zT4qXWUI}OkP*E|+dIdCZYoKNJR0{(ut3=h*JVPZ-urrHG3kvd!pl8Mtte-stU_pvI zqmn-S3NFu(YG;Hkr03Pl0-U)Nbo$3cDR{jMI?#Bc6el+{mkKZou}qZW6M*JY0S-`- z7n~yntCt0!xpa~g+cYUyy$lfqt-_Gv5QkXNT@*3Uo)jq#8)z;Ct78;m zn=ZvR8`VHiE}bC-&-M_7OhPO(r8u0B4P;~%VwnZ9d1@0a3$>@0?=V-#XrC&ji2)j&|1Sue#QkE|I~vTu;$a6u7c7Gl{5 zvg8{i%?Jsx?U!OXCB-K76q07l$VoHsk`^P)fJ;ws;hAEdXk}<V zWny7sY?Yj1X=Y_+WR_%QVr-D4ggR`*1#0(|rYd9>gJ))7{RHGv4wSMWT@0#s6OgST zD`A=8Ojw{r)cX+$3$!z0KO$iXF$=NmM>_j6y62 z5eW+-#w5gY2o&d#g{TlQ&>{gu!UBsi3b7rPVmpayAShuSLnJJSLM9=W5d+QgrV8twiMfac7i%NQ%Q6MT}X9<*^ip z9@j3&$d-r@+Yc$0KT_;cD_OzWA-*Uzjhrl@5R{shT9lfXocgF?N@`|FMrsjg#08Yr zi%XL;6cUR`$tF6WqB&6^C^aXsB(p4)3t_qf+%!FS7Aq`GEvkfNEgc1rop9HHis2Nv zK}hC+8pAlUAQv?n!U~{vFqgvM%!7z<0EG|s+(>zY86_gf%9x4-%M@ug@RDZeh$+!|36f{v!@97}J(mJn zUkRy>0gc)qjRa%u%%W=NQc!}mmJRd_mB86d3Dg$SL~Y-|n>U~pFR<1P%t@fU=@~%H zC9dc#XP8Z3_hlBqI(MWeA!sDvOhTfpLM)xq@V=KQBZv!6LZZwdE<6c|a)2UOY>G6j z?EP)QJTXaMT}X9Ws)?95!WY3J6cSLZGkk)YH2p9hfE3yrKOoE zL?=jCw1wYQHPJKFGlC~NE(JU-hEPuz z+=Ud$3vAGn8pwB~rz=oE;!IRxtU@dc5Q$2R5yXXe&cv8OTzKb9i~|%y!mHtlN(`E) z7D}@%g{J|CAfph=B58Q%3?jxP#IjhLL!C>Qi;G2=2_nWU#JWV9V*(>IRe=Q=h1ix! zv#mlk5tOQyOT&9-5QR)aEGwirY@n$MBE~GlvQnBupKCK@Vpd#;ZHF|=VQDt0Ma&8b zC7{qqP|zqT%FNA8O;G>^sV00WE`d3Fv{VLKnwgfFmzn|^`leDUgDlod%*jzmNi5C) z?H<3z5mDxW`~ubwS~?2Rj+q$AUj>ey8j*vL^i&Cs2%O1MoK=Wr2O?RDGlIDA z`5$p+5EnlGBhCSeVv)n}WGN0!mOG`{_QDepL=ZIVD-ECjfrv2)vFrvV7)aL+A_khk zmF5VArb@6FqY&FZX|_YC27*%Merfpp4@4o85X%8+_?Q7ij9G}~pfra8*M3Nns+0wRSeIfxju5X%*5jzDM%1&c8Xv0am9yM<~XD1}}}(+sMO zZ%A`EA{z)A3B3s_OV>eCsH70v3u%_m(ri+5!6~%Ngu*n+1?tqpCn~@R1U0@0CQVow zh_4Dp9x%mX2O33d?Xzr#w^70 zMw%l8njpbqpySV^**>8f2uhIer8!{T6No}4A(ju)9O1~8fD+_KX%0Ks77ZyOwgwrN zZW%W47LCk;GBau$>G|9lz zB}9x#h^1Kuo-QF`pl#qX@N@|lV-#X*lVR&ZH4v08+hySC5~7evh^0dYo-QF`pq`ct zhaJ~G$SAF}5ZhcCmK8E=QfpbjRXL; zUdjP+5@g8)WJNJS6NMzu!V1v*DR_M|#1)_-I0d{;y$G}j2)w5SrW97Bf}BRw z7_DajG%A`8AKYILFUP@ErO?Kh#;d7%X}I5Fe^liNr+{E47?=)5n~o&St!Hd3oW?8VvIs; zi)GlBqZ$axAWLL8RG@`4L?M$9%TgJ5O9CRsEX1-*hQowwA|!*z2(j&wVL2+pCe;AW zAY~@dAR!}*l$pSC2Fkn!q8V$T2hJCu!~ zVvIs;2V~fepc)8D^9N<%&2flA&|IJlhch%EK*T_EfifIsT&o~yURH?hx(v%B88)ez zkTh>bL7F#%#yO-ifUhRd!B;F(EdyX}^METC`j`Be6WEYcO54J}3>N<6Wd>NxJdCM$U-y24H>pO@U#mNWE5h#iO38PG0@sn zL}q}9f!3xXG6PtQQHbrX4BJCg13{VL9wIY96fz01+(%>vh#0dF%L7nmcnaxC$_cT( zlVSNO!zOiw8M4?lJ~1yP9<*qR-vNW&ROQw~-Z1{RQ&WuTe> zq!G3X6}0LrKMlOl)e$_`jOdnT=7H9*lCfA$R{>%aqC=6Imz-$ozz-V>xI#elNrJ z8J_GQf}pLuG90k^1c(@u5X(my4nJra3=sn@5=FH1!D5U;Y+q#9exMo%O2=OjT?dFl zCLxw@G8|sW27-E^-(@%~xaL68vAhslt1Qa|SvIL|W`%^(9C9Wod^}yiM~aoD78NIg zHXwxNWfp=Cy!1(}$V^78_|K_S2+l7mNleN~MVgn@M4j9Kx1ys%L1u!sNo1yFCW9AX zBGyBsXO?7?CV>}ufZVGOQU+T1NN|}V$WYwHD&gjjlI;WHT!G0@gk zSq^=!MI7MeKng-^D`i=>$g)XI+cOb3W1#iAd1uc69?KeP3fR;uSl@!B@ zwwzSZsUDyi;nKX6)S}{K(3xC`c`2Zh60#Qp)cdUj_lv=!;NYWHAi9cm6cCz8ao{9JFo8N+Bb&xFo*_wEZWiQU`yC6)Tj3{10D-4Lw6dArW+B9cXU>=uC(b z)cJJOr>qfzRhu($rJ%%}+@!%0omPx9r52WE7Nw?uf}LO) zhBuLcoQyhPKz;`byjvI2$wZoy!D~I~g&!uL?S%z{!_g;Z=Gg2XnQZ9iCN#2C`A!Y=%#O0KP4rD1ZgSaws&_0hM2dM0lzXdOE6`|$r1zEN$@X`??2s*P&mO~X<5<o&hzHpa@8Rm`;u;hm=I9gZihL@bg0ix*vI6*U zfFRI;YFu2{0vx&}wpgJ!KNq@34kfHXM*}F-DEKAjrYhhBH41)^!>(%-TvCgZi!#C6 zPH`%SNYrpC)ac>|aCtZbzk0AN$Xt|A3iS&PaCP={^K^BA*sM^aU;ts&Kq#U}40Dk~ z-_Oz4RY9Rf!4P6pjRHs#7Wtq8AFZATFYq!qGB!YTt_dCu08$QLMrf>Oq^Y2Q*G=FQ zp-`h>gw0KmvJkavMh!>lnP@81Ku?okQeqQg`5?;z+i9-^8hVlCmxq>TN}yBJWR+B) z<(U%b6g64ZHaS=)S_xX7eUxSU0WW`a}1KAo-f%ZohzW)oX zj!B5^pDbIe9IT@c5d@tcCd*-jteJ^Ph^0Y}!vxulp!1*`GQ}I=Uh)u{5VdM*-fzDJo3`m5wRMX&JozD>DT=E(Ga^ff9{w za(-?>PG(|W3E9b6Q%@l{wWI{RaXnulEitD!6;jWq=7HUwlb>G<+E{^bHMB{dp9ea3 zBNH_Im0zxq5K|PB7h{lsbP53MbbaWV0i~cZR8XKKRTd-`7sEmWTt$Fv2cKO4YC--9V8AwWkSC!iBrzzuL{ z+(2ztP#|%s8C1J1B;=5*0mx{oG7C6S!7`>YD~Jodt3;SdnGwVlfDNuHgZ9PBNlJ4~ zf{awE2(c}eV_GK1D%HrWkN_PbMy{*Do`=QTtNY5-w&4V--h>31c2I)r`^pxip<)kQqv|vW43X2fSVmbJ+btwY*w2S2wmdU}6E`$g&3b8DagCA%Q z5n~o&St0iAr$Ua)GW! zWLq!Cuw9N(N*}371v<|Jd`=_2))O}uGb^ZeSue-R&&9*V!pg|RAjGmkjtP9G9EisV z=7O8~ATAS#DR~j^Q^h^-$0$tUxO^ylNhG1o60@bM7 z%(nDzzAL>Su9Df&z3@5>%e1 zLdqv-sZ){807qjJ(Gpk*IO0CK$3Q8}3t(6SFE z$SA~eOpZkq+=7FNF$=LAmt)D~S_X+gMj^J#a!l9cSf!>iL9YwK7k!}W5G~fw?oXiM zJ$azi36D8&DuJ%zhqPLfQWZ=fN3(;ET7@4S1dbmfV-a$55A^tb&@q6J28Rgv=AO%l zCvA*m zGIL9FL1USqo0UK(%YZ|SKt!gdmVgri)Y{zCyb`M9NUZS+ce)X!5e&Lz0TRhZnE4WP z$<8B0B!e#5d4z~$P|5uW)+P}FU$XN^4tmLs2>6nnM{;sc;gJlvPX`oN@=xKB3=w1$ zVtFhFJx&><6O@6Og;<`*vE*?bfkZO15ZgyNrY~}=Qky}M47uF}f6)rgn~5cmc_*|u zPOKmyj#Dd=Q&UqQ#W%I17gpfH+-XK}ghGM_+np#P;CpXAA|e!Y@6AU z$uqUevr0WhzES}izL3C%bUPq9pd=r1_#rrxA#yA9HUrSAH_#bVkgF7;5qBvd+D=Go z9U$`s`bDWm`sJC~nfk$yYZpK(A3!}oh=tHoRv~VHMH9HUst_9DrfUJ3DF$=FeP>AD zHZ3tT2RgzGx{olgBr`V^QYRBAJ;2u_fYK=Rasw(B9bl(}d&7{?2H2ph0{r#_16|8l zZ57Z`A_WDqY6+;LLER$QjSNsTG0POtwLDGo(0+~x6X;r=CV6KIU-Du8+e4+n&jnM}rNH+guO0#OKB>o3nz%Jl$J zrmzaJO_XPvBF`#y9_da7d?^rIq=2FckpRJU1L(Lu@EUep_b_1H{sC{k(YEdYSp!}g z33feXNu&a-RR-#ek{9Rj`xjtFVVuP-0=}PqA|kRucl1n@hd1m%cl1n@7lhoXDgwTr zexf||MpY5;9X%7}Wv0L*8*=pyD9&W3z#|(X$SA}zNuEUv+Dd?kF$=LwmS@S}x&?`B zHX*hJ@=S~6A@|cmZW17({=gmCpo9l_QHW(BBAOv$%t9=SdU-+cmwh=Ln* zna+B7xlQm$hTM?@iYxg|@JNOTf(B3Ip#xMROb{{9k!qkw249pd!o(rOwojhvpgiQF zY|u&S_zG?0^V1@VL=I&WvP7JNPv9YQi?uV%FgtEJ7^%5J>@ax6VF9 zQUKkpvrk@77(6Q^0=`>kpS&1ki;@WVZk>Jd5(nW)0dkEFC`Kg@!jl3-kWq+bzdVZ& zct#tf6VfkWJs{6AiIJ-l(vIX5Vml|#bWxsF>Ic%v`UKJgXz&?+#R9fSg{KLuSsmP( z0-dFbKRG~Je(*E%GK+~ghX{2z1}Qm!oJ{K^0Y5dLykr5oedin^S#Yunv7AFB3rp*)5K+Bo(ilL((;M4T4%5w;D?S(WdxP;j5 z%d@XK6guy*!68*Dgr< z;1**0D$mlOz$Ud4oIXJ3#gdXj5T~xfhM@*e!a%rW9R z$B_>cpd@YJI@?u*33QcDmjb-t0A1zNr2tPGpsRek6yRwCbd^t+g78#$+JM~JBgEFN zz}Bw-D>xv6j6y6u3h=Z65d-bLP=Kcmh!|+^g#w2cG;M&z7=_p-D6maIH4v0GCMv+w z21Fr~5X&S5c-nx7fqJV79Hv~mA*~u-A+|*dENd0mq*gJ*4m-n9a6sA!sd>ryDXGN@ zhDH{;Ntq?!ldBX!E0B<;@3^RV5|tk0LKC%kB zBxjETe6S64NzNVxK?!KD40K7(9tG%)RwCd_a`q_5?N@+}tU~U&0mZZYeg)XbDnyV` zh-I$=d}I|O2HG&M09!!H31yKnu zH!da?Q6aVy3Je<+8Ko=~T=PnEiCUSEnpc_&Uzotf#m&Xc0y?SlgaU^k*9=hR5(OSfR{}Hvfu*d)J%Rx?*|!qpbI2UDZ)CiyU#M?9 z^6Fe$1p{kPzJSSvfz~t|f>v8$3L7cdDj0#UX+v&lK=-{ttkBm7tvrJ+p@4O%V6BhT zibU{A3RJtmQ{PZ$B4t*rcgKOe1Yfsch@Qbj!OIkGz%rO9c$vZtSOyaXFH^Xo09~da z3SOpgLjk%>K@_}9;f8|rZDE#iOL-@!QVl)%^c35y^fFp^tAx&g6Alip`fdJaT3|b|M2qySZuCP&CQj>#%twOaz zW}cNow4siIQ7p#Ts3;R?&BQ-MQUI-)_=hMYKx-!cDL}_YMZs$({wYAmMn%DECjKc% zG%7-BVUPf%kocz{*{BGqg<*oA)#wT=g3v+&vW^0DTACtD9@jNU!eSO;o2KWWN?pQ;sT8`!V_2`e1H=z3Wz|WPV$086rLo; z%p$}xSrOXM6J-Le;+U)mZRm+IfmU%$R)mhTiZX##aZFZ(&Z>$sfmU%$R+O0rPhyb8 z8K9IRI}M)1AcBlSEK?Mr%jG~iL1_`Rl3bA`mg@*4iLnT=tx#lIt;j01c_j1%z$qV6 zkw97`u*L=r>p2{K3XoHfx)g@^dJ<@AjZiO27G(mh$yk9%P@pv#D-a0^v?gPPBDCiq z3SN`3LQz@oH&=vC(}Q$E6Vxh@*bYWe zTbNadWrN~S>^X>ndJY>Dp*;staL-|bBDCiq3hp^XzqEyiH~XcTKwmXcsGqF}3F zjNYjf19u%RAW{;j>u>>)l0YML7Z524)OEOkNJ)&KgToaSF2hq2WPLO!K`35^rzD6V zXva7rB|*eM7bPpQ)NU&@)(YBq0iBx!6$7BPGvE$7c>7~AcoSerzCv+wVoqX_0_dO} zP&LH`y4gCl7<%AXF67i3h!}VcTT&`0jlggDB08>N85>*4Q3`f0II@c=ifX7q*e|ga z19uJXBcdAAHMp+`?Insafw~6w6`_0f#K2vH`-;%<12J&d;J%{tV|Y|UTKu3GlX(n} zYKS1C5X%Ea7Rb$qAf2Gng-M9zp(2YpxVr!oV-{k0q{w2x)dt$8D#pYt#PVH{NkQs6 zs22*UFj7)W5;Joki*dpGXF%O4&`eK3K~83B3Wx_e5TqnEmkW{_pgUaj6kNf(@$!p6 zCuxAIIQT*caON)70YwFL-ybM_q*kORmzJcaa4Ep>o|S?zC;@`@cY_LYu!x}nM5q=-p>#M=?ypZpsz$*sRsbzs0_Pb} zhO5+rH#J}hJvlKCR0|}fDwO6Gr$#45Fq33hr~DkSS+w!E@a zD|NwRGztZYnMK8*QaQgoH3c@S25nnqBo=ciq@|Xp7Ac@>D9K2~Uq|B}`GpnAzLma- z1&}0+dtes4<`~|hG(vX=I9KH+7T7>Cln%6=W(O)8VB2!B=B(Vr0&p7*;(c2Mj5eA$ zxQ*7O1nnk^gWG6bO3*f%IJk}0r37uGiG$l{T}se4nmD+P)}^G-s{|Q|0SSOgQ6aW2 zCB0 z=H%dtTksKd=<4YmwcvBzjHw&5i52m%n8j!_iG$lrGY~NgYBSA1#4M=IGy@T{pf=MC zM9hNPOf!@eX2W9^(i8$kkm783%t8big;-`HViqC>x)EB5rJSo3)cO_&ZC72R#3V2E z4Lx4*RhQYRl_+%(t||;Ye`TjuLfc%>T9%~Bi`;Amwsl~uE>J*#oq@g15(hV})*w;@ z6Ppmr8YO6*B@S*{tx*z#T)!g@Zd$ETl9l871c^h?IS4%Ko5J)sRVTiK=~s9 zvd2yjymSn7kf>u`Wr70e_$%nqUnP|VptGStGvE@8LM+EYEg|rTh6Ly|g)5-&1>b#! zbvjJFSRnyqKzwOY4(LQ2P^S|v0d99)fn_ZTaJ%aY ztag?Fx4W(=2|+G%kN~&4t|*C1a9sqI#S)-S@B<|#S*b&KXROqV!9xO|=m#Y@@KNO@ z3h>Ya`4qZ+6W&dP1P^3JG_wS-U;$q zi9?X98&qydf=1#0DX}ywvrGK~of}e;0BY<(4yOY3;la}_utqyL7?V;Jz++nA-Wn)D zpp{farV|~|&Ua|-3$ENiMuF2IJgI=Ht0K?{ToGjO9CURTY-t2YHQZw8+&pH=foyIk zX2~r)0l=D~n0}>J%E>7K?Kj7oY#7;uSpF%oK$>ro;Pp5Elw`q246#Upi^nH3UpN`r3tf8b>ZsOp=h%ptI>kdypcwBp=lF zO-Ih(C=m(G(+V2!Hah&Oa!4G5(i7OO%wmO{#G>?6Xep71)bLEmDM{c0op%9F&7cz{ zQxzarzJTU;QbCuMq^5w14aikykfaT9A1wdpmzES~rhpIZLT;6VO+hWUpw3FkPb~(w z(9w3yr-06AOUVRXy27)7MvkSqCp5&>aMtFGJKHh6bokjL#w9%EUbI z5smSXNU~Kh!qI1dy92cDExp7_0oB1e3OT4^h6W0?#QOu}?EE66dJrw7<8w;l;prBC zu%mj%m<;cLoQ~liQ>b?oYPmogp&(Nvl1$8OLM$s0#RzD)V5Ks=f&tBmtW<_qFrXz@ zE0tAFC`0z9NisoJR|&DLQf6DP4B3AK69k=DsLUbB1v>KqcVpKvUWyhA+}A*99xv5khL=jv29ak+l{InR2ptq=1@b{4l0y( zD0A#mjzreZEX1})ne8B|c2FqnRpyXI*3Qf<#I{eFNM4RAeHR#l)AGpaan%*P$R6nHU#Cg63Y+Nh&c3 zmQTomi|WT9V>)F_{HvDoG_NNq#akCPKac&&nmp+F69yzACf*Le&lm#Ba(R8pzr~rRsNOj-SfKkgMw? znOKF`I#igtRan7S*MSaHNh~f-1znB>T|9=~b%hK9l;k7tCqXMgi$N_*P+OG1NC(al z4rl`~IlrhVwYVTZ54_w2&(IjQ)*!}8GoqV>xuwM=pi($9PXT&Ix1~AHE6+nly+wlLe!@0EuF73W0i$cBuq1P^)08Kuk&j9mF*Qky6;$ zgji-EQVQrGt{EySTF{gNI*4nAimo0srGO6NnxUdU8=g`i=WBr?++a35r9cEh6XPn- z6=sr55HZlixC%=w*G@=EVHaXsslv2Ig;i?BNZ1*GSQLYrT8JqoKz%+OtNda8FQk?J zNR1vtg<5>e{_$%zf@;PrZ$Kw~twf|Ec2H*rk%~YkeXUec(T1iXcF?79Dth|RR0KNd zYo&_88h9##9N+~?42EmqsR$y-D8#Z#g+&UyCQ%Y`(${Jg*hXSWRw0&+DnoImRT4DQ zx={r>(<;ddI*NXy3UsDb5jC z2kBEBaFcM&aY!)=v20c05aXHx%CJ(PIgUdrOh;89oBE;qj;TDyf#j%xpX7i$2KOvS zv{E8uniD-6BlJS*0^Bnl;C^U)ZenpZe69mLHV#_&P>@)XQEa6Ew*@o+gfh|~1)c#p z1j~z3;2DrZD%>jI%qIn&0Xd|i3MtX0z>}$mU?sW~cn0K zMVKI?5X)guUKHlyVu6V<3$Y$iVQFFH`TV2?JA(O7)_1mz+^Pt0%;2sNPNlbBR5?C>$S%}o!0|z9;c=ck?A+{OCXw&$h z#wIAE=ztm-Xth1+u%JS4YO0k&MoCFQv6a3)e2qd%YMDO#az41QK3e4KD}$Y)3yM?F z9b8zhl|dvAa9r!_E1;|aKuCkz5TK3QXdwk^azd^KPsq$m0I!lnbA=wl5aN>{X!|Op zwt>z6CxRNENgyIw51Q7oq(aaX?o~u81Wn;yRe>(0lmgGMTvdTi=}UpLP~;=HYjb9A;fG@A(#eRbO1_?nYp>8;Oosn9DGTWiA9Lz1tMvJHdVYp zBu!A~`-KX0b%7Lkg6D+_bnl`Rc!KAJitKB6(uB<0fD)M8Yk1Ox2r>$>yi{S4h9*sj z7_$)TD;1WDj9lj-1thZ&+fNmyKPs$J`$krq7f}|0%R8cqDo8nmb#(x^e1+7R(95Yn z1s5dmf>H{!(29k&U~#ma37G;)RXPevDM~PN&|AV%;H?lp5s8bLO^D?uqV@$1&HPkR z)qz&Vpke=?D*6V{at}1@|5L^A4?Ju1YQ@7HJ{2R#k?pYK&6Z z3h+C8(OSZ=O1>C=;}3j>2+!(6xM{dr!_tgGEUl^>Vq9xMSy>v?8tzkNnyAVuH3xAD z4|;1DwtrR!=fDM$y`ToK4rD$Lc2o_x*1)!S2k(q1>VOz1JP;#qpbH1U7s%rt@F2W` z4en^%4Pzq%BV%0yQ(Xh&5JLki0}zS=WoY!Ak1#43JVl)np9q@v#5FbscRr%644)r0 zG&eH0w6w4=Fu-V_NQ0ZneX7tQGHGx#xla|^OqK>WllxSm&17kCGr3O{I`AP4ZYKAs z>Q7XK43U8ZKtxGMFHv5KF%*bl?Lf#w^4#K^3}CNE)Qums7ZD5W{40m4Xy(8^%W3a0k?8dR=KD{5|E*;fnf+Z^;j7gM5!x8!}aLl zn88L^1Mv~=ju)~~;OYaBlFZDFOiWBH3=9;oBqh+|q1mb|vfxr(8r-;@tttmuA|efL z+|E{oE)kIiH*ROEsxDB4EFJ<0K+?`^Rka1Gki|nVK}I2#IjYblA}}##A-1`yZ1YvE zAO}B6gIc$%R9V)mvP*#uegrQbLTTEfRiRN169 zabQ_7hOe+dSx^Lu9e5)GIwqqqx?~JKSL9oXnk+%93=n|}tKLw86k1T>so`*}BtzIl zd<~ZlYE)Gt){+^b)^MQQg0(^>#>J=$BoN**!cqRFmq6D{;3kUfd9 za0?`9YHSEDs1-o9JgH%T;ysi_YKS1f)ph{~0fCikh-fBml?B0|fcX|{P{5iy;GjSm zVwPqCExWjgsI@>V1}-8RJfLM47ZD8}(6Wn*s;VDUA&c3h;a6~7Qf0fY3R%ns6J!)( zxvUCZb^#M(7Gk@i%5hB6vDZW{S*jm(B z7OSyIy#g2C=!@A9agN-GA4;p)xZu?{C9By$=aQyEj|fUdSpZDCz6wgwh@}OJw_vAU z3p5?Pnw`?3oe1}nUd+SR#?hdhM_WUNRGVQf-C#u@xO5|~pvN8}lok4*0HJZIj~Os9 zzhey;e8oO!8BvQGyq^nNM%1DP@2r595w)nnJ1d}NL@jD+i`8JoK5Q9Ls~TIE8m!od z2r>$>w5f4ObAi?{OGDQiF$=M^t8sLyK~~#AMHz+Iy4BbwplSyd`#oy#f){KgXbp3( z8b`kxtl$NUG6}IwRAZZlsvT79Pf~*yyby(;gHY8trm8_!+d*w-7Gj&O#x@64JE+*7 zp~fMP?0C>>yP0Ynv(+H0?V#FOgxKb)u`NQ?4l4HNsc~o`YX_~io3F;PP_3BjE~KNz zCd9T@jb(!xyVMybP^%r(`=D+|4W;k`HSW>+YU$t}Pl5tf`)Z&@G1jIqwfk!j*W>K3 zfvz81s|N3{fmS`ORfG4}K&zhCs=@nfpjA(6)kHR^!3J<3iWWE5gq zrv~q@LByDaSk|kt1#!)WjP9`uvF%o4+NZ`U)y1p;@9)C8R-oBQ@J>Jq`rwd0G}@ps zY^4uu92zyck+*+A27DAU!Fy|v%z~MKW!xBb>Kwce0z6}m*btAcdjv}fP&*72GV)7{ zK<=Yb88xHx@U#WF{SYK*d>)>*AcBlS zEN9hNq`_Z)L6tIr;|vt3b9;K8;Uc9(x92bD{9c0LTT_!;T1LLOrbP* zrtpfIs07y}CN3rx86mb?Y7BSP7^Rw-6cWNR^K-zb0~CYL224~)&dGyb`dR6cUj<3-^p3TRPLxT zsYsP8B*2Y?1rY3HLg+LCcx)KD{sVlP9JqqP7H$fN=||84nv#65N`(0eDXAr?$t9^N z(DMXkz;knVU|B;3JU4eojT>?Rhzxjc?v9!$WOKF*cy8{Fnvyct4R$Uj7Fi*-E_H^} z>WtvsUhr53&6z<%rC1?Rp*S-yJttKmGq1QLF)tY$QAMc=&?^kV$pReIItr;3AYH}a zGbTYAAyJ~EP@Y%}UL=&8TB4`mSptgUvP{rzTA6ui`9--9BNCJHOF@Sw!9+zr%SsYuK%$Vt@!trP=or3LNNO;vy#SP6;=XiB!ix+E>MIJF2e zn+sa_Rbp(UkeCjd+Q2mto1B^hnll5Hb)bcc3JRGiRtid`#i>PliRr0I&{fAEDQK8x z=I2={D3xSn7Ax2&7+NUUE2x9*gf#;aOB8Za6N^hga+RrxMa2sFIVtK2RtoCsFw;Pz zbnu}mP&6dxm*%8^A|s_V8N72C$+z&phWaTgI4v)g379S z>P+%dEi4KN;2?(<911FLK{*br5R$b(x79%IVuqHk;0x+9ixnV= z2y|wxLSAYys1yb3(g7X7SDulXoS~4MSPV&BV57m?LNfD`b4pWELC0!>i?@PA@Jtt2 zXE7)tr+{)CwCqbvE&-L=U`dE$uoVha$;Pll<-iFpDL+3)Ara4kPtb|u6f8X@QnOb{ zYDEdS;^RW9oxrYuc)MBwnlBWpAvH=oKQhwu1WZ*=Dk;W8?OA?DpKq&=& zNDp|S9H=1x-yH`I3rMoTW<03mPcAJk$zQ9&n`t@qIkV zQ?T*|rOr_VS333z>Phgi=8~e)RCRFG0!mo5v50g8BkMG7IX~i0Z?Y!%LvJA@tG;${02&n z&{NSfQ&RIvGSf0s!J_~m2}tt`lJ;RuFnHStR(@xK22D~GiW75F6%s(Eg4$i6mK4&S z_{=twd@uxvf$H152y>lf&#Rx;{YfqzCeP4S%~EbC@7vXDkMOi0uG8qysbP)D5Rto zCl_TVK^KzeC6<&HrNVMG#Fd%(d9aWH4c;X|YOa*jw8YY!60|)J@KOh&H8n*65<)tl z{o$D<;L$&Dp#{leum&f@De0MIsgP?OKn+e%Hc&vcfFTBg-2zPzpz}13Ac72ZC(#i^ zkb#C(k062!v{Lp6D9F;mB{ho>%W-uk8L3ExgtYvk`0Uh5IGg3=3z^BrrBGo;h zc!O56NtFtT3b}~|pc9C76p9j|suMw-HSj0`v?n49zUSsR!jYhRZjP%%58#(&0uA0C zR~Hb6`Uo`fd0bse8hlBkET<65Np&V(E*_AJIT=7aZXSqAP6kF)E)$dsZs%|bv7Az8 zl7MLBVgT{D#ku-GeONgmwu|ZvSJWA${xd2hfUAap#FC5zP?Hx_u!FiN;5JD~zCt3b z^#ZPilSMREJ_96Q4HP(taaS930#zk=gI4ve-<`qNE-Gz$7 zvIr=!Ku3t9*Sbad`6b{RCW=y%ON&6=MTM07^1SlIB3Lg~4&2tb3`-kw;I_tPb#BNp z_j2I2#$|OG6>yBpf!i9F)zvk)Rx@)kvB(Rt-BxGl)L;a!6^6&7E9908r0B#RC!jEb zhYh?p3y%SC0|6Az&|%!-RM5yqY6>I)fQASlsUP{K1+X4aQz2C$p|~^&RHzh`fX5z6 zQxzZvfD#wHi31BqaEXywqQ|A+2uT#+!WO(G3fwLN_o+)$VQH}xnwX%O4^*t6jVpjV z1(4(8VCP+;y9ZpHr9xZ%;6`Br)N9~_&x;jG^AdBDGSf@*ON$kXQj7C*N@3$Vusejn zNjEc10V%O6Bo?KD`}B!riJ3W|UNtn4$ukPE+*aq1;PU5UVUY(Vvb*X`N>Yw62SD1b z;1U~DM}b2Ub~rn@r2$H0@YXIoXrN_`JUAWQg{32T@ad;_)uETO$%E6;U3Edo;!=5V zI=ZW_pvbifRCmaO+N=-NnG~gFGs2ydn45}}>XDoTsu1xyNC8|hKx+xeC`GqH}cneepWTt^@QxuDJPO(q_D~&KNy>wl%|28Yf*ipo z4{k3!RF_xaIsoeK$%EPpZ`GMxq&C93d(eo7rbuW72x`AT@^&WlrYb#!aL_1RDL4-o zrKW+hLS}v*D5+<{j>gP{4Cdserlx@FbVxXWYXVSifwr2#r%B`{W~VA-mM9eD7Z+zH z<)nf;mPLu+1I-j*#S}OoK&=o+>4j8kK^#*8(Ew^&z~yv6M@Qsl=B1>S=74;VlwXpe zP>`6MothGD7z-|oQc}SeDy61G8^tPsuWtvLP@J5Rnwwe-%J0#Ju{ic978K>D7bWKA zX6B{qrk8?7s`Oz7=qp2P;Zle;B1?OEQGRJbF&CGDpPrMRm4ahVNd~ykhKPf+RZ40> zQ7Y(4pcGKJr=pDr!u^HrDNx|&<(GhFm{LFkXL+D7O06i!$xO~H$*EKTC(+D2$f#>_ zevX1uX5exRkdZ`HX?zy-8CxQ+H!-P9h&WdT=nl2eNnl2S{`Q&aP}6kviS3YmE&si0H) zU?~8Uc0k={dGPL~PlzA_?OytX2qI9M?322wEi{NgZL&}54vt)NAwk3<#PSW4L;Dz^ zmkeM85-5NmVSqD;K*a_ma5B>rD)UPf$`c{Cz`=_l*kaSnJkXdX!61W2Kj_NBZwSAE z*06m;_zl#Q`-X^qP*d)kx|1_k4XDfqZPECt&eWm7ER};&JtD@x5qB0rDlpJ!EUbYE zE*>)>JCNb^0H_WJ9kvf{sln?&s5Wo`0&_WdL3E5`%m7*OBhw>pccl$AmP*e>LS z0Er3)x=D$k@nx_SD5^m%LAd!S1~aj;3bFiFXMvniC(i_0(D55l^MDp~{8kqh;j#f+ z3pycHokdJaPay%+Cx?ufLFI;$#MKC8REWWA@`<6!PFw z;o<}x$)8l3nUf;KkyM%nW=Zja+nliZE+GX|17jmo6BC0pi!>t>15@K90~2!-3zI~% zq%<>wBy%$f2Cjn!Tr$N4nRzKWsrrfOsd**(Wrh$2nI%G^Tx?nSNgxwI4&e-9&B{;mOcAo=l1xra&Pa_fDJlgGFvk~^Cgo%% zgCen5f=Pi{gEL4BtRMs;9{`s2POU5!GUO5k%fy4Ge&P#?@+&Ii3-XIfz)s{15&}yG zr6#5XfTRNQi%Nt{xx~Su@udYh`H3mWnm|tH4H5xMhk|7h+ElpsQu2zCRkMNvlP4vw z7_LgGfsu>TwIVe+7&LGxB*ev%Sd?BS!3g#Pm@Op0#hjW~2K5$*2@**zhXpEga(N0! zA84jX0^}afAQli?NS%uxG*_QrS`rUkrXc}w03TF17;+A(k^&bO#04q&r6nMvQW8rN z!7kTe%(YDH#Ad~$wDD%cp-AP$g#bAC#ykO&tm zc&<$Xl42knkQR`mQsVPVAu+)b!~quYFD+5p&&b8+m|T)smROSNoS&DLnJy&D#gzbK091&_Lqy^;5{omS_VIzlA%=J)7H5ExA}Fw+D#3|>Gl&Z$ z1W_a;#l=yQUyzepmYM?%T{xfk5KiY3LG?gTYH?{!2_bJF+YJdWgx&CPgk&v9I4W6j zaXFP{=9GX2HHtxD4eDbQLz5FLm?I>{#a5J>Se%~+4H76%$&`!B8I-A9iV`#Pu=x#? z1$l$eQ_l`YE&+r-sK?;01L@%mg1Amdk&6eMQ{oFtQ%h4*!1nV7af8JI!9oZniJ%Bi z1xE->2{;H-g;crtz&fDWF*OAoFT6p#U{OdeOie+U1R3LprhSNIZi$&WsVPcaTpR)U z<*7yaX=zGoTzoJ#K0hd};NW71j>-#(b1`M6KvJI*`z{ZbA|?UGAg0U|Azd!k3{a5@ z_93^CxE^|utOqV}gCMGew76I@^D-d`pW8?bQ#Gh83<9eWvf|7oqQWFHWEEudFS$k?xkp!CpqXs_h zsYOL_?Vw`A98@f%CdY${0tt4oW-07;gGv!lk&&7VVhMS2@q<%CVo542pK^e;D`08| zrxAEL5yS`62+hqgTw*X0s8R7n;H)UY2{uuUpow7BK_W0yAm&0-j4c;8s9G6ave*2$#kO%_xNP`XBs?twS&9er#o5~EmuE=n)39&FRA4AO|;u$;HLQ!XU)L!NAMN#l^+U z!oVoR%)!CS#m&XS!ob8J#LUSg$Hl|N3gWPWcpNYuqYyJErzDKWEX2$yBMa5WD8$Ug zBn_2fWCihH+89A{oMJFJW+7%SNhzo{CLv~SCQ+yq6Dx=Z)5gRo#LUep0Fz@DV&)bW zfofwGV&-9jIfR)N#Di&LW)x!P;S_;tV+N@dm*4{VnNf&^pFtGjXONrtIe5UvF@pRo zzy#642pSn=7GQ!H$H)W<905*#uyKr_aY1GQAz`RCMj>WFCIhGxBMXSf1>>;_F$;1F z!gwH+LfSAMvkcugHZyS z20%kX%#ut}V0)QBS4c8T!VLi>BuTg-ph%Mxf}|EE(9k8bq=Y2YB}_uhQcRHWW?}*b zqZBken7|fDaq>VdU}6?xmf{zHYGW2+mSz%yN`Zn=8mhCeT%L%+dld zf3pZN%P`48wSj_B2CfYhj53^{c_olG7G@!48EF}CFft3V$TKKHf)Ql6Jckh270jSu zRDimI860p5P~(`v0jB_U1v5C{6a)mp#<2*oC^M)+j00J!3^k4gWSk1rI2N#RDp2EC zz{aUSjbi~Drvf#ORft8MK^I~iE29vzItL_-SV6{VFhQb@6>OXa6Fb=JtYG6bI3U5! z3N{YZjRzGZte}LV$s`O_2#QEes4G}OgV4;HoP01jW+7%xK_RXspaCvcCMF?f9VSt! zX^aY>MVgSoyNpzY^wQMg5`}E=`aJMBcyUQ;fkJ+sLS{*^0%+C>)Dr+rk%8zU9fe}Z z0?3T~;u6T9W^ztqadBo^C3rXwG|K`~2-1K&Gntu}mReK<8sP&q2tl*F;Hei-YXsa= z(Ssy!R&Wy2VPb=N0pwX7c1T&p3eKrI!Xn^UViRJ~XE22XD;uK_vp$CyIMUfb!D;}W ze`IB00|%=CG2|G4$uo{SoL$xsqF&je5XEr8Kuo}X(f%1nTw0vd*=MO_@ z`OF3iRU>GL$p&`35nLO{@kY=RlMU>6BXJ3+ZOlT<#?ZKD18ulxHil~hC1GPuNZhl5 z^M|pBC{!DZ5VHw1f3Puu@`njr8z`xoK=TJ16DWU}K=TK?5Q{m3H6*Cm8HJe5Ib^`b zv4euj0vc58;GnXA8pjR}Dhp^(v4exk0vc58Ab(jx9nTI9DoeOFP*7Py9nTI9Dod#2 z*+D^N1x>i@;GnXCYXcc+1x>i@;GnV+6avRHhY*V`g9F6z9E?KDw$R+j0dl+@w8-TE zmyveR;*A6B6+3A0#sT(LyN*o1CHX!wGhUGt@Xvuq&LQ@xuvrg)=mMxP(~T z7?sep zHc;;VnE^t!zf@=dM zWiM#e#|2KxUeJ)|Vqy_u_J-RAO3L1FZJ?y=4Yv)Hl)WXu%`X;iA!a`YP$V$3a5D-q z`*8?>BZNnYC4eCa5`sL8Ld*fs!hi=9f`QNwT>M<@EWGUPoS^0&SU`Y_6Dq*SAjBNb#><5y z!YITX&M%5A!X(5TF0PC$!Ysrbu4;rV!Xm^RZsLe6!YafZ?(Byw!Y0HV!6uF@!Y;%d zA!&##!Xd;QVeEh`!YRZY;pC4j!X?BU$tHy?!Y#xcDWirg!Xv~SsbR^*&&2`tDU%R$ zGz%oj@q)rRj>!sKYV(4_ISv}myx?$-syr83~nBxR2A@(paaI|ys zF$pop30i;)C|*#qOMuq6yiAO2Ld*%!Dv}qRJrg(}%?Ms_N=|?_BX~h6IgtrcYw$8L zu?aCJLTeOWaHA!W9p)NPN>1bh#V!jAFNl)_ZFTW7F|!FVCo#bkf>Lr452Ui@1*haB zXgiA+l#-L7nVuJPxBzo96QohZ%LGcv$-Iz83op1(PL_rgzPz9emIBwt$|l5|0@ns= zC#1l&u`&xWr@*zb2{ETaD;!=XHZTuTck+U!C74sWA?n4mJ=E76_oyI9*&9CdVwq zoNizUjWfN z6RwSmS%^6ku8muWIg1I>3*cqqW&`mcO;27XP$`im3~A5vGI28tF=uILK^@5>#GDNe zNgg&34;GT3LL{3T(oo=K;$aqI&K83d%DhaxLd-c#YEW, + #[prost(uint32, tag = "3")] + pub cache_read_proxy_port: u32, + #[prost(uint32, tag = "4")] + pub cache_upload_proxy_port: u32, + #[prost(uint32, tag = "5")] + pub dns_proxy_port: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStart { + #[prost(string, repeated, tag = "1")] + pub argv: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, repeated, tag = "2")] + pub env: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "3")] + pub cwd: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub user: ::prost::alloc::string::String, + #[prost(uint32, tag = "5")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStdout { + #[prost(string, tag = "1")] + pub data: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecStderr { + #[prost(string, tag = "1")] + pub data: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ExecExit { + #[prost(int32, tag = "1")] + pub exit_code: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + /// set when the guest killed the step on its own timeout timer, so the host + /// can classify it as a timeout rather than inferring failure from exit_code. + #[prost(bool, tag = "3")] + pub timed_out: bool, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ActivateConfig { + #[prost(string, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub base_config_hash: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub user_config: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub toplevel: ::prost::alloc::string::String, + #[prost(uint32, tag = "5")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ActivateConfigResult { + #[prost(string, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub toplevel: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct BuiltPaths { + #[prost(string, repeated, tag = "1")] + pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "2")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheDrain { + #[prost(uint32, tag = "1")] + pub timeout_seconds: u32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheDrainResult { + #[prost(string, tag = "1")] + pub error: ::prost::alloc::string::String, + #[prost(uint32, tag = "2")] + pub cache_queued: u32, + #[prost(uint32, tag = "3")] + pub cache_active: u32, + #[prost(uint32, tag = "4")] + pub cache_uploaded: u32, + #[prost(uint32, tag = "5")] + pub cache_failed: u32, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Poweroff {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PoweroffResult { + #[prost(string, tag = "1")] + pub error: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Message { + #[prost(string, tag = "1")] + pub id: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub hello: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub init: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub exec_start: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub exec_stdout: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub exec_stderr: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub exec_exit: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub activate_config: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub activate_config_result: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub built_paths: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub cache_drain: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub cache_drain_result: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub poweroff: ::core::option::Option, + #[prost(message, optional, tag = "14")] + pub poweroff_result: ::core::option::Option, +} +// @@protoc_insertion_point(module) diff --git a/shuttle/src/host_proxy.rs b/shuttle/src/host_proxy.rs new file mode 100644 index 00000000..b19d2ff1 --- /dev/null +++ b/shuttle/src/host_proxy.rs @@ -0,0 +1,92 @@ +use anyhow::{Context, Result}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::{JoinError, JoinHandle, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +// this implements a vsock <-> tcp proxy for communicating with spindle +pub struct VsockTcpProxy { + url: String, + handle: JoinHandle<()>, +} + +impl VsockTcpProxy { + pub async fn start( + name: &'static str, + bind_addr: &str, + host_cid: u32, + host_port: u32, + ) -> Result { + if host_port == 0 { + anyhow::bail!("port 0 cant be requested"); + } + + let listener = TcpListener::bind(bind_addr) + .await + .with_context(|| format!("bind {name} listener {bind_addr}"))?; + let local_addr = listener + .local_addr() + .with_context(|| format!("{name} local address"))?; + let url = format!("http://{local_addr}"); + + let handle = tokio::spawn(async move { + accept_loop(name, listener, host_cid, host_port).await; + }); + + info!(%url, host_cid, host_port, "{name} ready"); + Ok(Self { url, handle }) + } + + pub fn url(&self) -> &str { + &self.url + } +} + +impl Drop for VsockTcpProxy { + fn drop(&mut self) { + self.handle.abort(); + } +} + +async fn accept_loop(name: &'static str, listener: TcpListener, host_cid: u32, host_port: u32) { + let mut tasks = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => match accepted { + Ok((conn, _addr)) => { + tasks.spawn(async move { + if let Err(error) = proxy_conn(name, conn, host_cid, host_port).await { + warn!(%error, "{name} connection failed"); + } + }); + } + Err(error) => warn!(%error, "{name} accept failed"), + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_proxy_task_result(result); + } + } + } +} + +fn log_proxy_task_result(result: Result<(), JoinError>) { + if let Err(error) = result { + warn!(%error, "proxy task failed"); + } +} + +async fn proxy_conn( + name: &'static str, + mut tcp: TcpStream, + host_cid: u32, + host_port: u32, +) -> Result<()> { + let mut host = VsockStream::connect(VsockAddr::new(host_cid, host_port)) + .await + .with_context(|| format!("dial host {name} cid={host_cid} port={host_port}"))?; + + tokio::io::copy_bidirectional(&mut tcp, &mut host) + .await + .context("proxy connection copy")?; + Ok(()) +} diff --git a/shuttle/src/logging.rs b/shuttle/src/logging.rs new file mode 100644 index 00000000..ef9222f8 --- /dev/null +++ b/shuttle/src/logging.rs @@ -0,0 +1,49 @@ +use std::fs::{File, OpenOptions}; +use std::io::{self, Write}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt::MakeWriter; + +pub fn init() { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + tracing_subscriber::fmt() + .with_env_filter(filter) + .with_writer(ConsoleAndStderr) + .init(); +} + +#[derive(Clone, Copy, Debug)] +struct ConsoleAndStderr; + +struct TeeWriter { + stderr: io::Stderr, + console: Option, +} + +impl<'a> MakeWriter<'a> for ConsoleAndStderr { + type Writer = TeeWriter; + + fn make_writer(&'a self) -> Self::Writer { + TeeWriter { + stderr: io::stderr(), + console: OpenOptions::new().write(true).open("/dev/console").ok(), + } + } +} + +impl Write for TeeWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.stderr.write_all(buf)?; + if let Some(console) = &mut self.console { + console.write_all(buf)?; + } + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.stderr.flush()?; + if let Some(console) = &mut self.console { + console.flush()?; + } + Ok(()) + } +} diff --git a/shuttle/src/main.rs b/shuttle/src/main.rs new file mode 100644 index 00000000..c85872a5 --- /dev/null +++ b/shuttle/src/main.rs @@ -0,0 +1,62 @@ +#![cfg(target_os = "linux")] + +mod activation; +mod cache; +mod command; +mod dns_proxy; +mod exec; +mod host_proxy; +mod logging; +mod nix_config; +mod protocol; +mod session; + +use std::env; +use std::time::Duration; +use tracing::warn; + +#[macro_export] +macro_rules! cfg { + (@val $key:expr) => { + std::env::var(concat!("SHUTTLE_", $key)) + }; + ($key:expr, $default:expr) => { + cfg!(@val $key) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or($default.to_owned()) + .into() + }; +} + +fn cmdline_param(key: &str) -> Option { + let cmdline = std::fs::read_to_string("/proc/cmdline").ok()?; + cmdline + .split_whitespace() + .find_map(|tok| Some(tok.strip_prefix(key)?.strip_prefix('=')?.to_owned())) +} + +#[tokio::main] +async fn main() { + logging::init(); + + let args: Vec = env::args().collect(); + if args.get(1).map(String::as_str) == Some("enqueue-built-paths") { + cache::enqueue_built_paths(&args[2..]).await; + return; + } + + let port: u32 = cfg!(@val "VSOCK_PORT") + .ok() + .or_else(|| cmdline_param("shuttle.vsock_port")) + .and_then(|s| s.parse().ok()) + .unwrap_or(protocol::DEFAULT_PORT); + let host_cid: u32 = cfg!("HOST_CID", tokio_vsock::VMADDR_CID_HOST); + + loop { + if let Err(error) = session::run(host_cid, port).await { + warn!(host_cid, port, %error, "agent session failed"); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } +} diff --git a/shuttle/src/nix_config.rs b/shuttle/src/nix_config.rs new file mode 100644 index 00000000..033b1721 --- /dev/null +++ b/shuttle/src/nix_config.rs @@ -0,0 +1,223 @@ +use crate::command::{self, Spec}; +use crate::protocol::v1; +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::fs; +use std::io::Write as _; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use tempfile::Builder; +use tracing::{info, warn}; + +pub const SPINDLE_RUN_DIR: &str = "/run/spindle"; +pub const SPINDLE_NIX_CONFIG: &str = "/run/spindle/nix.conf"; +pub const SPINDLE_CACHE_CONFIG: &str = "/run/spindle/cache.json"; +pub const SYSTEMCTL_EXECUTABLE: &str = "/run/current-system/sw/bin/systemctl"; + +// nix lives in different places depending on the guest OS (NixOS system +// profile vs. plain /usr/local on e.g. alpine) +pub fn nix_executable() -> &'static str { + static NIX: once_cell::sync::Lazy<&'static str> = once_cell::sync::Lazy::new(|| { + let paths = [ + "/run/current-system/sw/bin/nix", + "/usr/local/bin/nix", + "/usr/bin/nix", + ]; + for candidate in paths { + if Path::new(candidate).exists() { + return candidate; + } + } + "/run/current-system/sw/bin/nix" + }); + &NIX +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub struct RuntimeCacheConfig { + pub read_urls: Vec, + pub trusted_public_keys: Vec, +} + +// configures nix daemon with the configuration passed from host +pub async fn configure(init: &v1::Init, read_proxy_url: &str) -> Result { + let read_urls = vec![read_proxy_url.to_owned()]; + let cfg = RuntimeCacheConfig { + read_urls, + trusted_public_keys: clean_strings(&init.cache_trusted_public_keys), + }; + + if cfg.read_urls.is_empty() && cfg.trusted_public_keys.is_empty() { + remove_if_exists(SPINDLE_NIX_CONFIG)?; + remove_if_exists(SPINDLE_CACHE_CONFIG)?; + return Ok(cfg); + } + + fs::create_dir_all(SPINDLE_RUN_DIR).with_context(|| format!("create {SPINDLE_RUN_DIR}"))?; + + let cache_json = serde_json::to_vec_pretty(&cfg)?; + write_file_atomic(SPINDLE_CACHE_CONFIG, &cache_json, 0o600)?; + + let mut nix_conf = String::new(); + if !cfg.read_urls.is_empty() { + writeln!( + &mut nix_conf, + "extra-substituters = {}", + cfg.read_urls.join(" ") + ) + .unwrap(); + } + if !cfg.trusted_public_keys.is_empty() { + writeln!( + &mut nix_conf, + "extra-trusted-public-keys = {}", + cfg.trusted_public_keys.join(" ") + ) + .unwrap(); + } + + if nix_conf.is_empty() { + remove_if_exists(SPINDLE_NIX_CONFIG)?; + return Ok(cfg); + } + + write_file_atomic(SPINDLE_NIX_CONFIG, nix_conf.as_bytes(), 0o644)?; + restart_nix_daemon().await; + info!( + read_urls = ?cfg.read_urls, + trusted_public_keys = cfg.trusted_public_keys.len(), + "configured nix cache" + ); + + Ok(cfg) +} + +pub fn clean_strings(values: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::with_capacity(values.len()); + + for value in values { + let value = value.trim(); + if value.is_empty() || !seen.insert(value.to_owned()) { + continue; + } + out.push(value.to_owned()); + } + + out +} + +pub fn clean_store_paths(values: &[String]) -> Vec { + clean_strings(values) + .into_iter() + .filter(|value| value.starts_with("/nix/store/")) + .collect() +} + +pub async fn nix_version() -> String { + let spec = Spec::new(nix_executable()) + .arg("--version") + .timeout(Duration::from_secs(1)); + + let Ok(output) = command::run_capture(spec).await else { + return String::new(); + }; + if !output.success() { + return String::new(); + } + + String::from_utf8_lossy(&output.stdout).trim().to_owned() +} + +fn write_file_atomic(path: impl AsRef, data: &[u8], mode: u32) -> Result<()> { + let path = path.as_ref(); + let dir = path.parent().unwrap_or_else(|| Path::new(".")); + let prefix = path + .file_name() + .and_then(|name| name.to_str()) + .map(|name| format!(".{name}.tmp-")) + .unwrap_or_else(|| ".tmp-".to_owned()); + + let mut tmp = Builder::new() + .prefix(&prefix) + .permissions(fs::Permissions::from_mode(mode)) + .tempfile_in(dir) + .with_context(|| format!("create temp file for {}", path.display()))?; + + // no separate sync here because we don't need to be crash-safe (this is an + // ephemeral vm) only atomicity is needed + tmp.write_all(data) + .with_context(|| format!("write temp file for {}", path.display()))?; + tmp.persist(path) + .map(|_| ()) + .map_err(|err| err.error) + .with_context(|| format!("install {}", path.display())) +} + +const NIX_DAEMON_SOCKET: &str = "/nix/var/nix/daemon-socket/socket"; + +async fn restart_nix_daemon() { + let systemd = Path::new(SYSTEMCTL_EXECUTABLE).exists(); + let spec = if systemd { + Spec::new(SYSTEMCTL_EXECUTABLE) + .args(["try-restart", "nix-daemon.service"]) + .timeout(Duration::from_secs(5)) + } else { + // on non-systemd we can just kill the daemon and it should restart + Spec::new("pkill") + .args(["-f", "nix-daemon"]) + .timeout(Duration::from_secs(5)) + }; + + match command::run_capture(spec).await { + Ok(output) if output.success() => { + if !systemd { + // init has to respawn the daemon before any step needs it + wait_for_nix_daemon_socket(Duration::from_secs(5)).await; + } + } + // pkill exits 1 when nothing matched, ie. no daemon to restart + Ok(output) if !systemd && output.exit.exit_code == 1 => { + info!("no nix-daemon running, skipping restart") + } + Ok(output) => warn!( + exit_code = output.exit.exit_code, + error = ?output.exit.error, + output = %output.combined_lossy(), + "nix-daemon restart failed" + ), + Err(error) => warn!(%error, "nix-daemon restart failed"), + } +} + +async fn wait_for_nix_daemon_socket(timeout: Duration) { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if tokio::net::UnixStream::connect(NIX_DAEMON_SOCKET) + .await + .is_ok() + { + return; + } + if tokio::time::Instant::now() >= deadline { + warn!( + socket = NIX_DAEMON_SOCKET, + "nix-daemon did not come back after restart" + ); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +fn remove_if_exists(path: impl AsRef) -> Result<()> { + let path: PathBuf = path.as_ref().to_owned(); + match fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("remove {}", path.display())), + } +} diff --git a/shuttle/src/protocol.rs b/shuttle/src/protocol.rs new file mode 100644 index 00000000..979e314d --- /dev/null +++ b/shuttle/src/protocol.rs @@ -0,0 +1,212 @@ +use once_cell::sync::Lazy; +use prost::Message as ProstMessage; +use prost_reflect::DescriptorPool; +use std::io; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; + +pub mod v1 { + include!("gen/spindle/agent/v1/spindle.agent.v1.rs"); +} + +pub use v1::Message; + +pub static DESCRIPTOR_POOL: Lazy = Lazy::new(|| { + let bytes = include_bytes!("gen/file_descriptor_set.bin"); + DescriptorPool::decode(&bytes[..]).unwrap() +}); + +macro_rules! impl_reflect { + ($($t:ident),* $(,)?) => { + $( + impl prost_reflect::ReflectMessage for v1::$t { + fn descriptor(&self) -> prost_reflect::MessageDescriptor { + DESCRIPTOR_POOL + .get_message_by_name(concat!("spindle.agent.v1.", stringify!($t))) + .unwrap() + } + } + )* + }; +} + +impl_reflect!( + Hello, + Init, + ExecStart, + ExecStdout, + ExecStderr, + ExecExit, + ActivateConfig, + ActivateConfigResult, + BuiltPaths, + CacheDrain, + CacheDrainResult, + Poweroff, + PoweroffResult, + Message, +); + +pub const PROTOCOL_VERSION: u32 = 1; +pub const DEFAULT_PORT: u32 = 10240; +pub const MAX_MESSAGE_BYTES: usize = 1024 * 1024; + +#[macro_export] +macro_rules! on_payload { + (ref $msg:expr, { $( $field:ident => $body:expr ),* $(,)? }) => { + #[allow(unused_variables)] + $(if let Some(ref $field) = $msg.$field { Some($body) } else)* { None } + }; + ($msg:expr, { $( $field:ident => $body:expr ),* $(,)? }) => { + $(if let Some($field) = $msg.$field { Some($body) } else )* { None } + }; +} + +pub fn kind(msg: &Message) -> &'static str { + // todo(dawn): maybe eventually we should have a custom protoc plugin for + // generating an enum, right now not worth it, when we have more needs for + // it imo we can consider it again + on_payload!(ref msg, { + hello => "hello", + init => "init", + exec_start => "exec_start", + exec_stdout => "exec_stdout", + exec_stderr => "exec_stderr", + exec_exit => "exec_exit", + activate_config => "activate_config", + activate_config_result => "activate_config_result", + built_paths => "built_paths", + cache_drain => "cache_drain", + cache_drain_result => "cache_drain_result", + poweroff => "poweroff", + poweroff_result => "poweroff_result", + }) + .unwrap_or_else(|| unreachable!("validated message has no payload")) +} + +pub fn error_or_empty(error: Option) -> String { + error.filter(|error| !error.is_empty()).unwrap_or_default() +} + +pub async fn write_message(writer: &mut W, msg: &Message) -> io::Result<()> { + if let Err(err) = prost_protovalidate::validate(msg) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("validate agent message: {err}"), + )); + } + + let mut data = Vec::new(); + msg.encode(&mut data) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + if data.len() > MAX_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("agent message exceeded {MAX_MESSAGE_BYTES} bytes"), + )); + } + + writer.write_all(&(data.len() as u32).to_be_bytes()).await?; + writer.write_all(&data).await?; + writer.flush().await +} + +pub async fn read_message(reader: &mut R) -> io::Result> { + let Some(header) = read_header(reader).await? else { + return Ok(None); + }; + let size = u32::from_be_bytes(header) as usize; + if size > MAX_MESSAGE_BYTES { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("agent message exceeded {MAX_MESSAGE_BYTES} bytes"), + )); + } + + let mut data = vec![0; size]; + reader.read_exact(&mut data).await?; + let msg = Message::decode(&data[..]) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + + if let Err(err) = prost_protovalidate::validate(&msg) { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("validate agent message: {err}"), + )); + } + + Ok(Some(msg)) +} + +async fn read_header(reader: &mut R) -> io::Result> { + let mut header = [0; 4]; + let mut read = 0; + while read < header.len() { + match reader.read(&mut header[read..]).await { + Ok(0) if read == 0 => return Ok(None), + Ok(0) => { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "partial agent message header", + )); + } + Ok(n) => read += n, + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) => return Err(err), + } + } + Ok(Some(header)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn round_trips_protobuf_message() { + let msg = Message { + id: "built-paths".to_owned(), + built_paths: Some(v1::BuiltPaths { + paths: vec!["/nix/store/abc-package".to_owned()], + reason: "post_build_hook".to_owned(), + }), + ..Default::default() + }; + + let mut encoded = Vec::new(); + write_message(&mut encoded, &msg).await.unwrap(); + + let decoded = read_message(&mut &encoded[..]).await.unwrap().unwrap(); + assert!(decoded.built_paths.is_some()); + if let Some(p) = decoded.built_paths { + assert_eq!(p.paths, ["/nix/store/abc-package"]); + assert_eq!(p.reason, "post_build_hook"); + } + } + + #[test] + fn validates_messages() { + // 1. valid message (exactly one field set) + let valid = Message { + id: "test-1".to_owned(), + hello: Some(v1::Hello::default()), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&valid).is_ok()); + + // 2. invalid message (zero fields set) + let invalid_zero = Message { + id: "test-2".to_owned(), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&invalid_zero).is_err()); + + // 3. invalid message (multiple fields set) + let invalid_multi = Message { + id: "test-3".to_owned(), + hello: Some(v1::Hello::default()), + init: Some(v1::Init::default()), + ..Default::default() + }; + assert!(prost_protovalidate::validate(&invalid_multi).is_err()); + } +} diff --git a/shuttle/src/session.rs b/shuttle/src/session.rs new file mode 100644 index 00000000..4743cc53 --- /dev/null +++ b/shuttle/src/session.rs @@ -0,0 +1,244 @@ +use crate::cache::{CacheUploadManager, ReadCacheProxy, WriteCacheProxy}; +use crate::command::Spec; +use crate::dns_proxy::DnsProxy; +use crate::exec; +use crate::nix_config::{self, SYSTEMCTL_EXECUTABLE}; +use crate::on_payload; +use crate::protocol::{self, Message, v1}; +use crate::{activation, command}; +use anyhow::{Context, Result, bail}; +use std::time::Duration; +use tokio::io::{AsyncWrite, BufReader}; +use tokio::sync::mpsc::{self, Sender}; +use tokio::task::{JoinError, JoinSet}; +use tokio_vsock::{VsockAddr, VsockStream}; +use tracing::{info, warn}; + +pub async fn run(host_cid: u32, port: u32) -> Result<()> { + let mut conn = VsockStream::connect(VsockAddr::new(host_cid, port)) + .await + .with_context(|| format!("dial host vsock cid={host_cid} port={port}"))?; + + send_hello(&mut conn).await?; + + let (reader_conn, writer_conn) = tokio::io::split(conn); + let (out_tx, out_rx) = mpsc::channel::(256); + let writer = tokio::spawn(async move { writer_loop(writer_conn, out_rx).await }); + let mut reader = BufReader::new(reader_conn); + + let init = match protocol::read_message(&mut reader).await? { + Some(Message { + init: Some(init), .. + }) => init, + Some(other) => bail!("expected init, got {}", protocol::kind(&other)), + None => bail!("read init: EOF"), + }; + info!(job_id = %init.job_id, "received init"); + + let read_proxy = ReadCacheProxy::start(host_cid, init.cache_read_proxy_port) + .await + .context("start read cache proxy")?; + let write_proxy = WriteCacheProxy::start(host_cid, init.cache_upload_proxy_port) + .await + .context("start write cache proxy")?; + let _dns_proxy = DnsProxy::start(host_cid, init.dns_proxy_port) + .await + .context("start dns proxy")?; + let _cache_cfg = nix_config::configure( + &init, + read_proxy.as_ref().map(ReadCacheProxy::url).unwrap_or(""), + ) + .await + .context("configure nix cache")?; + let uploader = CacheUploadManager::start( + write_proxy.as_ref().map(WriteCacheProxy::url).unwrap_or(""), + out_tx.clone(), + ) + .await + .context("start cache upload manager")?; + + let mut tasks = JoinSet::new(); + let read_result: Result<()> = loop { + tokio::select! { + read = protocol::read_message(&mut reader) => match read { + Ok(Some(msg)) => spawn_message_task(&mut tasks, msg, &out_tx, uploader.clone()), + Ok(None) => break Ok(()), + Err(error) => break Err(error).context("read message"), + }, + Some(result) = tasks.join_next(), if !tasks.is_empty() => { + log_task_result(result, false); + } + } + }; + + tasks.abort_all(); + while let Some(result) = tasks.join_next().await { + log_task_result(result, true); + } + + drop(out_tx); + let _ = writer.await; + read_result?; + Ok(()) +} + +fn spawn_message_task( + tasks: &mut JoinSet<()>, + msg: Message, + out_tx: &Sender, + uploader: Option, +) { + let kind = protocol::kind(&msg); + let handle = on_payload!(msg, { + activate_config => tasks.spawn(activation::run(msg.id, activate_config, out_tx.clone())), + exec_start => tasks.spawn(exec::run(msg.id, exec_start, out_tx.clone())), + cache_drain => tasks.spawn(run_cache_drain(msg.id, cache_drain, out_tx.clone(), uploader)), + poweroff => tasks.spawn(run_poweroff(msg.id, poweroff, out_tx.clone())), + }); + if handle.is_none() { + warn!(kind, "ignoring unsupported message"); + } +} + +fn log_task_result(result: Result<(), JoinError>, shutting_down: bool) { + match result { + Ok(()) => {} + Err(error) if shutting_down && error.is_cancelled() => {} + Err(error) => warn!(%error, "session handler task failed"), + } +} + +async fn writer_loop(mut conn: W, mut rx: mpsc::Receiver) +where + W: AsyncWrite + Unpin, +{ + while let Some(msg) = rx.recv().await { + if let Err(error) = protocol::write_message(&mut conn, &msg).await { + warn!(%error, "failed to write protocol message"); + break; + } + } +} + +async fn send_hello(conn: &mut VsockStream) -> Result<()> { + let boot_id = tokio::fs::read_to_string("/proc/sys/kernel/random/boot_id") + .await + .unwrap_or_default() + .trim() + .to_owned(); + let nix_version = nix_config::nix_version().await; + + let hello_payload = v1::Hello { + protocol_version: protocol::PROTOCOL_VERSION, + agent_version: env!("CARGO_PKG_VERSION").to_string(), + boot_id: boot_id.clone(), + nix_version: nix_version.clone(), + }; + info!( + protocol = hello_payload.protocol_version, + version = %hello_payload.agent_version, + boot = %hello_payload.boot_id, + nix = %hello_payload.nix_version, + "sent hello" + ); + let hello = Message { + id: "hello".to_owned(), + hello: Some(hello_payload), + ..Default::default() + }; + + protocol::write_message(conn, &hello) + .await + .context("send hello")?; + Ok(()) +} + +async fn run_cache_drain( + id: String, + req: v1::CacheDrain, + out: Sender, + uploader: Option, +) { + let timeout = + (req.timeout_seconds > 0).then(|| Duration::from_secs(u64::from(req.timeout_seconds))); + let stats = match uploader.as_ref() { + Some(uploader) => uploader.drain(timeout).await, + None => Default::default(), + }; + + if let Some(error) = &stats.last_error { + warn!( + %id, + pending = stats.pending, + active = stats.active, + uploaded = stats.uploaded, + failed = stats.failed, + %error, + "cache drain completed with error" + ); + } else { + info!( + %id, + uploaded = stats.uploaded, + failed = stats.failed, + "cache drain completed" + ); + } + + let result = Message { + id, + cache_drain_result: Some(v1::CacheDrainResult { + error: protocol::error_or_empty(stats.last_error), + cache_queued: stats.pending, + cache_active: stats.active, + cache_uploaded: stats.uploaded, + cache_failed: stats.failed, + }), + ..Default::default() + }; + let _ = out.send(result).await; +} + +async fn run_poweroff(id: String, _req: v1::Poweroff, out: Sender) { + let result = Message { + id, + poweroff_result: Some(v1::PoweroffResult { + error: String::new(), + }), + ..Default::default() + }; + let _ = out.send(result).await; + + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(100)).await; + + // prefer a clean shutdown through the init system when one is around + // (systemd on NixOS, busybox/openrc elsewhere), then fall back to the + // raw reboot(2) syscall on minimal guests + for poweroff in [SYSTEMCTL_EXECUTABLE, "/sbin/poweroff", "/usr/sbin/poweroff"] { + if !std::path::Path::new(poweroff).exists() { + continue; + } + let mut spec = Spec::new(poweroff).timeout(Duration::from_secs(5)); + if poweroff == SYSTEMCTL_EXECUTABLE { + spec = spec.args(["poweroff"]); + } + match command::run_capture(spec).await { + Ok(output) if output.success() => return, + Ok(output) => warn!( + %poweroff, + exit_code = output.exit.exit_code, + error = ?output.exit.error, + output = %output.combined_lossy(), + "poweroff command failed" + ), + Err(error) => warn!(%poweroff, %error, "poweroff command failed"), + } + } + + // only ever returns on failure + let error = + nix::sys::reboot::reboot(nix::sys::reboot::RebootMode::RB_POWER_OFF).unwrap_err(); + warn!(%error, "reboot(RB_POWER_OFF) syscall failed"); + }); +} diff --git a/spindle/agentproto/gen/agent.pb.go b/spindle/agentproto/gen/agent.pb.go new file mode 100644 index 00000000..f4a02799 --- /dev/null +++ b/spindle/agentproto/gen/agent.pb.go @@ -0,0 +1,1102 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc (unknown) +// source: spindle/agent/v1/agent.proto + +package agentv1 + +import ( + _ "buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go/buf/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Hello struct { + state protoimpl.MessageState `protogen:"open.v1"` + ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + AgentVersion string `protobuf:"bytes,2,opt,name=agent_version,json=agentVersion,proto3" json:"agent_version,omitempty"` + BootId string `protobuf:"bytes,3,opt,name=boot_id,json=bootId,proto3" json:"boot_id,omitempty"` + NixVersion string `protobuf:"bytes,4,opt,name=nix_version,json=nixVersion,proto3" json:"nix_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Hello) Reset() { + *x = Hello{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Hello) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Hello) ProtoMessage() {} + +func (x *Hello) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Hello.ProtoReflect.Descriptor instead. +func (*Hello) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{0} +} + +func (x *Hello) GetProtocolVersion() uint32 { + if x != nil { + return x.ProtocolVersion + } + return 0 +} + +func (x *Hello) GetAgentVersion() string { + if x != nil { + return x.AgentVersion + } + return "" +} + +func (x *Hello) GetBootId() string { + if x != nil { + return x.BootId + } + return "" +} + +func (x *Hello) GetNixVersion() string { + if x != nil { + return x.NixVersion + } + return "" +} + +type Init struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + CacheTrustedPublicKeys []string `protobuf:"bytes,2,rep,name=cache_trusted_public_keys,json=cacheTrustedPublicKeys,proto3" json:"cache_trusted_public_keys,omitempty"` + CacheReadProxyPort uint32 `protobuf:"varint,3,opt,name=cache_read_proxy_port,json=cacheReadProxyPort,proto3" json:"cache_read_proxy_port,omitempty"` + CacheUploadProxyPort uint32 `protobuf:"varint,4,opt,name=cache_upload_proxy_port,json=cacheUploadProxyPort,proto3" json:"cache_upload_proxy_port,omitempty"` + DnsProxyPort uint32 `protobuf:"varint,5,opt,name=dns_proxy_port,json=dnsProxyPort,proto3" json:"dns_proxy_port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Init) Reset() { + *x = Init{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Init) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Init) ProtoMessage() {} + +func (x *Init) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Init.ProtoReflect.Descriptor instead. +func (*Init) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{1} +} + +func (x *Init) GetJobId() string { + if x != nil { + return x.JobId + } + return "" +} + +func (x *Init) GetCacheTrustedPublicKeys() []string { + if x != nil { + return x.CacheTrustedPublicKeys + } + return nil +} + +func (x *Init) GetCacheReadProxyPort() uint32 { + if x != nil { + return x.CacheReadProxyPort + } + return 0 +} + +func (x *Init) GetCacheUploadProxyPort() uint32 { + if x != nil { + return x.CacheUploadProxyPort + } + return 0 +} + +func (x *Init) GetDnsProxyPort() uint32 { + if x != nil { + return x.DnsProxyPort + } + return 0 +} + +type ExecStart struct { + state protoimpl.MessageState `protogen:"open.v1"` + Argv []string `protobuf:"bytes,1,rep,name=argv,proto3" json:"argv,omitempty"` + Env []string `protobuf:"bytes,2,rep,name=env,proto3" json:"env,omitempty"` + Cwd string `protobuf:"bytes,3,opt,name=cwd,proto3" json:"cwd,omitempty"` + User string `protobuf:"bytes,4,opt,name=user,proto3" json:"user,omitempty"` + TimeoutSeconds uint32 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStart) Reset() { + *x = ExecStart{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStart) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStart) ProtoMessage() {} + +func (x *ExecStart) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStart.ProtoReflect.Descriptor instead. +func (*ExecStart) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{2} +} + +func (x *ExecStart) GetArgv() []string { + if x != nil { + return x.Argv + } + return nil +} + +func (x *ExecStart) GetEnv() []string { + if x != nil { + return x.Env + } + return nil +} + +func (x *ExecStart) GetCwd() string { + if x != nil { + return x.Cwd + } + return "" +} + +func (x *ExecStart) GetUser() string { + if x != nil { + return x.User + } + return "" +} + +func (x *ExecStart) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type ExecStdout struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStdout) Reset() { + *x = ExecStdout{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStdout) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStdout) ProtoMessage() {} + +func (x *ExecStdout) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStdout.ProtoReflect.Descriptor instead. +func (*ExecStdout) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{3} +} + +func (x *ExecStdout) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +type ExecStderr struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecStderr) Reset() { + *x = ExecStderr{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecStderr) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecStderr) ProtoMessage() {} + +func (x *ExecStderr) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecStderr.ProtoReflect.Descriptor instead. +func (*ExecStderr) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{4} +} + +func (x *ExecStderr) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +type ExecExit struct { + state protoimpl.MessageState `protogen:"open.v1"` + ExitCode int32 `protobuf:"varint,1,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + Error string `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"` + // set when the guest killed the step on its own timeout timer, so the host + // can classify it as a timeout rather than inferring failure from exit_code. + TimedOut bool `protobuf:"varint,3,opt,name=timed_out,json=timedOut,proto3" json:"timed_out,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ExecExit) Reset() { + *x = ExecExit{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ExecExit) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExecExit) ProtoMessage() {} + +func (x *ExecExit) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExecExit.ProtoReflect.Descriptor instead. +func (*ExecExit) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{5} +} + +func (x *ExecExit) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *ExecExit) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *ExecExit) GetTimedOut() bool { + if x != nil { + return x.TimedOut + } + return false +} + +type ActivateConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfigKey string `protobuf:"bytes,1,opt,name=config_key,json=configKey,proto3" json:"config_key,omitempty"` + BaseConfigHash string `protobuf:"bytes,2,opt,name=base_config_hash,json=baseConfigHash,proto3" json:"base_config_hash,omitempty"` + UserConfig string `protobuf:"bytes,3,opt,name=user_config,json=userConfig,proto3" json:"user_config,omitempty"` + Toplevel string `protobuf:"bytes,4,opt,name=toplevel,proto3" json:"toplevel,omitempty"` + TimeoutSeconds uint32 `protobuf:"varint,5,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActivateConfig) Reset() { + *x = ActivateConfig{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActivateConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActivateConfig) ProtoMessage() {} + +func (x *ActivateConfig) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActivateConfig.ProtoReflect.Descriptor instead. +func (*ActivateConfig) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{6} +} + +func (x *ActivateConfig) GetConfigKey() string { + if x != nil { + return x.ConfigKey + } + return "" +} + +func (x *ActivateConfig) GetBaseConfigHash() string { + if x != nil { + return x.BaseConfigHash + } + return "" +} + +func (x *ActivateConfig) GetUserConfig() string { + if x != nil { + return x.UserConfig + } + return "" +} + +func (x *ActivateConfig) GetToplevel() string { + if x != nil { + return x.Toplevel + } + return "" +} + +func (x *ActivateConfig) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type ActivateConfigResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConfigKey string `protobuf:"bytes,1,opt,name=config_key,json=configKey,proto3" json:"config_key,omitempty"` + Toplevel string `protobuf:"bytes,2,opt,name=toplevel,proto3" json:"toplevel,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActivateConfigResult) Reset() { + *x = ActivateConfigResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActivateConfigResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActivateConfigResult) ProtoMessage() {} + +func (x *ActivateConfigResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActivateConfigResult.ProtoReflect.Descriptor instead. +func (*ActivateConfigResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{7} +} + +func (x *ActivateConfigResult) GetConfigKey() string { + if x != nil { + return x.ConfigKey + } + return "" +} + +func (x *ActivateConfigResult) GetToplevel() string { + if x != nil { + return x.Toplevel + } + return "" +} + +func (x *ActivateConfigResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +type BuiltPaths struct { + state protoimpl.MessageState `protogen:"open.v1"` + Paths []string `protobuf:"bytes,1,rep,name=paths,proto3" json:"paths,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BuiltPaths) Reset() { + *x = BuiltPaths{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BuiltPaths) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BuiltPaths) ProtoMessage() {} + +func (x *BuiltPaths) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BuiltPaths.ProtoReflect.Descriptor instead. +func (*BuiltPaths) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{8} +} + +func (x *BuiltPaths) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + +func (x *BuiltPaths) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +type CacheDrain struct { + state protoimpl.MessageState `protogen:"open.v1"` + TimeoutSeconds uint32 `protobuf:"varint,1,opt,name=timeout_seconds,json=timeoutSeconds,proto3" json:"timeout_seconds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheDrain) Reset() { + *x = CacheDrain{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheDrain) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheDrain) ProtoMessage() {} + +func (x *CacheDrain) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheDrain.ProtoReflect.Descriptor instead. +func (*CacheDrain) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{9} +} + +func (x *CacheDrain) GetTimeoutSeconds() uint32 { + if x != nil { + return x.TimeoutSeconds + } + return 0 +} + +type CacheDrainResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + CacheQueued uint32 `protobuf:"varint,2,opt,name=cache_queued,json=cacheQueued,proto3" json:"cache_queued,omitempty"` + CacheActive uint32 `protobuf:"varint,3,opt,name=cache_active,json=cacheActive,proto3" json:"cache_active,omitempty"` + CacheUploaded uint32 `protobuf:"varint,4,opt,name=cache_uploaded,json=cacheUploaded,proto3" json:"cache_uploaded,omitempty"` + CacheFailed uint32 `protobuf:"varint,5,opt,name=cache_failed,json=cacheFailed,proto3" json:"cache_failed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheDrainResult) Reset() { + *x = CacheDrainResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheDrainResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheDrainResult) ProtoMessage() {} + +func (x *CacheDrainResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheDrainResult.ProtoReflect.Descriptor instead. +func (*CacheDrainResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{10} +} + +func (x *CacheDrainResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *CacheDrainResult) GetCacheQueued() uint32 { + if x != nil { + return x.CacheQueued + } + return 0 +} + +func (x *CacheDrainResult) GetCacheActive() uint32 { + if x != nil { + return x.CacheActive + } + return 0 +} + +func (x *CacheDrainResult) GetCacheUploaded() uint32 { + if x != nil { + return x.CacheUploaded + } + return 0 +} + +func (x *CacheDrainResult) GetCacheFailed() uint32 { + if x != nil { + return x.CacheFailed + } + return 0 +} + +type Poweroff struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Poweroff) Reset() { + *x = Poweroff{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Poweroff) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Poweroff) ProtoMessage() {} + +func (x *Poweroff) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Poweroff.ProtoReflect.Descriptor instead. +func (*Poweroff) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{11} +} + +type PoweroffResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Error string `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PoweroffResult) Reset() { + *x = PoweroffResult{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PoweroffResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PoweroffResult) ProtoMessage() {} + +func (x *PoweroffResult) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PoweroffResult.ProtoReflect.Descriptor instead. +func (*PoweroffResult) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{12} +} + +func (x *PoweroffResult) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +type Message struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Hello *Hello `protobuf:"bytes,2,opt,name=hello,proto3" json:"hello,omitempty"` + Init *Init `protobuf:"bytes,3,opt,name=init,proto3" json:"init,omitempty"` + ExecStart *ExecStart `protobuf:"bytes,4,opt,name=exec_start,json=execStart,proto3" json:"exec_start,omitempty"` + ExecStdout *ExecStdout `protobuf:"bytes,5,opt,name=exec_stdout,json=execStdout,proto3" json:"exec_stdout,omitempty"` + ExecStderr *ExecStderr `protobuf:"bytes,6,opt,name=exec_stderr,json=execStderr,proto3" json:"exec_stderr,omitempty"` + ExecExit *ExecExit `protobuf:"bytes,7,opt,name=exec_exit,json=execExit,proto3" json:"exec_exit,omitempty"` + ActivateConfig *ActivateConfig `protobuf:"bytes,8,opt,name=activate_config,json=activateConfig,proto3" json:"activate_config,omitempty"` + ActivateConfigResult *ActivateConfigResult `protobuf:"bytes,9,opt,name=activate_config_result,json=activateConfigResult,proto3" json:"activate_config_result,omitempty"` + BuiltPaths *BuiltPaths `protobuf:"bytes,10,opt,name=built_paths,json=builtPaths,proto3" json:"built_paths,omitempty"` + CacheDrain *CacheDrain `protobuf:"bytes,11,opt,name=cache_drain,json=cacheDrain,proto3" json:"cache_drain,omitempty"` + CacheDrainResult *CacheDrainResult `protobuf:"bytes,12,opt,name=cache_drain_result,json=cacheDrainResult,proto3" json:"cache_drain_result,omitempty"` + Poweroff *Poweroff `protobuf:"bytes,13,opt,name=poweroff,proto3" json:"poweroff,omitempty"` + PoweroffResult *PoweroffResult `protobuf:"bytes,14,opt,name=poweroff_result,json=poweroffResult,proto3" json:"poweroff_result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Message) Reset() { + *x = Message{} + mi := &file_spindle_agent_v1_agent_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Message) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Message) ProtoMessage() {} + +func (x *Message) ProtoReflect() protoreflect.Message { + mi := &file_spindle_agent_v1_agent_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Message.ProtoReflect.Descriptor instead. +func (*Message) Descriptor() ([]byte, []int) { + return file_spindle_agent_v1_agent_proto_rawDescGZIP(), []int{13} +} + +func (x *Message) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *Message) GetHello() *Hello { + if x != nil { + return x.Hello + } + return nil +} + +func (x *Message) GetInit() *Init { + if x != nil { + return x.Init + } + return nil +} + +func (x *Message) GetExecStart() *ExecStart { + if x != nil { + return x.ExecStart + } + return nil +} + +func (x *Message) GetExecStdout() *ExecStdout { + if x != nil { + return x.ExecStdout + } + return nil +} + +func (x *Message) GetExecStderr() *ExecStderr { + if x != nil { + return x.ExecStderr + } + return nil +} + +func (x *Message) GetExecExit() *ExecExit { + if x != nil { + return x.ExecExit + } + return nil +} + +func (x *Message) GetActivateConfig() *ActivateConfig { + if x != nil { + return x.ActivateConfig + } + return nil +} + +func (x *Message) GetActivateConfigResult() *ActivateConfigResult { + if x != nil { + return x.ActivateConfigResult + } + return nil +} + +func (x *Message) GetBuiltPaths() *BuiltPaths { + if x != nil { + return x.BuiltPaths + } + return nil +} + +func (x *Message) GetCacheDrain() *CacheDrain { + if x != nil { + return x.CacheDrain + } + return nil +} + +func (x *Message) GetCacheDrainResult() *CacheDrainResult { + if x != nil { + return x.CacheDrainResult + } + return nil +} + +func (x *Message) GetPoweroff() *Poweroff { + if x != nil { + return x.Poweroff + } + return nil +} + +func (x *Message) GetPoweroffResult() *PoweroffResult { + if x != nil { + return x.PoweroffResult + } + return nil +} + +var File_spindle_agent_v1_agent_proto protoreflect.FileDescriptor + +const file_spindle_agent_v1_agent_proto_rawDesc = "" + + "\n" + + "\x1cspindle/agent/v1/agent.proto\x12\x10spindle.agent.v1\x1a\x1bbuf/validate/validate.proto\"\x91\x01\n" + + "\x05Hello\x12)\n" + + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12#\n" + + "\ragent_version\x18\x02 \x01(\tR\fagentVersion\x12\x17\n" + + "\aboot_id\x18\x03 \x01(\tR\x06bootId\x12\x1f\n" + + "\vnix_version\x18\x04 \x01(\tR\n" + + "nixVersion\"\xe8\x01\n" + + "\x04Init\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\tR\x05jobId\x129\n" + + "\x19cache_trusted_public_keys\x18\x02 \x03(\tR\x16cacheTrustedPublicKeys\x121\n" + + "\x15cache_read_proxy_port\x18\x03 \x01(\rR\x12cacheReadProxyPort\x125\n" + + "\x17cache_upload_proxy_port\x18\x04 \x01(\rR\x14cacheUploadProxyPort\x12$\n" + + "\x0edns_proxy_port\x18\x05 \x01(\rR\fdnsProxyPort\"\x80\x01\n" + + "\tExecStart\x12\x12\n" + + "\x04argv\x18\x01 \x03(\tR\x04argv\x12\x10\n" + + "\x03env\x18\x02 \x03(\tR\x03env\x12\x10\n" + + "\x03cwd\x18\x03 \x01(\tR\x03cwd\x12\x12\n" + + "\x04user\x18\x04 \x01(\tR\x04user\x12'\n" + + "\x0ftimeout_seconds\x18\x05 \x01(\rR\x0etimeoutSeconds\" \n" + + "\n" + + "ExecStdout\x12\x12\n" + + "\x04data\x18\x01 \x01(\tR\x04data\" \n" + + "\n" + + "ExecStderr\x12\x12\n" + + "\x04data\x18\x01 \x01(\tR\x04data\"Z\n" + + "\bExecExit\x12\x1b\n" + + "\texit_code\x18\x01 \x01(\x05R\bexitCode\x12\x14\n" + + "\x05error\x18\x02 \x01(\tR\x05error\x12\x1b\n" + + "\ttimed_out\x18\x03 \x01(\bR\btimedOut\"\xbf\x01\n" + + "\x0eActivateConfig\x12\x1d\n" + + "\n" + + "config_key\x18\x01 \x01(\tR\tconfigKey\x12(\n" + + "\x10base_config_hash\x18\x02 \x01(\tR\x0ebaseConfigHash\x12\x1f\n" + + "\vuser_config\x18\x03 \x01(\tR\n" + + "userConfig\x12\x1a\n" + + "\btoplevel\x18\x04 \x01(\tR\btoplevel\x12'\n" + + "\x0ftimeout_seconds\x18\x05 \x01(\rR\x0etimeoutSeconds\"g\n" + + "\x14ActivateConfigResult\x12\x1d\n" + + "\n" + + "config_key\x18\x01 \x01(\tR\tconfigKey\x12\x1a\n" + + "\btoplevel\x18\x02 \x01(\tR\btoplevel\x12\x14\n" + + "\x05error\x18\x03 \x01(\tR\x05error\":\n" + + "\n" + + "BuiltPaths\x12\x14\n" + + "\x05paths\x18\x01 \x03(\tR\x05paths\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\"5\n" + + "\n" + + "CacheDrain\x12'\n" + + "\x0ftimeout_seconds\x18\x01 \x01(\rR\x0etimeoutSeconds\"\xb8\x01\n" + + "\x10CacheDrainResult\x12\x14\n" + + "\x05error\x18\x01 \x01(\tR\x05error\x12!\n" + + "\fcache_queued\x18\x02 \x01(\rR\vcacheQueued\x12!\n" + + "\fcache_active\x18\x03 \x01(\rR\vcacheActive\x12%\n" + + "\x0ecache_uploaded\x18\x04 \x01(\rR\rcacheUploaded\x12!\n" + + "\fcache_failed\x18\x05 \x01(\rR\vcacheFailed\"\n" + + "\n" + + "\bPoweroff\"&\n" + + "\x0ePoweroffResult\x12\x14\n" + + "\x05error\x18\x01 \x01(\tR\x05error\"\xa8\b\n" + + "\aMessage\x12\x17\n" + + "\x02id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x02id\x12-\n" + + "\x05hello\x18\x02 \x01(\v2\x17.spindle.agent.v1.HelloR\x05hello\x12*\n" + + "\x04init\x18\x03 \x01(\v2\x16.spindle.agent.v1.InitR\x04init\x12:\n" + + "\n" + + "exec_start\x18\x04 \x01(\v2\x1b.spindle.agent.v1.ExecStartR\texecStart\x12=\n" + + "\vexec_stdout\x18\x05 \x01(\v2\x1c.spindle.agent.v1.ExecStdoutR\n" + + "execStdout\x12=\n" + + "\vexec_stderr\x18\x06 \x01(\v2\x1c.spindle.agent.v1.ExecStderrR\n" + + "execStderr\x127\n" + + "\texec_exit\x18\a \x01(\v2\x1a.spindle.agent.v1.ExecExitR\bexecExit\x12I\n" + + "\x0factivate_config\x18\b \x01(\v2 .spindle.agent.v1.ActivateConfigR\x0eactivateConfig\x12\\\n" + + "\x16activate_config_result\x18\t \x01(\v2&.spindle.agent.v1.ActivateConfigResultR\x14activateConfigResult\x12=\n" + + "\vbuilt_paths\x18\n" + + " \x01(\v2\x1c.spindle.agent.v1.BuiltPathsR\n" + + "builtPaths\x12=\n" + + "\vcache_drain\x18\v \x01(\v2\x1c.spindle.agent.v1.CacheDrainR\n" + + "cacheDrain\x12P\n" + + "\x12cache_drain_result\x18\f \x01(\v2\".spindle.agent.v1.CacheDrainResultR\x10cacheDrainResult\x126\n" + + "\bpoweroff\x18\r \x01(\v2\x1a.spindle.agent.v1.PoweroffR\bpoweroff\x12I\n" + + "\x0fpoweroff_result\x18\x0e \x01(\v2 .spindle.agent.v1.PoweroffResultR\x0epoweroffResult:\xb9\x01\xbaH\xb5\x01\"\xb2\x01\n" + + "\x05hello\n" + + "\x04init\n" + + "\n" + + "exec_start\n" + + "\vexec_stdout\n" + + "\vexec_stderr\n" + + "\texec_exit\n" + + "\x0factivate_config\n" + + "\x16activate_config_result\n" + + "\vbuilt_paths\n" + + "\vcache_drain\n" + + "\x12cache_drain_result\n" + + "\bpoweroff\n" + + "\x0fpoweroff_result\x10\x01B1Z/tangled.org/core/spindle/agentproto/gen;agentv1b\x06proto3" + +var ( + file_spindle_agent_v1_agent_proto_rawDescOnce sync.Once + file_spindle_agent_v1_agent_proto_rawDescData []byte +) + +func file_spindle_agent_v1_agent_proto_rawDescGZIP() []byte { + file_spindle_agent_v1_agent_proto_rawDescOnce.Do(func() { + file_spindle_agent_v1_agent_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_spindle_agent_v1_agent_proto_rawDesc), len(file_spindle_agent_v1_agent_proto_rawDesc))) + }) + return file_spindle_agent_v1_agent_proto_rawDescData +} + +var file_spindle_agent_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_spindle_agent_v1_agent_proto_goTypes = []any{ + (*Hello)(nil), // 0: spindle.agent.v1.Hello + (*Init)(nil), // 1: spindle.agent.v1.Init + (*ExecStart)(nil), // 2: spindle.agent.v1.ExecStart + (*ExecStdout)(nil), // 3: spindle.agent.v1.ExecStdout + (*ExecStderr)(nil), // 4: spindle.agent.v1.ExecStderr + (*ExecExit)(nil), // 5: spindle.agent.v1.ExecExit + (*ActivateConfig)(nil), // 6: spindle.agent.v1.ActivateConfig + (*ActivateConfigResult)(nil), // 7: spindle.agent.v1.ActivateConfigResult + (*BuiltPaths)(nil), // 8: spindle.agent.v1.BuiltPaths + (*CacheDrain)(nil), // 9: spindle.agent.v1.CacheDrain + (*CacheDrainResult)(nil), // 10: spindle.agent.v1.CacheDrainResult + (*Poweroff)(nil), // 11: spindle.agent.v1.Poweroff + (*PoweroffResult)(nil), // 12: spindle.agent.v1.PoweroffResult + (*Message)(nil), // 13: spindle.agent.v1.Message +} +var file_spindle_agent_v1_agent_proto_depIdxs = []int32{ + 0, // 0: spindle.agent.v1.Message.hello:type_name -> spindle.agent.v1.Hello + 1, // 1: spindle.agent.v1.Message.init:type_name -> spindle.agent.v1.Init + 2, // 2: spindle.agent.v1.Message.exec_start:type_name -> spindle.agent.v1.ExecStart + 3, // 3: spindle.agent.v1.Message.exec_stdout:type_name -> spindle.agent.v1.ExecStdout + 4, // 4: spindle.agent.v1.Message.exec_stderr:type_name -> spindle.agent.v1.ExecStderr + 5, // 5: spindle.agent.v1.Message.exec_exit:type_name -> spindle.agent.v1.ExecExit + 6, // 6: spindle.agent.v1.Message.activate_config:type_name -> spindle.agent.v1.ActivateConfig + 7, // 7: spindle.agent.v1.Message.activate_config_result:type_name -> spindle.agent.v1.ActivateConfigResult + 8, // 8: spindle.agent.v1.Message.built_paths:type_name -> spindle.agent.v1.BuiltPaths + 9, // 9: spindle.agent.v1.Message.cache_drain:type_name -> spindle.agent.v1.CacheDrain + 10, // 10: spindle.agent.v1.Message.cache_drain_result:type_name -> spindle.agent.v1.CacheDrainResult + 11, // 11: spindle.agent.v1.Message.poweroff:type_name -> spindle.agent.v1.Poweroff + 12, // 12: spindle.agent.v1.Message.poweroff_result:type_name -> spindle.agent.v1.PoweroffResult + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_spindle_agent_v1_agent_proto_init() } +func file_spindle_agent_v1_agent_proto_init() { + if File_spindle_agent_v1_agent_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_agent_v1_agent_proto_rawDesc), len(file_spindle_agent_v1_agent_proto_rawDesc)), + NumEnums: 0, + NumMessages: 14, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_spindle_agent_v1_agent_proto_goTypes, + DependencyIndexes: file_spindle_agent_v1_agent_proto_depIdxs, + MessageInfos: file_spindle_agent_v1_agent_proto_msgTypes, + }.Build() + File_spindle_agent_v1_agent_proto = out.File + file_spindle_agent_v1_agent_proto_goTypes = nil + file_spindle_agent_v1_agent_proto_depIdxs = nil +} diff --git a/spindle/agentproto/protocol.go b/spindle/agentproto/protocol.go new file mode 100644 index 00000000..ac12e909 --- /dev/null +++ b/spindle/agentproto/protocol.go @@ -0,0 +1,98 @@ +package agentproto + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + + "google.golang.org/protobuf/proto" + + "buf.build/go/protovalidate" + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +const ( + ProtocolVersion = 1 + DefaultPort = 10240 + MaxMessageBytes = 1024 * 1024 +) + +type Message = agentv1.Message + +var validator protovalidate.Validator + +func init() { + var err error + validator, err = protovalidate.New() + if err != nil { + panic(fmt.Errorf("failed to initialize protovalidate validator: %w", err)) + } +} + +type Encoder struct { + mu sync.Mutex + w io.Writer +} + +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (e *Encoder) Encode(msg *Message) error { + if err := validator.Validate(msg); err != nil { + return fmt.Errorf("validate agent message: %w", err) + } + + data, err := proto.Marshal(msg) + if err != nil { + return fmt.Errorf("marshal agent message: %w", err) + } + if len(data) > MaxMessageBytes { + return fmt.Errorf("agent message exceeded %d bytes", MaxMessageBytes) + } + + var header [4]byte + binary.BigEndian.PutUint32(header[:], uint32(len(data))) + + e.mu.Lock() + defer e.mu.Unlock() + if _, err := e.w.Write(header[:]); err != nil { + return err + } + _, err = e.w.Write(data) + return err +} + +type Decoder struct { + r io.Reader +} + +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +func (d *Decoder) Decode() (*Message, error) { + msg := &Message{} + var header [4]byte + if _, err := io.ReadFull(d.r, header[:]); err != nil { + return msg, err + } + + size := binary.BigEndian.Uint32(header[:]) + if size > MaxMessageBytes { + return msg, fmt.Errorf("agent message exceeded %d bytes", MaxMessageBytes) + } + + data := make([]byte, size) + if _, err := io.ReadFull(d.r, data); err != nil { + return msg, err + } + if err := proto.Unmarshal(data, msg); err != nil { + return msg, fmt.Errorf("parse agent message: %w", err) + } + if err := validator.Validate(msg); err != nil { + return msg, fmt.Errorf("validate agent message: %w", err) + } + return msg, nil +} diff --git a/spindle/agentproto/protocol_test.go b/spindle/agentproto/protocol_test.go new file mode 100644 index 00000000..ccc7dc94 --- /dev/null +++ b/spindle/agentproto/protocol_test.go @@ -0,0 +1,57 @@ +package agentproto + +import ( + "bytes" + "encoding/binary" + "testing" + + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +func TestDecoderRejectsOversizedMessage(t *testing.T) { + var tooLarge bytes.Buffer + var header [4]byte + binary.BigEndian.PutUint32(header[:], MaxMessageBytes+1) + tooLarge.Write(header[:]) + + _, err := NewDecoder(&tooLarge).Decode() + if err == nil { + t.Fatal("expected oversized message error") + } +} + +func TestValidation(t *testing.T) { + // 1. Valid message (exactly one of the payload fields is set) + validMsg := &Message{ + Id: "test-1", + Hello: &agentv1.Hello{ + ProtocolVersion: 1, + AgentVersion: "1.0", + }, + } + if err := validator.Validate(validMsg); err != nil { + t.Fatalf("expected valid message to pass validation, got: %v", err) + } + + // 2. Invalid message: zero payloads set + invalidZeroMsg := &Message{ + Id: "test-2", + } + if err := validator.Validate(invalidZeroMsg); err == nil { + t.Fatal("expected message with zero payloads to fail validation") + } + + // 3. Invalid message: multiple payloads set + invalidMultiMsg := &Message{ + Id: "test-3", + Hello: &agentv1.Hello{ + ProtocolVersion: 1, + }, + Init: &agentv1.Init{ + JobId: "job-1", + }, + } + if err := validator.Validate(invalidMultiMsg); err == nil { + t.Fatal("expected message with multiple payloads to fail validation") + } +} diff --git a/spindle/agentproto/spindle/agent/v1/agent.proto b/spindle/agentproto/spindle/agent/v1/agent.proto new file mode 100644 index 00000000..6b3d4dc2 --- /dev/null +++ b/spindle/agentproto/spindle/agent/v1/agent.proto @@ -0,0 +1,110 @@ +syntax = "proto3"; + +package spindle.agent.v1; + +import "buf/validate/validate.proto"; + +option go_package = "tangled.org/core/spindle/agentproto/gen;agentv1"; + +message Hello { + uint32 protocol_version = 1; + string agent_version = 2; + string boot_id = 3; + string nix_version = 4; +} + +message Init { + string job_id = 1; + repeated string cache_trusted_public_keys = 2; + uint32 cache_read_proxy_port = 3; + uint32 cache_upload_proxy_port = 4; + uint32 dns_proxy_port = 5; +} + +message ExecStart { + repeated string argv = 1; + repeated string env = 2; + string cwd = 3; + string user = 4; + uint32 timeout_seconds = 5; +} + +message ExecStdout { + string data = 1; +} + +message ExecStderr { + string data = 1; +} + +message ExecExit { + int32 exit_code = 1; + string error = 2; + // set when the guest killed the step on its own timeout timer, so the host + // can classify it as a timeout rather than inferring failure from exit_code. + bool timed_out = 3; +} + +message ActivateConfig { + string config_key = 1; + string base_config_hash = 2; + string user_config = 3; + string toplevel = 4; + uint32 timeout_seconds = 5; +} + +message ActivateConfigResult { + string config_key = 1; + string toplevel = 2; + string error = 3; +} + +message BuiltPaths { + repeated string paths = 1; + string reason = 2; +} + +message CacheDrain { + uint32 timeout_seconds = 1; +} + +message CacheDrainResult { + string error = 1; + uint32 cache_queued = 2; + uint32 cache_active = 3; + uint32 cache_uploaded = 4; + uint32 cache_failed = 5; +} + +message Poweroff {} + +message PoweroffResult { + string error = 1; +} + +message Message { + option (buf.validate.message).oneof = { + fields: [ + "hello", "init", "exec_start", "exec_stdout", "exec_stderr", "exec_exit", + "activate_config", "activate_config_result", "built_paths", "cache_drain", + "cache_drain_result", "poweroff", "poweroff_result" + ], + required: true + }; + + string id = 1 [(buf.validate.field).string.min_len = 1]; + + Hello hello = 2; + Init init = 3; + ExecStart exec_start = 4; + ExecStdout exec_stdout = 5; + ExecStderr exec_stderr = 6; + ExecExit exec_exit = 7; + ActivateConfig activate_config = 8; + ActivateConfigResult activate_config_result = 9; + BuiltPaths built_paths = 10; + CacheDrain cache_drain = 11; + CacheDrainResult cache_drain_result = 12; + Poweroff poweroff = 13; + PoweroffResult poweroff_result = 14; +} diff --git a/spindle/config/config.go b/spindle/config/config.go index 990b2941..1d8ef04d 100644 --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -2,6 +2,7 @@ package config import ( "context" + "time" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/sethvargo/go-envconfig" @@ -9,20 +10,20 @@ import ( ) type Server struct { - ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"` - DBPath string `env:"DB_PATH, default=spindle.db"` - Hostname string `env:"HOSTNAME, required"` - JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"` - Tap Tap `env:",prefix=TAP_"` - PlcUrl string `env:"PLC_URL, default=https://plc.directory"` - Dev bool `env:"DEV, default=false"` - Owner string `env:"OWNER, required"` - Secrets Secrets `env:",prefix=SECRETS_"` - LogDir string `env:"LOG_DIR, default=/var/log/spindle"` - QueueSize int `env:"QUEUE_SIZE, default=100"` - MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time - MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) - DockerSocket string `env:"DOCKER_SOCKET"` // path to a docker socket to expose to workflow containers + ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:6555"` + DBPath string `env:"DB_PATH, default=spindle.db"` + Hostname string `env:"HOSTNAME, required"` + JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"` + Tap Tap `env:",prefix=TAP_"` + PlcUrl string `env:"PLC_URL, default=https://plc.directory"` + Dev bool `env:"DEV, default=false"` + DevExtraHosts []string `env:"DEV_EXTRA_HOSTS"` + Owner string `env:"OWNER, required"` + Secrets Secrets `env:",prefix=SECRETS_"` + LogDir string `env:"LOG_DIR, default=/var/log/spindle"` + QueueSize int `env:"QUEUE_SIZE, default=100"` + MaxJobCount int `env:"MAX_JOB_COUNT, default=2"` // max number of pipelines that run at a time + DockerSocket string `env:"DOCKER_SOCKET"` // path to a docker socket to expose to workflow containers } type Tap struct { @@ -49,19 +50,54 @@ type OpenBaoConfig struct { } type NixeryPipelines struct { - Nixery string `env:"NIXERY, default=nixery.tangled.sh"` - WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` - MaxJobMemoryMB int64 `env:"MAX_JOB_MEMORY_MB, default=6144"` // per-container memory limit in MiB (default 6 GiB) + Nixery string `env:"NIXERY, default=nixery.tangled.sh"` + WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` + MaxJobMemoryMB int64 `env:"MAX_JOB_MEMORY_MB, default=6144"` // per-container memory limit in MiB (default 6 GiB) + MaxConcurrentWorkflows int `env:"MAX_CONCURRENT_WORKFLOWS, default=8"` // max number of workflow containers running at once (memory cap) } type S3 struct { LogBucket string `env:"LOG_BUCKET"` } +type MicroVMPipelines struct { + ImageDir string `env:"IMAGE_DIR, required"` + OverlayDir string `env:"OVERLAY_DIR, default="` // where microVM temporary disks will live + DefaultImage string `env:"DEFAULT_IMAGE, default=nixos-x86_64"` + AgentPort uint32 `env:"AGENT_PORT, default=10240"` + EnableKVM bool `env:"ENABLE_KVM, default=true"` + WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` + + MaxTotalMemoryMiB int64 `env:"MAX_TOTAL_MEMORY_MIB, default=0"` + MaxTotalVCPUs int64 `env:"MAX_TOTAL_VCPUS, default=0"` + MaxTotalDiskMiB int64 `env:"MAX_TOTAL_DISK_MIB, default=0"` + + MaxWorkflowMemoryMiB int64 `env:"MAX_WORKFLOW_MEMORY_MIB, default=0"` + MaxWorkflowVCPUs int64 `env:"MAX_WORKFLOW_VCPUS, default=0"` + MaxWorkflowDiskMiB int64 `env:"MAX_WORKFLOW_DISK_MIB, default=0"` + + AgingThreshold time.Duration `env:"AGING_THRESHOLD, default=30s"` + + EnableCgroups bool `env:"ENABLE_CGROUPS, default=false"` + CgroupParent string `env:"CGROUP_PARENT, default=self"` + CgroupPidsMax int64 `env:"CGROUP_PIDS_MAX, default=4096"` + CgroupSwapMaxMiB *int64 `env:"CGROUP_SWAP_MAX_MIB"` + // memory.min that will get assigned to the supervisor (spindle itself) cgroup + CgroupSupervisorMemoryMinMiB int64 `env:"CGROUP_SUPERVISOR_MEMORY_MIN_MIB, default=512"` +} + +type NixCache struct { + ReadURLs []string `env:"READ_URLS"` + TrustedPublicKeys []string `env:"TRUSTED_PUBLIC_KEYS"` + UploadURL string `env:"UPLOAD_URL"` +} + type Config struct { - Server Server `env:",prefix=SPINDLE_SERVER_"` - NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` - S3 S3 `env:",prefix=SPINDLE_S3_"` + Server Server `env:",prefix=SPINDLE_SERVER_"` + NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` + MicroVMPipelines MicroVMPipelines `env:",prefix=SPINDLE_MICROVM_PIPELINES_"` + NixCache NixCache `env:",prefix=SPINDLE_NIX_CACHE_"` + S3 S3 `env:",prefix=SPINDLE_S3_"` } func Load(ctx context.Context) (*Config, error) { diff --git a/spindle/db/db.go b/spindle/db/db.go index 57d4dfb1..d6ac2c72 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -101,6 +101,12 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { created integer not null -- unix nanos ); + create table if not exists nixos_toplevel_cache ( + config_key text primary key, + toplevel text not null, + updated_at text not null + ); + create table if not exists migrations ( id integer primary key autoincrement, name text unique diff --git a/spindle/db/nixos_toplevel_cache.go b/spindle/db/nixos_toplevel_cache.go new file mode 100644 index 00000000..56d18e75 --- /dev/null +++ b/spindle/db/nixos_toplevel_cache.go @@ -0,0 +1,41 @@ +package db + +import ( + "time" +) + +type NixOSToplevelCacheRecord struct { + ConfigKey string + Toplevel string + UpdatedAt time.Time +} + +func (d *DB) GetNixOSToplevelCacheRecord(configKey string) (*NixOSToplevelCacheRecord, error) { + var record NixOSToplevelCacheRecord + var updatedAtStr string + err := d.QueryRow( + `select config_key, toplevel, updated_at from nixos_toplevel_cache where config_key = ?`, + configKey, + ).Scan(&record.ConfigKey, &record.Toplevel, &updatedAtStr) + if err != nil { + return nil, err + } + updatedAt, err := time.Parse(time.RFC3339, updatedAtStr) + if err != nil { + return nil, err + } + record.UpdatedAt = updatedAt + return &record, nil +} + +func (d *DB) SaveNixOSToplevelCacheRecord(configKey, toplevel string) error { + _, err := d.Exec( + `insert into nixos_toplevel_cache (config_key, toplevel, updated_at) + values (?, ?, ?) + on conflict(config_key) do update set + toplevel = excluded.toplevel, + updated_at = excluded.updated_at`, + configKey, toplevel, time.Now().UTC().Format(time.RFC3339), + ) + return err +} diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index b7927d2e..b83fdae6 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -20,7 +20,11 @@ var ( ErrWorkflowFailed = errors.New("workflow failed") ) -func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, workflowSem chan struct{}, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { +type workflowFinalizer interface { + FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error +} + +func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { l.Info("starting all workflows in parallel", "pipeline", pipelineId) // extract secrets @@ -74,16 +78,28 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, d defer wfLogger.Close() } + l.Info("waiting for slot", "wid", wid) + slot := WorkflowSlot(NoopSlot{}) + if s, ok := eng.(WorkflowSlotter); ok { + var err error + slot, err = s.AcquireWorkflowSlot(ctx, wid, &w) + if err != nil { + l.Error("failed to acquire slot", "wid", wid, "err", err) + dbErr := db.StatusFailed(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) + } + return + } + } + defer slot.Release() + err = db.StatusRunning(wid, n) if err != nil { l.Error("failed to set workflow status to running", "wid", wid, "err", err) return } - // acquire semaphore slot before starting the container - workflowSem <- struct{}{} - defer func() { <-workflowSem }() - err = eng.SetupWorkflow(ctx, wid, &w, wfLogger) if err != nil { // TODO(winter): Should this always set StatusFailed? @@ -139,6 +155,16 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, d } } + if finalizer, ok := eng.(workflowFinalizer); ok { + if err := finalizer.FinalizeWorkflow(ctx, wid, &w, wfLogger); err != nil { + dbErr := db.StatusFailed(wid, err.Error(), -1, n) + if dbErr != nil { + l.Error("failed to set workflow status to failed", "wid", wid, "err", dbErr) + } + return + } + } + err = db.StatusSuccess(wid, n) if err != nil { l.Error("failed to set workflow status to success", "wid", wid, "err", err) diff --git a/spindle/engine/scheduler.go b/spindle/engine/scheduler.go new file mode 100644 index 00000000..c067a63f --- /dev/null +++ b/spindle/engine/scheduler.go @@ -0,0 +1,139 @@ +package engine + +import ( + "context" + "fmt" + "slices" + "sync" + "time" +) + +const defaultAgingThreshold = 30 * time.Second + +type Resources[Self any] interface { + Fits(Self) bool + Add(Self) Self + Sub(Self) Self +} + +type ResourceScheduler[R Resources[R]] struct { + mu sync.Mutex + budget R + max R + used R + queue []*resourceWaiter[R] + now func() time.Time // get time now, is a field for mocking + agingThreshold time.Duration +} + +type resourceWaiter[R Resources[R]] struct { + req R + ready chan struct{} + enqueuedAt time.Time +} + +type resourceLease[R Resources[R]] struct { + scheduler *ResourceScheduler[R] + req R + once sync.Once +} + +func NewResourceScheduler[R Resources[R]](budget, max R, agingThreshold time.Duration) *ResourceScheduler[R] { + if agingThreshold <= 0 { + agingThreshold = defaultAgingThreshold + } + return &ResourceScheduler[R]{ + budget: budget, + max: max, + now: time.Now, + agingThreshold: agingThreshold, + } +} + +func (s *ResourceScheduler[R]) Acquire(ctx context.Context, req R) (WorkflowSlot, error) { + if s == nil { + return NoopSlot{}, nil + } + + s.mu.Lock() + if !req.Fits(s.budget) || !req.Fits(s.max) { + s.mu.Unlock() + return nil, fmt.Errorf("%w: request=%v budget=%v max=%v", ErrNoWorkflowSlots, req, s.budget, s.max) + } + if len(s.queue) == 0 && s.used.Add(req).Fits(s.budget) { + s.used = s.used.Add(req) + s.mu.Unlock() + return &resourceLease[R]{scheduler: s, req: req}, nil + } + + waiter := &resourceWaiter[R]{req: req, ready: make(chan struct{}), enqueuedAt: s.now()} + s.queue = append(s.queue, waiter) + s.schedule() + s.mu.Unlock() + + select { + case <-waiter.ready: + return &resourceLease[R]{scheduler: s, req: req}, nil + case <-ctx.Done(): + s.mu.Lock() + select { + case <-waiter.ready: + // undo committed resources, schedule already did that + s.used = s.used.Sub(req) + default: + // still in queue, just remove + s.remove(waiter) + } + s.schedule() + s.mu.Unlock() + return nil, ctx.Err() + } +} + +func (l *resourceLease[R]) Release() { + if l == nil || l.scheduler == nil { + return + } + l.once.Do(func() { + l.scheduler.release(l.req) + }) +} + +func (s *ResourceScheduler[R]) release(req R) { + s.mu.Lock() + defer s.mu.Unlock() + s.used = s.used.Sub(req) + s.schedule() +} + +// start every waiter whose request fits. once a waiter is older than +// agingThreshold, count its request as already used so younger waiters +// stop being scheduled ahead of it. +func (s *ResourceScheduler[R]) schedule() { + var reserved R + now := s.now() + i := 0 + for i < len(s.queue) { + w := s.queue[i] + if s.used.Add(reserved).Add(w.req).Fits(s.budget) { + s.queue = slices.Delete(s.queue, i, i+1) + s.used = s.used.Add(w.req) + close(w.ready) + continue + } + if now.Sub(w.enqueuedAt) >= s.agingThreshold { + reserved = reserved.Add(w.req) + } + i++ + } +} + +func (s *ResourceScheduler[R]) remove(waiter *resourceWaiter[R]) { + for i, candidate := range s.queue { + if candidate != waiter { + continue + } + s.queue = slices.Delete(s.queue, i, i+1) + return + } +} diff --git a/spindle/engine/scheduler_test.go b/spindle/engine/scheduler_test.go new file mode 100644 index 00000000..99e0630c --- /dev/null +++ b/spindle/engine/scheduler_test.go @@ -0,0 +1,200 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// resources for testing +type ru struct{ a, b int64 } + +func (r ru) Fits(limit ru) bool { + if limit.a > 0 && r.a > limit.a { + return false + } + if limit.b > 0 && r.b > limit.b { + return false + } + return true +} +func (r ru) Add(o ru) ru { return ru{r.a + o.a, r.b + o.b} } +func (r ru) Sub(o ru) ru { return ru{max(0, r.a-o.a), max(0, r.b-o.b)} } +func (r ru) String() string { + return fmt.Sprintf("a=%d b=%d", r.a, r.b) +} + +type acquireResult struct { + slot WorkflowSlot + err error +} + +func TestResourceSchedulerZeroLimitsDoNotApply(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{}, ru{}, 0) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 1 << 20, b: 1 << 20}) + if err != nil { + t.Fatalf("Acquire() error = %v", err) + } + slot.Release() +} + +func TestResourceSchedulerRejectsRequestsThatCanNeverFit(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024, b: 10_000}, ru{a: 512, b: 5_000}, 0) + + _, err := scheduler.Acquire(context.Background(), ru{a: 768, b: 100}) + if !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) + } + + _, err = scheduler.Acquire(context.Background(), ru{a: 128, b: 12_000}) + if !errors.Is(err, ErrNoWorkflowSlots) { + t.Fatalf("Acquire() error = %v, want ErrNoWorkflowSlots", err) + } +} + +func TestResourceSchedulerWaitsUntilResourcesAreReleased(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 0) + + first, err := scheduler.Acquire(context.Background(), ru{a: 1024}) + if err != nil { + t.Fatalf("first Acquire() error = %v", err) + } + defer first.Release() + + ch := acquireAsync(context.Background(), scheduler, ru{a: 1}) + assertAcquireBlocked(t, ch) + + first.Release() + first = NoopSlot{} + + second := waitAcquireOK(t, ch) + second.Release() +} + +func TestResourceSchedulerReleaseIsIdempotent(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1}, ru{}, 0) + + slot, err := scheduler.Acquire(context.Background(), ru{a: 1}) + if err != nil { + t.Fatalf("Acquire() error = %v", err) + } + + slot.Release() + slot.Release() + + second, err := scheduler.Acquire(context.Background(), ru{a: 1}) + if err != nil { + t.Fatalf("Acquire() after double release error = %v", err) + } + second.Release() +} + +func TestResourceSchedulerBackfillsPastBlockedHead(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, time.Hour) // disable aging so we test pure backfill + + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + if err != nil { + t.Fatalf("hold Acquire() error = %v", err) + } + defer hold.Release() + + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) + assertAcquireBlocked(t, bigCh) + + smallCh := acquireAsync(context.Background(), scheduler, ru{a: 256}) + small := waitAcquireOK(t, smallCh) + small.Release() + + assertAcquireBlocked(t, bigCh) +} + +func TestResourceSchedulerAgingReservesCapacityForBlockedHead(t *testing.T) { + t.Parallel() + + scheduler := NewResourceScheduler(ru{a: 1024}, ru{}, 10*time.Millisecond) + fakeNow := time.Now() + scheduler.now = func() time.Time { return fakeNow } + + hold, err := scheduler.Acquire(context.Background(), ru{a: 512}) + if err != nil { + t.Fatalf("hold Acquire() error = %v", err) + } + + bigCh := acquireAsync(context.Background(), scheduler, ru{a: 768}) + assertAcquireBlocked(t, bigCh) + + fakeNow = fakeNow.Add(time.Second) + + // big is now aged and reserves its 768. a 256 request would fit + // alongside the held 512, but the reservation blocks it. + smallCh := acquireAsync(context.Background(), scheduler, ru{a: 256}) + assertAcquireBlocked(t, smallCh) + + hold.Release() + + big := waitAcquireOK(t, bigCh) + small := waitAcquireOK(t, smallCh) + small.Release() + big.Release() +} + +func acquireAsync(ctx context.Context, scheduler *ResourceScheduler[ru], req ru) <-chan acquireResult { + ch := make(chan acquireResult, 1) + go func() { + slot, err := scheduler.Acquire(ctx, req) + ch <- acquireResult{slot: slot, err: err} + }() + return ch +} + +func assertAcquireBlocked(t *testing.T, ch <-chan acquireResult) { + t.Helper() + + select { + case res := <-ch: + if res.slot != nil { + res.slot.Release() + } + t.Fatalf("Acquire() returned before resources were available: err=%v", res.err) + case <-time.After(25 * time.Millisecond): + } +} + +func waitAcquireOK(t *testing.T, ch <-chan acquireResult) WorkflowSlot { + t.Helper() + + res := waitAcquireResult(t, ch) + if res.err != nil { + t.Fatalf("Acquire() error = %v", res.err) + } + if res.slot == nil { + t.Fatal("Acquire() returned nil slot") + } + return res.slot +} + +func waitAcquireResult(t *testing.T, ch <-chan acquireResult) acquireResult { + t.Helper() + + select { + case res := <-ch: + return res + case <-time.After(time.Second): + t.Fatal("timed out waiting for Acquire() result") + } + + return acquireResult{} +} diff --git a/spindle/engine/slot.go b/spindle/engine/slot.go new file mode 100644 index 00000000..667989e5 --- /dev/null +++ b/spindle/engine/slot.go @@ -0,0 +1,54 @@ +package engine + +import ( + "context" + "errors" + + "tangled.org/core/spindle/models" +) + +var ErrNoWorkflowSlots = errors.New("no workflow slots available") + +type WorkflowSlot interface { + Release() +} + +type WorkflowSlotter interface { + AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) +} + +type releaseFunc func() + +func (f releaseFunc) Release() { + if f != nil { + f() + } +} + +type NoopSlot struct{} + +func (NoopSlot) Release() {} + +// limit by concurrent workflow count +type SemaphoreSlotter struct { + slots chan struct{} +} + +func NewSemaphoreSlotter(maxConcurrent int) *SemaphoreSlotter { + if maxConcurrent <= 0 { + return &SemaphoreSlotter{} + } + return &SemaphoreSlotter{slots: make(chan struct{}, maxConcurrent)} +} + +func (a *SemaphoreSlotter) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (WorkflowSlot, error) { + if a == nil || a.slots == nil { + return NoopSlot{}, nil + } + select { + case a.slots <- struct{}{}: + return releaseFunc(func() { <-a.slots }), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} diff --git a/spindle/engine/slot_test.go b/spindle/engine/slot_test.go new file mode 100644 index 00000000..506e8b75 --- /dev/null +++ b/spindle/engine/slot_test.go @@ -0,0 +1,112 @@ +package engine + +import ( + "context" + "errors" + "testing" + "time" + + "tangled.org/core/spindle/models" +) + +func TestSemaphoreSlotterDisabledDoesNotBlock(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(0) + + for range 10 { + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("AcquireWorkflowSlot() error = %v", err) + } + slot.Release() + } +} + +func TestSemaphoreSlotterBlocksUntilRelease(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(1) + + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("first AcquireWorkflowSlot() error = %v", err) + } + releasedFirst := false + defer func() { + if !releasedFirst { + first.Release() + } + }() + + acquired := make(chan WorkflowSlot, 1) + errs := make(chan error, 1) + go func() { + slot, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + errs <- err + return + } + acquired <- slot + }() + + assertNotAcquired(t, acquired, errs) + + first.Release() + releasedFirst = true + + second := waitForSlot(t, acquired, errs) + second.Release() +} + +func TestSemaphoreSlotterHonorsContextCancellation(t *testing.T) { + t.Parallel() + + slotter := NewSemaphoreSlotter(1) + + first, err := slotter.AcquireWorkflowSlot(context.Background(), zeroWorkflowID(), nil) + if err != nil { + t.Fatalf("first AcquireWorkflowSlot() error = %v", err) + } + defer first.Release() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err = slotter.AcquireWorkflowSlot(ctx, zeroWorkflowID(), nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("AcquireWorkflowSlot() error = %v, want context.Canceled", err) + } +} + +func assertNotAcquired(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) { + t.Helper() + + select { + case slot := <-acquired: + slot.Release() + t.Fatal("AcquireWorkflowSlot() acquired a slot before one was released") + case err := <-errs: + t.Fatalf("AcquireWorkflowSlot() returned unexpected error: %v", err) + case <-time.After(25 * time.Millisecond): + } +} + +func waitForSlot(t *testing.T, acquired <-chan WorkflowSlot, errs <-chan error) WorkflowSlot { + t.Helper() + + select { + case slot := <-acquired: + return slot + case err := <-errs: + t.Fatalf("AcquireWorkflowSlot() returned error: %v", err) + case <-time.After(time.Second): + t.Fatal("timed out waiting for slot acquisition") + } + + return nil +} + +func zeroWorkflowID() models.WorkflowId { + return models.WorkflowId{} +} diff --git a/spindle/engines/microvm/README.md b/spindle/engines/microvm/README.md new file mode 100644 index 00000000..b014f98d --- /dev/null +++ b/spindle/engines/microvm/README.md @@ -0,0 +1,208 @@ +# spindle microVM engine + +This document describes the architecture of the microvm engine for spindle. In +short it allows the spindle to spin up microvm guests, and implements a guest +[agent protocol](../../agentproto) for communicating with those guests (via the +[shuttle](../../../shuttle) implementation of that proto). It implements some +fairly simple resource budgeting and optionally sets up cgroups for better +enforcing resource limits, and hardens the VM network access. It has Nix cache +integration for any paths built in the VM, those will get pushed to a Nix cache +by the spindle (if one is configured). The runner is abstracted behind an +interface; right now only the QEMU microVM impl is supported, but others (e.g. +firecracker) can slot in later. + +Currently two kinds of images are supported: + +- NixOS images: these allow configuration such as `dependencies`, `services`, + `virtualisation`, `registry`, `caches` in the workflow file itself. The guest + agent will build (or if it's cached, spindle will send the store path for + realization) and activate it before any workflow steps are ran. +- Non-NixOS: this is mainly just Alpine for now, but can be anything else. + Workflow-level configuration like NixOS aren't supported while using these. If + Nix exists inside the image (like in our Alpine image) it will still be able + to make use of the spindle cache. + +(For testing, you can run `bash spindle/engines/microvm/test-spindle-microvm.sh` +from repo root. These test the Alpine & NixOS, and features like if Docker +works, public internet is reachable, and so on.) + +## Image builds + +Image builds right now are done via Nix: + +- For NixOS, we use [microvm.nix](https://github.com/microvm-nix/microvm.nix), + and layer our own configs on-top, see [here](../../../nix/microvm). +- For Alpine we have a small-ish Nix definition that includes fetching the + kernel, initrd, kernel modules; setting up the init script that configures the + VM proper; copying dependencies (like `nix` or `git`) into a rootfs and + creating a squashfs from it. + +This does not mean it *has* to be done via Nix, as long as your images are what +spindle expects, they should work. That is: +- a guest agent is present inside of the image and when that image boots it will + get started, +- `spindle-workflow` user exists, +- and the work directory is configured (`/workspace`). + +## Image discovery + +Each built image ships with a `spec.json` next to its artifacts. This spec +describes everything needed to run the image: the kernel, initrd and read-only +store disk paths, boot args, memory/vCPU sizing, the shell used for workflow +steps, writable volumes, network interfaces, and runner-specific config (machine +type, CPU, extra args for QEMU). NixOS images also carry a `baseConfigHash` +identifying the base configuration baked into the image. + +An image lives in the configured image directory either as a directory +containing a `spec.json` (alongside the kernel/initrd/store-disk artifacts) or, +for a self-contained spec, as a flat `.json` file. An operator keeping +multiple arches side by side can name them `-` (eg. `nixos-x86_64`, +`alpine-aarch64`); that arch suffix is just part of the name, not something +resolution infers. + +A workflow names an image with the `image` key at top-level (falling back to +`SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE` if unset). The name is matched +literally: we look for `` (a directory with a `spec.json`) then +`.json`. Resolution depends only on the name and what is on disk, never on +the host, so the same workflow resolves identically on every spindle. If for +example an operator wants `nixos` to work, they can symlink `nixos` to +`nixos-x86_64`. + +The spec is validated at resolve time (required fields, positive sizes etc.), +and right before launch we also check the referenced files actually exist on +disk and that the host has the commands we need: `mkfs.ext4` for volume +formatting, plus whatever the selected runner requires. For QEMU that's the QEMU +binary for the spec's arch, `/dev/vhost-vsock`, `/dev/kvm` (if KVM is enabled), +and the `ip`, `mount`, `slirp4netns`, `unshare` toolchain when the image has +network interfaces. + +## microVM lifecycle + +```mermaid +flowchart LR + Init["InitWorkflow
parse manifest, resolve image, build steps"] + Acquire["AcquireWorkflowSlot
queue until resources fit budget"] + Setup["SetupWorkflow
proxies, VM, agent handshake"] + Run["RunStep ×N
exec via agent"] + Destroy["DestroyWorkflow
drain cache, poweroff, cleanup"] + + Init --> Acquire --> Setup --> Run --> Destroy +``` + +While a workflow is running, things look like this (everything inside the cgroup +box is what gets resource-limited): + +```mermaid +flowchart LR + subgraph Host["spindle host"] + Hub["agent hub"] + ReadProxy["read cache proxy"] + UploadProxy["upload cache proxy"] + subgraph Cgroup["per-workflow cgroup"] + QEMU["qemu"] + Slirp["slirp4netns"] + end + end + + subgraph Guest["guest"] + Agent["guest agent"] + end + + Agent -->|"vsock"| Hub + Agent -->|substitutions| ReadProxy + Agent -->|built paths| UploadProxy + QEMU --- Guest + Slirp -->|outbound only| Internet["the internet"] + ReadProxy --> Substituters["upstream caches"] + UploadProxy --> NixCache["spindle nix cache"] +``` + +`InitWorkflow` parses the workflow manifest, resolves the image, and assembles +the step list: the clone step first, then (for NixOS images with a workflow +config) a "NixOS config activation" system step, then the user steps. Before any +of this actually runs the workflow has to acquire a slot from the resource +scheduler, each image declares its memory/vCPUs/disk and workflows queue until +their request fits within the configured budget. The scheduler is +work-conserving with aging and per-user fairness, so one user submitting a pile +of jobs won't starve everyone else, and slots don't sit idle while there's +queued work that fits in the budget. + +### Configuration + +Setup allocates a random vsock CID for the guest and registers it with the agent +hub, which listens on a single host vsock port. Incoming agent connections are +matched to workflows by CID, anything with an unknown CID is dropped. It then +creates a per-workflow work directory and starts three host-side proxies the guest +reaches over vsock: a read cache proxy (fronting the configured Nix substituters +plus any workflow-level `caches`) and an upload cache proxy (for pushing paths +built in the guest to the spindle's cache), plus a DNS proxy that resolves +through the host's resolver and filters private/special-purpose address answers. + +Then the VM itself. Writable volumes from the spec are created as sparse files +and formatted ext4, the store disk is attached read-only. QEMU runs with +`-sandbox on`, `-nodefaults`, no display/monitor, etc., serial output to a log +file, and a QMP socket for control. + +For network hardening: if the image has network interfaces, QEMU doesn't run in +the host network namespace at all. We `unshare` into fresh user/net/mount +namespaces, and a small wrapper script inside the namespace bind-mounts a +resolv.conf that disables qemu's slirp DNS and adds blackhole routes for every +special-use IPv4/IPv6 range (RFC 6890, so private networks, link-local, +loopback, CGNAT, multicast, ULAs and so on) before exec'ing QEMU. `slirp4netns` +(with `--disable-host-loopback`, sandbox and seccomp enabled) then provides +outbound connectivity for the namespace. The guest's `/etc/resolv.conf` points +at shuttle on localhost; shuttle forwards DNS packets over vsock to the +host-side DNS proxy. The guest sits behind a second layer of QEMU user-mode +networking inside that namespace, so guest traffic can only ever reach the +outside world, never the host or anything on its local networks. + +Optionally the whole thing (QEMU and slirp4netns) is placed in a per-workflow +cgroup with memory, swap and pids limits, so the budget above is actually +enforced and not just bookkeeping. That also allows us to, for example, if the +cgroup OOM-kills the VM we can detect that and report it as such instead of a +generic crash. The spindle supervisor itself also gets a cgroup with a +protected `memory.min`, so under host memory pressure it's the workflows that +get OOM-killed first, not spindle. + +### Boot - run - death + +Once QEMU is up we poll the QMP socket until it accepts a connection and reports +the guest as running, then wait for the guest agent to send handshake message +over vsock from the expected CID. It reports its protocol and versions, and +spindle sends it the job id, trusted cache public keys, and the cache/DNS proxy +ports. + +First the activation step is ran (if on a NixOS image and the workflow is +configured with anything), spindle sends the user config (or a cached toplevel +store path, if we've built this exact base + config combo before) and the agent +builds and activates it before the user steps run. Afterwards, each step is sent +as an exec request (`$shell -lc ` as an unprivileged workflow user in +`/workspace/repo`, with workflow/step environment and unlocked secrets), and +stdout/stderr stream back as messages until an exit message arrives. Timeouts +are cooperative: we derive a deadline from the workflow timeout and ship it to +the guest, with a little grace on the host side so the guest gets to report the +timeout itself. While a step runs we also watch for the VM crashing, if it does +we tail the serial (and qemu) logs into the step's stderr so you get something +more useful than "guest agent connection lost: EOF". + +Teardown is same whether the workflow succeeded, failed or timed out: drain the +guest's pending Nix cache uploads, ask the agent to power off and wait for QEMU +to exit (falling back to QMP `system_powerdown` and finally a kill if it +doesn't), then close the proxies and remove the work directory. + +### Nix cache + +The two host-side proxies are how the guest talks to spindle's Nix cache without +ever needing credentials or direct network access; like the agent they reach the +host over vsock. + +The read proxy fronts the configured substituters plus any workflow-level +`caches`. When the guest needs to realize a store path it asks the proxy, which +queries the read caches concurrently and returns the first successful response, +with a 404 only winning if every upstream returns 404. + +The upload proxy goes the other way: paths built inside the guest are pushed to +spindle's configured upload cache (if any) so the next workflow that needs them +doesn't rebuild. Paths already present on any configured read cache are skipped. +The agent queues built paths and they're uploaded eagerly as they appear; any +still in flight at teardown block the drain step until they finish. diff --git a/spindle/engines/microvm/agent.go b/spindle/engines/microvm/agent.go new file mode 100644 index 00000000..4173c8a4 --- /dev/null +++ b/spindle/engines/microvm/agent.go @@ -0,0 +1,368 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "sync" + "time" + + "github.com/mdlayher/vsock" + + "tangled.org/core/spindle/agentproto" + agentv1 "tangled.org/core/spindle/agentproto/gen" +) + +const guestWorkflowUser = "spindle-workflow" + +var errGuestTimedOut = errors.New("guest reported step timed out") + +type agentHub struct { + l *slog.Logger + ln *vsock.Listener + pending map[uint32]chan net.Conn + mu sync.Mutex +} + +func newAgentHub(port uint32, l *slog.Logger) (*agentHub, error) { + ln, err := vsock.Listen(port, nil) + if err != nil { + return nil, fmt.Errorf("listen for agent on vsock port %d: %w", port, err) + } + h := &agentHub{ + l: l, + ln: ln, + pending: make(map[uint32]chan net.Conn), + } + go h.acceptLoop() + return h, nil +} + +func (h *agentHub) expect(cid uint32) (<-chan net.Conn, func(), error) { + h.mu.Lock() + defer h.mu.Unlock() + if _, exists := h.pending[cid]; exists { + return nil, nil, fmt.Errorf("already waiting for agent cid %d", cid) + } + ch := make(chan net.Conn, 1) + h.pending[cid] = ch + unregister := func() { + h.mu.Lock() + delete(h.pending, cid) + h.mu.Unlock() + close(ch) + for conn := range ch { + if conn != nil { + _ = conn.Close() + } + } + } + return ch, unregister, nil +} + +func (h *agentHub) acceptLoop() { + for { + conn, err := h.ln.Accept() + if err != nil { + h.l.Error("agent vsock accept failed", "error", err) + return + } + + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if !ok { + h.l.Warn("agent connection has unexpected remote address", "remote", conn.RemoteAddr()) + _ = conn.Close() + continue + } + + h.mu.Lock() + ch, ok := h.pending[addr.ContextID] + if ok { + delete(h.pending, addr.ContextID) + } + h.mu.Unlock() + + // todo: if / when we add agent recovery (reconnect) we should add a + // boot-initialized session credential to prevent random connections... + // checking cid here works to ensure for now since we dont attempt to + // reconnect, so we block anything else thats not expected (and agent + // runs first in the boot sequence always). + if !ok { + h.l.Warn("dropping agent connection for unknown cid", "cid", addr.ContextID) + _ = conn.Close() + continue + } + + select { + case ch <- conn: + default: + _ = conn.Close() + } + } +} + +type AgentExec struct { + *agentv1.ExecStart + ID string + Stdout io.Writer + Stderr io.Writer +} + +type AgentSession struct { + conn net.Conn + enc *agentproto.Encoder + dec *agentproto.Decoder + l *slog.Logger + mu sync.Mutex +} + +func NewAgentSession(conn net.Conn, l *slog.Logger) *AgentSession { + return &AgentSession{ + conn: conn, + enc: agentproto.NewEncoder(conn), + dec: agentproto.NewDecoder(conn), + l: l, + } +} + +func (s *AgentSession) Init(ctx context.Context, init *agentv1.Init) error { + s.mu.Lock() + defer s.mu.Unlock() + + hello, err := s.decode(ctx) + if err != nil { + return fmt.Errorf("read agent hello: %w", err) + } + helloPayload := hello.Hello + if helloPayload == nil { + return fmt.Errorf("expected agent hello, got nil") + } + s.l.Info("agent connected", "protocol", helloPayload.ProtocolVersion, "version", helloPayload.AgentVersion, "boot", helloPayload.BootId, "nix", helloPayload.NixVersion) + + if err := s.enc.Encode(&agentproto.Message{ + Id: "init", + Init: init, + }); err != nil { + return fmt.Errorf("send agent init: %w", err) + } + return nil +} + +func (s *AgentSession) Exec(ctx context.Context, exec AgentExec) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if exec.ID == "" { + return 0, fmt.Errorf("empty ID passed to Exec") + } + + if exec.ExecStart.TimeoutSeconds == 0 { + exec.ExecStart.TimeoutSeconds = timeoutSeconds(ctx, guestTimeoutGrace) + } + + if err := s.enc.Encode(&agentproto.Message{ + Id: exec.ID, + ExecStart: exec.ExecStart, + }); err != nil { + return 0, fmt.Errorf("send exec_start: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return 0, err + } + if msg.BuiltPaths == nil && msg.Id != exec.ID { + continue + } + + if p := msg.ExecStdout; p != nil { + _, _ = io.WriteString(exec.Stdout, p.Data) + } else if p := msg.ExecStderr; p != nil { + _, _ = io.WriteString(exec.Stderr, p.Data) + } else if p := msg.BuiltPaths; p != nil { + // s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths)) + } else if p := msg.ExecExit; p != nil { + if p.Error != "" { + s.l.Warn("guest exec error", "id", msg.Id, "error", p.Error) + } + if p.TimedOut { + return int(p.ExitCode), errGuestTimedOut + } + return int(p.ExitCode), nil + } + } +} + +func (s *AgentSession) ActivateConfig(ctx context.Context, id string, req *agentv1.ActivateConfig) (*agentv1.ActivateConfigResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if id == "" { + return nil, fmt.Errorf("empty ID passed to ActivateConfig") + } + if req.TimeoutSeconds == 0 { + req.TimeoutSeconds = timeoutSeconds(ctx, guestTimeoutGrace) + } + if err := s.enc.Encode(&agentproto.Message{ + Id: id, + ActivateConfig: req, + }); err != nil { + return nil, fmt.Errorf("send activate_config: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return nil, err + } + if msg.BuiltPaths == nil && msg.Id != id { + continue + } + + if p := msg.BuiltPaths; p != nil { + // s.l.Debug("guest built paths", "reason", p.Reason, "count", len(p.Paths)) + } else if p := msg.ActivateConfigResult; p != nil { + if p.Error != "" { + return nil, fmt.Errorf("activate config failed: %s", p.Error) + } + if p.Toplevel == "" { + return nil, fmt.Errorf("activate config returned empty toplevel") + } + return p, nil + } + } +} + +func (s *AgentSession) Poweroff(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + + id := "poweroff" + if err := s.enc.Encode(&agentproto.Message{ + Id: id, + Poweroff: &agentv1.Poweroff{}, + }); err != nil { + return fmt.Errorf("send poweroff: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return err + } + if msg.Id != id { + continue + } + p := msg.PoweroffResult + if p == nil { + continue + } + if p.Error != "" { + return fmt.Errorf("guest poweroff failed: %s", p.Error) + } + return nil + } +} + +func (s *AgentSession) Drain(ctx context.Context) (uint32, error) { + s.mu.Lock() + defer s.mu.Unlock() + + drainID := "cache-drain" + if err := s.enc.Encode(&agentproto.Message{ + Id: drainID, + CacheDrain: &agentv1.CacheDrain{ + TimeoutSeconds: timeoutSeconds(ctx, 0), + }, + }); err != nil { + return 0, fmt.Errorf("send cache_drain: %w", err) + } + + for { + msg, err := s.decode(ctx) + if err != nil { + return 0, err + } + if msg.Id != drainID { + continue + } + p := msg.CacheDrainResult + if p == nil { + continue + } + s.l.Info("cache drain complete", "uploaded", p.CacheUploaded, "failed", p.CacheFailed, "queued", p.CacheQueued, "active", p.CacheActive) + if p.Error != "" { + return 0, fmt.Errorf("cache drain failed: %s", p.Error) + } + if p.CacheFailed > 0 { + return 0, fmt.Errorf("cache drain failed for %d paths", p.CacheFailed) + } + if p.CacheQueued > 0 || p.CacheActive > 0 { + return 0, fmt.Errorf("cache drain incomplete: queued=%d active=%d", p.CacheQueued, p.CacheActive) + } + return p.CacheUploaded, nil + } +} + +func (s *AgentSession) decode(ctx context.Context) (*agentproto.Message, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + if deadline, ok := ctx.Deadline(); ok { + _ = s.conn.SetReadDeadline(deadline) + } else { + _ = s.conn.SetReadDeadline(time.Time{}) + } + + // a blocked vsock read wont wake up just from the ctx being cancelled, + // only a deadline will wake it up, so if the VM crashes mid-step the read would + // hang until workflow timeout. so we will set a deadline in the past to cancel it. + // + // we set a deadline here instead of closing the connection, this is the long-lived + // connection that everything reuses, so we only really want to interrupt it for this + // current read. this also lands as a timeout error which the netErr.Timeout() check + // below maps to ctx.Err() correctly + stop := context.AfterFunc(ctx, func() { + _ = s.conn.SetReadDeadline(time.Now()) + }) + defer stop() + + msg, err := s.dec.Decode() + if err != nil { + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() && ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("read agent message: %w", err) + } + return msg, nil +} + +func (s *AgentSession) Close() error { + if s == nil || s.conn == nil { + return nil + } + return s.conn.Close() +} + +// this pulls the deadline from the context and converts it to what the +// agentproto expects +func timeoutSeconds(ctx context.Context, lead time.Duration) uint32 { + deadline, ok := ctx.Deadline() + if !ok { + return 0 + } + seconds := int64((time.Until(deadline) - lead).Round(time.Second) / time.Second) + if seconds < 1 { + return 1 + } + if seconds > int64(^uint32(0)) { + return ^uint32(0) + } + return uint32(seconds) +} diff --git a/spindle/engines/microvm/args.go b/spindle/engines/microvm/args.go new file mode 100644 index 00000000..05b370af --- /dev/null +++ b/spindle/engines/microvm/args.go @@ -0,0 +1,69 @@ +package microvm + +import ( + "fmt" + "strings" +) + +type argBuilder struct { + args []string +} + +func newArgBuilder(capacity int) argBuilder { + return argBuilder{ + args: make([]string, 0, capacity), + } +} + +func (b *argBuilder) Add(args ...string) *argBuilder { + b.args = append(b.args, args...) + return b +} + +func (b *argBuilder) Flag(name string) *argBuilder { + b.args = append(b.args, name) + return b +} + +func (b *argBuilder) Opt(name, value string) *argBuilder { + b.args = append(b.args, name, value) + return b +} + +func (b *argBuilder) Optf(name, format string, values ...any) *argBuilder { + return b.Opt(name, fmt.Sprintf(format, values...)) +} + +func (b *argBuilder) Args() []string { + args := make([]string, len(b.args)) + copy(args, b.args) + return args +} + +type optionBuilder struct { + parts []string +} + +func newOptionBuilder(capacity int) optionBuilder { + return optionBuilder{ + parts: make([]string, 0, capacity), + } +} + +func (b *optionBuilder) Add(parts ...string) *optionBuilder { + b.parts = append(b.parts, parts...) + return b +} + +func (b *optionBuilder) KV(key, value string) *optionBuilder { + b.parts = append(b.parts, key+"="+value) + return b +} + +func (b *optionBuilder) KVf(key, format string, values ...any) *optionBuilder { + return b.KV(key, fmt.Sprintf(format, values...)) +} + +func (b optionBuilder) String() string { + return strings.Join(b.parts, ",") +} diff --git a/spindle/engines/microvm/bench-boot.sh b/spindle/engines/microvm/bench-boot.sh new file mode 100755 index 00000000..f700eb6a --- /dev/null +++ b/spindle/engines/microvm/bench-boot.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# quick boot-time benchmark for the spindle nixos microvm. +# boots N times running a trivial command, reports wall-clock + systemd-analyze. +# needs: sudo modprobe vhost_vsock +set -euo pipefail + +N="${1:-5}" +cd "$(git rev-parse --show-toplevel)" + +strip_ansi() { sed -E "s/$(printf '\033')\[[0-9;]*[a-zA-Z]//g; s/$(printf '\033')\([a-zA-Z]//g"; } + +echo ">>> building runner + image" +nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run +TARBALL=$(nix build .#spindle-nixos-image-tarball --no-link --print-out-paths) + +WORK=$(mktemp -d -t spindle-bench-XXXXXX) +trap 'chmod -R +w "$WORK" 2>/dev/null || true; rm -rf "$WORK"' EXIT +mkdir -p "$WORK/image" +tar -C "$WORK/image" -xzf "$TARBALL" +SPEC="$WORK/image/spec.json" + +echo ">>> systemd-analyze breakdown" +spindle/spindle-microvm-run --image-spec "$SPEC" --work-dir "$WORK/analyze" --exec-timeout 60s -- \ + /run/current-system/sw/bin/systemd-analyze time 2>/dev/null | strip_ansi | grep -i startup || true + +echo ">>> $N timed boot+exec(true) runs" +total=0 +for i in $(seq 1 "$N"); do + start=$EPOCHREALTIME + spindle/spindle-microvm-run --image-spec "$SPEC" --work-dir "$WORK/run$i" --exec-timeout 60s -- \ + /run/current-system/sw/bin/true >/dev/null 2>&1 + end=$EPOCHREALTIME + ms=$(( (${end%.*} - ${start%.*}) * 1000 + (10#${end#*.} - 10#${start#*.}) / 1000 )) + echo " run $i: ${ms}ms" + total=$((total + ms)) + rm -rf "$WORK/run$i" +done +echo ">>> mean wall-clock: $((total / N))ms over $N runs" diff --git a/spindle/engines/microvm/budget.go b/spindle/engines/microvm/budget.go new file mode 100644 index 00000000..084d5061 --- /dev/null +++ b/spindle/engines/microvm/budget.go @@ -0,0 +1,91 @@ +package microvm + +import ( + "context" + "fmt" + "time" + + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" +) + +type Resources struct { + MemoryMiB int64 + VCPUs int64 + DiskMiB int64 +} + +func (r Resources) Fits(limit Resources) bool { + if limit.MemoryMiB > 0 && r.MemoryMiB > limit.MemoryMiB { + return false + } + if limit.VCPUs > 0 && r.VCPUs > limit.VCPUs { + return false + } + if limit.DiskMiB > 0 && r.DiskMiB > limit.DiskMiB { + return false + } + return true +} + +func (r Resources) Add(other Resources) Resources { + return Resources{ + MemoryMiB: r.MemoryMiB + other.MemoryMiB, + VCPUs: r.VCPUs + other.VCPUs, + DiskMiB: r.DiskMiB + other.DiskMiB, + } +} + +func (r Resources) Sub(other Resources) Resources { + return Resources{ + MemoryMiB: max(0, r.MemoryMiB-other.MemoryMiB), + VCPUs: max(0, r.VCPUs-other.VCPUs), + DiskMiB: max(0, r.DiskMiB-other.DiskMiB), + } +} + +func (r Resources) String() string { + return fmt.Sprintf("memory=%dMiB vcpus=%d disk=%dMiB", r.MemoryMiB, r.VCPUs, r.DiskMiB) +} + +func newVMBudgetConfig(cfg config.MicroVMPipelines) (Resources, Resources, time.Duration) { + budget := Resources{ + MemoryMiB: cfg.MaxTotalMemoryMiB, + VCPUs: cfg.MaxTotalVCPUs, + DiskMiB: cfg.MaxTotalDiskMiB, + } + maxReq := Resources{ + MemoryMiB: cfg.MaxWorkflowMemoryMiB, + VCPUs: cfg.MaxWorkflowVCPUs, + DiskMiB: cfg.MaxWorkflowDiskMiB, + } + return budget, maxReq, cfg.AgingThreshold +} + +func (e *Engine) AcquireWorkflowSlot(ctx context.Context, wid models.WorkflowId, wf *models.Workflow) (engine.WorkflowSlot, error) { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil { + return nil, fmt.Errorf("microVM workflow state is not initialized") + } + if e.scheduler == nil { + return engine.NoopSlot{}, nil + } + req := resourcesForImage(state.ImageSpec) + if req.MemoryMiB < 0 || req.VCPUs < 0 || req.DiskMiB < 0 { + return nil, fmt.Errorf("microVM resource request must not be negative: %s", req) + } + return e.scheduler.Acquire(ctx, req) +} + +func resourcesForImage(spec ImageSpec) Resources { + var diskMiB int64 + for _, volume := range spec.Volumes { + diskMiB += volume.SizeMiB + } + return Resources{ + MemoryMiB: int64(spec.MemoryMiB), + VCPUs: int64(spec.VCPUs), + DiskMiB: diskMiB, + } +} diff --git a/spindle/engines/microvm/cgroup.go b/spindle/engines/microvm/cgroup.go new file mode 100644 index 00000000..184a27b1 --- /dev/null +++ b/spindle/engines/microvm/cgroup.go @@ -0,0 +1,267 @@ +package microvm + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "strings" + + cgroups "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/prometheus/procfs" +) + +var ( + cgroupInvalidChar = regexp.MustCompile(`[^a-zA-Z0-9\-_.]`) + cgroupConsecutiveSep = regexp.MustCompile(`[-_.]{2,}`) +) + +const ( + cgroupParentSelf = "self" + supervisorCgroupName = "supervisor" +) + +type CgroupLimits struct { + Enabled bool + Parent *CgroupParent + Name string + MemoryMaxMiB int64 + SwapMaxMiB *int64 + PidsMax int64 +} + +type CgroupParent struct { + root *cgroup2.Manager + mountpoint string + group string +} + +type CgroupHandle struct { + manager *cgroup2.Manager +} + +func initCgroupParent(parent string, supervisorMemoryMinMiB int64, logger *slog.Logger) (*CgroupParent, error) { + if parent == "" { + parent = cgroupParentSelf + } + if cgroups.Mode() != cgroups.Unified { + return nil, fmt.Errorf("microVM cgroups require cgroup v2 unified mode") + } + + mountpoint, group, err := resolveCgroupParent(parent) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(mountpoint, strings.TrimPrefix(group, "/"))); err != nil { + return nil, fmt.Errorf("stat cgroup parent %q:%q: %w", mountpoint, group, err) + } + + root, err := cgroup2.Load(group, cgroup2.WithMountpoint(mountpoint)) + if err != nil { + return nil, fmt.Errorf("load cgroup parent %q:%q: %w", mountpoint, group, err) + } + + if group != "/" { + if err := moveParentProcesses(root, supervisorMemoryMinMiB, logger); err != nil { + return nil, err + } + } + + if logger != nil { + logger.Info("initialized microVM cgroup parent", "mountpoint", mountpoint, "group", group) + } + return &CgroupParent{root: root, mountpoint: mountpoint, group: group}, nil +} + +func prepareCgroup(limits CgroupLimits, logger *slog.Logger) (*CgroupHandle, error) { + if !limits.Enabled { + return nil, nil + } + if limits.Parent == nil || limits.Parent.root == nil { + return nil, fmt.Errorf("cgroup parent is not initialized") + } + name := sanitizeCgroupName(limits.Name) + if name == "" { + return nil, fmt.Errorf("cgroup name is empty") + } + + manager, err := limits.Parent.root.NewChild(name, cgroupResources(limits)) + if err != nil { + return nil, fmt.Errorf("create cgroup %q: %w", name, err) + } + + if logger != nil { + logger.Info("created microVM cgroup", "name", name, "parentGroup", limits.Parent.group) + } + return &CgroupHandle{manager: manager}, nil +} + +func cgroupResources(limits CgroupLimits) *cgroup2.Resources { + resources := &cgroup2.Resources{} + if limits.MemoryMaxMiB > 0 || limits.SwapMaxMiB != nil { + memory := &cgroup2.Memory{} + if limits.MemoryMaxMiB > 0 { + maxBytes := limits.MemoryMaxMiB * 1024 * 1024 + memory.Max = &maxBytes + } + if limits.SwapMaxMiB != nil { + swapBytes := *limits.SwapMaxMiB * 1024 * 1024 + memory.Swap = &swapBytes + } + oomGroup := true + memory.OOMGroup = &oomGroup + resources.Memory = memory + } + if limits.PidsMax > 0 { + resources.Pids = &cgroup2.Pids{Max: limits.PidsMax} + } + return resources +} + +func supervisorResources(memoryMinMiB int64) *cgroup2.Resources { + if memoryMinMiB <= 0 { + return nil + } + minBytes := memoryMinMiB * 1024 * 1024 + return &cgroup2.Resources{ + Memory: &cgroup2.Memory{Min: &minBytes}, + } +} + +func (h *CgroupHandle) AddProcess(pid int, logger *slog.Logger) error { + if h == nil || h.manager == nil { + return nil + } + if pid <= 0 { + return fmt.Errorf("invalid pid %d", pid) + } + if err := h.manager.AddProc(uint64(pid)); err != nil { + return fmt.Errorf("add pid %d to cgroup: %w", pid, err) + } + if logger != nil { + logger.Info("added process to microVM cgroup", "pid", pid) + } + return nil +} + +func (h *CgroupHandle) Close() error { + if h == nil || h.manager == nil { + return nil + } + return h.manager.Delete() +} + +func (h *CgroupHandle) OOMKilled() bool { + if h == nil || h.manager == nil { + return false + } + metrics, err := h.manager.Stat() + if err != nil || metrics == nil || metrics.MemoryEvents == nil { + return false + } + return metrics.MemoryEvents.OomKill > 0 +} + +func resolveCgroupParent(parent string) (string, string, error) { + mountpoint, err := cgroup2Mountpoint() + if err != nil { + return "", "", err + } + + if parent == "" || parent == cgroupParentSelf { + group, err := selfCgroupV2Path() + if err != nil { + return "", "", err + } + return mountpoint, group, nil + } + if !filepath.IsAbs(parent) { + return "", "", fmt.Errorf("cgroup parent must be %q or an absolute delegated cgroupfs path: %q", cgroupParentSelf, parent) + } + + cleanParent := filepath.Clean(parent) + rel, err := filepath.Rel(mountpoint, cleanParent) + if err != nil { + return "", "", fmt.Errorf("resolve cgroup parent %q relative to cgroup2 mount %q: %w", cleanParent, mountpoint, err) + } + if rel == ".." || strings.HasPrefix(rel, "../") { + return "", "", fmt.Errorf("cgroup parent %q is outside cgroup2 mount %q", cleanParent, mountpoint) + } + if rel == "." { + return mountpoint, "/", nil + } + + group := "/" + filepath.ToSlash(rel) + if err := cgroup2.VerifyGroupPath(group); err != nil { + return "", "", fmt.Errorf("invalid cgroup parent path %q: %w", group, err) + } + return mountpoint, group, nil +} + +func cgroup2Mountpoint() (string, error) { + mounts, err := procfs.GetMounts() + if err != nil { + return "", fmt.Errorf("read procfs mountinfo: %w", err) + } + for _, mount := range mounts { + if mount.FSType == "cgroup2" { + return mount.MountPoint, nil + } + } + return "", fmt.Errorf("cgroup v2 mountpoint not found") +} + +func selfCgroupV2Path() (string, error) { + self, err := procfs.Self() + if err != nil { + return "", fmt.Errorf("open procfs self: %w", err) + } + groups, err := self.Cgroups() + if err != nil { + return "", fmt.Errorf("read procfs self cgroups: %w", err) + } + for _, group := range groups { + if group.HierarchyID != 0 { + continue + } + path := group.Path + if path == "" { + path = "/" + } + if err := cgroup2.VerifyGroupPath(path); err != nil { + return "", fmt.Errorf("invalid self cgroup path %q: %w", path, err) + } + return path, nil + } + return "", fmt.Errorf("current process has no cgroup v2 hierarchy entry") +} + +func moveParentProcesses(parent *cgroup2.Manager, supervisorMemoryMinMiB int64, logger *slog.Logger) error { + supervisor, err := parent.NewChild(supervisorCgroupName, supervisorResources(supervisorMemoryMinMiB)) + if err != nil { + return fmt.Errorf("create supervisor cgroup: %w", err) + } + + procs, err := parent.Procs(false) + if err != nil { + return fmt.Errorf("list parent cgroup processes: %w", err) + } + for _, pid := range procs { + if err := supervisor.AddProc(pid); err != nil { + return fmt.Errorf("move pid %d to supervisor cgroup: %w", pid, err) + } + } + + if logger != nil && len(procs) > 0 { + logger.Info("moved spindle processes to supervisor cgroup", "processes", len(procs)) + } + return nil +} + +func sanitizeCgroupName(name string) string { + name = cgroupInvalidChar.ReplaceAllLiteralString(name, "-") + name = cgroupConsecutiveSep.ReplaceAllLiteralString(name, "-") + return strings.Trim(name, "-_.") +} diff --git a/spindle/engines/microvm/cgroup_oom_test.go b/spindle/engines/microvm/cgroup_oom_test.go new file mode 100644 index 00000000..5754b96a --- /dev/null +++ b/spindle/engines/microvm/cgroup_oom_test.go @@ -0,0 +1,126 @@ +package microvm + +import ( + "log/slog" + "os" + "os/exec" + "runtime" + "testing" + "time" + + cgroups "github.com/containerd/cgroups/v3" +) + +const memhogEnv = "SPINDLE_CGROUP_MEMHOG" + +func TestMain(m *testing.M) { + if os.Getenv(memhogEnv) == "1" { + runMemhogChild() + return + } + os.Exit(m.Run()) +} + +// this will allocate memory in steps until either the cgroup kills the process +// this is running on, or if the limit is reached. the limit is there so that if +// the cgroup somehow does not work, we don't kill the host and can observe that +// failure. +func runMemhogChild() { + var b [1]byte + _, _ = os.Stdin.Read(b[:]) + + const chunk = 4 << 20 // 4 MiB + const limit = 512 << 20 // safety cap + hold := make([][]byte, 0, limit/chunk) + for total := 0; total < limit; total += chunk { + c := make([]byte, chunk) + for i := range c { + c[i] = 1 // fault the pages in so they count against memory.current + } + hold = append(hold, c) + time.Sleep(5 * time.Millisecond) + } + runtime.KeepAlive(hold) + os.Exit(0) +} + +// creates a cgroup parent, adds a memory limited child to it, and creates a +// process that hogs memory and observes if it OOMs or not. +// +// run with: +// +// SPINDLE_CGROUP_INTEGRATION=1 systemd-run --user --scope -p Delegate=yes \ +// go test -run TestCgroupOOMEnforcement ./spindle/engines/microvm/ +func TestCgroupOOMEnforcement(t *testing.T) { + if os.Getenv("SPINDLE_CGROUP_INTEGRATION") != "1" { + t.Skip("see test doc comment on how to run") + } + if cgroups.Mode() != cgroups.Unified { + t.Skip("requires cgroup v2 unified mode") + } + + logger := slog.Default() + + parent, err := initCgroupParent(cgroupParentSelf, 0, logger) + if err != nil { + t.Skipf("cannot initialize cgroup parent (need cgroup v2 delegation): %v", err) + } + + swap := int64(0) // disable swap so the limit forces an OOM promptly + handle, err := prepareCgroup(CgroupLimits{ + Enabled: true, + Parent: parent, + Name: "cgtest-oom", + MemoryMaxMiB: 64, + SwapMaxMiB: &swap, + PidsMax: 256, + }, logger) + if err != nil { + t.Skipf("cannot create a memory-limited child cgroup (need the memory controller delegated): %v", err) + } + if handle == nil { + t.Fatal("prepareCgroup returned a nil handle for enabled limits") + } + t.Cleanup(func() { _ = handle.Close() }) + + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), memhogEnv+"=1") + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + + if err := handle.AddProcess(cmd.Process.Pid, logger); err != nil { + t.Fatalf("add memhog to cgroup: %v", err) + } + + // let the child process start allocating memory + if _, err := stdin.Write([]byte("g")); err != nil { + t.Fatalf("release memhog: %v", err) + } + _ = stdin.Close() + + waitErr := make(chan error, 1) + go func() { waitErr <- cmd.Wait() }() + + select { + case err := <-waitErr: + if err == nil { + t.Fatal("memhog exited cleanly: the cgroup memory limit was not enforced") + } + t.Logf("memhog died as expected: %v", err) + case <-time.After(30 * time.Second): + t.Fatal("memhog did not die within 30s, cgroup memory limit not enforced") + } + + if !handle.OOMKilled() { + t.Fatal("OOMKilled() is false after the memhog was killed, memory.events oom_kill was not observed") + } +} diff --git a/spindle/engines/microvm/cgroup_test.go b/spindle/engines/microvm/cgroup_test.go new file mode 100644 index 00000000..e8c54108 --- /dev/null +++ b/spindle/engines/microvm/cgroup_test.go @@ -0,0 +1,39 @@ +package microvm + +import ( + "testing" +) + +func TestSanitizeCgroupName(t *testing.T) { + cases := []struct { + in string + want string + }{ + {"workflow-abc123", "workflow-abc123"}, + {"a/b:c", "a-b-c"}, + {"--lead--", "lead"}, + {"a__b..c", "a-b-c"}, + {"keep.dots_and-dashes", "keep.dots_and-dashes"}, + {"", ""}, + {"///", ""}, + } + for _, tc := range cases { + if got := sanitizeCgroupName(tc.in); got != tc.want { + t.Errorf("sanitizeCgroupName(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestCgroupResourcesSwapOnlyStillSetsMemory(t *testing.T) { + swap := int64(8) + r := cgroupResources(CgroupLimits{SwapMaxMiB: &swap}) + if r.Memory == nil { + t.Fatal("a swap limit alone should still produce a memory controller config") + } + if r.Memory.Max != nil { + t.Errorf("memory max should be unset when only swap is limited, got %v", *r.Memory.Max) + } + if r.Memory.Swap == nil || *r.Memory.Swap != 8*1024*1024 { + t.Errorf("swap = %v, want %d bytes", r.Memory.Swap, 8*1024*1024) + } +} diff --git a/spindle/engines/microvm/dns_proxy.go b/spindle/engines/microvm/dns_proxy.go new file mode 100644 index 00000000..16e6183a --- /dev/null +++ b/spindle/engines/microvm/dns_proxy.go @@ -0,0 +1,380 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "sync" + "time" + + "github.com/miekg/dns" +) + +const ( + dnsProxyIOTimeout = 10 * time.Second + dnsProxyIdleTimeout = 30 * time.Second + dnsProxyShutdownTimeout = 10 * time.Second + dnsProxyMaxConnections = 64 + dnsProxyMaxTCPQueries = 128 + dnsProxyResolvConfPath = "/etc/resolv.conf" +) + +type DNSProxy struct { + port uint32 + srv *dns.Server + + closeOnce sync.Once + closeErr error +} + +func StartDNSProxy(ctx context.Context, cid uint32, logger *slog.Logger) (*DNSProxy, error) { + if ctx == nil { + ctx = context.Background() + } + + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "dns_proxy", "cid", cid) + + ln, port, err := listenRandomVsockPort(ctx) + if err != nil { + return nil, fmt.Errorf("listen for dns proxy: %w", err) + } + + resolver, err := newHostDNSResolver(dnsProxyResolvConfPath, logger) + if err != nil { + _ = ln.Close() + return nil, err + } + + listener := newLimitedListener( + &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + }, + dnsProxyMaxConnections, + logger, + ) + + proxy := &DNSProxy{ + port: port, + srv: &dns.Server{ + Net: "tcp", + Listener: listener, + Handler: dns.HandlerFunc(resolver.ServeDNS), + ReadTimeout: dnsProxyIOTimeout, + WriteTimeout: dnsProxyIOTimeout, + IdleTimeout: func() time.Duration { return dnsProxyIdleTimeout }, + MaxTCPQueries: dnsProxyMaxTCPQueries, + MsgInvalidFunc: func(_ []byte, err error) { + logger.Warn("dns proxy invalid message", "error", err) + }, + }, + } + + go func() { + <-ctx.Done() + _ = proxy.Close() + }() + + go func() { + if err := proxy.srv.ActivateAndServe(); err != nil && !errors.Is(err, net.ErrClosed) { + logger.Warn("dns proxy stopped", "error", err) + } + }() + + logger.Info("started dns proxy", "port", port) + return proxy, nil +} + +func (p *DNSProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *DNSProxy) Close() error { + if p == nil || p.srv == nil { + return nil + } + + p.closeOnce.Do(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), dnsProxyShutdownTimeout) + defer cancel() + + p.closeErr = p.srv.ShutdownContext(shutdownCtx) + }) + return p.closeErr +} + +type limitedListener struct { + net.Listener + slots chan struct{} + logger *slog.Logger +} + +func newLimitedListener(listener net.Listener, limit int, logger *slog.Logger) net.Listener { + if limit <= 0 { + return listener + } + return &limitedListener{ + Listener: listener, + slots: make(chan struct{}, limit), + logger: logger, + } +} + +func (l *limitedListener) Accept() (net.Conn, error) { + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + select { + case l.slots <- struct{}{}: + return &limitedConn{ + Conn: conn, + release: func() { + <-l.slots + }, + }, nil + default: + l.logger.Warn("dns proxy dropped connection because workers are busy") + _ = conn.Close() + } + } +} + +type limitedConn struct { + net.Conn + once sync.Once + release func() +} + +func (c *limitedConn) Close() error { + err := c.Conn.Close() + c.once.Do(c.release) + return err +} + +type hostDNSResolver struct { + upstreams []string + attempts int + timeout time.Duration + logger *slog.Logger +} + +func newHostDNSResolver(path string, logger *slog.Logger) (*hostDNSResolver, error) { + config, err := dns.ClientConfigFromFile(path) + if err != nil { + return nil, fmt.Errorf("read host resolv.conf: %w", err) + } + if len(config.Servers) == 0 { + return nil, fmt.Errorf("host resolv.conf has no nameservers") + } + + port := config.Port + if port == "" { + port = "53" + } + + upstreams := make([]string, 0, len(config.Servers)) + for _, server := range config.Servers { + upstreams = append(upstreams, net.JoinHostPort(server, port)) + } + + timeout := time.Duration(config.Timeout) * time.Second + if timeout <= 0 { + timeout = dnsProxyIOTimeout + } + + return &hostDNSResolver{ + upstreams: upstreams, + attempts: max(config.Attempts, 1), + timeout: timeout, + logger: logger, + }, nil +} + +func (r *hostDNSResolver) ServeDNS(w dns.ResponseWriter, req *dns.Msg) { + resp, err := r.exchange(req) + if err != nil { + r.logger.Warn( + "dns upstream exchange failed", + "question", dnsQuestionLogValue(req), + "error", err, + ) + if err := w.WriteMsg(rcodeResponse(req, dns.RcodeServerFailure)); err != nil { + r.logger.Warn("dns proxy response write failed", "error", err) + } + return + } + + filterDNSResponse(resp) + + if err := w.WriteMsg(resp); err != nil { + r.logger.Warn("dns proxy response write failed", "error", err) + } +} + +func (r *hostDNSResolver) exchange(req *dns.Msg) (*dns.Msg, error) { + var errs []error + + for range r.attempts { + for _, upstream := range r.upstreams { + resp, err := exchangeDNSAt(req, upstream, r.timeout) + if err == nil { + return resp, nil + } + errs = append(errs, fmt.Errorf("%s: %w", upstream, err)) + } + } + + return nil, errors.Join(errs...) +} + +func exchangeDNSAt(req *dns.Msg, addr string, timeout time.Duration) (*dns.Msg, error) { + resp, _, err := (&dns.Client{Net: "udp", Timeout: timeout}).Exchange(req, addr) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("empty udp response") + } + if !resp.Truncated { + return resp, nil + } + + resp, _, err = (&dns.Client{Net: "tcp", Timeout: timeout}).Exchange(req, addr) + if err != nil { + return nil, err + } + if resp == nil { + return nil, fmt.Errorf("empty tcp response") + } + return resp, nil +} + +func filterDNSResponse(msg *dns.Msg) { + if msg == nil { + return + } + msg.Answer = filterDNSRRs(msg.Answer) + msg.Ns = filterDNSRRs(msg.Ns) + msg.Extra = filterDNSRRs(msg.Extra) +} + +func filterDNSRRs(rrs []dns.RR) []dns.RR { + filtered := rrs[:0] + for _, rr := range rrs { + if rr := filterDNSRR(rr); rr != nil { + filtered = append(filtered, rr) + } + } + return filtered +} + +func filterDNSRR(rr dns.RR) dns.RR { + switch rr := rr.(type) { + case *dns.A: + if isBlockedNamespaceIP(rr.A) { + return nil + } + case *dns.AAAA: + if isBlockedNamespaceIP(rr.AAAA) { + return nil + } + case *dns.SVCB: + filterSVCBValues(&rr.Value) + case *dns.HTTPS: + filterSVCBValues(&rr.Value) + } + return rr +} + +// this removes any blocked namespaces in ipv4/v6 hints +func filterSVCBValues(values *[]dns.SVCBKeyValue) { + filtered := (*values)[:0] + for _, value := range *values { + switch value := value.(type) { + case *dns.SVCBIPv4Hint: + value.Hint = filterDNSIPs(value.Hint) + if len(value.Hint) == 0 { + continue + } + case *dns.SVCBIPv6Hint: + value.Hint = filterDNSIPs(value.Hint) + if len(value.Hint) == 0 { + continue + } + } + filtered = append(filtered, value) + } + *values = filtered +} + +func filterDNSIPs(ips []net.IP) []net.IP { + filtered := ips[:0] + for _, ip := range ips { + if !isBlockedNamespaceIP(ip) { + filtered = append(filtered, ip) + } + } + return filtered +} + +func isBlockedNamespaceIP(ip net.IP) bool { + if ip == nil { + return true + } + if ip4 := ip.To4(); ip4 != nil { + return isBlockedByNamespaceNets(ip4, 32) + } + return isBlockedByNamespaceNets(ip, 128) +} + +func isBlockedByNamespaceNets(ip net.IP, bits int) bool { + for _, blockedNet := range blockedNamespaceNets { + if blockedNet == nil { + continue + } + + _, blockedBits := blockedNet.Mask.Size() + if blockedBits != bits { + continue + } + if blockedNet.Contains(ip) { + return true + } + } + return false +} + +func rcodeResponse(req *dns.Msg, rcode int) *dns.Msg { + resp := new(dns.Msg) + if req == nil { + resp.Rcode = rcode + return resp + } + resp.SetRcode(req, rcode) + return resp +} + +func dnsQuestionLogValue(msg *dns.Msg) string { + if msg == nil || len(msg.Question) == 0 { + return "" + } + + q := msg.Question[0] + qtype := dns.TypeToString[q.Qtype] + if qtype == "" { + qtype = fmt.Sprintf("TYPE%d", q.Qtype) + } + return fmt.Sprintf("%s/%s", q.Name, qtype) +} diff --git a/spindle/engines/microvm/dns_proxy_test.go b/spindle/engines/microvm/dns_proxy_test.go new file mode 100644 index 00000000..a38b0f4e --- /dev/null +++ b/spindle/engines/microvm/dns_proxy_test.go @@ -0,0 +1,73 @@ +package microvm + +import ( + "net" + "testing" + + "github.com/miekg/dns" +) + +func TestFilterDNSResponseDropsBlockedAddressRecords(t *testing.T) { + msg := new(dns.Msg) + msg.Answer = []dns.RR{ + &dns.CNAME{Hdr: dns.RR_Header{Name: "cache.example.", Rrtype: dns.TypeCNAME, Class: dns.ClassINET}, Target: "edge.example."}, + &dns.A{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("1.1.1.1")}, + &dns.A{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("10.0.0.1")}, + &dns.AAAA{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("2606:4700:4700::1111")}, + &dns.AAAA{Hdr: dns.RR_Header{Name: "edge.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("fd7a:115c:a1e0::53")}, + } + msg.Extra = []dns.RR{ + &dns.A{Hdr: dns.RR_Header{Name: "private.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("192.168.1.2")}, + &dns.A{Hdr: dns.RR_Header{Name: "public.example.", Rrtype: dns.TypeA, Class: dns.ClassINET}, A: net.ParseIP("8.8.8.8")}, + } + + filterDNSResponse(msg) + + if len(msg.Answer) != 3 { + t.Fatalf("filtered answer len = %d, want 3: %#v", len(msg.Answer), msg.Answer) + } + if _, ok := msg.Answer[0].(*dns.CNAME); !ok { + t.Fatalf("answer[0] = %T, want CNAME", msg.Answer[0]) + } + if a, ok := msg.Answer[1].(*dns.A); !ok || !a.A.Equal(net.ParseIP("1.1.1.1")) { + t.Fatalf("answer[1] = %#v, want public A", msg.Answer[1]) + } + if aaaa, ok := msg.Answer[2].(*dns.AAAA); !ok || !aaaa.AAAA.Equal(net.ParseIP("2606:4700:4700::1111")) { + t.Fatalf("answer[2] = %#v, want public AAAA", msg.Answer[2]) + } + if len(msg.Extra) != 1 { + t.Fatalf("filtered extra len = %d, want 1: %#v", len(msg.Extra), msg.Extra) + } +} + +func TestFilterDNSResponseFiltersSVCBAddressHints(t *testing.T) { + msg := new(dns.Msg) + msg.Answer = []dns.RR{ + &dns.HTTPS{ + SVCB: dns.SVCB{ + Hdr: dns.RR_Header{Name: "svc.example.", Rrtype: dns.TypeHTTPS, Class: dns.ClassINET}, + Priority: 1, + Target: ".", + Value: []dns.SVCBKeyValue{ + &dns.SVCBIPv4Hint{Hint: []net.IP{net.ParseIP("10.0.0.1"), net.ParseIP("8.8.8.8")}}, + &dns.SVCBIPv6Hint{Hint: []net.IP{net.ParseIP("fd7a:115c:a1e0::53"), net.ParseIP("2001:4860:4860::8888")}}, + }, + }, + }, + } + + filterDNSResponse(msg) + + https := msg.Answer[0].(*dns.HTTPS) + if len(https.Value) != 2 { + t.Fatalf("https values len = %d, want 2", len(https.Value)) + } + ipv4 := https.Value[0].(*dns.SVCBIPv4Hint) + if len(ipv4.Hint) != 1 || !ipv4.Hint[0].Equal(net.ParseIP("8.8.8.8")) { + t.Fatalf("ipv4 hints = %v, want [8.8.8.8]", ipv4.Hint) + } + ipv6 := https.Value[1].(*dns.SVCBIPv6Hint) + if len(ipv6.Hint) != 1 || !ipv6.Hint[0].Equal(net.ParseIP("2001:4860:4860::8888")) { + t.Fatalf("ipv6 hints = %v, want [2001:4860:4860::8888]", ipv6.Hint) + } +} diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go new file mode 100644 index 00000000..671b5011 --- /dev/null +++ b/spindle/engines/microvm/engine.go @@ -0,0 +1,517 @@ +package microvm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "slices" + "sync" + "sync/atomic" + "time" + + "gopkg.in/yaml.v3" + + "tangled.org/core/api/tangled" + "tangled.org/core/log" + "tangled.org/core/spindle/agentproto" + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/config" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/secrets" +) + +const ( + guestWorkDir = "/workspace/repo" + activationStepAction = "activate-config" + agentAcceptTimeout = 2 * time.Minute + agentHandshakeTimeout = 30 * time.Second + cacheDrainTimeout = 5 * time.Minute + vmShutdownTimeout = 10 * time.Second + guestTimeoutGrace = 5 * time.Second +) + +type cleanupFunc func(context.Context) error + +type Engine struct { + l *slog.Logger + cfg *config.Config + db *db.DB + agent *agentHub + scheduler *engine.ResourceScheduler[Resources] + cgroupParent *CgroupParent + + cleanupMu sync.Mutex + cleanup map[string][]cleanupFunc +} + +type Step struct { + name string + kind models.StepKind + command string + environment map[string]string + action string + config manifestConfig + configKey string +} + +func (s Step) Name() string { return s.name } +func (s Step) Command() string { return s.command } +func (s Step) Kind() models.StepKind { return s.kind } + +func New(ctx context.Context, cfg *config.Config, d *db.DB) (*Engine, error) { + l := log.FromContext(ctx).With("component", "engine.microvm") + port := cfg.MicroVMPipelines.AgentPort + if port == 0 { + port = agentproto.DefaultPort + } + agent, err := newAgentHub(port, l) + if err != nil { + return nil, err + } + budget, max, agingThreshold := newVMBudgetConfig(cfg.MicroVMPipelines) + l.Info("initialized microVM workflow budget", "budget", budget.String(), "maxWorkflow", max.String(), "agingThreshold", agingThreshold) + + var cgroupParent *CgroupParent + if cfg.MicroVMPipelines.EnableCgroups { + cgroupParent, err = initCgroupParent(cfg.MicroVMPipelines.CgroupParent, cfg.MicroVMPipelines.CgroupSupervisorMemoryMinMiB, l) + if err != nil { + return nil, err + } + } + + return &Engine{ + l: l, + cfg: cfg, + db: d, + agent: agent, + scheduler: engine.NewResourceScheduler[Resources](budget, max, agingThreshold), + cgroupParent: cgroupParent, + cleanup: make(map[string][]cleanupFunc), + }, nil +} + +func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { + swf := &models.Workflow{} + var dwf manifestWorkflow + + if err := yaml.Unmarshal([]byte(twf.Raw), &dwf); err != nil { + return nil, err + } + + for _, dstep := range dwf.Steps { + swf.Steps = append(swf.Steps, Step{ + name: dstep.Name, + kind: models.StepKindUser, + command: dstep.Command, + environment: dstep.Environment, + }) + } + swf.Name = twf.Name + swf.Environment = dwf.Environment + + if tpl.TriggerMetadata != nil { + if clone := models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev); clone.Command() != "" { + swf.Steps = append([]models.Step{clone}, swf.Steps...) + } + } + + imageSpec, imageSpecPath, imageName, err := e.resolveImage(dwf.Image) + if err != nil { + return nil, err + } + configKey := "" + config := manifestConfig{ + Services: dwf.Services, + Virtualisation: dwf.Virtualisation, + Dependencies: dwf.Dependencies, + Registry: dwf.Registry, + } + if config.Enabled() { + if !imageSpec.SupportsConfigActivation() { + return nil, fmt.Errorf( + "microVM image %q is not a NixOS image: services, virtualisation, dependencies and registry workflow options require a NixOS image", + imageName, + ) + } + var err error + configKey, err = buildConfigKey(imageSpec, config) + if err != nil { + return nil, fmt.Errorf("build config key: %w", err) + } + activationStep := Step{ + name: "NixOS config activation", + kind: models.StepKindSystem, + command: "activate nixos config", + action: activationStepAction, + config: config, + configKey: configKey, + } + + insertAt := 0 + if len(swf.Steps) > 0 && swf.Steps[0].Kind() == models.StepKindSystem { + insertAt = 1 + } + swf.Steps = append(swf.Steps, nil) + copy(swf.Steps[insertAt+1:], swf.Steps[insertAt:]) + swf.Steps[insertAt] = activationStep + } + + cacheURLs, cacheKeys, err := workflowCaches(dwf.Caches) + if err != nil { + return nil, err + } + + swf.Data = &workflowState{ + ImageSpec: imageSpec, + ImageSpecPath: imageSpecPath, + Config: config, + ConfigKey: configKey, + Image: imageName, + CacheReadURLs: cacheURLs, + CacheTrustedPublicKeys: cacheKeys, + NixOSToplevelCache: newNixOSToplevelCacheStore(e.db), + } + return swf, nil +} + +func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { + l := e.l.With("workflow", wid) + setupStep := Step{name: "microVM setup", kind: models.StepKindSystem} + + wfLogger.ControlWriter(-1, setupStep, models.StepStatusStart).Write([]byte{0}) + defer wfLogger.ControlWriter(-1, setupStep, models.StepStatusEnd).Write([]byte{0}) + + state, ok := wf.Data.(*workflowState) + if !ok || state == nil { + return fmt.Errorf("workflow state is not initialized") + } + + cid, err := AllocateCID() + if err != nil { + return err + } + connCh, unregister, err := e.agent.expect(cid) + if err != nil { + return err + } + defer unregister() + + workDirBase := e.cfg.MicroVMPipelines.OverlayDir + if workDirBase == "" { + workDirBase = os.TempDir() + } + workDir, err := os.MkdirTemp(workDirBase, "spindle-microvm-"+wid.String()+"-*") + if err != nil { + return fmt.Errorf("create workflow microVM directory: %w", err) + } + state.WorkDir = workDir + + setupDone := false + defer func() { + if setupDone { + return + } + if err := e.cleanupState(context.Background(), wid, state); err != nil { + l.Error("failed to cleanup failed setup", "error", err) + } + }() + + upstreams, err := BuildCacheUpstreams(e.cfg.NixCache.ReadURLs, state.CacheReadURLs) + if err != nil { + return err + } + readCache, err := StartReadCacheProxy(ctx, cid, upstreams, l) + if err != nil { + return err + } + state.ReadCache = readCache + uploadCache, err := StartUploadCacheProxy(ctx, cid, e.cfg.NixCache.UploadURL, upstreams, l) + if err != nil { + return err + } + state.UploadCache = uploadCache + dnsProxy, err := StartDNSProxy(ctx, cid, l) + if err != nil { + return err + } + state.DNSProxy = dnsProxy + + port := e.cfg.MicroVMPipelines.AgentPort + if port == 0 { + port = agentproto.DefaultPort + } + state.ImageSpec.BootArgs = fmt.Sprintf("%s shuttle.vsock_port=%d", state.ImageSpec.BootArgs, port) + + fmt.Fprintf(wfLogger.DataWriter(-1, "stdout"), "starting microVM image %s\n", state.Image) + l.Info("starting microVM workflow", "image", state.Image, "imageSpec", state.ImageSpecPath, "cid", cid, "workDir", workDir) + + var vm VMHandle + vm, err = StartVM(ctx, VMConfig{ + Image: state.ImageSpec, + CID: cid, + EnableKVM: e.cfg.MicroVMPipelines.EnableKVM, + WorkDir: workDir, + Cgroup: e.cgroupLimits(wid, state.ImageSpec), + Dev: e.cfg.Server.Dev, + }, l) + if err != nil { + return err + } + state.VM = vm + + acceptCtx, cancelAccept := context.WithTimeout(ctx, agentAcceptTimeout) + defer cancelAccept() + conn, err := waitAgentConn(acceptCtx, connCh) + if err != nil { + return err + } + + agentSession := NewAgentSession(conn, l) + initCtx, cancelInit := context.WithTimeout(ctx, agentHandshakeTimeout) + defer cancelInit() + if err := agentSession.Init(initCtx, &agentv1.Init{ + JobId: wid.String(), + CacheTrustedPublicKeys: append(slices.Clone(e.cfg.NixCache.TrustedPublicKeys), state.CacheTrustedPublicKeys...), + CacheReadProxyPort: readCache.Port(), + CacheUploadProxyPort: uploadCache.Port(), + DnsProxyPort: dnsProxy.Port(), + }); err != nil { + _ = agentSession.Close() + return err + } + state.Agent = agentSession + wf.Data = state + + e.registerCleanup(wid, func(ctx context.Context) error { + return e.cleanupState(ctx, wid, state) + }) + setupDone = true + + fmt.Fprintf(wfLogger.DataWriter(-1, "stdout"), + "agent connected; serial log: %s\n", vm.Logs().Serial, + ) + return nil +} + +func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { + state, ok := w.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + stderr := wfLogger.DataWriter(idx, "stderr") + + execCtx, vmExited, cancelWatch := watchVMExit(ctx, state.VM) + defer cancelWatch() + + step := w.Steps[idx] + if s, ok := step.(Step); ok && s.action == activationStepAction { + err := e.activateConfig(execCtx, wid, state, s, wfLogger.DataWriter(idx, "stdout")) + return e.classifyStepError(ctx, wid, step, state, stderr, vmExited, err) + } + env := []string{ + "HOME=/workspace", + "LOGNAME=" + guestWorkflowUser, + "PATH=/run/current-system/sw/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "USER=" + guestWorkflowUser, + } + for k, v := range w.Environment { + env = append(env, k+"="+v) + } + for _, s := range secrets { + env = append(env, s.Key+"="+s.Value) + } + if s, ok := step.(Step); ok { + for k, v := range s.environment { + env = append(env, k+"="+v) + } + } + + stdout := wfLogger.DataWriter(idx, "stdout") + exitCode, err := state.Agent.Exec(execCtx, AgentExec{ + ID: fmt.Sprintf("%s-%d", wid.String(), idx), + ExecStart: &agentv1.ExecStart{ + Argv: []string{state.ImageSpec.Shell, "-lc", step.Command()}, + Env: env, + Cwd: guestWorkDir, + User: guestWorkflowUser, + // timeout not set here, Exec will fill it + }, + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + return e.classifyStepError(ctx, wid, step, state, stderr, vmExited, err) + } + + if exitCode != 0 { + e.l.Debug("step exited non-zero", "workflow", wid, "step", step.Name(), "exitCode", exitCode) + return engine.ErrWorkflowFailed + } + return nil +} + +// reads the vm serial logs so we report the tail of that as an error instead of +// just "guest agent connection lost: EOF" +func (e *Engine) classifyStepError(ctx context.Context, wid models.WorkflowId, step models.Step, state *workflowState, stderr io.Writer, vmExited *atomic.Bool, err error) error { + if err == nil { + return nil + } + l := e.l.With("workflow", wid, "step", step.Name()) + + if vmExited != nil && vmExited.Load() { + reason := "microVM exited unexpectedly" + oom := state.VM != nil && state.VM.OOMKilled() + if oom { + reason = "microVM killed by OOM (cgroup memory limit exceeded)" + } + if detail := vmCrashLog(state.VM); detail != "" { + fmt.Fprintf(stderr, "%s:\n%s\n", reason, detail) + l.Debug(reason, "oom", oom, "detail", detail) + } else { + fmt.Fprintln(stderr, reason) + l.Debug(reason, "oom", oom) + } + return errors.New(reason + "; see workflow logs for serial output") + } + + if errors.Is(err, errGuestTimedOut) || ctx.Err() != nil { + l.Debug("step timed out", "guestReported", errors.Is(err, errGuestTimedOut)) + return engine.ErrTimedOut + } + + // the agent connection dropped while qemu stayed up (eg. the guest kernel + // OOM-killed the agent or a guest panic), so surface serial logs, those + // will be more helpful. + if detail := vmCrashLog(state.VM); detail != "" { + fmt.Fprintf(stderr, "step failed (%v):\n%s\n", err, detail) + l.Debug("step failed", "error", err, "detail", detail) + } else { + l.Debug("step failed", "error", err) + } + return err +} + +func (e *Engine) activateConfig(ctx context.Context, wid models.WorkflowId, state *workflowState, step Step, out io.Writer) error { + cfg := step.config + if !cfg.Enabled() { + return nil + } + + configKey := step.configKey + if configKey == "" { + configKey = state.ConfigKey + } + + userConfigJSON, err := json.Marshal(cfg) + if err != nil { + return fmt.Errorf("encode user config: %w", err) + } + + var cachedToplevel string + if configKey != "" { + if record, ok, err := state.NixOSToplevelCache.Lookup(configKey); err != nil { + return err + } else if ok { + cachedToplevel = record.Toplevel + fmt.Fprintf(out, "realizing cached NixOS config %s\n", cachedToplevel) + } + } + if cachedToplevel == "" { + fmt.Fprintf(out, "building NixOS config from user config\n") + } + + baseHash, err := BaseConfigHash(state.ImageSpec) + if err != nil { + return fmt.Errorf("calculate base config hash: %w", err) + } + + result, err := state.Agent.ActivateConfig(ctx, fmt.Sprintf("%s-config", wid.String()), &agentv1.ActivateConfig{ + ConfigKey: configKey, + BaseConfigHash: baseHash, + UserConfig: string(userConfigJSON), + Toplevel: cachedToplevel, + }) + if err != nil { + return err + } + fmt.Fprintf(out, "activated NixOS config toplevel %s\n", result.Toplevel) + + if cachedToplevel != "" || configKey == "" { + return nil + } + if e.cfg.NixCache.UploadURL == "" { + e.l.Warn("not committing config cache metadata: no upload URL configured", "workflow", wid, "configKey", configKey, "toplevel", result.Toplevel) + return nil + } + + drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) + defer cancel() + if _, err := state.Agent.Drain(drainCtx); err != nil { + return fmt.Errorf("drain config cache uploads before metadata commit: %w", err) + } + if err := state.NixOSToplevelCache.Commit(configKey, result.Toplevel); err != nil { + return err + } + fmt.Fprintf(out, "committed config cache metadata %s -> %s\n", configKey, result.Toplevel) + return nil +} + +func (e *Engine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) error { + fns := e.drainCleanups(wid) + + var cleanupErr error + for i := len(fns) - 1; i >= 0; i-- { + if err := fns[i](ctx); err != nil { + e.l.Error("failed to cleanup workflow resource", "workflowId", wid, "error", err) + cleanupErr = errors.Join(cleanupErr, err) + } + } + return cleanupErr +} + +func (e *Engine) FinalizeWorkflow(ctx context.Context, wid models.WorkflowId, w *models.Workflow, wfLogger models.WorkflowLogger) error { + return nil +} + +func (e *Engine) WorkflowTimeout() time.Duration { + d, err := time.ParseDuration(e.cfg.MicroVMPipelines.WorkflowTimeout) + if err != nil { + d = 5 * time.Minute + } + return d + guestTimeoutGrace +} + +func (e *Engine) registerCleanup(wid models.WorkflowId, fn cleanupFunc) { + e.cleanupMu.Lock() + defer e.cleanupMu.Unlock() + key := wid.String() + e.cleanup[key] = append(e.cleanup[key], fn) +} + +func (e *Engine) drainCleanups(wid models.WorkflowId) []cleanupFunc { + e.cleanupMu.Lock() + defer e.cleanupMu.Unlock() + key := wid.String() + fns := e.cleanup[key] + delete(e.cleanup, key) + return fns +} + +func (e *Engine) cgroupLimits(wid models.WorkflowId, spec ImageSpec) CgroupLimits { + cfg := e.cfg.MicroVMPipelines + return CgroupLimits{ + Enabled: cfg.EnableCgroups, + Parent: e.cgroupParent, + Name: "workflow-" + wid.String(), + MemoryMaxMiB: resourcesForImage(spec).MemoryMiB, + SwapMaxMiB: cfg.CgroupSwapMaxMiB, + PidsMax: cfg.CgroupPidsMax, + } +} diff --git a/spindle/engines/microvm/engine_test.go b/spindle/engines/microvm/engine_test.go new file mode 100644 index 00000000..0585121c --- /dev/null +++ b/spindle/engines/microvm/engine_test.go @@ -0,0 +1,116 @@ +package microvm + +import ( + "encoding/json" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/config" +) + +func writeTestImageSpec(t *testing.T, dir, name string, spec ImageSpec) { + t.Helper() + data, err := json.Marshal(spec) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name+".json"), data, 0o644); err != nil { + t.Fatal(err) + } +} + +func testEngine(t *testing.T, imageDir string) *Engine { + t.Helper() + return &Engine{ + l: slog.Default(), + cfg: &config.Config{ + MicroVMPipelines: config.MicroVMPipelines{ + ImageDir: imageDir, + DefaultImage: "alpine", + }, + }, + } +} + +func TestInitWorkflowRejectsConfigOnNonNixOSImage(t *testing.T) { + dir := t.TempDir() + writeTestImageSpec(t, dir, "alpine", validImageSpec()) + + e := testEngine(t, dir) + _, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: alpine +dependencies: + - nixpkgs#hello +steps: + - name: hello + command: hello +`, + }, tangled.Pipeline{}) + if err == nil { + t.Fatal("expected error for NixOS config options on a non-NixOS image") + } + if !strings.Contains(err.Error(), "NixOS") { + t.Fatalf("error should mention NixOS images, got: %v", err) + } +} + +func TestInitWorkflowPlainStepsOnNonNixOSImage(t *testing.T) { + dir := t.TempDir() + writeTestImageSpec(t, dir, "alpine", validImageSpec()) + + e := testEngine(t, dir) + wf, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: alpine +steps: + - name: hello + command: echo hello +`, + }, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if len(wf.Steps) != 1 { + t.Fatalf("expected exactly the user step, got %d steps", len(wf.Steps)) + } + state, ok := wf.Data.(*workflowState) + if !ok { + t.Fatal("workflow data is not workflowState") + } + if state.ConfigKey != "" { + t.Fatalf("non-NixOS workflow should not have a config key, got %q", state.ConfigKey) + } +} + +func TestInitWorkflowConfigOnNixOSImage(t *testing.T) { + dir := t.TempDir() + spec := validImageSpec() + spec.BaseConfigHash = "abcdef123456" + writeTestImageSpec(t, dir, "nixos", spec) + + e := testEngine(t, dir) + wf, err := e.InitWorkflow(tangled.Pipeline_Workflow{ + Raw: ` +image: nixos +dependencies: + - nixpkgs#hello +steps: + - name: hello + command: hello +`, + }, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if len(wf.Steps) != 2 { + t.Fatalf("expected activation step + user step, got %d steps", len(wf.Steps)) + } + if step, ok := wf.Steps[0].(Step); !ok || step.action != activationStepAction { + t.Fatalf("first step should be the activation step, got %+v", wf.Steps[0]) + } +} diff --git a/spindle/engines/microvm/image.go b/spindle/engines/microvm/image.go new file mode 100644 index 00000000..89db1552 --- /dev/null +++ b/spindle/engines/microvm/image.go @@ -0,0 +1,254 @@ +package microvm + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const imageSpecFileName = "spec.json" + +type RunnerConfig struct { + CPU string `json:"cpu,omitempty"` + Machine string `json:"machine,omitempty"` + Console string `json:"console,omitempty"` + ExtraArgs []string `json:"extraArgs,omitempty"` +} + +type ImageSpec struct { + Arch string `json:"arch"` + BootArgs string `json:"bootArgs"` + Initrd string `json:"initrd"` + Kernel string `json:"kernel"` + RunnerType string `json:"runnerType"` + RunnerConfig RunnerConfig `json:"runnerConfig"` + MemoryMiB int `json:"memoryMiB"` + NetworkInterfaces []NetworkInterface `json:"networkInterfaces"` + StoreDisk string `json:"storeDisk"` + StoreDiskType string `json:"storeDiskType"` + // baseConfigHash identifies the base nixos configuration baked into the + // image. its only for nixos images as other images won't have a system + // to rebuild. + BaseConfigHash string `json:"baseConfigHash,omitempty"` + // shell is the login shell used to run workflow step commands in the guest. + Shell string `json:"shell"` + VCPUs int `json:"vcpus"` + Volumes []Volume `json:"volumes"` +} + +func (s ImageSpec) SupportsConfigActivation() bool { + return s.BaseConfigHash != "" +} + +type NetworkInterface struct { + Type string `json:"type"` + ID string `json:"id"` + MAC string `json:"mac"` +} + +type Volume struct { + FSType string `json:"fsType"` + Image string `json:"image"` + ImageType string `json:"imageType"` + MountPoint string `json:"mountPoint"` + ReadOnly bool `json:"readOnly"` + SizeMiB int64 `json:"sizeMiB"` +} + +func LoadImageSpec(path string) (ImageSpec, error) { + data, err := os.ReadFile(path) + if err != nil { + return ImageSpec{}, fmt.Errorf("read microvm image spec: %w", err) + } + + var spec ImageSpec + if err := json.Unmarshal(data, &spec); err != nil { + return ImageSpec{}, fmt.Errorf("parse microvm image spec: %w", err) + } + + base := filepath.Dir(path) + spec.Kernel = resolveImageSpecPath(base, spec.Kernel) + spec.Initrd = resolveImageSpecPath(base, spec.Initrd) + spec.StoreDisk = resolveImageSpecPath(base, spec.StoreDisk) + + if err := spec.Validate(); err != nil { + return ImageSpec{}, err + } + return spec, nil +} + +func (s ImageSpec) Validate() error { + if s.Kernel == "" { + return fmt.Errorf("microvm image spec missing kernel") + } + if s.Initrd == "" { + return fmt.Errorf("microvm image spec missing initrd") + } + if s.StoreDisk == "" { + return fmt.Errorf("microvm image spec missing storeDisk") + } + if s.BootArgs == "" { + return fmt.Errorf("microvm image spec missing bootArgs") + } + if s.Shell == "" { + return fmt.Errorf("microvm image spec missing shell") + } + if s.RunnerType == "qemu" || s.RunnerType == "" { + if s.RunnerConfig.Machine == "" { + return fmt.Errorf("microvm image spec missing runnerConfig.machine for qemu runner") + } + } + if s.MemoryMiB <= 0 { + return fmt.Errorf("microvm image spec memoryMiB must be positive") + } + if s.VCPUs <= 0 { + return fmt.Errorf("microvm image spec vcpus must be positive") + } + for _, networkInterface := range s.NetworkInterfaces { + if networkInterface.Type == "" { + return fmt.Errorf("microvm image spec network interface missing type") + } + if networkInterface.ID == "" { + return fmt.Errorf("microvm image spec network interface missing id") + } + if networkInterface.MAC == "" { + return fmt.Errorf("microvm image spec network interface %q missing mac", networkInterface.ID) + } + } + for _, volume := range s.Volumes { + if volume.Image == "" { + return fmt.Errorf("microvm image spec volume missing image") + } + if volume.FSType == "" { + return fmt.Errorf("microvm image spec volume %q missing fsType", volume.Image) + } + if volume.SizeMiB <= 0 { + return fmt.Errorf("microvm image spec volume %q sizeMiB must be positive", volume.Image) + } + } + return nil +} + +func (s ImageSpec) RunnerCmd() string { + switch s.RunnerType { + case "qemu", "": + return "qemu-system-" + s.Arch + case "firecracker": + return "firecracker" + default: + return "" + } +} + +// also see Runner.Validate for where Runner specific files are validated +func (s ImageSpec) validateImageFiles() error { + required := map[string]string{ + "kernel": s.Kernel, + "initrd": s.Initrd, + "storeDisk": s.StoreDisk, + } + for name, path := range required { + if !filepath.IsAbs(path) { + continue + } + if _, err := os.Stat(path); err != nil { + return fmt.Errorf("required image spec file %s not found at %q: %w", name, path, err) + } + } + + return nil +} + +func resolveImageSpecPath(base, path string) string { + if path == "" || filepath.IsAbs(path) { + return path + } + return filepath.Join(base, path) +} + +func (e *Engine) resolveImage(name string) (ImageSpec, string, string, error) { + name = strings.TrimSpace(name) + if name == "" { + name = strings.TrimSpace(e.cfg.MicroVMPipelines.DefaultImage) + } + if name == "" { + return ImageSpec{}, "", "", fmt.Errorf("no image specified in workflow and SPINDLE_MICROVM_PIPELINES_DEFAULT_IMAGE is not set") + } + if !isPlainImageName(name) { + return ImageSpec{}, "", "", fmt.Errorf("invalid microVM image name %q: must be a plain name, not a path", name) + } + + candidates := imageCandidates(e.cfg.MicroVMPipelines.ImageDir, name) + for _, candidate := range candidates { + path, ok, err := imageSpecPath(candidate) + if err != nil { + return ImageSpec{}, "", "", err + } + if !ok { + continue + } + imageSpec, err := LoadImageSpec(path) + if err != nil { + return ImageSpec{}, "", "", err + } + return imageSpec, path, name, nil + } + + return ImageSpec{}, "", "", fmt.Errorf("microVM image %q was not found; looked in: %s", name, strings.Join(candidates, ", ")) +} + +// check if image name is not a path +func isPlainImageName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + if filepath.IsAbs(name) || strings.ContainsRune(name, '/') || strings.ContainsRune(name, filepath.Separator) { + return false + } + return true +} + +// returns candidates, which is either a directory or spec file itself +func imageCandidates(imageDir, name string) []string { + if imageDir == "" { + return nil + } + return []string{ + filepath.Join(imageDir, name), + filepath.Join(imageDir, name+".json"), + } +} + +// resolve the candidate to a spec: +// - first check if its a file, if yes, return +// - otherwise assume its a directory and check and return `/spec.json` +func imageSpecPath(candidate string) (string, bool, error) { + info, err := os.Stat(candidate) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", false, nil + } + return "", false, err + } + if !info.IsDir() { + return candidate, true, nil + } + + spec := filepath.Join(candidate, imageSpecFileName) + info, err = os.Stat(spec) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return "", false, fmt.Errorf("microVM image directory %q does not contain %s", candidate, imageSpecFileName) + } + return "", false, err + } + // this only happens if there is a directory named `spec.json` which would be very silly. + // but better output an error for it anyway :p + if info.IsDir() { + return "", false, fmt.Errorf("microVM image spec %q is a directory", spec) + } + return spec, true, nil +} diff --git a/spindle/engines/microvm/image_test.go b/spindle/engines/microvm/image_test.go new file mode 100644 index 00000000..3000005c --- /dev/null +++ b/spindle/engines/microvm/image_test.go @@ -0,0 +1,127 @@ +package microvm + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeSpecFile(t *testing.T, path string) { + t.Helper() + data, err := json.Marshal(validImageSpec()) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } +} + +func TestResolveImageConventionalLayouts(t *testing.T) { + cases := []struct { + name string + layout func(t *testing.T, dir string) + }{ + { + name: "directory with spec.json", + layout: func(t *testing.T, dir string) { + writeSpecFile(t, filepath.Join(dir, "nixos", "spec.json")) + }, + }, + { + name: "flat .json", + layout: func(t *testing.T, dir string) { + writeSpecFile(t, filepath.Join(dir, "nixos.json")) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + tc.layout(t, dir) + + e := testEngine(t, dir) + spec, path, name, err := e.resolveImage("nixos") + if err != nil { + t.Fatalf("resolveImage: %v", err) + } + if name != "nixos" { + t.Fatalf("name = %q, want nixos", name) + } + if !strings.HasPrefix(path, dir) { + t.Fatalf("resolved path %q not under image dir %q", path, dir) + } + if spec.Shell == "" { + t.Fatal("resolved spec not loaded") + } + }) + } +} + +func TestResolveImageDirectoryMissingSpec(t *testing.T) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "nixos"), 0o755); err != nil { + t.Fatal(err) + } + + e := testEngine(t, dir) + _, _, _, err := e.resolveImage("nixos") + if err == nil || !strings.Contains(err.Error(), imageSpecFileName) { + t.Fatalf("directory without %s should error, got: %v", imageSpecFileName, err) + } +} + +func TestResolveImageRejectsPaths(t *testing.T) { + e := testEngine(t, t.TempDir()) + for _, name := range []string{"/etc/passwd", "../evil", "sub/evil", "..", "."} { + if _, _, _, err := e.resolveImage(name); err == nil || !strings.Contains(err.Error(), "must be a plain name") { + t.Fatalf("name %q should be rejected as a path, got: %v", name, err) + } + } +} + +func validImageSpec() ImageSpec { + return ImageSpec{ + Arch: "x86_64", + BootArgs: "console=ttyS0", + Initrd: "initrd", + Kernel: "kernel", + RunnerConfig: RunnerConfig{ + Machine: "microvm", + }, + MemoryMiB: 2048, + Shell: "/bin/sh", + StoreDisk: "store-disk", + VCPUs: 2, + } +} + +func TestImageSpecValidateWithoutBaseConfigHash(t *testing.T) { + spec := validImageSpec() + if err := spec.Validate(); err != nil { + t.Fatalf("non-NixOS image spec should validate: %v", err) + } + if spec.SupportsConfigActivation() { + t.Fatal("spec without baseConfigHash should not support config activation") + } + + spec.BaseConfigHash = "abcdef" + if !spec.SupportsConfigActivation() { + t.Fatal("spec with baseConfigHash should support config activation") + } +} + +func TestImageSpecRequiresShell(t *testing.T) { + spec := validImageSpec() + spec.Shell = "" + err := spec.Validate() + if err == nil || !strings.Contains(err.Error(), "shell") { + t.Fatalf("spec without shell should fail validation, got: %v", err) + } +} diff --git a/spindle/engines/microvm/models.go b/spindle/engines/microvm/models.go new file mode 100644 index 00000000..02f3c096 --- /dev/null +++ b/spindle/engines/microvm/models.go @@ -0,0 +1,48 @@ +package microvm + +import ( + "fmt" + "slices" +) + +type manifestWorkflow struct { + Image string `yaml:"image"` + Services map[string]any `yaml:"services"` + Virtualisation map[string]any `yaml:"virtualisation"` + Dependencies []string `yaml:"dependencies"` + Registry map[string]any `yaml:"registry"` + Environment map[string]string `yaml:"environment"` + Caches map[string]string `yaml:"caches"` + Steps []struct { + Name string `yaml:"name"` + Command string `yaml:"command"` + Environment map[string]string `yaml:"environment"` + } `yaml:"steps"` +} + +// flattens the caches map into sorted substituter URLs and trusted public keys +func workflowCaches(caches map[string]string) (urls []string, keys []string, err error) { + for cacheURL, key := range caches { + urls = append(urls, cacheURL) + if key != "" { + keys = append(keys, key) + } + } + if _, err := parseCacheUpstreams(urls); err != nil { + return nil, nil, fmt.Errorf("caches: %w", err) + } + slices.Sort(urls) + slices.Sort(keys) + return urls, keys, nil +} + +type manifestConfig struct { + Services map[string]any `yaml:"services" json:"services,omitempty"` + Virtualisation map[string]any `yaml:"virtualisation" json:"virtualisation,omitempty"` + Dependencies []string `yaml:"dependencies" json:"dependencies,omitempty"` + Registry map[string]any `yaml:"registry" json:"registry,omitempty"` +} + +func (c manifestConfig) Enabled() bool { + return len(c.Services) > 0 || len(c.Virtualisation) > 0 || len(c.Dependencies) > 0 || len(c.Registry) > 0 +} diff --git a/spindle/engines/microvm/models_test.go b/spindle/engines/microvm/models_test.go new file mode 100644 index 00000000..b103e4f5 --- /dev/null +++ b/spindle/engines/microvm/models_test.go @@ -0,0 +1,41 @@ +package microvm + +import ( + "slices" + "testing" +) + +func TestWorkflowCaches(t *testing.T) { + urls, keys, err := workflowCaches(map[string]string{ + "https://hydra.nixos.org/": "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=", + "https://cache.garnix.io/": "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=", + "https://unsigned.example/": "", + }) + if err != nil { + t.Fatal(err) + } + + wantURLs := []string{ + "https://cache.garnix.io/", + "https://hydra.nixos.org/", + "https://unsigned.example/", + } + if !slices.Equal(urls, wantURLs) { + t.Fatalf("urls: got %v, want %v", urls, wantURLs) + } + wantKeys := []string{ + "cache.garnix.io:CTFPyKSLcx5RMJKfLo5EEPUObbA78b0YQ2DTCJXqr9g=", + "hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=", + } + if !slices.Equal(keys, wantKeys) { + t.Fatalf("keys: got %v, want %v", keys, wantKeys) + } +} + +func TestWorkflowCachesRejectsBadURLs(t *testing.T) { + for _, bad := range []string{"ftp://cache.example/", "not a url"} { + if _, _, err := workflowCaches(map[string]string{bad: ""}); err == nil { + t.Errorf("workflowCaches(%q): expected error, got nil", bad) + } + } +} diff --git a/spindle/engines/microvm/netns_wrapper.sh.tmpl b/spindle/engines/microvm/netns_wrapper.sh.tmpl new file mode 100644 index 00000000..c7b40417 --- /dev/null +++ b/spindle/engines/microvm/netns_wrapper.sh.tmpl @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +pid_file="$1"; shift +ip="$1"; shift +mount="$1"; shift +resolv_conf="$1"; shift + +printf '%s\n' "$$" > "$pid_file" + +while ! "$ip" link show {{.TapName}} >/dev/null 2>&1; do + sleep 0.02 +done + +"$mount" --bind "$resolv_conf" /etc/resolv.conf + +# fail if we can't add a route, ipv6 is skipped if not enabled on host +for route in{{range .BlockedRoutes}} {{.}}{{end}}; do + case "$route" in + *:*) [ -d /proc/sys/net/ipv6 ] || continue ;; + esac + "$ip" route add blackhole "$route" +done + +exec "$@" diff --git a/spindle/engines/microvm/networking.go b/spindle/engines/microvm/networking.go new file mode 100644 index 00000000..67df330d --- /dev/null +++ b/spindle/engines/microvm/networking.go @@ -0,0 +1,152 @@ +package microvm + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "log/slog" + "net" + "os" + "os/exec" + "text/template" +) + +// https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml +// https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +// https://datatracker.ietf.org/doc/rfc6890/ +var blockedNamespaceRoutes = []string{ + "0.0.0.0/8", // unspecified / "this network" addresses + "10.0.0.0/8", // private network + "100.64.0.0/10", // shared carrier-grade nat space + "127.0.0.0/8", // loopback + "169.254.0.0/16", // link-local / autoconfiguration + "172.16.0.0/12", // private network + "192.0.0.0/24", // ietf protocol assignments + "192.0.2.0/24", // documentation / examples + "192.88.99.0/24", // deprecated 6to4 relay anycast + "192.168.0.0/16", // private network + "198.18.0.0/15", // benchmarking / testing + "198.51.100.0/24", // documentation / examples + "203.0.113.0/24", // documentation / examples + "224.0.0.0/4", // multicast + "240.0.0.0/4", // reserved / future use, includes limited broadcast + "::/128", // unspecified address + "::1/128", // loopback + "::ffff:0:0/96", // ipv4-mapped addresses + "64:ff9b::/96", // ipv4/ipv6 translation prefix + "100::/64", // discard-only prefix + "2001::/23", // ietf protocol assignments + "2001:db8::/32", // documentation / examples + "2002::/16", // deprecated 6to4 addressing + "fc00::/7", // unique local addresses + "fe80::/10", // link-local unicast + "ff00::/8", // multicast +} + +var blockedNamespaceNets = func() []*net.IPNet { + nets := make([]*net.IPNet, 0, len(blockedNamespaceRoutes)) + for _, route := range blockedNamespaceRoutes { + _, ipnet, err := net.ParseCIDR(route) + if err != nil { + panic(fmt.Sprintf("parse blocked route %q: %v", route, err)) + } + nets = append(nets, ipnet) + } + return nets +}() + +//go:embed netns_wrapper.sh.tmpl +var netnsWrapperTemplate string + +type netnsWrapperData struct { + TapName string + BlockedRoutes []string +} + +func writeNetnsWrapper(path string, dev bool) error { + tmpl, err := template.New("netns-wrapper").Parse(netnsWrapperTemplate) + if err != nil { + return fmt.Errorf("parse qemu network namespace wrapper template: %w", err) + } + + var script bytes.Buffer + + var routes []string + if !dev { + routes = blockedNamespaceRoutes + } + + err = tmpl.Execute(&script, netnsWrapperData{ + TapName: netnsTapName, + BlockedRoutes: routes, + }) + if err != nil { + return fmt.Errorf("render qemu network namespace wrapper template: %w", err) + } + + if err := os.WriteFile(path, script.Bytes(), 0o700); err != nil { + return fmt.Errorf("write qemu network namespace wrapper: %w", err) + } + + return nil +} + +type slirpNamespace struct { + spec ImageSpec + pidFile string + dev bool +} + +func (n *slirpNamespace) Start(ctx context.Context, logFile *os.File, logger *slog.Logger) (*exec.Cmd, *os.File, error) { + pid, err := waitForPIDFile(ctx, n.pidFile) + if err != nil { + return nil, nil, err + } + + exitR, exitW, err := os.Pipe() + if err != nil { + return nil, nil, fmt.Errorf("create slirp4netns exit pipe: %w", err) + } + defer exitR.Close() // always close our read end; child gets it via ExtraFiles dup + + var ok bool + defer func() { + if !ok { + _ = exitW.Close() + } + }() + + slirpPath, err := exec.LookPath("slirp4netns") + if err != nil { + return nil, nil, fmt.Errorf("slirp4netns command not found in PATH: %w", err) + } + + args := []string{ + "--configure", + "--mtu=" + netnsMTU, + } + if !n.dev { + args = append(args, "--disable-host-loopback") + } + args = append(args, + "--enable-sandbox", + "--enable-seccomp", + "--exit-fd=3", + "--cidr="+outerSlirpCIDR, + pid, + netnsTapName, + ) + + cmd := exec.CommandContext(ctx, slirpPath, args...) + cmd.ExtraFiles = []*os.File{exitR} + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Start(); err != nil { + return nil, nil, fmt.Errorf("start slirp4netns: %w", err) + } + logger.Info("started slirp4netns network namespace", "pid", pid, "cidr", outerSlirpCIDR, "tap", netnsTapName) + + ok = true + return cmd, exitW, nil +} diff --git a/spindle/engines/microvm/nixos_toplevel_cache.go b/spindle/engines/microvm/nixos_toplevel_cache.go new file mode 100644 index 00000000..8333a65c --- /dev/null +++ b/spindle/engines/microvm/nixos_toplevel_cache.go @@ -0,0 +1,100 @@ +package microvm + +import ( + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "time" + + "tangled.org/core/spindle/db" +) + +const nixosToplevelCacheSchemaVersion = 1 + +type nixosToplevelCacheRecord struct { + ConfigKey string `json:"config_key"` + Toplevel string `json:"toplevel"` + UpdatedAt time.Time `json:"updated_at"` +} + +type nixosToplevelCacheStore struct { + db *db.DB +} + +func newNixOSToplevelCacheStore(d *db.DB) nixosToplevelCacheStore { + return nixosToplevelCacheStore{db: d} +} + +func (s nixosToplevelCacheStore) Lookup(configKey string) (nixosToplevelCacheRecord, bool, error) { + if s.db == nil { + return nixosToplevelCacheRecord{}, false, nil + } + r, err := s.db.GetNixOSToplevelCacheRecord(configKey) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nixosToplevelCacheRecord{}, false, nil + } + return nixosToplevelCacheRecord{}, false, err + } + return nixosToplevelCacheRecord{ + ConfigKey: r.ConfigKey, + Toplevel: r.Toplevel, + UpdatedAt: r.UpdatedAt, + }, true, nil +} + +func (s nixosToplevelCacheStore) Commit(configKey, toplevel string) error { + if configKey == "" { + return fmt.Errorf("config key is empty") + } + if toplevel == "" { + return fmt.Errorf("config toplevel is empty") + } + if s.db == nil { + return nil + } + return s.db.SaveNixOSToplevelCacheRecord(configKey, toplevel) +} + +func BaseConfigHash(imageSpec ImageSpec) (string, error) { + if imageSpec.BaseConfigHash == "" { + return "", fmt.Errorf("microvm image spec missing baseConfigHash") + } + return imageSpec.BaseConfigHash, nil +} + +func userConfigHash(cfg manifestConfig) string { + data, _ := json.Marshal(cfg) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func buildConfigKey(imageSpec ImageSpec, cfg manifestConfig) (string, error) { + baseHash, err := BaseConfigHash(imageSpec) + if err != nil { + return "", err + } + payload := struct { + Schema int `json:"schema"` + BaseConfig string `json:"base_config"` + UserConfig string `json:"user_config"` + }{ + Schema: nixosToplevelCacheSchemaVersion, + BaseConfig: baseHash, + UserConfig: userConfigHash(cfg), + } + data, _ := json.Marshal(payload) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func BuildConfigKey(imageSpec ImageSpec, userConfigJSON string) (string, error) { + var cfg manifestConfig + if err := json.Unmarshal([]byte(userConfigJSON), &cfg); err != nil { + return "", err + } + return buildConfigKey(imageSpec, cfg) +} diff --git a/spindle/engines/microvm/qemu.go b/spindle/engines/microvm/qemu.go new file mode 100644 index 00000000..0d1fe255 --- /dev/null +++ b/spindle/engines/microvm/qemu.go @@ -0,0 +1,676 @@ +package microvm + +import ( + "context" + _ "embed" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/digitalocean/go-qemu/qmp" + "github.com/google/uuid" +) + +const ( + defaultQMPTimeout = 10 * time.Second + outerSlirpCIDR = "10.0.2.0/24" + innerSlirpNet = "10.0.3.0/24" + innerSlirpHost = "10.0.3.2" + innerSlirpDNS = "10.0.3.3" + innerSlirpDHCP = "10.0.3.15" + netnsTapName = "tap0" + netnsMTU = "65520" +) + +type QEMUConfig struct { + Image ImageSpec + BootTimeout time.Duration + CID uint32 + EnableKVM bool + QEMULogPath string + QMPPath string + SerialLogPath string + WorkDir string + VolumePaths map[string]string + VolumeBaseName string + Cgroup CgroupLimits + Dev bool +} + +type QEMUVMHandle struct { + cid uint32 + Process *os.Process + qemuLogPath string + QMPMon *qmp.SocketMonitor + QMPPath string + serialLogPath string + workDir string + + cmd *exec.Cmd + done chan struct{} + qemuLogFile *os.File + cgroup *CgroupHandle + slirpCmd *exec.Cmd + slirpExit *os.File + waitErr error + waitErrMu sync.Mutex +} + +type qemuRunner struct{} + +func (qemuRunner) Validate(spec ImageSpec, enableKVM bool) error { + if _, err := exec.LookPath(spec.RunnerCmd()); err != nil { + return fmt.Errorf("required host command %q not found in PATH: %w", spec.RunnerCmd(), err) + } + if _, err := os.Stat("/dev/vhost-vsock"); err != nil { + return fmt.Errorf("microvm requires /dev/vhost-vsock for vhost-vsock-device: %w", err) + } + if enableKVM { + if _, err := os.Stat("/dev/kvm"); err != nil { + return fmt.Errorf("microvm KVM was requested but /dev/kvm is not accessible: %w", err) + } + } + if len(spec.NetworkInterfaces) > 0 { + if _, err := os.Stat("/dev/net/tun"); err != nil { + return fmt.Errorf("microvm slirp4netns networking requires /dev/net/tun: %w", err) + } + for _, cmd := range []string{"ip", "mount", "slirp4netns", "unshare"} { + if _, err := exec.LookPath(cmd); err != nil { + return fmt.Errorf("required host command %q not found in PATH: %w", cmd, err) + } + } + } + return nil +} + +func (qemuRunner) Start(ctx context.Context, cfg VMConfig, volumePaths map[string]string, logger *slog.Logger) (VMHandle, error) { + bootTimeout := cfg.BootTimeout + if bootTimeout == 0 { + bootTimeout = 10 * time.Second + } + return StartQEMU(ctx, QEMUConfig{ + Image: cfg.Image, + BootTimeout: bootTimeout, + CID: cfg.CID, + EnableKVM: cfg.EnableKVM, + WorkDir: cfg.WorkDir, + VolumePaths: volumePaths, + Cgroup: cfg.Cgroup, + Dev: cfg.Dev, + }, logger) +} + +func StartQEMU(ctx context.Context, cfg QEMUConfig, logger *slog.Logger) (VMHandle, error) { + if logger == nil { + logger = slog.Default() + } + + workDir := cfg.WorkDir + + handle := &QEMUVMHandle{ + workDir: workDir, + } + + var ok bool + defer func() { + if !ok { + _ = handle.Close() + } + }() + + cid := cfg.CID + if cid == 0 { + var err error + cid, err = AllocateCID() + if err != nil { + return nil, err + } + } + if cid < minGuestCID { + return nil, fmt.Errorf("guest CID must be >= %d", minGuestCID) + } + handle.cid = cid + + volumePaths := cfg.VolumePaths + + qemuLogPath := cfg.QEMULogPath + if qemuLogPath == "" { + qemuLogPath = filepath.Join(workDir, "qemu.log") + } + qemuLogFile, err := createParentedFile(qemuLogPath) + if err != nil { + return nil, err + } + handle.qemuLogPath = qemuLogPath + handle.qemuLogFile = qemuLogFile + + serialLogPath := cfg.SerialLogPath + if serialLogPath == "" { + serialLogPath = filepath.Join(workDir, "serial.log") + } + if err := os.MkdirAll(filepath.Dir(serialLogPath), 0o755); err != nil { + return nil, fmt.Errorf("create serial log directory: %w", err) + } + handle.serialLogPath = serialLogPath + + qmpPath := cfg.QMPPath + if qmpPath == "" { + qmpPath = filepath.Join(workDir, "qmp.sock") + } + handle.QMPPath = qmpPath + + qemuCmd := cfg.Image.RunnerCmd() + qemuBinary, err := exec.LookPath(qemuCmd) + if err != nil { + return nil, fmt.Errorf("%s command not found in PATH: %w", qemuCmd, err) + } + + args, err := qemuArgs(qemuArgsConfig{ + Image: cfg.Image, + CID: cid, + EnableKVM: cfg.EnableKVM, + QMPPath: qmpPath, + SerialLogPath: serialLogPath, + VolumePaths: volumePaths, + }) + if err != nil { + return nil, err + } + + cmd, slirpNet, err := qemuCommand(ctx, qemuBinary, args, cfg.Image, workDir, cfg.Dev) + if err != nil { + return nil, err + } + cmd.Env = append(os.Environ(), "TMPDIR="+workDir) + cmd.Stdout = qemuLogFile + cmd.Stderr = qemuLogFile + + cgroup, err := prepareCgroup(cfg.Cgroup, logger) + if err != nil { + return nil, err + } + handle.cgroup = cgroup + + logger.Info("starting qemu microvm", "cid", cid, "workDir", workDir, "serialLog", serialLogPath, "qmp", qmpPath) + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("starting qemu: %w", err) + } + handle.cmd = cmd + handle.Process = cmd.Process + handle.done = make(chan struct{}) + go func() { + err := cmd.Wait() + handle.waitErrMu.Lock() + handle.waitErr = err + handle.waitErrMu.Unlock() + close(handle.done) + }() + + if err := cgroup.AddProcess(cmd.Process.Pid, logger); err != nil { + return nil, err + } + + if slirpNet != nil { + handle.slirpCmd, handle.slirpExit, err = slirpNet.Start(ctx, qemuLogFile, logger) + if err != nil { + return nil, err + } + if handle.slirpCmd != nil && handle.slirpCmd.Process != nil { + if err := cgroup.AddProcess(handle.slirpCmd.Process.Pid, logger); err != nil { + return nil, err + } + } + } + + qmpTimeout := cfg.BootTimeout + if qmpTimeout == 0 { + qmpTimeout = defaultQMPTimeout + } + if err := handle.waitForQMP(ctx, qmpTimeout); err != nil { + return nil, err + } + + status, err := handle.QMPQueryStatus() + if err != nil { + return nil, err + } + if status != "running" { + return nil, fmt.Errorf("qemu guest not running (status: %s)", status) + } + logger.Info("qemu microvm running", "cid", cid, "status", status) + + ok = true + return handle, nil +} + +func (h *QEMUVMHandle) Wait() error { + if h == nil || h.done == nil { + return nil + } + <-h.done + h.waitErrMu.Lock() + defer h.waitErrMu.Unlock() + return h.waitErr +} + +func (h *QEMUVMHandle) WaitContext(ctx context.Context) error { + if h == nil || h.done == nil { + return nil + } + select { + case <-h.done: + h.waitErrMu.Lock() + defer h.waitErrMu.Unlock() + return h.waitErr + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *QEMUVMHandle) Kill() error { + if h == nil || h.Process == nil { + return nil + } + return h.Process.Kill() +} + +func (h *QEMUVMHandle) Shutdown(ctx context.Context) error { + if h == nil { + return nil + } + if h.QMPMon != nil { + if err := h.QMPSystemPowerdown(); err != nil { + return err + } + } + if h.done == nil { + return nil + } + select { + case <-h.done: + return h.Wait() + case <-ctx.Done(): + _ = h.Kill() + _ = h.Wait() + return ctx.Err() + } +} + +func (h *QEMUVMHandle) Close() error { + if h == nil { + return nil + } + + var closeErr error + if h.QMPMon != nil { + closeErr = errors.Join(closeErr, h.QMPMon.Disconnect()) + h.QMPMon = nil + } + if h.Process != nil { + _ = h.Process.Kill() + _ = h.Wait() + } + if h.slirpExit != nil { + _ = h.slirpExit.Close() + h.slirpExit = nil + } + if h.slirpCmd != nil && h.slirpCmd.Process != nil { + _ = h.slirpCmd.Process.Kill() + _ = h.slirpCmd.Wait() + h.slirpCmd = nil + } + if h.qemuLogFile != nil { + closeErr = errors.Join(closeErr, h.qemuLogFile.Close()) + h.qemuLogFile = nil + } + if h.cgroup != nil { + closeErr = errors.Join(closeErr, h.cgroup.Close()) + h.cgroup = nil + } + return closeErr +} + +func (h *QEMUVMHandle) QMPRun(command qmp.Command) ([]byte, error) { + if h == nil || h.QMPMon == nil { + return nil, fmt.Errorf("qmp monitor is not connected") + } + data, err := json.Marshal(command) + if err != nil { + return nil, err + } + return h.QMPMon.Run(data) +} + +func (h *QEMUVMHandle) QMPQueryStatus() (string, error) { + raw, err := h.QMPRun(qmp.Command{Execute: "query-status"}) + if err != nil { + return "", fmt.Errorf("qmp query-status failed: %w", err) + } + + var resp struct { + Return struct { + Status string `json:"status"` + } `json:"return"` + } + if err := json.Unmarshal(raw, &resp); err != nil { + return "", fmt.Errorf("qmp query-status parse: %w", err) + } + return resp.Return.Status, nil +} + +func (h *QEMUVMHandle) QMPSystemPowerdown() error { + _, err := h.QMPRun(qmp.Command{Execute: "system_powerdown"}) + return err +} + +func (h *QEMUVMHandle) Logs() VMLogs { + if h == nil { + return VMLogs{} + } + return VMLogs{ + Serial: h.serialLogPath, + Extra: map[string]string{ + "qemu": h.qemuLogPath, + }, + } +} + +func (h *QEMUVMHandle) CID() uint32 { + if h == nil { + return 0 + } + return h.cid +} + +func (h *QEMUVMHandle) WorkDir() string { + if h == nil { + return "" + } + return h.workDir +} + +func (h *QEMUVMHandle) OOMKilled() bool { + if h == nil { + return false + } + return h.cgroup.OOMKilled() +} + +func (h *QEMUVMHandle) waitForQMP(ctx context.Context, timeout time.Duration) error { + qmpCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + var lastErr error + for { + mon, err := qmp.NewSocketMonitor("unix", h.QMPPath, 2*time.Second) + if err == nil { + if err = mon.Connect(); err == nil { + h.QMPMon = mon + return nil + } + _ = mon.Disconnect() + } + lastErr = err + + select { + case <-qmpCtx.Done(): + return fmt.Errorf("qmp connect timeout: %w", lastErr) + case <-h.done: + return fmt.Errorf("qemu exited before qmp was ready: %w", h.Wait()) + case <-time.After(25 * time.Millisecond): + } + } +} + +func qemuCommand( + ctx context.Context, + qemuBinary string, + args []string, + spec ImageSpec, + workDir string, + dev bool, +) (*exec.Cmd, *slirpNamespace, error) { + if len(spec.NetworkInterfaces) == 0 { + return exec.CommandContext(ctx, qemuBinary, args...), nil, nil + } + + ipPath, err := exec.LookPath("ip") + if err != nil { + return nil, nil, fmt.Errorf("ip command not found in PATH: %w", err) + } + mountPath, err := exec.LookPath("mount") + if err != nil { + return nil, nil, fmt.Errorf("mount command not found in PATH: %w", err) + } + unsharePath, err := exec.LookPath("unshare") + if err != nil { + return nil, nil, fmt.Errorf("unshare command not found in PATH: %w", err) + } + + pidFile, resolvPath, wrapperPath, err := prepareQEMUNetnsFiles(workDir, dev) + if err != nil { + return nil, nil, err + } + + cmdArgs := append([]string{ + "--user", + "--map-root-user", + "--net", + "--mount", + "--propagation", "private", + "--", + wrapperPath, + pidFile, + ipPath, + mountPath, + resolvPath, + qemuBinary, + }, args...) + + cmd := exec.CommandContext(ctx, unsharePath, cmdArgs...) + + return cmd, &slirpNamespace{ + spec: spec, + pidFile: pidFile, + dev: dev, + }, nil +} + +func prepareQEMUNetnsFiles(workDir string, dev bool) (pidFile, resolvPath, wrapperPath string, err error) { + pidFile = filepath.Join(workDir, "qemu-netns.pid") + resolvPath = filepath.Join(workDir, "qemu-netns-resolv.conf") + wrapperPath = filepath.Join(workDir, "qemu-netns-wrapper") + + // the guest resolves through shuttle on 127.0.0.1. keep qemu's slirp DNS + // pointed at an unroutable local resolver inside this network namespace so + // direct guest queries to 10.0.3.3 don't bypass the shuttle dns policy. + if err := os.WriteFile(resolvPath, []byte("nameserver 127.0.0.1\n"), 0o644); err != nil { + return "", "", "", fmt.Errorf("write qemu network namespace resolv.conf: %w", err) + } + + if err := writeNetnsWrapper(wrapperPath, dev); err != nil { + return "", "", "", fmt.Errorf("write qemu network namespace wrapper: %w", err) + } + + return pidFile, resolvPath, wrapperPath, nil +} + +type qemuArgsConfig struct { + Image ImageSpec + CID uint32 + EnableKVM bool + QMPPath string + SerialLogPath string + VolumePaths map[string]string +} + +func qemuArgs(cfg qemuArgsConfig) ([]string, error) { + uuid := uuid.New() + + b := newArgBuilder(64) + + addQEMUMachineArgs(&b, cfg, uuid) + addQEMUStoreArgs(&b, cfg) + + if cfg.EnableKVM { + addQEMUKVMArgs(&b, cfg.Image) + } + + if err := addQEMUVolumeArgs(&b, cfg); err != nil { + return nil, err + } + + if err := addQEMUNetworkArgs(&b, cfg.Image.NetworkInterfaces); err != nil { + return nil, err + } + + b.Optf("-device", "vhost-vsock-device,guest-cid=%d", cfg.CID) + + if len(cfg.Image.RunnerConfig.ExtraArgs) > 0 { + b.Add(cfg.Image.RunnerConfig.ExtraArgs...) + } + + return b.Args(), nil +} + +func addQEMUMachineArgs(b *argBuilder, cfg qemuArgsConfig, uuid uuid.UUID) { + if cfg.Image.RunnerConfig.Machine != "" { + b.Opt("-M", cfg.Image.RunnerConfig.Machine) + } + b.Optf("-m", "%dM", cfg.Image.MemoryMiB) + b.Opt("-smp", strconv.Itoa(cfg.Image.VCPUs)) + + b.Add( + "-nodefaults", + "-no-user-config", + "-no-reboot", + ) + + b.Opt("-kernel", cfg.Image.Kernel) + b.Opt("-initrd", cfg.Image.Initrd) + + b.Opt("-device", "virtio-rng-device") + + b.Optf("-smbios", "type=1,uuid=%s", uuid) + b.Opt("-serial", "file:"+cfg.SerialLogPath) + + // use virtio console if requsted. this is faster than the serial UART logging + // because serial has a higher cost when being accesssed. we still have to + // support serial itself for early kernel boot but thats OK. + if cfg.Image.RunnerConfig.Console == "hvc0" { + b.Optf("-chardev", "file,id=virtiocon0,path=%s,append=on", cfg.SerialLogPath) + b.Add("-device", "virtio-serial-device") + b.Opt("-device", "virtconsole,chardev=virtiocon0") + } + b.Opt("-display", "none") + b.Opt("-monitor", "none") + b.Opt("-append", cfg.Image.BootArgs) + + b.Opt("-sandbox", "on") + b.Optf("-qmp", "unix:%s,server,nowait", cfg.QMPPath) +} + +func addQEMUStoreArgs(b *argBuilder, cfg qemuArgsConfig) { + drive := newOptionBuilder(8) + drive.KV("id", "store") + drive.KV("format", "raw") + drive.Add("read-only=on") + drive.KV("file", cfg.Image.StoreDisk) + drive.Add("if=none") + drive.Add("aio=io_uring") + + b.Opt("-drive", drive.String()) + b.Opt("-device", "virtio-blk-device,drive=store") +} + +func addQEMUKVMArgs(b *argBuilder, image ImageSpec) { + b.Flag("-enable-kvm") + if image.RunnerConfig.CPU != "" { + b.Opt("-cpu", image.RunnerConfig.CPU) + } + b.Opt("-device", "i8042") +} + +func addQEMUVolumeArgs(b *argBuilder, cfg qemuArgsConfig) error { + for index, volume := range cfg.Image.Volumes { + path := cfg.VolumePaths[volume.Image] + if path == "" { + return fmt.Errorf("missing prepared path for volume %q", volume.Image) + } + + driveID := fmt.Sprintf("volume%d", index) + + drive := newOptionBuilder(10) + drive.KV("id", driveID) + drive.KV("format", "raw") + drive.Add("read-only=off") + drive.KV("file", path) + drive.Add("if=none") + drive.Add("aio=io_uring") + drive.Add("discard=unmap") + drive.Add("cache=none") + + b.Opt("-drive", drive.String()) + b.Optf("-device", "virtio-blk-device,drive=%s", driveID) + } + + return nil +} + +func addQEMUNetworkArgs(b *argBuilder, interfaces []NetworkInterface) error { + for _, networkInterface := range interfaces { + if networkInterface.Type != "slirp4netns" { + return fmt.Errorf("unsupported microvm network interface type %q", networkInterface.Type) + } + + netdevOpts := newOptionBuilder(6) + netdevOpts.Add("user") + netdevOpts.KV("id", networkInterface.ID) + netdevOpts.KV("net", innerSlirpNet) + netdevOpts.KV("host", innerSlirpHost) + netdevOpts.KV("dns", innerSlirpDNS) + netdevOpts.KV("dhcpstart", innerSlirpDHCP) + + b.Opt("-netdev", netdevOpts.String()) + b.Optf( + "-device", "virtio-net-device,netdev=%s,mac=%s", + networkInterface.ID, networkInterface.MAC, + ) + } + + return nil +} + +func waitForPIDFile(ctx context.Context, path string) (string, error) { + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + + for { + data, err := os.ReadFile(path) + if err == nil { + pid := strings.TrimSpace(string(data)) + if pid != "" { + return pid, nil + } + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("read qemu network namespace pid: %w", err) + } + + select { + case <-waitCtx.Done(): + return "", fmt.Errorf("waiting for qemu network namespace pid: %w", waitCtx.Err()) + case <-ticker.C: + } + } +} diff --git a/spindle/engines/microvm/read_cache_proxy.go b/spindle/engines/microvm/read_cache_proxy.go new file mode 100644 index 00000000..2d1d749f --- /dev/null +++ b/spindle/engines/microvm/read_cache_proxy.go @@ -0,0 +1,427 @@ +package microvm + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" + "syscall" + "time" + + "github.com/mdlayher/vsock" +) + +const ( + readCacheProxyPortMin = 20000 + readCacheProxyPortMax = 60000 +) + +type ReadCacheProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server +} + +func StartReadCacheProxy(ctx context.Context, cid uint32, upstreams []CacheUpstream, logger *slog.Logger) (*ReadCacheProxy, error) { + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "read_cache", "cid", cid) + + if len(upstreams) == 0 { + return nil, nil + } + + ln, port, err := listenRandomVsockPort(ctx) + if err != nil { + return nil, err + } + + proxy := &ReadCacheProxy{ + port: port, + ln: ln, + } + proxy.server = &http.Server{ + Handler: cacheProxyHandler(upstreams, logger), + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 10 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("proxy stopped", "cid", cid, "port", port, "error", err) + } + }() + + logger.Info("started proxy", "cid", cid, "port", port, "upstreams", len(upstreams)) + return proxy, nil +} + +func (p *ReadCacheProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *ReadCacheProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + return closeErr +} + +type cidFilteredVsockListener struct { + *vsock.Listener + cid uint32 + logger *slog.Logger +} + +func (l *cidFilteredVsockListener) Accept() (net.Conn, error) { + for { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + + addr, ok := conn.RemoteAddr().(*vsock.Addr) + if ok && addr.ContextID == l.cid { + return conn, nil + } + + l.logger.Warn("dropping proxy connection from unexpected cid", "remote", conn.RemoteAddr(), "expectedCID", l.cid) + _ = conn.Close() + } +} + +func parseCacheUpstreams(raw []string) ([]*url.URL, error) { + upstreams := make([]*url.URL, 0, len(raw)) + seen := make(map[string]struct{}, len(raw)) + for _, value := range raw { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + + parsed, err := url.Parse(value) + if err != nil { + return nil, fmt.Errorf("parse URL %q: %w", value, err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return nil, fmt.Errorf("URL %q uses unsupported scheme %q", value, parsed.Scheme) + } + if parsed.Host == "" { + return nil, fmt.Errorf("URL %q is missing host", value) + } + upstreams = append(upstreams, parsed) + } + return upstreams, nil +} + +type CacheUpstream struct { + url *url.URL + // guarded upstreams come from the workflow file + // requests to them are refused for special-purpose address ranges + guarded bool +} + +func BuildCacheUpstreams(rawTrusted, rawGuarded []string) ([]CacheUpstream, error) { + trusted, err := parseCacheUpstreams(rawTrusted) + if err != nil { + return nil, err + } + guarded, err := parseCacheUpstreams(rawGuarded) + if err != nil { + return nil, err + } + return mergeCacheUpstreams(trusted, guarded), nil +} + +func mergeCacheUpstreams(trusted, guarded []*url.URL) []CacheUpstream { + merged := make([]CacheUpstream, 0, len(trusted)+len(guarded)) + seen := make(map[string]struct{}, len(trusted)+len(guarded)) + for _, u := range trusted { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, CacheUpstream{url: u}) + } + for _, u := range guarded { + if _, ok := seen[u.String()]; ok { + continue + } + seen[u.String()] = struct{}{} + merged = append(merged, CacheUpstream{url: u, guarded: true}) + } + return merged +} + +func listenRandomVsockPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.Listen(port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock port: %w", lastErr) +} + +func randomVsockPort() (uint32, error) { + var data [4]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, fmt.Errorf("allocate read vsock port: %w", err) + } + span := uint32(readCacheProxyPortMax - readCacheProxyPortMin) + return readCacheProxyPortMin + binary.BigEndian.Uint32(data[:])%span, nil +} + +var proxyTransport = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// for guarded upstreams, this will refuse requests made to blocked addresses +var guardedProxyTransport = &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + Control: refuseSpecialPurposeAddrs, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, +} + +// this should run after dns resolution, so it should cover any rebinding tricks +func refuseSpecialPurposeAddrs(network, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("split dial address %q: %w", address, err) + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("refusing to dial non-IP address %q", host) + } + for _, ipnet := range blockedNamespaceNets { + if ipnet.Contains(ip) { + return fmt.Errorf("refusing to dial %s: %s is blocked for workflow caches", ip, ipnet) + } + } + return nil +} + +// the proxy is the cache as far as the guest is concerned, so we answer +// /nix-cache-info ourselves instead of racing the upstreams for it. merging +// those also doesn't make any sense (none of the options make sense for +// merging) +const nixCacheInfo = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n" + +func cacheProxyHandler(upstreams []CacheUpstream, logger *slog.Logger) http.Handler { + proxy := &httputil.ReverseProxy{ + // nothing to do here: the racing transport builds the full URL per + // upstream, it just needs the guest's path/query left intact + Rewrite: func(*httputil.ProxyRequest) {}, + ErrorLog: slog.NewLogLogger(logger.Handler(), slog.LevelError), + Transport: ¶llelRacingTransport{ + upstreams: upstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + }, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/nix-cache-info" { + w.Header().Set("Content-Type", "text/x-nix-cache-info") + _, _ = io.WriteString(w, nixCacheInfo) + return + } + proxy.ServeHTTP(w, r) + }) +} + +func cacheProxyProtocols() *http.Protocols { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetUnencryptedHTTP2(true) + return protocols +} + +func mergeQuery(base, extra string) string { + switch { + case base == "": + return extra + case extra == "": + return base + default: + return base + "&" + extra + } +} + +type parallelRacingTransport struct { + upstreams []CacheUpstream + underlying http.RoundTripper + guardedUnderlying http.RoundTripper + logger *slog.Logger +} + +func (t *parallelRacingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + type result struct { + resp *http.Response + err error + is404 bool + idx int + } + + resCh := make(chan result, len(t.upstreams)) + cancels := make([]context.CancelFunc, len(t.upstreams)) + var wg sync.WaitGroup + + for i, upstream := range t.upstreams { + wg.Add(1) + ctx, cancel := context.WithCancel(req.Context()) + cancels[i] = cancel + + go func(idx int, target CacheUpstream, uCtx context.Context) { + defer wg.Done() + + raceReq := req.Clone(uCtx) + // rewrite to the target, joining the upstream's base path/query + // with what the guest asked for + raceReq.URL.Scheme = target.url.Scheme + raceReq.URL.Host = target.url.Host + raceReq.URL.Path = strings.TrimSuffix(target.url.Path, "/") + req.URL.Path + raceReq.URL.RawQuery = mergeQuery(target.url.RawQuery, req.URL.RawQuery) + // Host wins over URL.Host for the outgoing Host header, and the + // reverse proxy preserves the guest's (127.0.0.1:), which + // host-routed upstreams like fastly reject with a 421 + raceReq.Host = target.url.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.url.User; user != nil { + password, _ := user.Password() + raceReq.SetBasicAuth(user.Username(), password) + } + + rt := t.underlying + if target.guarded { + rt = t.guardedUnderlying + } + resp, err := rt.RoundTrip(raceReq) + if err != nil { + resCh <- result{err: err, idx: idx} + return + } + if resp.StatusCode == http.StatusNotFound { + _ = resp.Body.Close() // don't care about the body of a 404 + resCh <- result{is404: true, idx: idx} + return + } + if resp.StatusCode >= 400 { + // an erroring upstream must not win over a healthy one + _ = resp.Body.Close() + resCh <- result{err: fmt.Errorf("upstream returned status %d", resp.StatusCode), idx: idx} + return + } + // yay, ok + resCh <- result{resp: resp, idx: idx} + }(i, upstream, ctx) + } + + go func() { + wg.Wait() + close(resCh) + }() + + var total404s int + for res := range resCh { + if res.is404 { + total404s++ + if total404s == len(t.upstreams) { + for _, cancel := range cancels { + cancel() + } + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(strings.NewReader("404 nix path not found")), + Header: make(http.Header), + Request: req, + }, nil + } + continue + } + + if res.err != nil { + if !errors.Is(res.err, context.Canceled) { + t.logger.Warn("upstream failed", + "path", req.URL.Path, + "error", res.err, + ) + } + continue + } + + // cancel other requests + for i, cancel := range cancels { + if i != res.idx { + cancel() + } + } + return res.resp, nil + } + + for _, cancel := range cancels { + cancel() + } + return nil, errors.New("all upstreams failed or timed out") +} diff --git a/spindle/engines/microvm/read_cache_proxy_test.go b/spindle/engines/microvm/read_cache_proxy_test.go new file mode 100644 index 00000000..9b92b137 --- /dev/null +++ b/spindle/engines/microvm/read_cache_proxy_test.go @@ -0,0 +1,171 @@ +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestCacheProxyFallsBackOnNotFound(t *testing.T) { + first := httptest.NewServer(http.NotFoundHandler()) + defer first.Close() + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Fatalf("path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "ok") + })) + defer second.Close() + + upstreams, err := parseCacheUpstreams([]string{first.URL, second.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyServesNixCacheInfoItself(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("upstream should not be hit, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/nix-cache-info", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != nixCacheInfo { + t.Fatalf("body: got %q, want %q", got, nixCacheInfo) + } +} + +func TestCacheProxyErrorStatusDoesNotWinRace(t *testing.T) { + erroring := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Error(w, "misdirected", http.StatusMisdirectedRequest) + })) + defer erroring.Close() + healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + time.Sleep(50 * time.Millisecond) // lose the race to the erroring upstream + _, _ = io.WriteString(w, "ok") + })) + defer healthy.Close() + + upstreams, err := parseCacheUpstreams([]string{erroring.URL, healthy.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyJoinsSubpathQueryAndAuth(t *testing.T) { + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/sub/cache/abc.narinfo" { + t.Errorf("path: got %q, want /sub/cache/abc.narinfo", req.URL.Path) + } + if got := req.URL.Query().Get("token"); got != "s3cret" { + t.Errorf("token: got %q, want s3cret", got) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/woof/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + + upstreamURL := "http://dawn:woof@" + strings.TrimPrefix(upstream.URL, "http://") + "/sub/cache/?token=s3cret" + upstreams, err := parseCacheUpstreams([]string{upstreamURL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Body.String(); got != "ok" { + t.Fatalf("body: got %q, want ok", got) + } +} + +func TestCacheProxyGuardedUpstreamCannotReachBlockedRanges(t *testing.T) { + // httptest listens on 127.0.0.1, which is in the blocked ranges; reaching + // it would mean a workflow-defined cache can hit the host's loopback + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + t.Errorf("guarded upstream on loopback should not be reachable, got request for %q", req.URL.Path) + })) + defer upstream.Close() + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://guest/abc.narinfo", nil) + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(nil, upstreams), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusBadGateway { + t.Fatalf("status: got %d, want 502; body=%q", rec.Code, rec.Body.String()) + } +} + +func TestCacheProxyRewritesHostHeader(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + upstreams, err := parseCacheUpstreams([]string{upstream.URL}) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10500/abc.narinfo", nil) + req.Host = "127.0.0.1:10500" + rec := httptest.NewRecorder() + cacheProxyHandler(mergeCacheUpstreams(upstreams, nil), slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} diff --git a/spindle/engines/microvm/runner.go b/spindle/engines/microvm/runner.go new file mode 100644 index 00000000..c08e3d84 --- /dev/null +++ b/spindle/engines/microvm/runner.go @@ -0,0 +1,24 @@ +package microvm + +import ( + "context" + "fmt" + "log/slog" +) + +type Runner interface { + // check the host has what this backend needs for spec. + Validate(spec ImageSpec, enableKVM bool) error + Start(ctx context.Context, cfg VMConfig, volumePaths map[string]string, logger *slog.Logger) (VMHandle, error) +} + +func runnerFor(runnerType string) (Runner, error) { + switch runnerType { + case "qemu", "": + return qemuRunner{}, nil + case "firecracker": + return nil, fmt.Errorf("runner type %q not implemented yet", runnerType) + default: + return nil, fmt.Errorf("unsupported runner type %q", runnerType) + } +} diff --git a/spindle/engines/microvm/start-test-cache.sh b/spindle/engines/microvm/start-test-cache.sh new file mode 100755 index 00000000..30ccb2ab --- /dev/null +++ b/spindle/engines/microvm/start-test-cache.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail + +# start a local ncps binary cache +# usage: ./start-test-cache.sh [port] + +if [ "$#" -lt 1 ]; then + echo "Usage: $0 [ncps-port]" + exit 1 +fi + +TEST_DIR="$(mkdir -p "$1" && cd "$1" && pwd)" +PORT="${2:-8501}" + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +SECRET_KEY_PATH="$TEST_DIR/test-cache-key.secret" +PUBLIC_KEY_PATH="$TEST_DIR/test-cache-key.pub" +DB_PATH="$TEST_DIR/ncps.sqlite" +CONFIG_PATH="$TEST_DIR/ncps-config.yaml" +STORAGE_DIR="$TEST_DIR/storage" +ENV_PATH="$TEST_DIR/env.sh" +PID_PATH="$TEST_DIR/ncps.pid" + +mkdir -p "$STORAGE_DIR" + +echo "generating binary cache keys.." +nix-store --generate-binary-cache-key test-cache-key "$SECRET_KEY_PATH" "$PUBLIC_KEY_PATH" +PUBKEY_VAL=$(cat "$PUBLIC_KEY_PATH") + +echo "initializing ncps db..." +nix shell nixpkgs#dbmate --command dbmate \ + --migrations-dir "$(nix build --no-link --print-out-paths nixpkgs#ncps)/share/ncps/db/migrations/sqlite" \ + -u "sqlite:$DB_PATH" \ + up + +echo "writing ncps configuration..." +cat < "$CONFIG_PATH" +cache: + allow-delete-verb: true + allow-put-verb: true + hostname: "cache.local" + database-url: "sqlite:$DB_PATH" + secret-key-path: "$SECRET_KEY_PATH" + sign-narinfo: true + storage: + local: "$STORAGE_DIR" + upstream: + urls: + - https://cache.nixos.org + public-keys: + - cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +server: + addr: "127.0.0.1:$PORT" +EOF + +echo "starting ncps on port $PORT..." +export CACHE_ALLOW_PUT_VERB=true +nix shell nixpkgs#ncps --command ncps serve --config "$CONFIG_PATH" & +NCPS_PID=$! +echo "$NCPS_PID" > "$PID_PATH" + +# wait for connection +for i in {1..30}; do + if curl -s "http://127.0.0.1:$PORT/nix-cache-info" > /dev/null; then + echo "ncps is healthy." + break + fi + sleep 0.5 + if ! kill -0 "$NCPS_PID" 2>/dev/null; then + echo "ncps exited unexpectedly during startup." + exit 1 + fi +done + +cat < "$ENV_PATH" +export CACHE_PUBKEY="$PUBKEY_VAL" +export CACHE_PORT="$PORT" +export CACHE_URL="http://127.0.0.1:$PORT" +export CACHE_UPLOAD_URL="http://127.0.0.1:$PORT/upload" +export CACHE_SECRET_KEY_PATH="$SECRET_KEY_PATH" +export NCPS_PID="$NCPS_PID" +export TEST_DIR="$TEST_DIR" +EOF + +echo "cache server started successfully. source $ENV_PATH to use, and kill PID $NCPS_PID or check $PID_PATH to stop it." diff --git a/spindle/engines/microvm/test-spindle-microvm.sh b/spindle/engines/microvm/test-spindle-microvm.sh new file mode 100755 index 00000000..40b1e2e4 --- /dev/null +++ b/spindle/engines/microvm/test-spindle-microvm.sh @@ -0,0 +1,850 @@ +#!/usr/bin/env bash +set -euo pipefail +# note: needs `sudo modprobe vhost_vsock`! + +log() { + printf "\n\033[1;36m>>> %s\033[0m\n" "$*" +} + +strip_ansi() { + local esc + esc=$(printf '\033') + sed -E "s/${esc}\[[0-9;]*[a-zA-Z]//g; s/${esc}\([a-zA-Z]//g" "$@" +} + +declare -a TEST_NAMES=() +declare -a TEST_STATUSES=() +declare -a TEST_TIMES=() + +get_time_ms() { + local t="${EPOCHREALTIME:-}" + if [[ "$t" == *.* ]]; then + local secs="${t%.*}" + local subs="${t#*.}" + subs="${subs:0:3}" + while [ "${#subs}" -lt 3 ]; do + subs="${subs}0" + done + echo "${secs}${subs}" + else + echo "$(date +%s)000" + fi +} + +format_duration() { + local ms=$1 + local secs=$((ms / 1000)) + local rem=$((ms % 1000)) + printf "%d.%03ds" "$secs" "$rem" +} + +print_summary() { + if [ "${#TEST_NAMES[@]}" -eq 0 ]; then + return + fi + printf "\n" + log "test summary" + echo "=========================================" + local passed_count=0 + local failed_count=0 + local total_time=0 + for i in "${!TEST_NAMES[@]}"; do + local name="${TEST_NAMES[$i]}" + local status="${TEST_STATUSES[$i]}" + local duration_ms="${TEST_TIMES[$i]}" + local duration_str + duration_str=$(format_duration "$duration_ms") + + local status_color="\033[0;32m" + if [ "$status" = "Failed" ]; then + status_color="\033[0;31m" + failed_count=$((failed_count + 1)) + else + passed_count=$((passed_count + 1)) + fi + total_time=$((total_time + duration_ms)) + + printf " %-30s %b%-8b\033[0m %s\n" "$name" "$status_color" "$status" "$duration_str" + done + echo "-----------------------------------------" + local total_tests="${#TEST_NAMES[@]}" + local total_time_str + total_time_str=$(format_duration "$total_time") + printf " total: %d tests, %d passed, %d failed\n" "$total_tests" "$passed_count" "$failed_count" + printf " total execution time: %s\n" "$total_time_str" + echo "=========================================" +} + +JOBS="${JOBS:-4}" +while [[ $# -gt 0 ]]; do + case "$1" in + -j | --jobs) + JOBS="$2" + shift 2 + ;; + --jobs=*) + JOBS="${1#*=}" + shift + ;; + --only) + TEST_ONLY="$2" + shift 2 + ;; + --only=*) + TEST_ONLY="${1#*=}" + shift + ;; + *) + echo "unknown argument: $1" >&2 + echo "usage: $0 [-j N|--jobs N] [--only TEST]" >&2 + exit 1 + ;; + esac +done +if ! [[ "$JOBS" =~ ^[0-9]+$ ]] || [ "$JOBS" -lt 1 ]; then + echo "error: --jobs must be a positive integer (got '$JOBS')" >&2 + exit 1 +fi + +pick_free_port() { + local port + for _ in $(seq 1 50); do + port=$(((RANDOM % 16384) + 20000)) + if ! (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + echo "$port" + return 0 + fi + done + echo "error: could not find a free port for the cache" >&2 + return 1 +} + +SUCCESS=0 +rm -rf /tmp/test-spindle-microvm-logs + +log "setup local cache & temp environment" +TEMP_DIR=$(mktemp -d -t test-spindle-microvm-XXXXXX) + +log "build spindle & microvm image tarball" +nix develop --command go build -o spindle/spindle-microvm-run ./cmd/spindle-microvm-run +TARBALL_PATH=$(nix build .#spindle-nixos-image-tarball --no-link --print-out-paths) +mkdir -p "$TEMP_DIR/image" +tar -C "$TEMP_DIR/image" -xzf "$TARBALL_PATH" +IMAGE_SPEC_JSON="$TEMP_DIR/image/spec.json" + +log "build alpine microvm image tarball" +ALPINE_TARBALL_PATH=$(nix build .#spindle-alpine-image-tarball --no-link --print-out-paths) +mkdir -p "$TEMP_DIR/alpine-image" +tar -C "$TEMP_DIR/alpine-image" -xzf "$ALPINE_TARBALL_PATH" +ALPINE_IMAGE_SPEC_JSON="$TEMP_DIR/alpine-image/spec.json" + +kill_temp_dir_procs() { + if [ -f "$TEMP_DIR/ncps.pid" ]; then + kill "$(cat "$TEMP_DIR/ncps.pid")" 2>/dev/null || true + fi + pkill -TERM -f "$TEMP_DIR" 2>/dev/null || true + local i + for i in $(seq 1 20); do + pgrep -f "$TEMP_DIR" >/dev/null 2>&1 || break + sleep 0.25 + done + pkill -KILL -f "$TEMP_DIR" 2>/dev/null || true +} + +collect_logs() { + echo "test failed. copying logs to /tmp/test-spindle-microvm-logs" + mkdir -p /tmp/test-spindle-microvm-logs + local f + for f in "$TEMP_DIR"/*; do + [ -f "$f" ] && cp "$f" /tmp/test-spindle-microvm-logs/ + done + local work logf + for work in "$TEMP_DIR"/work-*; do + [ -d "$work" ] || continue + for logf in "$work"/*.log; do + [ -f "$logf" ] || continue + strip_ansi "$logf" > "/tmp/test-spindle-microvm-logs/$(basename "$work")-$(basename "$logf")" + done + done +} + +CLEANED=0 +cleanup() { + [ "$CLEANED" -eq 1 ] && return + CLEANED=1 + + print_summary + log "cleaning up..." + + local jobs_pids + jobs_pids=$(jobs -p) + [ -n "$jobs_pids" ] && kill $jobs_pids 2>/dev/null || true + + kill_temp_dir_procs + + [ "$SUCCESS" -ne 1 ] && collect_logs + + chmod -R +w "$TEMP_DIR" 2>/dev/null || true + rm -rf "$TEMP_DIR" + echo "done" +} +trap cleanup EXIT +# route signals through the EXIT trap so an interrupt still tears down VMs. +trap 'exit 130' INT +trap 'exit 143' TERM + +CACHE_PORT=$(pick_free_port) +./spindle/engines/microvm/start-test-cache.sh "$TEMP_DIR" "$CACHE_PORT" +source "$TEMP_DIR/env.sh" + +run_vm() { + local name="" + local timeout="60s" + local upload=0 + local activate="" + local no_cache=0 + local db="" + local spec="$IMAGE_SPEC_JSON" + + while [[ $# -gt 0 ]]; do + case "$1" in + --spec) + spec="$2" + shift 2 + ;; + --name) + name="$2" + shift 2 + ;; + --timeout) + timeout="$2" + shift 2 + ;; + --upload) + upload=1 + shift + ;; + --activate) + activate="$2" + shift 2 + ;; + --no-cache) + no_cache=1 + shift + ;; + --db) + db="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "unknown argument: $1" >&2 + exit 1 + ;; + esac + done + + local work_dir="$TEMP_DIR/work-${name}" + mkdir -p "$work_dir" + + local args=( + --image-spec "$spec" + --work-dir "$work_dir" + --exec-timeout "$timeout" + --port "${SPINDLE_TEST_VSOCK_PORT:-10240}" + --memory-mib 2049 + ) + + if [ "$no_cache" -eq 0 ]; then + args+=( + --cache-read-url "$CACHE_URL" + --cache-trusted-public-key "$CACHE_PUBKEY" + ) + fi + + if [ "$upload" -eq 1 ]; then + args+=( + --cache-upload-url "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" + ) + fi + + if [ -n "$activate" ]; then + args+=( + --activate-config "$activate" + ) + fi + + if [ -n "$db" ]; then + args+=( + --db "$db" + ) + fi + + local out + if ! out=$(spindle/spindle-microvm-run "${args[@]}" -- "$@" 2>&1); then + echo "$out" | strip_ansi >&2 + strip_ansi "$work_dir/serial.log" >&2 + strip_ansi "$work_dir/qemu.log" >&2 + return 1 + fi + echo "$out" +} + +run_test_job() { + local name="$1" + local func="$2" + local port="$3" + export SPINDLE_TEST_VSOCK_PORT="$port" + + local logfile="$TEMP_DIR/test-${name}.log" + local start + start=$(get_time_ms) + log "[$name] start (vsock port $port)" + + local status="Passed" + if ! "$func" > "$logfile" 2>&1; then + status="Failed" + fi + + local duration_ms=$(($(get_time_ms) - start)) + printf '%s\t%s\n' "$status" "$duration_ms" > "$TEMP_DIR/test-${name}.status" + + local duration_str + duration_str=$(format_duration "$duration_ms") + if [ "$status" = "Failed" ]; then + printf "\n\033[0;31m>>> [%s] FAILED (%s)\033[0m\n" "$name" "$duration_str" + strip_ansi "$logfile" || true + else + printf "\n\033[0;32m>>> [%s] passed (%s)\033[0m\n" "$name" "$duration_str" + fi +} + +# schedules every selected test across at most $JOBS concurrent VMs (each on its +# own vsock port), then aggregates the per-test status files into the summary +# arrays. returns 1 if any test failed, 0 otherwise. +run_tests() { + local base_port=10240 + local idx=0 + local running=0 + + for func in "${TESTS[@]}"; do + local name="${func#test_}" + name="${name//_/-}" + if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then + continue + fi + + run_test_job "$name" "$func" "$((base_port + idx))" & + idx=$((idx + 1)) + running=$((running + 1)) + + if [ "$running" -ge "$JOBS" ]; then + wait -n || true + running=$((running - 1)) + fi + done + wait + + local failed=0 + for func in "${TESTS[@]}"; do + local name="${func#test_}" + name="${name//_/-}" + if [ -n "${TEST_ONLY:-}" ] && [ "${TEST_ONLY}" != "$name" ]; then + continue + fi + + local statusfile="$TEMP_DIR/test-${name}.status" + if [ ! -f "$statusfile" ]; then + TEST_NAMES+=("$name") + TEST_STATUSES+=("Failed") + TEST_TIMES+=(0) + failed=1 + continue + fi + + local status duration_ms + IFS=$'\t' read -r status duration_ms < "$statusfile" + TEST_NAMES+=("$name") + TEST_STATUSES+=("$status") + TEST_TIMES+=("$duration_ms") + if [ "$status" = "Failed" ]; then + failed=1 + fi + done + + return "$failed" +} + +test_realize() { + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "test-file" "hello from cache"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + local out + out=$(run_vm --name "realize" --timeout "60s" -- /run/current-system/sw/bin/bash -lc ' +set -euo pipefail +store_path=$1 +cache_url=$(sed -n "s/^extra-substituters = //p" /run/spindle/nix.conf) +cache_url=${cache_url%% *} +if [ -z "$cache_url" ]; then + echo "error: cache URL not found in /run/spindle/nix.conf" >&2 + exit 1 +fi + +http_version=$(/run/current-system/sw/bin/curl --http2-prior-knowledge -fsS -o /dev/null -w "%{http_version}" "$cache_url/nix-cache-info") +echo "http_version=$http_version" +case "$http_version" in + 2|2.0) ;; + *) + echo "error: cache proxy did not negotiate HTTP/2 (got $http_version)" >&2 + exit 1 + ;; +esac + +/run/current-system/sw/bin/nix-store --realise "$store_path" >/dev/null +' bash "$test_store_path") || return 1 + + if ! echo "$out" | strip_ansi | grep -q -E "^http_version=2(\\.0)?$"; then + echo "error: cache proxy did not report HTTP/2" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: store path realized from cache and cache proxy accepted cleartext HTTP/2" +} + +test_build_upload() { + local nix_expr='with import {}; writeText "uploaded-test-file" "hello from vm upload"' + local out + out=$(run_vm --name "build-upload" --timeout "120s" --upload -- /run/current-system/sw/bin/bash -l -c "nix-build -E '$nix_expr' --no-out-link") || return 1 + + local built_path + built_path=$(echo "$out" | strip_ansi | grep -v '\.drv' | grep -o '/nix/store/[a-z0-9]*-uploaded-test-file' | head -n 1 || true) + if [ -z "$built_path" ]; then + echo "error: could not find built store path in vm output" >&2 + return 1 + fi + echo "extracted path: $built_path" + + local hash + hash=$(basename "$built_path" | cut -d'-' -f1) + if ! curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then + echo "error: built store path was not uploaded to the binary cache" >&2 + return 1 + fi + echo "success: store path uploaded to cache" +} + +test_networking() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --name "networking" --timeout "120s" --no-cache -- /run/current-system/sw/bin/bash -c "/run/current-system/sw/bin/curl -I --connect-timeout 1 -m 1 http://10.0.2.2:$CACHE_PORT; /run/current-system/sw/bin/nix-store --realise $hello_path") || return 1 + + if echo "$out" | grep -qi -E "unreachable|timeout|failed to connect|timed out" || echo "$out" | grep -q "exited with code"; then + echo "success: host network access blocked" + else + echo "error: guest vm accessed host network or returned unexpected output" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: guest vm reached the internet and substituted hello" +} + +test_substitution_and_no_upload() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --name "nixpkgs-hello" --timeout "180s" --upload -- /run/current-system/sw/bin/nix-store --realise "$hello_path") || return 1 + + # Check that it was substituted from our proxy + if ! echo "$out" | strip_ansi | grep -q -E "copying path.*hello"; then + echo "error: hello package was not substituted (or output mismatch)" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + # Check that nothing was uploaded to the cache + if ! echo "$out" | strip_ansi | grep -q "cache uploaded: 0"; then + echo "error: hello package substitution triggered cache upload" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + echo "success: hello package substituted from upstream cache and was not uploaded" +} + +# a pinned registry, reused by the dependency and registry-pin tests. +ACTIVATION_REGISTRY='"registry": { + "nixpkgs": "github:nixos/nixpkgs/nixos-unstable", + "my-nixpkgs": "nixpkgs" + }' + +test_activation_services() { + local config='{ + "services": { + "openssh": { + "enable": true, + "authorizedKeysFiles": ["/etc/ssh/authorized_keys"] + } + } + }' + local out + out=$(run_vm --name "activation-services" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + if ! echo "$out" | strip_ansi | grep -q "^active$"; then + echo "error: sshd not active after activation" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: openssh service active after activation" +} + +test_activation_dependencies() { + # cowsay as a bare dependency (resolved via the pinned nixpkgs registry); + # hello via the github flakeref and (separately) the my-nixpkgs alias. + local config='{ + '"$ACTIVATION_REGISTRY"', + "dependencies": [ + "cowsay", + "github:nixos/nixpkgs#hello", + "my-nixpkgs#hello" + ] + }' + local out + out=$(run_vm --name "activation-dependencies" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +cowsay "registry pin ok" >/dev/null && echo "cowsay=ran" +echo "hello=$(hello)" +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -qF "cowsay=ran"; then + echo "error: bare dependency 'cowsay' (resolved via the pinned nixpkgs registry) did not run" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -qF "hello=Hello, world!"; then + echo "error: hello dependency (github flakeref + my-nixpkgs alias) did not run" >&2 + echo "$clean" >&2 + return 1 + fi + echo "success: bare, flakeref, and aliased dependencies all resolved and ran" +} + +test_activation_registry_pin() { + # the nixpkgs the image itself was built from; the registry override must NOT + # resolve to this. deterministic (locked in the repo flake), so safe to compare. + local base_nixpkgs + base_nixpkgs=$(nix eval --raw --impure --expr '(builtins.getFlake (toString ./.)).inputs.nixpkgs.outPath') + + local config='{ + '"$ACTIVATION_REGISTRY"' + }' + # the pinned nixpkgs must reach the system nix config: resolving it via the + # flakes CLI must not error "is not locked", and it must win the nixPath. + local out + out=$(run_vm --name "activation-registry-pin" --timeout "300s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +echo "lib_version=$(nix eval --raw nixpkgs#lib.version)" +echo "nix_path=$(nix eval --raw --impure --expr "toString ")" +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -qE "lib_version=[0-9]"; then + echo "error: guest could not resolve nixpkgs#lib.version from the user registry (locked-ref failure?)" >&2 + echo "$clean" >&2 + return 1 + fi + local guest_nixpath + guest_nixpath=$(echo "$clean" | sed -n 's/^nix_path=//p' | head -n1) + if [ -z "$guest_nixpath" ] || [ "$guest_nixpath" = "$base_nixpkgs" ]; then + echo "error: guest nixPath did not resolve to the registry override (got '$guest_nixpath', base '$base_nixpkgs')" >&2 + return 1 + fi + echo "success: pinned nixpkgs registry reached the guest nix config (flakes CLI + nixPath)" +} + +test_activation_cache_substitution() { + # a unique path that only exists in the configured (workflow) cache; the host + # seeds it so the guest can prove it substitutes through the read proxy. + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "activation-cache-test" "hello from the workflow cache"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + # a trivial config: this test only cares that the read proxy serves the + # workflow-cache path during an activated run. + local out + out=$(run_vm --name "activation-cache-substitution" --timeout "300s" --activate '{}' -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +store_path=$1 +# the unique path only exists in the configured cache, so realising it proves it +# was substituted through the read proxy and not built or found elsewhere. +nix-store --realise "$store_path" >/dev/null +echo "substituted=$(cat "$store_path")" +' bash "$test_store_path") || return 1 + + if ! echo "$out" | strip_ansi | grep -qF "substituted=hello from the workflow cache"; then + echo "error: unique path was not substituted from the configured cache" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + echo "success: unique path substituted from the workflow cache through the read proxy" +} + +test_activation_docker() { + local config='{ + "virtualisation": { + "docker": { "enable": true } + } + }' + # docker.service is up, but the daemon socket can lag a beat behind activation; + # wait for it to answer, then pull+run a real image. this drives the slimmed + # kernel modules: overlay.ko storage plus bridge/iptables networking out of the + # pruned tree, with outbound DNS/network over the guest slirp link. + local out + out=$(run_vm --name "activation-docker" --timeout "600s" --activate "$config" -- /run/current-system/sw/bin/bash -l -c ' +set -euo pipefail +echo "docker_unit=$(systemctl is-active docker)" +for i in $(seq 1 60); do docker info >/dev/null 2>&1 && break; sleep 1; done +docker info >/dev/null +echo "storage_driver=$(docker info --format "{{.Driver}}")" +docker run --rm alpine cat /etc/alpine-release | sed "s/^/alpine_release=/" +docker run --rm alpine echo container-ran-ok +') || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -q "^docker_unit=active$"; then + echo "error: docker service not active after activation" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -qE "storage_driver=overlay(2|fs)"; then + echo "error: docker is not using an overlay storage driver (overlay.ko missing?)" >&2 + return 1 + fi + if ! echo "$clean" | grep -qE "alpine_release=[0-9]+\."; then + echo "error: failed to pull and read the alpine image" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "container-ran-ok"; then + echo "error: command did not run inside the alpine container" >&2 + return 1 + fi + echo "success: docker service active, pulled and ran an alpine container on the overlay storage driver" +} + +test_activation_cached_realize() { + local config='{ + "services": { + "openssh": { + "enable": true, + "authorizedKeysFiles": ["/etc/ssh/authorized_keys"] + } + } + }' + local db_path="$TEMP_DIR/activation-cached.db" + + # first run: build the config, upload its closure, and record the toplevel in + # the db. nothing cached yet, so this builds from scratch. + local out + out=$(run_vm --name "activation-cached-first" --timeout "600s" --activate "$config" --db "$db_path" --upload -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + if ! echo "$out" | strip_ansi | grep -q "^active$"; then + echo "error: sshd not active after first activation" >&2 + echo "$out" | strip_ansi >&2 + return 1 + fi + + # second run: same config + db, no upload. must realize the recorded toplevel + # from the cache instead of rebuilding, and the cached system must come up. + out=$(run_vm --name "activation-cached-second" --timeout "300s" --activate "$config" --db "$db_path" -- /run/current-system/sw/bin/systemctl is-active sshd) || return 1 + + local clean + clean=$(echo "$out" | strip_ansi) + if ! echo "$clean" | grep -q "realizing cached NixOS config"; then + echo "error: second run did not realize cached configuration" >&2 + echo "$clean" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "^active$"; then + echo "error: sshd not active after cached config activation" >&2 + echo "$clean" >&2 + return 1 + fi + echo "success: second run realized the cached NixOS config from the cache and sshd came up" +} + +test_alpine() { + local hello_path + hello_path=$(nix-build -E 'with import {}; hello' --no-out-link) + + local out + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine" --timeout "180s" --no-cache -- /bin/sh -lc ' +set -eu +export HOME=/workspace +hello_path=$1 +echo "release=$(cat /etc/alpine-release)" +echo "user=$(id -un)" +git version +bash -c "echo bash=\$BASH_VERSION" +touch /workspace/write-test +echo "workspace writable" +git ls-remote https://tangled.org/@tangled.org/core HEAD >/dev/null +echo "git over https ok" +apk add make +echo "apk ok" +# substitute a real package from cache.nixos.org over HTTPS and run it +nix-store --realise "$hello_path" >/dev/null +echo "ran=$("$hello_path/bin/hello")" +' sh "$hello_path") || return 1 + + echo "$out" | strip_ansi >&2 + for needle in "release=" "user=spindle-workflow" "git version" "bash=" "workspace writable" "git over https ok" "apk ok" "ran=Hello, world!"; do + if ! echo "$out" | strip_ansi | grep -q "$needle"; then + echo "error: alpine guest output missing $needle" >&2 + return 1 + fi + done + echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned + installed over the network, and substituted+ran a package from cache.nixos.org over HTTPS" +} + +# asserts a store path's narinfo shows up in the local cache, retrying briefly +# since the post-build-hook enqueues uploads asynchronously. +cache_has_path() { + local path="$1" + local hash + hash=$(basename "$path" | cut -d'-' -f1) + local i + for i in $(seq 1 20); do + if curl -s -f "$CACHE_URL/${hash}.narinfo" > /dev/null; then + return 0 + fi + sleep 0.5 + done + return 1 +} + +test_alpine_nix() { + local test_store_path + test_store_path=$(nix-build -E 'with import {}; writeText "alpine-nix-test" "hello from cache to alpine"' --no-out-link) + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" + + # exercise the full local-cache path: daemon connectivity, substitution, + # store-db queries, and a build via *both* the classic (nix-build) and the + # new flakes/nix-command (nix build) frontends. the two build derivations + # use distinct names so we can confirm each got uploaded back to the cache. + local out + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-nix" --timeout "180s" --upload -- /bin/sh -lc ' +set -eu +export HOME=/workspace +store_path=$1 + +echo "nix_version=$(nix --version | head -n1)" +{ nix store info >/dev/null 2>&1 || nix store ping >/dev/null 2>&1; } && echo "daemon=ok" + +# substitute a path from the cache and query the store db about it +nix-store --realise "$store_path" >/dev/null +echo "substituted=$(cat "$store_path")" +echo "requisites=$(nix-store --query --requisites "$store_path" | wc -l | tr -d " ")" +nix path-info --json "$store_path" >/dev/null && echo "path_info=ok" + +# build via the new CLI; the substituted path is declared as a real input +# (builtins.storePath) so nix must realise it into the build sandbox first. +# heredoc is unquoted (the outer guest script is single-quoted, so a quoted +# delimiter would close it), hence \$ escapes what nix/the builder must expand. +export DEP="$store_path" +cat > /workspace/new.nix < \$out" ]; +} +NIXEOF +new_path=$(nix build --impure --file /workspace/new.nix --no-link --print-out-paths) +echo "new_path=$new_path" +echo "new_content=$(tr "\n" "|" < "$new_path")" + +# build via the classic CLI +cat > /workspace/old.nix < \$out" ]; +} +NIXEOF +old_path=$(nix-build /workspace/old.nix --no-out-link) +echo "old_path=$old_path" +' sh "$test_store_path") || return 1 + + echo "$out" | strip_ansi >&2 + local clean + clean=$(echo "$out" | strip_ansi) + + local needle + for needle in "daemon=ok" "substituted=hello from cache to alpine" "path_info=ok"; do + if ! echo "$clean" | grep -q "$needle"; then + echo "error: alpine nix output missing '$needle'" >&2 + return 1 + fi + done + if ! echo "$clean" | grep -qE "requisites=[1-9][0-9]*"; then + echo "error: store db query returned no requisites for the substituted path" >&2 + return 1 + fi + if ! echo "$clean" | grep -q "new_content=via-nix-build-with-dep"; then + echo "error: 'nix build' (new CLI) did not realise its substituted dependency and build" >&2 + return 1 + fi + + local new_path old_path + new_path=$(echo "$clean" | grep -o 'new_path=/nix/store/[a-z0-9]*-alpine-nix-build-new' | cut -d= -f2) + old_path=$(echo "$clean" | grep -o 'old_path=/nix/store/[a-z0-9]*-alpine-nix-build-old' | cut -d= -f2) + if [ -z "$new_path" ] || [ -z "$old_path" ]; then + echo "error: could not extract both built store paths from alpine guest output" >&2 + return 1 + fi + if ! cache_has_path "$new_path"; then + echo "error: nix-build (new CLI) output was not uploaded to the cache" >&2 + return 1 + fi + if ! cache_has_path "$old_path"; then + echo "error: nix-build (classic CLI) output was not uploaded to the cache" >&2 + return 1 + fi + echo "success: alpine guest substituted, queried the store db, built via both CLIs, and uploaded both outputs" +} + +TESTS=( + test_alpine + test_alpine_nix + test_realize + test_build_upload + test_networking + test_substitution_and_no_upload + test_activation_services + test_activation_dependencies + test_activation_registry_pin + test_activation_cache_substitution + test_activation_docker + test_activation_cached_realize +) + +log "running ${#TESTS[@]} tests" +if ! run_tests; then + exit 1 +fi + +SUCCESS=1 +log "passed!!" diff --git a/spindle/engines/microvm/upload_cache_proxy.go b/spindle/engines/microvm/upload_cache_proxy.go new file mode 100644 index 00000000..54491325 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_proxy.go @@ -0,0 +1,192 @@ +package microvm + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" + + "github.com/mdlayher/vsock" +) + +type UploadCacheProxy struct { + port uint32 + + ln *vsock.Listener + server *http.Server +} + +func StartUploadCacheProxy(ctx context.Context, cid uint32, uploadURL string, readUpstreams []CacheUpstream, logger *slog.Logger) (*UploadCacheProxy, error) { + if strings.TrimSpace(uploadURL) == "" { + return nil, nil + } + + if logger == nil { + logger = slog.Default() + } + logger = logger.With("where", "upload_cache_proxy", "cid", cid, "uploadURL", uploadURL) + + target, err := url.Parse(uploadURL) + if err != nil { + return nil, fmt.Errorf("parse upload URL %q: %w", uploadURL, err) + } + if target.Scheme != "http" && target.Scheme != "https" { + return nil, fmt.Errorf("upload URL %q uses unsupported scheme %q (must be http or https)", uploadURL, target.Scheme) + } + if target.Host == "" { + return nil, fmt.Errorf("upload URL %q is missing host", uploadURL) + } + + ln, port, err := listenRandomVsockUploadPort(ctx) + if err != nil { + return nil, fmt.Errorf("listen for cache upload proxy: %w", err) + } + + proxy := &UploadCacheProxy{ + port: port, + ln: ln, + } + proxy.server = &http.Server{ + Handler: uploadProxyHandler(target, readUpstreams, logger), + Protocols: cacheProxyProtocols(), + ReadHeaderTimeout: 30 * time.Second, + } + + filtered := &cidFilteredVsockListener{ + Listener: ln, + cid: cid, + logger: logger, + } + go func() { + if err := proxy.server.Serve(filtered); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) { + logger.Warn("upload cache proxy stopped", "port", port, "error", err) + } + }() + + logger.Info("started upload cache proxy", "port", port, "target", uploadURL, "readUpstreams", len(readUpstreams)) + return proxy, nil +} + +func (p *UploadCacheProxy) Port() uint32 { + if p == nil { + return 0 + } + return p.port +} + +func (p *UploadCacheProxy) Close() error { + if p == nil { + return nil + } + + var closeErr error + if p.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + closeErr = errors.Join(closeErr, p.server.Shutdown(ctx)) + cancel() + p.server = nil + } + if p.ln != nil { + closeErr = errors.Join(closeErr, p.ln.Close()) + p.ln = nil + } + return closeErr +} + +func uploadProxyHandler(target *url.URL, readUpstreams []CacheUpstream, logger *slog.Logger) http.Handler { + rp := httputil.NewSingleHostReverseProxy(target) + rp.ErrorLog = slog.NewLogLogger(logger.Handler(), slog.LevelError) + + origDirector := rp.Director + rp.Director = func(req *http.Request) { + origDirector(req) + // ensure host matches target + req.Host = target.Host + // the transport doesn't turn URL userinfo into basic auth, only + // http.Client does, so do it ourselves + if user := target.User; user != nil { + password, _ := user.Password() + req.SetBasicAuth(user.Username(), password) + } + } + + // before uploading, nix copy asks the destination whether it already has each + // path by GET/HEAD-ing .narinfo and skips the ones it does. we answer + // that check across the upload target *and* the read caches: if any of them + // already serves the path there is no point uploading it (the guest would + // just substitute it from there anyway). + narinfoUpstreams := append([]CacheUpstream{{url: target}}, readUpstreams...) + exists := ¶llelRacingTransport{ + upstreams: narinfoUpstreams, + underlying: proxyTransport, + guardedUnderlying: guardedProxyTransport, + logger: logger, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if isNarinfoExistenceCheck(r) { + serveNarinfoExistence(w, r, exists, logger) + return + } + rp.ServeHTTP(w, r) + }) +} + +func isNarinfoExistenceCheck(r *http.Request) bool { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + return strings.HasSuffix(r.URL.Path, ".narinfo") +} + +func serveNarinfoExistence(w http.ResponseWriter, r *http.Request, exists http.RoundTripper, logger *slog.Logger) { + probe := r.Clone(r.Context()) + probe.RequestURI = "" + + resp, err := exists.RoundTrip(probe) + if err != nil { + logger.Warn("upload proxy narinfo check failed, treating as not present", "path", r.URL.Path, "error", err) + w.WriteHeader(http.StatusNotFound) + return + } + defer resp.Body.Close() + + for key, values := range resp.Header { + for _, value := range values { + w.Header().Add(key, value) + } + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, resp.Body); err != nil && !errors.Is(err, context.Canceled) { + logger.Warn("upload proxy narinfo copy failed", "path", r.URL.Path, "error", err) + } +} + +func listenRandomVsockUploadPort(ctx context.Context) (*vsock.Listener, uint32, error) { + var lastErr error + for range 32 { + port, err := randomVsockPort() + if err != nil { + return nil, 0, err + } + ln, err := vsock.Listen(port, nil) + if err == nil { + return ln, port, nil + } + lastErr = err + + select { + case <-ctx.Done(): + return nil, 0, ctx.Err() + default: + } + } + return nil, 0, fmt.Errorf("listen on random vsock upload port: %w", lastErr) +} diff --git a/spindle/engines/microvm/upload_cache_proxy_test.go b/spindle/engines/microvm/upload_cache_proxy_test.go new file mode 100644 index 00000000..ef4e6405 --- /dev/null +++ b/spindle/engines/microvm/upload_cache_proxy_test.go @@ -0,0 +1,128 @@ +package microvm + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +func TestUploadProxyRewritesHostAndAuth(t *testing.T) { + var upstreamHost string + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Host != upstreamHost { + t.Errorf("host: got %q, want %q", req.Host, upstreamHost) + } + if req.URL.Path != "/sub/abc.narinfo" { + t.Errorf("path: got %q, want /sub/abc.narinfo", req.URL.Path) + } + if user, pass, ok := req.BasicAuth(); !ok || user != "dawn" || pass != "woof" { + t.Errorf("basic auth: got %q/%q/%v, want dawn/hunter2/true", user, pass, ok) + } + _, _ = io.WriteString(w, "ok") + })) + defer upstream.Close() + upstreamHost = strings.TrimPrefix(upstream.URL, "http://") + + target, err := url.Parse("http://dawn:woof@" + upstreamHost + "/sub/") + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodPut, "http://127.0.0.1:10501/abc.narinfo", strings.NewReader("narinfo")) + req.Host = "127.0.0.1:10501" + rec := httptest.NewRecorder() + uploadProxyHandler(target, nil, slog.Default()).ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200; body=%q", rec.Code, rec.Body.String()) + } +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +func TestUploadProxySkipsNarinfoAvailableUpstream(t *testing.T) { + var uploadHits int + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + uploadHits++ + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.URL.Path != "/abc.narinfo" { + t.Errorf("upstream path: got %q, want /abc.narinfo", req.URL.Path) + } + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200 (so nix treats the path as present and skips upload)", rec.Code) + } + if !strings.Contains(rec.Body.String(), "StorePath: /nix/store/abc") { + t.Fatalf("body: got %q, want the upstream narinfo body", rec.Body.String()) + } +} + +func TestUploadProxyUploadsNarinfoNobodyHas(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer target.Close() + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer upstream.Close() + + handler := uploadProxyHandler( + mustParseURL(t, target.URL), + []CacheUpstream{{url: mustParseURL(t, upstream.URL)}}, + slog.Default(), + ) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Fatalf("status: got %d, want 404 (so nix uploads the path)", rec.Code) + } +} + +func TestUploadProxySkipsNarinfoAlreadyOnTarget(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, _ = io.WriteString(w, "StorePath: /nix/store/abc\n") + })) + defer target.Close() + + handler := uploadProxyHandler(mustParseURL(t, target.URL), nil, slog.Default()) + + req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1:10501/abc.narinfo", nil) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status: got %d, want 200", rec.Code) + } +} diff --git a/spindle/engines/microvm/vm.go b/spindle/engines/microvm/vm.go new file mode 100644 index 00000000..2ad0b07f --- /dev/null +++ b/spindle/engines/microvm/vm.go @@ -0,0 +1,370 @@ +package microvm + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "log/slog" + "maps" + "math" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "time" + + "tangled.org/core/spindle/models" +) + +const ( + minGuestCID = 3 + vmCrashLogTailBytes = 4096 +) + +func AllocateCID() (uint32, error) { + var data [4]byte + if _, err := rand.Read(data[:]); err != nil { + return 0, fmt.Errorf("allocate guest CID: %w", err) + } + return minGuestCID + binary.BigEndian.Uint32(data[:])%60000, nil +} + +func prepareWorkDir(workDir string) error { + if workDir == "" { + return fmt.Errorf("microvm work directory is required") + } + if err := os.MkdirAll(workDir, 0o755); err != nil { + return fmt.Errorf("create microvm work directory: %w", err) + } + return nil +} + +func prepareVolumes(ctx context.Context, workDir string, volumes []Volume, mkfsExt4 string) (map[string]string, error) { + paths := make(map[string]string, len(volumes)) + for _, volume := range volumes { + if volume.ReadOnly { + return nil, fmt.Errorf("read-only microvm volume %q is not supported yet", volume.Image) + } + if volume.FSType != "ext4" { + return nil, fmt.Errorf("microvm volume %q uses unsupported fsType %q", volume.Image, volume.FSType) + } + if volume.ImageType != "" && volume.ImageType != "raw" { + return nil, fmt.Errorf("microvm volume %q uses unsupported imageType %q", volume.Image, volume.ImageType) + } + + path := filepath.Join(workDir, filepath.Base(volume.Image)) + if err := createSparseFile(path, volume.SizeMiB); err != nil { + return nil, err + } + noJournal := volume.MountPoint == "/workspace" + if err := runMkfsExt4(ctx, mkfsExt4, path, noJournal); err != nil { + return nil, err + } + paths[volume.Image] = path + } + return paths, nil +} + +func createSparseFile(path string, sizeMiB int64) error { + if sizeMiB <= 0 { + return fmt.Errorf("sparse file %q size must be positive", path) + } + if sizeMiB > math.MaxInt64/(1024*1024) { + return fmt.Errorf("sparse file %q size is too large", path) + } + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("create sparse file %q: %w", path, err) + } + defer file.Close() + + if err := file.Truncate(sizeMiB * 1024 * 1024); err != nil { + return fmt.Errorf("resize sparse file %q: %w", path, err) + } + return nil +} + +func runMkfsExt4(ctx context.Context, mkfsExt4, path string, noJournal bool) error { + if mkfsExt4 == "" { + return fmt.Errorf("mkfs.ext4 path is required") + } + args := []string{"-F"} + if noJournal { + args = append(args, "-O", "^has_journal") + } + args = append(args, path) + + cmd := exec.CommandContext(ctx, mkfsExt4, args...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("mkfs.ext4 %q: %w: %s", path, err, strings.TrimSpace(string(output))) + } + return nil +} + +func createParentedFile(path string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("create log directory: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return nil, fmt.Errorf("create log file %q: %w", path, err) + } + return file, nil +} + +type VMLogs struct { + Serial string + Extra map[string]string +} + +type VMHandle interface { + Shutdown(ctx context.Context) error + WaitContext(ctx context.Context) error + Close() error + Logs() VMLogs + CID() uint32 + WorkDir() string + OOMKilled() bool +} + +type VMConfig struct { + Image ImageSpec + CID uint32 + EnableKVM bool + WorkDir string + Cgroup CgroupLimits + + BootTimeout time.Duration + MkfsExt4 string + Dev bool +} + +type workflowState struct { + ImageSpec ImageSpec + ImageSpecPath string + Config manifestConfig + ConfigKey string + Image string + CacheReadURLs []string + CacheTrustedPublicKeys []string + VM VMHandle + Agent *AgentSession + ReadCache *ReadCacheProxy + UploadCache *UploadCacheProxy + DNSProxy *DNSProxy + WorkDir string + NixOSToplevelCache nixosToplevelCacheStore +} + +func (e *Engine) cleanupState(ctx context.Context, wid models.WorkflowId, state *workflowState) error { + if state == nil { + return nil + } + + ctx = context.WithoutCancel(ctx) + + var err error + err = errors.Join(err, e.drainNixCache(ctx, state)) + err = errors.Join(err, e.shutdownVM(ctx, wid, state)) + err = errors.Join(err, closeIO(&state.Agent)) + err = errors.Join(err, closeIO(&state.ReadCache)) + err = errors.Join(err, closeIO(&state.UploadCache)) + err = errors.Join(err, closeIO(&state.DNSProxy)) + err = errors.Join(err, removeWorkDir(state)) + return err +} + +func (e *Engine) drainNixCache(ctx context.Context, state *workflowState) error { + if state.Agent == nil || e.cfg.NixCache.UploadURL == "" { + return nil + } + + drainCtx, cancel := context.WithTimeout(ctx, cacheDrainTimeout) + defer cancel() + + if _, err := state.Agent.Drain(drainCtx); err != nil { + return fmt.Errorf("drain nix cache: %w", err) + } + return nil +} + +func (e *Engine) shutdownVM(ctx context.Context, wid models.WorkflowId, state *workflowState) error { + if state.VM == nil { + return nil + } + + var err error + + if state.Agent != nil { + gracefulCtx, cancel := context.WithTimeout(ctx, vmShutdownTimeout) + poweredOff, poweroffErr := e.poweroffViaAgent(gracefulCtx, wid, state) + cancel() + + err = errors.Join(err, poweroffErr) + if poweredOff { + return errors.Join(err, closeIO(&state.VM)) + } + } + + fallbackCtx, cancel := context.WithTimeout(ctx, vmShutdownTimeout) + defer cancel() + + if shutdownErr := state.VM.Shutdown(fallbackCtx); shutdownErr != nil { + e.l.Warn("microVM shutdown fallback failed", "workflow", wid, "error", shutdownErr) + err = errors.Join(err, shutdownErr) + } + + return errors.Join(err, closeIO(&state.VM)) +} + +func (e *Engine) poweroffViaAgent(ctx context.Context, wid models.WorkflowId, state *workflowState) (bool, error) { + if err := state.Agent.Poweroff(ctx); err != nil { + e.l.Warn("agent poweroff request failed", "workflow", wid, "error", err) + return false, err + } + + if err := state.VM.WaitContext(ctx); err != nil { + e.l.Warn("agent poweroff did not stop microVM", "workflow", wid, "error", err) + return false, nil + } + + return true, nil +} + +// helper for closing io interfaces, sets to nil to prevent double-close +func closeIO[T io.Closer](field *T) error { + closer := *field + var zero T + *field = zero + if any(closer) == any(zero) { + return nil + } + return closer.Close() +} + +func removeWorkDir(state *workflowState) error { + if state.WorkDir == "" { + return nil + } + + err := os.RemoveAll(state.WorkDir) + state.WorkDir = "" + return err +} + +// returns a context derived from ctx that is cancelled either when ctx itself +// is cancelled or when the microVM exits on its own. the returned flag reports +// whether the VM exited (as opposed to ctx being cancelled for another reason, +// e.g. the workflow timeout), letting callers tell a crash apart from a +// timeout. cancel must be called to release the watcher goroutine. +func watchVMExit(ctx context.Context, vm VMHandle) (context.Context, *atomic.Bool, context.CancelFunc) { + exited := &atomic.Bool{} + watchCtx, cancel := context.WithCancel(ctx) + if vm == nil { + return watchCtx, exited, cancel + } + go func() { + _ = vm.WaitContext(watchCtx) // returns when VM exits or watchCtx is cancelled + if watchCtx.Err() == nil { + exited.Store(true) + cancel() // don't forget to cancel the watchCtx... + } + }() + return watchCtx, exited, cancel +} + +func vmCrashLog(vm VMHandle) string { + if vm == nil { + return "" + } + logs := vm.Logs() + + var b strings.Builder + if tail := tailFile(logs.Serial, vmCrashLogTailBytes); tail != "" { + fmt.Fprintf(&b, "==== serial log ====\n%s\n", tail) + } + for _, name := range slices.Sorted(maps.Keys(logs.Extra)) { + if tail := tailFile(logs.Extra[name], vmCrashLogTailBytes); tail != "" { + fmt.Fprintf(&b, "==== %s log ====\n%s\n", name, tail) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func tailFile(path string, max int64) string { + if path == "" { + return "" + } + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + if info, err := f.Stat(); err == nil && info.Size() > max { + if _, err := f.Seek(-max, io.SeekEnd); err != nil { + return "" + } + } + data, err := io.ReadAll(f) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func waitAgentConn(ctx context.Context, connCh <-chan net.Conn) (net.Conn, error) { + select { + case conn := <-connCh: + if conn == nil { + return nil, fmt.Errorf("agent connection closed before setup") + } + return conn, nil + case <-ctx.Done(): + return nil, fmt.Errorf("waiting for agent: %w", ctx.Err()) + } +} + +func StartVM(ctx context.Context, cfg VMConfig, logger *slog.Logger) (VMHandle, error) { + if logger == nil { + logger = slog.Default() + } + + runner, err := runnerFor(cfg.Image.RunnerType) + if err != nil { + return nil, err + } + if err := cfg.Image.Validate(); err != nil { + return nil, err + } + if err := cfg.Image.validateImageFiles(); err != nil { + return nil, err + } + if err := runner.Validate(cfg.Image, cfg.EnableKVM); err != nil { + return nil, err + } + + if err := prepareWorkDir(cfg.WorkDir); err != nil { + return nil, err + } + + mkfsExt4 := cfg.MkfsExt4 + if mkfsExt4 == "" { + mkfsExt4, err = exec.LookPath("mkfs.ext4") + if err != nil { + return nil, fmt.Errorf("mkfs.ext4 command not found in PATH: %w", err) + } + } + volumePaths, err := prepareVolumes(ctx, cfg.WorkDir, cfg.Image.Volumes, mkfsExt4) + if err != nil { + return nil, err + } + + return runner.Start(ctx, cfg, volumePaths, logger) +} diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index 81746504..07ca225b 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -39,6 +39,8 @@ type Engine struct { l *slog.Logger cfg *config.Config + slotter engine.WorkflowSlotter + cleanupMu sync.Mutex cleanup map[string][]cleanupFunc } @@ -168,9 +170,10 @@ func New(ctx context.Context, cfg *config.Config) (*Engine, error) { l := log.FromContext(ctx).With("component", "spindle") e := &Engine{ - docker: dcli, - l: l, - cfg: cfg, + docker: dcli, + l: l, + cfg: cfg, + slotter: engine.NewSemaphoreSlotter(cfg.NixeryPipelines.MaxConcurrentWorkflows), } e.cleanup = make(map[string][]cleanupFunc) @@ -178,6 +181,18 @@ func New(ctx context.Context, cfg *config.Config) (*Engine, error) { return e, nil } +func (e *Engine) AcquireWorkflowSlot( + ctx context.Context, + wid models.WorkflowId, + wf *models.Workflow, +) (engine.WorkflowSlot, error) { + if e.slotter == nil { + return engine.NoopSlot{}, nil + } + + return e.slotter.AcquireWorkflowSlot(ctx, wid, wf) +} + func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, wfLogger models.WorkflowLogger) error { /// -------------------------INITIAL SETUP------------------------------------------ l := e.l.With("workflow", wid) @@ -235,6 +250,11 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m l.Info("creating container") wfLogger.DataWriter(setupStepIdx, "stdout").Write([]byte("Creating container...")) + extraHosts := []string{"host.docker.internal:host-gateway"} + for _, h := range e.cfg.Server.DevExtraHosts { + extraHosts = append(extraHosts, h+":host-gateway") + } + resp, err := e.docker.ContainerCreate(ctx, &container.Config{ Image: addl.image, Cmd: []string{"cat"}, @@ -265,7 +285,7 @@ func (e *Engine) SetupWorkflow(ctx context.Context, wid models.WorkflowId, wf *m CapDrop: []string{"ALL"}, CapAdd: []string{"CAP_DAC_OVERRIDE", "CAP_CHOWN", "CAP_FOWNER", "CAP_SETUID", "CAP_SETGID"}, SecurityOpt: []string{"no-new-privileges"}, - ExtraHosts: []string{"host.docker.internal:host-gateway"}, + ExtraHosts: extraHosts, Resources: container.Resources{ Memory: e.cfg.NixeryPipelines.MaxJobMemoryMB * 1024 * 1024, }, diff --git a/spindle/models/clone.go b/spindle/models/clone.go index d3a17574..e95575df 100644 --- a/spindle/models/clone.go +++ b/spindle/models/clone.go @@ -5,6 +5,7 @@ import ( "strings" "tangled.org/core/api/tangled" + "tangled.org/core/hostutil" "tangled.org/core/workflow" ) @@ -55,7 +56,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe } } - repoURL := BuildRepoURL(tr.Repo, dev) + repoURL := BuildRepoURL(tr.Repo) var cloneOpts tangled.Pipeline_CloneOpts if twf.Clone != nil { @@ -63,13 +64,20 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe } fetchArgs := buildFetchArgs(cloneOpts, commitSHA) + // In dev mode we point at Caddy via host-gateway with a self-signed cert, + // so skip the TLS check for the fetch call. + fetchCmd := "git fetch" + if dev { + fetchCmd = "git -c http.sslVerify=false fetch" + } + return CloneStep{ kind: StepKindSystem, name: "Clone repository into workspace", commands: []string{ "git init", fmt.Sprintf("git remote add origin %s", repoURL), - fmt.Sprintf("git fetch %s", strings.Join(fetchArgs, " ")), + fmt.Sprintf("%s %s", fetchCmd, strings.Join(fetchArgs, " ")), "git checkout FETCH_HEAD", }, } @@ -102,21 +110,18 @@ func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { } // BuildRepoURL constructs the repository URL from repo metadata. -func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo, devMode bool) string { - scheme := "https://" - if devMode { - scheme = "http://" +func BuildRepoURL(repo *tangled.Pipeline_TriggerRepo) string { + if repo == nil { + return "" } - // Get host from knot - host := repo.Knot - - // In dev mode, replace localhost with host.docker.internal for Docker networking - if devMode && strings.Contains(host, "localhost") { - host = strings.ReplaceAll(host, "localhost", "host.docker.internal") + host, noSSL, _ := hostutil.ParseHostname(repo.Knot) + scheme := "https" + if noSSL { + scheme = "http" } - return fmt.Sprintf("%s%s/%s", scheme, host, *repo.RepoDid) + return fmt.Sprintf("%s://%s/%s", scheme, host, *repo.RepoDid) } // buildFetchArgs constructs the arguments for git fetch based on clone options diff --git a/spindle/models/clone_test.go b/spindle/models/clone_test.go index eeedaa11..64fd792b 100644 --- a/spindle/models/clone_test.go +++ b/spindle/models/clone_test.go @@ -166,6 +166,7 @@ func TestBuildCloneStep_SkipFlag(t *testing.T) { func TestBuildCloneStep_DevMode(t *testing.T) { twf := tangled.Pipeline_Workflow{ + Engine: "nixery", Clone: &tangled.Pipeline_CloneOpts{ Depth: 1, Skip: false, @@ -177,7 +178,7 @@ func TestBuildCloneStep_DevMode(t *testing.T) { NewSha: "abc123", }, Repo: &tangled.Pipeline_TriggerRepo{ - Knot: "localhost:3000", + Knot: "knot.tngl.boltless.dev", Did: "did:plc:user123", Repo: sp("my-repo"), RepoDid: sp("did:plc:boltless"), @@ -186,11 +187,10 @@ func TestBuildCloneStep_DevMode(t *testing.T) { step := BuildCloneStep(twf, tr, true) - // In dev mode, should use http:// and replace localhost with host.docker.internal + // In dev mode, sslVerify should be disabled allCmds := strings.Join(step.Commands(), " ") - expectedURL := "http://host.docker.internal:3000/did:plc:boltless" - if !strings.Contains(allCmds, expectedURL) { - t.Errorf("Expected dev mode URL '%s' in commands", expectedURL) + if !strings.Contains(allCmds, "git -c http.sslVerify=false fetch") { + t.Error("Expected sslVerify to be disabled in dev mode clone commands") } } diff --git a/spindle/models/pipeline_env.go b/spindle/models/pipeline_env.go index cf9bf64c..5e9703c8 100644 --- a/spindle/models/pipeline_env.go +++ b/spindle/models/pipeline_env.go @@ -10,7 +10,7 @@ import ( // PipelineEnvVars extracts environment variables from pipeline trigger metadata. // These are framework-provided variables that are injected into workflow steps. -func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId, devMode bool) map[string]string { +func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId) map[string]string { if tr == nil { return nil } @@ -34,7 +34,7 @@ func PipelineEnvVars(tr *tangled.Pipeline_TriggerMetadata, pipelineId PipelineId env["TANGLED_REPO_REPO_DID"] = *tr.Repo.RepoDid } env["TANGLED_REPO_DEFAULT_BRANCH"] = tr.Repo.DefaultBranch - env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo, devMode) + env["TANGLED_REPO_URL"] = BuildRepoURL(tr.Repo) } switch workflow.TriggerKind(tr.Kind) { diff --git a/spindle/models/pipeline_env_test.go b/spindle/models/pipeline_env_test.go index 574f207a..65f2122e 100644 --- a/spindle/models/pipeline_env_test.go +++ b/spindle/models/pipeline_env_test.go @@ -27,7 +27,7 @@ func TestPipelineEnvVars_PushBranch(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check standard CI variable if env["CI"] != "true" { @@ -90,7 +90,7 @@ func TestPipelineEnvVars_PushTag(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) if env["TANGLED_REF"] != "refs/tags/v1.2.3" { t.Errorf("Expected TANGLED_REF='refs/tags/v1.2.3', got '%s'", env["TANGLED_REF"]) @@ -123,7 +123,7 @@ func TestPipelineEnvVars_PullRequest(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check ref variables for PR if env["TANGLED_REF"] != "refs/heads/feature-branch" { @@ -179,7 +179,7 @@ func TestPipelineEnvVars_ManualWithInputs(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Check manual input variables if env["TANGLED_INPUT_VERSION"] != "1.0.0" { @@ -216,10 +216,9 @@ func TestPipelineEnvVars_DevMode(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, true) + env := PipelineEnvVars(tr, id) - // Dev mode should use http:// and replace localhost with host.docker.internal - expectedURL := "http://host.docker.internal:3000/did:plc:boltless" + expectedURL := "http://localhost:3000/did:plc:boltless" if env["TANGLED_REPO_URL"] != expectedURL { t.Errorf("Expected TANGLED_REPO_URL='%s', got '%s'", expectedURL, env["TANGLED_REPO_URL"]) } @@ -230,7 +229,7 @@ func TestPipelineEnvVars_NilTrigger(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(nil, id, false) + env := PipelineEnvVars(nil, id) if env != nil { t.Error("Expected nil env for nil trigger") @@ -252,7 +251,7 @@ func TestPipelineEnvVars_NilPushData(t *testing.T) { Knot: "example.com", Rkey: "123123", } - env := PipelineEnvVars(tr, id, false) + env := PipelineEnvVars(tr, id) // Should still have repo variables if env["TANGLED_REPO_KNOT"] != "example.com" { diff --git a/spindle/queue/queue.go b/spindle/queue/queue.go index 754a265a..b90aa09e 100644 --- a/spindle/queue/queue.go +++ b/spindle/queue/queue.go @@ -1,7 +1,10 @@ package queue import ( + "slices" "sync" + + "github.com/bluesky-social/indigo/atproto/syntax" ) type Job struct { @@ -9,26 +12,43 @@ type Job struct { OnFail func(error) } +type ownedJob struct { + owner syntax.DID + job Job +} + +// prefers users with fewer running jobs, otherwise it's FIFO type Queue struct { - jobs chan Job + mu sync.Mutex + cond *sync.Cond + queue []ownedJob + running map[syntax.DID]int + maxSize int workers int + stopped bool wg sync.WaitGroup } func NewQueue(queueSize, numWorkers int) *Queue { - return &Queue{ - jobs: make(chan Job, queueSize), + q := &Queue{ + maxSize: queueSize, workers: numWorkers, + running: make(map[syntax.DID]int), } + q.cond = sync.NewCond(&q.mu) + return q } -func (q *Queue) Enqueue(job Job) bool { - select { - case q.jobs <- job: - return true - default: +// todo(dawn): add a per-user cap so a single user can't fill the queue +func (q *Queue) Enqueue(owner syntax.DID, job Job) bool { + q.mu.Lock() + defer q.mu.Unlock() + if q.stopped || len(q.queue) >= q.maxSize { return false } + q.queue = append(q.queue, ownedJob{owner: owner, job: job}) + q.cond.Signal() + return true } func (q *Queue) Start() { @@ -40,16 +60,68 @@ func (q *Queue) Start() { func (q *Queue) worker() { defer q.wg.Done() - for job := range q.jobs { - if err := job.Run(); err != nil { - if job.OnFail != nil { - job.OnFail(err) - } + for { + picked, ok := q.takeNext() + if !ok { + return } + + err := picked.job.Run() + if err != nil && picked.job.OnFail != nil { + picked.job.OnFail(err) + } + + q.finish(picked.owner) + } +} + +// get or wait for the next job +func (q *Queue) takeNext() (ownedJob, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + for len(q.queue) == 0 && !q.stopped { + q.cond.Wait() // waiting for jobs + } + if q.stopped && len(q.queue) == 0 { + return ownedJob{}, false // no jobs are left and the queue is stopped + } + + idx := q.pickBest() + picked := q.queue[idx] + q.queue = slices.Delete(q.queue, idx, idx+1) + q.running[picked.owner]++ + + return picked, true +} + +// index of the queued job whose owner has the fewest currently-running jobs, +// tiebreaking by arrival order. +func (q *Queue) pickBest() int { + best := 0 + for idx, job := range q.queue { + if q.running[job.owner] < q.running[q.queue[best].owner] { + best = idx + } + } + return best +} + +// called when finishing a job +func (q *Queue) finish(owner syntax.DID) { + q.mu.Lock() + defer q.mu.Unlock() + + q.running[owner]-- + if q.running[owner] <= 0 { + delete(q.running, owner) } } func (q *Queue) Stop() { - close(q.jobs) + q.mu.Lock() + q.stopped = true + q.cond.Broadcast() + q.mu.Unlock() q.wg.Wait() } diff --git a/spindle/queue/queue_test.go b/spindle/queue/queue_test.go new file mode 100644 index 00000000..cadb0ab6 --- /dev/null +++ b/spindle/queue/queue_test.go @@ -0,0 +1,103 @@ +package queue + +import ( + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +const ( + alice = syntax.DID("did:plc:alice") + eve = syntax.DID("did:plc:eve") + dawn = syntax.DID("did:plc:dawn") +) + +func TestQueueDrainsAllJobs(t *testing.T) { + t.Parallel() + + q := NewQueue(10, 2) + q.Start() + + var mu sync.Mutex + var ran []string + done := make(chan struct{}, 5) + + for _, name := range []string{"a", "b", "c", "d", "e"} { + q.Enqueue(dawn, Job{Run: func() error { + mu.Lock() + ran = append(ran, name) + mu.Unlock() + done <- struct{}{} + return nil + }}) + } + + for range 5 { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("timed out waiting for jobs to finish") + } + } + + q.Stop() + + if len(ran) != 5 { + t.Fatalf("expected 5 jobs, ran %d", len(ran)) + } +} + +func TestQueueRejectsWhenFull(t *testing.T) { + t.Parallel() + + // no workers, so the queue never drains + q := NewQueue(2, 0) + + if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("first Enqueue() returned false, want true") + } + if !q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("second Enqueue() returned false, want true") + } + if q.Enqueue(dawn, Job{Run: func() error { return nil }}) { + t.Fatal("third Enqueue() returned true on full queue, want false") + } +} + +func TestQueuePrefersOwnerWithFewestRunning(t *testing.T) { + t.Parallel() + + // 2 workers. alice gets the first slot; while she's holding it, eve's + // job should win the second slot over alice's own queued waiters. + q := NewQueue(20, 2) + + releaseAlice1 := make(chan struct{}) + gotEve := make(chan struct{}, 1) + + q.Enqueue(alice, Job{Run: func() error { + <-releaseAlice1 + return nil + }}) + // alice queues two more + q.Enqueue(alice, Job{Run: func() error { return nil }}) + q.Enqueue(alice, Job{Run: func() error { return nil }}) + // eve queues one + q.Enqueue(eve, Job{Run: func() error { + gotEve <- struct{}{} + return nil + }}) + + q.Start() + + // eve should run while alice's first is still held + select { + case <-gotEve: + case <-time.After(time.Second): + t.Fatal("eve's job did not run while alice was blocked") + } + + close(releaseAlice1) + q.Stop() +} diff --git a/spindle/server.go b/spindle/server.go index 74e8883b..82cbdf95 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -9,6 +9,7 @@ import ( "maps" "net/http" "sync" + "time" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-chi/chi/v5" @@ -25,6 +26,7 @@ import ( "tangled.org/core/spindle/db" "tangled.org/core/spindle/engine" "tangled.org/core/spindle/engines/dummy" + "tangled.org/core/spindle/engines/microvm" "tangled.org/core/spindle/engines/nixery" "tangled.org/core/spindle/models" "tangled.org/core/spindle/queue" @@ -41,34 +43,28 @@ const ( ) type Spindle struct { - jc *jetstream.JetstreamClient - tap *Tap - embedTap *embeddedTap - db *db.DB - e *rbac.Enforcer - l *slog.Logger - n *notifier.Notifier - engs map[string]models.Engine - jq *queue.Queue - cfg *config.Config - ks *eventconsumer.Consumer - res *idresolver.Resolver - vault secrets.Manager - motd []byte - motdMu sync.RWMutex - workflowSem chan struct{} - rootCtx context.Context + jc *jetstream.JetstreamClient + tap *Tap + embedTap *embeddedTap + db *db.DB + e *rbac.Enforcer + l *slog.Logger + n *notifier.Notifier + engs map[string]models.Engine + jq *queue.Queue + cfg *config.Config + ks *eventconsumer.Consumer + res *idresolver.Resolver + vault secrets.Manager + motd []byte + motdMu sync.RWMutex + rootCtx context.Context } // New creates a new Spindle server with the provided configuration and engines. -func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engine) (*Spindle, error) { +func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { logger := log.FromContext(ctx) - d, err := db.Make(ctx, cfg.Server.DBPath) - if err != nil { - return nil, fmt.Errorf("failed to setup db: %w", err) - } - e, err := rbac.NewEnforcer(cfg.Server.DBPath) if err != nil { return nil, fmt.Errorf("failed to setup rbac enforcer: %w", err) @@ -109,9 +105,6 @@ func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engi jq := queue.NewQueue(cfg.Server.QueueSize, cfg.Server.MaxJobCount) logger.Info("initialized queue", "queueSize", cfg.Server.QueueSize, "numWorkers", cfg.Server.MaxJobCount) - workflowSem := make(chan struct{}, cfg.Server.MaxConcurrentWorkflows) - logger.Info("initialized workflow semaphore", "maxConcurrentWorkflows", cfg.Server.MaxConcurrentWorkflows) - collections := []string{ tangled.SpindleMemberNSID, tangled.RepoNSID, @@ -145,19 +138,18 @@ func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engi resolver := idresolver.DefaultResolver(cfg.Server.PlcUrl) spindle := &Spindle{ - jc: jc, - e: e, - db: d, - l: logger, - n: &n, - engs: engines, - jq: jq, - cfg: cfg, - res: resolver, - vault: vault, - motd: defaultMotd, - workflowSem: workflowSem, - rootCtx: ctx, + jc: jc, + e: e, + db: d, + l: logger, + n: &n, + engs: engines, + jq: jq, + cfg: cfg, + res: resolver, + vault: vault, + motd: defaultMotd, + rootCtx: ctx, } err = e.AddSpindle(rbacDomain) @@ -185,9 +177,15 @@ func New(ctx context.Context, cfg *config.Config, engines map[string]models.Engi // job in the above registered queue. ccfg := eventconsumer.NewConsumerConfig() ccfg.Logger = log.SubLogger(logger, "eventconsumer") - ccfg.URLFunc = eventconsumer.DefaultURL(cfg.Server.Dev) ccfg.ProcessFunc = spindle.processPipeline ccfg.CursorStore = cursorStore + if cfg.Server.Dev { + ccfg.RetryInterval = 5 * time.Second + ccfg.MaxRetryInterval = 10 * time.Second + } else { + ccfg.RetryInterval = 1 * time.Minute + ccfg.MaxRetryInterval = 10 * time.Minute + } knownKnots, err := d.Knots() if err != nil { return nil, err @@ -330,14 +328,25 @@ func Run(ctx context.Context) error { return fmt.Errorf("failed to load config: %w", err) } + d, err := db.Make(ctx, cfg.Server.DBPath) + if err != nil { + return fmt.Errorf("failed to setup db: %w", err) + } + nixeryEng, err := nixery.New(ctx, cfg) if err != nil { return err } - s, err := New(ctx, cfg, map[string]models.Engine{ - "nixery": nixeryEng, - "dummy": dummy.New(log.FromContext(ctx)), + microvmEng, err := microvm.New(ctx, cfg, d) + if err != nil { + return err + } + + s, err := New(ctx, cfg, d, map[string]models.Engine{ + "nixery": nixeryEng, + "microvm": microvmEng, + "dummy": dummy.New(log.FromContext(ctx)), }) if err != nil { return err @@ -413,7 +422,7 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, workflows := make(map[models.Engine][]models.Workflow) // Build pipeline environment variables once for all workflows - pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId, s.cfg.Server.Dev) + pipelineEnv := models.PipelineEnvVars(tpl.TriggerMetadata, pipelineId) for _, w := range tpl.Workflows { if w != nil { @@ -467,9 +476,9 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, } } - ok := s.jq.Enqueue(queue.Job{ + ok := s.jq.Enqueue(repoDid, queue.Job{ Run: func() error { - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, s.workflowSem, ctx, &models.Pipeline{ + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.db, s.n, ctx, &models.Pipeline{ RepoDid: repoDid, Workflows: workflows, }, pipelineId)