diff --git a/docker-compose.mill.yml b/docker-compose.mill.yml index 082f1d87..46635396 100644 --- a/docker-compose.mill.yml +++ b/docker-compose.mill.yml @@ -27,6 +27,9 @@ x-mill-executor: &mill-executor SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk + SPINDLE_CACHE_BACKEND: disk + SPINDLE_CACHE_DISK_DIR: /var/lib/spindle/cache + SPINDLE_CACHE_STORE_ID: localinfra-shared-cache SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= @@ -72,8 +75,12 @@ services: SPINDLE_ROLE: mill SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk + SPINDLE_CACHE_BACKEND: disk + SPINDLE_CACHE_DISK_DIR: /var/lib/spindle/cache + SPINDLE_CACHE_STORE_ID: localinfra-shared-cache volumes: - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache mill-tokens: profiles: ["linux"] @@ -116,6 +123,7 @@ services: volumes: - spindle-executor-a-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -131,6 +139,7 @@ services: volumes: - spindle-executor-b-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -146,6 +155,7 @@ services: volumes: - spindle-executor-c-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -155,3 +165,4 @@ volumes: spindle-executor-b-data: spindle-executor-c-data: spindle-artifacts: + spindle-cache: diff --git a/docker-compose.yml b/docker-compose.yml index e6637339..ca90b5a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -182,6 +182,7 @@ services: SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" + SPINDLE_CACHE_BACKEND: disk # route guest nix substitution + uploads through the local ncps cache. # ncps re-signs on serve with cache.local's key, so the guest trusts the # matching public key below (no signing happens in spindle itself). diff --git a/docs/DOCS.md b/docs/DOCS.md index 28e97b2e..78928faa 100644 --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -965,6 +965,58 @@ triggered by a pull request: - `TANGLED_PR_SOURCE_SHA` - The commit SHA of the source branch +### Cache + +The `cache` field lets a workflow persist directories across +pipeline runs. Before the first step, the engine looks up +each entry's key and extracts the matching archive into the +workspace; after all steps succeed, the paths are archived +again and stored back under the key. + +- `key`: name this cache is saved under. Keys are scoped to + the repository and the engine specified. +- `hash`: **optional** list of repo files (lockfiles, + manifests) whose content is folded into the key. The entry + is stored as `-`, so editing + `go.sum` automatically rotates the cache without bumping + the key by hand. Paths are relative to the repository + root and are read from git at the commit being built. +- `paths`: paths to archive. Relative paths are anchored at + the repository checkout, the directory steps start in + (`/workspace/repo` on microvm, `/tangled/workspace` on + nixery). Absolute paths work too, for caching directories + outside the checkout, but note they name engine-specific + locations. All paths must be writable by the CI user and + contain no spaces. +- `compression-level`: **optional** zstd level, `1` + (fastest) to `19` (smallest). defaults to (`5`). +- `when`: **optional** save policy: `on-success` (the + default) or `always`. + +When the exact key (or generation, with `hash`) misses, the +newest older generation under the same key is restored. + +```yaml +cache: + - key: go-mod + hash: + - go.sum + - go.mod + paths: + - .gocache +``` + +Caches are only restored and saved for trusted pipelines +(pushes and same-repository pull requests). Pipelines +building untrusted code, like pull requests from forks, +skip the cache entirely. Saving follows each entry's `when` +policy, except on timeout, when nothing is saved. A cache +miss or failure never fails the workflow. + +The spindle operator chooses the storage backend. See +[Running spindle](#running-spindle). If no backend is +configured, `cache` entries are ignored. + ### Steps The `steps` field allows you to define what steps should run @@ -1516,6 +1568,51 @@ cache (and read from it), configure the cache (prefix - `SPINDLE_NIX_CACHE_UPLOAD_URL`: Cache URL that paths built in the guest are uploaded to. +The generic CI cache (the workflow-level +[`cache`](#cache) field) is configured via prefix +`SPINDLE_CACHE_`. + +- `SPINDLE_CACHE_BACKEND`: Storage backend, `disk` or `s3` + (default: `""`, caching disabled). +- `SPINDLE_CACHE_DISK_DIR`: Directory for the `disk` backend + (default: a `cache` directory next to the spindle + database). +- `SPINDLE_CACHE_S3_BUCKET`: Unversioned bucket for the `s3` + backend. Credentials come from the standard AWS chain and + need `s3:GetBucketVersioning` in addition to object access. +- `SPINDLE_CACHE_S3_PREFIX`: Key prefix inside the bucket + (default: `"spindle/cache"`). +- `SPINDLE_CACHE_STORE_ID`: Operator-defined identity for a + physically shared store. Set the same value only on mill and + executor processes that can read and write the same objects. +- `SPINDLE_CACHE_RETENTION`: Time since the last restore or + save before an entry is deleted (default: `720h`, or 30 + days). Set to `0` to keep entries indefinitely. +- `SPINDLE_CACHE_PRUNE_INTERVAL`: How often expired entries + are deleted (default: `1h`). +- `SPINDLE_CACHE_MAX_BYTES_PER_OWNER`: Maximum stored bytes + across all repositories owned by one DID (default: + `10737418240`, or 10 GiB). Set to `0` for no byte limit. +- `SPINDLE_CACHE_MAX_ENTRIES_PER_OWNER`: Maximum pending and + stored entries across all repositories owned by one DID + (default: `1000`). Set to `0` for no entry limit. + +Cache metadata, per-owner usage, and limits are enforced from +spindle's SQLite database. Objects are namespaced by the target +repository and authorized per mill lease; storage backends are +never listed during lookup or cleanup. + +In a mill deployment, configure the same cache backend and +`SPINDLE_CACHE_STORE_ID` on the mill and every executor that +should run cached workflows. Without a store ID, cache entries +are ignored and workflows run uncached. The mill owns cache +metadata and pruning; executors transfer cache objects directly. +A disk cache must therefore be a shared filesystem. An S3 cache +must use the same endpoint, bucket, and prefix. Executors +advertise their store ID and archive compatibility namespace, and +the mill only places cached workflows on a compatible store; OS, +architecture, and archive format are part of the cache namespace. + ### Running spindle 1. **Set the environment variables.** For example: diff --git a/nix/microvm/base.nix b/nix/microvm/base.nix index 992e9622..3b9f3026 100644 --- a/nix/microvm/base.nix +++ b/nix/microvm/base.nix @@ -258,6 +258,7 @@ in { gz-utils bzip2 lz4 + zstd p7zip ]; # disable default nixos packages ([perl rsync strace]) diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix index ba85d128..a5632060 100644 --- a/nix/pkgs/spindle-alpine-image.nix +++ b/nix/pkgs/spindle-alpine-image.nix @@ -20,7 +20,8 @@ git = pkgsStatic.callPackage ./spindle-static-git.nix {}; # we don't include gnused, xxd etc. here because busybox has them # we want to keep the image this image small! - guestTools = [nix bash git curl jq]; + # zstd is not a busybox applet, and the spindle cache saves tar|zstd + guestTools = [nix bash git curl jq pkgsStatic.zstd]; # run by busybox at sysinit setupScript = writeText "spindle-setup" '' diff --git a/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs b/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs new file mode 100644 index 00000000..60b1bcec --- /dev/null +++ b/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs @@ -0,0 +1,380 @@ +// @generated +// This file is @generated by prost-build. +/// executor identity, sent on connect +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Hello { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, + /// GOARCH of the node, informational only + #[prost(string, tag = "2")] + pub arch: ::prost::alloc::string::String, + /// operator-defined labels, matched against runs_on + #[prost(string, repeated, tag = "3")] + pub labels: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "4")] + pub epoch: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub cache_store_id: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub cache_namespace: ::prost::alloc::string::String, +} +/// reconnect state for an existing epoch +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Resume { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub ack_seqno: u64, +} +/// per-engine state of a node +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EngineAvailability { + #[prost(bool, tag = "1")] + pub available: bool, + /// opaque engine-defined load metrics, higher means more loaded + #[prost(map = "string, double", tag = "2")] + pub load: ::std::collections::HashMap<::prost::alloc::string::String, f64>, +} +/// full node state +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NodeSnapshot { + #[prost(uint64, tag = "1")] + pub seqno: u64, + #[prost(map = "string, message", tag = "2")] + pub engines: ::std::collections::HashMap<::prost::alloc::string::String, EngineAvailability>, + /// every lease the executor currently holds, reserved or running + #[prost(string, repeated, tag = "3")] + pub active_lease_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// a seat reservation, carries no secrets +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReserveSeat { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub target_engine: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub raw_pipeline_json: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub raw_workflow_json: ::prost::alloc::string::String, + /// pipeline id, the executor reconstructs the exact WorkflowId from it + #[prost(string, tag = "5")] + pub knot: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub rkey: ::prost::alloc::string::String, + #[prost(uint32, tag = "7")] + pub ttl_seconds: u32, + /// repository that owns the pipeline, used to distinguish fork source code + #[prost(string, tag = "8")] + pub repo_did: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReserveResult { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub accepted: bool, + #[prost(string, tag = "3")] + pub reject_reason: ::prost::alloc::string::String, + #[prost(enumeration = "RejectClass", tag = "4")] + pub reject_class: i32, +} +/// a single unlocked secret +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Secret { + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub value: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheBinding { + #[prost(uint32, tag = "1")] + pub entry_index: u32, + #[prost(string, tag = "2")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub restore_id: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub restore_key: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub restore_name: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub save_id: ::prost::alloc::string::String, + #[prost(string, tag = "7")] + pub save_key: ::prost::alloc::string::String, +} +/// promotes a reservation to a running job and hands over the secrets +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommitLease { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub secrets: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub cache_bindings: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Committed { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +/// drops a reservation that was never committed +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReleaseLease { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +/// cancels a running attempt +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelAttempt { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelAck { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StatusEvent { + #[prost(enumeration = "NonterminalStatus", tag = "1")] + pub status: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + #[prost(int64, tag = "3")] + pub exit_code: i64, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LogArtifact { + #[prost(string, tag = "1")] + pub r#ref: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub hash: ::prost::alloc::string::String, +} +/// terminal outcome of an attempt +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AttemptResult { + #[prost(enumeration = "TerminalStatus", tag = "1")] + pub status: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + #[prost(int64, tag = "3")] + pub exit_code: i64, + #[prost(message, optional, tag = "4")] + pub log_artifact: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheUpdate { + #[prost(enumeration = "CacheUpdateAction", tag = "1")] + pub action: i32, + #[prost(string, tag = "2")] + pub id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub r#ref: ::prost::alloc::string::String, + #[prost(int64, tag = "4")] + pub size_bytes: i64, +} +/// live non-replay log frame +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LiveLog { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub raw_json: ::prost::bytes::Bytes, +} +/// one event in the executor's stream back to the mill +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Event { + #[prost(uint64, tag = "1")] + pub seqno: u64, + #[prost(string, tag = "2")] + pub lease_id: ::prost::alloc::string::String, + #[prost(oneof = "event::Payload", tags = "3, 4, 5")] + pub payload: ::core::option::Option, +} +/// Nested message and enum types in `Event`. +pub mod event { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Payload { + #[prost(message, tag = "3")] + StatusEvent(super::StatusEvent), + #[prost(message, tag = "4")] + AttemptResult(super::AttemptResult), + #[prost(message, tag = "5")] + CacheUpdate(super::CacheUpdate), + } +} +/// a flushed bundle of events +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventBatch { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub events: ::prost::alloc::vec::Vec, +} +/// all events up to and including up_to_seqno are durably processed +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Ack { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub up_to_seqno: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Message { + #[prost(message, optional, tag = "1")] + pub hello: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub resume: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub node_snapshot: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub reserve_seat: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub reserve_result: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub commit_lease: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub committed: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub release_lease: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub cancel_attempt: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub cancel_ack: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub event_batch: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub ack: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub live_log: ::core::option::Option, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RejectClass { + Unspecified = 0, + Transient = 1, + Incompatible = 2, +} +impl RejectClass { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "REJECT_CLASS_UNSPECIFIED", + Self::Transient => "REJECT_CLASS_TRANSIENT", + Self::Incompatible => "REJECT_CLASS_INCOMPATIBLE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REJECT_CLASS_UNSPECIFIED" => Some(Self::Unspecified), + "REJECT_CLASS_TRANSIENT" => Some(Self::Transient), + "REJECT_CLASS_INCOMPATIBLE" => Some(Self::Incompatible), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum NonterminalStatus { + Unspecified = 0, + Running = 1, +} +impl NonterminalStatus { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "NONTERMINAL_STATUS_UNSPECIFIED", + Self::Running => "RUNNING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONTERMINAL_STATUS_UNSPECIFIED" => Some(Self::Unspecified), + "RUNNING" => Some(Self::Running), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TerminalStatus { + Unspecified = 0, + Success = 1, + Failed = 2, + Timeout = 3, + Cancelled = 4, +} +impl TerminalStatus { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "TERMINAL_STATUS_UNSPECIFIED", + Self::Success => "SUCCESS", + Self::Failed => "FAILED", + Self::Timeout => "TIMEOUT", + Self::Cancelled => "CANCELLED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TERMINAL_STATUS_UNSPECIFIED" => Some(Self::Unspecified), + "SUCCESS" => Some(Self::Success), + "FAILED" => Some(Self::Failed), + "TIMEOUT" => Some(Self::Timeout), + "CANCELLED" => Some(Self::Cancelled), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CacheUpdateAction { + Unspecified = 0, + CacheUsed = 1, + CacheStored = 2, + CacheDiscarded = 3, + CacheMissing = 4, +} +impl CacheUpdateAction { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "CACHE_UPDATE_ACTION_UNSPECIFIED", + Self::CacheUsed => "CACHE_USED", + Self::CacheStored => "CACHE_STORED", + Self::CacheDiscarded => "CACHE_DISCARDED", + Self::CacheMissing => "CACHE_MISSING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CACHE_UPDATE_ACTION_UNSPECIFIED" => Some(Self::Unspecified), + "CACHE_USED" => Some(Self::CacheUsed), + "CACHE_STORED" => Some(Self::CacheStored), + "CACHE_DISCARDED" => Some(Self::CacheDiscarded), + "CACHE_MISSING" => Some(Self::CacheMissing), + _ => None, + } + } +} +// @@protoc_insertion_point(module) diff --git a/spindle/config/config.go b/spindle/config/config.go index 3f7c3b44..535bab2b 100644 --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -76,6 +76,18 @@ type LegacyS3 struct { LogBucket string `env:"LOG_BUCKET"` } +type Cache struct { + Backend string `env:"BACKEND"` // "disk" or "s3" + DiskDir string `env:"DISK_DIR"` + S3Bucket string `env:"S3_BUCKET"` + S3Prefix string `env:"S3_PREFIX, default=spindle/cache"` + StoreID string `env:"STORE_ID"` + Retention time.Duration `env:"RETENTION, default=720h"` + PruneInterval time.Duration `env:"PRUNE_INTERVAL, default=1h"` + MaxBytesPerOwner int64 `env:"MAX_BYTES_PER_OWNER, default=10737418240"` + MaxEntriesPerOwner int64 `env:"MAX_ENTRIES_PER_OWNER, default=1000"` +} + type MicroVMPipelines struct { ImageDir string `env:"IMAGE_DIR"` OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live @@ -143,6 +155,7 @@ type Config struct { ArtifactStores ArtifactStores `env:",prefix=SPINDLE_ARTIFACT_STORES_"` LegacyS3 LegacyS3 `env:",prefix=SPINDLE_S3_"` Mill Mill `env:",prefix=SPINDLE_MILL_"` + Cache Cache `env:",prefix=SPINDLE_CACHE_"` } func (c *Config) validate() error { diff --git a/spindle/config/config_test.go b/spindle/config/config_test.go index 12a42e39..c7c6e75d 100644 --- a/spindle/config/config_test.go +++ b/spindle/config/config_test.go @@ -18,3 +18,17 @@ func TestLoadAllowsUnconfiguredMicroVMEngine(t *testing.T) { t.Fatalf("image directory = %q, want empty", cfg.MicroVMPipelines.ImageDir) } } + +func TestLoadCacheStoreID(t *testing.T) { + t.Setenv("SPINDLE_SERVER_HOSTNAME", "spindle.example.com") + t.Setenv("SPINDLE_SERVER_OWNER", "did:web:spindle.example.com") + t.Setenv("SPINDLE_CACHE_STORE_ID", "shared-cache") + + cfg, err := Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if cfg.Cache.StoreID != "shared-cache" { + t.Fatalf("cache store ID = %q, want shared-cache", cfg.Cache.StoreID) + } +} diff --git a/spindle/db/cache.go b/spindle/db/cache.go new file mode 100644 index 00000000..7ab446c6 --- /dev/null +++ b/spindle/db/cache.go @@ -0,0 +1,403 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +type CacheEntry struct { + ID string + StorageKey string + OwnerDID string + RepoDID string + Engine string + CacheKey string + CacheHash string + SizeBytes int64 + State string + CreatedAt time.Time + LastUsedAt time.Time +} + +const cacheEntryColumns = ` + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, + size_bytes, state, created_at, last_used_at` + +func (d *DB) InsertCacheEntry(ctx context.Context, entry CacheEntry) error { + _, err := d.ExecContext(ctx, ` + insert into cache_entries ( + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, + size_bytes, state, created_at, last_used_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + entry.ID, + entry.StorageKey, + entry.OwnerDID, + entry.RepoDID, + entry.Engine, + entry.CacheKey, + entry.CacheHash, + entry.SizeBytes, + entry.State, + entry.CreatedAt.UnixNano(), + entry.LastUsedAt.UnixNano(), + ) + return err +} +func (d *DB) InsertCacheEntryWithinQuota(ctx context.Context, entry CacheEntry, maxEntries int64) (bool, error) { + result, err := d.ExecContext(ctx, ` + insert into cache_entries ( + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, + size_bytes, state, created_at, last_used_at + ) + select ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + where ? <= 0 or ( + select count(*) from cache_entries + where owner_did = ? and state in ('pending', 'ready', 'deleting') + ) < ?`, + entry.ID, + entry.StorageKey, + entry.OwnerDID, + entry.RepoDID, + entry.Engine, + entry.CacheKey, + entry.CacheHash, + entry.SizeBytes, + entry.State, + entry.CreatedAt.UnixNano(), + entry.LastUsedAt.UnixNano(), + maxEntries, + entry.OwnerDID, + maxEntries, + ) + if err != nil { + return false, err + } + inserted, err := result.RowsAffected() + return inserted == 1, err +} + +func (d *DB) MarkCacheEntryReady(ctx context.Context, id string, sizeBytes, maxBytes int64, now time.Time) ([]CacheEntry, error) { + tx, err := d.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + superseded, ready, err := markCacheEntryReady(ctx, tx, id, sizeBytes, maxBytes, now) + if err != nil { + return nil, err + } + if !ready { + return nil, sql.ErrNoRows + } + if err := tx.Commit(); err != nil { + return nil, err + } + return superseded, nil +} + +type cacheEntryTx interface { + QueryRowContext(context.Context, string, ...any) *sql.Row + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func markCacheEntryReady(ctx context.Context, tx cacheEntryTx, id string, sizeBytes, maxBytes int64, now time.Time) ([]CacheEntry, bool, error) { + var repoDID, engine, key, hash string + err := tx.QueryRowContext(ctx, ` + update cache_entries as current + set state = 'ready', size_bytes = ?, last_used_at = ? + where current.id = ? and current.state = 'pending' + and (? <= 0 or ( + select coalesce(sum(other.size_bytes), 0) + from cache_entries as other + where other.owner_did = current.owner_did + and other.state in ('ready', 'deleting') + and other.id <> current.id + ) <= ? - ?) + returning repo_did, engine, cache_key, cache_hash`, + sizeBytes, + now.UnixNano(), + id, + maxBytes, + maxBytes, + sizeBytes, + ).Scan(&repoDID, &engine, &key, &hash) + if err == sql.ErrNoRows { + current, selectErr := scanCacheEntry(tx.QueryRowContext(ctx, `select `+cacheEntryColumns+` from cache_entries where id = ?`, id)) + if selectErr == sql.ErrNoRows { + return nil, false, nil + } + if selectErr != nil { + return nil, false, selectErr + } + if current.State == "ready" { + if current.SizeBytes != sizeBytes { + return nil, false, fmt.Errorf("cache entry %q size changed from %d to %d", id, current.SizeBytes, sizeBytes) + } + return nil, true, nil + } + return nil, false, nil + } + if err != nil { + return nil, false, err + } + + rows, err := tx.QueryContext(ctx, ` + update cache_entries + set state = 'deleting' + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? + and state = 'ready' and id <> ? + returning `+cacheEntryColumns, repoDID, engine, key, hash, id) + if err != nil { + return nil, false, err + } + var superseded []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + rows.Close() + return nil, false, err + } + superseded = append(superseded, *entry) + } + if err := rows.Close(); err != nil { + return nil, false, err + } + if err := rows.Err(); err != nil { + return nil, false, err + } + return superseded, true, nil +} + +func (tx *EventBatchTx) MarkCacheEntryReady(ctx context.Context, id string, sizeBytes, maxBytes int64, now time.Time) ([]CacheEntry, bool, error) { + return markCacheEntryReady(ctx, tx.tx, id, sizeBytes, maxBytes, now) +} + +func (tx *EventBatchTx) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { + _, err := tx.tx.ExecContext(ctx, ` + update cache_entries + set last_used_at = max(last_used_at, ?) + where id = ? and state = 'ready'`, now.UnixNano(), id) + return err +} + +func (tx *EventBatchTx) DiscardCacheEntry(ctx context.Context, id string, pendingOnly bool) (*CacheEntry, error) { + stateFilter := "" + if pendingOnly { + stateFilter = " and state in ('pending', 'deleting')" + } + current, err := scanCacheEntry(tx.tx.QueryRowContext(ctx, ` + update cache_entries + set state = 'deleting' + where id = ?`+stateFilter+` + returning `+cacheEntryColumns, id)) + if err == sql.ErrNoRows { + return nil, nil + } + return current, err +} + +func (d *DB) FindCacheEntry(ctx context.Context, repoDID, engine, key, hash string) (*CacheEntry, error) { + return scanCacheEntry(d.QueryRowContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state = 'ready' + order by created_at desc + limit 1`, repoDID, engine, key, hash)) +} + +func (d *DB) FindFallbackCacheEntry(ctx context.Context, repoDID, engine, key, excludeHash string) (*CacheEntry, error) { + return scanCacheEntry(d.QueryRowContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? + and cache_hash <> ? and cache_hash <> '' and state = 'ready' + order by created_at desc + limit 1`, repoDID, engine, key, excludeHash)) +} + +func (d *DB) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { + _, err := d.ExecContext(ctx, ` + update cache_entries set last_used_at = ? where id = ? and state = 'ready'`, now.UnixNano(), id) + return err +} + +func (d *DB) ClaimPendingCacheEntry(ctx context.Context, id string) (bool, error) { + result, err := d.ExecContext(ctx, ` + update cache_entries set state = 'deleting' where id = ? and state = 'pending'`, id) + if err != nil { + return false, err + } + changed, err := result.RowsAffected() + return changed == 1, err +} + +func (d *DB) ClaimCacheEntry(ctx context.Context, id, expectedState string, expectedLastUsed time.Time) (bool, error) { + result, err := d.ExecContext(ctx, ` + update cache_entries + set state = 'deleting' + where id = ? and state = ? and last_used_at = ?`, + id, expectedState, expectedLastUsed.UnixNano()) + if err != nil { + return false, err + } + changed, err := result.RowsAffected() + return changed == 1, err +} + +func (d *DB) RestoreCacheEntryState(ctx context.Context, id, state string) error { + _, err := d.ExecContext(ctx, ` + update cache_entries set state = ? where id = ? and state = 'deleting'`, state, id) + return err +} + +func (d *DB) ExpiredCacheEntries(ctx context.Context, readyBefore, pendingBefore time.Time, limit int) ([]CacheEntry, error) { + rows, err := d.QueryContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where (state = 'ready' and last_used_at < ?) + or (state in ('pending', 'deleting') and last_used_at < ?) + order by last_used_at + limit ?`, + readyBefore.UnixNano(), + pendingBefore.UnixNano(), + limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var entries []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + return nil, err + } + entries = append(entries, *entry) + } + return entries, rows.Err() +} + +func (d *DB) DeleteCacheEntry(ctx context.Context, id string) error { + _, err := d.ExecContext(ctx, `delete from cache_entries where id = ?`, id) + return err +} + +type MillCacheCapability struct { + Action string + CacheID string + StorageKey string +} + +func (d *DB) SaveMillCacheCapabilities(leaseID string, capabilities []MillCacheCapability) error { + tx, err := d.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`delete from mill_cache_capabilities where lease_id = ?`, leaseID); err != nil { + return err + } + for _, capability := range capabilities { + if capability.Action != "restore" && capability.Action != "save" { + return fmt.Errorf("cache capability has invalid action %q", capability.Action) + } + if capability.CacheID == "" || capability.StorageKey == "" { + return fmt.Errorf("cache capability has empty identity") + } + if _, err := tx.Exec(` + insert into mill_cache_capabilities ( + lease_id, action, cache_id, storage_key + ) values (?, ?, ?, ?)`, + leaseID, + capability.Action, + capability.CacheID, + capability.StorageKey, + ); err != nil { + return err + } + } + return tx.Commit() +} + +func (tx *EventBatchTx) ConsumeMillCacheCapability(ctx context.Context, leaseID, action, cacheID string) (string, bool, error) { + var storageKey string + err := tx.tx.QueryRowContext(ctx, ` + delete from mill_cache_capabilities + where lease_id = ? and action = ? and cache_id = ? + returning storage_key`, + leaseID, action, cacheID, + ).Scan(&storageKey) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, err + } + return storageKey, true, nil +} + +func (tx *EventBatchTx) QueueCacheObjectDeletion(ctx context.Context, storageKey string, now time.Time) error { + _, err := tx.tx.ExecContext(ctx, ` + insert into cache_object_deletions (storage_key, created_at) + values (?, ?) + on conflict(storage_key) do nothing`, + storageKey, now.UnixNano(), + ) + return err +} + +func (d *DB) PendingCacheObjectDeletions(ctx context.Context, limit int) ([]string, error) { + rows, err := d.QueryContext(ctx, ` + select storage_key + from cache_object_deletions + order by created_at + limit ?`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +func (d *DB) CompleteCacheObjectDeletion(ctx context.Context, storageKey string) error { + _, err := d.ExecContext(ctx, `delete from cache_object_deletions where storage_key = ?`, storageKey) + return err +} + +type cacheEntryScanner interface { + Scan(dest ...any) error +} + +func scanCacheEntry(row cacheEntryScanner) (*CacheEntry, error) { + var entry CacheEntry + var createdAt, lastUsedAt int64 + if err := row.Scan( + &entry.ID, + &entry.StorageKey, + &entry.OwnerDID, + &entry.RepoDID, + &entry.Engine, + &entry.CacheKey, + &entry.CacheHash, + &entry.SizeBytes, + &entry.State, + &createdAt, + &lastUsedAt, + ); err != nil { + return nil, err + } + entry.CreatedAt = time.Unix(0, createdAt) + entry.LastUsedAt = time.Unix(0, lastUsedAt) + return &entry, nil +} diff --git a/spindle/db/cache_test.go b/spindle/db/cache_test.go new file mode 100644 index 00000000..b5c7f64e --- /dev/null +++ b/spindle/db/cache_test.go @@ -0,0 +1,412 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" +) + +func testCacheEntry(id, hash, state string, createdAt time.Time) CacheEntry { + return CacheEntry{ + ID: id, + StorageKey: "objects/" + id, + OwnerDID: "did:plc:owner", + RepoDID: "did:plc:repo", + Engine: "microvm", + CacheKey: "dependencies", + CacheHash: hash, + SizeBytes: 10, + State: state, + CreatedAt: createdAt, + LastUsedAt: createdAt, + } +} + +func insertTestCacheEntry(t *testing.T, d *DB, entry CacheEntry) { + t.Helper() + if err := d.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatalf("InsertCacheEntry(%s): %v", entry.ID, err) + } +} +func TestInsertCacheEntryWithinQuota(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + now := time.Now() + + first := testCacheEntry("first", "a", "pending", now) + inserted, err := d.InsertCacheEntryWithinQuota(ctx, first, 1) + if err != nil || !inserted { + t.Fatalf("first quota insert = (%t, %v)", inserted, err) + } + + second := testCacheEntry("second", "b", "pending", now) + inserted, err = d.InsertCacheEntryWithinQuota(ctx, second, 1) + if err != nil { + t.Fatal(err) + } + if inserted { + t.Fatal("entry quota accepted a second object for the same owner") + } + + second.OwnerDID = "did:plc:other" + inserted, err = d.InsertCacheEntryWithinQuota(ctx, second, 1) + if err != nil || !inserted { + t.Fatalf("other owner quota insert = (%t, %v)", inserted, err) + } +} + +func TestMarkCacheEntryReadyEnforcesOwnerByteQuota(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + now := time.Now() + + existing := testCacheEntry("existing", "a", "ready", now) + existing.SizeBytes = 40 + insertTestCacheEntry(t, d, existing) + pending := testCacheEntry("pending", "b", "pending", now) + pending.SizeBytes = 0 + insertTestCacheEntry(t, d, pending) + + if _, err := d.MarkCacheEntryReady(ctx, pending.ID, 11, 50, now); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("over-quota ready error = %v, want sql.ErrNoRows", err) + } + var state string + if err := d.QueryRow(`select state from cache_entries where id = ?`, pending.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if state != "pending" { + t.Fatalf("over-quota entry state = %q, want pending", state) + } + + if _, err := d.MarkCacheEntryReady(ctx, pending.ID, 10, 50, now); err != nil { + t.Fatalf("at-quota ready: %v", err) + } +} + +func TestCacheEntryLookup(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 2, 3, 4, 5, 6, time.UTC) + + insertTestCacheEntry(t, d, testCacheEntry("exact-old", "requested", "ready", base)) + insertTestCacheEntry(t, d, testCacheEntry("exact-new", "requested", "ready", base.Add(time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("exact-pending", "requested", "pending", base.Add(2*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-old", "old-a", "ready", base.Add(3*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-new", "old-b", "ready", base.Add(4*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-pending", "old-c", "pending", base.Add(5*time.Second))) + + exact, err := d.FindCacheEntry(ctx, "did:plc:repo", "microvm", "dependencies", "requested") + if err != nil { + t.Fatalf("FindCacheEntry: %v", err) + } + if exact.ID != "exact-new" { + t.Fatalf("FindCacheEntry returned %q, want exact-new", exact.ID) + } + if !exact.CreatedAt.Equal(base.Add(time.Second)) || !exact.LastUsedAt.Equal(base.Add(time.Second)) { + t.Fatalf("timestamps = (%v, %v), want %v", exact.CreatedAt, exact.LastUsedAt, base.Add(time.Second)) + } + + fallback, err := d.FindFallbackCacheEntry(ctx, "did:plc:repo", "microvm", "dependencies", "requested") + if err != nil { + t.Fatalf("FindFallbackCacheEntry: %v", err) + } + if fallback.ID != "fallback-new" { + t.Fatalf("FindFallbackCacheEntry returned %q, want fallback-new", fallback.ID) + } + + if _, err := d.FindCacheEntry(ctx, "did:plc:repo", "microvm", "missing", "requested"); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing FindCacheEntry error = %v, want sql.ErrNoRows", err) + } +} + +func TestMarkCacheEntryReadySupersedesMatchingReady(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 3, 4, 5, 6, 7, time.UTC) + first := testCacheEntry("first-completion", "same-hash", "pending", base) + second := testCacheEntry("second-completion", "same-hash", "pending", base.Add(time.Second)) + insertTestCacheEntry(t, d, first) + insertTestCacheEntry(t, d, second) + + superseded, err := d.MarkCacheEntryReady(ctx, first.ID, 100, 0, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("mark first ready: %v", err) + } + if len(superseded) != 0 { + t.Fatalf("first completion superseded %d entries, want none", len(superseded)) + } + superseded, err = d.MarkCacheEntryReady(ctx, second.ID, 200, 0, base.Add(3*time.Second)) + if err != nil { + t.Fatalf("mark second ready: %v", err) + } + if len(superseded) != 1 || superseded[0].ID != first.ID || superseded[0].State != "deleting" { + t.Fatalf("second completion superseded %+v, want deleting %s", superseded, first.ID) + } + + ready, err := d.FindCacheEntry(ctx, second.RepoDID, second.Engine, second.CacheKey, second.CacheHash) + if err != nil { + t.Fatalf("find surviving ready entry: %v", err) + } + if ready.ID != second.ID || ready.SizeBytes != 200 || !ready.LastUsedAt.Equal(base.Add(3*time.Second)) { + t.Fatalf("surviving entry = %+v, want %s (200 bytes)", ready, second.ID) + } +} + +func TestMarkCacheEntryReadyConcurrentCompletions(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 4, 5, 6, 7, 8, time.UTC) + first := testCacheEntry("concurrent-a", "same-hash", "pending", base) + second := testCacheEntry("concurrent-b", "same-hash", "pending", base.Add(time.Second)) + insertTestCacheEntry(t, d, first) + insertTestCacheEntry(t, d, second) + + type result struct { + superseded []CacheEntry + err error + } + start := make(chan struct{}) + results := make(chan result, 2) + for _, entry := range []CacheEntry{first, second} { + entry := entry + go func() { + <-start + superseded, err := d.MarkCacheEntryReady(ctx, entry.ID, 100, 0, base.Add(2*time.Second)) + results <- result{superseded: superseded, err: err} + }() + } + close(start) + + var superseded []CacheEntry + for range 2 { + result := <-results + if result.err != nil { + t.Fatalf("concurrent MarkCacheEntryReady: %v", result.err) + } + superseded = append(superseded, result.superseded...) + } + if len(superseded) != 1 || superseded[0].State != "deleting" { + t.Fatalf("concurrent completions superseded %+v, want one deleting entry", superseded) + } + + var readyCount, deletingCount int + if err := d.QueryRowContext(ctx, ` + select sum(state = 'ready'), sum(state = 'deleting') + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ?`, + first.RepoDID, first.Engine, first.CacheKey, first.CacheHash).Scan(&readyCount, &deletingCount); err != nil { + t.Fatalf("count completion states: %v", err) + } + if readyCount != 1 || deletingCount != 1 { + t.Fatalf("completion states = %d ready, %d deleting; want one each", readyCount, deletingCount) + } +} + +func TestCacheEntryReadyTouchAndExpiry(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 2, 3, 4, 5, 6, 7, time.UTC) + + readyOld := testCacheEntry("ready-old", "a", "ready", base) + readyFresh := testCacheEntry("ready-fresh", "b", "ready", base) + pendingOld := testCacheEntry("pending-old", "c", "pending", base) + pendingFresh := testCacheEntry("pending-fresh", "d", "pending", base) + pendingFresh.LastUsedAt = base.Add(20 * time.Minute) + deletingOld := testCacheEntry("deleting-old", "e", "deleting", base) + deletingFresh := testCacheEntry("deleting-fresh", "f", "deleting", base) + deletingFresh.LastUsedAt = base.Add(20 * time.Minute) + for _, entry := range []CacheEntry{readyOld, readyFresh, pendingOld, pendingFresh, deletingOld, deletingFresh} { + insertTestCacheEntry(t, d, entry) + } + + touchedAt := base.Add(30 * time.Minute) + if err := d.TouchCacheEntry(ctx, readyFresh.ID, touchedAt); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + + expired, err := d.ExpiredCacheEntries(ctx, base.Add(10*time.Minute), base.Add(10*time.Minute), 10) + if err != nil { + t.Fatalf("ExpiredCacheEntries: %v", err) + } + got := make(map[string]bool, len(expired)) + for _, entry := range expired { + got[entry.ID] = true + } + if len(got) != 3 || !got[readyOld.ID] || !got[pendingOld.ID] || !got[deletingOld.ID] { + t.Fatalf("expired IDs = %v, want ready-old, pending-old, and deleting-old", got) + } + + limited, err := d.ExpiredCacheEntries(ctx, base.Add(10*time.Minute), base.Add(10*time.Minute), 1) + if err != nil { + t.Fatalf("limited ExpiredCacheEntries: %v", err) + } + if len(limited) != 1 { + t.Fatalf("limited expiry count = %d, want 1", len(limited)) + } +} + +func TestCacheEntryClaim(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 2, 4, 5, 6, 7, 8, time.UTC) + entry := testCacheEntry("claim-me", "hash", "ready", base) + insertTestCacheEntry(t, d, entry) + + touchedAt := base.Add(time.Minute) + if err := d.TouchCacheEntry(ctx, entry.ID, touchedAt); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + claimed, err := d.ClaimCacheEntry(ctx, entry.ID, "ready", base) + if err != nil { + t.Fatalf("stale ClaimCacheEntry: %v", err) + } + if claimed { + t.Fatal("stale ClaimCacheEntry claimed a touched entry") + } + claimed, err = d.ClaimCacheEntry(ctx, entry.ID, "ready", touchedAt) + if err != nil { + t.Fatalf("ClaimCacheEntry: %v", err) + } + if !claimed { + t.Fatal("ClaimCacheEntry did not claim unchanged entry") + } + + if err := d.TouchCacheEntry(ctx, entry.ID, base.Add(2*time.Minute)); err != nil { + t.Fatalf("TouchCacheEntry while deleting: %v", err) + } + if _, err := d.MarkCacheEntryReady(ctx, entry.ID, 999, 0, base.Add(3*time.Minute)); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("MarkCacheEntryReady while deleting error = %v, want sql.ErrNoRows", err) + } + var state string + var lastUsedAt int64 + var sizeBytes int64 + if err := d.QueryRowContext(ctx, ` + select state, last_used_at, size_bytes from cache_entries where id = ?`, entry.ID).Scan(&state, &lastUsedAt, &sizeBytes); err != nil { + t.Fatalf("query claimed entry: %v", err) + } + if state != "deleting" || lastUsedAt != touchedAt.UnixNano() || sizeBytes != entry.SizeBytes { + t.Fatalf("claimed entry = state %q, last used %d, size %d; want deleting, %d, %d", + state, lastUsedAt, sizeBytes, touchedAt.UnixNano(), entry.SizeBytes) + } + + if err := d.RestoreCacheEntryState(ctx, entry.ID, "ready"); err != nil { + t.Fatalf("RestoreCacheEntryState: %v", err) + } + restored, err := d.FindCacheEntry(ctx, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash) + if err != nil { + t.Fatalf("find restored entry: %v", err) + } + if restored.State != "ready" || !restored.LastUsedAt.Equal(touchedAt) { + t.Fatalf("restored entry = state %q, last used %v", restored.State, restored.LastUsedAt) + } +} + +func TestCacheEntryDelete(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + entry := testCacheEntry("delete-me", "hash", "ready", time.Now()) + insertTestCacheEntry(t, d, entry) + + if err := d.DeleteCacheEntry(ctx, entry.ID); err != nil { + t.Fatalf("DeleteCacheEntry: %v", err) + } + if err := d.DeleteCacheEntry(ctx, entry.ID); err != nil { + t.Fatalf("second DeleteCacheEntry: %v", err) + } + if _, err := d.FindCacheEntry(ctx, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("find deleted error = %v, want sql.ErrNoRows", err) + } +} + +func TestMillCacheCapabilitiesAreLeaseScopedAndConsumable(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + lease := MillLease{ + LeaseID: "lease-1", + NodeID: "node-1", + Epoch: "epoch-1", + Engine: "microvm", + Knot: "k", + Rkey: "r", + Workflow: "build", + State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatal(err) + } + capability := MillCacheCapability{ + Action: "save", + CacheID: "cache-1", + StorageKey: "objects/did:web:example.com/cache-1", + } + if err := d.SaveMillCacheCapabilities(lease.LeaseID, []MillCacheCapability{capability}); err != nil { + t.Fatal(err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + storageKey, consumed, err := tx.ConsumeMillCacheCapability( + ctx, + lease.LeaseID, + capability.Action, + capability.CacheID, + ) + if err != nil { + return err + } + if !consumed { + t.Fatal("planned capability was not consumed") + } + if storageKey != capability.StorageKey { + t.Fatalf("consumed storage key = %q, want %q", storageKey, capability.StorageKey) + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + _, consumed, err := tx.ConsumeMillCacheCapability( + ctx, + "another-lease", + capability.Action, + capability.CacheID, + ) + if err != nil { + return err + } + if consumed { + t.Fatal("foreign lease consumed capability") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestCacheObjectDeletionQueuePersistsUntilCompleted(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + key := "objects/did:web:example.com/orphan" + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + return tx.QueueCacheObjectDeletion(ctx, key, time.Now()) + }); err != nil { + t.Fatal(err) + } + keys, err := d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(keys) != 1 || keys[0] != key { + t.Fatalf("pending deletions = %v, want %q", keys, key) + } + if err := d.CompleteCacheObjectDeletion(ctx, key); err != nil { + t.Fatal(err) + } + keys, err = d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(keys) != 0 { + t.Fatalf("completed deletion remained queued: %v", keys) + } +} diff --git a/spindle/db/db.go b/spindle/db/db.go index de8de374..f1677ee5 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -101,6 +101,35 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { updated_at text not null ); + create table if not exists cache_entries ( + id text primary key, + storage_key text unique not null, + owner_did text not null, + repo_did text not null, + engine text not null, + cache_key text not null, + cache_hash text not null, + size_bytes integer not null default 0, + state text not null check (state in ('pending', 'ready', 'deleting')), + created_at integer not null, + last_used_at integer not null + ); + + create index if not exists cache_entries_lookup + on cache_entries (repo_did, engine, cache_key, cache_hash, created_at desc) + where state = 'ready'; + create index if not exists cache_entries_ready_expiry + on cache_entries (last_used_at) where state = 'ready'; + create index if not exists cache_entries_pending_deadline + on cache_entries (last_used_at) where state in ('pending', 'deleting'); + create index if not exists cache_entries_owner_state + on cache_entries (owner_did, state); + + create table if not exists cache_object_deletions ( + storage_key text primary key, + created_at integer not null + ); + create table if not exists pipelines ( id text primary key, repo_did text not null, @@ -147,6 +176,15 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { state text not null ); + create table if not exists mill_cache_capabilities ( + lease_id text not null, + action text not null check (action in ('restore', 'save')), + cache_id text not null, + storage_key text not null, + primary key (lease_id, action, cache_id), + foreign key (lease_id) references mill_leases(lease_id) on delete cascade + ); + create table if not exists mill_executor_cursors ( node_id text not null, epoch text not null, @@ -197,7 +235,7 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { return nil, err } - return &DB{db}, nil + return &DB{DB: db}, nil } func runMigrations(_ context.Context, conn *sql.Conn, logger *slog.Logger) error { diff --git a/spindle/engine/cache.go b/spindle/engine/cache.go new file mode 100644 index 00000000..8dc03e5d --- /dev/null +++ b/spindle/engine/cache.go @@ -0,0 +1,111 @@ +package engine + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "strings" + + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +// cache log steps live below the setup step (-1) +const ( + CacheRestoreStepIdx = -2 + CacheSaveStepIdx = -3 +) + +type cacheStep struct { + name string + command string +} + +func (s cacheStep) Name() string { return s.name } +func (s cacheStep) Command() string { return s.command } +func (s cacheStep) Kind() models.StepKind { return models.StepKindSystem } + +var ( + CacheRestoreStep models.Step = cacheStep{name: "restore cache", command: "restore cached paths"} + CacheSaveStep models.Step = cacheStep{name: "save cache", command: "persist changed paths"} +) + +type CacheRunner interface { + RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error + SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error +} + +const CacheExitNoPaths = 42 + +// avoids storing an empty archive when zstd is missing +const CacheExitNoCompressor = 43 + +// run tar from / so absolute paths survive extraction +// tar exits nonzero on missing paths, so only existing ones reach it +func CacheSaveScript(paths []string, workspaceRoot string, compressionLevel int) string { + trimmed := make([]string, 0, len(paths)) + for _, p := range paths { + if !strings.HasPrefix(p, "/") { + p = workspaceRoot + "/" + p + } + trimmed = append(trimmed, shellQuote(strings.TrimPrefix(p, "/"))) + } + tail := fmt.Sprintf(`tar -cf - -C / -- "$@" | %s`, CacheCompressCmd(compressionLevel)) + return fmt.Sprintf(`set -o pipefail +command -v zstd >/dev/null 2>&1 || { echo "zstd not found in image; cannot save cache" >&2; exit %d; } +set -- +for p in %s; do [ -e "/$p" ] && set -- "$@" "$p"; done +if [ $# -eq 0 ]; then echo "no cache paths exist; skipping" >&2; exit %d; fi +%s`, CacheExitNoCompressor, strings.Join(trimmed, " "), CacheExitNoPaths, tail) +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func CacheCompressCmd(level int) string { + if level == 0 { + return "zstd -T0 -5" + } + return fmt.Sprintf("zstd -T0 -%d", level) +} + +// old entries might still be gzip, so detect them instead of trusting config +func CacheDecompressCmd(br *bufio.Reader) string { + head, _ := br.Peek(4) + if bytes.HasPrefix(head, []byte{0x1f, 0x8b}) { + return "gzip -dc" + } + return "zstd -dc" +} + +// Put keeps draining the pipe after it returns, so the guest writer never +// blocks on a full pipe. +type CacheUpload struct { + Writer *io.PipeWriter + done chan error +} + +func NewCacheUpload(ctx context.Context, store storage.Storage, key string) *CacheUpload { + pr, pw := io.Pipe() + u := &CacheUpload{Writer: pw, done: make(chan error, 1)} + go func() { + err := store.Put(ctx, key, pr) + _, _ = io.Copy(io.Discard, pr) + u.done <- err + }() + return u +} + +func (u *CacheUpload) Abort(err error) { + u.Writer.CloseWithError(err) + <-u.done +} + +// storage treats EOF as a complete archive, so only a clean exec may Finish +func (u *CacheUpload) Finish() error { + u.Writer.Close() + return <-u.done +} diff --git a/spindle/engine/cache_controller.go b/spindle/engine/cache_controller.go new file mode 100644 index 00000000..43589eee --- /dev/null +++ b/spindle/engine/cache_controller.go @@ -0,0 +1,373 @@ +package engine + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "io" + "log/slog" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +type CacheUpdateAction int + +const ( + CacheUsed CacheUpdateAction = iota + 1 + CacheStored + CacheDiscarded + CacheMissing +) + +type CacheUpdate struct { + Action CacheUpdateAction + ID string + Ref string + SizeBytes int64 +} + +type CacheController interface { + Plan(ctx context.Context, pipeline *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) + Apply(ctx context.Context, update CacheUpdate) error +} + +type localCacheController struct { + index *db.DB + store storage.Storage + repoDir string + maxBytesPerOwner int64 + maxEntriesPerOwner int64 + logger *slog.Logger +} + +func NewLocalCacheController(index *db.DB, store storage.Storage, repoDir string, maxBytesPerOwner, maxEntriesPerOwner int64, logger *slog.Logger) CacheController { + return &localCacheController{ + index: index, + store: store, + repoDir: repoDir, + maxBytesPerOwner: maxBytesPerOwner, + maxEntriesPerOwner: maxEntriesPerOwner, + logger: logger, + } +} + +// on a hash miss, the newest older generation still warms the build +// unusable entries degrade to a plain miss +func resolveCaches(ctx context.Context, l *slog.Logger, index *db.DB, repoDID, engine, repoPath, rev string, entries []models.CacheEntry) []models.CacheBinding { + resolved := make([]models.CacheBinding, 0, len(entries)) + for entryIndex, entry := range entries { + hash := "" + if len(entry.Hash) > 0 && repoPath != "" { + sum, missing := hashKeyFiles(ctx, repoPath, rev, entry.Hash) + for _, m := range missing { + l.Warn("cache hash file not in repo", "key", entry.Key, "path", m) + } + hash = sum + } + binding := models.CacheBinding{ + EntryIndex: entryIndex, + Paths: entry.Paths, + Key: entry.Key, + Hash: hash, + CompressionLevel: entry.CompressionLevel, + When: entry.When, + } + + found, err := index.FindCacheEntry(ctx, repoDID, engine, entry.Key, hash) + if errors.Is(err, sql.ErrNoRows) && hash != "" { + found, err = index.FindFallbackCacheEntry(ctx, repoDID, engine, entry.Key, hash) + if err == nil { + binding.RestoreName = entry.Key + "-" + found.CacheHash + } + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + l.Warn("cache lookup failed; entry will save but not restore", "key", entry.Key, "err", err) + } else if err == nil { + binding.RestoreID = found.ID + binding.RestoreKey = found.StorageKey + } + resolved = append(resolved, binding) + } + return resolved +} + +func hashKeyFiles(ctx context.Context, repoPath, rev string, paths []string) (string, []string) { + h := sha256.New() + var missing []string + hashed := 0 + for _, p := range paths { + blob, err := gitBlobId(ctx, repoPath, rev, p) + if err != nil { + missing = append(missing, p) + continue + } + fmt.Fprintf(h, "%s=%s\n", p, blob) + hashed++ + } + if hashed == 0 { + return "", missing + } + return hex.EncodeToString(h.Sum(nil))[:12], missing +} + +// sparse checkouts might not have the file, but the object database does +func gitBlobId(ctx context.Context, repoPath, rev, path string) (string, error) { + out, err := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", rev+":"+path).Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} +func (c *localCacheController) Plan(ctx context.Context, pipeline *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) { + repo, err := c.index.GetRepoByDid(pipeline.RepoDid) + if err != nil { + return nil, fmt.Errorf("cache owner lookup: %w", err) + } + + repoPath, rev := "", "" + if metadata := pipeline.TriggerMetadata; metadata != nil { + if resolvedRev, err := models.ExtractCommitSHA(*metadata); err != nil { + c.logger.Warn("cannot resolve pipeline commit; cache hashing disabled", "err", err) + } else { + did := pipeline.RepoDid.String() + if metadata.SourceRepo != nil && *metadata.SourceRepo != "" { + did = *metadata.SourceRepo + } + repoPath, rev = filepath.Join(c.repoDir, did), resolvedRev + } + } + + cacheNamespace := workflow.CacheNamespace + if cacheNamespace == "" { + cacheNamespace = models.CacheNamespace() + } + cacheEngine := workflow.Engine + "/" + cacheNamespace + bindings := resolveCaches(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Caches) + now := time.Now() + pendingUntil := now + // pending rows stay live through the workflow deadline + if deadline, ok := ctx.Deadline(); ok { + pendingUntil = deadline + } + for i := range bindings { + id := uuid.NewString() + saveKey, err := models.CacheObjectKey(pipeline.RepoDid.String(), id) + if err != nil { + return nil, err + } + bindings[i].SaveID = id + bindings[i].SaveKey = saveKey + inserted, err := c.index.InsertCacheEntryWithinQuota(ctx, db.CacheEntry{ + ID: id, + StorageKey: bindings[i].SaveKey, + OwnerDID: repo.Owner.String(), + RepoDID: pipeline.RepoDid.String(), + Engine: cacheEngine, + CacheKey: bindings[i].Key, + CacheHash: bindings[i].Hash, + State: "pending", + CreatedAt: now, + LastUsedAt: pendingUntil, + }, c.maxEntriesPerOwner) + if err != nil { + for _, planned := range bindings[:i] { + _ = c.Apply(context.WithoutCancel(ctx), CacheUpdate{ + Action: CacheDiscarded, + ID: planned.SaveID, + Ref: planned.SaveKey, + }) + } + for j := range bindings { + bindings[j].SaveID = "" + bindings[j].SaveKey = "" + } + c.logger.Warn("cache save reservations failed; restoring only", "err", err) + return bindings, nil + } + if !inserted { + bindings[i].SaveID = "" + bindings[i].SaveKey = "" + } + } + return bindings, nil +} + +func (c *localCacheController) Apply(ctx context.Context, update CacheUpdate) error { + now := time.Now() + switch update.Action { + case CacheUsed: + return c.index.TouchCacheEntry(ctx, update.ID, now) + case CacheStored: + superseded, err := c.index.MarkCacheEntryReady(ctx, update.ID, update.SizeBytes, c.maxBytesPerOwner, now) + if err != nil { + _ = c.store.Delete(context.WithoutCancel(ctx), update.Ref) + return err + } + for _, old := range superseded { + if err := c.deleteEntry(context.WithoutCancel(ctx), old.StorageKey, old.ID); err != nil { + c.logger.Warn("delete replaced cache failed", "id", old.ID, "err", err) + } + } + case CacheDiscarded, CacheMissing: + if update.Action == CacheDiscarded { + claimed, err := c.index.ClaimPendingCacheEntry(ctx, update.ID) + if err != nil || !claimed { + return err + } + } + return c.deleteEntry(ctx, update.Ref, update.ID) + default: + return fmt.Errorf("unknown cache update action %d", update.Action) + } + return nil +} + +func (c *localCacheController) deleteEntry(ctx context.Context, ref, id string) error { + if err := c.store.Delete(ctx, ref); err != nil { + return err + } + return c.index.DeleteCacheEntry(ctx, id) +} + +type preplannedCacheController struct { + apply func(context.Context, CacheUpdate) error +} + +func NewPreplannedCacheController(apply func(context.Context, CacheUpdate) error) CacheController { + return &preplannedCacheController{apply: apply} +} + +func (c *preplannedCacheController) Plan(_ context.Context, _ *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) { + return workflow.CacheBindings, nil +} + +func (c *preplannedCacheController) Apply(ctx context.Context, update CacheUpdate) error { + return c.apply(ctx, update) +} + +type trackedCacheStore struct { + storage.Storage + controller CacheController + logger *slog.Logger + mu sync.Mutex + restores map[string]models.CacheBinding + saves map[string]models.CacheBinding +} + +func newTrackedCacheStore(base storage.Storage, controller CacheController, logger *slog.Logger, bindings []models.CacheBinding) *trackedCacheStore { + store := &trackedCacheStore{ + Storage: base, + controller: controller, + logger: logger, + restores: make(map[string]models.CacheBinding, len(bindings)), + saves: make(map[string]models.CacheBinding, len(bindings)), + } + for _, binding := range bindings { + if binding.RestoreKey != "" { + store.restores[binding.RestoreKey] = binding + } + if binding.SaveKey != "" { + store.saves[binding.SaveKey] = binding + } + } + return store +} + +func (s *trackedCacheStore) Get(ctx context.Context, key string) (io.ReadCloser, error) { + s.mu.Lock() + binding, tracked := s.restores[key] + s.mu.Unlock() + if !tracked { + return nil, fmt.Errorf("cache metadata missing for %q", key) + } + + reader, err := s.Storage.Get(ctx, key) + if err != nil { + if errors.Is(err, storage.ErrNotExist) { + _ = s.controller.Apply(context.WithoutCancel(ctx), CacheUpdate{ + Action: CacheMissing, + ID: binding.RestoreID, + Ref: binding.RestoreKey, + }) + } + return nil, err + } + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheUsed, + ID: binding.RestoreID, + Ref: binding.RestoreKey, + }); err != nil { + s.logger.Warn("cache usage update failed", "id", binding.RestoreID, "err", err) + } + return reader, nil +} + +func (s *trackedCacheStore) Put(ctx context.Context, key string, reader io.Reader) error { + s.mu.Lock() + binding, tracked := s.saves[key] + s.mu.Unlock() + if !tracked { + return fmt.Errorf("cache metadata missing for %q", key) + } + + counted := &countingReader{r: reader} + if err := s.Storage.Put(ctx, key, counted); err != nil { + return err + } + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheStored, + ID: binding.SaveID, + Ref: binding.SaveKey, + SizeBytes: counted.n, + }); err != nil { + _ = s.Storage.Delete(context.WithoutCancel(ctx), key) + return fmt.Errorf("record cache upload: %w", err) + } + + s.mu.Lock() + delete(s.saves, key) + s.mu.Unlock() + return nil +} + +func (s *trackedCacheStore) cleanup(ctx context.Context) { + s.mu.Lock() + pending := make([]models.CacheBinding, 0, len(s.saves)) + for _, binding := range s.saves { + pending = append(pending, binding) + } + clear(s.saves) + s.mu.Unlock() + + for _, binding := range pending { + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheDiscarded, + ID: binding.SaveID, + Ref: binding.SaveKey, + }); err != nil { + s.logger.Warn("discard incomplete cache failed", "id", binding.SaveID, "err", err) + } + } +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.r.Read(p) + r.n += int64(n) + return n, err +} diff --git a/spindle/engine/cache_prune.go b/spindle/engine/cache_prune.go new file mode 100644 index 00000000..c8cb56da --- /dev/null +++ b/spindle/engine/cache_prune.go @@ -0,0 +1,98 @@ +package engine + +import ( + "context" + "log/slog" + "time" + + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/storage" +) + +const ( + cachePruneBatch = 100 + cachePendingMaxAge = time.Hour +) + +func StartCachePruner(ctx context.Context, l *slog.Logger, index *db.DB, store storage.Storage, retention, interval time.Duration) { + if store == nil || interval <= 0 { + return + } + go func() { + prune := func() { + total := 0 + for { + n, err := PruneCaches(ctx, index, store, time.Now(), retention, cachePendingMaxAge, cachePruneBatch) + total += n + if err != nil { + l.Warn("cache prune failed", "count", total, "err", err) + return + } + if n < cachePruneBatch { + if total > 0 { + l.Info("pruned cache entries", "count", total) + } + return + } + } + } + prune() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }() +} + +func PruneCaches(ctx context.Context, index *db.DB, store storage.Storage, now time.Time, retention, pendingMaxAge time.Duration, limit int) (int, error) { + deletions, err := index.PendingCacheObjectDeletions(ctx, limit) + if err != nil { + return 0, err + } + pruned := 0 + for _, key := range deletions { + if err := store.Delete(ctx, key); err != nil { + return pruned, err + } + if err := index.CompleteCacheObjectDeletion(ctx, key); err != nil { + return pruned, err + } + pruned++ + } + if pruned == limit { + return pruned, nil + } + + readyBefore := time.Unix(0, 0) + if retention > 0 { + readyBefore = now.Add(-retention) + } + entries, err := index.ExpiredCacheEntries(ctx, readyBefore, now.Add(-pendingMaxAge), limit-pruned) + if err != nil { + return pruned, err + } + for _, entry := range entries { + claimed, err := index.ClaimCacheEntry(ctx, entry.ID, entry.State, entry.LastUsedAt) + if err != nil { + return pruned, err + } + if !claimed { + continue + } + if err := store.Delete(ctx, entry.StorageKey); err != nil { + _ = index.RestoreCacheEntryState(context.WithoutCancel(ctx), entry.ID, entry.State) + return pruned, err + } + if err := index.DeleteCacheEntry(ctx, entry.ID); err != nil { + return pruned, err + } + pruned++ + } + return pruned, nil +} diff --git a/spindle/engine/cache_prune_test.go b/spindle/engine/cache_prune_test.go new file mode 100644 index 00000000..2cd8b95f --- /dev/null +++ b/spindle/engine/cache_prune_test.go @@ -0,0 +1,188 @@ +package engine + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "tangled.org/core/spindle/db" +) + +func TestPruneCachesExpiryPolicies(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + type spec struct { + id string + state string + age time.Duration + } + tests := []struct { + name string + retention time.Duration + pendingMax time.Duration + entries []spec + wantPruned int + wantSurvive []string + }{ + { + name: "ready entries expire by last use", + retention: time.Hour, + pendingMax: 15 * time.Minute, + entries: []spec{{"old", "ready", 2 * time.Hour}, {"fresh", "ready", 30 * time.Minute}}, + wantPruned: 1, + wantSurvive: []string{"fresh"}, + }, + { + name: "zero retention keeps ready entries", + retention: 0, + pendingMax: time.Hour, + entries: []spec{{"ready", "ready", 24 * time.Hour}, {"pending", "pending", 2 * time.Hour}}, + wantPruned: 1, + wantSurvive: []string{"ready"}, + }, + { + name: "pending entries expire by age", + retention: time.Hour, + pendingMax: time.Hour, + entries: []spec{{"old", "pending", 2 * time.Hour}, {"fresh", "pending", 10 * time.Minute}}, + wantPruned: 1, + wantSurvive: []string{"fresh"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := newCacheTestDB(t) + store := &fakeStorage{objects: map[string][]byte{}} + keys := map[string]string{} + for _, e := range tt.entries { + entry := cacheTestEntry(e.id, "did:plc:repo", "microvm", "deps", "hash", e.state, now.Add(-e.age)) + insertCacheTestEntry(t, d, entry) + store.objects[entry.StorageKey] = []byte(e.id) + keys[e.id] = entry.StorageKey + } + + pruned, err := PruneCaches(ctx, d, store, now, tt.retention, tt.pendingMax, 10) + if err != nil { + t.Fatalf("PruneCaches: %v", err) + } + if pruned != tt.wantPruned { + t.Fatalf("pruned %d entries, want %d", pruned, tt.wantPruned) + } + for _, e := range tt.entries { + survives := slices.Contains(tt.wantSurvive, e.id) + if store.has(keys[e.id]) != survives || cacheTestEntryExists(t, d, e.id) != survives { + t.Fatalf("entry %q survived = %v, want %v", e.id, !survives, survives) + } + } + }) + } +} + +func TestPruneCachesSkipsEntryRefreshedAfterScan(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + first := cacheTestEntry("first", "did:plc:repo", "microvm", "deps", "first", "ready", now.Add(-3*time.Hour)) + refreshed := cacheTestEntry("refreshed", "did:plc:repo", "microvm", "deps", "refreshed", "ready", now.Add(-2*time.Hour)) + insertCacheTestEntry(t, d, first) + insertCacheTestEntry(t, d, refreshed) + store := &fakeStorage{objects: map[string][]byte{ + first.StorageKey: []byte("first"), + refreshed.StorageKey: []byte("refreshed"), + }} + store.onDelete = func(key string) { + if key != first.StorageKey { + return + } + if err := d.TouchCacheEntry(ctx, refreshed.ID, now); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + } + + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatalf("PruneCaches: %v", err) + } + if pruned != 1 { + t.Fatalf("pruned %d entries, want 1", pruned) + } + if !store.has(refreshed.StorageKey) || !cacheTestEntryExists(t, d, refreshed.ID) { + t.Fatal("entry refreshed after expiry scan was pruned") + } +} + +func TestPruneCachesDeleteFailureRemainsRetryable(t *testing.T) { + for _, initialState := range []string{"ready", "deleting"} { + t.Run(initialState, func(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + entry := cacheTestEntry("retry", "did:plc:repo", "microvm", "deps", "hash", initialState, now.Add(-2*time.Hour)) + insertCacheTestEntry(t, d, entry) + deleteErr := errors.New("delete failed") + store := &fakeStorage{ + objects: map[string][]byte{entry.StorageKey: []byte("archive")}, + deleteErr: deleteErr, + } + + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if !errors.Is(err, deleteErr) { + t.Fatalf("first PruneCaches error = %v, want %v", err, deleteErr) + } + if pruned != 0 { + t.Fatalf("first prune count = %d, want 0", pruned) + } + state, _, _ := cacheTestEntryState(t, d, entry.ID) + if state != initialState { + t.Fatalf("state after delete failure = %q, want %q", state, initialState) + } + if !store.has(entry.StorageKey) { + t.Fatal("failed delete removed object") + } + + pruned, err = PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatalf("retry PruneCaches: %v", err) + } + if pruned != 1 || store.has(entry.StorageKey) || cacheTestEntryExists(t, d, entry.ID) { + t.Fatalf("retry result = pruned %d, object %v, metadata %v", pruned, store.has(entry.StorageKey), cacheTestEntryExists(t, d, entry.ID)) + } + }) + } +} + +func TestPruneCachesRetriesQueuedObjectDeletion(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Now() + key := "objects/did:web:example.com/orphan" + if err := d.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { + return tx.QueueCacheObjectDeletion(ctx, key, now) + }); err != nil { + t.Fatal(err) + } + deleteErr := errors.New("delete failed") + store := &fakeStorage{ + objects: map[string][]byte{key: []byte("archive")}, + deleteErr: deleteErr, + } + if _, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10); !errors.Is(err, deleteErr) { + t.Fatalf("first prune error = %v, want %v", err, deleteErr) + } + pending, err := d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 || pending[0] != key { + t.Fatalf("pending object deletions = %v", pending) + } + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatal(err) + } + if pruned != 1 || store.has(key) { + t.Fatalf("retry pruned %d, object remains %t", pruned, store.has(key)) + } +} diff --git a/spindle/engine/cache_test.go b/spindle/engine/cache_test.go new file mode 100644 index 00000000..ed0ea456 --- /dev/null +++ b/spindle/engine/cache_test.go @@ -0,0 +1,512 @@ +package engine + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +type fakeStorage struct { + objects map[string][]byte + putErr error + deleteErr error // fails once, then clears + onDelete func(string) +} + +func (f *fakeStorage) Get(_ context.Context, key string) (io.ReadCloser, error) { + data, ok := f.objects[key] + if !ok { + return nil, storage.ErrNotExist + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (f *fakeStorage) Put(_ context.Context, key string, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.objects[key] = data + return f.putErr +} + +func (f *fakeStorage) Delete(_ context.Context, key string) error { + if f.onDelete != nil { + f.onDelete(key) + } + if f.deleteErr != nil { + err := f.deleteErr + f.deleteErr = nil + return err + } + delete(f.objects, key) + return nil +} + +func (f *fakeStorage) has(key string) bool { + _, ok := f.objects[key] + return ok +} + +func gitRepo(t *testing.T, files map[string]string) (string, string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.email=t@t", "-c", "user.name=t", "-c", "init.defaultBranch=main"}, args...)...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + run("init") + for name, content := range files { + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + run("add", ".") + run("commit", "-m", "init") + return dir, "HEAD" +} + +func newCacheTestDB(t *testing.T) *db.DB { + t.Helper() + d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { _ = d.Close() }) + return d +} +func addCacheTestRepo(t *testing.T, d *db.DB) (syntax.DID, syntax.DID) { + t.Helper() + repoDid := syntax.DID("did:plc:repo") + ownerDid := syntax.DID("did:plc:owner") + if err := d.AddRepo(db.Repo{ + Knot: "knot.example", + Owner: ownerDid, + Rkey: syntax.RecordKey("repo"), + RepoDid: repoDid, + }); err != nil { + t.Fatal(err) + } + return repoDid, ownerDid +} + +func cacheTestEntry(id, repoDID, engine, key, hash, state string, at time.Time) db.CacheEntry { + return db.CacheEntry{ + ID: id, + StorageKey: "objects/" + id, + OwnerDID: "did:plc:owner", + RepoDID: repoDID, + Engine: engine, + CacheKey: key, + CacheHash: hash, + State: state, + CreatedAt: at, + LastUsedAt: at, + } +} + +func insertCacheTestEntry(t *testing.T, d *db.DB, entry db.CacheEntry) { + t.Helper() + if err := d.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatalf("InsertCacheEntry(%s): %v", entry.ID, err) + } +} + +func cacheTestEntryState(t *testing.T, d *db.DB, id string) (string, int64, time.Time) { + t.Helper() + var state string + var size, lastUsed int64 + if err := d.QueryRow(`select state, size_bytes, last_used_at from cache_entries where id = ?`, id).Scan(&state, &size, &lastUsed); err != nil { + t.Fatalf("query cache entry %s: %v", id, err) + } + return state, size, time.Unix(0, lastUsed) +} + +func cacheTestEntryExists(t *testing.T, d *db.DB, id string) bool { + t.Helper() + var exists bool + if err := d.QueryRow(`select exists(select 1 from cache_entries where id = ?)`, id).Scan(&exists); err != nil { + t.Fatalf("query cache entry existence %s: %v", id, err) + } + return exists +} + +var discardLogger = slog.New(slog.NewTextHandler(io.Discard, nil)) + +func TestCachePlanPreservesRestoreAtEntryQuota(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoDid, ownerDid := addCacheTestRepo(t, d) + old := cacheTestEntry( + "restore", + repoDid.String(), + "microvm/"+models.CacheNamespace(), + "deps", + "", + "ready", + time.Now(), + ) + old.OwnerDID = ownerDid.String() + insertCacheTestEntry(t, d, old) + + controller := NewLocalCacheController( + d, + &fakeStorage{objects: map[string][]byte{old.StorageKey: []byte("archive")}}, + "", + 0, + 1, + discardLogger, + ) + bindings, err := controller.Plan(ctx, &models.Pipeline{RepoDid: repoDid}, &models.Workflow{ + Engine: "microvm", + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 1 || bindings[0].RestoreID != old.ID || bindings[0].SaveKey != "" { + t.Fatalf("quota binding = %+v, want restore-only %q", bindings, old.ID) + } +} + +func TestCachePlanKeepsPendingEntryThroughWorkflowDeadline(t *testing.T) { + d := newCacheTestDB(t) + repoDid, _ := addCacheTestRepo(t, d) + deadline := time.Now().Add(6 * time.Hour) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + controller := NewLocalCacheController( + d, + &fakeStorage{objects: make(map[string][]byte)}, + "", + 0, + 0, + discardLogger, + ) + bindings, err := controller.Plan(ctx, &models.Pipeline{RepoDid: repoDid}, &models.Workflow{ + Engine: "microvm", + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }) + if err != nil { + t.Fatal(err) + } + var pendingUntil int64 + if err := d.QueryRow( + `select last_used_at from cache_entries where id = ?`, + bindings[0].SaveID, + ).Scan(&pendingUntil); err != nil { + t.Fatal(err) + } + if pendingUntil != deadline.UnixNano() { + t.Fatalf("pending deadline = %d, want %d", pendingUntil, deadline.UnixNano()) + } +} + +func TestResolveCachesExactUnhashed(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) + + insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "deps", "", "ready", base)) + insertCacheTestEntry(t, d, cacheTestEntry("other-repo", "did:plc:other", "microvm", "deps", "", "ready", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("other-engine", "did:plc:repo", "nixery", "deps", "", "ready", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("pending", "did:plc:repo", "microvm", "deps", "", "pending", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("hashed-only", "did:plc:repo", "microvm", "tools", "old", "ready", base)) + + resolved := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", "", "", []models.CacheEntry{ + {Key: "deps", Paths: []string{"/x"}, CompressionLevel: 7, When: "always"}, + {Key: "tools", Paths: []string{"/y"}}, + }) + if len(resolved) != 2 { + t.Fatalf("got %d resolved entries, want 2", len(resolved)) + } + got := resolved[0] + if got.RestoreID != "exact" || got.RestoreKey != "objects/exact" || got.RestoreName != "" { + t.Fatalf("exact restore = (%q, %q, %q)", got.RestoreID, got.RestoreKey, got.RestoreName) + } + if got.SaveKey != "" { + t.Fatalf("resolve allocated a save key %q", got.SaveKey) + } + if got.CompressionLevel != 7 || got.When != "always" { + t.Fatalf("resolved metadata = %+v", got) + } + if resolved[1].RestoreKey != "" { + t.Fatalf("unhashed entry used hashed fallback %q", resolved[1].RestoreKey) + } +} + +func TestResolveCachesHashExactAndFallback(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoPath, rev := gitRepo(t, map[string]string{"go.sum": "v1 contents", "go.mod": "module x"}) + request := []models.CacheEntry{{Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}} + + first := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if first.Hash == "" { + t.Fatal("hash is empty") + } + + base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) + insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "go-mod", first.Hash, "ready", base.Add(time.Minute))) + insertCacheTestEntry(t, d, cacheTestEntry("sibling", "did:plc:repo", "microvm", "go-modules", "newer", "ready", base.Add(time.Hour))) + + exact := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if exact.RestoreID != "exact" || exact.RestoreKey != "objects/exact" || exact.RestoreName != "" { + t.Fatalf("exact generation restore = %+v", exact) + } + + if err := os.WriteFile(filepath.Join(repoPath, "go.sum"), []byte("v2 contents"), 0o644); err != nil { + t.Fatal(err) + } + commit := exec.Command("git", "-C", repoPath, "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "bump") + if out, err := commit.CombinedOutput(); err != nil { + t.Fatalf("commit: %v\n%s", err, out) + } + rotated := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if rotated.Hash == first.Hash { + t.Fatal("lockfile change did not rotate the cache hash") + } + if rotated.RestoreID != "exact" || rotated.RestoreKey != "objects/exact" || rotated.RestoreName != "go-mod-"+first.Hash { + t.Fatalf("fallback restore = %+v", rotated) + } +} + +func TestResolveCachesMissingHashFiles(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoPath, rev := gitRepo(t, map[string]string{"go.mod": "module x"}) + + partial := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}, + })[0] + if len(partial.Hash) != 12 { + t.Fatalf("partial hash = %q, want 12 characters", partial.Hash) + } + + none := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"nope.lock"}, Paths: []string{"/x"}}, + })[0] + if none.Hash != "" || none.RestoreKey != "" { + t.Fatalf("all-missing result = hash %q, restore %q", none.Hash, none.RestoreKey) + } +} + +func TestTrackedCacheStoreSaveLifecycle(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + old := cacheTestEntry("superseded", "did:plc:repo", "microvm", "deps", "hash", "ready", time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) + insertCacheTestEntry(t, d, old) + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "hash", "pending", time.Now()) + insertCacheTestEntry(t, d, pending) + base := &fakeStorage{objects: map[string][]byte{old.StorageKey: []byte("old")}} + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", Hash: "hash", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + payload := []byte("cache archive bytes") + if err := store.Put(ctx, pending.StorageKey, bytes.NewReader(payload)); err != nil { + t.Fatalf("tracked Put: %v", err) + } + state, size, _ := cacheTestEntryState(t, d, pending.ID) + if state != "ready" || size != int64(len(payload)) { + t.Fatalf("saved metadata = state %q, size %d", state, size) + } + if got := base.objects[pending.StorageKey]; !bytes.Equal(got, payload) { + t.Fatalf("stored payload = %q, want %q", got, payload) + } + if base.has(old.StorageKey) || cacheTestEntryExists(t, d, old.ID) { + t.Fatal("completed save left the superseded generation") + } + store.cleanup(ctx) + if !base.has(pending.StorageKey) || !cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("cleanup removed a completed cache") + } +} + +func TestTrackedCacheStoreRejectsOverQuotaSave(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + existing := cacheTestEntry("existing", "did:plc:repo", "microvm", "deps", "old", "ready", time.Now()) + existing.SizeBytes = 4 + insertCacheTestEntry(t, d, existing) + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "new", "pending", time.Now()) + pending.SizeBytes = 0 + insertCacheTestEntry(t, d, pending) + + base := &fakeStorage{objects: map[string][]byte{existing.StorageKey: []byte("kept")}} + controller := NewLocalCacheController(d, base, "", 4, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + if err := store.Put(ctx, pending.StorageKey, strings.NewReader("x")); err == nil { + t.Fatal("over-quota cache save succeeded") + } + store.cleanup(ctx) + if base.has(pending.StorageKey) || cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("over-quota save left an object or metadata") + } + if !base.has(existing.StorageKey) || !cacheTestEntryExists(t, d, existing.ID) { + t.Fatal("over-quota save removed the existing cache") + } +} + +func TestTrackedCacheStoreRestoreTouchesEntry(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + old := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC) + entry := cacheTestEntry("restore", "did:plc:repo", "microvm", "deps", "hash", "ready", old) + insertCacheTestEntry(t, d, entry) + base := &fakeStorage{objects: map[string][]byte{entry.StorageKey: []byte("archive")}} + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + RestoreID: entry.ID, RestoreKey: entry.StorageKey, + }}) + + r, err := store.Get(ctx, entry.StorageKey) + if err != nil { + t.Fatalf("tracked Get: %v", err) + } + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read restored object: %v", err) + } + if err := r.Close(); err != nil { + t.Fatalf("close restored object: %v", err) + } + if string(data) != "archive" { + t.Fatalf("restored payload = %q", data) + } + _, _, touched := cacheTestEntryState(t, d, entry.ID) + if !touched.After(old) { + t.Fatalf("last used = %v, want after %v", touched, old) + } +} + +func TestTrackedCacheStoreFailedPutCleanup(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + putErr := errors.New("put failed") + base := &fakeStorage{objects: make(map[string][]byte), putErr: putErr} + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "", "pending", time.Now()) + insertCacheTestEntry(t, d, pending) + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + if err := store.Put(ctx, pending.StorageKey, strings.NewReader("partial")); !errors.Is(err, putErr) { + t.Fatalf("tracked Put error = %v, want %v", err, putErr) + } + store.cleanup(ctx) + if base.has(pending.StorageKey) { + t.Fatal("cleanup left partial object") + } + if cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("cleanup left pending metadata") + } +} + +func TestTrackedCacheStoreRejectsUnpreparedKeys(t *testing.T) { + store := newTrackedCacheStore( + &fakeStorage{objects: map[string][]byte{"objects/missing": []byte("foreign")}}, + &preplannedCacheController{}, + discardLogger, + nil, + ) + if _, err := store.Get(context.Background(), "objects/missing"); err == nil { + t.Fatal("Get accepted a key without restore metadata") + } + if err := store.Put(context.Background(), "objects/missing", strings.NewReader("data")); err == nil { + t.Fatal("Put accepted a key without pending metadata") + } +} + +func TestCacheSaveScriptQuotesPaths(t *testing.T) { + workspace := t.TempDir() + cachePath := filepath.Join(workspace, "cache;name") + if err := os.Mkdir(cachePath, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cachePath, "value"), []byte("cached"), 0o644); err != nil { + t.Fatal(err) + } + + bin := t.TempDir() + zstd := filepath.Join(bin, "zstd") + if err := os.WriteFile(zstd, []byte("#!/bin/sh\nset --\nexec cat\n"), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", CacheSaveScript([]string{"cache;name"}, workspace, 0)) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + if output, err := cmd.Output(); err != nil || len(output) == 0 { + t.Fatalf("cache save script = %d bytes, %v", len(output), err) + } +} + +func TestCacheDecompressCmd(t *testing.T) { + cases := []struct { + name string + head []byte + want string + }{ + {"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd, 0x00}, "zstd -dc"}, + {"gzip", []byte{0x1f, 0x8b, 0x08, 0x00}, "gzip -dc"}, + {"empty", nil, "zstd -dc"}, + } + for _, tc := range cases { + got := CacheDecompressCmd(bufio.NewReader(bytes.NewReader(tc.head))) + if got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestCacheBindingSaveOn(t *testing.T) { + cases := []struct { + when string + onFail, onPass bool + }{ + {"", false, true}, + {"on-success", false, true}, + {"always", true, true}, + } + for _, tc := range cases { + binding := models.CacheBinding{When: tc.when} + if got := binding.SaveOn(true); got != tc.onFail { + t.Errorf("when=%q failed run: got %v, want %v", tc.when, got, tc.onFail) + } + if got := binding.SaveOn(false); got != tc.onPass { + t.Errorf("when=%q passing run: got %v, want %v", tc.when, got, tc.onPass) + } + } +} + +var _ storage.Storage = (*fakeStorage)(nil) diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index 3fa5a241..ebe9a9e7 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -18,6 +18,7 @@ import ( "tangled.org/core/spindle/db" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" ) var ( @@ -97,18 +98,33 @@ func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Noti } } -func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { +func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, cacheStore storage.Storage, cacheController CacheController, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { l.Info("starting all workflows in parallel", "pipeline", pipelineId) + isTrustedRepo := pipeline.TrustedSource && pipeline.RepoDid != "" var allSecrets []secrets.UnlockedSecret // never pass secrets to pipelines that run untrusted (e.g. fork) code - if pipeline.TrustedSource && pipeline.RepoDid != "" { + if isTrustedRepo { if res, err := vault.GetSecretsUnlocked(ctx, secrets.RepoIdentifier(pipeline.RepoDid.String())); err == nil { allSecrets = res } } else if !pipeline.TrustedSource { l.Info("skipping secrets for untrusted pipeline source", "pipeline", pipelineId) } + if cacheController == nil && cacheStore != nil && db != nil { + cacheController = NewLocalCacheController( + db, + cacheStore, + cfg.Server.RepoDir, + cfg.Cache.MaxBytesPerOwner, + cfg.Cache.MaxEntriesPerOwner, + l, + ) + } + cacheEnabled := cacheStore != nil && cacheController != nil && isTrustedRepo + if cacheStore != nil && !pipeline.TrustedSource { + l.Info("skipping caches for untrusted pipeline source", "pipeline", pipelineId) + } secretValues := make([]string, len(allSecrets)) for i, s := range allSecrets { @@ -130,6 +146,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s var wg sync.WaitGroup for eng, wfs := range pipeline.Workflows { workflowTimeout := eng.WorkflowTimeout() + cacheRunner, cachesSupported := eng.(CacheRunner) l.Info("using workflow timeout", "timeout", workflowTimeout) for _, w := range wfs { @@ -218,6 +235,54 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s } defer eng.DestroyWorkflow(ctx, wid) + var bindings []models.CacheBinding + var trackedStore *trackedCacheStore + if cacheEnabled && len(w.Caches) > 0 { + bindings, err = cacheController.Plan(wfCtx, pipeline, &w) + if err != nil { + l.Warn("cache planning failed", "wid", wid, "err", err) + } else { + w.CacheBindings = bindings + trackedStore = newTrackedCacheStore(cacheStore, cacheController, l, bindings) + if cachesSupported { + wfLogger.ControlWriter(CacheRestoreStepIdx, CacheRestoreStep, models.StepStatusStart).Write([]byte{0}) + // caches are an optimization, never a reason to fail the workflow + if err := cacheRunner.RestoreCache(wfCtx, wid, &w, trackedStore, bindings, wfLogger); err != nil { + l.Warn("cache restore failed", "wid", wid, "err", err) + } + wfLogger.ControlWriter(CacheRestoreStepIdx, CacheRestoreStep, models.StepStatusEnd).Write([]byte{0}) + } else if !remoteStatus { + l.Warn("engine does not support caches, skipping restore", "wid", wid) + } + } + } + + cleanupCaches := func() { + if trackedStore != nil { + trackedStore.cleanup(context.WithoutCancel(wfCtx)) + trackedStore = nil + } + } + // dont save on timeouts, their context is already dead + saveCaches := func(failed bool) { + if trackedStore == nil || !cachesSupported { + return + } + toSave := make([]models.CacheBinding, 0, len(bindings)) + for _, binding := range bindings { + if binding.SaveKey != "" && binding.SaveOn(failed) { + toSave = append(toSave, binding) + } + } + if len(toSave) == 0 { + return + } + wfLogger.ControlWriter(CacheSaveStepIdx, CacheSaveStep, models.StepStatusStart).Write([]byte{0}) + if err := cacheRunner.SaveCache(wfCtx, wid, &w, trackedStore, toSave, wfLogger); err != nil { + l.Warn("cache save failed", "wid", wid, "err", err) + } + wfLogger.ControlWriter(CacheSaveStepIdx, CacheSaveStep, models.StepStatusEnd).Write([]byte{0}) + } for stepIdx, step := range w.Steps { if wfLogger != nil { wfLogger. @@ -234,6 +299,10 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s } if err != nil { + if !errors.Is(err, ErrTimedOut) && !errors.Is(wfCtx.Err(), context.DeadlineExceeded) && !isCanceled(wfCtx) { + saveCaches(true) + } + cleanupCaches() if !remoteStatus { writeWfError(db, n, l, wfCtx, wid, "running step", err) } @@ -241,6 +310,9 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, s } } + saveCaches(false) + cleanupCaches() + if isCanceled(wfCtx) { if !remoteStatus { writeWfError(db, n, l, wfCtx, wid, "before success", nil) diff --git a/spindle/engine/engine_test.go b/spindle/engine/engine_test.go index 90155334..bf1856cf 100644 --- a/spindle/engine/engine_test.go +++ b/spindle/engine/engine_test.go @@ -5,6 +5,7 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "sync" "testing" "time" @@ -25,13 +26,44 @@ func (m mockStep) Name() string { return m.name } func (m mockStep) Command() string { return m.command } func (m mockStep) Kind() models.StepKind { return models.StepKindUser } +type emptyVault struct{} + +func (emptyVault) AddSecret(context.Context, secrets.UnlockedSecret) error { + return nil +} + +func (emptyVault) RemoveSecret(context.Context, secrets.Secret[any]) error { + return nil +} + +func (emptyVault) GetSecretsLocked(context.Context, secrets.RepoIdentifier) ([]secrets.LockedSecret, error) { + return nil, nil +} + +func (emptyVault) GetSecretsUnlocked(context.Context, secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) { + return nil, nil +} + +type staticCacheController struct { + bindings []models.CacheBinding +} + +func (c staticCacheController) Plan(context.Context, *models.Pipeline, *models.Workflow) ([]models.CacheBinding, error) { + return c.bindings, nil +} + +func (staticCacheController) Apply(context.Context, CacheUpdate) error { + return nil +} + type mockEngine struct { - mu sync.Mutex - setupCalls []models.WorkflowId - runStepCalls []models.WorkflowId - setupFunc func(ctx context.Context, wid models.WorkflowId) error - runStepFunc func(ctx context.Context, wid models.WorkflowId, idx int) error - timeout time.Duration + mu sync.Mutex + setupCalls []models.WorkflowId + runStepCalls []models.WorkflowId + setupFunc func(ctx context.Context, wid models.WorkflowId) error + runStepFunc func(ctx context.Context, wid models.WorkflowId, idx int) error + runWorkflowFunc func(*models.Workflow) + timeout time.Duration } func (m *mockEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { @@ -63,11 +95,15 @@ func (m *mockEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) func (m *mockEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { m.mu.Lock() m.runStepCalls = append(m.runStepCalls, wid) - fn := m.runStepFunc + runStepFn := m.runStepFunc + runWorkflowFn := m.runWorkflowFunc m.mu.Unlock() - if fn != nil { - return fn(ctx, wid, idx) + if runWorkflowFn != nil { + runWorkflowFn(w) + } + if runStepFn != nil { + return runStepFn(ctx, wid, idx) } return nil } @@ -114,7 +150,7 @@ func TestStartWorkflows_CollisionRejection(t *testing.T) { } cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} - StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) eng.mu.Lock() setupCalls := append([]models.WorkflowId(nil), eng.setupCalls...) @@ -195,7 +231,7 @@ func TestCancelWorkflow_NotOverwritten(t *testing.T) { cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} doneChan := make(chan struct{}) go func() { - StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) close(doneChan) }() @@ -250,7 +286,7 @@ func TestSetupTimeout_ReportsTimeout(t *testing.T) { } cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} - StartWorkflows(logger, nil, cfg, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) st, err := testDB.GetStatus(wid) if err != nil { @@ -264,3 +300,56 @@ func TestSetupTimeout_ReportsTimeout(t *testing.T) { t.Fatalf("expected no steps to run after setup timeout, got %d", len(eng.runStepCalls)) } } + +func TestStartWorkflowsPassesPlannedCacheBindingsToEngine(t *testing.T) { + t.Parallel() + + database := newTestDB(t) + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) + expected := []models.CacheBinding{{ + EntryIndex: 0, + Paths: []string{"deps"}, + Key: "deps", + SaveID: "cache-id", + SaveKey: "objects/cache-id", + }} + var got []models.CacheBinding + eng := &mockEngine{ + runWorkflowFunc: func(workflow *models.Workflow) { + got = append([]models.CacheBinding(nil), workflow.CacheBindings...) + }, + } + pipelineID := models.PipelineId{Knot: "test-knot", Rkey: "test-rkey"} + pipeline := &models.Pipeline{ + RepoDid: "did:web:example.com", + TrustedSource: true, + Workflows: map[models.Engine][]models.Workflow{ + eng: {{ + Name: "cache-job", + Engine: "microvm", + Steps: []models.Step{mockStep{name: "step1"}}, + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }}, + }, + } + cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} + store := &fakeStorage{objects: make(map[string][]byte)} + + StartWorkflows( + logger, + emptyVault{}, + cfg, + nil, + database, + nil, + store, + staticCacheController{bindings: expected}, + context.Background(), + pipeline, + pipelineID, + ) + + if !reflect.DeepEqual(got, expected) { + t.Fatalf("cache bindings passed to engine = %#v, want %#v", got, expected) + } +} diff --git a/spindle/engines/microvm/cache.go b/spindle/engines/microvm/cache.go new file mode 100644 index 00000000..175bc4f6 --- /dev/null +++ b/spindle/engines/microvm/cache.go @@ -0,0 +1,127 @@ +//go:build linux + +package microvm + +import ( + "bufio" + "context" + "fmt" + "io" + + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +func (e *Engine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + out := wfLogger.DataWriter(engine.CacheRestoreStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + if entry.RestoreKey == "" { + fmt.Fprintf(out, "cache %q: miss\n", entry.Key) + continue + } + if entry.RestoreName != "" { + fmt.Fprintf(out, "cache %q: restoring from %q\n", entry.Key, entry.RestoreName) + } + + rc, err := store.Get(ctx, entry.RestoreKey) + if err != nil { + fmt.Fprintf(out, "cache %q: fetch failed: %v\n", entry.Key, err) + continue + } + br := bufio.NewReader(rc) + decompress := engine.CacheDecompressCmd(br) + + var restored int64 + exit, err := state.Agent.Exec(ctx, AgentExec{ + ID: fmt.Sprintf("%s-cache-restore", wid.String()), + ExecStart: cacheExecStart(state, fmt.Sprintf("set -o pipefail\n%s | tar -x -C /", decompress)), + Stdin: &countingReader{r: br, n: &restored}, + Stderr: out, + }) + rc.Close() + if err != nil { + fmt.Fprintf(out, "cache %q: restore failed: %v\n", entry.Key, err) + continue + } + if exit != 0 { + fmt.Fprintf(out, "cache %q: restore failed: guest exited %d\n", entry.Key, exit) + continue + } + fmt.Fprintf(out, "cache %q: restored %d bytes\n", entry.Key, restored) + } + return nil +} + +func (e *Engine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + out := wfLogger.DataWriter(engine.CacheSaveStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + + script := engine.CacheSaveScript(entry.Paths, guestWorkDir, entry.CompressionLevel) + + up := engine.NewCacheUpload(ctx, store, entry.SaveKey) + exit, execErr := state.Agent.Exec(ctx, AgentExec{ + ID: fmt.Sprintf("%s-cache-save", wid.String()), + ExecStart: cacheExecStart(state, script), + Stdout: up.Writer, + Stderr: out, + }) + switch { + case exit == engine.CacheExitNoPaths: + up.Abort(fmt.Errorf("guest exited %d", exit)) + fmt.Fprintf(out, "cache %q: nothing to save\n", entry.Key) + continue + case exit == engine.CacheExitNoCompressor: + up.Abort(fmt.Errorf("guest exited %d", exit)) + fmt.Fprintf(out, "cache %q: zstd not available in image; skipping\n", entry.Key) + continue + case execErr != nil: + up.Abort(execErr) + return fmt.Errorf("save cache %q: %w", entry.Key, execErr) + case exit != 0: + up.Abort(fmt.Errorf("guest exited %d", exit)) + return fmt.Errorf("save cache %q: save script exited %d", entry.Key, exit) + } + if err := up.Finish(); err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + fmt.Fprintf(out, "cache %q: saved\n", entry.Key) + } + return nil +} + +func cacheExecStart(state *workflowState, script string) *agentv1.ExecStart { + return &agentv1.ExecStart{ + Argv: []string{state.ImageSpec.Shell, "-c", script}, + Env: guestBaseEnv(), + User: guestWorkflowUser, + } +} + +type countingReader struct { + r io.Reader + n *int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + *c.n += int64(n) + return n, err +} diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go index a39a3ea3..e690c0ae 100644 --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -45,6 +45,16 @@ const ( type cleanupFunc func(context.Context) error +// return a fresh slice since callers append to it +func guestBaseEnv() []string { + return []string{ + "HOME=/workspace", + "LOGNAME=" + guestWorkflowUser, + "PATH=" + guestBasePATH, + "USER=" + guestWorkflowUser, + } +} + type Engine struct { l *slog.Logger cfg *config.Config @@ -142,6 +152,12 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin swf.Name = twf.Name swf.Environment = dwf.Environment + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + swf.Caches = caches + if tpl.TriggerMetadata != nil { if clone := models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev); clone.Command() != "" { swf.Steps = append([]models.Step{clone}, swf.Steps...) @@ -374,12 +390,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W err := e.activateConfig(execCtx, wid, state, s, wfLogger.DataWriter(idx, "stdout")) return e.classifyStepError(ctx, wid, step, state, stderr, vmExited, "Failed to activate config", err) } - env := []string{ - "HOME=/workspace", - "LOGNAME=" + guestWorkflowUser, - "PATH=" + guestBasePATH, - "USER=" + guestWorkflowUser, - } + env := guestBaseEnv() for k, v := range w.Environment { env = append(env, k+"="+v) } diff --git a/spindle/engines/microvm/models.go b/spindle/engines/microvm/models.go index df4d20c8..7fdf9ccf 100644 --- a/spindle/engines/microvm/models.go +++ b/spindle/engines/microvm/models.go @@ -5,16 +5,19 @@ package microvm import ( "fmt" "slices" + + "tangled.org/core/spindle/models" ) type manifestWorkflow struct { - Image string `yaml:"image"` - Services map[string]any `yaml:"services"` - Virtualisation map[string]any `yaml:"virtualisation"` - Dependencies []string `yaml:"dependencies"` - Registry map[string]any `yaml:"registry"` - Environment map[string]string `yaml:"environment"` - Substituters map[string]string `yaml:"substituters"` + Image string `yaml:"image"` + Services map[string]any `yaml:"services"` + Virtualisation map[string]any `yaml:"virtualisation"` + Dependencies []string `yaml:"dependencies"` + Registry map[string]any `yaml:"registry"` + Environment map[string]string `yaml:"environment"` + Substituters map[string]string `yaml:"substituters"` + Cache []models.CacheEntry `yaml:"cache"` Steps []struct { Name string `yaml:"name"` Command string `yaml:"command"` diff --git a/spindle/engines/microvm/substituter_proxy.go b/spindle/engines/microvm/substituter_proxy.go index df217d02..44dec5d0 100644 --- a/spindle/engines/microvm/substituter_proxy.go +++ b/spindle/engines/microvm/substituter_proxy.go @@ -245,7 +245,6 @@ var guardedProxyTransport = &http.Transport{ ExpectContinueTimeout: 1 * time.Second, } - // the proxy is the cache as far as the guest is concerned, so we answer // /nix-cache-info ourselves instead of racing the upstreams for it. merging // those also doesn't make any sense (none of the options make sense for diff --git a/spindle/engines/nixery/cache.go b/spindle/engines/nixery/cache.go new file mode 100644 index 00000000..1422593d --- /dev/null +++ b/spindle/engines/nixery/cache.go @@ -0,0 +1,167 @@ +package nixery + +import ( + "bufio" + "context" + "fmt" + "io" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +func (e *Engine) baseEnv() EnvVars { + envs := EnvVars{} + envs.AddEnv("HOME", homeDir) + envs.AddEnv("PATH", fmt.Sprintf("%s/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", homeDir)) + return envs +} + +func (e *Engine) execAttached(ctx context.Context, containerID string, opts container.ExecOptions) (string, types.HijackedResponse, error) { + execResp, err := e.docker.ContainerExecCreate(ctx, containerID, opts) + if err != nil { + return "", types.HijackedResponse{}, fmt.Errorf("create exec: %w", err) + } + attach, err := e.docker.ContainerExecAttach(ctx, execResp.ID, container.ExecAttachOptions{}) + if err != nil { + return "", types.HijackedResponse{}, fmt.Errorf("attach exec: %w", err) + } + return execResp.ID, attach, nil +} + +func (e *Engine) containerID(wf *models.Workflow) (string, error) { + addl, ok := wf.Data.(addlFields) + if !ok || addl.container == "" { + return "", fmt.Errorf("nixery workflow has no container") + } + return addl.container, nil +} + +func (e *Engine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + containerID, err := e.containerID(wf) + if err != nil { + return err + } + + out := wfLogger.DataWriter(engine.CacheRestoreStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + if entry.RestoreKey == "" { + fmt.Fprintf(out, "cache %q: miss\n", entry.Key) + continue + } + if entry.RestoreName != "" { + fmt.Fprintf(out, "cache %q: restoring from %q\n", entry.Key, entry.RestoreName) + } + + rc, err := store.Get(ctx, entry.RestoreKey) + if err != nil { + fmt.Fprintf(out, "cache %q: fetch failed: %v\n", entry.Key, err) + continue + } + + br := bufio.NewReader(rc) + execID, attach, err := e.execAttached(ctx, containerID, container.ExecOptions{ + Cmd: []string{"bash", "-c", engine.CacheDecompressCmd(br) + " | tar -x -C /"}, + Env: e.baseEnv(), + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + }) + if err != nil { + rc.Close() + return fmt.Errorf("restore cache %q: %w", entry.Key, err) + } + + // drain this now or tar can block on stderr before reading stdin + copyDone := make(chan error, 1) + go func() { + _, err := io.Copy(attach.Conn, br) + _ = attach.CloseWrite() + copyDone <- err + }() + _, _ = stdcopy.StdCopy(out, out, attach.Reader) + copyErr := <-copyDone + rc.Close() + attach.Close() + if copyErr != nil { + return fmt.Errorf("restore cache %q: stream archive: %w", entry.Key, copyErr) + } + + inspect, err := e.docker.ContainerExecInspect(ctx, execID) + if err != nil { + return fmt.Errorf("restore cache %q: %w", entry.Key, err) + } + if inspect.ExitCode != 0 { + fmt.Fprintf(out, "cache %q: extract failed (exit %d)\n", entry.Key, inspect.ExitCode) + continue + } + fmt.Fprintf(out, "cache %q: restored\n", entry.Key) + } + return nil +} + +func (e *Engine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + containerID, err := e.containerID(wf) + if err != nil { + return err + } + + out := wfLogger.DataWriter(engine.CacheSaveStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + + script := engine.CacheSaveScript(entry.Paths, workspaceDir, entry.CompressionLevel) + + execID, attach, err := e.execAttached(ctx, containerID, container.ExecOptions{ + Cmd: []string{"bash", "-c", script}, + Env: e.baseEnv(), + AttachStdout: true, + AttachStderr: true, + }) + if err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + + up := engine.NewCacheUpload(ctx, store, entry.SaveKey) + + // StdCopy only returns once the archive is fully written + _, copyErr := stdcopy.StdCopy(up.Writer, out, attach.Reader) + attach.Close() + inspect, inspectErr := e.docker.ContainerExecInspect(ctx, execID) + + switch { + case inspectErr == nil && inspect.ExitCode == engine.CacheExitNoPaths: + up.Abort(fmt.Errorf("no cache paths")) + fmt.Fprintf(out, "cache %q: nothing to save\n", entry.Key) + continue + case inspectErr == nil && inspect.ExitCode == engine.CacheExitNoCompressor: + up.Abort(fmt.Errorf("zstd not available")) + fmt.Fprintf(out, "cache %q: zstd not available in image; skipping\n", entry.Key) + continue + case copyErr != nil: + up.Abort(copyErr) + return fmt.Errorf("save cache %q: stream archive: %w", entry.Key, copyErr) + case inspectErr != nil: + up.Abort(inspectErr) + return fmt.Errorf("save cache %q: %w", entry.Key, inspectErr) + case inspect.ExitCode != 0: + up.Abort(fmt.Errorf("exited %d", inspect.ExitCode)) + return fmt.Errorf("save cache %q: tar exited %d", entry.Key, inspect.ExitCode) + } + if err := up.Finish(); err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + fmt.Fprintf(out, "cache %q: saved\n", entry.Key) + } + return nil +} diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index fb5232c3..0648d054 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -91,6 +91,7 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin } `yaml:"steps"` Dependencies map[string][]string `yaml:"dependencies"` Environment map[string]string `yaml:"environment"` + Cache []models.CacheEntry `yaml:"cache"` }{} if err := engine.DescribeManifestError(twf.Raw, dwf); err != nil { return nil, err @@ -109,6 +110,11 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin } swf.Name = twf.Name swf.Environment = dwf.Environment + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + swf.Caches = caches addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery) if sock := e.cfg.Server.DockerSocket; sock != "" { @@ -155,7 +161,7 @@ func workflowImage(deps map[string][]string, nixery string) string { } // load defaults from somewhere else - dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") + dependencies = path.Join(dependencies, "bash", "git", "coreutils", "gnutar", "zstd", "nix") if runtime.GOARCH == "arm64" { dependencies = path.Join("arm64", dependencies) @@ -410,9 +416,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W } } - envs.AddEnv("HOME", homeDir) - existingPath := "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - envs.AddEnv("PATH", fmt.Sprintf("%s/.nix-profile/bin:/nix/var/nix/profiles/default/bin:%s", homeDir, existingPath)) + envs = append(envs, e.baseEnv()...) if sock := e.cfg.Server.DockerSocket; sock != "" { envs.AddEnv("DOCKER_HOST", fmt.Sprintf("unix://%s", sock)) } diff --git a/spindle/mill/auth_test.go b/spindle/mill/auth_test.go index 6876f791..b6b87dea 100644 --- a/spindle/mill/auth_test.go +++ b/spindle/mill/auth_test.go @@ -358,11 +358,11 @@ func TestAuthLabelEscalation(t *testing.T) { enc := millproto.NewEncoder(stream) dec := millproto.NewDecoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux", "gpu"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux", "gpu"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc.Encode(hello); err != nil { t.Fatalf("encode hello: %v", err) @@ -387,11 +387,11 @@ func TestAuthLabelEscalation(t *testing.T) { enc := millproto.NewEncoder(stream) dec := millproto.NewDecoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc.Encode(hello); err != nil { t.Fatalf("encode hello: %v", err) @@ -429,11 +429,11 @@ func TestHandshakeTimeoutAndConcurrency(t *testing.T) { stream := millproto.NewWSStream(conn) enc := millproto.NewEncoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} err = enc.Encode(hello) dec := millproto.NewDecoder(stream) @@ -457,11 +457,11 @@ func TestHandshakeTimeoutAndConcurrency(t *testing.T) { stream1 := millproto.NewWSStream(conn1) enc1 := millproto.NewEncoder(stream1) dec1 := millproto.NewDecoder(stream1) - hello1 := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello1 := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc1.Encode(hello1); err != nil { t.Fatalf("encode hello 1: %v", err) diff --git a/spindle/mill/cache.go b/spindle/mill/cache.go new file mode 100644 index 00000000..f03e09c3 --- /dev/null +++ b/spindle/mill/cache.go @@ -0,0 +1,49 @@ +package mill + +import ( + "fmt" + "tangled.org/core/spindle/db" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func cacheBindingsToProto(bindings []models.CacheBinding) []*millv1.CacheBinding { + result := make([]*millv1.CacheBinding, len(bindings)) + for i, binding := range bindings { + result[i] = &millv1.CacheBinding{ + EntryIndex: uint32(binding.EntryIndex), + RestoreId: binding.RestoreID, + RestoreKey: binding.RestoreKey, + RestoreName: binding.RestoreName, + SaveId: binding.SaveID, + SaveKey: binding.SaveKey, + } + } + return result +} + +func cacheCapabilities(repoDid string, bindings []models.CacheBinding) ([]db.MillCacheCapability, error) { + capabilities := make([]db.MillCacheCapability, 0, len(bindings)*2) + for _, binding := range bindings { + for _, capability := range []db.MillCacheCapability{ + {Action: "restore", CacheID: binding.RestoreID, StorageKey: binding.RestoreKey}, + {Action: "save", CacheID: binding.SaveID, StorageKey: binding.SaveKey}, + } { + if capability.CacheID == "" && capability.StorageKey == "" { + continue + } + if capability.CacheID == "" || capability.StorageKey == "" { + return nil, fmt.Errorf("%s cache id and key must both be set", capability.Action) + } + want, err := models.CacheObjectKey(repoDid, capability.CacheID) + if err != nil { + return nil, err + } + if capability.StorageKey != want { + return nil, fmt.Errorf("%s cache key %q does not match object id", capability.Action, capability.StorageKey) + } + capabilities = append(capabilities, capability) + } + } + return capabilities, nil +} diff --git a/spindle/mill/engine.go b/spindle/mill/engine.go index 8d269397..e91305f5 100644 --- a/spindle/mill/engine.go +++ b/spindle/mill/engine.go @@ -36,10 +36,25 @@ func NewEngine(name string, mill *Mill) *Engine { // and marks pending normally. the real InitWorkflow runs exactly once, on // the executor inside ReserveSeat, and commit reuses that workflow func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + if e.mill.cfg.CacheStoreID == "" { + caches = nil + } + repoDid := "" + if metadata := tpl.TriggerMetadata; metadata != nil && metadata.Repo != nil && metadata.Repo.RepoDid != nil { + repoDid = *metadata.Repo.RepoDid + } + return &models.Workflow{ Name: twf.Name, Environment: map[string]string{}, Steps: []models.Step{remoteStep{}}, + Caches: caches, + Engine: e.name, + RepoDid: repoDid, Data: &millWorkflowState{ RawWorkflow: twf, RawPipeline: tpl, diff --git a/spindle/mill/executor/cache.go b/spindle/mill/executor/cache.go new file mode 100644 index 00000000..f484af3d --- /dev/null +++ b/spindle/mill/executor/cache.go @@ -0,0 +1,61 @@ +package executor + +import ( + "fmt" + + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func cacheBindingsFromProto(repoDid string, entries []models.CacheEntry, bindings []*millv1.CacheBinding) ([]models.CacheBinding, error) { + result := make([]models.CacheBinding, 0, len(bindings)) + seen := make(map[int]struct{}, len(bindings)) + for _, binding := range bindings { + entryIndex := int(binding.GetEntryIndex()) + if entryIndex < 0 || entryIndex >= len(entries) { + return nil, fmt.Errorf("cache binding index %d is out of range", entryIndex) + } + if _, ok := seen[entryIndex]; ok { + return nil, fmt.Errorf("duplicate cache binding index %d", entryIndex) + } + seen[entryIndex] = struct{}{} + + for _, object := range []struct { + name string + id string + key string + }{ + {name: "restore", id: binding.GetRestoreId(), key: binding.GetRestoreKey()}, + {name: "save", id: binding.GetSaveId(), key: binding.GetSaveKey()}, + } { + if (object.id == "") != (object.key == "") { + return nil, fmt.Errorf("%s cache id and key must both be set", object.name) + } + if object.id == "" { + continue + } + want, err := models.CacheObjectKey(repoDid, object.id) + if err != nil { + return nil, err + } + if object.key != want { + return nil, fmt.Errorf("%s cache key %q does not match object id", object.name, object.key) + } + } + + entry := entries[entryIndex] + result = append(result, models.CacheBinding{ + EntryIndex: entryIndex, + Paths: entry.Paths, + Key: entry.Key, + SaveID: binding.GetSaveId(), + SaveKey: binding.GetSaveKey(), + RestoreID: binding.GetRestoreId(), + RestoreKey: binding.GetRestoreKey(), + RestoreName: binding.GetRestoreName(), + CompressionLevel: entry.CompressionLevel, + When: entry.When, + }) + } + return result, nil +} diff --git a/spindle/mill/executor/cache_test.go b/spindle/mill/executor/cache_test.go new file mode 100644 index 00000000..a7547693 --- /dev/null +++ b/spindle/mill/executor/cache_test.go @@ -0,0 +1,66 @@ +package executor + +import ( + "testing" + + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func TestCacheBindingsFromProtoUsesValidatedWorkflowEntries(t *testing.T) { + const ( + repoDid = "did:web:example.com" + restoreID = "11111111-1111-4111-8111-111111111111" + saveID = "22222222-2222-4222-8222-222222222222" + ) + entries := []models.CacheEntry{ + {Key: "deps", Paths: []string{"node_modules"}, CompressionLevel: 7, When: "always"}, + } + bindings, err := cacheBindingsFromProto(repoDid, entries, []*millv1.CacheBinding{{ + EntryIndex: 0, + RestoreId: restoreID, + RestoreKey: "objects/" + repoDid + "/" + restoreID, + SaveId: saveID, + SaveKey: "objects/" + repoDid + "/" + saveID, + }}) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 1 { + t.Fatalf("bindings = %+v", bindings) + } + got := bindings[0] + if got.Key != "deps" || got.RestoreID != restoreID || got.SaveID != saveID || got.CompressionLevel != 7 || got.When != "always" { + t.Fatalf("binding = %+v", got) + } +} + +func TestCacheBindingsFromProtoRejectsInvalidIndexes(t *testing.T) { + entries := []models.CacheEntry{{Key: "deps", Paths: []string{"node_modules"}}} + for _, bindings := range [][]*millv1.CacheBinding{ + {{EntryIndex: 1}}, + {{EntryIndex: 0}, {EntryIndex: 0}}, + } { + if _, err := cacheBindingsFromProto("did:web:example.com", entries, bindings); err == nil { + t.Fatalf("cacheBindingsFromProto(%+v) succeeded", bindings) + } + } +} + +func TestCacheBindingsFromProtoRejectsForeignAndUnpairedObjects(t *testing.T) { + const id = "11111111-1111-4111-8111-111111111111" + entries := []models.CacheEntry{{Key: "deps", Paths: []string{"node_modules"}}} + for _, binding := range []*millv1.CacheBinding{ + {EntryIndex: 0, SaveId: id}, + {EntryIndex: 0, SaveKey: "objects/did:web:example.com/" + id}, + {EntryIndex: 0, RestoreId: id, RestoreKey: "objects/did:web:other.example/" + id}, + } { + if _, err := cacheBindingsFromProto( + "did:web:example.com", + entries, + []*millv1.CacheBinding{binding}, + ); err == nil { + t.Fatalf("foreign cache binding %+v succeeded", binding) + } + } +} diff --git a/spindle/mill/executor/capability_test.go b/spindle/mill/executor/capability_test.go index a9b4cde7..018051a2 100644 --- a/spindle/mill/executor/capability_test.go +++ b/spindle/mill/executor/capability_test.go @@ -95,6 +95,7 @@ func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { RawPipelineJson: string(tpl), Knot: "k", Rkey: "r", + RepoDid: "did:web:example.com", }) result := (<-enc.messages).GetReserveResult() @@ -117,3 +118,51 @@ func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { t.Fatalf("active reservations = %d, want 0", len(e.active)) } } + +func TestHandleReserveKeepsTriggerMetadataForExecution(t *testing.T) { + e := testExecutor(t) + e.enc = newCaptureEncoder() + e.seats = 1 + e.engines = map[string]models.Engine{"microvm": &fakeEngine{}} + + repoDID := "did:web:example.com" + metadata := &tangled.Pipeline_TriggerMetadata{ + Kind: "push", + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "0123456789abcdef", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDID}, + } + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) + if err != nil { + t.Fatal(err) + } + tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: metadata}) + if err != nil { + t.Fatal(err) + } + + e.handleReserve(context.Background(), &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: string(twf), + RawPipelineJson: string(tpl), + Knot: "k", + Rkey: "r", + RepoDid: repoDID, + }) + + result := (<-e.enc.(*captureEncoder).messages).GetReserveResult() + if result == nil || !result.GetAccepted() { + t.Fatalf("ReserveResult = %+v, want accepted", result) + } + res := e.active["lease-1"] + if res == nil || res.pipeline.TriggerMetadata == nil || res.pipeline.TriggerMetadata.Push == nil { + t.Fatal("reservation dropped trigger metadata") + } + if got := res.pipeline.TriggerMetadata.Push.NewSha; got != metadata.Push.NewSha { + t.Fatalf("trigger commit = %q, want %q", got, metadata.Push.NewSha) + } + res.ttlTimer.Stop() +} diff --git a/spindle/mill/executor/executor.go b/spindle/mill/executor/executor.go index c198fd7f..41c5606a 100644 --- a/spindle/mill/executor/executor.go +++ b/spindle/mill/executor/executor.go @@ -25,6 +25,7 @@ import ( millproto "tangled.org/core/spindle/mill/proto" millv1 "tangled.org/core/spindle/mill/proto/gen" "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" ) const ( @@ -47,6 +48,7 @@ type Executor struct { cfg *config.Config l *slog.Logger writer artifactstore.Writer + cache storage.Storage epoch string outboxBytes int64 @@ -79,7 +81,7 @@ type reservation struct { realEngine models.Engine slot engine.WorkflowSlot wf *models.Workflow - repoDid syntax.DID + pipeline *models.Pipeline vault *memVault committed bool @@ -93,7 +95,7 @@ type messageEncoder interface { Encode(*millproto.Message) error } -func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { +func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writer artifactstore.Writer, cache storage.Storage) (*Executor, error) { seats := defaultSeats millURL := "" token := "" @@ -111,10 +113,6 @@ func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *noti if d == nil || n == nil { return nil, fmt.Errorf("executor requires a database and notifier") } - var writer artifactstore.Writer - if len(writers) > 0 { - writer = writers[0] - } e := &Executor{ millURL: millURL, token: token, @@ -127,6 +125,7 @@ func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *noti cfg: cfg, l: l.With("component", "mill.executor"), writer: writer, + cache: cache, active: make(map[string]*reservation), maxOutboxBytes: 10 * 1024 * 1024, } @@ -207,6 +206,8 @@ func (e *Executor) runSession(ctx context.Context) error { Arch: runtime.GOARCH, Labels: e.labels, Epoch: e.epoch, + CacheStoreId: e.cfg.Cache.StoreID, + CacheNamespace: models.CacheNamespace(), }} if err := enc.Encode(hello); err != nil { return fmt.Errorf("send hello: %w", err) @@ -356,6 +357,13 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { return } + repoDid, err := syntax.ParseDID(rs.GetRepoDid()) + if err != nil { + reject("bad repository did", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + trustedSource := models.TrustedPipelineSource(tpl.TriggerMetadata, repoDid.String()) + pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()} wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name} @@ -364,6 +372,8 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { reject("init workflow: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) return } + wf.Engine = rs.GetTargetEngine() + wf.RepoDid = rs.GetRepoDid() if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok { if err := validator.ValidateWorkflowPlacement(wf); err != nil { reject("validate workflow placement: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) @@ -385,18 +395,17 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { return } - var repoDid syntax.DID - if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil { - repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid) - } - res := &reservation{ leaseID: rs.GetLeaseId(), wid: wid, realEngine: realEngine, slot: slot, wf: wf, - repoDid: repoDid, + pipeline: &models.Pipeline{ + RepoDid: repoDid, + TrustedSource: trustedSource, + TriggerMetadata: tpl.TriggerMetadata, + }, } e.snapshotMu.Lock() @@ -426,29 +435,40 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { e.sendCommitted(cl.GetLeaseId()) return } + bindings, err := cacheBindingsFromProto(res.pipeline.RepoDid.String(), res.wf.Caches, cl.GetCacheBindings()) + if err != nil { + e.mu.Unlock() + e.sendReject(cl.GetLeaseId(), "invalid cache plan: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + res.wf.CacheBindings = bindings res.committed = true if res.ttlTimer != nil { res.ttlTimer.Stop() } + committedSecrets := cl.GetSecrets() + if !res.pipeline.TrustedSource { + committedSecrets = nil + } + res.vault = newMemVault(committedSecrets) jobCtx, cancel := context.WithCancel(e.lifecycleCtx) res.cancel = cancel e.mu.Unlock() - vault := newMemVault(cl.GetSecrets()) + vault := res.vault re := newReservedEngine(res.realEngine, res.slot) - pipeline := &models.Pipeline{ - RepoDid: res.repoDid, - Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}}, - TrustedSource: true, - } + res.pipeline.Workflows = map[models.Engine][]models.Workflow{re: {*res.wf}} + cacheController := engine.NewPreplannedCacheController(func(ctx context.Context, update engine.CacheUpdate) error { + return e.appendCacheUpdate(res.leaseID, update) + }) e.startTail(res) e.jobsWG.Add(1) go func() { defer e.jobsWG.Done() - engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) + engine.StartWorkflows(e.l, vault, e.cfg, nil, e.db, e.n, e.cache, cacheController, jobCtx, res.pipeline, res.wid.PipelineId) }() e.sendCommitted(cl.GetLeaseId()) diff --git a/spindle/mill/executor/outbox.go b/spindle/mill/executor/outbox.go index 20ee0650..44081680 100644 --- a/spindle/mill/executor/outbox.go +++ b/spindle/mill/executor/outbox.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/proto" "tangled.org/core/api/tangled" "tangled.org/core/spindle/db" + "tangled.org/core/spindle/engine" millproto "tangled.org/core/spindle/mill/proto" millv1 "tangled.org/core/spindle/mill/proto/gen" "tangled.org/core/spindle/models" @@ -64,12 +65,15 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro entry.Payload = payload case *millv1.Event_AttemptResult: entry.Payload = payload + case *millv1.Event_CacheUpdate: + entry.Payload = payload default: return fmt.Errorf("unsupported stream payload %T", payload) } - isTerminal := false - if _, ok := payload.(*millv1.Event_AttemptResult); ok { - isTerminal = true + durable := false + switch payload.(type) { + case *millv1.Event_AttemptResult, *millv1.Event_CacheUpdate: + durable = true } entry.Seqno = ^uint64(0) wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{ @@ -87,7 +91,7 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro } e.eventMu.Lock() - if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { + if control && !durable && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) e.eventMu.Unlock() return nil @@ -116,6 +120,28 @@ func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) erro return e.appendAndSend(leaseID, payload, true) } +func (e *Executor) appendCacheUpdate(leaseID string, update engine.CacheUpdate) error { + var action millv1.CacheUpdateAction + switch update.Action { + case engine.CacheUsed: + action = millv1.CacheUpdateAction_CACHE_USED + case engine.CacheStored: + action = millv1.CacheUpdateAction_CACHE_STORED + case engine.CacheDiscarded: + action = millv1.CacheUpdateAction_CACHE_DISCARDED + case engine.CacheMissing: + action = millv1.CacheUpdateAction_CACHE_MISSING + default: + return fmt.Errorf("unsupported cache update action %d", update.Action) + } + payload := &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: action, + Id: update.ID, + SizeBytes: update.SizeBytes, + }} + return e.appendAndSend(leaseID, payload, true) +} + func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { return e.appendTerminalWithArtifact(leaseID, status, st, "", "") } diff --git a/spindle/mill/executor/reserved.go b/spindle/mill/executor/reserved.go index 4a77d926..c7f53681 100644 --- a/spindle/mill/executor/reserved.go +++ b/spindle/mill/executor/reserved.go @@ -7,6 +7,7 @@ import ( "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" ) // wraps a real engine so StartWorkflows gets the slot ReserveSeat already @@ -18,8 +19,25 @@ type reservedEngine struct { once sync.Once } +type reservedCacheEngine struct { + *reservedEngine + runner engine.CacheRunner +} + func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine { - return &reservedEngine{Engine: inner, slot: slot} + reserved := &reservedEngine{Engine: inner, slot: slot} + if runner, ok := inner.(engine.CacheRunner); ok { + return &reservedCacheEngine{reservedEngine: reserved, runner: runner} + } + return reserved +} + +func (e *reservedCacheEngine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + return e.runner.RestoreCache(ctx, wid, wf, store, caches, wfLogger) +} + +func (e *reservedCacheEngine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + return e.runner.SaveCache(ctx, wid, wf, store, caches, wfLogger) } // hands back the pre-acquired slot exactly once, a second acquire would diff --git a/spindle/mill/executor/reserved_test.go b/spindle/mill/executor/reserved_test.go index 7fbc9816..1ed8d535 100644 --- a/spindle/mill/executor/reserved_test.go +++ b/spindle/mill/executor/reserved_test.go @@ -24,6 +24,7 @@ import ( millv1 "tangled.org/core/spindle/mill/proto/gen" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" ) type captureEncoder struct { @@ -79,6 +80,22 @@ func (e *fakeEngine) AcquireWorkflowSlot(ctx context.Context, wid models.Workflo return &fakeSlot{}, nil } +type fakeCacheEngine struct { + *fakeEngine + restored bool + saved bool +} + +func (e *fakeCacheEngine) RestoreCache(context.Context, models.WorkflowId, *models.Workflow, storage.Storage, []models.CacheBinding, models.WorkflowLogger) error { + e.restored = true + return nil +} + +func (e *fakeCacheEngine) SaveCache(context.Context, models.WorkflowId, *models.Workflow, storage.Storage, []models.CacheBinding, models.WorkflowLogger) error { + e.saved = true + return nil +} + type fakeStep struct{} func (fakeStep) Name() string { return "test" } @@ -92,7 +109,7 @@ func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) { } n := notifier.New() cfg := &config.Config{} - if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil))); err == nil { + if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil)), nil, nil); err == nil { t.Fatal("New succeeded with an unavailable outbox database") } } @@ -118,6 +135,26 @@ func TestReservedEngineHandsBackHeldSlotOnce(t *testing.T) { } } +func TestReservedEngineForwardsCacheRunner(t *testing.T) { + inner := &fakeCacheEngine{fakeEngine: &fakeEngine{}} + re := newReservedEngine(inner, &fakeSlot{}) + runner, ok := re.(engine.CacheRunner) + if !ok { + t.Fatal("reserved engine dropped CacheRunner") + } + + ctx := context.Background() + if err := runner.RestoreCache(ctx, models.WorkflowId{}, nil, nil, nil, nil); err != nil { + t.Fatal(err) + } + if err := runner.SaveCache(ctx, models.WorkflowId{}, nil, nil, nil, nil); err != nil { + t.Fatal(err) + } + if !inner.restored || !inner.saved { + t.Fatalf("cache calls were not forwarded: restored=%t saved=%t", inner.restored, inner.saved) + } +} + func TestHandleCommitIsIdempotent(t *testing.T) { enc := newCaptureEncoder() e := testExecutor(t) @@ -204,13 +241,20 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { if err != nil { t.Fatal(err) } + repoDidString := repoDid.String() res := &reservation{ leaseID: "lease-1", wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{ + RepoDid: repoDid, + TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDidString}, + }, + TrustedSource: true, + }, } e.active[res.leaseID] = res @@ -221,6 +265,9 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) } + if got := e.maskSecrets(res, "value=secret-value"); got != "value=***" { + t.Fatalf("masked log = %q", got) + } select { case got := <-inner.secrets: if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" { @@ -240,6 +287,83 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { e.jobsWG.Wait() } +func TestHandleCommitRejectsCachesAndSecretsForForkSource(t *testing.T) { + d := testDB(t) + n := notifier.New() + enc := newCaptureEncoder() + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + e := &Executor{ + db: d, + n: &n, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, + cache: cache, + enc: enc, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + e.lifecycleCtx = context.Background() + + inner := &fakeCacheEngine{fakeEngine: &fakeEngine{ + secrets: make(chan []secrets.UnlockedSecret, 1), + done: make(chan struct{}), + }} + targetRepoDid, err := syntax.ParseDID("did:plc:target") + if err != nil { + t.Fatal(err) + } + sourceRepoDid := "did:plc:fork" + cacheObjectID := "11111111-1111-4111-8111-111111111111" + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + realEngine: inner, + slot: &fakeSlot{}, + wf: &models.Workflow{ + Name: "build", + Steps: []models.Step{fakeStep{}}, + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }, + pipeline: &models.Pipeline{ + RepoDid: targetRepoDid, + TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + SourceRepo: &sourceRepoDid, + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &sourceRepoDid}, + }, + }, + } + e.active[res.leaseID] = res + + e.handleCommit(context.Background(), &millv1.CommitLease{ + LeaseId: res.leaseID, + Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}}, + CacheBindings: []*millv1.CacheBinding{{ + EntryIndex: 0, + RestoreId: cacheObjectID, + RestoreKey: "objects/" + targetRepoDid.String() + "/" + cacheObjectID, + SaveId: cacheObjectID, + SaveKey: "objects/" + targetRepoDid.String() + "/" + cacheObjectID, + }}, + }) + if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { + t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) + } + e.jobsWG.Wait() + + if inner.restored || inner.saved { + t.Fatalf("fork cache access = restored %t, saved %t", inner.restored, inner.saved) + } + if got := <-inner.secrets; len(got) != 0 { + t.Fatalf("fork secrets = %+v, want none", got) + } +} + func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) { connected := make(chan struct{}) release := make(chan struct{}) @@ -401,7 +525,7 @@ func TestSocketCancellationIndependence(t *testing.T) { realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{RepoDid: repoDid}, } e.active[res.leaseID] = res @@ -471,6 +595,7 @@ func TestTimerRace(t *testing.T) { RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), TtlSeconds: 1, + RepoDid: "did:web:example.com", }) e.mu.Lock() @@ -526,7 +651,7 @@ func TestStructuredShutdown(t *testing.T) { realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{RepoDid: repoDid}, } e.active[res.leaseID] = res diff --git a/spindle/mill/handler.go b/spindle/mill/handler.go index e4709e26..da679908 100644 --- a/spindle/mill/handler.go +++ b/spindle/mill/handler.go @@ -130,6 +130,9 @@ func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l) sess.closeTransport = conn.Close sess.labels = h.GetLabels() + sess.arch = h.GetArch() + sess.cacheStoreID = h.GetCacheStoreId() + sess.cacheNamespace = h.GetCacheNamespace() resume, ok := m.attachSession(sess) if !ok { diff --git a/spindle/mill/integration_test.go b/spindle/mill/integration_test.go index 409e0f31..354e527f 100644 --- a/spindle/mill/integration_test.go +++ b/spindle/mill/integration_test.go @@ -61,7 +61,7 @@ func TestEndToEndDummyJob(t *testing.T) { dummyEng := dummy.New(l) dummyEng.StepDelay = 50 * time.Millisecond engines := map[string]models.Engine{"dummy": dummyEng} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } @@ -72,7 +72,10 @@ func TestEndToEndDummyJob(t *testing.T) { Name: "build", Raw: "steps:\n - name: hello\n command: echo hi\n", } - wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + repoDid := "did:web:example.com" + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDid}, + }}) if err != nil { t.Fatalf("InitWorkflow: %v", err) } @@ -169,7 +172,7 @@ func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) { cfg.Mill.Labels = []string{"linux", "arm64", "gpu"} engines := map[string]models.Engine{"dummy": dummy.New(l)} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } @@ -223,7 +226,7 @@ func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { cfg.Mill.Labels = labels engines := map[string]models.Engine{"dummy": dummy.New(l)} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } @@ -245,7 +248,10 @@ func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { RunsOn: []string{"linux/arm64"}, Raw: "steps:\n - name: hello\n command: echo hi\n", } - wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + repoDid := "did:web:example.com" + wf, err := be.InitWorkflow(twf, tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDid}, + }}) if err != nil { t.Fatalf("InitWorkflow: %v", err) } diff --git a/spindle/mill/lease.go b/spindle/mill/lease.go index dee1332d..36fd5b6c 100644 --- a/spindle/mill/lease.go +++ b/spindle/mill/lease.go @@ -33,11 +33,12 @@ const ( // mill-side fencing token for one placed job type RemoteLease struct { - id string - nodeID string - epoch string - engine string - wid models.WorkflowId // job this lease carries, set once placed + id string + nodeID string + epoch string + engine string + cacheNamespace string + wid models.WorkflowId // job this lease carries, set once placed // restored after a mill restart. no RunStep waits on it, so terminals // and death are authored directly. set before publication, never mutated orphaned bool diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go index ed44f956..af6ce96e 100644 --- a/spindle/mill/mill.go +++ b/spindle/mill/mill.go @@ -18,6 +18,7 @@ import ( "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" "tangled.org/core/tid" millproto "tangled.org/core/spindle/mill/proto" @@ -43,13 +44,15 @@ func protoErrf(format string, args ...any) error { type Config struct { // mill appends live-tailed executor lines here so logview can follow running remote jobs - LogDir string - MaxPending int - ReconnectGrace time.Duration - JobTimeout time.Duration - BidTimeout time.Duration - TopK int - CancelTimeout time.Duration + LogDir string + MaxPending int + ReconnectGrace time.Duration + CacheStoreID string + CacheMaxBytesPerOwner int64 + JobTimeout time.Duration + BidTimeout time.Duration + TopK int + CancelTimeout time.Duration // how many protocol-violating session deaths in a row quarantine the node QuarantineStrikes int } @@ -58,8 +61,9 @@ type Mill struct { l *slog.Logger cfg Config - db *db.DB - n *notifier.Notifier + db *db.DB + n *notifier.Notifier + cache storage.Storage mu sync.Mutex sessions map[string]*millSession @@ -114,6 +118,12 @@ func (m *Mill) Attach(d *db.DB, n *notifier.Notifier) { m.mu.Unlock() } +func (m *Mill) AttachCache(store storage.Storage) { + m.mu.Lock() + m.cache = store + m.mu.Unlock() +} + func (m *Mill) nextLeaseID() string { m.mu.Lock() m.leaseSeq++ @@ -244,8 +254,7 @@ func (m *Mill) noteSessionError(sess *millSession, err error) { func (m *Mill) sessionReady(sess *millSession) { for _, lease := range m.cancelledLeasesForNode(sess.nodeID) { - _, reason := lease.cancelRequested() - m.sendCancel(sess, lease, reason) + m.sendCancel(sess, lease) } m.notifyChange() } @@ -363,6 +372,7 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow } if lease != nil { lease.wid = wid + wf.CacheNamespace = lease.cacheNamespace if err := m.persistLease(lease, leaseRowReserved); err != nil { m.releaseRemote(lease) m.mu.Lock() @@ -396,7 +406,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId } requiredLabels := requiredLabels(wf) - candidates := m.rankCandidates(engineName, requiredLabels) + candidates := m.rankCandidates(engineName, requiredLabels, len(wf.Caches) > 0) if len(candidates) == 0 { return nil, nil } @@ -421,6 +431,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId defer cancel() leaseID := m.nextLeaseID() lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName) + lease.cacheNamespace = sess.cacheNamespace m.mu.Lock() m.reservations[leaseID] = lease m.mu.Unlock() @@ -432,6 +443,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId Knot: wid.Knot, Rkey: wid.Rkey, TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), + RepoDid: wf.RepoDid, }} resp, err := sess.request(bidCtx, leaseID, msg) if err != nil { @@ -520,7 +532,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId // ranks nodes that are least busy first. if a resource is used a lot // then that node will lose to one that is more even across the board. -func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession { +func (m *Mill) rankCandidates(engineName string, requiredLabels []string, cacheRequired bool) []*millSession { m.mu.Lock() defer m.mu.Unlock() @@ -538,6 +550,9 @@ func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*mil if s.snapshot == nil { continue } + if cacheRequired && (m.cfg.CacheStoreID == "" || s.cacheStoreID != m.cfg.CacheStoreID) { + continue + } // the engine has to exist and have room right now ea, ok := s.snapshot.GetEngines()[engineName] if !ok || !ea.GetAvailable() { @@ -607,14 +622,25 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked } lease := st.Lease + capabilities, err := cacheCapabilities(wf.RepoDid, wf.CacheBindings) + if err != nil { + return err + } + if m.db != nil { + if err := m.db.SaveMillCacheCapabilities(lease.id, capabilities); err != nil { + return fmt.Errorf("persist mill cache capabilities: %w", err) + } + } + pbSecrets := make([]*millv1.Secret, len(unlocked)) for i, s := range unlocked { pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value} } commit := &millproto.Message{CommitLease: &millv1.CommitLease{ - LeaseId: lease.id, - Secrets: pbSecrets, + LeaseId: lease.id, + Secrets: pbSecrets, + CacheBindings: cacheBindingsToProto(wf.CacheBindings), }} // commit retries ride reconnects, a reservation outlives one @@ -672,8 +698,8 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked if err := m.persistLease(lease, leaseRowRunning); err != nil { m.l.Error("persist running mill lease", "lease", lease.id, "err", err) } - if cancelled, reason := lease.cancelRequested(); cancelled { - m.sendCancel(sess, lease, reason) + if cancelled, _ := lease.cancelRequested(); cancelled { + m.sendCancel(sess, lease) } break } @@ -753,7 +779,7 @@ func (m *Mill) destroy(wid models.WorkflowId) { lease.deliverCancelled(reason) case cancelRemote: if sess := m.sessionForNode(lease.nodeID); sess != nil { - m.sendCancel(sess, lease, reason) + m.sendCancel(sess, lease) } } } @@ -817,10 +843,9 @@ func (m *Mill) releaseRemote(lease *RemoteLease) { } } -func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease, reason string) { +func (m *Mill) sendCancel(sess *millSession, lease *RemoteLease) { if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{ LeaseId: lease.id, - Reason: reason, }}); err != nil { return } @@ -900,6 +925,8 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { } var pendingTerminals []pendingTerminal var artifactLeases []*RemoteLease + cacheDeletes := make(map[string]db.CacheEntry) + cacheObjectDeletes := make(map[string]struct{}) finishedInBatch := make(map[string]struct{}) var highestSeqno uint64 = current @@ -955,6 +982,77 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { } } + case entry.GetCacheUpdate() != nil: + update := entry.GetCacheUpdate() + if tx != nil { + at := time.Now() + capabilityAction := "" + switch update.GetAction() { + case millv1.CacheUpdateAction_CACHE_USED, millv1.CacheUpdateAction_CACHE_MISSING: + capabilityAction = "restore" + case millv1.CacheUpdateAction_CACHE_STORED, millv1.CacheUpdateAction_CACHE_DISCARDED: + capabilityAction = "save" + default: + return protoErrf("unsupported cache update action %v", update.GetAction()) + } + ref, authorized, err := tx.ConsumeMillCacheCapability( + context.Background(), + lease.id, + capabilityAction, + update.GetId(), + ) + if err != nil { + return err + } + if !authorized { + return protoErrf( + "cache update %q was not planned for lease %q", + update.GetId(), + lease.id, + ) + } + + switch update.GetAction() { + case millv1.CacheUpdateAction_CACHE_USED: + if err := tx.TouchCacheEntry(context.Background(), update.GetId(), at); err != nil { + return err + } + case millv1.CacheUpdateAction_CACHE_STORED: + superseded, ready, err := tx.MarkCacheEntryReady( + context.Background(), + update.GetId(), + update.GetSizeBytes(), + m.cfg.CacheMaxBytesPerOwner, + at, + ) + if err != nil { + return err + } + for _, old := range superseded { + cacheDeletes[old.ID] = old + } + if !ready { + if err := tx.QueueCacheObjectDeletion(context.Background(), ref, at); err != nil { + return err + } + cacheObjectDeletes[ref] = struct{}{} + } + case millv1.CacheUpdateAction_CACHE_DISCARDED, millv1.CacheUpdateAction_CACHE_MISSING: + pendingOnly := update.GetAction() == millv1.CacheUpdateAction_CACHE_DISCARDED + discarded, err := tx.DiscardCacheEntry( + context.Background(), + update.GetId(), + pendingOnly, + ) + if err != nil { + return err + } + if discarded != nil { + cacheDeletes[discarded.ID] = *discarded + } + } + } + case entry.GetAttemptResult() != nil: ar := entry.GetAttemptResult() statusStr := "success" @@ -1028,6 +1126,10 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { return err } + m.deleteCacheEntries(cacheDeletes) + + m.deleteQueuedCacheObjects(cacheObjectDeletes) + if m.cfg.LogDir != "" { for _, lease := range artifactLeases { path := models.LogFilePath(m.cfg.LogDir, lease.wid) @@ -1059,6 +1161,48 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { return m.sendAck(sess, highestSeqno) } +func (m *Mill) deleteCacheEntries(entries map[string]db.CacheEntry) { + if len(entries) == 0 { + return + } + if m.cache == nil { + m.l.Warn("cache objects need deletion but the mill has no cache store", "count", len(entries)) + return + } + for _, entry := range entries { + if err := m.cache.Delete(context.Background(), entry.StorageKey); err != nil { + m.l.Warn("delete cache object", "id", entry.ID, "ref", entry.StorageKey, "err", err) + continue + } + if m.db != nil { + if err := m.db.DeleteCacheEntry(context.Background(), entry.ID); err != nil { + m.l.Warn("delete cache metadata", "id", entry.ID, "err", err) + } + } + } +} + +func (m *Mill) deleteQueuedCacheObjects(keys map[string]struct{}) { + if len(keys) == 0 { + return + } + if m.cache == nil { + m.l.Warn("cache objects need deletion but the mill has no cache store", "count", len(keys)) + return + } + for key := range keys { + if err := m.cache.Delete(context.Background(), key); err != nil { + m.l.Warn("delete unindexed cache object", "ref", key, "err", err) + continue + } + if m.db != nil { + if err := m.db.CompleteCacheObjectDeletion(context.Background(), key); err != nil { + m.l.Warn("complete cache object deletion", "ref", key, "err", err) + } + } + } +} + func (m *Mill) sendAck(sess *millSession, seqno uint64) error { msg := &millproto.Message{Ack: &millv1.Ack{ Epoch: sess.epoch, diff --git a/spindle/mill/mill_test.go b/spindle/mill/mill_test.go index 1cf88d02..be8fb400 100644 --- a/spindle/mill/mill_test.go +++ b/spindle/mill/mill_test.go @@ -1,10 +1,12 @@ package mill import ( + "bytes" "context" "errors" "io" "log/slog" + "strings" "sync" "testing" "time" @@ -13,6 +15,7 @@ import ( "tangled.org/core/spindle/db" "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" millproto "tangled.org/core/spindle/mill/proto" millv1 "tangled.org/core/spindle/mill/proto/gen" @@ -23,16 +26,45 @@ type scriptedEncoder func(*millproto.Message) error func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) } func testWorkflow(name string) *models.Workflow { + repoDid := "did:web:example.com" return &models.Workflow{ Name: name, Environment: map[string]string{}, Steps: []models.Step{remoteStep{}}, + RepoDid: repoDid, Data: &millWorkflowState{ RawWorkflow: tangled.Pipeline_Workflow{Name: name}, - RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}, + RawPipeline: tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDid}, + }}, }, } } +func TestMillEngineRequiresSharedCacheStoreID(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Name: "build", + Raw: "cache:\n - key: deps\n paths: [deps]\n", + } + for _, tc := range []struct { + name string + storeID string + want int + }{ + {name: "unidentified store", want: 0}, + {name: "shared store", storeID: "fleet-cache", want: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + m := New(discardLogger(), Config{CacheStoreID: tc.storeID}) + wf, err := NewEngine("microvm", m).InitWorkflow(twf, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if got := len(wf.Caches); got != tc.want { + t.Fatalf("cache entries = %d, want %d", got, tc.want) + } + }) + } +} func testWorkflowWithRunsOn(name string, runsOn []string) *models.Workflow { wf := testWorkflow(name) @@ -139,6 +171,17 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second}) wf := testWorkflow("build") + restoreID := "11111111-1111-4111-8111-111111111111" + saveID := "22222222-2222-4222-8222-222222222222" + wf.CacheBindings = []models.CacheBinding{{ + EntryIndex: 0, + Hash: "abc123", + RestoreID: restoreID, + RestoreKey: "objects/did:web:example.com/" + restoreID, + RestoreName: "deps-abc123", + SaveID: saveID, + SaveKey: "objects/did:web:example.com/" + saveID, + }} wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} lease := newLease("lease-1", "node-1", "inc-1", "dummy") lease.wid = wid @@ -175,6 +218,12 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { if msg.GetCommitLease() == nil { return nil } + bindings := msg.GetCommitLease().GetCacheBindings() + if len(bindings) != 1 || + bindings[0].GetRestoreId() != restoreID || + bindings[0].GetSaveId() != saveID { + t.Errorf("CommitLease cache bindings = %+v", bindings) + } leaseID := msg.GetCommitLease().GetLeaseId() sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) _ = m.onEventBatch(sess2, &millv1.EventBatch{ @@ -288,11 +337,22 @@ func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want) + assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels, false), tt.want) }) } } +func TestRankCandidatesRequiresSharedCacheStore(t *testing.T) { + m := New(discardLogger(), Config{CacheStoreID: "disk:/cache"}) + matching := addCandidateSession(t, m, "matching", []string{"linux"}, 0.5, nil) + matching.cacheStoreID = "disk:/cache" + other := addCandidateSession(t, m, "other", []string{"linux"}, 0.1, nil) + other.cacheStoreID = "s3:bucket/cache" + + assertRankedNodes(t, m.rankCandidates("dummy", []string{"linux"}, true), []string{"matching"}) + assertRankedNodes(t, m.rankCandidates("dummy", []string{"linux"}, false), []string{"other", "matching"}) +} + func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{BidTimeout: 10 * time.Millisecond}) @@ -615,6 +675,205 @@ func TestAtomicBatchRollback(t *testing.T) { } } +func TestCacheUpdatesCommitWithStreamCursor(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + + at := time.Date(2026, 8, 11, 20, 0, 0, 0, time.UTC) + old := db.CacheEntry{ + ID: "old", StorageKey: "objects/old", OwnerDID: "did:plc:owner", + RepoDID: "did:plc:repo", Engine: "dummy", CacheKey: "deps", CacheHash: "hash", + SizeBytes: 3, State: "ready", CreatedAt: at.Add(-time.Hour), LastUsedAt: at.Add(-time.Hour), + } + pending := old + pending.ID = "pending" + pending.StorageKey = "objects/pending" + pending.SizeBytes = 0 + pending.State = "pending" + pending.CreatedAt = at + pending.LastUsedAt = at + for _, entry := range []db.CacheEntry{old, pending} { + if err := bdb.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatal(err) + } + if err := cache.Put(context.Background(), entry.StorageKey, bytes.NewReader([]byte(entry.ID))); err != nil { + t.Fatal(err) + } + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(owned, leaseRowRunning); err != nil { + t.Fatal(err) + } + if err := bdb.SaveMillCacheCapabilities(owned.id, []db.MillCacheCapability{ + {Action: "restore", CacheID: old.ID, StorageKey: old.StorageKey}, + {Action: "save", CacheID: pending.ID, StorageKey: pending.StorageKey}, + }); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + batch := &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, LeaseId: owned.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_USED, Id: old.ID, + }}, + }, + { + Seqno: 2, LeaseId: owned.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_STORED, Id: pending.ID, + SizeBytes: int64(len(pending.ID)), + }}, + }, + }, + } + if err := m.onEventBatch(sess, batch); err != nil { + t.Fatalf("onEventBatch: %v", err) + } + if err := m.onEventBatch(sess, batch); err != nil { + t.Fatalf("replay onEventBatch: %v", err) + } + + ready, err := bdb.FindCacheEntry(context.Background(), old.RepoDID, old.Engine, old.CacheKey, old.CacheHash) + if err != nil { + t.Fatal(err) + } + if ready.ID != pending.ID || ready.SizeBytes != int64(len(pending.ID)) { + t.Fatalf("ready cache = %+v", ready) + } + + if _, err := cache.Get(context.Background(), old.StorageKey); !errors.Is(err, storage.ErrNotExist) { + t.Fatalf("superseded object error = %v, want ErrNotExist", err) + } + if cursor, err := bdb.GetExecutorCursor(sess.nodeID, sess.epoch); err != nil || cursor != 2 { + t.Fatalf("cursor = %d, %v; want 2", cursor, err) + } +} +func TestCacheUpdateCannotDeleteUnplannedObject(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + victim := "objects/did:web:victim.example/11111111-1111-4111-8111-111111111111" + if err := cache.Put(context.Background(), victim, strings.NewReader("victim")); err != nil { + t.Fatal(err) + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + err = m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{{ + Seqno: 1, + LeaseId: lease.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_STORED, + Id: "unknown", + }}, + }}, + }) + if err == nil { + t.Fatal("unplanned cache update succeeded") + } + reader, err := cache.Get(context.Background(), victim) + if err != nil { + t.Fatalf("victim object was deleted: %v", err) + } + _ = reader.Close() + if cursor, err := bdb.GetExecutorCursor(sess.nodeID, sess.epoch); err != nil || cursor != 0 { + t.Fatalf("cursor = %d, %v; want 0", cursor, err) + } +} + +func TestMissingPendingRowQueuesAndDeletesPlannedObject(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + id := "11111111-1111-4111-8111-111111111111" + key := "objects/did:web:example.com/" + id + if err := cache.Put(context.Background(), key, strings.NewReader("orphan")); err != nil { + t.Fatal(err) + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatal(err) + } + if err := bdb.SaveMillCacheCapabilities(lease.id, []db.MillCacheCapability{{ + Action: "save", + CacheID: id, + StorageKey: key, + }}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + if err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{{ + Seqno: 1, + LeaseId: lease.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_STORED, + Id: id, + }}, + }}, + }); err != nil { + t.Fatal(err) + } + if _, err := cache.Get(context.Background(), key); !errors.Is(err, storage.ErrNotExist) { + t.Fatalf("planned orphan still exists: %v", err) + } + pending, err := bdb.PendingCacheObjectDeletions(context.Background(), 10) + if err != nil { + t.Fatal(err) + } + if len(pending) != 0 { + t.Fatalf("completed object deletion remains queued: %v", pending) + } +} + func TestTerminalBeforeACK(t *testing.T) { m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) diff --git a/spindle/mill/proto/gen/mill.pb.go b/spindle/mill/proto/gen/mill.pb.go index e31758f2..3a7b2670 100644 --- a/spindle/mill/proto/gen/mill.pb.go +++ b/spindle/mill/proto/gen/mill.pb.go @@ -172,6 +172,61 @@ func (TerminalStatus) EnumDescriptor() ([]byte, []int) { return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} } +type CacheUpdateAction int32 + +const ( + CacheUpdateAction_CACHE_UPDATE_ACTION_UNSPECIFIED CacheUpdateAction = 0 + CacheUpdateAction_CACHE_USED CacheUpdateAction = 1 + CacheUpdateAction_CACHE_STORED CacheUpdateAction = 2 + CacheUpdateAction_CACHE_DISCARDED CacheUpdateAction = 3 + CacheUpdateAction_CACHE_MISSING CacheUpdateAction = 4 +) + +// Enum value maps for CacheUpdateAction. +var ( + CacheUpdateAction_name = map[int32]string{ + 0: "CACHE_UPDATE_ACTION_UNSPECIFIED", + 1: "CACHE_USED", + 2: "CACHE_STORED", + 3: "CACHE_DISCARDED", + 4: "CACHE_MISSING", + } + CacheUpdateAction_value = map[string]int32{ + "CACHE_UPDATE_ACTION_UNSPECIFIED": 0, + "CACHE_USED": 1, + "CACHE_STORED": 2, + "CACHE_DISCARDED": 3, + "CACHE_MISSING": 4, + } +) + +func (x CacheUpdateAction) Enum() *CacheUpdateAction { + p := new(CacheUpdateAction) + *p = x + return p +} + +func (x CacheUpdateAction) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CacheUpdateAction) Descriptor() protoreflect.EnumDescriptor { + return file_spindle_mill_v1_mill_proto_enumTypes[3].Descriptor() +} + +func (CacheUpdateAction) Type() protoreflect.EnumType { + return &file_spindle_mill_v1_mill_proto_enumTypes[3] +} + +func (x CacheUpdateAction) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CacheUpdateAction.Descriptor instead. +func (CacheUpdateAction) EnumDescriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} +} + // executor identity, sent on connect type Hello struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -179,10 +234,12 @@ type Hello struct { // GOARCH of the node, informational only Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"` // operator-defined labels, matched against runs_on - Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` - Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` + Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + CacheStoreId string `protobuf:"bytes,5,opt,name=cache_store_id,json=cacheStoreId,proto3" json:"cache_store_id,omitempty"` + CacheNamespace string `protobuf:"bytes,6,opt,name=cache_namespace,json=cacheNamespace,proto3" json:"cache_namespace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Hello) Reset() { @@ -243,6 +300,20 @@ func (x *Hello) GetEpoch() string { return "" } +func (x *Hello) GetCacheStoreId() string { + if x != nil { + return x.CacheStoreId + } + return "" +} + +func (x *Hello) GetCacheNamespace() string { + if x != nil { + return x.CacheNamespace + } + return "" +} + // reconnect state for an existing epoch type Resume struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -420,9 +491,11 @@ type ReserveSeat struct { RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` // pipeline id, the executor reconstructs the exact WorkflowId from it - Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` - Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` - TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` + Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` + Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` + TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` + // repository that owns the pipeline, used to distinguish fork source code + RepoDid string `protobuf:"bytes,8,opt,name=repo_did,json=repoDid,proto3" json:"repo_did,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -506,6 +579,13 @@ func (x *ReserveSeat) GetTtlSeconds() uint32 { return 0 } +func (x *ReserveSeat) GetRepoDid() string { + if x != nil { + return x.RepoDid + } + return "" +} + type ReserveResult struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` @@ -627,18 +707,103 @@ func (x *Secret) GetValue() string { return "" } +type CacheBinding struct { + state protoimpl.MessageState `protogen:"open.v1"` + EntryIndex uint32 `protobuf:"varint,1,opt,name=entry_index,json=entryIndex,proto3" json:"entry_index,omitempty"` + RestoreId string `protobuf:"bytes,3,opt,name=restore_id,json=restoreId,proto3" json:"restore_id,omitempty"` + RestoreKey string `protobuf:"bytes,4,opt,name=restore_key,json=restoreKey,proto3" json:"restore_key,omitempty"` + RestoreName string `protobuf:"bytes,5,opt,name=restore_name,json=restoreName,proto3" json:"restore_name,omitempty"` + SaveId string `protobuf:"bytes,6,opt,name=save_id,json=saveId,proto3" json:"save_id,omitempty"` + SaveKey string `protobuf:"bytes,7,opt,name=save_key,json=saveKey,proto3" json:"save_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheBinding) Reset() { + *x = CacheBinding{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheBinding) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheBinding) ProtoMessage() {} + +func (x *CacheBinding) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheBinding.ProtoReflect.Descriptor instead. +func (*CacheBinding) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} +} + +func (x *CacheBinding) GetEntryIndex() uint32 { + if x != nil { + return x.EntryIndex + } + return 0 +} + +func (x *CacheBinding) GetRestoreId() string { + if x != nil { + return x.RestoreId + } + return "" +} + +func (x *CacheBinding) GetRestoreKey() string { + if x != nil { + return x.RestoreKey + } + return "" +} + +func (x *CacheBinding) GetRestoreName() string { + if x != nil { + return x.RestoreName + } + return "" +} + +func (x *CacheBinding) GetSaveId() string { + if x != nil { + return x.SaveId + } + return "" +} + +func (x *CacheBinding) GetSaveKey() string { + if x != nil { + return x.SaveKey + } + return "" +} + // promotes a reservation to a running job and hands over the secrets type CommitLease struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` Secrets []*Secret `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty"` + CacheBindings []*CacheBinding `protobuf:"bytes,3,rep,name=cache_bindings,json=cacheBindings,proto3" json:"cache_bindings,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *CommitLease) Reset() { *x = CommitLease{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -650,7 +815,7 @@ func (x *CommitLease) String() string { func (*CommitLease) ProtoMessage() {} func (x *CommitLease) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -663,7 +828,7 @@ func (x *CommitLease) ProtoReflect() protoreflect.Message { // Deprecated: Use CommitLease.ProtoReflect.Descriptor instead. func (*CommitLease) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} } func (x *CommitLease) GetLeaseId() string { @@ -680,6 +845,13 @@ func (x *CommitLease) GetSecrets() []*Secret { return nil } +func (x *CommitLease) GetCacheBindings() []*CacheBinding { + if x != nil { + return x.CacheBindings + } + return nil +} + type Committed struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` @@ -689,7 +861,7 @@ type Committed struct { func (x *Committed) Reset() { *x = Committed{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -701,7 +873,7 @@ func (x *Committed) String() string { func (*Committed) ProtoMessage() {} func (x *Committed) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -714,7 +886,7 @@ func (x *Committed) ProtoReflect() protoreflect.Message { // Deprecated: Use Committed.ProtoReflect.Descriptor instead. func (*Committed) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} } func (x *Committed) GetLeaseId() string { @@ -734,7 +906,7 @@ type ReleaseLease struct { func (x *ReleaseLease) Reset() { *x = ReleaseLease{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -746,7 +918,7 @@ func (x *ReleaseLease) String() string { func (*ReleaseLease) ProtoMessage() {} func (x *ReleaseLease) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -759,7 +931,7 @@ func (x *ReleaseLease) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead. func (*ReleaseLease) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} } func (x *ReleaseLease) GetLeaseId() string { @@ -773,14 +945,13 @@ func (x *ReleaseLease) GetLeaseId() string { type CancelAttempt struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` - Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *CancelAttempt) Reset() { *x = CancelAttempt{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -792,7 +963,7 @@ func (x *CancelAttempt) String() string { func (*CancelAttempt) ProtoMessage() {} func (x *CancelAttempt) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -805,7 +976,7 @@ func (x *CancelAttempt) ProtoReflect() protoreflect.Message { // Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead. func (*CancelAttempt) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} } func (x *CancelAttempt) GetLeaseId() string { @@ -815,13 +986,6 @@ func (x *CancelAttempt) GetLeaseId() string { return "" } -func (x *CancelAttempt) GetReason() string { - if x != nil { - return x.Reason - } - return "" -} - type CancelAck struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` @@ -831,7 +995,7 @@ type CancelAck struct { func (x *CancelAck) Reset() { *x = CancelAck{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -843,7 +1007,7 @@ func (x *CancelAck) String() string { func (*CancelAck) ProtoMessage() {} func (x *CancelAck) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -856,7 +1020,7 @@ func (x *CancelAck) ProtoReflect() protoreflect.Message { // Deprecated: Use CancelAck.ProtoReflect.Descriptor instead. func (*CancelAck) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} } func (x *CancelAck) GetLeaseId() string { @@ -877,7 +1041,7 @@ type StatusEvent struct { func (x *StatusEvent) Reset() { *x = StatusEvent{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -889,7 +1053,7 @@ func (x *StatusEvent) String() string { func (*StatusEvent) ProtoMessage() {} func (x *StatusEvent) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -902,7 +1066,7 @@ func (x *StatusEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. func (*StatusEvent) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} } func (x *StatusEvent) GetStatus() NonterminalStatus { @@ -936,7 +1100,7 @@ type LogArtifact struct { func (x *LogArtifact) Reset() { *x = LogArtifact{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -948,7 +1112,7 @@ func (x *LogArtifact) String() string { func (*LogArtifact) ProtoMessage() {} func (x *LogArtifact) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -961,7 +1125,7 @@ func (x *LogArtifact) ProtoReflect() protoreflect.Message { // Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. func (*LogArtifact) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} } func (x *LogArtifact) GetRef() string { @@ -991,7 +1155,7 @@ type AttemptResult struct { func (x *AttemptResult) Reset() { *x = AttemptResult{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1003,7 +1167,7 @@ func (x *AttemptResult) String() string { func (*AttemptResult) ProtoMessage() {} func (x *AttemptResult) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1016,7 +1180,7 @@ func (x *AttemptResult) ProtoReflect() protoreflect.Message { // Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead. func (*AttemptResult) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} } func (x *AttemptResult) GetStatus() TerminalStatus { @@ -1047,6 +1211,66 @@ func (x *AttemptResult) GetLogArtifact() *LogArtifact { return nil } +type CacheUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Action CacheUpdateAction `protobuf:"varint,1,opt,name=action,proto3,enum=spindle.mill.v1.CacheUpdateAction" json:"action,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + SizeBytes int64 `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheUpdate) Reset() { + *x = CacheUpdate{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheUpdate) ProtoMessage() {} + +func (x *CacheUpdate) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheUpdate.ProtoReflect.Descriptor instead. +func (*CacheUpdate) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} +} + +func (x *CacheUpdate) GetAction() CacheUpdateAction { + if x != nil { + return x.Action + } + return CacheUpdateAction_CACHE_UPDATE_ACTION_UNSPECIFIED +} + +func (x *CacheUpdate) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *CacheUpdate) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + // live non-replay log frame type LiveLog struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1058,7 +1282,7 @@ type LiveLog struct { func (x *LiveLog) Reset() { *x = LiveLog{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1070,7 +1294,7 @@ func (x *LiveLog) String() string { func (*LiveLog) ProtoMessage() {} func (x *LiveLog) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1083,7 +1307,7 @@ func (x *LiveLog) ProtoReflect() protoreflect.Message { // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. func (*LiveLog) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} } func (x *LiveLog) GetLeaseId() string { @@ -1109,6 +1333,7 @@ type Event struct { // // *Event_StatusEvent // *Event_AttemptResult + // *Event_CacheUpdate Payload isEvent_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1116,7 +1341,7 @@ type Event struct { func (x *Event) Reset() { *x = Event{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1128,7 +1353,7 @@ func (x *Event) String() string { func (*Event) ProtoMessage() {} func (x *Event) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1141,7 +1366,7 @@ func (x *Event) ProtoReflect() protoreflect.Message { // Deprecated: Use Event.ProtoReflect.Descriptor instead. func (*Event) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} } func (x *Event) GetSeqno() uint64 { @@ -1183,6 +1408,15 @@ func (x *Event) GetAttemptResult() *AttemptResult { return nil } +func (x *Event) GetCacheUpdate() *CacheUpdate { + if x != nil { + if x, ok := x.Payload.(*Event_CacheUpdate); ok { + return x.CacheUpdate + } + } + return nil +} + type isEvent_Payload interface { isEvent_Payload() } @@ -1195,10 +1429,16 @@ type Event_AttemptResult struct { AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` } +type Event_CacheUpdate struct { + CacheUpdate *CacheUpdate `protobuf:"bytes,5,opt,name=cache_update,json=cacheUpdate,proto3,oneof"` +} + func (*Event_StatusEvent) isEvent_Payload() {} func (*Event_AttemptResult) isEvent_Payload() {} +func (*Event_CacheUpdate) isEvent_Payload() {} + // a flushed bundle of events type EventBatch struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1210,7 +1450,7 @@ type EventBatch struct { func (x *EventBatch) Reset() { *x = EventBatch{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1222,7 +1462,7 @@ func (x *EventBatch) String() string { func (*EventBatch) ProtoMessage() {} func (x *EventBatch) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1235,7 +1475,7 @@ func (x *EventBatch) ProtoReflect() protoreflect.Message { // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. func (*EventBatch) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} } func (x *EventBatch) GetEpoch() string { @@ -1263,7 +1503,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1275,7 +1515,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1288,7 +1528,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{20} } func (x *Ack) GetEpoch() string { @@ -1326,7 +1566,7 @@ type Message struct { func (x *Message) Reset() { *x = Message{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1338,7 +1578,7 @@ func (x *Message) String() string { func (*Message) ProtoMessage() {} func (x *Message) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1351,7 +1591,7 @@ func (x *Message) ProtoReflect() protoreflect.Message { // Deprecated: Use Message.ProtoReflect.Descriptor instead. func (*Message) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{21} } func (x *Message) GetHello() *Hello { @@ -1449,12 +1689,14 @@ var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\n" + - "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" + + "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"\xd5\x01\n" + "\x05Hello\x12)\n" + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" + "\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" + "\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" + - "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" + + "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12$\n" + + "\x0ecache_store_id\x18\x05 \x01(\tR\fcacheStoreId\x120\n" + + "\x0fcache_namespace\x18\x06 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x0ecacheNamespace\"D\n" + "\x06Resume\x12\x1d\n" + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" + "\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" + @@ -1470,7 +1712,7 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + "\fEnginesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + - "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\x80\x02\n" + + "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\xa4\x02\n" + "\vReserveSeat\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + @@ -1479,7 +1721,8 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" + "\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" + "\vttl_seconds\x18\a \x01(\rR\n" + - "ttlSeconds\"\xbf\x01\n" + + "ttlSeconds\x12\"\n" + + "\brepo_did\x18\b \x01(\tB\a\xbaH\x04r\x02\x10\x01R\arepoDid\"\xbf\x01\n" + "\rReserveResult\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + @@ -1487,17 +1730,27 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\freject_class\x18\x04 \x01(\x0e2\x1c.spindle.mill.v1.RejectClassB\b\xbaH\x05\x82\x01\x02\x10\x01R\vrejectClass\"0\n" + "\x06Secret\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value\"d\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"\xd8\x01\n" + + "\fCacheBinding\x12\x1f\n" + + "\ventry_index\x18\x01 \x01(\rR\n" + + "entryIndex\x12\x1d\n" + + "\n" + + "restore_id\x18\x03 \x01(\tR\trestoreId\x12\x1f\n" + + "\vrestore_key\x18\x04 \x01(\tR\n" + + "restoreKey\x12!\n" + + "\frestore_name\x18\x05 \x01(\tR\vrestoreName\x12 \n" + + "\asave_id\x18\x06 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x06saveId\x12\"\n" + + "\bsave_key\x18\a \x01(\tB\a\xbaH\x04r\x02\x10\x01R\asaveKey\"\xaa\x01\n" + "\vCommitLease\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" + - "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\"/\n" + + "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\x12D\n" + + "\x0ecache_bindings\x18\x03 \x03(\v2\x1d.spindle.mill.v1.CacheBindingR\rcacheBindings\"/\n" + "\tCommitted\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" + "\fReleaseLease\x12\"\n" + - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"K\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"3\n" + "\rCancelAttempt\x12\"\n" + - "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x16\n" + - "\x06reason\x18\x02 \x01(\tR\x06reason\"/\n" + + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"/\n" + "\tCancelAck\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"\x93\x01\n" + "\vStatusEvent\x12F\n" + @@ -1513,15 +1766,22 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06status\x12\x1f\n" + "\x05error\x18\x02 \x01(\tB\t\xbaH\x06r\x04(\x80\x80\x04R\x05error\x12\x1b\n" + "\texit_code\x18\x03 \x01(\x03R\bexitCode\x12?\n" + - "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"Q\n" + + "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\"\x96\x01\n" + + "\vCacheUpdate\x12F\n" + + "\x06action\x18\x01 \x01(\x0e2\".spindle.mill.v1.CacheUpdateActionB\n" + + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06action\x12\x17\n" + + "\x02id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x02id\x12&\n" + + "\n" + + "size_bytes\x18\x04 \x01(\x03B\a\xbaH\x04\"\x02(\x00R\tsizeBytes\"Q\n" + "\aLiveLog\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + - "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + + "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xab\x02\n" + "\x05Event\x12\x1d\n" + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + - "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + + "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResult\x12A\n" + + "\fcache_update\x18\x05 \x01(\v2\x1c.spindle.mill.v1.CacheUpdateH\x00R\vcacheUpdateB\x10\n" + "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + "\n" + "EventBatch\x12\x1d\n" + @@ -1574,7 +1834,14 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\n" + "\x06FAILED\x10\x02\x12\v\n" + "\aTIMEOUT\x10\x03\x12\r\n" + - "\tCANCELLED\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3" + "\tCANCELLED\x10\x04*\x82\x01\n" + + "\x11CacheUpdateAction\x12#\n" + + "\x1fCACHE_UPDATE_ACTION_UNSPECIFIED\x10\x00\x12\x0e\n" + + "\n" + + "CACHE_USED\x10\x01\x12\x10\n" + + "\fCACHE_STORED\x10\x02\x12\x13\n" + + "\x0fCACHE_DISCARDED\x10\x03\x12\x11\n" + + "\rCACHE_MISSING\x10\x04B0Z.tangled.org/core/spindle/mill/proto/gen;millv1b\x06proto3" var ( file_spindle_mill_v1_mill_proto_rawDescOnce sync.Once @@ -1588,65 +1855,71 @@ func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte { return file_spindle_mill_v1_mill_proto_rawDescData } -var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_spindle_mill_v1_mill_proto_goTypes = []any{ (RejectClass)(0), // 0: spindle.mill.v1.RejectClass (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus (TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus - (*Hello)(nil), // 3: spindle.mill.v1.Hello - (*Resume)(nil), // 4: spindle.mill.v1.Resume - (*EngineAvailability)(nil), // 5: spindle.mill.v1.EngineAvailability - (*NodeSnapshot)(nil), // 6: spindle.mill.v1.NodeSnapshot - (*ReserveSeat)(nil), // 7: spindle.mill.v1.ReserveSeat - (*ReserveResult)(nil), // 8: spindle.mill.v1.ReserveResult - (*Secret)(nil), // 9: spindle.mill.v1.Secret - (*CommitLease)(nil), // 10: spindle.mill.v1.CommitLease - (*Committed)(nil), // 11: spindle.mill.v1.Committed - (*ReleaseLease)(nil), // 12: spindle.mill.v1.ReleaseLease - (*CancelAttempt)(nil), // 13: spindle.mill.v1.CancelAttempt - (*CancelAck)(nil), // 14: spindle.mill.v1.CancelAck - (*StatusEvent)(nil), // 15: spindle.mill.v1.StatusEvent - (*LogArtifact)(nil), // 16: spindle.mill.v1.LogArtifact - (*AttemptResult)(nil), // 17: spindle.mill.v1.AttemptResult - (*LiveLog)(nil), // 18: spindle.mill.v1.LiveLog - (*Event)(nil), // 19: spindle.mill.v1.Event - (*EventBatch)(nil), // 20: spindle.mill.v1.EventBatch - (*Ack)(nil), // 21: spindle.mill.v1.Ack - (*Message)(nil), // 22: spindle.mill.v1.Message - nil, // 23: spindle.mill.v1.EngineAvailability.LoadEntry - nil, // 24: spindle.mill.v1.NodeSnapshot.EnginesEntry + (CacheUpdateAction)(0), // 3: spindle.mill.v1.CacheUpdateAction + (*Hello)(nil), // 4: spindle.mill.v1.Hello + (*Resume)(nil), // 5: spindle.mill.v1.Resume + (*EngineAvailability)(nil), // 6: spindle.mill.v1.EngineAvailability + (*NodeSnapshot)(nil), // 7: spindle.mill.v1.NodeSnapshot + (*ReserveSeat)(nil), // 8: spindle.mill.v1.ReserveSeat + (*ReserveResult)(nil), // 9: spindle.mill.v1.ReserveResult + (*Secret)(nil), // 10: spindle.mill.v1.Secret + (*CacheBinding)(nil), // 11: spindle.mill.v1.CacheBinding + (*CommitLease)(nil), // 12: spindle.mill.v1.CommitLease + (*Committed)(nil), // 13: spindle.mill.v1.Committed + (*ReleaseLease)(nil), // 14: spindle.mill.v1.ReleaseLease + (*CancelAttempt)(nil), // 15: spindle.mill.v1.CancelAttempt + (*CancelAck)(nil), // 16: spindle.mill.v1.CancelAck + (*StatusEvent)(nil), // 17: spindle.mill.v1.StatusEvent + (*LogArtifact)(nil), // 18: spindle.mill.v1.LogArtifact + (*AttemptResult)(nil), // 19: spindle.mill.v1.AttemptResult + (*CacheUpdate)(nil), // 20: spindle.mill.v1.CacheUpdate + (*LiveLog)(nil), // 21: spindle.mill.v1.LiveLog + (*Event)(nil), // 22: spindle.mill.v1.Event + (*EventBatch)(nil), // 23: spindle.mill.v1.EventBatch + (*Ack)(nil), // 24: spindle.mill.v1.Ack + (*Message)(nil), // 25: spindle.mill.v1.Message + nil, // 26: spindle.mill.v1.EngineAvailability.LoadEntry + nil, // 27: spindle.mill.v1.NodeSnapshot.EnginesEntry } var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ - 23, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry - 24, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry + 26, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry + 27, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass - 9, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret - 1, // 4: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus - 2, // 5: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus - 16, // 6: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact - 15, // 7: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent - 17, // 8: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult - 19, // 9: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event - 3, // 10: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello - 4, // 11: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume - 6, // 12: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot - 7, // 13: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat - 8, // 14: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult - 10, // 15: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease - 11, // 16: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed - 12, // 17: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease - 13, // 18: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt - 14, // 19: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck - 20, // 20: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch - 21, // 21: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack - 18, // 22: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog - 5, // 23: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability - 24, // [24:24] is the sub-list for method output_type - 24, // [24:24] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 10, // 3: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret + 11, // 4: spindle.mill.v1.CommitLease.cache_bindings:type_name -> spindle.mill.v1.CacheBinding + 1, // 5: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus + 2, // 6: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus + 18, // 7: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact + 3, // 8: spindle.mill.v1.CacheUpdate.action:type_name -> spindle.mill.v1.CacheUpdateAction + 17, // 9: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent + 19, // 10: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult + 20, // 11: spindle.mill.v1.Event.cache_update:type_name -> spindle.mill.v1.CacheUpdate + 22, // 12: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event + 4, // 13: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello + 5, // 14: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume + 7, // 15: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot + 8, // 16: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat + 9, // 17: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult + 12, // 18: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease + 13, // 19: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed + 14, // 20: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease + 15, // 21: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt + 16, // 22: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck + 23, // 23: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch + 24, // 24: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack + 21, // 25: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog + 6, // 26: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability + 27, // [27:27] is the sub-list for method output_type + 27, // [27:27] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name } func init() { file_spindle_mill_v1_mill_proto_init() } @@ -1654,17 +1927,18 @@ func file_spindle_mill_v1_mill_proto_init() { if File_spindle_mill_v1_mill_proto != nil { return } - file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ + file_spindle_mill_v1_mill_proto_msgTypes[18].OneofWrappers = []any{ (*Event_StatusEvent)(nil), (*Event_AttemptResult)(nil), + (*Event_CacheUpdate)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), - NumEnums: 3, - NumMessages: 22, + NumEnums: 4, + NumMessages: 24, NumExtensions: 0, NumServices: 0, }, diff --git a/spindle/mill/proto/protocol.go b/spindle/mill/proto/protocol.go index 2e1d1402..0a739f8e 100644 --- a/spindle/mill/proto/protocol.go +++ b/spindle/mill/proto/protocol.go @@ -17,7 +17,7 @@ import ( ) const ( - ProtocolVersion = 1 + ProtocolVersion = 4 // generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and // workflow JSON, and streamed log lines can be chunky MaxMessageBytes = 8 * 1024 * 1024 diff --git a/spindle/mill/proto/protocol_test.go b/spindle/mill/proto/protocol_test.go index 0e0b9637..3fa8bb36 100644 --- a/spindle/mill/proto/protocol_test.go +++ b/spindle/mill/proto/protocol_test.go @@ -20,6 +20,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { Knot: "knot.example", Rkey: "abc123", TtlSeconds: 30, + RepoDid: "did:web:example.com", }, } if err := enc.Encode(want); err != nil { @@ -34,7 +35,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { if rs == nil { t.Fatal("decoded message missing reserve_seat") } - if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 { + if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 || rs.RepoDid != "did:web:example.com" { t.Fatalf("round-trip mismatch: %+v", rs) } } @@ -69,11 +70,11 @@ func TestValidationRules(t *testing.T) { { name: "valid hello message", msg: &Message{ - Hello: &millv1.Hello{ - ProtocolVersion: 1, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + Hello: &millv1.Hello{ProtocolVersion: 1, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }, }, wantErr: false, diff --git a/spindle/mill/proto/spindle/mill/v1/mill.proto b/spindle/mill/proto/spindle/mill/v1/mill.proto index df71b493..07b71d88 100644 --- a/spindle/mill/proto/spindle/mill/v1/mill.proto +++ b/spindle/mill/proto/spindle/mill/v1/mill.proto @@ -14,6 +14,8 @@ message Hello { // operator-defined labels, matched against runs_on repeated string labels = 3; string epoch = 4 [(buf.validate.field).string.min_len = 1]; + string cache_store_id = 5; + string cache_namespace = 6 [(buf.validate.field).string.min_len = 1]; } // reconnect state for an existing epoch @@ -47,6 +49,8 @@ message ReserveSeat { string knot = 5; string rkey = 6; uint32 ttl_seconds = 7; + // repository that owns the pipeline, used to distinguish fork source code + string repo_did = 8 [(buf.validate.field).string.min_len = 1]; } enum RejectClass { @@ -68,10 +72,20 @@ message Secret { string value = 2; } +message CacheBinding { + uint32 entry_index = 1; + string restore_id = 3; + string restore_key = 4; + string restore_name = 5; + string save_id = 6 [(buf.validate.field).string.min_len = 1]; + string save_key = 7 [(buf.validate.field).string.min_len = 1]; +} + // promotes a reservation to a running job and hands over the secrets message CommitLease { string lease_id = 1 [(buf.validate.field).string.min_len = 1]; repeated Secret secrets = 2; + repeated CacheBinding cache_bindings = 3; } message Committed { @@ -86,7 +100,6 @@ message ReleaseLease { // cancels a running attempt message CancelAttempt { string lease_id = 1 [(buf.validate.field).string.min_len = 1]; - string reason = 2; } message CancelAck { @@ -131,6 +144,23 @@ message AttemptResult { LogArtifact log_artifact = 4; } +enum CacheUpdateAction { + CACHE_UPDATE_ACTION_UNSPECIFIED = 0; + CACHE_USED = 1; + CACHE_STORED = 2; + CACHE_DISCARDED = 3; + CACHE_MISSING = 4; +} + +message CacheUpdate { + CacheUpdateAction action = 1 [(buf.validate.field).enum = { + defined_only: true + not_in: 0 + }]; + string id = 2 [(buf.validate.field).string.min_len = 1]; + int64 size_bytes = 4 [(buf.validate.field).int64.gte = 0]; +} + // live non-replay log frame message LiveLog { string lease_id = 1 [(buf.validate.field).string.min_len = 1]; @@ -146,6 +176,7 @@ message Event { option (buf.validate.oneof).required = true; StatusEvent status_event = 3; AttemptResult attempt_result = 4; + CacheUpdate cache_update = 5; } } diff --git a/spindle/mill/restore.go b/spindle/mill/restore.go index de77c943..b8b65cec 100644 --- a/spindle/mill/restore.go +++ b/spindle/mill/restore.go @@ -154,7 +154,7 @@ func (m *Mill) reconcileLeases(sess *millSession, activeLeaseIDs []string) error } } for _, id := range unknown { - if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id, Reason: "lease is not owned by this mill"}}); err != nil { + if err := sess.send(&millproto.Message{CancelAttempt: &millv1.CancelAttempt{LeaseId: id}}); err != nil { return fmt.Errorf("cancel unknown executor lease %q: %w", id, err) } } diff --git a/spindle/mill/session.go b/spindle/mill/session.go index 4a942ea1..f3f6b84c 100644 --- a/spindle/mill/session.go +++ b/spindle/mill/session.go @@ -19,7 +19,10 @@ var errSessionClosed = errors.New("mill: executor session closed") type millSession struct { nodeID string epoch string + arch string labels []string + cacheStoreID string + cacheNamespace string enc messageEncoder l *slog.Logger closeTransport func() error diff --git a/spindle/models/cache.go b/spindle/models/cache.go new file mode 100644 index 00000000..a235b2f1 --- /dev/null +++ b/spindle/models/cache.go @@ -0,0 +1,129 @@ +package models + +import ( + "fmt" + "regexp" + "runtime" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/google/uuid" + "gopkg.in/yaml.v3" +) + +type CacheEntry struct { + Key string `yaml:"key"` + Hash []string `yaml:"hash"` + Paths []string `yaml:"paths"` + // 1 is fastest, 19 is smallest, 0 is the zstd default + CompressionLevel int `yaml:"compression-level"` + // on-success is the default, always also saves on failed runs + When string `yaml:"when"` +} + +type CacheBinding struct { + EntryIndex int + Paths []string + Key string + Hash string + SaveID string + SaveKey string + RestoreID string + RestoreKey string + RestoreName string + CompressionLevel int + When string +} + +func (b CacheBinding) SaveOn(failed bool) bool { + return b.When == "always" || !failed +} + +func CacheNamespace() string { + return fmt.Sprintf("tar-zstd-v1/%s/%s", runtime.GOOS, runtime.GOARCH) +} + +func CacheObjectKey(repoDid, id string) (string, error) { + did, err := syntax.ParseDID(repoDid) + if err != nil { + return "", fmt.Errorf("cache: invalid repository did %q: %w", repoDid, err) + } + parsedID, err := uuid.Parse(id) + if err != nil { + return "", fmt.Errorf("cache: invalid object id %q: %w", id, err) + } + return fmt.Sprintf("objects/%s/%s", did, parsedID), nil +} + +// keys become storage paths, so no slashes +var cacheKeyRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +func (c CacheEntry) Validate() error { + if !cacheKeyRe.MatchString(c.Key) { + return fmt.Errorf("cache: invalid key %q (allowed: letters, digits, '.', '_', '-')", c.Key) + } + for _, f := range c.Hash { + // rev-parse needs plain repo-relative paths, not pathspecs + if f == "" || strings.HasPrefix(f, "/") || strings.HasPrefix(f, "..") { + return fmt.Errorf("cache %q: hash path %q is not repo-relative", c.Key, f) + } + if strings.ContainsAny(f, ": \t\n\"'`$\\*?[") || strings.Contains(f, "/../") || strings.HasSuffix(f, "/..") { + return fmt.Errorf("cache %q: hash path %q contains unsupported characters", c.Key, f) + } + } + if len(c.Paths) == 0 { + return fmt.Errorf("cache %q: no paths", c.Key) + } + if c.CompressionLevel < 0 || c.CompressionLevel > 19 { + return fmt.Errorf("cache %q: compression-level %d out of range (1-19)", c.Key, c.CompressionLevel) + } + switch c.When { + case "", "on-success", "always": + default: + return fmt.Errorf("cache %q: when %q is not one of on-success, always", c.Key, c.When) + } + seen := make(map[string]bool, len(c.Paths)) + for _, p := range c.Paths { + if p == "" { + return fmt.Errorf("cache %q: empty path", c.Key) + } + if strings.ContainsAny(p, " \t\n\"'`$\\;&|<>(){}") { + return fmt.Errorf("cache %q: path %q contains unsupported characters", c.Key, p) + } + if strings.Contains(p, "..") { + return fmt.Errorf("cache %q: path %q must not contain '..'", c.Key, p) + } + if seen[p] { + return fmt.Errorf("cache %q: duplicate path %q", c.Key, p) + } + seen[p] = true + } + return nil +} + +func ParseCacheEntries(raw string) ([]CacheEntry, error) { + var workflow struct { + Cache []CacheEntry `yaml:"cache"` + } + if err := yaml.Unmarshal([]byte(raw), &workflow); err != nil { + return nil, err + } + if err := ValidateCacheEntries(workflow.Cache); err != nil { + return nil, err + } + return workflow.Cache, nil +} + +func ValidateCacheEntries(entries []CacheEntry) error { + seen := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + if err := entry.Validate(); err != nil { + return err + } + if _, ok := seen[entry.Key]; ok { + return fmt.Errorf("cache: duplicate key %q", entry.Key) + } + seen[entry.Key] = struct{}{} + } + return nil +} diff --git a/spindle/models/cache_test.go b/spindle/models/cache_test.go new file mode 100644 index 00000000..2cc92e78 --- /dev/null +++ b/spindle/models/cache_test.go @@ -0,0 +1,78 @@ +package models + +import ( + "strings" + "testing" +) + +func TestCacheEntryValidate(t *testing.T) { + valid := CacheEntry{Key: "go-mod-v1", Hash: []string{"go.sum", "sub/dir/package-lock.json"}, Paths: []string{"/workspace/go/pkg/mod", "/root/.cache"}} + if err := valid.Validate(); err != nil { + t.Fatalf("valid entry: %v", err) + } + + cases := []struct { + name string + entry CacheEntry + want string + }{ + {"empty key", CacheEntry{Key: "", Paths: []string{"/x"}}, "invalid key"}, + {"key with slash", CacheEntry{Key: "a/b", Paths: []string{"/x"}}, "invalid key"}, + {"no paths", CacheEntry{Key: "ok"}, "no paths"}, + {"empty path", CacheEntry{Key: "ok", Paths: []string{""}}, "empty path"}, + {"path with space", CacheEntry{Key: "ok", Paths: []string{"/my dir"}}, "unsupported characters"}, + {"path with quote", CacheEntry{Key: "ok", Paths: []string{"/x'$(rm)"}}, "unsupported characters"}, + {"path with shell syntax", CacheEntry{Key: "ok", Paths: []string{"/x;touch-pwn"}}, "unsupported characters"}, + {"path traversal", CacheEntry{Key: "ok", Paths: []string{"/x/../y"}}, ".."}, + {"duplicate path", CacheEntry{Key: "ok", Paths: []string{"/x", "/x"}}, "duplicate"}, + {"level too high", CacheEntry{Key: "ok", Paths: []string{"/x"}, CompressionLevel: 20}, "out of range"}, + {"negative level", CacheEntry{Key: "ok", Paths: []string{"/x"}, CompressionLevel: -1}, "out of range"}, + {"bad when", CacheEntry{Key: "ok", Paths: []string{"/x"}, When: "sometimes"}, "not one of"}, + {"absolute hash path", CacheEntry{Key: "ok", Hash: []string{"/go.sum"}, Paths: []string{"/x"}}, "not repo-relative"}, + {"empty hash path", CacheEntry{Key: "ok", Hash: []string{""}, Paths: []string{"/x"}}, "not repo-relative"}, + {"hash path traversal", CacheEntry{Key: "ok", Hash: []string{"../secret"}, Paths: []string{"/x"}}, "not repo-relative"}, + {"hash path inner traversal", CacheEntry{Key: "ok", Hash: []string{"a/../../b"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with colon", CacheEntry{Key: "ok", Hash: []string{"rev:go.sum"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with space", CacheEntry{Key: "ok", Hash: []string{"my lock"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with glob", CacheEntry{Key: "ok", Hash: []string{"*.sum"}, Paths: []string{"/x"}}, "unsupported characters"}, + } + for _, tc := range cases { + err := tc.entry.Validate() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: got %v, want error containing %q", tc.name, err, tc.want) + } + } + + ok := CacheEntry{Key: "go-mod", Hash: []string{"go.sum"}, Paths: []string{"node_modules", "/root/.cache"}, CompressionLevel: 19, When: "always"} + if err := ok.Validate(); err != nil { + t.Errorf("relative and absolute paths should validate, got %v", err) + } +} + +func TestParseCacheEntries(t *testing.T) { + entries, err := ParseCacheEntries(` +cache: + - key: go-mod + hash: [go.sum] + paths: [/root/go/pkg/mod] + compression-level: 7 + when: always +`) + if err != nil { + t.Fatalf("ParseCacheEntries: %v", err) + } + if len(entries) != 1 || entries[0].Key != "go-mod" || entries[0].CompressionLevel != 7 || entries[0].When != "always" { + t.Fatalf("parsed entries = %+v", entries) + } + + _, err = ParseCacheEntries(` +cache: + - key: deps + paths: [/deps] + - key: deps + paths: [/other] +`) + if err == nil || !strings.Contains(err.Error(), "duplicate key") { + t.Fatalf("duplicate cache keys error = %v", err) + } +} diff --git a/spindle/models/clone.go b/spindle/models/clone.go index d9d54389..1f6be1cb 100644 --- a/spindle/models/clone.go +++ b/spindle/models/clone.go @@ -47,7 +47,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe return CloneStep{} } - commitSHA, err := extractCommitSHA(tr) + commitSHA, err := ExtractCommitSHA(tr) if err != nil { return CloneStep{ kind: StepKindSystem, @@ -83,8 +83,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe } } -// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type -func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { +func ExtractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { switch workflow.TriggerKind(tr.Kind) { case workflow.TriggerKindPush: if tr.Push == nil { diff --git a/spindle/models/pipeline.go b/spindle/models/pipeline.go index a794644b..0316bdc3 100644 --- a/spindle/models/pipeline.go +++ b/spindle/models/pipeline.go @@ -1,12 +1,36 @@ package models -import "github.com/bluesky-social/indigo/atproto/syntax" +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/api/tangled" +) type Pipeline struct { RepoDid syntax.DID Workflows map[Engine][]Workflow // whether the code being ran was checked out from RepoDid itself TrustedSource bool + // used to resolve cache hash files against the checkout the workflow builds + TriggerMetadata *tangled.Pipeline_TriggerMetadata +} + +func TrustedPipelineSource(metadata *tangled.Pipeline_TriggerMetadata, target string) bool { + targetDid, err := syntax.ParseDID(target) + if err != nil || metadata == nil || metadata.Repo == nil || metadata.Repo.RepoDid == nil { + return false + } + checkoutDid, err := syntax.ParseDID(*metadata.Repo.RepoDid) + if err != nil || checkoutDid != targetDid { + return false + } + if metadata.SourceRepo != nil { + sourceDid, err := syntax.ParseDID(*metadata.SourceRepo) + if err != nil || sourceDid != checkoutDid { + return false + } + } + return true } type Step interface { @@ -25,8 +49,13 @@ const ( ) type Workflow struct { - Steps []Step - Name string - Data any - Environment map[string]string + Steps []Step + Name string + Data any + Environment map[string]string + Caches []CacheEntry + CacheBindings []CacheBinding + CacheNamespace string + RepoDid string + Engine string } diff --git a/spindle/models/pipeline_test.go b/spindle/models/pipeline_test.go new file mode 100644 index 00000000..608de864 --- /dev/null +++ b/spindle/models/pipeline_test.go @@ -0,0 +1,34 @@ +package models + +import ( + "testing" + + "tangled.org/core/api/tangled" +) + +func TestTrustedPipelineSourceRequiresConsistentCheckoutMetadata(t *testing.T) { + target := "did:web:target.example" + fork := "did:web:fork.example" + for _, tc := range []struct { + name string + repo *string + source *string + trusted bool + }{ + {name: "target checkout", repo: &target, trusted: true}, + {name: "consistent target source", repo: &target, source: &target, trusted: true}, + {name: "fork checkout", repo: &fork, source: &fork}, + {name: "contradictory source", repo: &fork, source: &target}, + {name: "missing checkout", source: &target}, + } { + t.Run(tc.name, func(t *testing.T) { + metadata := &tangled.Pipeline_TriggerMetadata{SourceRepo: tc.source} + if tc.repo != nil { + metadata.Repo = &tangled.Pipeline_TriggerRepo{RepoDid: tc.repo} + } + if got := TrustedPipelineSource(metadata, target); got != tc.trusted { + t.Fatalf("TrustedPipelineSource() = %t, want %t", got, tc.trusted) + } + }) + } +} diff --git a/spindle/server.go b/spindle/server.go index 4895a1fb..ab9a82be 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -46,6 +46,7 @@ import ( "tangled.org/core/spindle/mill/executor" "tangled.org/core/spindle/models" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" "tangled.org/core/spindle/xrpc" "tangled.org/core/tid" "tangled.org/core/workflow" @@ -74,6 +75,7 @@ type Spindle struct { res *idresolver.Resolver verify repoverify.Verifier vault secrets.Manager + cache storage.Storage motd []byte motdMu sync.RWMutex rootCtx context.Context @@ -85,6 +87,23 @@ type Spindle struct { exec *executor.Executor } +func newCacheStore(ctx context.Context, cfg *config.Config) (storage.Storage, error) { + switch cfg.Cache.Backend { + case "": + return nil, nil + case "disk": + dir := cfg.Cache.DiskDir + if dir == "" { + dir = filepath.Join(filepath.Dir(cfg.Server.DBPath), "cache") + } + return storage.NewDisk(dir) + case "s3": + return storage.NewUnversionedS3(ctx, cfg.Cache.S3Bucket, cfg.Cache.S3Prefix) + default: + return nil, fmt.Errorf("storage: unknown backend %q", cfg.Cache.Backend) + } +} + // New creates a new Spindle server with the provided configuration and engines. func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine) (*Spindle, error) { logger := log.FromContext(ctx) @@ -98,12 +117,21 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m return nil, fmt.Errorf("failed to run startup migrations: %w", err) } + cacheStore, err := newCacheStore(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("failed to setup cache storage: %w", err) + } + if cacheStore != nil { + logger.Info("cache storage enabled", "backend", cfg.Cache.Backend, "storeID", cfg.Cache.StoreID) + } + spindle := &Spindle{ db: d, l: logger, n: &n, engs: engines, cfg: cfg, + cache: cacheStore, motd: defaultMotd, rootCtx: ctx, jobWake: make(chan struct{}, 1), @@ -123,6 +151,9 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m if cfg.LegacyS3.LogBucket != "" { logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") } + if cfg.Role != config.RoleExecutor { + engine.StartCachePruner(ctx, logger, d, cacheStore, cfg.Cache.Retention, cfg.Cache.PruneInterval) + } if cfg.Role == config.RoleStandalone { spindle.reader = stores } else { @@ -269,6 +300,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m cfg.Server.Tap.AdminPassword = pw logger.Info("embedded tap: using random admin password") } + spindle.tap = NewTapClient(spindle) return spindle, nil @@ -408,9 +440,11 @@ func Run(ctx context.Context) error { // on a mill host, engines place jobs on executors instead of running // them. all names share one Mill m = mill.New(log.SubLogger(logger, "mill"), mill.Config{ - LogDir: cfg.Server.LogDir, - MaxPending: cfg.Mill.MaxPending, - ReconnectGrace: cfg.Mill.ReconnectGrace, + LogDir: cfg.Server.LogDir, + MaxPending: cfg.Mill.MaxPending, + ReconnectGrace: cfg.Mill.ReconnectGrace, + CacheStoreID: cfg.Cache.StoreID, + CacheMaxBytesPerOwner: cfg.Cache.MaxBytesPerOwner, }) engines = map[string]models.Engine{ "nixery": mill.NewEngine("nixery", m), @@ -443,13 +477,14 @@ func Run(ctx context.Context) error { // the engines built above hold the mill, but the mill's db and // notifier only exist after New, so attach them here m.Attach(s.DB(), s.Notifier()) + m.AttachCache(s.cache) s.mill = m if err := m.RestoreState(); err != nil { return fmt.Errorf("restoring mill state: %w", err) } } if cfg.Role == config.RoleExecutor { - s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store, s.cache) if err != nil { return err } @@ -1005,11 +1040,6 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { } pipelineEnv := models.PipelineEnvVarsForSource(job.Tpl.TriggerMetadata, pipelineId, job.SourceRepo) - trustedSource := true - if tm := job.Tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && - *tm.SourceRepo != "" && *tm.SourceRepo != job.RepoDid { - trustedSource = false - } initTpl := job.Tpl if job.SourceRepo != nil && job.Tpl.TriggerMetadata != nil { @@ -1017,6 +1047,7 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { tm.Repo = job.SourceRepo initTpl.TriggerMetadata = &tm } + trustedSource := models.TrustedPipelineSource(initTpl.TriggerMetadata, job.RepoDid) workflows := make(map[models.Engine][]models.Workflow) for _, w := range job.Tpl.Workflows { @@ -1045,13 +1076,20 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { ewf.Environment = make(map[string]string) } maps.Copy(ewf.Environment, pipelineEnv) + + ewf.Engine = w.Engine + ewf.RepoDid = job.RepoDid + if !trustedSource { + ewf.Caches = nil + } workflows[eng] = append(workflows[eng], *ewf) } - engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.rootCtx, &models.Pipeline{ - RepoDid: syntax.DID(job.RepoDid), - Workflows: workflows, - TrustedSource: trustedSource, + engine.StartWorkflows(log.SubLogger(s.l, "engine"), s.vault, s.cfg, s.stores, s.db, s.n, s.cache, nil, s.rootCtx, &models.Pipeline{ + RepoDid: syntax.DID(job.RepoDid), + Workflows: workflows, + TrustedSource: trustedSource, + TriggerMetadata: job.Tpl.TriggerMetadata, }, pipelineId) } diff --git a/spindle/storage/storage_test.go b/spindle/storage/storage_test.go index 3a6b2036..362dcd80 100644 --- a/spindle/storage/storage_test.go +++ b/spindle/storage/storage_test.go @@ -45,6 +45,7 @@ func (m *mockS3Client) DeleteObject(_ context.Context, params *s3.DeleteObjectIn delete(m.objects, *params.Bucket+"/"+*params.Key) return &s3.DeleteObjectOutput{}, nil } + func TestValidateKey(t *testing.T) { valid := []string{ "did:plc:xyz123/go-mod-v1", diff --git a/spindle/storage/unversioned_s3.go b/spindle/storage/unversioned_s3.go new file mode 100644 index 00000000..7263a918 --- /dev/null +++ b/spindle/storage/unversioned_s3.go @@ -0,0 +1,30 @@ +package storage + +import ( + "context" + "fmt" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func NewUnversionedS3(ctx context.Context, bucket, prefix string) (*S3, error) { + if bucket == "" { + return nil, fmt.Errorf("storage: s3 backend requires a bucket") + } + sdkConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("storage: load s3 config: %w", err) + } + client := s3.NewFromConfig(sdkConfig) + versioning, err := client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + if err != nil { + return nil, fmt.Errorf("storage: check s3 bucket versioning: %w", err) + } + if versioning.Status != "" { + return nil, fmt.Errorf("storage: s3 cache bucket must not use versioning") + } + return NewS3(client, bucket, prefix) +}