diff --git a/docker-compose.mill.yml b/docker-compose.mill.yml index 12d3ad3b..bc9bc420 100644 --- a/docker-compose.mill.yml +++ b/docker-compose.mill.yml @@ -28,6 +28,9 @@ x-mill-executor: &mill-executor SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_ARTIFACT_STORES_DISK_DIR: /var/lib/spindle/artifacts SPINDLE_MILL_ARTIFACT_STORE: disk + SPINDLE_CACHE_BACKEND: disk + SPINDLE_CACHE_DISK_DIR: /var/lib/spindle/cache + SPINDLE_CACHE_STORE_ID: localinfra-shared-cache SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= @@ -89,8 +92,12 @@ services: SPINDLE_MILL_JUMP_HOST_KEY_PATH: /var/lib/spindle/debug_jump_host_key SPINDLE_MILL_DEBUG_EXECUTOR_PORT: "2223" SPINDLE_MILL_MAX_JUMP_CONNECTIONS: "128" + SPINDLE_CACHE_BACKEND: disk + SPINDLE_CACHE_DISK_DIR: /var/lib/spindle/cache + SPINDLE_CACHE_STORE_ID: localinfra-shared-cache volumes: - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache ports: !override - "127.0.0.1:2224:2224" @@ -147,6 +154,7 @@ services: volumes: - spindle-executor-a-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -166,6 +174,7 @@ services: volumes: - spindle-executor-b-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -185,6 +194,7 @@ services: volumes: - spindle-executor-c-data:/var/lib/spindle - spindle-artifacts:/var/lib/spindle/artifacts + - spindle-cache:/var/lib/spindle/cache - ./out/localinfra-spindle-images-alpine:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -197,3 +207,4 @@ volumes: spindle-executor-b-data: spindle-executor-c-data: spindle-artifacts: + spindle-cache: diff --git a/docker-compose.yml b/docker-compose.yml index 18898d4a..d186487a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -184,6 +184,7 @@ services: SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" + SPINDLE_CACHE_BACKEND: disk SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload diff --git a/docs/DOCS.md b/docs/DOCS.md index 5606c96e..00823b15 100644 --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -1345,6 +1345,58 @@ triggered by a pull request: - `TANGLED_PR_SOURCE_SHA` - The commit SHA of the source branch +### Cache + +The `cache` field lets a workflow persist directories across +pipeline runs. Before the first step, the engine looks up +each entry's key and extracts the matching archive into the +workspace; after all steps succeed, the paths are archived +again and stored back under the key. + +- `key`: name this cache is saved under. Keys are scoped to + the repository and the engine specified. +- `hash`: **optional** list of repo files (lockfiles, + manifests) whose content is folded into the key. The entry + is stored as `-`, so editing + `go.sum` automatically rotates the cache without bumping + the key by hand. Paths are relative to the repository + root and are read from git at the commit being built. +- `paths`: paths to archive. Relative paths are anchored at + the repository checkout, the directory steps start in + (`/workspace/repo` on microvm, `/tangled/workspace` on + nixery). Absolute paths work too, for caching directories + outside the checkout, but note they name engine-specific + locations. All paths must be writable by the CI user and + contain no spaces. +- `compression-level`: **optional** zstd level, `1` + (fastest) to `19` (smallest). defaults to (`5`). +- `when`: **optional** save policy: `on-success` (the + default) or `always`. + +When the exact key (or generation, with `hash`) misses, the +newest older generation under the same key is restored. + +```yaml +cache: + - key: go-mod + hash: + - go.sum + - go.mod + paths: + - .gocache +``` + +Caches are only restored and saved for trusted pipelines +(pushes and same-repository pull requests). Pipelines +building untrusted code, like pull requests from forks, +skip the cache entirely. Saving follows each entry's `when` +policy, except on timeout, when nothing is saved. A cache +miss or failure never fails the workflow. + +The spindle operator chooses the storage backend. See +[Running spindle](#running-spindle). If no backend is +configured, `cache` entries are ignored. + ### Steps The `steps` field allows you to define what steps should run @@ -1899,6 +1951,51 @@ cache (and read from it), configure the cache (prefix narinfos to be signed by one of the operator-configured trusted public keys. Defaults to `false` for caches such as NCPS that sign on ingestion. +The generic CI cache (the workflow-level +[`cache`](#cache) field) is configured via prefix +`SPINDLE_CACHE_`. + +- `SPINDLE_CACHE_BACKEND`: Storage backend, `disk` or `s3` + (default: `""`, caching disabled). +- `SPINDLE_CACHE_DISK_DIR`: Directory for the `disk` backend + (default: a `cache` directory next to the spindle + database). +- `SPINDLE_CACHE_S3_BUCKET`: Unversioned bucket for the `s3` + backend. Credentials come from the standard AWS chain and + need `s3:GetBucketVersioning` in addition to object access. +- `SPINDLE_CACHE_S3_PREFIX`: Key prefix inside the bucket + (default: `"spindle/cache"`). +- `SPINDLE_CACHE_STORE_ID`: Operator-defined identity for a + physically shared store. Set the same value only on mill and + executor processes that can read and write the same objects. +- `SPINDLE_CACHE_RETENTION`: Time since the last restore or + save before an entry is deleted (default: `720h`, or 30 + days). Set to `0` to keep entries indefinitely. +- `SPINDLE_CACHE_PRUNE_INTERVAL`: How often expired entries + are deleted (default: `1h`). +- `SPINDLE_CACHE_MAX_BYTES_PER_OWNER`: Maximum stored bytes + across all repositories owned by one DID (default: + `10737418240`, or 10 GiB). Set to `0` for no byte limit. +- `SPINDLE_CACHE_MAX_ENTRIES_PER_OWNER`: Maximum pending and + stored entries across all repositories owned by one DID + (default: `1000`). Set to `0` for no entry limit. + +Cache metadata, per-owner usage, and limits are enforced from +spindle's SQLite database. Objects are namespaced by the target +repository and authorized per mill lease; storage backends are +never listed during lookup or cleanup. + +In a mill deployment, configure the same cache backend and +`SPINDLE_CACHE_STORE_ID` on the mill and every executor that +should run cached workflows. Without a store ID, cache entries +are ignored and workflows run uncached. The mill owns cache +metadata and pruning; executors transfer cache objects directly. +A disk cache must therefore be a shared filesystem. An S3 cache +must use the same endpoint, bucket, and prefix. Executors +advertise their store ID and archive compatibility namespace, and +the mill only places cached workflows on a compatible store; OS, +architecture, and archive format are part of the cache namespace. + ### Running spindle 1. **Set the environment variables.** For example: diff --git a/nix/microvm/base.nix b/nix/microvm/base.nix index 68dd40d9..a5e03865 100644 --- a/nix/microvm/base.nix +++ b/nix/microvm/base.nix @@ -259,6 +259,7 @@ in { gz-utils bzip2 lz4 + zstd p7zip ]; # disable default nixos packages ([perl rsync strace]) diff --git a/nix/pkgs/spindle-alpine-image.nix b/nix/pkgs/spindle-alpine-image.nix index ba85d128..a5632060 100644 --- a/nix/pkgs/spindle-alpine-image.nix +++ b/nix/pkgs/spindle-alpine-image.nix @@ -20,7 +20,8 @@ git = pkgsStatic.callPackage ./spindle-static-git.nix {}; # we don't include gnused, xxd etc. here because busybox has them # we want to keep the image this image small! - guestTools = [nix bash git curl jq]; + # zstd is not a busybox applet, and the spindle cache saves tar|zstd + guestTools = [nix bash git curl jq pkgsStatic.zstd]; # run by busybox at sysinit setupScript = writeText "spindle-setup" '' diff --git a/shuttle/src/gen/file_descriptor_set.bin b/shuttle/src/gen/file_descriptor_set.bin index b733f9ae6d1f9366cdaf1e0224065c0d6aafeb1e..6d4b1ec4f8947843596476ee0fe454fd4823d932 100644 GIT binary patch delta 11484 zcmdnDKzPD-;f5B*7N!>FEi9>1nOF2MPEVi8;?1;^X}Vnni*cv{mq>1AUVK4Oeo20E zeolN@YEf}!ex3xo0;2|RkWg-BUI0`o3?d^0(vet!se=QmBe5c2`t_+Skr6^FTztui z$r-8f#U=SgsqvX9608c08k|8qV9{WZpl6DZ0T(}*AD@?)n_66un4BuX22$q4zRQE9 zh)IAk2x5vKOnSS)G?t%?jL)`f&0uk6R1s3*;wVZj$d6CSOp)M1)yWQ$aLG*Bo;#Bz zim84h6Bmy&$VE<>c`2EB=|b{c+^KmbMV0ZHc`2zC5{zIkaDjz9K_Wu3TwFz|aOX3F zOb_COig>1g^g}uE*{PKhEMR$<4)4@TAw@18n1Z~-`rK4#2yjCML4Fca;9@UMECYuR zn#b8d;+`oWAA(q5J3tY`i3l68R*+4aTmn!-l;i!-ZIDTg$DWdjdNHmnI$(da&dzdg%+eFmZS=~ zaj_*Pmt^MWOJ}eAOOsqo8yi6Qi++e3LfjCMaIgk{i5Hl}a z9}}|>GcQCR3j-4~s}M6k69<x@hc%tFkPP_HvGF>nYmOG3R4;z7JF3H3T76CDbtXlyPnkhZ zR%gN>hm9utcYvnCT4mk$>U3o{dw5VPj=HS1aYv^1fKpBe0Z zO(94IWM%?oIZahHE)y;$78W5ET?VV^DjQf#>UBAU!6veRJ*WruAPdNYdQ1*5DMle? zJx)ljVgY+lPXrPjEMO1niP?kw2jW3eg`T(_R4*u1=);X=6k^tAa)n7T3e_{~bBe*Z z%tFlil8}gEVPfPEV%C>-ftt$*P1yP}&QR5$!o~n@E|UjA$ma&K za!@%?VPgo_#%v|TYzWr}if}`?K2U*U2-gQHSd5uCxPrJ?SXe-TZ#;d%MwXCzV`u@z z0!mKI#?S(ag$Y!=n858}6=F7Ff~0;HCRRovW)rwJR%Rh)6Sy`|>0t`j#wNsU$|MPO zIjHn7<%A?h7I5idDlSnEl>`+ZW^jG%Ld<4xee8@v%w}+XpyIItr!5sVNF2`3fnS#U+_}>7|*)8K5dDTcJ3= zv?w`MAvr%K6;j!-FmW;pFq7hCLv~9NQ|>GfXXapJ0=OJ zLPjBGJ0@_w2a;kEVzvX9!S$@H4B(t*&m;|1$t1*V&!hl~7myT)Cnpbb6AP;lixY!C z*i9^;G~mPm$wRE*tmh1k23AnkbB1O;R4z2UI~R6-kTY3W!3DC55G1l$!S%Q++&N4_%&tt3YKj%? zRaa<^Wo2Swt`}l!8eF_wtRSTvLM-l@-Yi@KTpVC-JEyP^i@TOLGt@2?j&@FACLv~z>5N-h zgzG&;6u5-AxM50J_*jLQJ(cv4#aKb*fS0H=su;5nvzM$X)D57V?*k7#Rv~5|CJ&et zqY$$XJoG?0-v=IetQ_@1%s$fYP^W=g79h@#V^OOQf;KGJ!09*`>O?kBIu2%nR2ppHbR5jd z12&2coQ{L}!C8`p4V+9v;My34m_wK#Ns$elOhcemD;qeOh6oCQa}&D|OE^OUBNtaa z7c&dUq2U~mw9O87Mg-J6c91h7poJkj*clPf!jK*8j0k99$PRW!BwQP#5OX9Gq%dR$ zJ0lWW7_x(%5eY2}*_rA=HBl5?ACnMs6x=#co{NH82g(vraO*%dQ8Zi|vk-GM6C{7I zGl6nTG$*8VUc5sCi3oV1#nLx=l7Fq_eGl7$B zEVK*)aUtf$Ldzg_aCt&itJ46g3Ln9@v?GU0bC$IvLrKPO@F$L#iyQwQHVJi+PdKY zr->A(GdMtLA_eL_4se=CfqIVvoF-DB-s1ojvCJt@-+_1#-=#o($HBzNBE+1^#07RH z2Pn2vp}ym&X9BhCQlZT;4sa4m6@qw{1Du3Xp`PUc7r1F~$1w>pr$JK?2e`mZgQg%3 za4Rkinu0jM9jA1-HfAB_bZCO*U}9#h7h+C_rdSSe5=w`rSPpO!$^d7178VXrTPA}E z(#Yfhr=SdWNO8&mPC*&!8sOB$!NkfU#GDEDD=Ubj4U=LMV$S4-G!Hny=_RvXLJcO# zEX16tsRhopoI)%)40(_`5LCVAa6k$OPH+U}LOsI?ilAI*1aX2RC>I(*oZtw`g+>r3 zID&Ga5d`8vA}Ckfj+@Jhi;0Cxh^3IBc>3oZEN1mwj6%$X&_aR>Y+n)7J}!`bMNs>= z!1fhE?c)O5R|K_>3v6Ez)IJaoVqcNC9hW*66AQNxOBq8Yqm;Bla$;U`Y7VG3T~wNv z2kLB3U$K*gx1O6(h`9_}@pFS6Q4VzkH^>p?P)Bfs9Z?Q-1UJ|b_y& ztn~~{j9?2{85xC`>siG>H8(4`8_Lqa2yR8OfVe^+o+?a|3B*-afpQrcg;*LHnLwRv zkXi^&5hlq5l9UHE$Cy}|gxKm^7#X@48M(N*m_ZF9mKH`7|aaj!3<_*0&(F6gL?O!jNqmj$Pp|;ES;br?`-@=HL>!?im{^5aCe<@Cu|rK~5@MOe$OKMtEWDs{mSqwnhagNb zGe{Pkl9^cegxIDsGR%M^8x}rBA(pAI(BlK;4VGyzH}f$u2!VLkU}N~eX=)lHCl8Fv zTo01w7l83OKwLp9u%r2y7(k6PCXkSjB~%3{M5ZH5XB1+Y&d6i~lLT`)A)&_y%F8U% z83iC--~)A_S*9}zSwjs6hX|7p$N*t0E>FUvwk5m*p0vk9>*Vq^l1PqMJ^GckjC^3V_g<>N(+e2`4T52}Az z7BR|#GcpT5r~+nLjPMhS5KH}HMkWuaUQoVX%*Y8znf#zim}N1eB+Mul4k4DsjMDDl zXys=DbzwlBTFfZp2G!5XD#Wq`VLF8836o?5Npiwsj+Gh2m6EQ9NppbsG9FOVp}i26 zC5*D}P!(*fLM%%enb^R&h@XiK%+rKPGJ?45;G&s@AJh|JS<0vmYCwRRjch9z8P-8^ z5i2NwRxm;fd}gM421X&4m5fZFv1Arj(D)jNrv@?>#A5#o=Sarb+>Xxo$WP(%|%;3^@EvUNS;bLMD5MtZF z$gl(AaTWnaA(joW;zIzO(Ka$N)k8{O0Z>lc$jGz>n$rZpIc*~&wDc7KRj@1@LB0c( zz5?K!wvkbO3pA&J1i(3MBcs9=XikHQF$uA30_QX?E*7W|D5GtLX0&=10dP*+3QIQv zpq#dqk;w|=U63TB5X)9ZPHq^N8N}t|hjBSTTmegPvJqf{mNhI}83iq%DnO<0Hc(Cj zWi$a$M%xA}$OM>}80&>twlQ)*3UdK)PTR&P3dsQiOiUagS#d~72I97Jf(BEzF-pK( z$IL3kvK`@cW-t$?9hB6zGjc-8MFDVL+z!gdnp{jQf>#MY&#(QHW(X zBL~D+g5aWk56o0SP|?1Jk;wtdmXRO^Kqg;>tP@`x}v9?!#q zS{M|M=NXx7!7dO6$K!ceJPL#3@jNUZg~9Q7o>ABa9FHI_B&5$Xia^>e!k}&~%LPUz z&_FB;i!dm0UWj94@`9Sq$R@;cfsq@MIEBH{dI8oL5C%u<1yExE6z0OUFEpd&2a^&*TyELRvgAf=25*pF9XeiQ-u@hZ%ZB49sW zh51ng?8mDxKZ=0;copVH5EtUdt1v%`fYa(VgyD=rEZ6EmEpL#sMZliA#>ffDaUx(( zU1J3IyqQ=;h1hN~GJJy;=%S25EH@#kkeNl4U5M=#Bhw>B7A`?9b{0|gc1{)sxDd!_ zQE-v}1eWtfLB4vz$OJC>>RCj=-g*MduA*RXJ%KgqL_uQ;EKgyM76nDjQ${9;BSpb( ze#*!RDZWI(Zhp!r1Zn1rf)oEUMkdfCAPb8qDDgi7)wd8gflPYFSkDhh{i5I!_!%e} zLZm@$o#(K&jwq?n^>Zt_65sJ zgtcsJLM*QsnYh8t6j4yde+6nWK-|Q}2$JN1wB$s=&6HP+5|Ui?5NUQcA(q#OkYWe( zAjO0z6FVb_3lAuEW)K%zOo%dZunDoeLHL#f%!B!sgAv4q`<8a0iq{l!=>7h~+&a6KFu41(g1{!CY7x;${SK**KwkxtT#+9!MWdl!=E;h~)z# z6DKr~dB8lFJv@vcE*CgqvxqYBFoU?FkcOuy6EB+(%O^w_^MZNxFeih0sGk`5VMg&X zgQVe3=3^6L`HTo;J}?hb_=+;|F@m@Pkmj-|6CX2(D=QD`@w13>3bA~J=NC>;{qdEN z8`6Ro3|u}pGC}+6VxW?& zk%=i7oL0oZq0-322?v|Z$RrU2uB<`ac1}K_dX`2e$v|*T0Tlub z-!NG~di78tW+9eFCYJ!HEugxni3!?K6=Px)VrybzYGr~9UWkE%yNO8~}0u=(apIexq zgBN1pzGNFBFhGMqOiY1LXM!TSjS1QY6$2NqZA{Rvq!_q?+r}go0GiF@U;*(#p~2F| zB<>Fl3?}H%KpT^p1F{ga5K9}AlONO;P&2q4;RI$OmUbp+w^9t8K-!t0JwGvUG2PB2 z1t~$qn3y?)Sla8EWFY-Y5Wk%hly};hWFh@ZF>tT717SLg5K9LWw81I{PEs9A(6I zk%?g@G&_oehQyelgEr#e>^KP)gyNv=I0=>=#lhKe5-dB4gR|o#SauW#XU9oQ_0aqX z;)7G*Bv^KYih#z^!&0C)I5SR$r9g2| zW~^tK3`>C^J|r3^!&0C)IEzhTf{s0igPJf?n3y01j5v6-W(pHKq;3)iXR#?v(6JzK z&|C@2RD{Erg+M%eaJ~@-XN{>$ocxgZ7YFB!sZ7w8lQ=kUOl1M-J%__t)orww3co7F@jpT<#6iiPWd@TNr0@lCA;V`gn8Y2E*ubj_jW2y;X5nCvU;qHX0Nv&Q delta 8677 zcmbQRU3lLD;f5B*7N!>FEi9>1nXZ>k&zQ>M%~(6#u7bsQy3aHgukG)qvHWCYJhc7A z3>Ig`?f$b^qL`RoGETogm*wSlzWFQ`%+r4_VhLt?!8ko&G0U~>nM+vwIj5JdW4Xuq z%9D#-h?Rli^Yr=6ETYp3*RyCd3QV87o~4q@l#7Lhfr(LwnP>XL4J^ibj36c_FBcCN zD@dGKh?z$~5Xxg>5n|?J;^MLg>tPaN=9})ck;S&2iB*W1j~lLvQHYsONCKvZS%{fW zS_aBvW)))QXX4=E2AjYP;@HBZ7=@VmIr(5bker|pjK?9w%r9)iCBVhW!obAL(ay=o zB*e@wV$EgD#l*tKD8wSnAU1u{CKls*VGf89j7$uyLd+siyBL`mKpco&j7$uSLd+si zyBL`mn1z@{pms4bF>nYmi$LuH@gR1Ih**Q&!pOwPD#R=bH`6;Oi#U+U)sa#xKT+A%Yj6%!`91x?K!7fsS`i>doB1I+(u&vDC zKvRSU8Z$W16rq8}3=TBK6k&6441u`qoNO#Y%!(prP|YAGD={g8eaZ}SvJ#UXOo~y6 zSxHF~#$y&@R?^Xh@<1t0g^7#Hhl_=UnTbh=S!FuURu(@k6=-T>276ycNC~E%S%_Ih zRgKGpi;0Crh(&|JczVlL7L$4n4q>ng7O)33p&n!bc~Fzd0Vc&L#H`5)NyaQ-4{C}) zqJstOK}|7xu>U|jNUG2jw}a{hr3x*$v5Z2@T1>7mDMq1sW-U%J7?)XySxXWUQ7lZ1 z974=m(k@VQ8KEg#OU45o)!=xC6n6)`!1~D-UF>A}pLFGXCOb4!w z*-D662d)nk;W}`A%*;Z}I&gg~tU}CsOdMQ6Tr4aspupFg{$U$SNWC7kL|_3WCuTip ziNM0d$|}UH54Q)*fuw#GCRRovW_`FeR%Rh)eYiF@Rv~5sxHdKrM-u9CHbx<415QYC zWMN`s7GgFKm#BwIva<>?8^ZOmgE%mK?2JOphH!oC%tFkD;*c_kg^7bzh}j6PjRVAi zOEC&D8*#!C2M4ndvyqrM7Z(>3D}xY=DT5^=D37q#g9;2&R&Z`)WdP>{GbSleVg~Vq zn9U%83*s;dF`Gexla+x9RG6AGNkA1c3Nf2A$%4EBl425KHiwjptPJ2(Y{4WARmdd7 zY{8@eRR~J;^%ipSAV;yV3b9x-cz_+n0t#Sj4oCuJ1*dBps2^ED>DmUGu35p(wSlH< zR&ctufu?I#aJsgErfU!nlCEu_>6#T>+SxMILy9O?P{y@ofy*@2abnN5hs|l zXJKJwVi96-U~*xF$uSBsJBX^lc+5h~4(b|Qyj&nVSvZ7P95h|(S-1qaI6yi%+Bt=V zSRAxmn4xyDfJB&tm>pRJxrDg5VQN_TScI6JL=;fPScRCKl=P9sSV1|?SyUQTj9G}; zS=JQlB34$RdS+L6Sh5N+yE1ve;BVs@2whq{v$RJ4Hf%D6%G zf-<2S+*mdtW;Z5Jm=vQBvl}O*T4QBmW3Crsc9Vj|9~*}dvzv?u)Lb^Gxo)!VP|cuB z>JF{cSee*`nBAGQU{Z`i%^d@MDycQb<%N&df{CNmVG# z%gIm9PEAoLPMsdRhlQ`+8(OuqffKtA)ah)X#O}icDYn_biQR{j2W$l!II;WigYzv5 z8#poh!nH99G5az>(kmM{E&D<%H#Tru_7xQ33gBX5VHaWvWC&-Ja+&^XAB$%_sGJFe z)+X#=hXg@QWCuAU2%78I!43(6<~nw;LxP~Wjvbuqf}pt$#DnCzAZV^*XJTX#Vh(2F z0%rquP<{z!f|Ts+^-Q3eG#Fa_v4hK}t(@CT7NZA?8qMamfzOgP~%OWWvq_O37i+ zcxMOI9brt6N}ZjFg;9t(j2)82*uhmun7RhIK;sZ%iDZa|6lffbLd=mIkci{}2W=G8 z862RXjRFO&D;FyZ2RLk_pkd1a4%;Yb*m8iwHVPWHARZ)aqr~mFthkt1IE7f^7!sy? z9b_@9=VTOOj)S&*IKlSCL+#@P*%uG8kBgOs6Kq{P)H+VEb@5Q^IKkG%L#+exAlAi; z+i|ILF|lw7u_QC3F-l1*Bq!!2r{)wZBq|h@=H+GPr7KMTe~^W@o{Le4IT>1tbAcU^ z0(Arz$Pp<}M{t21kpguD7uXRgP)Bfq9gzZc1c(Q5L<%%Lae)(lD%@B`A?8$YS;)%5 zRS!-JOsF%t!OqNtIupce=j5wr5@OC2w*%S7$|%H=!;lZQj}=sN=CFd>OstUhdM=X! zC_jLBLd>~L;6en%VG?4_RR$+5Rz|RA@|a|x3K@l%^OzJtMFB{PNr*X5UIA%r22w0iG?k{4J1)BhFk{2<78TZHX-IJCSk669xfIZ9wr7MmMW%NNR7w?4udKYNKMNFj>9ThIdF~01ByeID*0MS zg$Uxda|#HtR4LR#DnzIVlMr(?ix`(67Z=EIj&@D~76u{a8Wu>uk_Vh1>p%&p9&A0M z5OW<9q{+(z4%j+qQOpBQkaf_01`oKoTqo=Vu6sbdc1}K4A?7*}M^GfQ@PLcFdM0*I zq=0xr%=Jv*Gz8)>3NhD1Q!o#>$g39-1^J(eg;$8JiIJg|k&%m=i>6fY>Gni-j_VUmnOEX|CZJTNXZh|4bk<8pwwf>vCDT%0VtObno^ zk_jXvWC>Nln8+%`(tjbMbSrf>d*Wl*{xmvTzA;aexItiGZz>QMQMX8SETBuxACBgjl*5 zS;V-6>bbaJ>R1?rSh^WmAfDm_r3980@MK2Sqc;pG;t`7)+KC#N~j5Cm$$Du=FvCz=DXGO^BtR zkqMMyL0Xx?Jb9>AW=0T~PYT8b%R=)AAE+i|nSk&Uix5lw1V$zgsA3jIA(jb@oG@2_ z`sWiEC1FN^TAULYrQM;P0yQuBSV0l}B*3J*kS<_)2 z;Rofc>5NPcFiA!smg$U~kjUW&6+$f2VUfcRE`_EuirItx0pfydJeKK<;&xE|jI2T| zGeA)f%C`KVbTz}DkqIu#2$JQ1WK@1oA;U6*QCI|~mjfg#Y6~?Tl(6`jKtf_RT+Uof zECNDoa~K(xPG5S4#jaj}QHW&@BL^fs3xGW}7v?Dekf-J{GFgBP7628jEOQy5=~)2m zskw~M^eh1O)LcekbFj4_E_e`yWiF$L8B{;WQ}YmpGYYZHtA_=d0I1+(nFk9p0kEg$ z!GcTxRCKb;g9RCg3kkA$pdbU40ss`h}+J|D8y0^76SE_1eidnWf3C_s4c<5BEZBX#IlHyMG;~es2XHh#3&#G<1&L} zgkKlv8dJy zG77P*VB~<55Q5;ySP2U+K~Q9@WMr}hn<@y7jFqs+5CliYN?2qFf+J%kqp%G)GC*8N zaIIt%ft3A%phh~&Dn=$Qa83~frLa|Tj7(lo(;3->SXMD|LsFO^IBr%kLQ7OZaNMk7 zl#+%i<^U;{@q}6iN^nAqAR$=~a34fSh;1Ds!^Y`L&a-&egJjk*a&Uw73W5E&9_B|O zkRR8>{3rzW<9e7Mg}{DX5A&lC*pKUBegttLeq0apqY$`s-heQiQHW(jJtLDO)M`-i zyn&Gul5K>*p4z|&?nf}O2n(@oW@NYkty6^=g;+L2k`gnEFuM@j7DlE$j4WJ&T*$abH@Bv09&=4{Ui!dk!A7EsHq+nri3O>MC z&ksq$!r%(y03$RB3xmckSPsG}3}Gf_ArKEzVF-hob}R=Op%sQOsOQ9TkP%v82s5#; z39%evWP)TIVbJ)*Ax0);a3~9dI!`Qz7zH2=Fkz;8P=#@bQBesh&B`Xkau{JPE0_nf z7SyIbjIb6|9UMkj%f=?ea)gnI8=QiLLAmz`BNL<|5@rI8svlwGfmDaWppiJ1Ba9M~ zT=kF$VrLU#If@7=b}$bXQtXT%ES6O2rtp#f0ba)EiU zP~~C-ad{=6fy%`U;!4Xxx!i0*EGHS6K<#4|Q2OHrb75(Sn-RoiCd6_U5y*UC9;5^nW&*Va&oTLg;%d>)-NS7E{!75q7 zvj;4f8Nodt77$km!~>^c5RVDORRAX+R%UP>y#gv`c(_1n8HHG`FfxI?&C1MF&&UMg z%YfsC6+8jJaup;EQOqO+;(=2WND|D2Bp+60a4x*Y2p-M=8O$uiat-7pUM^OUBol}$ z4Ngp~%;0Qy9pMNTA(rbPTX^fiia`@P*BQYbGA0%gA+}qL3@;%bXAxl(V!6f00V!}q zz~%F8SZO2zD#>m$G6jRviU_zMyUoZ64kQ*4a6xvPQ4A9QBH;4*HlsukxW)x>+d27! z>RE0xN(O>!3aAj15X)^w3rI5pH0T-_iVC@$XCMFJ$VzGdFXlO8j zdQ%`FaeruFfJMMYnmHf~F@sb&`9W=AW))(2h;RZkmJ&lp7%xKIRHg;<_3D(NE&va-~(3b8zA6qQC5Wd;d?hs>E+M1|O1GctUJW=By* zA(q#$p06l4JHCMhp(rRjzJX;&QE+yA1Iv!0;OzJYmK{aG+3^iyJv2Xp_@MmA@&=Y2 zp(0E|EN@`h5h}zC5`tz&QE;Q`Ey4+mLM(4#y+cuOW_$}vfui8d_!gD|MM0Udp5-kp z1%mjHXnYGxfui6n_72v869qM4-Z3&k3K&sv6XqQwJEU$B1!u8$u=b!RXyljWJ;Gtk zLLi%g_0-}8?yyS8ahfK n%EZP2;)+2EUl12OBE|BFQQR?!t)M8sB;WYbH)a+N1_=fLody*q diff --git a/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs b/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs new file mode 100644 index 00000000..60b1bcec --- /dev/null +++ b/shuttle/src/gen/spindle/mill/v1/spindle.mill.v1.rs @@ -0,0 +1,380 @@ +// @generated +// This file is @generated by prost-build. +/// executor identity, sent on connect +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Hello { + #[prost(uint32, tag = "1")] + pub protocol_version: u32, + /// GOARCH of the node, informational only + #[prost(string, tag = "2")] + pub arch: ::prost::alloc::string::String, + /// operator-defined labels, matched against runs_on + #[prost(string, repeated, tag = "3")] + pub labels: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "4")] + pub epoch: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub cache_store_id: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub cache_namespace: ::prost::alloc::string::String, +} +/// reconnect state for an existing epoch +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Resume { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub ack_seqno: u64, +} +/// per-engine state of a node +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EngineAvailability { + #[prost(bool, tag = "1")] + pub available: bool, + /// opaque engine-defined load metrics, higher means more loaded + #[prost(map = "string, double", tag = "2")] + pub load: ::std::collections::HashMap<::prost::alloc::string::String, f64>, +} +/// full node state +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NodeSnapshot { + #[prost(uint64, tag = "1")] + pub seqno: u64, + #[prost(map = "string, message", tag = "2")] + pub engines: ::std::collections::HashMap<::prost::alloc::string::String, EngineAvailability>, + /// every lease the executor currently holds, reserved or running + #[prost(string, repeated, tag = "3")] + pub active_lease_ids: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// a seat reservation, carries no secrets +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReserveSeat { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub target_engine: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub raw_pipeline_json: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub raw_workflow_json: ::prost::alloc::string::String, + /// pipeline id, the executor reconstructs the exact WorkflowId from it + #[prost(string, tag = "5")] + pub knot: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub rkey: ::prost::alloc::string::String, + #[prost(uint32, tag = "7")] + pub ttl_seconds: u32, + /// repository that owns the pipeline, used to distinguish fork source code + #[prost(string, tag = "8")] + pub repo_did: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReserveResult { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(bool, tag = "2")] + pub accepted: bool, + #[prost(string, tag = "3")] + pub reject_reason: ::prost::alloc::string::String, + #[prost(enumeration = "RejectClass", tag = "4")] + pub reject_class: i32, +} +/// a single unlocked secret +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Secret { + #[prost(string, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub value: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheBinding { + #[prost(uint32, tag = "1")] + pub entry_index: u32, + #[prost(string, tag = "2")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub restore_id: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub restore_key: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub restore_name: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub save_id: ::prost::alloc::string::String, + #[prost(string, tag = "7")] + pub save_key: ::prost::alloc::string::String, +} +/// promotes a reservation to a running job and hands over the secrets +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommitLease { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub secrets: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub cache_bindings: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Committed { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +/// drops a reservation that was never committed +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReleaseLease { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +/// cancels a running attempt +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelAttempt { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub reason: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelAck { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct StatusEvent { + #[prost(enumeration = "NonterminalStatus", tag = "1")] + pub status: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + #[prost(int64, tag = "3")] + pub exit_code: i64, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LogArtifact { + #[prost(string, tag = "1")] + pub r#ref: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub hash: ::prost::alloc::string::String, +} +/// terminal outcome of an attempt +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AttemptResult { + #[prost(enumeration = "TerminalStatus", tag = "1")] + pub status: i32, + #[prost(string, tag = "2")] + pub error: ::prost::alloc::string::String, + #[prost(int64, tag = "3")] + pub exit_code: i64, + #[prost(message, optional, tag = "4")] + pub log_artifact: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CacheUpdate { + #[prost(enumeration = "CacheUpdateAction", tag = "1")] + pub action: i32, + #[prost(string, tag = "2")] + pub id: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub r#ref: ::prost::alloc::string::String, + #[prost(int64, tag = "4")] + pub size_bytes: i64, +} +/// live non-replay log frame +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LiveLog { + #[prost(string, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(bytes = "bytes", tag = "2")] + pub raw_json: ::prost::bytes::Bytes, +} +/// one event in the executor's stream back to the mill +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Event { + #[prost(uint64, tag = "1")] + pub seqno: u64, + #[prost(string, tag = "2")] + pub lease_id: ::prost::alloc::string::String, + #[prost(oneof = "event::Payload", tags = "3, 4, 5")] + pub payload: ::core::option::Option, +} +/// Nested message and enum types in `Event`. +pub mod event { + #[derive(Clone, PartialEq, Eq, Hash, ::prost::Oneof)] + pub enum Payload { + #[prost(message, tag = "3")] + StatusEvent(super::StatusEvent), + #[prost(message, tag = "4")] + AttemptResult(super::AttemptResult), + #[prost(message, tag = "5")] + CacheUpdate(super::CacheUpdate), + } +} +/// a flushed bundle of events +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct EventBatch { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub events: ::prost::alloc::vec::Vec, +} +/// all events up to and including up_to_seqno are durably processed +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct Ack { + #[prost(string, tag = "1")] + pub epoch: ::prost::alloc::string::String, + #[prost(uint64, tag = "2")] + pub up_to_seqno: u64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Message { + #[prost(message, optional, tag = "1")] + pub hello: ::core::option::Option, + #[prost(message, optional, tag = "2")] + pub resume: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub node_snapshot: ::core::option::Option, + #[prost(message, optional, tag = "4")] + pub reserve_seat: ::core::option::Option, + #[prost(message, optional, tag = "5")] + pub reserve_result: ::core::option::Option, + #[prost(message, optional, tag = "6")] + pub commit_lease: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub committed: ::core::option::Option, + #[prost(message, optional, tag = "8")] + pub release_lease: ::core::option::Option, + #[prost(message, optional, tag = "9")] + pub cancel_attempt: ::core::option::Option, + #[prost(message, optional, tag = "10")] + pub cancel_ack: ::core::option::Option, + #[prost(message, optional, tag = "11")] + pub event_batch: ::core::option::Option, + #[prost(message, optional, tag = "12")] + pub ack: ::core::option::Option, + #[prost(message, optional, tag = "13")] + pub live_log: ::core::option::Option, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum RejectClass { + Unspecified = 0, + Transient = 1, + Incompatible = 2, +} +impl RejectClass { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "REJECT_CLASS_UNSPECIFIED", + Self::Transient => "REJECT_CLASS_TRANSIENT", + Self::Incompatible => "REJECT_CLASS_INCOMPATIBLE", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "REJECT_CLASS_UNSPECIFIED" => Some(Self::Unspecified), + "REJECT_CLASS_TRANSIENT" => Some(Self::Transient), + "REJECT_CLASS_INCOMPATIBLE" => Some(Self::Incompatible), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum NonterminalStatus { + Unspecified = 0, + Running = 1, +} +impl NonterminalStatus { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "NONTERMINAL_STATUS_UNSPECIFIED", + Self::Running => "RUNNING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "NONTERMINAL_STATUS_UNSPECIFIED" => Some(Self::Unspecified), + "RUNNING" => Some(Self::Running), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum TerminalStatus { + Unspecified = 0, + Success = 1, + Failed = 2, + Timeout = 3, + Cancelled = 4, +} +impl TerminalStatus { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "TERMINAL_STATUS_UNSPECIFIED", + Self::Success => "SUCCESS", + Self::Failed => "FAILED", + Self::Timeout => "TIMEOUT", + Self::Cancelled => "CANCELLED", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "TERMINAL_STATUS_UNSPECIFIED" => Some(Self::Unspecified), + "SUCCESS" => Some(Self::Success), + "FAILED" => Some(Self::Failed), + "TIMEOUT" => Some(Self::Timeout), + "CANCELLED" => Some(Self::Cancelled), + _ => None, + } + } +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] +#[repr(i32)] +pub enum CacheUpdateAction { + Unspecified = 0, + CacheUsed = 1, + CacheStored = 2, + CacheDiscarded = 3, + CacheMissing = 4, +} +impl CacheUpdateAction { + /// String value of the enum field names used in the ProtoBuf definition. + /// + /// The values are not transformed in any way and thus are considered stable + /// (if the ProtoBuf definition does not change) and safe for programmatic use. + pub fn as_str_name(&self) -> &'static str { + match self { + Self::Unspecified => "CACHE_UPDATE_ACTION_UNSPECIFIED", + Self::CacheUsed => "CACHE_USED", + Self::CacheStored => "CACHE_STORED", + Self::CacheDiscarded => "CACHE_DISCARDED", + Self::CacheMissing => "CACHE_MISSING", + } + } + /// Creates an enum from field names used in the ProtoBuf definition. + pub fn from_str_name(value: &str) -> ::core::option::Option { + match value { + "CACHE_UPDATE_ACTION_UNSPECIFIED" => Some(Self::Unspecified), + "CACHE_USED" => Some(Self::CacheUsed), + "CACHE_STORED" => Some(Self::CacheStored), + "CACHE_DISCARDED" => Some(Self::CacheDiscarded), + "CACHE_MISSING" => Some(Self::CacheMissing), + _ => None, + } + } +} +// @@protoc_insertion_point(module) diff --git a/spindle/config/config.go b/spindle/config/config.go index 2f86ccc3..c791b05e 100644 --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -79,6 +79,18 @@ type LegacyS3 struct { LogBucket string `env:"LOG_BUCKET"` } +type Cache struct { + Backend string `env:"BACKEND"` // "disk" or "s3" + DiskDir string `env:"DISK_DIR"` + S3Bucket string `env:"S3_BUCKET"` + S3Prefix string `env:"S3_PREFIX, default=spindle/cache"` + StoreID string `env:"STORE_ID"` + Retention time.Duration `env:"RETENTION, default=720h"` + PruneInterval time.Duration `env:"PRUNE_INTERVAL, default=1h"` + MaxBytesPerOwner int64 `env:"MAX_BYTES_PER_OWNER, default=10737418240"` + MaxEntriesPerOwner int64 `env:"MAX_ENTRIES_PER_OWNER, default=1000"` +} + type MicroVMPipelines struct { ImageDir string `env:"IMAGE_DIR"` OverlayDir string `env:"OVERLAY_DIR"` // where microVM temporary disks will live @@ -205,6 +217,7 @@ type Config struct { Mill Mill `env:",prefix=SPINDLE_MILL_"` Tracing Tracing `env:",prefix=SPINDLE_TRACING_"` Logging Logging `env:",prefix=SPINDLE_LOGGING_"` + Cache Cache `env:",prefix=SPINDLE_CACHE_"` } func (c *Config) validate() error { diff --git a/spindle/config/config_test.go b/spindle/config/config_test.go index e269e61e..300b8dcc 100644 --- a/spindle/config/config_test.go +++ b/spindle/config/config_test.go @@ -381,3 +381,17 @@ func TestLoadQuotaDefaultsRejectsNegativeLimits(t *testing.T) { t.Fatal("negative quota limit accepted") } } + +func TestLoadCacheStoreID(t *testing.T) { + t.Setenv("SPINDLE_SERVER_HOSTNAME", "spindle.example.com") + t.Setenv("SPINDLE_SERVER_OWNER", "did:web:spindle.example.com") + t.Setenv("SPINDLE_CACHE_STORE_ID", "shared-cache") + + cfg, err := Load(context.Background()) + if err != nil { + t.Fatal(err) + } + if cfg.Cache.StoreID != "shared-cache" { + t.Fatalf("cache store ID = %q, want shared-cache", cfg.Cache.StoreID) + } +} diff --git a/spindle/db/cache.go b/spindle/db/cache.go new file mode 100644 index 00000000..12afd3ab --- /dev/null +++ b/spindle/db/cache.go @@ -0,0 +1,619 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +type CacheEntry struct { + ID string + StorageKey string + OwnerDID string + RepoDID string + Engine string + CacheKey string + CacheHash string + Checksum string + SizeBytes int64 + RestoreCount int64 + State string + CreatedAt time.Time + LastUsedAt time.Time +} + +const cacheEntryColumns = ` + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, + size_bytes, restore_count, state, created_at, last_used_at` + +func (d *DB) InsertCacheEntry(ctx context.Context, entry CacheEntry) error { + _, err := d.ExecContext(ctx, ` + insert into cache_entries ( + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, + size_bytes, restore_count, state, created_at, last_used_at + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + entry.ID, + entry.StorageKey, + entry.OwnerDID, + entry.RepoDID, + entry.Engine, + entry.CacheKey, + entry.CacheHash, + entry.Checksum, + entry.SizeBytes, + entry.RestoreCount, + entry.State, + entry.CreatedAt.UnixNano(), + entry.LastUsedAt.UnixNano(), + ) + return err +} +func (d *DB) InsertCacheEntryWithinQuota(ctx context.Context, entry CacheEntry, maxEntries int64) (bool, error) { + result, err := d.ExecContext(ctx, ` + insert into cache_entries ( + id, storage_key, owner_did, repo_did, engine, cache_key, cache_hash, checksum, + size_bytes, restore_count, state, created_at, last_used_at + ) + select ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + where ? <= 0 or ( + select count(*) from cache_entries + where owner_did = ? and state in ('pending', 'ready', 'deleting') + ) < ?`, + entry.ID, + entry.StorageKey, + entry.OwnerDID, + entry.RepoDID, + entry.Engine, + entry.CacheKey, + entry.CacheHash, + entry.Checksum, + entry.SizeBytes, + entry.RestoreCount, + entry.State, + entry.CreatedAt.UnixNano(), + entry.LastUsedAt.UnixNano(), + maxEntries, + entry.OwnerDID, + maxEntries, + ) + if err != nil { + return false, err + } + inserted, err := result.RowsAffected() + return inserted == 1, err +} + +func (d *DB) MarkCacheEntryReady(ctx context.Context, id string, sizeBytes, maxBytes int64, now time.Time) ([]CacheEntry, error) { + return d.MarkCacheEntryReadyWithChecksum(ctx, id, sizeBytes, "", maxBytes, now) +} + +func (d *DB) MarkCacheEntryReadyWithChecksum(ctx context.Context, id string, sizeBytes int64, checksum string, maxBytes int64, now time.Time) ([]CacheEntry, error) { + tx, err := d.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer tx.Rollback() + superseded, ready, err := markCacheEntryReady(ctx, tx, id, sizeBytes, checksum, maxBytes, now) + if err != nil { + return nil, err + } + if !ready { + return nil, sql.ErrNoRows + } + if err := tx.Commit(); err != nil { + return nil, err + } + return superseded, nil +} + +type cacheEntryTx interface { + QueryRowContext(context.Context, string, ...any) *sql.Row + QueryContext(context.Context, string, ...any) (*sql.Rows, error) +} + +func markCacheEntryReady(ctx context.Context, tx cacheEntryTx, id string, sizeBytes int64, checksum string, maxBytes int64, now time.Time) ([]CacheEntry, bool, error) { + current, err := scanCacheEntry(tx.QueryRowContext(ctx, `select `+cacheEntryColumns+` from cache_entries where id = ?`, id)) + if err == sql.ErrNoRows { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if current.State != "pending" { + if current.State == "ready" { + if current.SizeBytes != sizeBytes { + return nil, false, fmt.Errorf("cache entry %q size changed from %d to %d", id, current.SizeBytes, sizeBytes) + } + return nil, true, nil + } + return nil, false, nil + } + + // A replacement frees its old generation. Exclude it while deciding how + // much room is needed, then evict cold unrelated entries if necessary. + rows, err := tx.QueryContext(ctx, ` + update cache_entries + set state = 'deleting' + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? + and state = 'ready' + returning `+cacheEntryColumns, current.RepoDID, current.Engine, current.CacheKey, current.CacheHash) + if err != nil { + return nil, false, err + } + var superseded []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + rows.Close() + return nil, false, err + } + superseded = append(superseded, *entry) + } + if err := rows.Close(); err != nil { + return nil, false, err + } + if err := rows.Err(); err != nil { + return nil, false, err + } + + if maxBytes > 0 { + var used int64 + if err := tx.QueryRowContext(ctx, ` + select coalesce(sum(size_bytes), 0) from cache_entries + where owner_did = ? and state in ('pending', 'ready', 'deleting') + and id <> ? + and not (repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state = 'deleting')`, + current.OwnerDID, current.ID, current.RepoDID, current.Engine, current.CacheKey, current.CacheHash).Scan(&used); err != nil { + return nil, false, err + } + need := used + sizeBytes - maxBytes + if need > 0 { + rows, err := tx.QueryContext(ctx, ` + select `+cacheEntryColumns+` from cache_entries + where owner_did = ? and state = 'ready' + and not (repo_did = ? and engine = ? and cache_key = ? and cache_hash = ?) + order by last_used_at, created_at`, + current.OwnerDID, current.RepoDID, current.Engine, current.CacheKey, current.CacheHash) + if err != nil { + return nil, false, err + } + var candidates []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + rows.Close() + return nil, false, err + } + candidates = append(candidates, *entry) + } + if err := rows.Close(); err != nil { + return nil, false, err + } + if err := rows.Err(); err != nil { + return nil, false, err + } + for _, entry := range candidates { + if need <= 0 { + break + } + claimed, err := tx.QueryContext(ctx, ` + update cache_entries set state = 'deleting' + where id = ? and state = 'ready' + returning `+cacheEntryColumns, entry.ID) + if err != nil { + return nil, false, err + } + if claimed.Next() { + deleted, scanErr := scanCacheEntry(claimed) + claimed.Close() + if scanErr != nil { + return nil, false, scanErr + } + superseded = append(superseded, *deleted) + need -= deleted.SizeBytes + } else { + claimed.Close() + } + } + if need > 0 { + return nil, false, nil + } + } + } + + updated, err := tx.QueryContext(ctx, ` + update cache_entries + set state = 'ready', checksum = ?, size_bytes = ?, last_used_at = ? + where id = ? and state = 'pending' + returning id`, checksum, sizeBytes, now.UnixNano(), id) + if err != nil { + return nil, false, err + } + ready := updated.Next() + updated.Close() + return superseded, ready, nil +} + +func (tx *EventBatchTx) MarkCacheEntryReady(ctx context.Context, id string, sizeBytes, maxBytes int64, now time.Time) ([]CacheEntry, bool, error) { + return markCacheEntryReady(ctx, tx.tx, id, sizeBytes, "", maxBytes, now) +} +func (tx *EventBatchTx) MarkCacheEntryReadyWithChecksum(ctx context.Context, id string, sizeBytes int64, checksum string, maxBytes int64, now time.Time) ([]CacheEntry, bool, error) { + return markCacheEntryReady(ctx, tx.tx, id, sizeBytes, checksum, maxBytes, now) +} + +// ReserveCacheEntryBytes atomically reserves the owner's currently available +// byte budget in a pending row before its object is written. +func (d *DB) ReserveCacheEntryBytes(ctx context.Context, id string, maxBytes int64) (int64, error) { + if maxBytes <= 0 { + return 0, nil + } + tx, err := d.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + defer tx.Rollback() + entry, err := scanCacheEntry(tx.QueryRowContext(ctx, `select `+cacheEntryColumns+` from cache_entries where id = ?`, id)) + if err != nil { + return 0, err + } + if entry.State != "pending" { + return 0, sql.ErrNoRows + } + var used int64 + if err := tx.QueryRowContext(ctx, ` + select coalesce(sum(size_bytes), 0) from cache_entries + where owner_did = ? and state in ('pending', 'ready', 'deleting') and id <> ? + and not (repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state in ('ready', 'deleting'))`, + entry.OwnerDID, entry.ID, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash).Scan(&used); err != nil { + return 0, err + } + available := maxBytes - used + if available <= 0 { + return 0, fmt.Errorf("%w: owner %q has %d bytes in use", ErrCacheQuota, entry.OwnerDID, used) + } + if _, err := tx.ExecContext(ctx, `update cache_entries set size_bytes = ? where id = ? and state = 'pending'`, available, id); err != nil { + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err + } + return available, nil +} + +var ErrCacheQuota = errors.New("cache byte quota exceeded") + +func (tx *EventBatchTx) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { + _, err := tx.tx.ExecContext(ctx, ` + update cache_entries + set last_used_at = max(last_used_at, ?) + where id = ? and state = 'ready'`, now.UnixNano(), id) + return err +} + +func (tx *EventBatchTx) DiscardCacheEntry(ctx context.Context, id string, pendingOnly bool) (*CacheEntry, error) { + stateFilter := "" + if pendingOnly { + stateFilter = " and state in ('pending', 'deleting')" + } + current, err := scanCacheEntry(tx.tx.QueryRowContext(ctx, ` + update cache_entries + set state = 'deleting' + where id = ?`+stateFilter+` + returning `+cacheEntryColumns, id)) + if err == sql.ErrNoRows { + return nil, nil + } + return current, err +} + +func (d *DB) FindCacheEntry(ctx context.Context, repoDID, engine, key, hash string) (*CacheEntry, error) { + return scanCacheEntry(d.QueryRowContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ? and state = 'ready' + order by created_at desc + limit 1`, repoDID, engine, key, hash)) +} + +func (d *DB) FindFallbackCacheEntry(ctx context.Context, repoDID, engine, key, excludeHash string) (*CacheEntry, error) { + return scanCacheEntry(d.QueryRowContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? + and cache_hash <> ? and cache_hash <> '' and state = 'ready' + order by created_at desc + limit 1`, repoDID, engine, key, excludeHash)) +} +func (d *DB) FindFallbackCacheEntryForIdentity(ctx context.Context, repoDID, engine, prefix, excludeHash string) (*CacheEntry, error) { + return scanCacheEntry(d.QueryRowContext(ctx, ` + select `+cacheEntryColumns+` from cache_entries + where repo_did = ? and engine = ? and cache_key like ? || '%' + and cache_hash <> ? and cache_hash <> '' and state = 'ready' + order by created_at desc limit 1`, repoDID, engine, prefix, excludeHash)) +} + +func (d *DB) BeginCacheRestore(ctx context.Context, id string) (bool, error) { + result, err := d.ExecContext(ctx, `update cache_entries set restore_count = restore_count + 1 where id = ? and state = 'ready'`, id) + if err != nil { + return false, err + } + n, err := result.RowsAffected() + return n == 1, err +} + +func (d *DB) EndCacheRestore(ctx context.Context, id string) error { + _, err := d.ExecContext(ctx, `update cache_entries set restore_count = restore_count - 1 where id = ? and restore_count > 0`, id) + return err +} + +func (d *DB) CacheEntryPinned(ctx context.Context, id string) (bool, error) { + var count int64 + err := d.QueryRowContext(ctx, `select restore_count from cache_entries where id = ?`, id).Scan(&count) + if err == sql.ErrNoRows { + return false, nil + } + return count > 0, err +} + +func (d *DB) TouchCacheEntry(ctx context.Context, id string, now time.Time) error { + _, err := d.ExecContext(ctx, ` + update cache_entries set last_used_at = max(last_used_at, ?) where id = ? and state = 'ready'`, now.UnixNano(), id) + return err +} + +func (d *DB) ClaimPendingCacheEntry(ctx context.Context, id string) (bool, error) { + result, err := d.ExecContext(ctx, ` + update cache_entries set state = 'deleting' where id = ? and state = 'pending'`, id) + if err != nil { + return false, err + } + changed, err := result.RowsAffected() + return changed == 1, err +} + +func (d *DB) ClaimCacheEntry(ctx context.Context, id, expectedState string, expectedLastUsed time.Time) (bool, error) { + result, err := d.ExecContext(ctx, ` + update cache_entries + set state = 'deleting' + where id = ? and state = ? and last_used_at = ? and restore_count = 0`, + id, expectedState, expectedLastUsed.UnixNano()) + if err != nil { + return false, err + } + changed, err := result.RowsAffected() + return changed == 1, err +} + +func (d *DB) RestoreCacheEntryState(ctx context.Context, id, state string) error { + _, err := d.ExecContext(ctx, ` + update cache_entries set state = ? where id = ? and state = 'deleting'`, state, id) + return err +} + +func (d *DB) ExpiredCacheEntries(ctx context.Context, readyBefore, pendingBefore time.Time, limit int) ([]CacheEntry, error) { + rows, err := d.QueryContext(ctx, ` + select `+cacheEntryColumns+` + from cache_entries + where (state = 'ready' and last_used_at < ?) + or (state in ('pending', 'deleting') and last_used_at < ?) + order by last_used_at + limit ?`, + readyBefore.UnixNano(), + pendingBefore.UnixNano(), + limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var entries []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + return nil, err + } + entries = append(entries, *entry) + } + return entries, rows.Err() +} + +// discardPendingCacheEntriesForLease fences uploads that can arrive after a lease dies. +// Their tombstones let the cache pruner remove a late object without metadata. +func (d *DB) DiscardPendingCacheEntriesForLease(leaseID string, now time.Time) ([]string, error) { + tx, err := d.Begin() + if err != nil { + return nil, err + } + defer tx.Rollback() + + rows, err := tx.Query(` + select c.id, c.storage_key + from cache_entries c + join mill_cache_capabilities cap on cap.cache_id = c.id + where cap.lease_id = ? and cap.action = 'save' and c.state = 'pending'`, leaseID) + if err != nil { + return nil, err + } + var keys []string + var ids []string + for rows.Next() { + var id, key string + if err := rows.Scan(&id, &key); err != nil { + _ = rows.Close() + return nil, err + } + ids = append(ids, id) + keys = append(keys, key) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return nil, err + } + if err := rows.Close(); err != nil { + return nil, err + } + for i, id := range ids { + if _, err := tx.Exec(`update cache_entries set state = 'deleting' where id = ? and state = 'pending'`, id); err != nil { + return nil, err + } + if _, err := tx.Exec(` + insert into cache_object_deletions (storage_key, created_at) + values (?, ?) + on conflict(storage_key) do nothing`, keys[i], now.UnixNano()); err != nil { + return nil, err + } + } + if err := tx.Commit(); err != nil { + return nil, err + } + return keys, nil +} + +func (d *DB) DeleteCacheEntry(ctx context.Context, id string) error { + _, err := d.ExecContext(ctx, `delete from cache_entries where id = ?`, id) + return err +} + +func (d *DB) DeleteCacheEntriesByStorageKey(ctx context.Context, key string) error { + _, err := d.ExecContext(ctx, `delete from cache_entries where storage_key = ? and restore_count = 0`, key) + return err +} + +type MillCacheCapability struct { + Action string + CacheID string + StorageKey string +} + +func (d *DB) SaveMillCacheCapabilities(leaseID string, capabilities []MillCacheCapability) error { + tx, err := d.Begin() + if err != nil { + return err + } + defer tx.Rollback() + if _, err := tx.Exec(`delete from mill_cache_capabilities where lease_id = ?`, leaseID); err != nil { + return err + } + for _, capability := range capabilities { + if capability.Action != "restore" && capability.Action != "save" { + return fmt.Errorf("cache capability has invalid action %q", capability.Action) + } + if capability.CacheID == "" || capability.StorageKey == "" { + return fmt.Errorf("cache capability has empty identity") + } + if _, err := tx.Exec(` + insert into mill_cache_capabilities ( + lease_id, action, cache_id, storage_key + ) values (?, ?, ?, ?)`, + leaseID, + capability.Action, + capability.CacheID, + capability.StorageKey, + ); err != nil { + return err + } + } + return tx.Commit() +} + +func (tx *EventBatchTx) ConsumeMillCacheCapability(ctx context.Context, leaseID, action, cacheID string) (string, bool, error) { + var storageKey string + err := tx.tx.QueryRowContext(ctx, ` + delete from mill_cache_capabilities + where lease_id = ? and action = ? and cache_id = ? + returning storage_key`, + leaseID, action, cacheID, + ).Scan(&storageKey) + if err == sql.ErrNoRows { + return "", false, nil + } + if err != nil { + return "", false, err + } + return storageKey, true, nil +} + +func (tx *EventBatchTx) QueueCacheObjectDeletion(ctx context.Context, storageKey string, now time.Time) error { + _, err := tx.tx.ExecContext(ctx, ` + insert into cache_object_deletions (storage_key, created_at) + values (?, ?) + on conflict(storage_key) do nothing`, + storageKey, now.UnixNano(), + ) + return err +} + +func (d *DB) PendingCacheObjectDeletions(ctx context.Context, limit int) ([]string, error) { + rows, err := d.QueryContext(ctx, ` + select storage_key + from cache_object_deletions + order by created_at + limit ?`, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var keys []string + for rows.Next() { + var key string + if err := rows.Scan(&key); err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, rows.Err() +} + +func (d *DB) CacheEntriesByStorageKey(ctx context.Context, key string) ([]CacheEntry, error) { + rows, err := d.QueryContext(ctx, `select `+cacheEntryColumns+` from cache_entries where storage_key = ?`, key) + if err != nil { + return nil, err + } + defer rows.Close() + var entries []CacheEntry + for rows.Next() { + entry, err := scanCacheEntry(rows) + if err != nil { + return nil, err + } + entries = append(entries, *entry) + } + return entries, rows.Err() +} + +func (d *DB) CompleteCacheObjectDeletion(ctx context.Context, storageKey string) error { + _, err := d.ExecContext(ctx, `delete from cache_object_deletions where storage_key = ?`, storageKey) + return err +} + +type cacheEntryScanner interface { + Scan(dest ...any) error +} + +func scanCacheEntry(row cacheEntryScanner) (*CacheEntry, error) { + var entry CacheEntry + var createdAt, lastUsedAt int64 + if err := row.Scan( + &entry.ID, + &entry.StorageKey, + &entry.OwnerDID, + &entry.RepoDID, + &entry.Engine, + &entry.CacheKey, + &entry.CacheHash, + &entry.Checksum, + &entry.SizeBytes, + &entry.RestoreCount, + &entry.State, + &createdAt, + &lastUsedAt, + ); err != nil { + return nil, err + } + entry.CreatedAt = time.Unix(0, createdAt) + entry.LastUsedAt = time.Unix(0, lastUsedAt) + return &entry, nil +} diff --git a/spindle/db/cache_test.go b/spindle/db/cache_test.go new file mode 100644 index 00000000..f451a00b --- /dev/null +++ b/spindle/db/cache_test.go @@ -0,0 +1,414 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" +) + +func testCacheEntry(id, hash, state string, createdAt time.Time) CacheEntry { + return CacheEntry{ + ID: id, + StorageKey: "objects/" + id, + OwnerDID: "did:plc:owner", + RepoDID: "did:plc:repo", + Engine: "microvm", + CacheKey: "dependencies", + CacheHash: hash, + SizeBytes: 10, + State: state, + CreatedAt: createdAt, + LastUsedAt: createdAt, + } +} + +func insertTestCacheEntry(t *testing.T, d *DB, entry CacheEntry) { + t.Helper() + if err := d.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatalf("InsertCacheEntry(%s): %v", entry.ID, err) + } +} +func TestInsertCacheEntryWithinQuota(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + now := time.Now() + + first := testCacheEntry("first", "a", "pending", now) + inserted, err := d.InsertCacheEntryWithinQuota(ctx, first, 1) + if err != nil || !inserted { + t.Fatalf("first quota insert = (%t, %v)", inserted, err) + } + + second := testCacheEntry("second", "b", "pending", now) + inserted, err = d.InsertCacheEntryWithinQuota(ctx, second, 1) + if err != nil { + t.Fatal(err) + } + if inserted { + t.Fatal("entry quota accepted a second object for the same owner") + } + + second.OwnerDID = "did:plc:other" + inserted, err = d.InsertCacheEntryWithinQuota(ctx, second, 1) + if err != nil || !inserted { + t.Fatalf("other owner quota insert = (%t, %v)", inserted, err) + } +} + +func TestMarkCacheEntryReadyEvictsOverQuotaOwner(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + now := time.Now() + + existing := testCacheEntry("existing", "a", "ready", now) + existing.SizeBytes = 40 + insertTestCacheEntry(t, d, existing) + pending := testCacheEntry("pending", "b", "pending", now) + pending.SizeBytes = 0 + insertTestCacheEntry(t, d, pending) + + if _, err := d.MarkCacheEntryReady(ctx, pending.ID, 11, 50, now); err != nil { + t.Fatalf("over-quota ready should evict LRU instead of failing: %v", err) + } + var state string + if err := d.QueryRow(`select state from cache_entries where id = ?`, existing.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if state != "deleting" { + t.Fatalf("evicted entry state = %q, want deleting", state) + } + if err := d.QueryRow(`select state from cache_entries where id = ?`, pending.ID).Scan(&state); err != nil { + t.Fatal(err) + } + if state != "ready" { + t.Fatalf("pending entry state = %q, want ready", state) + } +} + +func TestCacheEntryLookup(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 2, 3, 4, 5, 6, time.UTC) + + insertTestCacheEntry(t, d, testCacheEntry("exact-old", "requested", "ready", base)) + insertTestCacheEntry(t, d, testCacheEntry("exact-new", "requested", "ready", base.Add(time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("exact-pending", "requested", "pending", base.Add(2*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-old", "old-a", "ready", base.Add(3*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-new", "old-b", "ready", base.Add(4*time.Second))) + insertTestCacheEntry(t, d, testCacheEntry("fallback-pending", "old-c", "pending", base.Add(5*time.Second))) + + exact, err := d.FindCacheEntry(ctx, "did:plc:repo", "microvm", "dependencies", "requested") + if err != nil { + t.Fatalf("FindCacheEntry: %v", err) + } + if exact.ID != "exact-new" { + t.Fatalf("FindCacheEntry returned %q, want exact-new", exact.ID) + } + if !exact.CreatedAt.Equal(base.Add(time.Second)) || !exact.LastUsedAt.Equal(base.Add(time.Second)) { + t.Fatalf("timestamps = (%v, %v), want %v", exact.CreatedAt, exact.LastUsedAt, base.Add(time.Second)) + } + + fallback, err := d.FindFallbackCacheEntry(ctx, "did:plc:repo", "microvm", "dependencies", "requested") + if err != nil { + t.Fatalf("FindFallbackCacheEntry: %v", err) + } + if fallback.ID != "fallback-new" { + t.Fatalf("FindFallbackCacheEntry returned %q, want fallback-new", fallback.ID) + } + + if _, err := d.FindCacheEntry(ctx, "did:plc:repo", "microvm", "missing", "requested"); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing FindCacheEntry error = %v, want sql.ErrNoRows", err) + } +} + +func TestMarkCacheEntryReadySupersedesMatchingReady(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 3, 4, 5, 6, 7, time.UTC) + first := testCacheEntry("first-completion", "same-hash", "pending", base) + second := testCacheEntry("second-completion", "same-hash", "pending", base.Add(time.Second)) + insertTestCacheEntry(t, d, first) + insertTestCacheEntry(t, d, second) + + superseded, err := d.MarkCacheEntryReady(ctx, first.ID, 100, 0, base.Add(2*time.Second)) + if err != nil { + t.Fatalf("mark first ready: %v", err) + } + if len(superseded) != 0 { + t.Fatalf("first completion superseded %d entries, want none", len(superseded)) + } + superseded, err = d.MarkCacheEntryReady(ctx, second.ID, 200, 0, base.Add(3*time.Second)) + if err != nil { + t.Fatalf("mark second ready: %v", err) + } + if len(superseded) != 1 || superseded[0].ID != first.ID || superseded[0].State != "deleting" { + t.Fatalf("second completion superseded %+v, want deleting %s", superseded, first.ID) + } + + ready, err := d.FindCacheEntry(ctx, second.RepoDID, second.Engine, second.CacheKey, second.CacheHash) + if err != nil { + t.Fatalf("find surviving ready entry: %v", err) + } + if ready.ID != second.ID || ready.SizeBytes != 200 || !ready.LastUsedAt.Equal(base.Add(3*time.Second)) { + t.Fatalf("surviving entry = %+v, want %s (200 bytes)", ready, second.ID) + } +} + +func TestMarkCacheEntryReadyConcurrentCompletions(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 1, 4, 5, 6, 7, 8, time.UTC) + first := testCacheEntry("concurrent-a", "same-hash", "pending", base) + second := testCacheEntry("concurrent-b", "same-hash", "pending", base.Add(time.Second)) + insertTestCacheEntry(t, d, first) + insertTestCacheEntry(t, d, second) + + type result struct { + superseded []CacheEntry + err error + } + start := make(chan struct{}) + results := make(chan result, 2) + for _, entry := range []CacheEntry{first, second} { + entry := entry + go func() { + <-start + superseded, err := d.MarkCacheEntryReady(ctx, entry.ID, 100, 0, base.Add(2*time.Second)) + results <- result{superseded: superseded, err: err} + }() + } + close(start) + + var superseded []CacheEntry + for range 2 { + result := <-results + if result.err != nil { + t.Fatalf("concurrent MarkCacheEntryReady: %v", result.err) + } + superseded = append(superseded, result.superseded...) + } + if len(superseded) != 1 || superseded[0].State != "deleting" { + t.Fatalf("concurrent completions superseded %+v, want one deleting entry", superseded) + } + + var readyCount, deletingCount int + if err := d.QueryRowContext(ctx, ` + select sum(state = 'ready'), sum(state = 'deleting') + from cache_entries + where repo_did = ? and engine = ? and cache_key = ? and cache_hash = ?`, + first.RepoDID, first.Engine, first.CacheKey, first.CacheHash).Scan(&readyCount, &deletingCount); err != nil { + t.Fatalf("count completion states: %v", err) + } + if readyCount != 1 || deletingCount != 1 { + t.Fatalf("completion states = %d ready, %d deleting; want one each", readyCount, deletingCount) + } +} + +func TestCacheEntryReadyTouchAndExpiry(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 2, 3, 4, 5, 6, 7, time.UTC) + + readyOld := testCacheEntry("ready-old", "a", "ready", base) + readyFresh := testCacheEntry("ready-fresh", "b", "ready", base) + pendingOld := testCacheEntry("pending-old", "c", "pending", base) + pendingFresh := testCacheEntry("pending-fresh", "d", "pending", base) + pendingFresh.LastUsedAt = base.Add(20 * time.Minute) + deletingOld := testCacheEntry("deleting-old", "e", "deleting", base) + deletingFresh := testCacheEntry("deleting-fresh", "f", "deleting", base) + deletingFresh.LastUsedAt = base.Add(20 * time.Minute) + for _, entry := range []CacheEntry{readyOld, readyFresh, pendingOld, pendingFresh, deletingOld, deletingFresh} { + insertTestCacheEntry(t, d, entry) + } + + touchedAt := base.Add(30 * time.Minute) + if err := d.TouchCacheEntry(ctx, readyFresh.ID, touchedAt); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + + expired, err := d.ExpiredCacheEntries(ctx, base.Add(10*time.Minute), base.Add(10*time.Minute), 10) + if err != nil { + t.Fatalf("ExpiredCacheEntries: %v", err) + } + got := make(map[string]bool, len(expired)) + for _, entry := range expired { + got[entry.ID] = true + } + if len(got) != 3 || !got[readyOld.ID] || !got[pendingOld.ID] || !got[deletingOld.ID] { + t.Fatalf("expired IDs = %v, want ready-old, pending-old, and deleting-old", got) + } + + limited, err := d.ExpiredCacheEntries(ctx, base.Add(10*time.Minute), base.Add(10*time.Minute), 1) + if err != nil { + t.Fatalf("limited ExpiredCacheEntries: %v", err) + } + if len(limited) != 1 { + t.Fatalf("limited expiry count = %d, want 1", len(limited)) + } +} + +func TestCacheEntryClaim(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + base := time.Date(2026, 2, 4, 5, 6, 7, 8, time.UTC) + entry := testCacheEntry("claim-me", "hash", "ready", base) + insertTestCacheEntry(t, d, entry) + + touchedAt := base.Add(time.Minute) + if err := d.TouchCacheEntry(ctx, entry.ID, touchedAt); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + claimed, err := d.ClaimCacheEntry(ctx, entry.ID, "ready", base) + if err != nil { + t.Fatalf("stale ClaimCacheEntry: %v", err) + } + if claimed { + t.Fatal("stale ClaimCacheEntry claimed a touched entry") + } + claimed, err = d.ClaimCacheEntry(ctx, entry.ID, "ready", touchedAt) + if err != nil { + t.Fatalf("ClaimCacheEntry: %v", err) + } + if !claimed { + t.Fatal("ClaimCacheEntry did not claim unchanged entry") + } + + if err := d.TouchCacheEntry(ctx, entry.ID, base.Add(2*time.Minute)); err != nil { + t.Fatalf("TouchCacheEntry while deleting: %v", err) + } + if _, err := d.MarkCacheEntryReady(ctx, entry.ID, 999, 0, base.Add(3*time.Minute)); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("MarkCacheEntryReady while deleting error = %v, want sql.ErrNoRows", err) + } + var state string + var lastUsedAt int64 + var sizeBytes int64 + if err := d.QueryRowContext(ctx, ` + select state, last_used_at, size_bytes from cache_entries where id = ?`, entry.ID).Scan(&state, &lastUsedAt, &sizeBytes); err != nil { + t.Fatalf("query claimed entry: %v", err) + } + if state != "deleting" || lastUsedAt != touchedAt.UnixNano() || sizeBytes != entry.SizeBytes { + t.Fatalf("claimed entry = state %q, last used %d, size %d; want deleting, %d, %d", + state, lastUsedAt, sizeBytes, touchedAt.UnixNano(), entry.SizeBytes) + } + + if err := d.RestoreCacheEntryState(ctx, entry.ID, "ready"); err != nil { + t.Fatalf("RestoreCacheEntryState: %v", err) + } + restored, err := d.FindCacheEntry(ctx, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash) + if err != nil { + t.Fatalf("find restored entry: %v", err) + } + if restored.State != "ready" || !restored.LastUsedAt.Equal(touchedAt) { + t.Fatalf("restored entry = state %q, last used %v", restored.State, restored.LastUsedAt) + } +} + +func TestCacheEntryDelete(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + entry := testCacheEntry("delete-me", "hash", "ready", time.Now()) + insertTestCacheEntry(t, d, entry) + + if err := d.DeleteCacheEntry(ctx, entry.ID); err != nil { + t.Fatalf("DeleteCacheEntry: %v", err) + } + if err := d.DeleteCacheEntry(ctx, entry.ID); err != nil { + t.Fatalf("second DeleteCacheEntry: %v", err) + } + if _, err := d.FindCacheEntry(ctx, entry.RepoDID, entry.Engine, entry.CacheKey, entry.CacheHash); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("find deleted error = %v, want sql.ErrNoRows", err) + } +} + +func TestMillCacheCapabilitiesAreLeaseScopedAndConsumable(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + lease := MillLease{ + LeaseID: "lease-1", + NodeID: "node-1", + Epoch: "epoch-1", + Engine: "microvm", + Knot: "k", + Rkey: "r", + Workflow: "build", + State: "running", + } + if err := d.SaveMillLease(lease); err != nil { + t.Fatal(err) + } + capability := MillCacheCapability{ + Action: "save", + CacheID: "cache-1", + StorageKey: "objects/did:web:example.com/cache-1", + } + if err := d.SaveMillCacheCapabilities(lease.LeaseID, []MillCacheCapability{capability}); err != nil { + t.Fatal(err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + storageKey, consumed, err := tx.ConsumeMillCacheCapability( + ctx, + lease.LeaseID, + capability.Action, + capability.CacheID, + ) + if err != nil { + return err + } + if !consumed { + t.Fatal("planned capability was not consumed") + } + if storageKey != capability.StorageKey { + t.Fatalf("consumed storage key = %q, want %q", storageKey, capability.StorageKey) + } + return nil + }); err != nil { + t.Fatal(err) + } + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + _, consumed, err := tx.ConsumeMillCacheCapability( + ctx, + "another-lease", + capability.Action, + capability.CacheID, + ) + if err != nil { + return err + } + if consumed { + t.Fatal("foreign lease consumed capability") + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +func TestCacheObjectDeletionQueuePersistsUntilCompleted(t *testing.T) { + ctx := context.Background() + d := newTestDB(t) + key := "objects/did:web:example.com/orphan" + if err := d.ApplyEventBatch(nil, func(tx *EventBatchTx) error { + return tx.QueueCacheObjectDeletion(ctx, key, time.Now()) + }); err != nil { + t.Fatal(err) + } + keys, err := d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(keys) != 1 || keys[0] != key { + t.Fatalf("pending deletions = %v, want %q", keys, key) + } + if err := d.CompleteCacheObjectDeletion(ctx, key); err != nil { + t.Fatal(err) + } + keys, err = d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(keys) != 0 { + t.Fatalf("completed deletion remained queued: %v", keys) + } +} diff --git a/spindle/db/db.go b/spindle/db/db.go index d4e91e8d..3a3a5faf 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -91,6 +91,37 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { updated_at text not null ); + create table if not exists cache_entries ( + id text primary key, + storage_key text unique not null, + owner_did text not null, + repo_did text not null, + engine text not null, + cache_key text not null, + cache_hash text not null, + checksum text not null default '', + size_bytes integer not null default 0, + restore_count integer not null default 0 check (restore_count >= 0), + state text not null check (state in ('pending', 'ready', 'deleting')), + created_at integer not null, + last_used_at integer not null + ); + + create index if not exists cache_entries_lookup + on cache_entries (repo_did, engine, cache_key, cache_hash, created_at desc) + where state = 'ready'; + create index if not exists cache_entries_ready_expiry + on cache_entries (last_used_at) where state = 'ready'; + create index if not exists cache_entries_pending_deadline + on cache_entries (last_used_at) where state in ('pending', 'deleting'); + create index if not exists cache_entries_owner_state + on cache_entries (owner_did, state); + + create table if not exists cache_object_deletions ( + storage_key text primary key, + created_at integer not null + ); + create table if not exists pipelines ( id text primary key, repo_did text not null, @@ -149,6 +180,15 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { mill_records_terminal_metrics integer not null default 0 ); + create table if not exists mill_cache_capabilities ( + lease_id text not null, + action text not null check (action in ('restore', 'save')), + cache_id text not null, + storage_key text not null, + primary key (lease_id, action, cache_id), + foreign key (lease_id) references mill_leases(lease_id) on delete cascade + ); + create table if not exists mill_executor_cursors ( node_id text not null, epoch text not null, @@ -207,7 +247,7 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { return nil, err } - return &DB{db}, nil + return &DB{DB: db}, nil } func runMigrations(_ context.Context, conn *sql.Conn, logger *slog.Logger) error { @@ -399,6 +439,26 @@ func runMigrations(_ context.Context, conn *sql.Conn, logger *slog.Logger) error return err } + if err := orm.RunMigration(conn, logger, "cache-entry-integrity", func(tx *sql.Tx) error { + for _, column := range []struct{ name, definition string }{ + {"checksum", "text not null default ''"}, + {"restore_count", "integer not null default 0"}, + } { + var present int + if err := tx.QueryRow(`select count(*) from pragma_table_info('cache_entries') where name = ?`, column.name).Scan(&present); err != nil { + return err + } + if present == 0 { + if _, err := tx.Exec(`alter table cache_entries add column ` + column.name + ` ` + column.definition); err != nil { + return err + } + } + } + return nil + }); err != nil { + return err + } + if err := orm.RunMigration(conn, logger, "cache-quotas-schema", func(tx *sql.Tx) error { _, err := tx.Exec(` create table if not exists cache_objects ( diff --git a/spindle/engine/cache.go b/spindle/engine/cache.go new file mode 100644 index 00000000..8dc03e5d --- /dev/null +++ b/spindle/engine/cache.go @@ -0,0 +1,111 @@ +package engine + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "strings" + + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +// cache log steps live below the setup step (-1) +const ( + CacheRestoreStepIdx = -2 + CacheSaveStepIdx = -3 +) + +type cacheStep struct { + name string + command string +} + +func (s cacheStep) Name() string { return s.name } +func (s cacheStep) Command() string { return s.command } +func (s cacheStep) Kind() models.StepKind { return models.StepKindSystem } + +var ( + CacheRestoreStep models.Step = cacheStep{name: "restore cache", command: "restore cached paths"} + CacheSaveStep models.Step = cacheStep{name: "save cache", command: "persist changed paths"} +) + +type CacheRunner interface { + RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error + SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error +} + +const CacheExitNoPaths = 42 + +// avoids storing an empty archive when zstd is missing +const CacheExitNoCompressor = 43 + +// run tar from / so absolute paths survive extraction +// tar exits nonzero on missing paths, so only existing ones reach it +func CacheSaveScript(paths []string, workspaceRoot string, compressionLevel int) string { + trimmed := make([]string, 0, len(paths)) + for _, p := range paths { + if !strings.HasPrefix(p, "/") { + p = workspaceRoot + "/" + p + } + trimmed = append(trimmed, shellQuote(strings.TrimPrefix(p, "/"))) + } + tail := fmt.Sprintf(`tar -cf - -C / -- "$@" | %s`, CacheCompressCmd(compressionLevel)) + return fmt.Sprintf(`set -o pipefail +command -v zstd >/dev/null 2>&1 || { echo "zstd not found in image; cannot save cache" >&2; exit %d; } +set -- +for p in %s; do [ -e "/$p" ] && set -- "$@" "$p"; done +if [ $# -eq 0 ]; then echo "no cache paths exist; skipping" >&2; exit %d; fi +%s`, CacheExitNoCompressor, strings.Join(trimmed, " "), CacheExitNoPaths, tail) +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" +} + +func CacheCompressCmd(level int) string { + if level == 0 { + return "zstd -T0 -5" + } + return fmt.Sprintf("zstd -T0 -%d", level) +} + +// old entries might still be gzip, so detect them instead of trusting config +func CacheDecompressCmd(br *bufio.Reader) string { + head, _ := br.Peek(4) + if bytes.HasPrefix(head, []byte{0x1f, 0x8b}) { + return "gzip -dc" + } + return "zstd -dc" +} + +// Put keeps draining the pipe after it returns, so the guest writer never +// blocks on a full pipe. +type CacheUpload struct { + Writer *io.PipeWriter + done chan error +} + +func NewCacheUpload(ctx context.Context, store storage.Storage, key string) *CacheUpload { + pr, pw := io.Pipe() + u := &CacheUpload{Writer: pw, done: make(chan error, 1)} + go func() { + err := store.Put(ctx, key, pr) + _, _ = io.Copy(io.Discard, pr) + u.done <- err + }() + return u +} + +func (u *CacheUpload) Abort(err error) { + u.Writer.CloseWithError(err) + <-u.done +} + +// storage treats EOF as a complete archive, so only a clean exec may Finish +func (u *CacheUpload) Finish() error { + u.Writer.Close() + return <-u.done +} diff --git a/spindle/engine/cache_controller.go b/spindle/engine/cache_controller.go new file mode 100644 index 00000000..5094e0d3 --- /dev/null +++ b/spindle/engine/cache_controller.go @@ -0,0 +1,577 @@ +package engine + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "log/slog" + "net/url" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +type CacheUpdateAction int + +const ( + CacheUsed CacheUpdateAction = iota + 1 + CacheStored + CacheDiscarded + CacheMissing +) + +type CacheUpdate struct { + Action CacheUpdateAction + ID string + Ref string + SizeBytes int64 + Checksum string +} + +type CacheController interface { + Plan(ctx context.Context, pipeline *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) + Apply(ctx context.Context, update CacheUpdate) error +} + +type localCacheController struct { + index *db.DB + store storage.Storage + repoDir string + maxBytesPerOwner int64 + maxEntriesPerOwner int64 + logger *slog.Logger +} + +func NewLocalCacheController(index *db.DB, store storage.Storage, repoDir string, maxBytesPerOwner, maxEntriesPerOwner int64, logger *slog.Logger) CacheController { + return &localCacheController{ + index: index, + store: store, + repoDir: repoDir, + maxBytesPerOwner: maxBytesPerOwner, + maxEntriesPerOwner: maxEntriesPerOwner, + logger: logger, + } +} + +// on a hash miss, the newest older generation still warms the build +// unusable entries degrade to a plain miss +func resolveCaches(ctx context.Context, l *slog.Logger, index *db.DB, repoDID, engine, repoPath, rev string, entries []models.CacheEntry) []models.CacheBinding { + resolved := make([]models.CacheBinding, 0, len(entries)) + for entryIndex, entry := range entries { + hash := "" + if len(entry.Hash) > 0 && repoPath != "" { + sum, missing := hashKeyFilesLegacy(ctx, repoPath, rev, entry.Hash) + for _, m := range missing { + l.Warn("cache hash file not in repo", "key", entry.Key, "path", m) + } + hash = sum + } + binding := models.CacheBinding{ + EntryIndex: entryIndex, + Paths: entry.Paths, + Key: entry.Key, + Hash: hash, + CompressionLevel: entry.CompressionLevel, + When: entry.When, + } + + found, err := index.FindCacheEntry(ctx, repoDID, engine, entry.Key, hash) + if errors.Is(err, sql.ErrNoRows) && hash != "" { + found, err = index.FindFallbackCacheEntry(ctx, repoDID, engine, entry.Key, hash) + if err == nil { + binding.RestoreName = entry.Key + "-" + found.CacheHash + } + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + l.Warn("cache lookup failed; entry will save but not restore", "key", entry.Key, "err", err) + } else if err == nil { + binding.RestoreID = found.ID + binding.RestoreKey = found.StorageKey + } + resolved = append(resolved, binding) + } + return resolved +} + +func hashKeyFilesLegacy(ctx context.Context, repoPath, rev string, paths []string) (string, []string) { + h := sha256.New() + var missing []string + hashed := 0 + for _, p := range paths { + blob, err := gitBlobId(ctx, repoPath, rev, p) + if err != nil { + missing = append(missing, p) + continue + } + fmt.Fprintf(h, "%s=%s\n", p, blob) + hashed++ + } + if hashed == 0 { + return "", missing + } + return hex.EncodeToString(h.Sum(nil))[:12], missing +} + +func hashKeyFiles(ctx context.Context, repoPath, rev string, paths []string) (string, []string) { + h := sha256.New() + var missing []string + hashed := 0 + for _, p := range paths { + blob, err := gitBlobId(ctx, repoPath, rev, p) + if err != nil { + missing = append(missing, p) + continue + } + fmt.Fprintf(h, "%s=%s\n", p, blob) + hashed++ + } + if hashed == 0 { + return "", missing + } + return hex.EncodeToString(h.Sum(nil)), missing +} + +// sparse checkouts might not have the file, but the object database does +func gitBlobId(ctx context.Context, repoPath, rev, path string) (string, error) { + out, err := exec.CommandContext(ctx, "git", "-C", repoPath, "rev-parse", rev+":"+path).Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(out)), nil +} +func cacheIdentityKey(workflowName, ref, rev, key string) string { + return url.PathEscape(workflowName) + ":" + url.PathEscape(ref) + ":" + url.PathEscape(rev) + ":" + key +} + +func pipelineRef(metadata *tangled.Pipeline_TriggerMetadata) string { + if metadata == nil { + return "" + } + if metadata.Push != nil { + return metadata.Push.Ref + } + if metadata.PullRequest != nil { + return "refs/heads/" + metadata.PullRequest.SourceBranch + } + if metadata.Manual != nil && metadata.Manual.Ref != nil { + return *metadata.Manual.Ref + } + return "" +} + +func resolveCachesIdentity(ctx context.Context, l *slog.Logger, index *db.DB, repoDID, engine, repoPath, rev, workflowName, ref string, entries []models.CacheEntry) []models.CacheBinding { + resolved := make([]models.CacheBinding, 0, len(entries)) + for i, entry := range entries { + hash := "" + if len(entry.Hash) > 0 && repoPath != "" { + hash, _ = hashKeyFiles(ctx, repoPath, rev, entry.Hash) + } + lookupKey := cacheIdentityKey(workflowName, ref, rev, entry.Key) + b := models.CacheBinding{EntryIndex: i, Paths: entry.Paths, Key: entry.Key, Hash: hash, CompressionLevel: entry.CompressionLevel, When: entry.When} + found, err := index.FindCacheEntry(ctx, repoDID, engine, lookupKey, hash) + if errors.Is(err, sql.ErrNoRows) && entry.Fallback && hash != "" { + found, err = index.FindFallbackCacheEntryForIdentity(ctx, repoDID, engine, url.PathEscape(workflowName)+":"+url.PathEscape(ref)+":", hash) + if err == nil { + b.RestoreName = entry.Key + "-" + found.CacheHash + } + } + if err == nil { + b.RestoreID, b.RestoreKey, b.Checksum, b.SizeBytes = found.ID, found.StorageKey, found.Checksum, found.SizeBytes + } + if err != nil && !errors.Is(err, sql.ErrNoRows) { + l.Warn("cache lookup failed; entry will save but not restore", "key", entry.Key, "err", err) + } + resolved = append(resolved, b) + } + return resolved +} + +func (c *localCacheController) Plan(ctx context.Context, pipeline *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) { + repo, err := c.index.GetRepoByDid(pipeline.RepoDid) + if err != nil { + return nil, fmt.Errorf("cache owner lookup: %w", err) + } + + repoPath, rev := "", "" + if metadata := pipeline.TriggerMetadata; metadata != nil { + if resolvedRev, err := models.ExtractCommitSHA(*metadata); err != nil { + c.logger.Warn("cannot resolve pipeline commit; cache hashing disabled", "err", err) + } else { + did := pipeline.RepoDid.String() + if metadata.SourceRepo != nil && *metadata.SourceRepo != "" { + did = *metadata.SourceRepo + } + repoPath, rev = filepath.Join(c.repoDir, did), resolvedRev + } + } + + cacheNamespace := workflow.CacheNamespace + if cacheNamespace == "" { + cacheNamespace = models.CacheNamespace() + } + cacheEngine := workflow.Engine + "/" + cacheNamespace + var bindings []models.CacheBinding + if workflow.Name == "" && pipeline.TriggerMetadata == nil { + bindings = resolveCaches(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Caches) + } else { + bindings = resolveCachesIdentity(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Name, pipelineRef(pipeline.TriggerMetadata), workflow.Caches) + } + now := time.Now() + pendingUntil := now + // pending rows stay live through the workflow deadline + if deadline, ok := ctx.Deadline(); ok { + pendingUntil = deadline + } + for i := range bindings { + id := uuid.NewString() + cacheKey := bindings[i].Key + if workflow.Name != "" || pipeline.TriggerMetadata != nil { + cacheKey = cacheIdentityKey(workflow.Name, pipelineRef(pipeline.TriggerMetadata), rev, cacheKey) + } + saveKey, err := models.CacheObjectKey(pipeline.RepoDid.String(), id) + if err != nil { + return nil, err + } + bindings[i].SaveID = id + bindings[i].SaveKey = saveKey + inserted, err := c.index.InsertCacheEntryWithinQuota(ctx, db.CacheEntry{ + ID: id, + StorageKey: bindings[i].SaveKey, + OwnerDID: repo.Owner.String(), + RepoDID: pipeline.RepoDid.String(), + Engine: cacheEngine, + CacheKey: cacheKey, + CacheHash: bindings[i].Hash, + State: "pending", + CreatedAt: now, + LastUsedAt: pendingUntil, + }, c.maxEntriesPerOwner) + if err != nil { + for _, planned := range bindings[:i] { + _ = c.Apply(context.WithoutCancel(ctx), CacheUpdate{ + Action: CacheDiscarded, + ID: planned.SaveID, + Ref: planned.SaveKey, + }) + } + for j := range bindings { + bindings[j].SaveID = "" + bindings[j].SaveKey = "" + } + c.logger.Warn("cache save reservations failed; restoring only", "err", err) + return bindings, nil + } + if !inserted { + bindings[i].SaveID = "" + bindings[i].SaveKey = "" + } + } + return bindings, nil +} + +type cacheByteReserver interface { + reserveBytes(context.Context, models.CacheBinding) (int64, error) +} + +func (c *localCacheController) reserveBytes(ctx context.Context, binding models.CacheBinding) (int64, error) { + return c.index.ReserveCacheEntryBytes(ctx, binding.SaveID, c.maxBytesPerOwner) +} + +func (c *localCacheController) Apply(ctx context.Context, update CacheUpdate) error { + now := time.Now() + switch update.Action { + case CacheUsed: + return c.index.TouchCacheEntry(ctx, update.ID, now) + case CacheStored: + superseded, err := c.index.MarkCacheEntryReadyWithChecksum(ctx, update.ID, update.SizeBytes, update.Checksum, c.maxBytesPerOwner, now) + if err != nil { + _ = c.store.Delete(context.WithoutCancel(ctx), update.Ref) + return err + } + for _, old := range superseded { + if err := c.deleteEntry(context.WithoutCancel(ctx), old.StorageKey, old.ID); err != nil { + c.logger.Warn("delete replaced cache failed", "id", old.ID, "err", err) + } + } + case CacheDiscarded, CacheMissing: + if update.Action == CacheDiscarded { + claimed, err := c.index.ClaimPendingCacheEntry(ctx, update.ID) + if err != nil || !claimed { + return err + } + } + return c.deleteEntry(ctx, update.Ref, update.ID) + default: + return fmt.Errorf("unknown cache update action %d", update.Action) + } + return nil +} + +func (c *localCacheController) deleteEntry(ctx context.Context, ref, id string) error { + pinned, err := c.index.CacheEntryPinned(ctx, id) + if err != nil { + return err + } + if pinned { + return c.index.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { return tx.QueueCacheObjectDeletion(ctx, ref, time.Now()) }) + } + if err := c.store.Delete(ctx, ref); err != nil { + return err + } + return c.index.DeleteCacheEntry(ctx, id) +} + +type preplannedCacheController struct { + apply func(context.Context, CacheUpdate) error +} + +func NewPreplannedCacheController(apply func(context.Context, CacheUpdate) error) CacheController { + return &preplannedCacheController{apply: apply} +} + +func (c *preplannedCacheController) Plan(_ context.Context, _ *models.Pipeline, workflow *models.Workflow) ([]models.CacheBinding, error) { + return workflow.CacheBindings, nil +} + +func (c *preplannedCacheController) Apply(ctx context.Context, update CacheUpdate) error { + return c.apply(ctx, update) +} + +type trackedCacheStore struct { + storage.Storage + controller CacheController + logger *slog.Logger + mu sync.Mutex + restores map[string]models.CacheBinding + saves map[string]models.CacheBinding +} + +func newTrackedCacheStore(base storage.Storage, controller CacheController, logger *slog.Logger, bindings []models.CacheBinding) *trackedCacheStore { + store := &trackedCacheStore{ + Storage: base, + controller: controller, + logger: logger, + restores: make(map[string]models.CacheBinding, len(bindings)), + saves: make(map[string]models.CacheBinding, len(bindings)), + } + for _, binding := range bindings { + if binding.RestoreKey != "" { + store.restores[binding.RestoreKey] = binding + } + if binding.SaveKey != "" { + store.saves[binding.SaveKey] = binding + } + } + return store +} + +func (s *trackedCacheStore) beginRestore(ctx context.Context, id string) (bool, *db.DB, error) { + if c, ok := s.controller.(*localCacheController); ok { + pinned, err := c.index.BeginCacheRestore(ctx, id) + return pinned, c.index, err + } + return true, nil, nil +} +func (s *trackedCacheStore) endRestore(ctx context.Context, id string) error { + if c, ok := s.controller.(*localCacheController); ok { + return c.index.EndCacheRestore(ctx, id) + } + return nil +} + +func (s *trackedCacheStore) Get(ctx context.Context, key string) (io.ReadCloser, error) { + s.mu.Lock() + binding, tracked := s.restores[key] + s.mu.Unlock() + if !tracked { + return nil, fmt.Errorf("cache metadata missing for %q", key) + } + + pinned, index, err := s.beginRestore(ctx, binding.RestoreID) + if err != nil { + return nil, err + } + if !pinned { + return nil, storage.ErrNotExist + } + reader, err := s.Storage.Get(ctx, key) + if err != nil { + _ = s.endRestore(context.WithoutCancel(ctx), binding.RestoreID) + if errors.Is(err, storage.ErrNotExist) { + _ = s.controller.Apply(context.WithoutCancel(ctx), CacheUpdate{Action: CacheMissing, ID: binding.RestoreID, Ref: binding.RestoreKey}) + } + return nil, err + } + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheUsed, + ID: binding.RestoreID, + Ref: binding.RestoreKey, + }); err != nil { + s.logger.Warn("cache usage update failed", "id", binding.RestoreID, "err", err) + } + return &verifiedCacheReader{ReadCloser: reader, expected: binding.Checksum, expectedSize: binding.SizeBytes, id: binding.RestoreID, index: index}, nil +} + +type verifiedCacheReader struct { + io.ReadCloser + expected string + expectedSize int64 + id string + index *db.DB + h hash.Hash + n int64 + checked bool +} + +func (r *verifiedCacheReader) verify() error { + if r.expectedSize > 0 && r.n != r.expectedSize { + return fmt.Errorf("cache object size mismatch: got %d, want %d", r.n, r.expectedSize) + } + if r.expected != "" && hex.EncodeToString(r.h.Sum(nil)) != r.expected { + return fmt.Errorf("cache object checksum mismatch") + } + return nil +} + +func (r *verifiedCacheReader) Read(p []byte) (int, error) { + if r.h == nil { + r.h = sha256.New() + } + n, err := r.ReadCloser.Read(p) + if n > 0 { + _, _ = r.h.Write(p[:n]) + r.n += int64(n) + } + if err == io.EOF { + r.checked = true + if verifyErr := r.verify(); verifyErr != nil { + return n, verifyErr + } + } + return n, err +} + +func (r *verifiedCacheReader) Close() error { + var err error + if !r.checked { + if r.h == nil { + r.h = sha256.New() + } + _, err = io.Copy(io.Discard, r) + if err == nil { + r.checked = true + err = r.verify() + } + } + if closeErr := r.ReadCloser.Close(); err == nil { + err = closeErr + } + if r.index != nil { + if endErr := r.index.EndCacheRestore(context.Background(), r.id); err == nil { + err = endErr + } + } + return err +} + +func (s *trackedCacheStore) Put(ctx context.Context, key string, reader io.Reader) error { + s.mu.Lock() + binding, tracked := s.saves[key] + s.mu.Unlock() + if !tracked { + return fmt.Errorf("cache metadata missing for %q", key) + } + + limit := int64(0) + if reserver, ok := s.controller.(cacheByteReserver); ok { + var err error + limit, err = reserver.reserveBytes(ctx, binding) + if err != nil { + return err + } + } + counted := &countingReader{r: reader, limit: limit, h: sha256.New()} + put := s.Storage.Put + if conditional, ok := s.Storage.(storage.ConditionalStorage); ok { + put = func(ctx context.Context, key string, r io.Reader) error { return conditional.PutIfAbsent(ctx, key, r) } + } + if err := put(ctx, key, counted); err != nil { + _ = s.Storage.Delete(context.WithoutCancel(ctx), key) + return err + } + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheStored, + ID: binding.SaveID, + Ref: binding.SaveKey, + SizeBytes: counted.n, + Checksum: hex.EncodeToString(counted.h.Sum(nil)), + }); err != nil { + _ = s.Storage.Delete(context.WithoutCancel(ctx), key) + return fmt.Errorf("record cache upload: %w", err) + } + + s.mu.Lock() + delete(s.saves, key) + s.mu.Unlock() + return nil +} + +func (s *trackedCacheStore) cleanup(ctx context.Context) { + s.mu.Lock() + pending := make([]models.CacheBinding, 0, len(s.saves)) + for _, binding := range s.saves { + pending = append(pending, binding) + } + clear(s.saves) + s.mu.Unlock() + + for _, binding := range pending { + if err := s.controller.Apply(ctx, CacheUpdate{ + Action: CacheDiscarded, + ID: binding.SaveID, + Ref: binding.SaveKey, + }); err != nil { + s.logger.Warn("discard incomplete cache failed", "id", binding.SaveID, "err", err) + } + } +} + +var ErrCacheObjectTooLarge = errors.New("cache object exceeds owner byte quota") + +type countingReader struct { + r io.Reader + n int64 + limit int64 + h hash.Hash +} + +func (r *countingReader) Read(p []byte) (int, error) { + if r.limit > 0 { + remaining := r.limit - r.n + if remaining <= 0 { + return 0, ErrCacheObjectTooLarge + } + if int64(len(p)) > remaining { + p = p[:remaining] + } + } + n, err := r.r.Read(p) + r.n += int64(n) + if n > 0 && r.h != nil { + _, _ = r.h.Write(p[:n]) + } + return n, err +} diff --git a/spindle/engine/cache_prune.go b/spindle/engine/cache_prune.go new file mode 100644 index 00000000..6417eaef --- /dev/null +++ b/spindle/engine/cache_prune.go @@ -0,0 +1,115 @@ +package engine + +import ( + "context" + "log/slog" + "time" + + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/storage" +) + +const ( + cachePruneBatch = 100 + cachePendingMaxAge = time.Hour +) + +func StartCachePruner(ctx context.Context, l *slog.Logger, index *db.DB, store storage.Storage, retention, interval time.Duration) { + if store == nil || interval <= 0 { + return + } + go func() { + prune := func() { + total := 0 + for { + n, err := PruneCaches(ctx, index, store, time.Now(), retention, cachePendingMaxAge, cachePruneBatch) + total += n + if err != nil { + l.Warn("cache prune failed", "count", total, "err", err) + return + } + if n < cachePruneBatch { + if total > 0 { + l.Info("pruned cache entries", "count", total) + } + return + } + } + } + prune() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + prune() + } + } + }() +} + +func PruneCaches(ctx context.Context, index *db.DB, store storage.Storage, now time.Time, retention, pendingMaxAge time.Duration, limit int) (int, error) { + deletions, err := index.PendingCacheObjectDeletions(ctx, limit) + if err != nil { + return 0, err + } + pruned := 0 + for _, key := range deletions { + entries, err := index.CacheEntriesByStorageKey(ctx, key) + if err != nil { + return pruned, err + } + pinned := false + for _, entry := range entries { + if entry.RestoreCount > 0 { + pinned = true + break + } + } + if pinned { + continue + } + if err := store.Delete(ctx, key); err != nil { + return pruned, err + } + if err := index.CompleteCacheObjectDeletion(ctx, key); err != nil { + return pruned, err + } + if err := index.DeleteCacheEntriesByStorageKey(ctx, key); err != nil { + return pruned, err + } + pruned++ + } + if pruned == limit { + return pruned, nil + } + + readyBefore := time.Unix(0, 0) + if retention > 0 { + readyBefore = now.Add(-retention) + } + entries, err := index.ExpiredCacheEntries(ctx, readyBefore, now.Add(-pendingMaxAge), limit-pruned) + if err != nil { + return pruned, err + } + for _, entry := range entries { + claimed, err := index.ClaimCacheEntry(ctx, entry.ID, entry.State, entry.LastUsedAt) + if err != nil { + return pruned, err + } + if !claimed { + continue + } + if err := store.Delete(ctx, entry.StorageKey); err != nil { + _ = index.RestoreCacheEntryState(context.WithoutCancel(ctx), entry.ID, entry.State) + return pruned, err + } + if err := index.DeleteCacheEntry(ctx, entry.ID); err != nil { + return pruned, err + } + pruned++ + } + return pruned, nil +} diff --git a/spindle/engine/cache_prune_test.go b/spindle/engine/cache_prune_test.go new file mode 100644 index 00000000..2cd8b95f --- /dev/null +++ b/spindle/engine/cache_prune_test.go @@ -0,0 +1,188 @@ +package engine + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "tangled.org/core/spindle/db" +) + +func TestPruneCachesExpiryPolicies(t *testing.T) { + ctx := context.Background() + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + type spec struct { + id string + state string + age time.Duration + } + tests := []struct { + name string + retention time.Duration + pendingMax time.Duration + entries []spec + wantPruned int + wantSurvive []string + }{ + { + name: "ready entries expire by last use", + retention: time.Hour, + pendingMax: 15 * time.Minute, + entries: []spec{{"old", "ready", 2 * time.Hour}, {"fresh", "ready", 30 * time.Minute}}, + wantPruned: 1, + wantSurvive: []string{"fresh"}, + }, + { + name: "zero retention keeps ready entries", + retention: 0, + pendingMax: time.Hour, + entries: []spec{{"ready", "ready", 24 * time.Hour}, {"pending", "pending", 2 * time.Hour}}, + wantPruned: 1, + wantSurvive: []string{"ready"}, + }, + { + name: "pending entries expire by age", + retention: time.Hour, + pendingMax: time.Hour, + entries: []spec{{"old", "pending", 2 * time.Hour}, {"fresh", "pending", 10 * time.Minute}}, + wantPruned: 1, + wantSurvive: []string{"fresh"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := newCacheTestDB(t) + store := &fakeStorage{objects: map[string][]byte{}} + keys := map[string]string{} + for _, e := range tt.entries { + entry := cacheTestEntry(e.id, "did:plc:repo", "microvm", "deps", "hash", e.state, now.Add(-e.age)) + insertCacheTestEntry(t, d, entry) + store.objects[entry.StorageKey] = []byte(e.id) + keys[e.id] = entry.StorageKey + } + + pruned, err := PruneCaches(ctx, d, store, now, tt.retention, tt.pendingMax, 10) + if err != nil { + t.Fatalf("PruneCaches: %v", err) + } + if pruned != tt.wantPruned { + t.Fatalf("pruned %d entries, want %d", pruned, tt.wantPruned) + } + for _, e := range tt.entries { + survives := slices.Contains(tt.wantSurvive, e.id) + if store.has(keys[e.id]) != survives || cacheTestEntryExists(t, d, e.id) != survives { + t.Fatalf("entry %q survived = %v, want %v", e.id, !survives, survives) + } + } + }) + } +} + +func TestPruneCachesSkipsEntryRefreshedAfterScan(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + first := cacheTestEntry("first", "did:plc:repo", "microvm", "deps", "first", "ready", now.Add(-3*time.Hour)) + refreshed := cacheTestEntry("refreshed", "did:plc:repo", "microvm", "deps", "refreshed", "ready", now.Add(-2*time.Hour)) + insertCacheTestEntry(t, d, first) + insertCacheTestEntry(t, d, refreshed) + store := &fakeStorage{objects: map[string][]byte{ + first.StorageKey: []byte("first"), + refreshed.StorageKey: []byte("refreshed"), + }} + store.onDelete = func(key string) { + if key != first.StorageKey { + return + } + if err := d.TouchCacheEntry(ctx, refreshed.ID, now); err != nil { + t.Fatalf("TouchCacheEntry: %v", err) + } + } + + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatalf("PruneCaches: %v", err) + } + if pruned != 1 { + t.Fatalf("pruned %d entries, want 1", pruned) + } + if !store.has(refreshed.StorageKey) || !cacheTestEntryExists(t, d, refreshed.ID) { + t.Fatal("entry refreshed after expiry scan was pruned") + } +} + +func TestPruneCachesDeleteFailureRemainsRetryable(t *testing.T) { + for _, initialState := range []string{"ready", "deleting"} { + t.Run(initialState, func(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Date(2026, 5, 6, 7, 8, 9, 0, time.UTC) + entry := cacheTestEntry("retry", "did:plc:repo", "microvm", "deps", "hash", initialState, now.Add(-2*time.Hour)) + insertCacheTestEntry(t, d, entry) + deleteErr := errors.New("delete failed") + store := &fakeStorage{ + objects: map[string][]byte{entry.StorageKey: []byte("archive")}, + deleteErr: deleteErr, + } + + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if !errors.Is(err, deleteErr) { + t.Fatalf("first PruneCaches error = %v, want %v", err, deleteErr) + } + if pruned != 0 { + t.Fatalf("first prune count = %d, want 0", pruned) + } + state, _, _ := cacheTestEntryState(t, d, entry.ID) + if state != initialState { + t.Fatalf("state after delete failure = %q, want %q", state, initialState) + } + if !store.has(entry.StorageKey) { + t.Fatal("failed delete removed object") + } + + pruned, err = PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatalf("retry PruneCaches: %v", err) + } + if pruned != 1 || store.has(entry.StorageKey) || cacheTestEntryExists(t, d, entry.ID) { + t.Fatalf("retry result = pruned %d, object %v, metadata %v", pruned, store.has(entry.StorageKey), cacheTestEntryExists(t, d, entry.ID)) + } + }) + } +} + +func TestPruneCachesRetriesQueuedObjectDeletion(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + now := time.Now() + key := "objects/did:web:example.com/orphan" + if err := d.ApplyEventBatch(nil, func(tx *db.EventBatchTx) error { + return tx.QueueCacheObjectDeletion(ctx, key, now) + }); err != nil { + t.Fatal(err) + } + deleteErr := errors.New("delete failed") + store := &fakeStorage{ + objects: map[string][]byte{key: []byte("archive")}, + deleteErr: deleteErr, + } + if _, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10); !errors.Is(err, deleteErr) { + t.Fatalf("first prune error = %v, want %v", err, deleteErr) + } + pending, err := d.PendingCacheObjectDeletions(ctx, 10) + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 || pending[0] != key { + t.Fatalf("pending object deletions = %v", pending) + } + pruned, err := PruneCaches(ctx, d, store, now, time.Hour, time.Hour, 10) + if err != nil { + t.Fatal(err) + } + if pruned != 1 || store.has(key) { + t.Fatalf("retry pruned %d, object remains %t", pruned, store.has(key)) + } +} diff --git a/spindle/engine/cache_test.go b/spindle/engine/cache_test.go new file mode 100644 index 00000000..ed0ea456 --- /dev/null +++ b/spindle/engine/cache_test.go @@ -0,0 +1,512 @@ +package engine + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/spindle/db" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +type fakeStorage struct { + objects map[string][]byte + putErr error + deleteErr error // fails once, then clears + onDelete func(string) +} + +func (f *fakeStorage) Get(_ context.Context, key string) (io.ReadCloser, error) { + data, ok := f.objects[key] + if !ok { + return nil, storage.ErrNotExist + } + return io.NopCloser(bytes.NewReader(data)), nil +} + +func (f *fakeStorage) Put(_ context.Context, key string, r io.Reader) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.objects[key] = data + return f.putErr +} + +func (f *fakeStorage) Delete(_ context.Context, key string) error { + if f.onDelete != nil { + f.onDelete(key) + } + if f.deleteErr != nil { + err := f.deleteErr + f.deleteErr = nil + return err + } + delete(f.objects, key) + return nil +} + +func (f *fakeStorage) has(key string) bool { + _, ok := f.objects[key] + return ok +} + +func gitRepo(t *testing.T, files map[string]string) (string, string) { + t.Helper() + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + dir := t.TempDir() + run := func(args ...string) { + t.Helper() + cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.email=t@t", "-c", "user.name=t", "-c", "init.defaultBranch=main"}, args...)...) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + } + run("init") + for name, content := range files { + p := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + run("add", ".") + run("commit", "-m", "init") + return dir, "HEAD" +} + +func newCacheTestDB(t *testing.T) *db.DB { + t.Helper() + d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { _ = d.Close() }) + return d +} +func addCacheTestRepo(t *testing.T, d *db.DB) (syntax.DID, syntax.DID) { + t.Helper() + repoDid := syntax.DID("did:plc:repo") + ownerDid := syntax.DID("did:plc:owner") + if err := d.AddRepo(db.Repo{ + Knot: "knot.example", + Owner: ownerDid, + Rkey: syntax.RecordKey("repo"), + RepoDid: repoDid, + }); err != nil { + t.Fatal(err) + } + return repoDid, ownerDid +} + +func cacheTestEntry(id, repoDID, engine, key, hash, state string, at time.Time) db.CacheEntry { + return db.CacheEntry{ + ID: id, + StorageKey: "objects/" + id, + OwnerDID: "did:plc:owner", + RepoDID: repoDID, + Engine: engine, + CacheKey: key, + CacheHash: hash, + State: state, + CreatedAt: at, + LastUsedAt: at, + } +} + +func insertCacheTestEntry(t *testing.T, d *db.DB, entry db.CacheEntry) { + t.Helper() + if err := d.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatalf("InsertCacheEntry(%s): %v", entry.ID, err) + } +} + +func cacheTestEntryState(t *testing.T, d *db.DB, id string) (string, int64, time.Time) { + t.Helper() + var state string + var size, lastUsed int64 + if err := d.QueryRow(`select state, size_bytes, last_used_at from cache_entries where id = ?`, id).Scan(&state, &size, &lastUsed); err != nil { + t.Fatalf("query cache entry %s: %v", id, err) + } + return state, size, time.Unix(0, lastUsed) +} + +func cacheTestEntryExists(t *testing.T, d *db.DB, id string) bool { + t.Helper() + var exists bool + if err := d.QueryRow(`select exists(select 1 from cache_entries where id = ?)`, id).Scan(&exists); err != nil { + t.Fatalf("query cache entry existence %s: %v", id, err) + } + return exists +} + +var discardLogger = slog.New(slog.NewTextHandler(io.Discard, nil)) + +func TestCachePlanPreservesRestoreAtEntryQuota(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoDid, ownerDid := addCacheTestRepo(t, d) + old := cacheTestEntry( + "restore", + repoDid.String(), + "microvm/"+models.CacheNamespace(), + "deps", + "", + "ready", + time.Now(), + ) + old.OwnerDID = ownerDid.String() + insertCacheTestEntry(t, d, old) + + controller := NewLocalCacheController( + d, + &fakeStorage{objects: map[string][]byte{old.StorageKey: []byte("archive")}}, + "", + 0, + 1, + discardLogger, + ) + bindings, err := controller.Plan(ctx, &models.Pipeline{RepoDid: repoDid}, &models.Workflow{ + Engine: "microvm", + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 1 || bindings[0].RestoreID != old.ID || bindings[0].SaveKey != "" { + t.Fatalf("quota binding = %+v, want restore-only %q", bindings, old.ID) + } +} + +func TestCachePlanKeepsPendingEntryThroughWorkflowDeadline(t *testing.T) { + d := newCacheTestDB(t) + repoDid, _ := addCacheTestRepo(t, d) + deadline := time.Now().Add(6 * time.Hour) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + + controller := NewLocalCacheController( + d, + &fakeStorage{objects: make(map[string][]byte)}, + "", + 0, + 0, + discardLogger, + ) + bindings, err := controller.Plan(ctx, &models.Pipeline{RepoDid: repoDid}, &models.Workflow{ + Engine: "microvm", + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }) + if err != nil { + t.Fatal(err) + } + var pendingUntil int64 + if err := d.QueryRow( + `select last_used_at from cache_entries where id = ?`, + bindings[0].SaveID, + ).Scan(&pendingUntil); err != nil { + t.Fatal(err) + } + if pendingUntil != deadline.UnixNano() { + t.Fatalf("pending deadline = %d, want %d", pendingUntil, deadline.UnixNano()) + } +} + +func TestResolveCachesExactUnhashed(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) + + insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "deps", "", "ready", base)) + insertCacheTestEntry(t, d, cacheTestEntry("other-repo", "did:plc:other", "microvm", "deps", "", "ready", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("other-engine", "did:plc:repo", "nixery", "deps", "", "ready", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("pending", "did:plc:repo", "microvm", "deps", "", "pending", base.Add(time.Hour))) + insertCacheTestEntry(t, d, cacheTestEntry("hashed-only", "did:plc:repo", "microvm", "tools", "old", "ready", base)) + + resolved := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", "", "", []models.CacheEntry{ + {Key: "deps", Paths: []string{"/x"}, CompressionLevel: 7, When: "always"}, + {Key: "tools", Paths: []string{"/y"}}, + }) + if len(resolved) != 2 { + t.Fatalf("got %d resolved entries, want 2", len(resolved)) + } + got := resolved[0] + if got.RestoreID != "exact" || got.RestoreKey != "objects/exact" || got.RestoreName != "" { + t.Fatalf("exact restore = (%q, %q, %q)", got.RestoreID, got.RestoreKey, got.RestoreName) + } + if got.SaveKey != "" { + t.Fatalf("resolve allocated a save key %q", got.SaveKey) + } + if got.CompressionLevel != 7 || got.When != "always" { + t.Fatalf("resolved metadata = %+v", got) + } + if resolved[1].RestoreKey != "" { + t.Fatalf("unhashed entry used hashed fallback %q", resolved[1].RestoreKey) + } +} + +func TestResolveCachesHashExactAndFallback(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoPath, rev := gitRepo(t, map[string]string{"go.sum": "v1 contents", "go.mod": "module x"}) + request := []models.CacheEntry{{Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}} + + first := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if first.Hash == "" { + t.Fatal("hash is empty") + } + + base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) + insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "go-mod", first.Hash, "ready", base.Add(time.Minute))) + insertCacheTestEntry(t, d, cacheTestEntry("sibling", "did:plc:repo", "microvm", "go-modules", "newer", "ready", base.Add(time.Hour))) + + exact := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if exact.RestoreID != "exact" || exact.RestoreKey != "objects/exact" || exact.RestoreName != "" { + t.Fatalf("exact generation restore = %+v", exact) + } + + if err := os.WriteFile(filepath.Join(repoPath, "go.sum"), []byte("v2 contents"), 0o644); err != nil { + t.Fatal(err) + } + commit := exec.Command("git", "-C", repoPath, "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "bump") + if out, err := commit.CombinedOutput(); err != nil { + t.Fatalf("commit: %v\n%s", err, out) + } + rotated := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] + if rotated.Hash == first.Hash { + t.Fatal("lockfile change did not rotate the cache hash") + } + if rotated.RestoreID != "exact" || rotated.RestoreKey != "objects/exact" || rotated.RestoreName != "go-mod-"+first.Hash { + t.Fatalf("fallback restore = %+v", rotated) + } +} + +func TestResolveCachesMissingHashFiles(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + repoPath, rev := gitRepo(t, map[string]string{"go.mod": "module x"}) + + partial := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}, + })[0] + if len(partial.Hash) != 12 { + t.Fatalf("partial hash = %q, want 12 characters", partial.Hash) + } + + none := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"nope.lock"}, Paths: []string{"/x"}}, + })[0] + if none.Hash != "" || none.RestoreKey != "" { + t.Fatalf("all-missing result = hash %q, restore %q", none.Hash, none.RestoreKey) + } +} + +func TestTrackedCacheStoreSaveLifecycle(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + old := cacheTestEntry("superseded", "did:plc:repo", "microvm", "deps", "hash", "ready", time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)) + insertCacheTestEntry(t, d, old) + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "hash", "pending", time.Now()) + insertCacheTestEntry(t, d, pending) + base := &fakeStorage{objects: map[string][]byte{old.StorageKey: []byte("old")}} + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", Hash: "hash", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + payload := []byte("cache archive bytes") + if err := store.Put(ctx, pending.StorageKey, bytes.NewReader(payload)); err != nil { + t.Fatalf("tracked Put: %v", err) + } + state, size, _ := cacheTestEntryState(t, d, pending.ID) + if state != "ready" || size != int64(len(payload)) { + t.Fatalf("saved metadata = state %q, size %d", state, size) + } + if got := base.objects[pending.StorageKey]; !bytes.Equal(got, payload) { + t.Fatalf("stored payload = %q, want %q", got, payload) + } + if base.has(old.StorageKey) || cacheTestEntryExists(t, d, old.ID) { + t.Fatal("completed save left the superseded generation") + } + store.cleanup(ctx) + if !base.has(pending.StorageKey) || !cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("cleanup removed a completed cache") + } +} + +func TestTrackedCacheStoreRejectsOverQuotaSave(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + existing := cacheTestEntry("existing", "did:plc:repo", "microvm", "deps", "old", "ready", time.Now()) + existing.SizeBytes = 4 + insertCacheTestEntry(t, d, existing) + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "new", "pending", time.Now()) + pending.SizeBytes = 0 + insertCacheTestEntry(t, d, pending) + + base := &fakeStorage{objects: map[string][]byte{existing.StorageKey: []byte("kept")}} + controller := NewLocalCacheController(d, base, "", 4, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + if err := store.Put(ctx, pending.StorageKey, strings.NewReader("x")); err == nil { + t.Fatal("over-quota cache save succeeded") + } + store.cleanup(ctx) + if base.has(pending.StorageKey) || cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("over-quota save left an object or metadata") + } + if !base.has(existing.StorageKey) || !cacheTestEntryExists(t, d, existing.ID) { + t.Fatal("over-quota save removed the existing cache") + } +} + +func TestTrackedCacheStoreRestoreTouchesEntry(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + old := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC) + entry := cacheTestEntry("restore", "did:plc:repo", "microvm", "deps", "hash", "ready", old) + insertCacheTestEntry(t, d, entry) + base := &fakeStorage{objects: map[string][]byte{entry.StorageKey: []byte("archive")}} + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + RestoreID: entry.ID, RestoreKey: entry.StorageKey, + }}) + + r, err := store.Get(ctx, entry.StorageKey) + if err != nil { + t.Fatalf("tracked Get: %v", err) + } + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read restored object: %v", err) + } + if err := r.Close(); err != nil { + t.Fatalf("close restored object: %v", err) + } + if string(data) != "archive" { + t.Fatalf("restored payload = %q", data) + } + _, _, touched := cacheTestEntryState(t, d, entry.ID) + if !touched.After(old) { + t.Fatalf("last used = %v, want after %v", touched, old) + } +} + +func TestTrackedCacheStoreFailedPutCleanup(t *testing.T) { + ctx := context.Background() + d := newCacheTestDB(t) + putErr := errors.New("put failed") + base := &fakeStorage{objects: make(map[string][]byte), putErr: putErr} + pending := cacheTestEntry("save", "did:plc:repo", "microvm", "deps", "", "pending", time.Now()) + insertCacheTestEntry(t, d, pending) + controller := NewLocalCacheController(d, base, "", 0, 0, discardLogger) + store := newTrackedCacheStore(base, controller, discardLogger, []models.CacheBinding{{ + Key: "deps", SaveID: pending.ID, SaveKey: pending.StorageKey, + }}) + + if err := store.Put(ctx, pending.StorageKey, strings.NewReader("partial")); !errors.Is(err, putErr) { + t.Fatalf("tracked Put error = %v, want %v", err, putErr) + } + store.cleanup(ctx) + if base.has(pending.StorageKey) { + t.Fatal("cleanup left partial object") + } + if cacheTestEntryExists(t, d, pending.ID) { + t.Fatal("cleanup left pending metadata") + } +} + +func TestTrackedCacheStoreRejectsUnpreparedKeys(t *testing.T) { + store := newTrackedCacheStore( + &fakeStorage{objects: map[string][]byte{"objects/missing": []byte("foreign")}}, + &preplannedCacheController{}, + discardLogger, + nil, + ) + if _, err := store.Get(context.Background(), "objects/missing"); err == nil { + t.Fatal("Get accepted a key without restore metadata") + } + if err := store.Put(context.Background(), "objects/missing", strings.NewReader("data")); err == nil { + t.Fatal("Put accepted a key without pending metadata") + } +} + +func TestCacheSaveScriptQuotesPaths(t *testing.T) { + workspace := t.TempDir() + cachePath := filepath.Join(workspace, "cache;name") + if err := os.Mkdir(cachePath, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cachePath, "value"), []byte("cached"), 0o644); err != nil { + t.Fatal(err) + } + + bin := t.TempDir() + zstd := filepath.Join(bin, "zstd") + if err := os.WriteFile(zstd, []byte("#!/bin/sh\nset --\nexec cat\n"), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command("bash", "-c", CacheSaveScript([]string{"cache;name"}, workspace, 0)) + cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + if output, err := cmd.Output(); err != nil || len(output) == 0 { + t.Fatalf("cache save script = %d bytes, %v", len(output), err) + } +} + +func TestCacheDecompressCmd(t *testing.T) { + cases := []struct { + name string + head []byte + want string + }{ + {"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd, 0x00}, "zstd -dc"}, + {"gzip", []byte{0x1f, 0x8b, 0x08, 0x00}, "gzip -dc"}, + {"empty", nil, "zstd -dc"}, + } + for _, tc := range cases { + got := CacheDecompressCmd(bufio.NewReader(bytes.NewReader(tc.head))) + if got != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestCacheBindingSaveOn(t *testing.T) { + cases := []struct { + when string + onFail, onPass bool + }{ + {"", false, true}, + {"on-success", false, true}, + {"always", true, true}, + } + for _, tc := range cases { + binding := models.CacheBinding{When: tc.when} + if got := binding.SaveOn(true); got != tc.onFail { + t.Errorf("when=%q failed run: got %v, want %v", tc.when, got, tc.onFail) + } + if got := binding.SaveOn(false); got != tc.onPass { + t.Errorf("when=%q passing run: got %v, want %v", tc.when, got, tc.onPass) + } + } +} + +var _ storage.Storage = (*fakeStorage)(nil) diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index 87095c2b..2dace54a 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -24,6 +24,7 @@ import ( "tangled.org/core/spindle/observability" "tangled.org/core/spindle/quota" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" ) var ( @@ -224,17 +225,33 @@ func reportWorkflowStatusError(l *slog.Logger, database *db.DB, n *notifier.Noti } } -func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, qm *quota.Manager, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { +func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, qm *quota.Manager, stores *artifactstore.Stores, db *db.DB, n *notifier.Notifier, cacheStore storage.Storage, cacheController CacheController, ctx context.Context, pipeline *models.Pipeline, pipelineId models.PipelineId) { + l.Info("starting all workflows in parallel", "pipeline", pipelineId) + isTrustedRepo := pipeline.TrustedSource && pipeline.RepoDid != "" var allSecrets []secrets.UnlockedSecret // never pass secrets to pipelines that run untrusted (e.g. fork) code - if pipeline.TrustedSource && pipeline.RepoDid != "" { + if isTrustedRepo { if res, err := vault.GetSecretsUnlocked(ctx, secrets.RepoIdentifier(pipeline.RepoDid.String())); err == nil { allSecrets = res } } else if !pipeline.TrustedSource { l.Info("skipping secrets for untrusted pipeline source", "pipeline", pipelineId) } + if cacheController == nil && cacheStore != nil && db != nil { + cacheController = NewLocalCacheController( + db, + cacheStore, + cfg.Server.RepoDir, + cfg.Cache.MaxBytesPerOwner, + cfg.Cache.MaxEntriesPerOwner, + l, + ) + } + cacheEnabled := cacheStore != nil && cacheController != nil && isTrustedRepo + if cacheStore != nil && !pipeline.TrustedSource { + l.Info("skipping caches for untrusted pipeline source", "pipeline", pipelineId) + } secretValues := make([]string, len(allSecrets)) for i, s := range allSecrets { @@ -256,6 +273,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q var wg sync.WaitGroup for eng, wfs := range pipeline.Workflows { workflowTimeout := eng.WorkflowTimeout() + cacheRunner, cachesSupported := eng.(CacheRunner) l.Info("using workflow timeout", "timeout", workflowTimeout) for _, w := range wfs { @@ -564,6 +582,54 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q } destroyWorkflow = true + var bindings []models.CacheBinding + var trackedStore *trackedCacheStore + if cacheEnabled && len(w.Caches) > 0 { + bindings, err = cacheController.Plan(wfCtx, pipeline, &w) + if err != nil { + l.Warn("cache planning failed", "wid", wid, "err", err) + } else { + w.CacheBindings = bindings + trackedStore = newTrackedCacheStore(cacheStore, cacheController, l, bindings) + if cachesSupported { + wfLogger.ControlWriter(CacheRestoreStepIdx, CacheRestoreStep, models.StepStatusStart).Write([]byte{0}) + // caches are an optimization, never a reason to fail the workflow + if err := cacheRunner.RestoreCache(wfCtx, wid, &w, trackedStore, bindings, wfLogger); err != nil { + l.Warn("cache restore failed", "wid", wid, "err", err) + } + wfLogger.ControlWriter(CacheRestoreStepIdx, CacheRestoreStep, models.StepStatusEnd).Write([]byte{0}) + } else if !remoteStatus { + l.Warn("engine does not support caches, skipping restore", "wid", wid) + } + } + } + + cleanupCaches := func() { + if trackedStore != nil { + trackedStore.cleanup(context.WithoutCancel(wfCtx)) + trackedStore = nil + } + } + // dont save on timeouts, their context is already dead + saveCaches := func(failed bool) { + if trackedStore == nil || !cachesSupported { + return + } + toSave := make([]models.CacheBinding, 0, len(bindings)) + for _, binding := range bindings { + if binding.SaveKey != "" && binding.SaveOn(failed) { + toSave = append(toSave, binding) + } + } + if len(toSave) == 0 { + return + } + wfLogger.ControlWriter(CacheSaveStepIdx, CacheSaveStep, models.StepStatusStart).Write([]byte{0}) + if err := cacheRunner.SaveCache(wfCtx, wid, &w, trackedStore, toSave, wfLogger); err != nil { + l.Warn("cache save failed", "wid", wid, "err", err) + } + wfLogger.ControlWriter(CacheSaveStepIdx, CacheSaveStep, models.StepStatusEnd).Write([]byte{0}) + } for stepIdx, step := range w.Steps { if wfLogger != nil { wfLogger. @@ -618,6 +684,10 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q } if err != nil { + if !errors.Is(err, ErrTimedOut) && !errors.Is(wfCtx.Err(), context.DeadlineExceeded) && !isCanceled(wfCtx) { + saveCaches(true) + } + cleanupCaches() if !remoteStatus { setTerminalError("running step", err) } @@ -625,6 +695,9 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q } } + saveCaches(false) + cleanupCaches() + if isCanceled(wfCtx) { if !remoteStatus { setTerminalError("before success", nil) diff --git a/spindle/engine/engine_quota_test.go b/spindle/engine/engine_quota_test.go index 0c122340..7909603a 100644 --- a/spindle/engine/engine_quota_test.go +++ b/spindle/engine/engine_quota_test.go @@ -101,7 +101,7 @@ func TestStartWorkflows_NoDoubleAcquisitionByNoReporterEngine(t *testing.T) { } pipelineId := models.PipelineId{Knot: "knot", Rkey: "rkey"} - StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) store.mu.Lock() resCount := len(store.reserveCalls) @@ -143,7 +143,7 @@ func TestStartWorkflows_QuotaAcquisitionAndIdentity(t *testing.T) { } pipelineId := models.PipelineId{Knot: "knot", Rkey: "rkey"} - StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) store.mu.Lock() resCount := len(store.reserveCalls) @@ -192,7 +192,7 @@ func TestStartWorkflows_QuotaAcquisitionAndIdentity(t *testing.T) { engNoDisk: {wf2}, }, } - StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, context.Background(), pipelineNoDisk, pipelineId) + StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, context.Background(), pipelineNoDisk, pipelineId) store.mu.Lock() resCount2 := len(store.reserveCalls) @@ -236,7 +236,7 @@ func TestStartWorkflows_QuotaFailureRecordsWorkflowFailure(t *testing.T) { } metrics := observability.NewMetrics() ctx := observability.WithMetrics(context.Background(), metrics) - StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, ctx, pipeline, models.PipelineId{Knot: "knot", Rkey: "rkey"}) + StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, ctx, pipeline, models.PipelineId{Knot: "knot", Rkey: "rkey"}) families, err := metrics.Registry().Gather() if err != nil { diff --git a/spindle/engine/engine_test.go b/spindle/engine/engine_test.go index de7ce8c4..6da72c30 100644 --- a/spindle/engine/engine_test.go +++ b/spindle/engine/engine_test.go @@ -6,6 +6,7 @@ import ( "log/slog" "os" "path/filepath" + "reflect" "strings" "sync" "testing" @@ -33,14 +34,49 @@ func (m mockStep) Name() string { return m.name } func (m mockStep) Command() string { return m.command } func (m mockStep) Kind() models.StepKind { return models.StepKindUser } +type emptyVault struct{} + +func (emptyVault) AddSecret(context.Context, secrets.UnlockedSecret) error { + return nil +} + +func (emptyVault) RemoveSecret(context.Context, secrets.Secret[any]) error { + return nil +} + +func (emptyVault) RemoveAllSecrets(context.Context, secrets.RepoIdentifier) error { + return nil +} + +func (emptyVault) GetSecretsLocked(context.Context, secrets.RepoIdentifier) ([]secrets.LockedSecret, error) { + return nil, nil +} + +func (emptyVault) GetSecretsUnlocked(context.Context, secrets.RepoIdentifier) ([]secrets.UnlockedSecret, error) { + return nil, nil +} + +type staticCacheController struct { + bindings []models.CacheBinding +} + +func (c staticCacheController) Plan(context.Context, *models.Pipeline, *models.Workflow) ([]models.CacheBinding, error) { + return c.bindings, nil +} + +func (staticCacheController) Apply(context.Context, CacheUpdate) error { + return nil +} + type mockEngine struct { - mu sync.Mutex - setupCalls []models.WorkflowId - runStepCalls []models.WorkflowId - setupFunc func(ctx context.Context, wid models.WorkflowId) error - runStepFunc func(ctx context.Context, wid models.WorkflowId, idx int, wfLogger models.WorkflowLogger) error - destroyFunc func(ctx context.Context, wid models.WorkflowId) error - timeout time.Duration + mu sync.Mutex + setupCalls []models.WorkflowId + runStepCalls []models.WorkflowId + setupFunc func(ctx context.Context, wid models.WorkflowId) error + runStepFunc func(ctx context.Context, wid models.WorkflowId, idx int, wfLogger models.WorkflowLogger) error + destroyFunc func(ctx context.Context, wid models.WorkflowId) error + runWorkflowFunc func(*models.Workflow) + timeout time.Duration } func (m *mockEngine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipeline) (*models.Workflow, error) { @@ -86,11 +122,15 @@ func (m *mockEngine) DestroyWorkflow(ctx context.Context, wid models.WorkflowId) func (m *mockEngine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.Workflow, idx int, secrets []secrets.UnlockedSecret, wfLogger models.WorkflowLogger) error { m.mu.Lock() m.runStepCalls = append(m.runStepCalls, wid) - fn := m.runStepFunc + runStepFn := m.runStepFunc + runWorkflowFn := m.runWorkflowFunc m.mu.Unlock() - if fn != nil { - return fn(ctx, wid, idx, wfLogger) + if runWorkflowFn != nil { + runWorkflowFn(w) + } + if runStepFn != nil { + return runStepFn(ctx, wid, idx, wfLogger) } return nil } @@ -137,7 +177,7 @@ func TestStartWorkflows_CollisionRejection(t *testing.T) { } cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} - StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) eng.mu.Lock() setupCalls := append([]models.WorkflowId(nil), eng.setupCalls...) @@ -218,7 +258,7 @@ func TestCancelWorkflow_NotOverwritten(t *testing.T) { cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} doneChan := make(chan struct{}) go func() { - StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) close(doneChan) }() @@ -305,7 +345,7 @@ func TestStartWorkflows_FlushesLogBeforeTerminalStatus(t *testing.T) { done := make(chan struct{}) go func() { - StartWorkflows(logger, vault, cfg, nil, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, vault, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) close(done) }() @@ -364,7 +404,7 @@ func TestSetupTimeout_ReportsTimeout(t *testing.T) { } cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} - StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) st, err := testDB.GetStatus(wid) if err != nil { @@ -416,7 +456,7 @@ func TestStartWorkflows_SpanEnrichment(t *testing.T) { } cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} - StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, context.Background(), pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) spans := recorder.Ended() var workflowRunSpan, stepRunSpan sdktrace.ReadOnlySpan @@ -579,7 +619,7 @@ func TestStartWorkflows_ResourceTracking(t *testing.T) { metrics := observability.NewMetrics() ctx := observability.WithMetrics(context.Background(), metrics) - StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, ctx, pipeline, pipelineId) + StartWorkflows(logger, nil, cfg, nil, nil, testDB, nil, nil, nil, ctx, pipeline, pipelineId) handler.mu.Lock() records = *handler.records @@ -694,3 +734,57 @@ func (h *captureHandler) WithAttrs(attrs []slog.Attr) slog.Handler { func (h *captureHandler) WithGroup(name string) slog.Handler { return h } + +func TestStartWorkflowsPassesPlannedCacheBindingsToEngine(t *testing.T) { + t.Parallel() + + database := newTestDB(t) + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) + expected := []models.CacheBinding{{ + EntryIndex: 0, + Paths: []string{"deps"}, + Key: "deps", + SaveID: "cache-id", + SaveKey: "objects/cache-id", + }} + var got []models.CacheBinding + eng := &mockEngine{ + runWorkflowFunc: func(workflow *models.Workflow) { + got = append([]models.CacheBinding(nil), workflow.CacheBindings...) + }, + } + pipelineID := models.PipelineId{Knot: "test-knot", Rkey: "test-rkey"} + pipeline := &models.Pipeline{ + RepoDid: "did:web:example.com", + TrustedSource: true, + Workflows: map[models.Engine][]models.Workflow{ + eng: {{ + Name: "cache-job", + Engine: "microvm", + Steps: []models.Step{mockStep{name: "step1"}}, + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }}, + }, + } + cfg := &config.Config{Server: config.Server{LogDir: t.TempDir()}} + store := &fakeStorage{objects: make(map[string][]byte)} + + StartWorkflows( + logger, + emptyVault{}, + cfg, + nil, + nil, + database, + nil, + store, + staticCacheController{bindings: expected}, + context.Background(), + pipeline, + pipelineID, + ) + + if !reflect.DeepEqual(got, expected) { + t.Fatalf("cache bindings passed to engine = %#v, want %#v", got, expected) + } +} diff --git a/spindle/engines/microvm/cache.go b/spindle/engines/microvm/cache.go new file mode 100644 index 00000000..5db06135 --- /dev/null +++ b/spindle/engines/microvm/cache.go @@ -0,0 +1,132 @@ +//go:build linux + +package microvm + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + + agentv1 "tangled.org/core/spindle/agentproto/gen" + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +func (e *Engine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + out := wfLogger.DataWriter(engine.CacheRestoreStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + if entry.RestoreKey == "" { + fmt.Fprintf(out, "cache %q: miss\n", entry.Key) + continue + } + if entry.RestoreName != "" { + fmt.Fprintf(out, "cache %q: restoring from %q\n", entry.Key, entry.RestoreName) + } + + rc, err := store.Get(ctx, entry.RestoreKey) + if err != nil { + if errors.Is(err, storage.ErrNotExist) { + fmt.Fprintf(out, "cache %q: miss\n", entry.Key) + continue + } + return fmt.Errorf("restore cache %q: fetch: %w", entry.Key, err) + } + br := bufio.NewReader(rc) + decompress := engine.CacheDecompressCmd(br) + + var restored int64 + exit, err := state.Agent.Exec(ctx, AgentExec{ + ID: fmt.Sprintf("%s-cache-restore", wid.String()), + ExecStart: cacheExecStart(state, fmt.Sprintf("set -o pipefail\n%s | tar -x -C /", decompress)), + Stdin: &countingReader{r: br, n: &restored}, + Stderr: out, + }) + closeErr := rc.Close() + if err != nil { + return fmt.Errorf("restore cache %q: %w", entry.Key, err) + } + if closeErr != nil { + return fmt.Errorf("restore cache %q: %w", entry.Key, closeErr) + } + if exit != 0 { + return fmt.Errorf("restore cache %q: guest exited %d", entry.Key, exit) + } + fmt.Fprintf(out, "cache %q: restored %d bytes\n", entry.Key, restored) + } + return nil +} + +func (e *Engine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + state, ok := wf.Data.(*workflowState) + if !ok || state == nil || state.Agent == nil { + return fmt.Errorf("microVM workflow is not connected to agent") + } + + out := wfLogger.DataWriter(engine.CacheSaveStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + + script := engine.CacheSaveScript(entry.Paths, guestWorkDir, entry.CompressionLevel) + + up := engine.NewCacheUpload(ctx, store, entry.SaveKey) + exit, execErr := state.Agent.Exec(ctx, AgentExec{ + ID: fmt.Sprintf("%s-cache-save", wid.String()), + ExecStart: cacheExecStart(state, script), + Stdout: up.Writer, + Stderr: out, + }) + switch { + case exit == engine.CacheExitNoPaths: + up.Abort(fmt.Errorf("guest exited %d", exit)) + fmt.Fprintf(out, "cache %q: nothing to save\n", entry.Key) + continue + case exit == engine.CacheExitNoCompressor: + up.Abort(fmt.Errorf("guest exited %d", exit)) + fmt.Fprintf(out, "cache %q: zstd not available in image; skipping\n", entry.Key) + continue + case execErr != nil: + up.Abort(execErr) + return fmt.Errorf("save cache %q: %w", entry.Key, execErr) + case exit != 0: + up.Abort(fmt.Errorf("guest exited %d", exit)) + return fmt.Errorf("save cache %q: save script exited %d", entry.Key, exit) + } + if err := up.Finish(); err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + fmt.Fprintf(out, "cache %q: saved\n", entry.Key) + } + return nil +} + +func cacheExecStart(state *workflowState, script string) *agentv1.ExecStart { + return &agentv1.ExecStart{ + Argv: []string{state.ImageSpec.Shell, "-c", script}, + Env: guestBaseEnv(), + User: guestWorkflowUser, + } +} + +type countingReader struct { + r io.Reader + n *int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + *c.n += int64(n) + return n, err +} diff --git a/spindle/engines/microvm/engine.go b/spindle/engines/microvm/engine.go index 02516f89..b2cd7c57 100644 --- a/spindle/engines/microvm/engine.go +++ b/spindle/engines/microvm/engine.go @@ -47,6 +47,16 @@ const ( type cleanupFunc func(context.Context) error +// return a fresh slice since callers append to it +func guestBaseEnv() []string { + return []string{ + "HOME=/workspace", + "LOGNAME=" + guestWorkflowUser, + "PATH=" + guestBasePATH, + "USER=" + guestWorkflowUser, + } +} + type Engine struct { l *slog.Logger cfg *config.Config @@ -177,6 +187,12 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin swf.Name = twf.Name swf.Environment = dwf.Environment + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + swf.Caches = caches + if tpl.TriggerMetadata != nil { if clone := models.BuildCloneStep(twf, *tpl.TriggerMetadata, e.cfg.Server.Dev); clone.Command() != "" { swf.Steps = append([]models.Step{clone}, swf.Steps...) @@ -462,12 +478,7 @@ func applyDepsSource(command string) string { } func stepEnvironment(w *models.Workflow, step models.Step, unlocked []secrets.UnlockedSecret) []string { - env := []string{ - "HOME=/workspace", - "LOGNAME=" + guestWorkflowUser, - "PATH=" + guestBasePATH, - "USER=" + guestWorkflowUser, - } + env := guestBaseEnv() for k, v := range w.Environment { env = append(env, k+"="+v) } diff --git a/spindle/engines/microvm/models.go b/spindle/engines/microvm/models.go index df4d20c8..7fdf9ccf 100644 --- a/spindle/engines/microvm/models.go +++ b/spindle/engines/microvm/models.go @@ -5,16 +5,19 @@ package microvm import ( "fmt" "slices" + + "tangled.org/core/spindle/models" ) type manifestWorkflow struct { - Image string `yaml:"image"` - Services map[string]any `yaml:"services"` - Virtualisation map[string]any `yaml:"virtualisation"` - Dependencies []string `yaml:"dependencies"` - Registry map[string]any `yaml:"registry"` - Environment map[string]string `yaml:"environment"` - Substituters map[string]string `yaml:"substituters"` + Image string `yaml:"image"` + Services map[string]any `yaml:"services"` + Virtualisation map[string]any `yaml:"virtualisation"` + Dependencies []string `yaml:"dependencies"` + Registry map[string]any `yaml:"registry"` + Environment map[string]string `yaml:"environment"` + Substituters map[string]string `yaml:"substituters"` + Cache []models.CacheEntry `yaml:"cache"` Steps []struct { Name string `yaml:"name"` Command string `yaml:"command"` diff --git a/spindle/engines/nixery/cache.go b/spindle/engines/nixery/cache.go new file mode 100644 index 00000000..1422593d --- /dev/null +++ b/spindle/engines/nixery/cache.go @@ -0,0 +1,167 @@ +package nixery + +import ( + "bufio" + "context" + "fmt" + "io" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/pkg/stdcopy" + + "tangled.org/core/spindle/engine" + "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" +) + +func (e *Engine) baseEnv() EnvVars { + envs := EnvVars{} + envs.AddEnv("HOME", homeDir) + envs.AddEnv("PATH", fmt.Sprintf("%s/.nix-profile/bin:/nix/var/nix/profiles/default/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", homeDir)) + return envs +} + +func (e *Engine) execAttached(ctx context.Context, containerID string, opts container.ExecOptions) (string, types.HijackedResponse, error) { + execResp, err := e.docker.ContainerExecCreate(ctx, containerID, opts) + if err != nil { + return "", types.HijackedResponse{}, fmt.Errorf("create exec: %w", err) + } + attach, err := e.docker.ContainerExecAttach(ctx, execResp.ID, container.ExecAttachOptions{}) + if err != nil { + return "", types.HijackedResponse{}, fmt.Errorf("attach exec: %w", err) + } + return execResp.ID, attach, nil +} + +func (e *Engine) containerID(wf *models.Workflow) (string, error) { + addl, ok := wf.Data.(addlFields) + if !ok || addl.container == "" { + return "", fmt.Errorf("nixery workflow has no container") + } + return addl.container, nil +} + +func (e *Engine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + containerID, err := e.containerID(wf) + if err != nil { + return err + } + + out := wfLogger.DataWriter(engine.CacheRestoreStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + if entry.RestoreKey == "" { + fmt.Fprintf(out, "cache %q: miss\n", entry.Key) + continue + } + if entry.RestoreName != "" { + fmt.Fprintf(out, "cache %q: restoring from %q\n", entry.Key, entry.RestoreName) + } + + rc, err := store.Get(ctx, entry.RestoreKey) + if err != nil { + fmt.Fprintf(out, "cache %q: fetch failed: %v\n", entry.Key, err) + continue + } + + br := bufio.NewReader(rc) + execID, attach, err := e.execAttached(ctx, containerID, container.ExecOptions{ + Cmd: []string{"bash", "-c", engine.CacheDecompressCmd(br) + " | tar -x -C /"}, + Env: e.baseEnv(), + AttachStdin: true, + AttachStdout: true, + AttachStderr: true, + }) + if err != nil { + rc.Close() + return fmt.Errorf("restore cache %q: %w", entry.Key, err) + } + + // drain this now or tar can block on stderr before reading stdin + copyDone := make(chan error, 1) + go func() { + _, err := io.Copy(attach.Conn, br) + _ = attach.CloseWrite() + copyDone <- err + }() + _, _ = stdcopy.StdCopy(out, out, attach.Reader) + copyErr := <-copyDone + rc.Close() + attach.Close() + if copyErr != nil { + return fmt.Errorf("restore cache %q: stream archive: %w", entry.Key, copyErr) + } + + inspect, err := e.docker.ContainerExecInspect(ctx, execID) + if err != nil { + return fmt.Errorf("restore cache %q: %w", entry.Key, err) + } + if inspect.ExitCode != 0 { + fmt.Fprintf(out, "cache %q: extract failed (exit %d)\n", entry.Key, inspect.ExitCode) + continue + } + fmt.Fprintf(out, "cache %q: restored\n", entry.Key) + } + return nil +} + +func (e *Engine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + containerID, err := e.containerID(wf) + if err != nil { + return err + } + + out := wfLogger.DataWriter(engine.CacheSaveStepIdx, "stdout") + for _, entry := range caches { + if err := ctx.Err(); err != nil { + return err + } + + script := engine.CacheSaveScript(entry.Paths, workspaceDir, entry.CompressionLevel) + + execID, attach, err := e.execAttached(ctx, containerID, container.ExecOptions{ + Cmd: []string{"bash", "-c", script}, + Env: e.baseEnv(), + AttachStdout: true, + AttachStderr: true, + }) + if err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + + up := engine.NewCacheUpload(ctx, store, entry.SaveKey) + + // StdCopy only returns once the archive is fully written + _, copyErr := stdcopy.StdCopy(up.Writer, out, attach.Reader) + attach.Close() + inspect, inspectErr := e.docker.ContainerExecInspect(ctx, execID) + + switch { + case inspectErr == nil && inspect.ExitCode == engine.CacheExitNoPaths: + up.Abort(fmt.Errorf("no cache paths")) + fmt.Fprintf(out, "cache %q: nothing to save\n", entry.Key) + continue + case inspectErr == nil && inspect.ExitCode == engine.CacheExitNoCompressor: + up.Abort(fmt.Errorf("zstd not available")) + fmt.Fprintf(out, "cache %q: zstd not available in image; skipping\n", entry.Key) + continue + case copyErr != nil: + up.Abort(copyErr) + return fmt.Errorf("save cache %q: stream archive: %w", entry.Key, copyErr) + case inspectErr != nil: + up.Abort(inspectErr) + return fmt.Errorf("save cache %q: %w", entry.Key, inspectErr) + case inspect.ExitCode != 0: + up.Abort(fmt.Errorf("exited %d", inspect.ExitCode)) + return fmt.Errorf("save cache %q: tar exited %d", entry.Key, inspect.ExitCode) + } + if err := up.Finish(); err != nil { + return fmt.Errorf("save cache %q: %w", entry.Key, err) + } + fmt.Fprintf(out, "cache %q: saved\n", entry.Key) + } + return nil +} diff --git a/spindle/engines/nixery/engine.go b/spindle/engines/nixery/engine.go index 22a5aa56..c3f9b6e0 100644 --- a/spindle/engines/nixery/engine.go +++ b/spindle/engines/nixery/engine.go @@ -92,6 +92,7 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin } `yaml:"steps"` Dependencies map[string][]string `yaml:"dependencies"` Environment map[string]string `yaml:"environment"` + Cache []models.CacheEntry `yaml:"cache"` }{} if err := engine.DescribeManifestError(twf.Raw, dwf); err != nil { return nil, engine.ClassifiedFailure(engine.FailureClassUser, engine.FailureReasonWorkflowInvalid, err) @@ -135,6 +136,11 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin swf.Name = twf.Name swf.Environment = dwf.Environment + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + swf.Caches = caches addl.image = workflowImage(dwf.Dependencies, e.cfg.NixeryPipelines.Nixery) if sock := e.cfg.Server.DockerSocket; sock != "" { @@ -181,7 +187,7 @@ func workflowImage(deps map[string][]string, nixery string) string { } // load defaults from somewhere else - dependencies = path.Join(dependencies, "bash", "git", "coreutils", "nix") + dependencies = path.Join(dependencies, "bash", "git", "coreutils", "gnutar", "zstd", "nix") if runtime.GOARCH == "arm64" { dependencies = path.Join("arm64", dependencies) @@ -436,9 +442,7 @@ func (e *Engine) RunStep(ctx context.Context, wid models.WorkflowId, w *models.W } } - envs.AddEnv("HOME", homeDir) - existingPath := "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - envs.AddEnv("PATH", fmt.Sprintf("%s/.nix-profile/bin:/nix/var/nix/profiles/default/bin:%s", homeDir, existingPath)) + envs = append(envs, e.baseEnv()...) if sock := e.cfg.Server.DockerSocket; sock != "" { envs.AddEnv("DOCKER_HOST", fmt.Sprintf("unix://%s", sock)) } diff --git a/spindle/mill/auth_test.go b/spindle/mill/auth_test.go index 6464d1df..77a3b0a8 100644 --- a/spindle/mill/auth_test.go +++ b/spindle/mill/auth_test.go @@ -379,11 +379,11 @@ func TestAuthLabelEscalation(t *testing.T) { enc := millproto.NewEncoder(stream) dec := millproto.NewDecoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux", "gpu"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux", "gpu"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc.Encode(hello); err != nil { t.Fatalf("encode hello: %v", err) @@ -408,11 +408,11 @@ func TestAuthLabelEscalation(t *testing.T) { enc := millproto.NewEncoder(stream) dec := millproto.NewDecoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc.Encode(hello); err != nil { t.Fatalf("encode hello: %v", err) @@ -450,11 +450,11 @@ func TestHandshakeTimeoutAndConcurrency(t *testing.T) { stream := millproto.NewWSStream(conn) enc := millproto.NewEncoder(stream) - hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} err = enc.Encode(hello) dec := millproto.NewDecoder(stream) @@ -478,11 +478,11 @@ func TestHandshakeTimeoutAndConcurrency(t *testing.T) { stream1 := millproto.NewWSStream(conn1) enc1 := millproto.NewEncoder(stream1) dec1 := millproto.NewDecoder(stream1) - hello1 := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + hello1 := &millproto.Message{Hello: &millv1.Hello{ProtocolVersion: millproto.ProtocolVersion, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }} if err := enc1.Encode(hello1); err != nil { t.Fatalf("encode hello 1: %v", err) diff --git a/spindle/mill/cache.go b/spindle/mill/cache.go new file mode 100644 index 00000000..eb7515c0 --- /dev/null +++ b/spindle/mill/cache.go @@ -0,0 +1,51 @@ +package mill + +import ( + "fmt" + "tangled.org/core/spindle/db" + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func cacheBindingsToProto(bindings []models.CacheBinding) []*millv1.CacheBinding { + result := make([]*millv1.CacheBinding, len(bindings)) + for i, binding := range bindings { + result[i] = &millv1.CacheBinding{ + EntryIndex: uint32(binding.EntryIndex), + RestoreId: binding.RestoreID, + RestoreKey: binding.RestoreKey, + RestoreName: binding.RestoreName, + RestoreChecksum: binding.Checksum, + RestoreSizeBytes: binding.SizeBytes, + SaveId: binding.SaveID, + SaveKey: binding.SaveKey, + } + } + return result +} + +func cacheCapabilities(repoDid string, bindings []models.CacheBinding) ([]db.MillCacheCapability, error) { + capabilities := make([]db.MillCacheCapability, 0, len(bindings)*2) + for _, binding := range bindings { + for _, capability := range []db.MillCacheCapability{ + {Action: "restore", CacheID: binding.RestoreID, StorageKey: binding.RestoreKey}, + {Action: "save", CacheID: binding.SaveID, StorageKey: binding.SaveKey}, + } { + if capability.CacheID == "" && capability.StorageKey == "" { + continue + } + if capability.CacheID == "" || capability.StorageKey == "" { + return nil, fmt.Errorf("%s cache id and key must both be set", capability.Action) + } + want, err := models.CacheObjectKey(repoDid, capability.CacheID) + if err != nil { + return nil, err + } + if capability.StorageKey != want { + return nil, fmt.Errorf("%s cache key %q does not match object id", capability.Action, capability.StorageKey) + } + capabilities = append(capabilities, capability) + } + } + return capabilities, nil +} diff --git a/spindle/mill/engine.go b/spindle/mill/engine.go index 2fd35045..43cf1d25 100644 --- a/spindle/mill/engine.go +++ b/spindle/mill/engine.go @@ -48,12 +48,22 @@ func (e *Engine) InitWorkflow(twf tangled.Pipeline_Workflow, tpl tangled.Pipelin repoDID = *md.Repo.RepoDid } } + caches, err := models.ParseCacheEntries(twf.Raw) + if err != nil { + return nil, err + } + if e.mill.cfg.CacheStoreID == "" { + caches = nil + } + return &models.Workflow{ Name: twf.Name, Environment: map[string]string{}, OwnerDID: ownerDID, RepoDID: repoDID, Steps: []models.Step{remoteStep{}}, + Caches: caches, + Engine: e.name, Data: &millWorkflowState{ RawWorkflow: twf, RawPipeline: tpl, diff --git a/spindle/mill/executor/cache.go b/spindle/mill/executor/cache.go new file mode 100644 index 00000000..5c684720 --- /dev/null +++ b/spindle/mill/executor/cache.go @@ -0,0 +1,63 @@ +package executor + +import ( + "fmt" + + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func cacheBindingsFromProto(repoDid string, entries []models.CacheEntry, bindings []*millv1.CacheBinding) ([]models.CacheBinding, error) { + result := make([]models.CacheBinding, 0, len(bindings)) + seen := make(map[int]struct{}, len(bindings)) + for _, binding := range bindings { + entryIndex := int(binding.GetEntryIndex()) + if entryIndex < 0 || entryIndex >= len(entries) { + return nil, fmt.Errorf("cache binding index %d is out of range", entryIndex) + } + if _, ok := seen[entryIndex]; ok { + return nil, fmt.Errorf("duplicate cache binding index %d", entryIndex) + } + seen[entryIndex] = struct{}{} + + for _, object := range []struct { + name string + id string + key string + }{ + {name: "restore", id: binding.GetRestoreId(), key: binding.GetRestoreKey()}, + {name: "save", id: binding.GetSaveId(), key: binding.GetSaveKey()}, + } { + if (object.id == "") != (object.key == "") { + return nil, fmt.Errorf("%s cache id and key must both be set", object.name) + } + if object.id == "" { + continue + } + want, err := models.CacheObjectKey(repoDid, object.id) + if err != nil { + return nil, err + } + if object.key != want { + return nil, fmt.Errorf("%s cache key %q does not match object id", object.name, object.key) + } + } + + entry := entries[entryIndex] + result = append(result, models.CacheBinding{ + EntryIndex: entryIndex, + Paths: entry.Paths, + Key: entry.Key, + SaveID: binding.GetSaveId(), + SaveKey: binding.GetSaveKey(), + RestoreID: binding.GetRestoreId(), + RestoreKey: binding.GetRestoreKey(), + RestoreName: binding.GetRestoreName(), + Checksum: binding.GetRestoreChecksum(), + SizeBytes: binding.GetRestoreSizeBytes(), + CompressionLevel: entry.CompressionLevel, + When: entry.When, + }) + } + return result, nil +} diff --git a/spindle/mill/executor/cache_test.go b/spindle/mill/executor/cache_test.go new file mode 100644 index 00000000..a7547693 --- /dev/null +++ b/spindle/mill/executor/cache_test.go @@ -0,0 +1,66 @@ +package executor + +import ( + "testing" + + millv1 "tangled.org/core/spindle/mill/proto/gen" + "tangled.org/core/spindle/models" +) + +func TestCacheBindingsFromProtoUsesValidatedWorkflowEntries(t *testing.T) { + const ( + repoDid = "did:web:example.com" + restoreID = "11111111-1111-4111-8111-111111111111" + saveID = "22222222-2222-4222-8222-222222222222" + ) + entries := []models.CacheEntry{ + {Key: "deps", Paths: []string{"node_modules"}, CompressionLevel: 7, When: "always"}, + } + bindings, err := cacheBindingsFromProto(repoDid, entries, []*millv1.CacheBinding{{ + EntryIndex: 0, + RestoreId: restoreID, + RestoreKey: "objects/" + repoDid + "/" + restoreID, + SaveId: saveID, + SaveKey: "objects/" + repoDid + "/" + saveID, + }}) + if err != nil { + t.Fatal(err) + } + if len(bindings) != 1 { + t.Fatalf("bindings = %+v", bindings) + } + got := bindings[0] + if got.Key != "deps" || got.RestoreID != restoreID || got.SaveID != saveID || got.CompressionLevel != 7 || got.When != "always" { + t.Fatalf("binding = %+v", got) + } +} + +func TestCacheBindingsFromProtoRejectsInvalidIndexes(t *testing.T) { + entries := []models.CacheEntry{{Key: "deps", Paths: []string{"node_modules"}}} + for _, bindings := range [][]*millv1.CacheBinding{ + {{EntryIndex: 1}}, + {{EntryIndex: 0}, {EntryIndex: 0}}, + } { + if _, err := cacheBindingsFromProto("did:web:example.com", entries, bindings); err == nil { + t.Fatalf("cacheBindingsFromProto(%+v) succeeded", bindings) + } + } +} + +func TestCacheBindingsFromProtoRejectsForeignAndUnpairedObjects(t *testing.T) { + const id = "11111111-1111-4111-8111-111111111111" + entries := []models.CacheEntry{{Key: "deps", Paths: []string{"node_modules"}}} + for _, binding := range []*millv1.CacheBinding{ + {EntryIndex: 0, SaveId: id}, + {EntryIndex: 0, SaveKey: "objects/did:web:example.com/" + id}, + {EntryIndex: 0, RestoreId: id, RestoreKey: "objects/did:web:other.example/" + id}, + } { + if _, err := cacheBindingsFromProto( + "did:web:example.com", + entries, + []*millv1.CacheBinding{binding}, + ); err == nil { + t.Fatalf("foreign cache binding %+v succeeded", binding) + } + } +} diff --git a/spindle/mill/executor/capability_test.go b/spindle/mill/executor/capability_test.go index 1aa4001c..37e2c5ff 100644 --- a/spindle/mill/executor/capability_test.go +++ b/spindle/mill/executor/capability_test.go @@ -96,6 +96,7 @@ func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { RawPipelineJson: string(tpl), Knot: "k", Rkey: "r", + RepoDid: "did:web:example.com", }) result := (<-enc.messages).GetReserveResult() @@ -150,3 +151,51 @@ func TestHandleReservePreservesTypedInitFailure(t *testing.T) { ) } } + +func TestHandleReserveKeepsTriggerMetadataForExecution(t *testing.T) { + e := testExecutor(t) + e.enc = newCaptureEncoder() + e.seats = 1 + e.engines = map[string]models.Engine{"microvm": &fakeEngine{}} + + repoDID := "did:web:example.com" + metadata := &tangled.Pipeline_TriggerMetadata{ + Kind: "push", + Push: &tangled.Pipeline_PushTriggerData{ + NewSha: "0123456789abcdef", + Ref: "refs/heads/main", + }, + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDID}, + } + twf, err := json.Marshal(tangled.Pipeline_Workflow{Name: "build"}) + if err != nil { + t.Fatal(err) + } + tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: metadata}) + if err != nil { + t.Fatal(err) + } + + e.handleReserve(context.Background(), &millv1.ReserveSeat{ + LeaseId: "lease-1", + TargetEngine: "microvm", + RawWorkflowJson: string(twf), + RawPipelineJson: string(tpl), + Knot: "k", + Rkey: "r", + RepoDid: repoDID, + }) + + result := (<-e.enc.(*captureEncoder).messages).GetReserveResult() + if result == nil || !result.GetAccepted() { + t.Fatalf("ReserveResult = %+v, want accepted", result) + } + res := e.active["lease-1"] + if res == nil || res.pipeline.TriggerMetadata == nil || res.pipeline.TriggerMetadata.Push == nil { + t.Fatal("reservation dropped trigger metadata") + } + if got := res.pipeline.TriggerMetadata.Push.NewSha; got != metadata.Push.NewSha { + t.Fatalf("trigger commit = %q, want %q", got, metadata.Push.NewSha) + } + res.ttlTimer.Stop() +} diff --git a/spindle/mill/executor/executor.go b/spindle/mill/executor/executor.go index 0d11d20a..976c56b4 100644 --- a/spindle/mill/executor/executor.go +++ b/spindle/mill/executor/executor.go @@ -1,10 +1,12 @@ package executor import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "io" "log/slog" "maps" "net/http" @@ -30,6 +32,7 @@ import ( "tangled.org/core/spindle/models" "tangled.org/core/spindle/observability" "tangled.org/core/spindle/quota" + "tangled.org/core/spindle/storage" ) const ( @@ -53,6 +56,7 @@ type Executor struct { cfg *config.Config l *slog.Logger writer artifactstore.Writer + cache storage.Storage epoch string outboxBytes int64 @@ -91,7 +95,7 @@ type reservation struct { realEngine models.Engine slot engine.WorkflowSlot wf *models.Workflow - repoDid syntax.DID + pipeline *models.Pipeline vault *memVault committed bool @@ -110,7 +114,7 @@ type messageEncoder interface { Encode(*millproto.Message) error } -func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writers ...artifactstore.Writer) (*Executor, error) { +func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *notifier.Notifier, l *slog.Logger, writer artifactstore.Writer, cache storage.Storage) (*Executor, error) { seats := defaultSeats millURL := "" token := "" @@ -128,10 +132,6 @@ func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *noti if d == nil || n == nil { return nil, fmt.Errorf("executor requires a database and notifier") } - var writer artifactstore.Writer - if len(writers) > 0 { - writer = writers[0] - } e := &Executor{ millURL: millURL, token: token, @@ -144,6 +144,7 @@ func New(cfg *config.Config, engines map[string]models.Engine, d *db.DB, n *noti cfg: cfg, l: l.With("component", "mill.executor"), writer: writer, + cache: cache, active: make(map[string]*reservation), maxOutboxBytes: 10 * 1024 * 1024, } @@ -228,11 +229,19 @@ func (e *Executor) runSession(ctx context.Context) error { enc := millproto.NewEncoder(stream) dec := millproto.NewDecoder(stream) + cacheStoreID := "" + if e.cfg != nil && e.cfg.Cache.StoreID != "" && e.cacheStoreAvailable(ctx) { + cacheStoreID = e.cfg.Cache.StoreID + } hello := &millproto.Message{Hello: &millv1.Hello{ - ProtocolVersion: millproto.ProtocolVersion, - Arch: runtime.GOARCH, - Labels: e.labels, - Epoch: e.epoch, + ProtocolVersion: millproto.ProtocolVersion, + MinProtocolVersion: millproto.ProtocolMinVersion, + MaxProtocolVersion: millproto.ProtocolMaxVersion, + Arch: runtime.GOARCH, + Labels: e.labels, + Epoch: e.epoch, + CacheStoreId: cacheStoreID, + CacheNamespace: models.CacheNamespace(), }} if err := enc.Encode(hello); err != nil { return fmt.Errorf("send hello: %w", err) @@ -308,6 +317,24 @@ func (e *Executor) send(msg *millproto.Message) { } } +func (e *Executor) cacheStoreAvailable(ctx context.Context) bool { + if e.cache == nil { + return false + } + rc, err := e.cache.Get(ctx, millproto.CacheSentinelKey) + if err != nil { + e.l.Warn("cache store is not shared with mill; disabling cache placement", "key", millproto.CacheSentinelKey, "err", err) + return false + } + defer rc.Close() + value, err := io.ReadAll(io.LimitReader(rc, int64(len(millproto.CacheSentinelValue)+1))) + if err != nil || !bytes.Equal(value, []byte(millproto.CacheSentinelValue)) { + e.l.Warn("cache store sentinel is invalid; disabling cache placement", "key", millproto.CacheSentinelKey) + return false + } + return true +} + func (e *Executor) dispatch(ctx context.Context, msg *millproto.Message) { switch { case msg.GetReserveSeat() != nil: @@ -463,6 +490,13 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { reject("pipeline missing trigger metadata", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) return } + repoDid, err := syntax.ParseDID(rs.GetRepoDid()) + if err != nil { + reject("bad repository did", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + trustedSource := models.TrustedPipelineSource(tpl.TriggerMetadata, repoDid.String()) + wf, err := realEngine.InitWorkflow(twf, tpl) if err != nil { rejectError("init workflow: ", err, millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) @@ -476,6 +510,8 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { } } } + wf.Engine = rs.GetTargetEngine() + wf.RepoDID = rs.GetRepoDid() if validator, ok := realEngine.(engine.WorkflowPlacementValidator); ok { if err := validator.ValidateWorkflowPlacement(wf); err != nil { rejectError("validate workflow placement: ", err, millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) @@ -497,14 +533,8 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { return } - var repoDid syntax.DID - if tpl.TriggerMetadata != nil && tpl.TriggerMetadata.Repo != nil && tpl.TriggerMetadata.Repo.RepoDid != nil { - repoDid, _ = syntax.ParseDID(*tpl.TriggerMetadata.Repo.RepoDid) - } - if span.IsRecording() { - if repoDid.String() != "" { - span.SetAttributes(attribute.String(observability.RepoDIDKey, repoDid.String())) - } + if span.IsRecording() && repoDid.String() != "" { + span.SetAttributes(attribute.String(observability.RepoDIDKey, repoDid.String())) } res := &reservation{ @@ -513,8 +543,12 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { realEngine: realEngine, slot: slot, wf: wf, - repoDid: repoDid, traceParent: trace.SpanContextFromContext(reserveCtx), + pipeline: &models.Pipeline{ + RepoDid: repoDid, + TrustedSource: trustedSource, + TriggerMetadata: tpl.TriggerMetadata, + }, } e.snapshotMu.Lock() @@ -579,12 +613,29 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { e.sendCommitted(cl.GetLeaseId()) return } + var bindings []models.CacheBinding + if res.pipeline.TrustedSource { + var err error + bindings, err = cacheBindingsFromProto(res.pipeline.RepoDid.String(), res.wf.Caches, cl.GetCacheBindings()) + if err != nil { + e.mu.Unlock() + e.sendReject(cl.GetLeaseId(), "invalid cache plan: "+err.Error(), millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) + return + } + } + res.wf.CacheBindings = bindings res.committed = true res.millRecordsTerminalMetrics = cl.GetMillRecordsTerminalMetrics() if res.ttlTimer != nil { res.ttlTimer.Stop() } + committedSecrets := cl.GetSecrets() + if !res.pipeline.TrustedSource { + committedSecrets = nil + } + res.vault = newMemVault(committedSecrets) + parentCtx := e.lifecycleCtx if parentCtx == nil { parentCtx = ctx @@ -604,8 +655,8 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { attribute.String(observability.ExecutorNodeIDKey, e.nodeID), attribute.String(observability.PipelineIDKey, res.wid.PipelineId.AtUri().String()), } - if res.repoDid.String() != "" { - attrs = append(attrs, attribute.String(observability.RepoDIDKey, res.repoDid.String())) + if res.pipeline.RepoDid.String() != "" { + attrs = append(attrs, attribute.String(observability.RepoDIDKey, res.pipeline.RepoDid.String())) } if res.wf != nil && res.wf.OwnerDID != "" { attrs = append(attrs, attribute.String(observability.OwnerDIDKey, res.wf.OwnerDID)) @@ -629,13 +680,12 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { res.runDone = make(chan struct{}) e.mu.Unlock() - vault := newMemVault(cl.GetSecrets()) + vault := res.vault re := newReservedEngine(res.realEngine, res.slot) - pipeline := &models.Pipeline{ - RepoDid: res.repoDid, - Workflows: map[models.Engine][]models.Workflow{re: {*res.wf}}, - TrustedSource: true, - } + res.pipeline.Workflows = map[models.Engine][]models.Workflow{re: {*res.wf}} + cacheController := engine.NewPreplannedCacheController(func(ctx context.Context, update engine.CacheUpdate) error { + return e.appendCacheUpdate(res.leaseID, update) + }) e.startTail(res) @@ -648,7 +698,7 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { "lease_id", res.leaseID, "node_id", e.nodeID, ) - engine.StartWorkflows(el, vault, e.cfg, nil, nil, e.db, e.n, jobCtx, pipeline, res.wid.PipelineId) + engine.StartWorkflows(el, vault, e.cfg, nil, nil, e.db, e.n, e.cache, cacheController, jobCtx, res.pipeline, res.wid.PipelineId) }() e.sendCommitted(cl.GetLeaseId()) } diff --git a/spindle/mill/executor/outbox.go b/spindle/mill/executor/outbox.go index a01fca2d..d67feb1d 100644 --- a/spindle/mill/executor/outbox.go +++ b/spindle/mill/executor/outbox.go @@ -76,12 +76,19 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro entry.Payload = payload case *millv1.Event_AttemptResult: entry.Payload = payload + case *millv1.Event_CacheUpdate: + entry.Payload = payload default: return fmt.Errorf("unsupported stream payload %T", payload) } + durable := false isTerminal := false - if _, ok := payload.(*millv1.Event_AttemptResult); ok { + switch payload.(type) { + case *millv1.Event_AttemptResult: + durable = true isTerminal = true + case *millv1.Event_CacheUpdate: + durable = true } entry.Seqno = ^uint64(0) wireSize := proto.Size(&millproto.Message{EventBatch: &millv1.EventBatch{ @@ -104,7 +111,7 @@ func (e *Executor) appendAndSend(leaseID string, payload any, control bool) erro e.eventMu.Unlock() return nil } - if control && !isTerminal && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { + if control && !durable && e.maxOutboxBytes > 0 && e.outboxBytes+int64(len(encoded)) > e.maxOutboxBytes { e.l.Warn("outbox reserve exhausted; dropping nonterminal status", "cap", e.maxOutboxBytes) e.eventMu.Unlock() return nil @@ -140,6 +147,29 @@ func (e *Executor) appendStatus(leaseID string, st *tangled.PipelineStatus) erro return e.appendAndSend(leaseID, payload, true) } +func (e *Executor) appendCacheUpdate(leaseID string, update engine.CacheUpdate) error { + var action millv1.CacheUpdateAction + switch update.Action { + case engine.CacheUsed: + action = millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_USED + case engine.CacheStored: + action = millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED + case engine.CacheDiscarded: + action = millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_DISCARDED + case engine.CacheMissing: + action = millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_MISSING + default: + return fmt.Errorf("unsupported cache update action %d", update.Action) + } + payload := &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: action, + Id: update.ID, + SizeBytes: update.SizeBytes, + Checksum: update.Checksum, + }} + return e.appendAndSend(leaseID, payload, true) +} + func (e *Executor) appendTerminal(leaseID, status string, st *tangled.PipelineStatus) error { failureClass, failureReason := defaultTerminalAttribution(status) return e.appendTerminalWithArtifact(leaseID, status, st, "", "", failureClass, failureReason, true) diff --git a/spindle/mill/executor/reserved.go b/spindle/mill/executor/reserved.go index d3309da9..54a327c0 100644 --- a/spindle/mill/executor/reserved.go +++ b/spindle/mill/executor/reserved.go @@ -9,6 +9,7 @@ import ( "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" "tangled.org/core/spindle/quota" + "tangled.org/core/spindle/storage" ) type reservedEngine struct { @@ -17,8 +18,25 @@ type reservedEngine struct { once sync.Once } +type reservedCacheEngine struct { + *reservedEngine + runner engine.CacheRunner +} + func newReservedEngine(inner models.Engine, slot engine.WorkflowSlot) models.Engine { - return &reservedEngine{Engine: inner, slot: slot} + reserved := &reservedEngine{Engine: inner, slot: slot} + if runner, ok := inner.(engine.CacheRunner); ok { + return &reservedCacheEngine{reservedEngine: reserved, runner: runner} + } + return reserved +} + +func (e *reservedCacheEngine) RestoreCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + return e.runner.RestoreCache(ctx, wid, wf, store, caches, wfLogger) +} + +func (e *reservedCacheEngine) SaveCache(ctx context.Context, wid models.WorkflowId, wf *models.Workflow, store storage.Storage, caches []models.CacheBinding, wfLogger models.WorkflowLogger) error { + return e.runner.SaveCache(ctx, wid, wf, store, caches, wfLogger) } func (e *reservedEngine) MetricEngineName() string { diff --git a/spindle/mill/executor/reserved_test.go b/spindle/mill/executor/reserved_test.go index efed26d7..cba9e478 100644 --- a/spindle/mill/executor/reserved_test.go +++ b/spindle/mill/executor/reserved_test.go @@ -28,6 +28,7 @@ import ( "tangled.org/core/spindle/models" "tangled.org/core/spindle/quota" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" ) type captureEncoder struct { @@ -145,6 +146,22 @@ func (e *resourceReportingEngine) WorkflowResourceUsage(*models.Workflow) (engin return e.usage, true } +type fakeCacheEngine struct { + *fakeEngine + restored bool + saved bool +} + +func (e *fakeCacheEngine) RestoreCache(context.Context, models.WorkflowId, *models.Workflow, storage.Storage, []models.CacheBinding, models.WorkflowLogger) error { + e.restored = true + return nil +} + +func (e *fakeCacheEngine) SaveCache(context.Context, models.WorkflowId, *models.Workflow, storage.Storage, []models.CacheBinding, models.WorkflowLogger) error { + e.saved = true + return nil +} + type fakeStep struct{} func (fakeStep) Name() string { return "test" } @@ -157,7 +174,10 @@ func testReserveSeat(t *testing.T, leaseID, engineName string) *millv1.ReserveSe if err != nil { t.Fatal(err) } - tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{}}) + repoDID := "did:web:example.com" + tpl, err := json.Marshal(tangled.Pipeline{TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDID}, + }}) if err != nil { t.Fatal(err) } @@ -166,6 +186,9 @@ func testReserveSeat(t *testing.T, leaseID, engineName string) *millv1.ReserveSe TargetEngine: engineName, RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), + Knot: "knot.example", + Rkey: "rkey", + RepoDid: repoDID, } } @@ -176,7 +199,7 @@ func TestNewFailsWhenOutboxCannotInitialize(t *testing.T) { } n := notifier.New() cfg := &config.Config{} - if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil)), nil); err == nil { + if _, err := New(cfg, nil, d, &n, slog.New(slog.NewTextHandler(io.Discard, nil)), nil, nil); err == nil { t.Fatal("New succeeded with an unavailable outbox database") } } @@ -234,6 +257,26 @@ func TestReservedEngineForwardsResourceReporting(t *testing.T) { } } +func TestReservedEngineForwardsCacheRunner(t *testing.T) { + inner := &fakeCacheEngine{fakeEngine: &fakeEngine{}} + re := newReservedEngine(inner, &fakeSlot{}) + runner, ok := re.(engine.CacheRunner) + if !ok { + t.Fatal("reserved engine dropped CacheRunner") + } + + ctx := context.Background() + if err := runner.RestoreCache(ctx, models.WorkflowId{}, nil, nil, nil, nil); err != nil { + t.Fatal(err) + } + if err := runner.SaveCache(ctx, models.WorkflowId{}, nil, nil, nil, nil); err != nil { + t.Fatal(err) + } + if !inner.restored || !inner.saved { + t.Fatalf("cache calls were not forwarded: restored=%t saved=%t", inner.restored, inner.saved) + } +} + func TestHandleCommitIsIdempotent(t *testing.T) { enc := newCaptureEncoder() e := testExecutor(t) @@ -563,13 +606,20 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { if err != nil { t.Fatal(err) } + repoDidString := repoDid.String() res := &reservation{ leaseID: "lease-1", wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{ + RepoDid: repoDid, + TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &repoDidString}, + }, + TrustedSource: true, + }, } e.active[res.leaseID] = res @@ -580,6 +630,9 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) } + if got := e.maskSecrets(res, "value=secret-value"); got != "value=***" { + t.Fatalf("masked log = %q", got) + } select { case got := <-inner.secrets: if len(got) != 1 || got[0].Key != "TOKEN" || got[0].Value != "secret-value" { @@ -703,6 +756,83 @@ func TestConnectHandshakesWhilePendingArtifactRecoveryIsBlocked(t *testing.T) { } } +func TestHandleCommitRejectsCachesAndSecretsForForkSource(t *testing.T) { + d := testDB(t) + n := notifier.New() + enc := newCaptureEncoder() + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + e := &Executor{ + db: d, + n: &n, + l: slog.New(slog.NewTextHandler(io.Discard, nil)), + active: make(map[string]*reservation), + maxOutboxBytes: 10 * 1024 * 1024, + cfg: &config.Config{Server: config.Server{LogDir: t.TempDir()}}, + cache: cache, + enc: enc, + } + if err := e.initOutbox(); err != nil { + t.Fatal(err) + } + e.lifecycleCtx = context.Background() + + inner := &fakeCacheEngine{fakeEngine: &fakeEngine{ + secrets: make(chan []secrets.UnlockedSecret, 1), + done: make(chan struct{}), + }} + targetRepoDid, err := syntax.ParseDID("did:plc:target") + if err != nil { + t.Fatal(err) + } + sourceRepoDid := "did:plc:fork" + cacheObjectID := "11111111-1111-4111-8111-111111111111" + res := &reservation{ + leaseID: "lease-1", + wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + realEngine: inner, + slot: &fakeSlot{}, + wf: &models.Workflow{ + Name: "build", + Steps: []models.Step{fakeStep{}}, + Caches: []models.CacheEntry{{Key: "deps", Paths: []string{"deps"}}}, + }, + pipeline: &models.Pipeline{ + RepoDid: targetRepoDid, + TriggerMetadata: &tangled.Pipeline_TriggerMetadata{ + SourceRepo: &sourceRepoDid, + Repo: &tangled.Pipeline_TriggerRepo{RepoDid: &sourceRepoDid}, + }, + }, + } + e.active[res.leaseID] = res + + e.handleCommit(context.Background(), &millv1.CommitLease{ + LeaseId: res.leaseID, + Secrets: []*millv1.Secret{{Key: "TOKEN", Value: "secret-value"}}, + CacheBindings: []*millv1.CacheBinding{{ + EntryIndex: 0, + RestoreId: cacheObjectID, + RestoreKey: "objects/" + targetRepoDid.String() + "/" + cacheObjectID, + SaveId: cacheObjectID, + SaveKey: "objects/" + targetRepoDid.String() + "/" + cacheObjectID, + }}, + }) + if got := (<-enc.messages).GetCommitted().GetLeaseId(); got != res.leaseID { + t.Fatalf("Committed lease = %q, want %q", got, res.leaseID) + } + e.jobsWG.Wait() + + if inner.restored || inner.saved { + t.Fatalf("fork cache access = restored %t, saved %t", inner.restored, inner.saved) + } + if got := <-inner.secrets; len(got) != 0 { + t.Fatalf("fork secrets = %+v, want none", got) + } +} + func TestRunSessionCancellationClosesStalledWebsocket(t *testing.T) { connected := make(chan struct{}) release := make(chan struct{}) @@ -986,7 +1116,7 @@ func TestSocketCancellationIndependence(t *testing.T) { realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{RepoDid: repoDid}, } e.active[res.leaseID] = res @@ -1346,6 +1476,7 @@ func TestTimerRace(t *testing.T) { RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), TtlSeconds: 1, + RepoDid: "did:web:example.com", }) e.mu.Lock() @@ -1401,7 +1532,7 @@ func TestStructuredShutdown(t *testing.T) { realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, - repoDid: repoDid, + pipeline: &models.Pipeline{RepoDid: repoDid}, } e.active[res.leaseID] = res diff --git a/spindle/mill/handler.go b/spindle/mill/handler.go index e4709e26..8986af1f 100644 --- a/spindle/mill/handler.go +++ b/spindle/mill/handler.go @@ -111,8 +111,18 @@ func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { m.l.Error("fleet first frame was not hello") return } - if h.GetProtocolVersion() != millproto.ProtocolVersion { - m.l.Error("fleet protocol version mismatch", "got", h.GetProtocolVersion(), "want", millproto.ProtocolVersion) + protocolVersion := h.GetProtocolVersion() + minProtocol, maxProtocol := h.GetMinProtocolVersion(), h.GetMaxProtocolVersion() + legacyProtocol := minProtocol == 0 && maxProtocol == 0 + if legacyProtocol { + // v4 executors predate capability negotiation. keep them available for + // ordinary jobs, but never admit them for cache placement. + minProtocol, maxProtocol = protocolVersion, protocolVersion + } + if minProtocol == 0 || maxProtocol == 0 || minProtocol > maxProtocol || + protocolVersion < minProtocol || protocolVersion > maxProtocol || + protocolVersion < millproto.ProtocolMinVersion || protocolVersion > millproto.ProtocolMaxVersion { + m.l.Error("fleet protocol version range mismatch", "version", protocolVersion, "min", minProtocol, "max", maxProtocol, "supportedMin", millproto.ProtocolMinVersion, "supportedMax", millproto.ProtocolMaxVersion) return } if h.GetEpoch() == "" { @@ -130,6 +140,10 @@ func (m *Mill) HandleExecutorConn(w http.ResponseWriter, r *http.Request) { sess := newSession(name, h.GetEpoch(), authorizedLabels, enc, m.l) sess.closeTransport = conn.Close sess.labels = h.GetLabels() + sess.arch = h.GetArch() + sess.cacheStoreID = h.GetCacheStoreId() + sess.cacheNamespace = h.GetCacheNamespace() + sess.legacyProtocol = legacyProtocol resume, ok := m.attachSession(sess) if !ok { diff --git a/spindle/mill/integration_test.go b/spindle/mill/integration_test.go index 375fc506..3c39299d 100644 --- a/spindle/mill/integration_test.go +++ b/spindle/mill/integration_test.go @@ -61,7 +61,7 @@ func TestEndToEndDummyJob(t *testing.T) { dummyEng := dummy.New(l) dummyEng.StepDelay = 50 * time.Millisecond engines := map[string]models.Engine{"dummy": dummyEng} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } @@ -229,7 +229,7 @@ func TestExecutorConfiguredLabelsAreStoredOnSession(t *testing.T) { cfg.Mill.Labels = []string{"linux", "arm64", "gpu"} engines := map[string]models.Engine{"dummy": dummy.New(l)} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } @@ -279,7 +279,7 @@ func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { cfg.Mill.Labels = labels engines := map[string]models.Engine{"dummy": dummy.New(l)} - exec, err := executor.New(cfg, engines, edb, &en, l) + exec, err := executor.New(cfg, engines, edb, &en, l, nil, nil) if err != nil { t.Fatalf("executor.New: %v", err) } diff --git a/spindle/mill/lease.go b/spindle/mill/lease.go index 2adc6ca9..788a043a 100644 --- a/spindle/mill/lease.go +++ b/spindle/mill/lease.go @@ -40,11 +40,12 @@ type remoteQuota struct { } type RemoteLease struct { - id string - nodeID string - epoch string - engine string - wid models.WorkflowId // job this lease carries, set once placed + id string + nodeID string + epoch string + engine string + cacheNamespace string + wid models.WorkflowId // job this lease carries, set once placed // identity comes from the mill's pipeline record ownerDID string repoDID string diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go index 2b09f817..a7d7c1c5 100644 --- a/spindle/mill/mill.go +++ b/spindle/mill/mill.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "maps" "os" @@ -24,6 +25,7 @@ import ( "tangled.org/core/spindle/observability" "tangled.org/core/spindle/quota" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" "tangled.org/core/tid" millproto "tangled.org/core/spindle/mill/proto" @@ -38,6 +40,7 @@ const ( defaultMaxPending = 100 defaultCancelAckTimeout = 10 * time.Second defaultCancelTeardownTimeout = 10 * time.Minute + defaultCacheDeleteTimeout = 30 * time.Second ) var errProtocolViolation = errors.New("executor protocol violation") @@ -48,15 +51,26 @@ func protoErrf(format string, args ...any) error { type Config struct { // mill appends live-tailed executor lines here so logview can follow running remote jobs - LogDir string - MaxPending int - ReconnectGrace time.Duration - JobTimeout time.Duration - BidTimeout time.Duration - TopK int - CancelAckTimeout time.Duration + LogDir string + MaxPending int + ReconnectGrace time.Duration + CacheStoreID string + CacheMaxBytesPerOwner int64 + JobTimeout time.Duration + BidTimeout time.Duration + TopK int + CancelAckTimeout time.Duration // teardown can take minutes after the cancel is acknowledged CancelTeardownTimeout time.Duration + // bounds time spent deleting an object from the shared cache + CacheDeleteTimeout time.Duration + // how many protocol-violating session deaths in a row quarantine the node + QuarantineStrikes int +} + +type cacheDeletion struct { + entries []db.CacheEntry + objects []string } type Mill struct { @@ -67,6 +81,7 @@ type Mill struct { n *notifier.Notifier metrics *observability.Metrics qm *quota.Manager + cache storage.Storage mu sync.Mutex quotaLifecycleMu sync.Mutex sessions map[string]*millSession @@ -74,9 +89,15 @@ type Mill struct { reservations map[string]*RemoteLease nodeSeqno map[string]uint64 quotaLeases map[string]*RemoteLease + protoStrikes map[string]int pending int changeCh chan struct{} + cacheDeleteOnce sync.Once + cacheDeleteMu sync.Mutex + cacheDeleteQueue []cacheDeletion + cacheDeleteWake chan struct{} + leaseSeq uint64 } @@ -102,7 +123,10 @@ func New(l *slog.Logger, cfg Config) *Mill { if cfg.CancelTeardownTimeout <= 0 { cfg.CancelTeardownTimeout = defaultCancelTeardownTimeout } - return &Mill{ + if cfg.CacheDeleteTimeout <= 0 { + cfg.CacheDeleteTimeout = defaultCacheDeleteTimeout + } + m := &Mill{ l: l, cfg: cfg, sessions: make(map[string]*millSession), @@ -112,6 +136,8 @@ func New(l *slog.Logger, cfg Config) *Mill { changeCh: make(chan struct{}), quotaLeases: make(map[string]*RemoteLease), } + m.startCacheDeletionWorker() + return m } func (m *Mill) Attach(d *db.DB, n *notifier.Notifier, qm *quota.Manager) { @@ -122,6 +148,35 @@ func (m *Mill) Attach(d *db.DB, n *notifier.Notifier, qm *quota.Manager) { m.mu.Unlock() } +func (m *Mill) AttachCache(store storage.Storage) { + m.mu.Lock() + m.cache = store + m.mu.Unlock() + m.ensureCacheSentinel(store) +} + +func (m *Mill) ensureCacheSentinel(store storage.Storage) { + if store == nil { + return + } + rc, err := store.Get(context.Background(), millproto.CacheSentinelKey) + if err == nil { + defer rc.Close() + value, readErr := io.ReadAll(io.LimitReader(rc, int64(len(millproto.CacheSentinelValue)+1))) + if readErr != nil || string(value) != millproto.CacheSentinelValue { + m.l.Warn("cache store sentinel has unexpected contents", "key", millproto.CacheSentinelKey) + } + return + } + if !errors.Is(err, storage.ErrNotExist) { + m.l.Warn("failed to read cache store sentinel", "key", millproto.CacheSentinelKey, "err", err) + return + } + if err := store.Put(context.Background(), millproto.CacheSentinelKey, strings.NewReader(millproto.CacheSentinelValue)); err != nil { + m.l.Warn("failed to initialize cache store sentinel", "key", millproto.CacheSentinelKey, "err", err) + } +} + func (m *Mill) nextLeaseID() string { m.mu.Lock() m.leaseSeq++ @@ -407,6 +462,7 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow return nil, err } if lease != nil { + wf.CacheNamespace = lease.cacheNamespace slot, retry, err := m.admit(ctx, engineName, wid, wf, lease) if err != nil { var failure *engine.WorkflowFailure @@ -419,8 +475,6 @@ func (m *Mill) place(ctx context.Context, engineName string, wid models.Workflow placementResult = "success" return slot, nil } - // quota churn handed the seat back, bid again without waiting - // on a fleet change that already happened if retry { continue } @@ -596,7 +650,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId } requiredLabels := requiredLabels(wf) - candidates := m.rankCandidates(engineName, requiredLabels) + candidates := m.rankCandidates(engineName, requiredLabels, len(wf.Caches) > 0) if len(candidates) == 0 { return nil, nil } @@ -623,6 +677,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId defer cancel() leaseID := m.nextLeaseID() lease := newLease(leaseID, sess.nodeID, sess.epoch, engineName) + lease.cacheNamespace = sess.cacheNamespace m.mu.Lock() m.reservations[leaseID] = lease m.mu.Unlock() @@ -675,6 +730,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), Traceparent: traceparent, Tracestate: tracestate, + RepoDid: wf.RepoDID, }} resp, err := sess.request(bidCtx, leaseID, msg) if err != nil { @@ -801,7 +857,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId // ranks nodes that are least busy first. if a resource is used a lot // then that node will lose to one that is more even across the board. -func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*millSession { +func (m *Mill) rankCandidates(engineName string, requiredLabels []string, cacheRequired bool) []*millSession { m.mu.Lock() defer m.mu.Unlock() @@ -819,6 +875,9 @@ func (m *Mill) rankCandidates(engineName string, requiredLabels []string) []*mil if s.snapshot == nil { continue } + if cacheRequired && (m.cfg.CacheStoreID == "" || s.cacheStoreID != m.cfg.CacheStoreID || s.legacyProtocol) { + continue + } // the engine has to exist and have room right now ea, ok := s.snapshot.GetEngines()[engineName] if !ok || !ea.GetAvailable() { @@ -913,6 +972,16 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked span.End() }() + capabilities, err := cacheCapabilities(wf.RepoDID, wf.CacheBindings) + if err != nil { + return err + } + if m.db != nil { + if err := m.db.SaveMillCacheCapabilities(lease.id, capabilities); err != nil { + return fmt.Errorf("persist mill cache capabilities: %w", err) + } + } + pbSecrets := make([]*millv1.Secret, len(unlocked)) for i, s := range unlocked { pbSecrets[i] = &millv1.Secret{Key: s.Key, Value: s.Value} @@ -926,6 +995,7 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked Traceparent: traceparent, Tracestate: tracestate, MillRecordsTerminalMetrics: lease.millRecordsTerminalMetrics, + CacheBindings: cacheBindingsToProto(wf.CacheBindings), }} // commit retries ride reconnects, a reservation outlives one @@ -1092,6 +1162,20 @@ func (m *Mill) cleanupLeaseLocked(lease *RemoteLease) error { return nil } if m.db != nil { + // fence unreported save uploads before dropping the lease capabilities; + // a late object is then covered by a durable deletion tombstone + pendingKeys, err := m.db.DiscardPendingCacheEntriesForLease(lease.id, time.Now()) + if err != nil { + m.scheduleCleanupLocked(lease) + return err + } + if len(pendingKeys) > 0 { + objects := make(map[string]struct{}, len(pendingKeys)) + for _, key := range pendingKeys { + objects[key] = struct{}{} + } + m.enqueueCacheDeletions(nil, objects) + } // delete the row before releasing the reservation if err := m.db.DeleteMillLease(lease.id); err != nil { m.scheduleCleanupLocked(lease) @@ -1314,6 +1398,8 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { lease *RemoteLease ar *millv1.AttemptResult } + cacheDeletes := make(map[string]db.CacheEntry) + cacheObjectDeletes := make(map[string]struct{}) type startupObservation struct { engine string delay time.Duration @@ -1429,6 +1515,85 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { } } + case entry.GetCacheUpdate() != nil: + update := entry.GetCacheUpdate() + if tx != nil { + at := time.Now() + capabilityAction := "" + switch update.GetAction() { + case millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_USED, millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_MISSING: + capabilityAction = "restore" + case millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED, millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_DISCARDED: + capabilityAction = "save" + default: + return batchApply{}, protoErrf("unsupported cache update action %v", update.GetAction()) + } + ref, authorized, err := tx.ConsumeMillCacheCapability( + context.Background(), + lease.id, + capabilityAction, + update.GetId(), + ) + if err != nil { + return batchApply{}, err + } + if !authorized { + return batchApply{}, protoErrf( + "cache update %q was not planned for lease %q", + update.GetId(), + lease.id, + ) + } + + switch update.GetAction() { + case millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_USED: + if err := tx.TouchCacheEntry(context.Background(), update.GetId(), at); err != nil { + return batchApply{}, err + } + case millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED: + actualSize, err := m.cacheObjectSize(context.Background(), ref) + if err != nil { + return batchApply{}, fmt.Errorf("stat stored cache object %q: %w", ref, err) + } + superseded, ready, err := tx.MarkCacheEntryReadyWithChecksum( + context.Background(), + update.GetId(), + actualSize, + update.GetChecksum(), + m.cfg.CacheMaxBytesPerOwner, + at, + ) + if err != nil { + return batchApply{}, err + } + for _, old := range superseded { + cacheDeletes[old.ID] = old + } + if !ready { + if err := tx.QueueCacheObjectDeletion(context.Background(), ref, at); err != nil { + return batchApply{}, err + } + cacheObjectDeletes[ref] = struct{}{} + } + case millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_DISCARDED, millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_MISSING: + // A missing restore object must not let one executor delete a + // ready generation that another executor published. Both + // actions can only discard entries still owned by this save. + pendingOnly := true + discarded, err := tx.DiscardCacheEntry( + context.Background(), + update.GetId(), + pendingOnly, + ) + if err != nil { + return batchApply{}, err + } + if discarded != nil { + cacheDeletes[discarded.ID] = *discarded + } + } + } + case entry.GetAttemptResult() != nil: ar := entry.GetAttemptResult() statusStr := "success" @@ -1520,6 +1685,8 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { m.metrics.RecordWorkflowStartupDelay(context.Background(), observation.engine, "mill", observation.delay) } + m.enqueueCacheDeletions(cacheDeletes, cacheObjectDeletes) + if m.cfg.LogDir != "" { for _, lease := range applied.artifactLeases { path := models.LogFilePath(m.cfg.LogDir, lease.wid) @@ -1578,6 +1745,147 @@ func terminalMetricResult(status millv1.TerminalStatus) string { } } +func (m *Mill) cacheObjectSize(ctx context.Context, key string) (int64, error) { + m.mu.Lock() + store := m.cache + m.mu.Unlock() + if store == nil { + return 0, errors.New("cache store is not configured") + } + if statter, ok := store.(storage.StatStorage); ok { + return statter.Stat(ctx, key) + } + r, err := store.Get(ctx, key) + if err != nil { + return 0, err + } + defer r.Close() + n, err := io.Copy(io.Discard, r) + if err != nil { + return 0, err + } + return n, nil +} + +func (m *Mill) startCacheDeletionWorker() { + m.cacheDeleteOnce.Do(func() { + m.cacheDeleteWake = make(chan struct{}, 1) + go m.cacheDeletionLoop() + }) +} + +func (m *Mill) enqueueCacheDeletions(entries map[string]db.CacheEntry, objects map[string]struct{}) { + if len(entries) == 0 && len(objects) == 0 { + return + } + m.startCacheDeletionWorker() + deletion := cacheDeletion{ + entries: make([]db.CacheEntry, 0, len(entries)), + objects: make([]string, 0, len(objects)), + } + for _, entry := range entries { + deletion.entries = append(deletion.entries, entry) + } + for key := range objects { + deletion.objects = append(deletion.objects, key) + } + m.cacheDeleteMu.Lock() + m.cacheDeleteQueue = append(m.cacheDeleteQueue, deletion) + m.cacheDeleteMu.Unlock() + select { + case m.cacheDeleteWake <- struct{}{}: + default: + } +} + +func (m *Mill) cacheDeletionLoop() { + for range m.cacheDeleteWake { + for { + m.cacheDeleteMu.Lock() + if len(m.cacheDeleteQueue) == 0 { + m.cacheDeleteMu.Unlock() + break + } + deletion := m.cacheDeleteQueue[0] + m.cacheDeleteQueue[0] = cacheDeletion{} + m.cacheDeleteQueue = m.cacheDeleteQueue[1:] + m.cacheDeleteMu.Unlock() + m.deleteCacheEntries(deletion.entries) + m.deleteQueuedCacheObjects(deletion.objects) + } + } +} + +func (m *Mill) cacheDeleteContext() (context.Context, context.CancelFunc) { + timeout := m.cfg.CacheDeleteTimeout + if timeout <= 0 { + timeout = defaultCacheDeleteTimeout + } + return context.WithTimeout(context.Background(), timeout) +} + +func (m *Mill) deleteCacheEntries(entries []db.CacheEntry) { + if len(entries) == 0 { + return + } + m.mu.Lock() + cache := m.cache + database := m.db + m.mu.Unlock() + if cache == nil { + m.l.Warn("cache objects need deletion but the mill has no cache store", "count", len(entries)) + return + } + for _, entry := range entries { + ctx, cancel := m.cacheDeleteContext() + err := cache.Delete(ctx, entry.StorageKey) + cancel() + if err != nil { + m.l.Warn("delete cache object", "id", entry.ID, "ref", entry.StorageKey, "err", err) + continue + } + if database != nil { + ctx, cancel = m.cacheDeleteContext() + err = database.DeleteCacheEntry(ctx, entry.ID) + cancel() + if err != nil { + m.l.Warn("delete cache metadata", "id", entry.ID, "err", err) + } + } + } +} + +func (m *Mill) deleteQueuedCacheObjects(keys []string) { + if len(keys) == 0 { + return + } + m.mu.Lock() + cache := m.cache + database := m.db + m.mu.Unlock() + if cache == nil { + m.l.Warn("cache objects need deletion but the mill has no cache store", "count", len(keys)) + return + } + for _, key := range keys { + ctx, cancel := m.cacheDeleteContext() + err := cache.Delete(ctx, key) + cancel() + if err != nil { + m.l.Warn("delete unindexed cache object", "ref", key, "err", err) + continue + } + if database != nil { + ctx, cancel = m.cacheDeleteContext() + err = database.CompleteCacheObjectDeletion(ctx, key) + cancel() + if err != nil { + m.l.Warn("complete cache object deletion", "ref", key, "err", err) + } + } + } +} + func (m *Mill) sendAck(sess *millSession, seqno uint64) error { msg := &millproto.Message{Ack: &millv1.Ack{ Epoch: sess.epoch, diff --git a/spindle/mill/mill_test.go b/spindle/mill/mill_test.go index 4c1501e1..4ed06775 100644 --- a/spindle/mill/mill_test.go +++ b/spindle/mill/mill_test.go @@ -1,10 +1,12 @@ package mill import ( + "bytes" "context" "errors" "io" "log/slog" + "strings" "sync" "testing" "time" @@ -13,6 +15,7 @@ import ( "tangled.org/core/spindle/db" "tangled.org/core/spindle/engine" "tangled.org/core/spindle/models" + "tangled.org/core/spindle/storage" millproto "tangled.org/core/spindle/mill/proto" millv1 "tangled.org/core/spindle/mill/proto/gen" @@ -22,17 +25,69 @@ type scriptedEncoder func(*millproto.Message) error func (e scriptedEncoder) Encode(msg *millproto.Message) error { return e(msg) } +func waitForCacheDeletion(t *testing.T, cache storage.Storage, index *db.DB, key string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + reader, err := cache.Get(context.Background(), key) + if errors.Is(err, storage.ErrNotExist) { + if index != nil { + pending, queryErr := index.PendingCacheObjectDeletions(context.Background(), 10) + if queryErr != nil { + t.Fatal(queryErr) + } + if len(pending) == 0 { + return + } + } else { + return + } + } else if err == nil { + _ = reader.Close() + } + time.Sleep(time.Millisecond) + } + t.Fatalf("cache object %q was not deleted", key) +} + func testWorkflow(name string) *models.Workflow { + repoDid := "did:web:example.com" return &models.Workflow{ Name: name, Environment: map[string]string{}, Steps: []models.Step{remoteStep{}}, + RepoDID: repoDid, Data: &millWorkflowState{ RawWorkflow: tangled.Pipeline_Workflow{Name: name}, RawPipeline: testPipeline(), }, } } +func TestMillEngineRequiresSharedCacheStoreID(t *testing.T) { + twf := tangled.Pipeline_Workflow{ + Name: "build", + Raw: "cache:\n - key: deps\n paths: [deps]\n", + } + for _, tc := range []struct { + name string + storeID string + want int + }{ + {name: "unidentified store", want: 0}, + {name: "shared store", storeID: "fleet-cache", want: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + m := New(discardLogger(), Config{CacheStoreID: tc.storeID}) + wf, err := NewEngine("microvm", m).InitWorkflow(twf, tangled.Pipeline{}) + if err != nil { + t.Fatal(err) + } + if got := len(wf.Caches); got != tc.want { + t.Fatalf("cache entries = %d, want %d", got, tc.want) + } + }) + } +} func TestEngineInitWorkflowCarriesRepositoryIdentity(t *testing.T) { m := &Mill{l: slog.New(slog.NewTextHandler(io.Discard, nil))} @@ -154,6 +209,17 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{BidTimeout: 25 * time.Millisecond, ReconnectGrace: time.Second}) wf := testWorkflow("build") + restoreID := "11111111-1111-4111-8111-111111111111" + saveID := "22222222-2222-4222-8222-222222222222" + wf.CacheBindings = []models.CacheBinding{{ + EntryIndex: 0, + Hash: "abc123", + RestoreID: restoreID, + RestoreKey: "objects/did:web:example.com/" + restoreID, + RestoreName: "deps-abc123", + SaveID: saveID, + SaveKey: "objects/did:web:example.com/" + saveID, + }} wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} lease := newLease("lease-1", "node-1", "inc-1", "dummy") lease.wid = wid @@ -190,6 +256,12 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { if msg.GetCommitLease() == nil { return nil } + bindings := msg.GetCommitLease().GetCacheBindings() + if len(bindings) != 1 || + bindings[0].GetRestoreId() != restoreID || + bindings[0].GetSaveId() != saveID { + t.Errorf("CommitLease cache bindings = %+v", bindings) + } leaseID := msg.GetCommitLease().GetLeaseId() sess2.deliver(leaseID, &millproto.Message{Committed: &millv1.Committed{LeaseId: leaseID}}) _ = m.onEventBatch(sess2, &millv1.EventBatch{ @@ -303,11 +375,22 @@ func TestRankCandidatesFiltersRequiredLabelsWithANDSemantics(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels), tt.want) + assertRankedNodes(t, m.rankCandidates("dummy", tt.requiredLabels, false), tt.want) }) } } +func TestRankCandidatesRequiresSharedCacheStore(t *testing.T) { + m := New(discardLogger(), Config{CacheStoreID: "disk:/cache"}) + matching := addCandidateSession(t, m, "matching", []string{"linux"}, 0.5, nil) + matching.cacheStoreID = "disk:/cache" + other := addCandidateSession(t, m, "other", []string{"linux"}, 0.1, nil) + other.cacheStoreID = "s3:bucket/cache" + + assertRankedNodes(t, m.rankCandidates("dummy", []string{"linux"}, true), []string{"matching"}) + assertRankedNodes(t, m.rankCandidates("dummy", []string{"linux"}, false), []string{"other", "matching"}) +} + func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{BidTimeout: 10 * time.Millisecond}) @@ -670,6 +753,194 @@ func TestAtomicBatchRollback(t *testing.T) { } } +func TestCacheUpdatesCommitWithStreamCursor(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + + at := time.Date(2026, 8, 11, 20, 0, 0, 0, time.UTC) + old := db.CacheEntry{ + ID: "old", StorageKey: "objects/old", OwnerDID: "did:plc:owner", + RepoDID: "did:plc:repo", Engine: "dummy", CacheKey: "deps", CacheHash: "hash", + SizeBytes: 3, State: "ready", CreatedAt: at.Add(-time.Hour), LastUsedAt: at.Add(-time.Hour), + } + pending := old + pending.ID = "pending" + pending.StorageKey = "objects/pending" + pending.SizeBytes = 0 + pending.State = "pending" + pending.CreatedAt = at + pending.LastUsedAt = at + for _, entry := range []db.CacheEntry{old, pending} { + if err := bdb.InsertCacheEntry(context.Background(), entry); err != nil { + t.Fatal(err) + } + if err := cache.Put(context.Background(), entry.StorageKey, bytes.NewReader([]byte(entry.ID))); err != nil { + t.Fatal(err) + } + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + owned := newLease("lease-1", "node-1", "inc-1", "dummy") + owned.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(owned, leaseRowRunning); err != nil { + t.Fatal(err) + } + if err := bdb.SaveMillCacheCapabilities(owned.id, []db.MillCacheCapability{ + {Action: "restore", CacheID: old.ID, StorageKey: old.StorageKey}, + {Action: "save", CacheID: pending.ID, StorageKey: pending.StorageKey}, + }); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[owned.id] = owned + m.mu.Unlock() + + batch := &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{ + { + Seqno: 1, LeaseId: owned.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_USED, Id: old.ID, + }}, + }, + { + Seqno: 2, LeaseId: owned.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED, Id: pending.ID, + SizeBytes: int64(len(pending.ID)), + }}, + }, + }, + } + if err := m.onEventBatch(sess, batch); err != nil { + t.Fatalf("onEventBatch: %v", err) + } + if err := m.onEventBatch(sess, batch); err != nil { + t.Fatalf("replay onEventBatch: %v", err) + } + + ready, err := bdb.FindCacheEntry(context.Background(), old.RepoDID, old.Engine, old.CacheKey, old.CacheHash) + if err != nil { + t.Fatal(err) + } + if ready.ID != pending.ID || ready.SizeBytes != int64(len(pending.ID)) { + t.Fatalf("ready cache = %+v", ready) + } + + waitForCacheDeletion(t, cache, nil, old.StorageKey) + if cursor, err := bdb.GetExecutorCursor(sess.nodeID, sess.epoch); err != nil || cursor != 2 { + t.Fatalf("cursor = %d, %v; want 2", cursor, err) + } +} +func TestCacheUpdateCannotDeleteUnplannedObject(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + victim := "objects/did:web:victim.example/11111111-1111-4111-8111-111111111111" + if err := cache.Put(context.Background(), victim, strings.NewReader("victim")); err != nil { + t.Fatal(err) + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + err = m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{{ + Seqno: 1, + LeaseId: lease.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED, + Id: "unknown", + }}, + }}, + }) + if err == nil { + t.Fatal("unplanned cache update succeeded") + } + reader, err := cache.Get(context.Background(), victim) + if err != nil { + t.Fatalf("victim object was deleted: %v", err) + } + _ = reader.Close() + if cursor, err := bdb.GetExecutorCursor(sess.nodeID, sess.epoch); err != nil || cursor != 0 { + t.Fatalf("cursor = %d, %v; want 0", cursor, err) + } +} + +func TestMissingPendingRowQueuesAndDeletesPlannedObject(t *testing.T) { + m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) + cache, err := storage.NewDisk(t.TempDir()) + if err != nil { + t.Fatal(err) + } + m.AttachCache(cache) + id := "11111111-1111-4111-8111-111111111111" + key := "objects/did:web:example.com/" + id + if err := cache.Put(context.Background(), key, strings.NewReader("orphan")); err != nil { + t.Fatal(err) + } + + sess := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) + m.attachSession(sess) + lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") + lease.wid = models.WorkflowId{ + PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + Name: "build", + } + if err := m.persistLease(lease, leaseRowRunning); err != nil { + t.Fatal(err) + } + if err := bdb.SaveMillCacheCapabilities(lease.id, []db.MillCacheCapability{{ + Action: "save", + CacheID: id, + StorageKey: key, + }}); err != nil { + t.Fatal(err) + } + m.mu.Lock() + m.leases[lease.id] = lease + m.mu.Unlock() + + if err := m.onEventBatch(sess, &millv1.EventBatch{ + Epoch: sess.epoch, + Events: []*millv1.Event{{ + Seqno: 1, + LeaseId: lease.id, + Payload: &millv1.Event_CacheUpdate{CacheUpdate: &millv1.CacheUpdate{ + Action: millv1.CacheUpdateAction_CACHE_UPDATE_ACTION_STORED, + Id: id, + }}, + }}, + }); err != nil { + t.Fatal(err) + } + waitForCacheDeletion(t, cache, bdb, key) +} + func TestTerminalBeforeACK(t *testing.T) { m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) @@ -912,12 +1183,12 @@ func TestPendingCancelBlocksPlacementUntilAcknowledged(t *testing.T) { m.leases[lease.id] = lease m.mu.Unlock() - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) m.destroy(lease.wid) - assertRankedNodes(t, m.rankCandidates("dummy", nil), nil) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), nil) m.onCancelAck(sess, &millv1.CancelAck{LeaseId: lease.id}) - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) } func TestTerminalSettlesPendingCancelPlacementGate(t *testing.T) { @@ -931,7 +1202,7 @@ func TestTerminalSettlesPendingCancelPlacementGate(t *testing.T) { m.mu.Unlock() m.destroy(lease.wid) - assertRankedNodes(t, m.rankCandidates("dummy", nil), nil) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), nil) if err := m.onEventBatch(sess, &millv1.EventBatch{ Epoch: sess.epoch, Events: []*millv1.Event{{ @@ -944,7 +1215,7 @@ func TestTerminalSettlesPendingCancelPlacementGate(t *testing.T) { }); err != nil { t.Fatalf("onEventBatch: %v", err) } - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) } func TestCancelSentAfterTerminalDoesNotLeakPlacementGate(t *testing.T) { @@ -972,7 +1243,7 @@ func TestCancelSentAfterTerminalDoesNotLeakPlacementGate(t *testing.T) { if pending != 0 { t.Fatalf("pending cancels after terminal = %d, want 0", pending) } - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) } func TestValidSnapshotRecoversAfterProtocolError(t *testing.T) { @@ -1008,7 +1279,7 @@ func TestValidSnapshotRecoversAfterProtocolError(t *testing.T) { if recovering { t.Fatal("valid reconnect snapshot did not end recovery") } - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) } func TestReconnectRetriesKeepOriginalFailureDeadline(t *testing.T) { @@ -1236,7 +1507,7 @@ func TestCancelAckDeadlineDoesNotCloseReplacementSession(t *testing.T) { }, } m.sessionReady(replacement) - assertRankedNodes(t, m.rankCandidates("dummy", nil), nil) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), nil) m.checkCancelAck(oldSession, lease) select { diff --git a/spindle/mill/proto/gen/mill.pb.go b/spindle/mill/proto/gen/mill.pb.go index 30904ff6..387c9d02 100644 --- a/spindle/mill/proto/gen/mill.pb.go +++ b/spindle/mill/proto/gen/mill.pb.go @@ -172,6 +172,61 @@ func (TerminalStatus) EnumDescriptor() ([]byte, []int) { return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{2} } +type CacheUpdateAction int32 + +const ( + CacheUpdateAction_CACHE_UPDATE_ACTION_UNSPECIFIED CacheUpdateAction = 0 + CacheUpdateAction_CACHE_UPDATE_ACTION_USED CacheUpdateAction = 1 + CacheUpdateAction_CACHE_UPDATE_ACTION_STORED CacheUpdateAction = 2 + CacheUpdateAction_CACHE_UPDATE_ACTION_DISCARDED CacheUpdateAction = 3 + CacheUpdateAction_CACHE_UPDATE_ACTION_MISSING CacheUpdateAction = 4 +) + +// Enum value maps for CacheUpdateAction. +var ( + CacheUpdateAction_name = map[int32]string{ + 0: "CACHE_UPDATE_ACTION_UNSPECIFIED", + 1: "CACHE_UPDATE_ACTION_USED", + 2: "CACHE_UPDATE_ACTION_STORED", + 3: "CACHE_UPDATE_ACTION_DISCARDED", + 4: "CACHE_UPDATE_ACTION_MISSING", + } + CacheUpdateAction_value = map[string]int32{ + "CACHE_UPDATE_ACTION_UNSPECIFIED": 0, + "CACHE_UPDATE_ACTION_USED": 1, + "CACHE_UPDATE_ACTION_STORED": 2, + "CACHE_UPDATE_ACTION_DISCARDED": 3, + "CACHE_UPDATE_ACTION_MISSING": 4, + } +) + +func (x CacheUpdateAction) Enum() *CacheUpdateAction { + p := new(CacheUpdateAction) + *p = x + return p +} + +func (x CacheUpdateAction) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CacheUpdateAction) Descriptor() protoreflect.EnumDescriptor { + return file_spindle_mill_v1_mill_proto_enumTypes[3].Descriptor() +} + +func (CacheUpdateAction) Type() protoreflect.EnumType { + return &file_spindle_mill_v1_mill_proto_enumTypes[3] +} + +func (x CacheUpdateAction) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CacheUpdateAction.Descriptor instead. +func (CacheUpdateAction) EnumDescriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} +} + type QuotaOperation int32 const ( @@ -211,11 +266,11 @@ func (x QuotaOperation) String() string { } func (QuotaOperation) Descriptor() protoreflect.EnumDescriptor { - return file_spindle_mill_v1_mill_proto_enumTypes[3].Descriptor() + return file_spindle_mill_v1_mill_proto_enumTypes[4].Descriptor() } func (QuotaOperation) Type() protoreflect.EnumType { - return &file_spindle_mill_v1_mill_proto_enumTypes[3] + return &file_spindle_mill_v1_mill_proto_enumTypes[4] } func (x QuotaOperation) Number() protoreflect.EnumNumber { @@ -224,20 +279,25 @@ func (x QuotaOperation) Number() protoreflect.EnumNumber { // Deprecated: Use QuotaOperation.Descriptor instead. func (QuotaOperation) EnumDescriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{3} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{4} } // executor identity, sent on connect type Hello struct { state protoimpl.MessageState `protogen:"open.v1"` ProtocolVersion uint32 `protobuf:"varint,1,opt,name=protocol_version,json=protocolVersion,proto3" json:"protocol_version,omitempty"` + // supported protocol range for rolling upgrades. zero means a legacy v4 executor. + MinProtocolVersion uint32 `protobuf:"varint,7,opt,name=min_protocol_version,json=minProtocolVersion,proto3" json:"min_protocol_version,omitempty"` + MaxProtocolVersion uint32 `protobuf:"varint,8,opt,name=max_protocol_version,json=maxProtocolVersion,proto3" json:"max_protocol_version,omitempty"` // GOARCH of the node, informational only Arch string `protobuf:"bytes,2,opt,name=arch,proto3" json:"arch,omitempty"` // operator-defined labels, matched against runs_on - Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` - Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Labels []string `protobuf:"bytes,3,rep,name=labels,proto3" json:"labels,omitempty"` + Epoch string `protobuf:"bytes,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + CacheStoreId string `protobuf:"bytes,5,opt,name=cache_store_id,json=cacheStoreId,proto3" json:"cache_store_id,omitempty"` + CacheNamespace string `protobuf:"bytes,6,opt,name=cache_namespace,json=cacheNamespace,proto3" json:"cache_namespace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Hello) Reset() { @@ -277,6 +337,20 @@ func (x *Hello) GetProtocolVersion() uint32 { return 0 } +func (x *Hello) GetMinProtocolVersion() uint32 { + if x != nil { + return x.MinProtocolVersion + } + return 0 +} + +func (x *Hello) GetMaxProtocolVersion() uint32 { + if x != nil { + return x.MaxProtocolVersion + } + return 0 +} + func (x *Hello) GetArch() string { if x != nil { return x.Arch @@ -298,6 +372,20 @@ func (x *Hello) GetEpoch() string { return "" } +func (x *Hello) GetCacheStoreId() string { + if x != nil { + return x.CacheStoreId + } + return "" +} + +func (x *Hello) GetCacheNamespace() string { + if x != nil { + return x.CacheNamespace + } + return "" +} + // reconnect state for an existing epoch type Resume struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -475,11 +563,13 @@ type ReserveSeat struct { RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` // pipeline id, the executor reconstructs the exact WorkflowId from it - Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` - Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` - TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` - Traceparent string `protobuf:"bytes,8,opt,name=traceparent,proto3" json:"traceparent,omitempty"` - Tracestate string `protobuf:"bytes,9,opt,name=tracestate,proto3" json:"tracestate,omitempty"` + Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` + Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` + TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` + Traceparent string `protobuf:"bytes,8,opt,name=traceparent,proto3" json:"traceparent,omitempty"` + Tracestate string `protobuf:"bytes,9,opt,name=tracestate,proto3" json:"tracestate,omitempty"` + // repository that owns the pipeline, used to distinguish fork source code + RepoDid string `protobuf:"bytes,10,opt,name=repo_did,json=repoDid,proto3" json:"repo_did,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -577,6 +667,13 @@ func (x *ReserveSeat) GetTracestate() string { return "" } +func (x *ReserveSeat) GetRepoDid() string { + if x != nil { + return x.RepoDid + } + return "" +} + type ReserveResult struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` @@ -730,6 +827,106 @@ func (x *Secret) GetValue() string { return "" } +type CacheBinding struct { + state protoimpl.MessageState `protogen:"open.v1"` + EntryIndex uint32 `protobuf:"varint,1,opt,name=entry_index,json=entryIndex,proto3" json:"entry_index,omitempty"` + RestoreId string `protobuf:"bytes,3,opt,name=restore_id,json=restoreId,proto3" json:"restore_id,omitempty"` + RestoreKey string `protobuf:"bytes,4,opt,name=restore_key,json=restoreKey,proto3" json:"restore_key,omitempty"` + RestoreName string `protobuf:"bytes,5,opt,name=restore_name,json=restoreName,proto3" json:"restore_name,omitempty"` + SaveId string `protobuf:"bytes,6,opt,name=save_id,json=saveId,proto3" json:"save_id,omitempty"` + SaveKey string `protobuf:"bytes,7,opt,name=save_key,json=saveKey,proto3" json:"save_key,omitempty"` + RestoreChecksum string `protobuf:"bytes,8,opt,name=restore_checksum,json=restoreChecksum,proto3" json:"restore_checksum,omitempty"` + RestoreSizeBytes int64 `protobuf:"varint,9,opt,name=restore_size_bytes,json=restoreSizeBytes,proto3" json:"restore_size_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheBinding) Reset() { + *x = CacheBinding{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheBinding) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheBinding) ProtoMessage() {} + +func (x *CacheBinding) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheBinding.ProtoReflect.Descriptor instead. +func (*CacheBinding) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} +} + +func (x *CacheBinding) GetEntryIndex() uint32 { + if x != nil { + return x.EntryIndex + } + return 0 +} + +func (x *CacheBinding) GetRestoreId() string { + if x != nil { + return x.RestoreId + } + return "" +} + +func (x *CacheBinding) GetRestoreKey() string { + if x != nil { + return x.RestoreKey + } + return "" +} + +func (x *CacheBinding) GetRestoreName() string { + if x != nil { + return x.RestoreName + } + return "" +} + +func (x *CacheBinding) GetSaveId() string { + if x != nil { + return x.SaveId + } + return "" +} + +func (x *CacheBinding) GetSaveKey() string { + if x != nil { + return x.SaveKey + } + return "" +} + +func (x *CacheBinding) GetRestoreChecksum() string { + if x != nil { + return x.RestoreChecksum + } + return "" +} + +func (x *CacheBinding) GetRestoreSizeBytes() int64 { + if x != nil { + return x.RestoreSizeBytes + } + return 0 +} + // promotes a reservation to a running job and hands over the secrets type CommitLease struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -738,13 +935,14 @@ type CommitLease struct { Traceparent string `protobuf:"bytes,3,opt,name=traceparent,proto3" json:"traceparent,omitempty"` Tracestate string `protobuf:"bytes,4,opt,name=tracestate,proto3" json:"tracestate,omitempty"` MillRecordsTerminalMetrics bool `protobuf:"varint,5,opt,name=mill_records_terminal_metrics,json=millRecordsTerminalMetrics,proto3" json:"mill_records_terminal_metrics,omitempty"` + CacheBindings []*CacheBinding `protobuf:"bytes,6,rep,name=cache_bindings,json=cacheBindings,proto3" json:"cache_bindings,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *CommitLease) Reset() { *x = CommitLease{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -756,7 +954,7 @@ func (x *CommitLease) String() string { func (*CommitLease) ProtoMessage() {} func (x *CommitLease) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[7] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -769,7 +967,7 @@ func (x *CommitLease) ProtoReflect() protoreflect.Message { // Deprecated: Use CommitLease.ProtoReflect.Descriptor instead. func (*CommitLease) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{7} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} } func (x *CommitLease) GetLeaseId() string { @@ -807,6 +1005,13 @@ func (x *CommitLease) GetMillRecordsTerminalMetrics() bool { return false } +func (x *CommitLease) GetCacheBindings() []*CacheBinding { + if x != nil { + return x.CacheBindings + } + return nil +} + type Committed struct { state protoimpl.MessageState `protogen:"open.v1"` LeaseId string `protobuf:"bytes,1,opt,name=lease_id,json=leaseId,proto3" json:"lease_id,omitempty"` @@ -816,7 +1021,7 @@ type Committed struct { func (x *Committed) Reset() { *x = Committed{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -828,7 +1033,7 @@ func (x *Committed) String() string { func (*Committed) ProtoMessage() {} func (x *Committed) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[8] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -841,7 +1046,7 @@ func (x *Committed) ProtoReflect() protoreflect.Message { // Deprecated: Use Committed.ProtoReflect.Descriptor instead. func (*Committed) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{8} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} } func (x *Committed) GetLeaseId() string { @@ -861,7 +1066,7 @@ type ReleaseLease struct { func (x *ReleaseLease) Reset() { *x = ReleaseLease{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -873,7 +1078,7 @@ func (x *ReleaseLease) String() string { func (*ReleaseLease) ProtoMessage() {} func (x *ReleaseLease) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[9] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -886,7 +1091,7 @@ func (x *ReleaseLease) ProtoReflect() protoreflect.Message { // Deprecated: Use ReleaseLease.ProtoReflect.Descriptor instead. func (*ReleaseLease) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{9} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} } func (x *ReleaseLease) GetLeaseId() string { @@ -907,7 +1112,7 @@ type CancelAttempt struct { func (x *CancelAttempt) Reset() { *x = CancelAttempt{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -919,7 +1124,7 @@ func (x *CancelAttempt) String() string { func (*CancelAttempt) ProtoMessage() {} func (x *CancelAttempt) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[10] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -932,7 +1137,7 @@ func (x *CancelAttempt) ProtoReflect() protoreflect.Message { // Deprecated: Use CancelAttempt.ProtoReflect.Descriptor instead. func (*CancelAttempt) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{10} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} } func (x *CancelAttempt) GetLeaseId() string { @@ -958,7 +1163,7 @@ type CancelAck struct { func (x *CancelAck) Reset() { *x = CancelAck{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -970,7 +1175,7 @@ func (x *CancelAck) String() string { func (*CancelAck) ProtoMessage() {} func (x *CancelAck) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[11] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -983,7 +1188,7 @@ func (x *CancelAck) ProtoReflect() protoreflect.Message { // Deprecated: Use CancelAck.ProtoReflect.Descriptor instead. func (*CancelAck) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{11} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} } func (x *CancelAck) GetLeaseId() string { @@ -1004,7 +1209,7 @@ type StatusEvent struct { func (x *StatusEvent) Reset() { *x = StatusEvent{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1016,7 +1221,7 @@ func (x *StatusEvent) String() string { func (*StatusEvent) ProtoMessage() {} func (x *StatusEvent) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[12] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1029,7 +1234,7 @@ func (x *StatusEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StatusEvent.ProtoReflect.Descriptor instead. func (*StatusEvent) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{12} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} } func (x *StatusEvent) GetStatus() NonterminalStatus { @@ -1063,7 +1268,7 @@ type LogArtifact struct { func (x *LogArtifact) Reset() { *x = LogArtifact{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1075,7 +1280,7 @@ func (x *LogArtifact) String() string { func (*LogArtifact) ProtoMessage() {} func (x *LogArtifact) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[13] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1088,7 +1293,7 @@ func (x *LogArtifact) ProtoReflect() protoreflect.Message { // Deprecated: Use LogArtifact.ProtoReflect.Descriptor instead. func (*LogArtifact) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{13} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} } func (x *LogArtifact) GetRef() string { @@ -1121,7 +1326,7 @@ type AttemptResult struct { func (x *AttemptResult) Reset() { *x = AttemptResult{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1133,7 +1338,7 @@ func (x *AttemptResult) String() string { func (*AttemptResult) ProtoMessage() {} func (x *AttemptResult) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[14] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1146,7 +1351,7 @@ func (x *AttemptResult) ProtoReflect() protoreflect.Message { // Deprecated: Use AttemptResult.ProtoReflect.Descriptor instead. func (*AttemptResult) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{14} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} } func (x *AttemptResult) GetStatus() TerminalStatus { @@ -1198,6 +1403,74 @@ func (x *AttemptResult) GetMillRecordsTerminalMetrics() bool { return false } +type CacheUpdate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Action CacheUpdateAction `protobuf:"varint,1,opt,name=action,proto3,enum=spindle.mill.v1.CacheUpdateAction" json:"action,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + SizeBytes int64 `protobuf:"varint,4,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Checksum string `protobuf:"bytes,5,opt,name=checksum,proto3" json:"checksum,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CacheUpdate) Reset() { + *x = CacheUpdate{} + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CacheUpdate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CacheUpdate) ProtoMessage() {} + +func (x *CacheUpdate) ProtoReflect() protoreflect.Message { + mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CacheUpdate.ProtoReflect.Descriptor instead. +func (*CacheUpdate) Descriptor() ([]byte, []int) { + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} +} + +func (x *CacheUpdate) GetAction() CacheUpdateAction { + if x != nil { + return x.Action + } + return CacheUpdateAction_CACHE_UPDATE_ACTION_UNSPECIFIED +} + +func (x *CacheUpdate) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *CacheUpdate) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *CacheUpdate) GetChecksum() string { + if x != nil { + return x.Checksum + } + return "" +} + // live non-replay log frame type LiveLog struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1209,7 +1482,7 @@ type LiveLog struct { func (x *LiveLog) Reset() { *x = LiveLog{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1221,7 +1494,7 @@ func (x *LiveLog) String() string { func (*LiveLog) ProtoMessage() {} func (x *LiveLog) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[15] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1234,7 +1507,7 @@ func (x *LiveLog) ProtoReflect() protoreflect.Message { // Deprecated: Use LiveLog.ProtoReflect.Descriptor instead. func (*LiveLog) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{15} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} } func (x *LiveLog) GetLeaseId() string { @@ -1260,6 +1533,7 @@ type Event struct { // // *Event_StatusEvent // *Event_AttemptResult + // *Event_CacheUpdate Payload isEvent_Payload `protobuf_oneof:"payload"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1267,7 +1541,7 @@ type Event struct { func (x *Event) Reset() { *x = Event{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1279,7 +1553,7 @@ func (x *Event) String() string { func (*Event) ProtoMessage() {} func (x *Event) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[16] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1292,7 +1566,7 @@ func (x *Event) ProtoReflect() protoreflect.Message { // Deprecated: Use Event.ProtoReflect.Descriptor instead. func (*Event) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{16} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} } func (x *Event) GetSeqno() uint64 { @@ -1334,6 +1608,15 @@ func (x *Event) GetAttemptResult() *AttemptResult { return nil } +func (x *Event) GetCacheUpdate() *CacheUpdate { + if x != nil { + if x, ok := x.Payload.(*Event_CacheUpdate); ok { + return x.CacheUpdate + } + } + return nil +} + type isEvent_Payload interface { isEvent_Payload() } @@ -1346,10 +1629,16 @@ type Event_AttemptResult struct { AttemptResult *AttemptResult `protobuf:"bytes,4,opt,name=attempt_result,json=attemptResult,proto3,oneof"` } +type Event_CacheUpdate struct { + CacheUpdate *CacheUpdate `protobuf:"bytes,5,opt,name=cache_update,json=cacheUpdate,proto3,oneof"` +} + func (*Event_StatusEvent) isEvent_Payload() {} func (*Event_AttemptResult) isEvent_Payload() {} +func (*Event_CacheUpdate) isEvent_Payload() {} + // a flushed bundle of events type EventBatch struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1361,7 +1650,7 @@ type EventBatch struct { func (x *EventBatch) Reset() { *x = EventBatch{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1373,7 +1662,7 @@ func (x *EventBatch) String() string { func (*EventBatch) ProtoMessage() {} func (x *EventBatch) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[17] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1386,7 +1675,7 @@ func (x *EventBatch) ProtoReflect() protoreflect.Message { // Deprecated: Use EventBatch.ProtoReflect.Descriptor instead. func (*EventBatch) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{17} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} } func (x *EventBatch) GetEpoch() string { @@ -1414,7 +1703,7 @@ type Ack struct { func (x *Ack) Reset() { *x = Ack{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1426,7 +1715,7 @@ func (x *Ack) String() string { func (*Ack) ProtoMessage() {} func (x *Ack) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[18] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1439,7 +1728,7 @@ func (x *Ack) ProtoReflect() protoreflect.Message { // Deprecated: Use Ack.ProtoReflect.Descriptor instead. func (*Ack) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{18} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{20} } func (x *Ack) GetEpoch() string { @@ -1479,7 +1768,7 @@ type Message struct { func (x *Message) Reset() { *x = Message{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1491,7 +1780,7 @@ func (x *Message) String() string { func (*Message) ProtoMessage() {} func (x *Message) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[19] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1504,7 +1793,7 @@ func (x *Message) ProtoReflect() protoreflect.Message { // Deprecated: Use Message.ProtoReflect.Descriptor instead. func (*Message) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{19} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{21} } func (x *Message) GetHello() *Hello { @@ -1627,7 +1916,7 @@ type QuotaRequest struct { func (x *QuotaRequest) Reset() { *x = QuotaRequest{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1639,7 +1928,7 @@ func (x *QuotaRequest) String() string { func (*QuotaRequest) ProtoMessage() {} func (x *QuotaRequest) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[20] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1652,7 +1941,7 @@ func (x *QuotaRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use QuotaRequest.ProtoReflect.Descriptor instead. func (*QuotaRequest) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{20} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{22} } func (x *QuotaRequest) GetRequestId() string { @@ -1719,7 +2008,7 @@ type QuotaResponse struct { func (x *QuotaResponse) Reset() { *x = QuotaResponse{} - mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1731,7 +2020,7 @@ func (x *QuotaResponse) String() string { func (*QuotaResponse) ProtoMessage() {} func (x *QuotaResponse) ProtoReflect() protoreflect.Message { - mi := &file_spindle_mill_v1_mill_proto_msgTypes[21] + mi := &file_spindle_mill_v1_mill_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1744,7 +2033,7 @@ func (x *QuotaResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use QuotaResponse.ProtoReflect.Descriptor instead. func (*QuotaResponse) Descriptor() ([]byte, []int) { - return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{21} + return file_spindle_mill_v1_mill_proto_rawDescGZIP(), []int{23} } func (x *QuotaResponse) GetRequestId() string { @@ -1800,12 +2089,16 @@ var File_spindle_mill_v1_mill_proto protoreflect.FileDescriptor const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\n" + - "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"}\n" + + "\x1aspindle/mill/v1/mill.proto\x12\x0fspindle.mill.v1\x1a\x1bbuf/validate/validate.proto\"\xb9\x02\n" + "\x05Hello\x12)\n" + - "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x12\n" + + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x120\n" + + "\x14min_protocol_version\x18\a \x01(\rR\x12minProtocolVersion\x120\n" + + "\x14max_protocol_version\x18\b \x01(\rR\x12maxProtocolVersion\x12\x12\n" + "\x04arch\x18\x02 \x01(\tR\x04arch\x12\x16\n" + "\x06labels\x18\x03 \x03(\tR\x06labels\x12\x1d\n" + - "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\"D\n" + + "\x05epoch\x18\x04 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12$\n" + + "\x0ecache_store_id\x18\x05 \x01(\tR\fcacheStoreId\x120\n" + + "\x0fcache_namespace\x18\x06 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x0ecacheNamespace\"D\n" + "\x06Resume\x12\x1d\n" + "\x05epoch\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x05epoch\x12\x1b\n" + "\tack_seqno\x18\x02 \x01(\x04R\backSeqno\"\xae\x01\n" + @@ -1821,7 +2114,7 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + "\fEnginesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + - "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\xc2\x02\n" + + "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\xe6\x02\n" + "\vReserveSeat\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + @@ -1834,7 +2127,9 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\vtraceparent\x18\b \x01(\tR\vtraceparent\x12\x1e\n" + "\n" + "tracestate\x18\t \x01(\tR\n" + - "tracestate\"\xa4\x04\n" + + "tracestate\x12\"\n" + + "\brepo_did\x18\n" + + " \x01(\tB\a\xbaH\x04r\x02\x10\x01R\arepoDid\"\xa4\x04\n" + "\rReserveResult\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + @@ -1849,7 +2144,19 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\x03R\x05value:\x028\x01\"0\n" + "\x06Secret\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value\"\xe9\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"\xb1\x02\n" + + "\fCacheBinding\x12\x1f\n" + + "\ventry_index\x18\x01 \x01(\rR\n" + + "entryIndex\x12\x1d\n" + + "\n" + + "restore_id\x18\x03 \x01(\tR\trestoreId\x12\x1f\n" + + "\vrestore_key\x18\x04 \x01(\tR\n" + + "restoreKey\x12!\n" + + "\frestore_name\x18\x05 \x01(\tR\vrestoreName\x12 \n" + + "\asave_id\x18\x06 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x06saveId\x12\"\n" + + "\bsave_key\x18\a \x01(\tB\a\xbaH\x04r\x02\x10\x01R\asaveKey\x12)\n" + + "\x10restore_checksum\x18\b \x01(\tR\x0frestoreChecksum\x12,\n" + + "\x12restore_size_bytes\x18\t \x01(\x03R\x10restoreSizeBytes\"\xaf\x02\n" + "\vCommitLease\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x121\n" + "\asecrets\x18\x02 \x03(\v2\x17.spindle.mill.v1.SecretR\asecrets\x12 \n" + @@ -1857,7 +2164,8 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\n" + "tracestate\x18\x04 \x01(\tR\n" + "tracestate\x12A\n" + - "\x1dmill_records_terminal_metrics\x18\x05 \x01(\bR\x1amillRecordsTerminalMetrics\"/\n" + + "\x1dmill_records_terminal_metrics\x18\x05 \x01(\bR\x1amillRecordsTerminalMetrics\x12D\n" + + "\x0ecache_bindings\x18\x06 \x03(\v2\x1d.spindle.mill.v1.CacheBindingR\rcacheBindings\"/\n" + "\tCommitted\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\"2\n" + "\fReleaseLease\x12\"\n" + @@ -1883,15 +2191,23 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\flog_artifact\x18\x04 \x01(\v2\x1c.spindle.mill.v1.LogArtifactR\vlogArtifact\x12,\n" + "\rfailure_class\x18\x05 \x01(\tB\a\xbaH\x04r\x02( R\ffailureClass\x12.\n" + "\x0efailure_reason\x18\x06 \x01(\tB\a\xbaH\x04r\x02(@R\rfailureReason\x12A\n" + - "\x1dmill_records_terminal_metrics\x18\a \x01(\bR\x1amillRecordsTerminalMetrics\"Q\n" + + "\x1dmill_records_terminal_metrics\x18\a \x01(\bR\x1amillRecordsTerminalMetrics\"\xb2\x01\n" + + "\vCacheUpdate\x12F\n" + + "\x06action\x18\x01 \x01(\x0e2\".spindle.mill.v1.CacheUpdateActionB\n" + + "\xbaH\a\x82\x01\x04\x10\x01 \x00R\x06action\x12\x17\n" + + "\x02id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\x02id\x12&\n" + + "\n" + + "size_bytes\x18\x04 \x01(\x03B\a\xbaH\x04\"\x02(\x00R\tsizeBytes\x12\x1a\n" + + "\bchecksum\x18\x05 \x01(\tR\bchecksum\"Q\n" + "\aLiveLog\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\"\n" + - "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xe8\x01\n" + + "\braw_json\x18\x02 \x01(\fB\a\xbaH\x04z\x02\x10\x01R\arawJson\"\xab\x02\n" + "\x05Event\x12\x1d\n" + "\x05seqno\x18\x01 \x01(\x04B\a\xbaH\x042\x02 \x00R\x05seqno\x12\"\n" + "\blease_id\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12A\n" + "\fstatus_event\x18\x03 \x01(\v2\x1c.spindle.mill.v1.StatusEventH\x00R\vstatusEvent\x12G\n" + - "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResultB\x10\n" + + "\x0eattempt_result\x18\x04 \x01(\v2\x1e.spindle.mill.v1.AttemptResultH\x00R\rattemptResult\x12A\n" + + "\fcache_update\x18\x05 \x01(\v2\x1c.spindle.mill.v1.CacheUpdateH\x00R\vcacheUpdateB\x10\n" + "\apayload\x12\x05\xbaH\x02\b\x01\"e\n" + "\n" + "EventBatch\x12\x1d\n" + @@ -1974,7 +2290,13 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x17TERMINAL_STATUS_SUCCESS\x10\x01\x12\x1a\n" + "\x16TERMINAL_STATUS_FAILED\x10\x02\x12\x1b\n" + "\x17TERMINAL_STATUS_TIMEOUT\x10\x03\x12\x1d\n" + - "\x19TERMINAL_STATUS_CANCELLED\x10\x04*\xa9\x01\n" + + "\x19TERMINAL_STATUS_CANCELLED\x10\x04*\xba\x01\n" + + "\x11CacheUpdateAction\x12#\n" + + "\x1fCACHE_UPDATE_ACTION_UNSPECIFIED\x10\x00\x12\x1c\n" + + "\x18CACHE_UPDATE_ACTION_USED\x10\x01\x12\x1e\n" + + "\x1aCACHE_UPDATE_ACTION_STORED\x10\x02\x12!\n" + + "\x1dCACHE_UPDATE_ACTION_DISCARDED\x10\x03\x12\x1f\n" + + "\x1bCACHE_UPDATE_ACTION_MISSING\x10\x04*\xa9\x01\n" + "\x0eQuotaOperation\x12\x1f\n" + "\x1bQUOTA_OPERATION_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17QUOTA_OPERATION_RESERVE\x10\x01\x12 \n" + @@ -1994,75 +2316,81 @@ func file_spindle_mill_v1_mill_proto_rawDescGZIP() []byte { return file_spindle_mill_v1_mill_proto_rawDescData } -var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_spindle_mill_v1_mill_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_spindle_mill_v1_mill_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_spindle_mill_v1_mill_proto_goTypes = []any{ (RejectClass)(0), // 0: spindle.mill.v1.RejectClass (NonterminalStatus)(0), // 1: spindle.mill.v1.NonterminalStatus (TerminalStatus)(0), // 2: spindle.mill.v1.TerminalStatus - (QuotaOperation)(0), // 3: spindle.mill.v1.QuotaOperation - (*Hello)(nil), // 4: spindle.mill.v1.Hello - (*Resume)(nil), // 5: spindle.mill.v1.Resume - (*EngineAvailability)(nil), // 6: spindle.mill.v1.EngineAvailability - (*NodeSnapshot)(nil), // 7: spindle.mill.v1.NodeSnapshot - (*ReserveSeat)(nil), // 8: spindle.mill.v1.ReserveSeat - (*ReserveResult)(nil), // 9: spindle.mill.v1.ReserveResult - (*Secret)(nil), // 10: spindle.mill.v1.Secret - (*CommitLease)(nil), // 11: spindle.mill.v1.CommitLease - (*Committed)(nil), // 12: spindle.mill.v1.Committed - (*ReleaseLease)(nil), // 13: spindle.mill.v1.ReleaseLease - (*CancelAttempt)(nil), // 14: spindle.mill.v1.CancelAttempt - (*CancelAck)(nil), // 15: spindle.mill.v1.CancelAck - (*StatusEvent)(nil), // 16: spindle.mill.v1.StatusEvent - (*LogArtifact)(nil), // 17: spindle.mill.v1.LogArtifact - (*AttemptResult)(nil), // 18: spindle.mill.v1.AttemptResult - (*LiveLog)(nil), // 19: spindle.mill.v1.LiveLog - (*Event)(nil), // 20: spindle.mill.v1.Event - (*EventBatch)(nil), // 21: spindle.mill.v1.EventBatch - (*Ack)(nil), // 22: spindle.mill.v1.Ack - (*Message)(nil), // 23: spindle.mill.v1.Message - (*QuotaRequest)(nil), // 24: spindle.mill.v1.QuotaRequest - (*QuotaResponse)(nil), // 25: spindle.mill.v1.QuotaResponse - nil, // 26: spindle.mill.v1.EngineAvailability.LoadEntry - nil, // 27: spindle.mill.v1.NodeSnapshot.EnginesEntry - nil, // 28: spindle.mill.v1.ReserveResult.QuotaResourcesEntry - nil, // 29: spindle.mill.v1.QuotaRequest.ResourcesEntry + (CacheUpdateAction)(0), // 3: spindle.mill.v1.CacheUpdateAction + (QuotaOperation)(0), // 4: spindle.mill.v1.QuotaOperation + (*Hello)(nil), // 5: spindle.mill.v1.Hello + (*Resume)(nil), // 6: spindle.mill.v1.Resume + (*EngineAvailability)(nil), // 7: spindle.mill.v1.EngineAvailability + (*NodeSnapshot)(nil), // 8: spindle.mill.v1.NodeSnapshot + (*ReserveSeat)(nil), // 9: spindle.mill.v1.ReserveSeat + (*ReserveResult)(nil), // 10: spindle.mill.v1.ReserveResult + (*Secret)(nil), // 11: spindle.mill.v1.Secret + (*CacheBinding)(nil), // 12: spindle.mill.v1.CacheBinding + (*CommitLease)(nil), // 13: spindle.mill.v1.CommitLease + (*Committed)(nil), // 14: spindle.mill.v1.Committed + (*ReleaseLease)(nil), // 15: spindle.mill.v1.ReleaseLease + (*CancelAttempt)(nil), // 16: spindle.mill.v1.CancelAttempt + (*CancelAck)(nil), // 17: spindle.mill.v1.CancelAck + (*StatusEvent)(nil), // 18: spindle.mill.v1.StatusEvent + (*LogArtifact)(nil), // 19: spindle.mill.v1.LogArtifact + (*AttemptResult)(nil), // 20: spindle.mill.v1.AttemptResult + (*CacheUpdate)(nil), // 21: spindle.mill.v1.CacheUpdate + (*LiveLog)(nil), // 22: spindle.mill.v1.LiveLog + (*Event)(nil), // 23: spindle.mill.v1.Event + (*EventBatch)(nil), // 24: spindle.mill.v1.EventBatch + (*Ack)(nil), // 25: spindle.mill.v1.Ack + (*Message)(nil), // 26: spindle.mill.v1.Message + (*QuotaRequest)(nil), // 27: spindle.mill.v1.QuotaRequest + (*QuotaResponse)(nil), // 28: spindle.mill.v1.QuotaResponse + nil, // 29: spindle.mill.v1.EngineAvailability.LoadEntry + nil, // 30: spindle.mill.v1.NodeSnapshot.EnginesEntry + nil, // 31: spindle.mill.v1.ReserveResult.QuotaResourcesEntry + nil, // 32: spindle.mill.v1.QuotaRequest.ResourcesEntry } var file_spindle_mill_v1_mill_proto_depIdxs = []int32{ - 26, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry - 27, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry + 29, // 0: spindle.mill.v1.EngineAvailability.load:type_name -> spindle.mill.v1.EngineAvailability.LoadEntry + 30, // 1: spindle.mill.v1.NodeSnapshot.engines:type_name -> spindle.mill.v1.NodeSnapshot.EnginesEntry 0, // 2: spindle.mill.v1.ReserveResult.reject_class:type_name -> spindle.mill.v1.RejectClass - 28, // 3: spindle.mill.v1.ReserveResult.quota_resources:type_name -> spindle.mill.v1.ReserveResult.QuotaResourcesEntry - 10, // 4: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret - 1, // 5: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus - 2, // 6: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus - 17, // 7: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact - 16, // 8: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent - 18, // 9: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult - 20, // 10: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event - 4, // 11: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello - 5, // 12: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume - 7, // 13: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot - 8, // 14: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat - 9, // 15: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult - 11, // 16: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease - 12, // 17: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed - 13, // 18: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease - 14, // 19: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt - 15, // 20: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck - 21, // 21: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch - 22, // 22: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack - 19, // 23: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog - 24, // 24: spindle.mill.v1.Message.quota_req:type_name -> spindle.mill.v1.QuotaRequest - 25, // 25: spindle.mill.v1.Message.quota_resp:type_name -> spindle.mill.v1.QuotaResponse - 3, // 26: spindle.mill.v1.QuotaRequest.operation:type_name -> spindle.mill.v1.QuotaOperation - 29, // 27: spindle.mill.v1.QuotaRequest.resources:type_name -> spindle.mill.v1.QuotaRequest.ResourcesEntry - 6, // 28: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability - 29, // [29:29] is the sub-list for method output_type - 29, // [29:29] is the sub-list for method input_type - 29, // [29:29] is the sub-list for extension type_name - 29, // [29:29] is the sub-list for extension extendee - 0, // [0:29] is the sub-list for field type_name + 31, // 3: spindle.mill.v1.ReserveResult.quota_resources:type_name -> spindle.mill.v1.ReserveResult.QuotaResourcesEntry + 11, // 4: spindle.mill.v1.CommitLease.secrets:type_name -> spindle.mill.v1.Secret + 12, // 5: spindle.mill.v1.CommitLease.cache_bindings:type_name -> spindle.mill.v1.CacheBinding + 1, // 6: spindle.mill.v1.StatusEvent.status:type_name -> spindle.mill.v1.NonterminalStatus + 2, // 7: spindle.mill.v1.AttemptResult.status:type_name -> spindle.mill.v1.TerminalStatus + 19, // 8: spindle.mill.v1.AttemptResult.log_artifact:type_name -> spindle.mill.v1.LogArtifact + 3, // 9: spindle.mill.v1.CacheUpdate.action:type_name -> spindle.mill.v1.CacheUpdateAction + 18, // 10: spindle.mill.v1.Event.status_event:type_name -> spindle.mill.v1.StatusEvent + 20, // 11: spindle.mill.v1.Event.attempt_result:type_name -> spindle.mill.v1.AttemptResult + 21, // 12: spindle.mill.v1.Event.cache_update:type_name -> spindle.mill.v1.CacheUpdate + 23, // 13: spindle.mill.v1.EventBatch.events:type_name -> spindle.mill.v1.Event + 5, // 14: spindle.mill.v1.Message.hello:type_name -> spindle.mill.v1.Hello + 6, // 15: spindle.mill.v1.Message.resume:type_name -> spindle.mill.v1.Resume + 8, // 16: spindle.mill.v1.Message.node_snapshot:type_name -> spindle.mill.v1.NodeSnapshot + 9, // 17: spindle.mill.v1.Message.reserve_seat:type_name -> spindle.mill.v1.ReserveSeat + 10, // 18: spindle.mill.v1.Message.reserve_result:type_name -> spindle.mill.v1.ReserveResult + 13, // 19: spindle.mill.v1.Message.commit_lease:type_name -> spindle.mill.v1.CommitLease + 14, // 20: spindle.mill.v1.Message.committed:type_name -> spindle.mill.v1.Committed + 15, // 21: spindle.mill.v1.Message.release_lease:type_name -> spindle.mill.v1.ReleaseLease + 16, // 22: spindle.mill.v1.Message.cancel_attempt:type_name -> spindle.mill.v1.CancelAttempt + 17, // 23: spindle.mill.v1.Message.cancel_ack:type_name -> spindle.mill.v1.CancelAck + 24, // 24: spindle.mill.v1.Message.event_batch:type_name -> spindle.mill.v1.EventBatch + 25, // 25: spindle.mill.v1.Message.ack:type_name -> spindle.mill.v1.Ack + 22, // 26: spindle.mill.v1.Message.live_log:type_name -> spindle.mill.v1.LiveLog + 27, // 27: spindle.mill.v1.Message.quota_req:type_name -> spindle.mill.v1.QuotaRequest + 28, // 28: spindle.mill.v1.Message.quota_resp:type_name -> spindle.mill.v1.QuotaResponse + 4, // 29: spindle.mill.v1.QuotaRequest.operation:type_name -> spindle.mill.v1.QuotaOperation + 32, // 30: spindle.mill.v1.QuotaRequest.resources:type_name -> spindle.mill.v1.QuotaRequest.ResourcesEntry + 7, // 31: spindle.mill.v1.NodeSnapshot.EnginesEntry.value:type_name -> spindle.mill.v1.EngineAvailability + 32, // [32:32] is the sub-list for method output_type + 32, // [32:32] is the sub-list for method input_type + 32, // [32:32] is the sub-list for extension type_name + 32, // [32:32] is the sub-list for extension extendee + 0, // [0:32] is the sub-list for field type_name } func init() { file_spindle_mill_v1_mill_proto_init() } @@ -2070,17 +2398,18 @@ func file_spindle_mill_v1_mill_proto_init() { if File_spindle_mill_v1_mill_proto != nil { return } - file_spindle_mill_v1_mill_proto_msgTypes[16].OneofWrappers = []any{ + file_spindle_mill_v1_mill_proto_msgTypes[18].OneofWrappers = []any{ (*Event_StatusEvent)(nil), (*Event_AttemptResult)(nil), + (*Event_CacheUpdate)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_spindle_mill_v1_mill_proto_rawDesc), len(file_spindle_mill_v1_mill_proto_rawDesc)), - NumEnums: 4, - NumMessages: 26, + NumEnums: 5, + NumMessages: 28, NumExtensions: 0, NumServices: 0, }, diff --git a/spindle/mill/proto/protocol.go b/spindle/mill/proto/protocol.go index b9db8cbb..f30eb03a 100644 --- a/spindle/mill/proto/protocol.go +++ b/spindle/mill/proto/protocol.go @@ -17,7 +17,17 @@ import ( ) const ( - ProtocolVersion = 3 + // ProtocolVersion is the wire version selected by the current implementation. + ProtocolVersion = 4 + // ProtocolMinVersion and ProtocolMaxVersion delimit versions this endpoint + // can negotiate during rolling upgrades. + ProtocolMinVersion = ProtocolVersion + ProtocolMaxVersion = ProtocolVersion + // CacheSentinelKey is initialized by the mill and read by executors before + // they advertise cache support. Its value prevents an executor-local store + // from passing the shared-store check by merely containing the key. + CacheSentinelKey = "spindle-cache-sentinel" + CacheSentinelValue = "spindle-cache-sentinel-v1" // generous vs agentproto's 1 MiB. a ReserveSeat carries the raw pipeline and // workflow JSON, and streamed log lines can be chunky MaxMessageBytes = 8 * 1024 * 1024 diff --git a/spindle/mill/proto/protocol_test.go b/spindle/mill/proto/protocol_test.go index b1d860e6..292211a3 100644 --- a/spindle/mill/proto/protocol_test.go +++ b/spindle/mill/proto/protocol_test.go @@ -22,6 +22,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { Knot: "knot.example", Rkey: "abc123", TtlSeconds: 30, + RepoDid: "did:web:example.com", }, } if err := enc.Encode(want); err != nil { @@ -36,7 +37,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { if rs == nil { t.Fatal("decoded message missing reserve_seat") } - if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 { + if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 || rs.RepoDid != "did:web:example.com" { t.Fatalf("round-trip mismatch: %+v", rs) } } @@ -71,11 +72,11 @@ func TestValidationRules(t *testing.T) { { name: "valid hello message", msg: &Message{ - Hello: &millv1.Hello{ - ProtocolVersion: 1, - Arch: "amd64", - Labels: []string{"linux"}, - Epoch: "inc-1", + Hello: &millv1.Hello{ProtocolVersion: 1, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", }, }, wantErr: false, diff --git a/spindle/mill/proto/spindle/mill/v1/mill.proto b/spindle/mill/proto/spindle/mill/v1/mill.proto index 0ff2e217..f59007df 100644 --- a/spindle/mill/proto/spindle/mill/v1/mill.proto +++ b/spindle/mill/proto/spindle/mill/v1/mill.proto @@ -9,11 +9,16 @@ option go_package = "tangled.org/core/spindle/mill/proto/gen;millv1"; // executor identity, sent on connect message Hello { uint32 protocol_version = 1; + // supported protocol range for rolling upgrades. zero means a legacy v4 executor. + uint32 min_protocol_version = 7; + uint32 max_protocol_version = 8; // GOARCH of the node, informational only string arch = 2; // operator-defined labels, matched against runs_on repeated string labels = 3; string epoch = 4 [(buf.validate.field).string.min_len = 1]; + string cache_store_id = 5; + string cache_namespace = 6 [(buf.validate.field).string.min_len = 1]; } // reconnect state for an existing epoch @@ -49,6 +54,8 @@ message ReserveSeat { uint32 ttl_seconds = 7; string traceparent = 8; string tracestate = 9; + // repository that owns the pipeline, used to distinguish fork source code + string repo_did = 10 [(buf.validate.field).string.min_len = 1]; } enum RejectClass { @@ -78,6 +85,17 @@ message Secret { string value = 2; } +message CacheBinding { + uint32 entry_index = 1; + string restore_id = 3; + string restore_key = 4; + string restore_name = 5; + string save_id = 6 [(buf.validate.field).string.min_len = 1]; + string save_key = 7 [(buf.validate.field).string.min_len = 1]; + string restore_checksum = 8; + int64 restore_size_bytes = 9; +} + // promotes a reservation to a running job and hands over the secrets message CommitLease { string lease_id = 1 [(buf.validate.field).string.min_len = 1]; @@ -85,6 +103,7 @@ message CommitLease { string traceparent = 3; string tracestate = 4; bool mill_records_terminal_metrics = 5; + repeated CacheBinding cache_bindings = 6; } message Committed { @@ -147,6 +166,24 @@ message AttemptResult { bool mill_records_terminal_metrics = 7; } +enum CacheUpdateAction { + CACHE_UPDATE_ACTION_UNSPECIFIED = 0; + CACHE_UPDATE_ACTION_USED = 1; + CACHE_UPDATE_ACTION_STORED = 2; + CACHE_UPDATE_ACTION_DISCARDED = 3; + CACHE_UPDATE_ACTION_MISSING = 4; +} + +message CacheUpdate { + CacheUpdateAction action = 1 [(buf.validate.field).enum = { + defined_only: true + not_in: 0 + }]; + string id = 2 [(buf.validate.field).string.min_len = 1]; + int64 size_bytes = 4 [(buf.validate.field).int64.gte = 0]; + string checksum = 5; +} + // live non-replay log frame message LiveLog { string lease_id = 1 [(buf.validate.field).string.min_len = 1]; @@ -162,6 +199,7 @@ message Event { option (buf.validate.oneof).required = true; StatusEvent status_event = 3; AttemptResult attempt_result = 4; + CacheUpdate cache_update = 5; } } diff --git a/spindle/mill/restore_test.go b/spindle/mill/restore_test.go index d0b3f301..5a13f2a3 100644 --- a/spindle/mill/restore_test.go +++ b/spindle/mill/restore_test.go @@ -233,7 +233,7 @@ func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) { default: t.Fatal("snapshot reconciliation left an executor-only lease running") } - assertRankedNodes(t, m.rankCandidates("dummy", nil), nil) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), nil) if err := m.onEventBatch(sess, &millv1.EventBatch{ Epoch: sess.epoch, Events: []*millv1.Event{{ @@ -246,7 +246,7 @@ func TestSnapshotReconciliationCancelsUnknownExecutorLease(t *testing.T) { }); err != nil { t.Fatalf("onEventBatch: %v", err) } - assertRankedNodes(t, m.rankCandidates("dummy", nil), []string{"node-1"}) + assertRankedNodes(t, m.rankCandidates("dummy", nil, false), []string{"node-1"}) } func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { diff --git a/spindle/mill/session.go b/spindle/mill/session.go index 08851420..2ce71327 100644 --- a/spindle/mill/session.go +++ b/spindle/mill/session.go @@ -19,7 +19,11 @@ var errSessionClosed = errors.New("mill: executor session closed") type millSession struct { nodeID string epoch string + arch string labels []string + cacheStoreID string + cacheNamespace string + legacyProtocol bool enc messageEncoder l *slog.Logger closeTransport func() error diff --git a/spindle/models/cache.go b/spindle/models/cache.go new file mode 100644 index 00000000..3b792f75 --- /dev/null +++ b/spindle/models/cache.go @@ -0,0 +1,132 @@ +package models + +import ( + "fmt" + "regexp" + "runtime" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/google/uuid" + "gopkg.in/yaml.v3" +) + +type CacheEntry struct { + Key string `yaml:"key"` + Hash []string `yaml:"hash"` + Paths []string `yaml:"paths"` + Fallback bool `yaml:"fallback"` + // 1 is fastest, 19 is smallest, 0 is the zstd default + CompressionLevel int `yaml:"compression-level"` + // on-success is the default, always also saves on failed runs + When string `yaml:"when"` +} + +type CacheBinding struct { + EntryIndex int + Paths []string + Key string + Hash string + Checksum string + SizeBytes int64 + SaveID string + SaveKey string + RestoreID string + RestoreKey string + RestoreName string + CompressionLevel int + When string +} + +func (b CacheBinding) SaveOn(failed bool) bool { + return b.When == "always" || !failed +} + +func CacheNamespace() string { + return fmt.Sprintf("tar-zstd-v1/%s/%s", runtime.GOOS, runtime.GOARCH) +} + +func CacheObjectKey(repoDid, id string) (string, error) { + did, err := syntax.ParseDID(repoDid) + if err != nil { + return "", fmt.Errorf("cache: invalid repository did %q: %w", repoDid, err) + } + parsedID, err := uuid.Parse(id) + if err != nil { + return "", fmt.Errorf("cache: invalid object id %q: %w", id, err) + } + return fmt.Sprintf("objects/%s/%s", did, parsedID), nil +} + +// keys become storage paths, so no slashes +var cacheKeyRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + +func (c CacheEntry) Validate() error { + if !cacheKeyRe.MatchString(c.Key) { + return fmt.Errorf("cache: invalid key %q (allowed: letters, digits, '.', '_', '-')", c.Key) + } + for _, f := range c.Hash { + // rev-parse needs plain repo-relative paths, not pathspecs + if f == "" || strings.HasPrefix(f, "/") || strings.HasPrefix(f, "..") { + return fmt.Errorf("cache %q: hash path %q is not repo-relative", c.Key, f) + } + if strings.ContainsAny(f, ": \t\n\"'`$\\*?[") || strings.Contains(f, "/../") || strings.HasSuffix(f, "/..") { + return fmt.Errorf("cache %q: hash path %q contains unsupported characters", c.Key, f) + } + } + if len(c.Paths) == 0 { + return fmt.Errorf("cache %q: no paths", c.Key) + } + if c.CompressionLevel < 0 || c.CompressionLevel > 19 { + return fmt.Errorf("cache %q: compression-level %d out of range (1-19)", c.Key, c.CompressionLevel) + } + switch c.When { + case "", "on-success", "always": + default: + return fmt.Errorf("cache %q: when %q is not one of on-success, always", c.Key, c.When) + } + seen := make(map[string]bool, len(c.Paths)) + for _, p := range c.Paths { + if p == "" { + return fmt.Errorf("cache %q: empty path", c.Key) + } + if strings.ContainsAny(p, " \t\n\"'`$\\;&|<>(){}") { + return fmt.Errorf("cache %q: path %q contains unsupported characters", c.Key, p) + } + if strings.Contains(p, "..") { + return fmt.Errorf("cache %q: path %q must not contain '..'", c.Key, p) + } + if seen[p] { + return fmt.Errorf("cache %q: duplicate path %q", c.Key, p) + } + seen[p] = true + } + return nil +} + +func ParseCacheEntries(raw string) ([]CacheEntry, error) { + var workflow struct { + Cache []CacheEntry `yaml:"cache"` + } + if err := yaml.Unmarshal([]byte(raw), &workflow); err != nil { + return nil, err + } + if err := ValidateCacheEntries(workflow.Cache); err != nil { + return nil, err + } + return workflow.Cache, nil +} + +func ValidateCacheEntries(entries []CacheEntry) error { + seen := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + if err := entry.Validate(); err != nil { + return err + } + if _, ok := seen[entry.Key]; ok { + return fmt.Errorf("cache: duplicate key %q", entry.Key) + } + seen[entry.Key] = struct{}{} + } + return nil +} diff --git a/spindle/models/cache_test.go b/spindle/models/cache_test.go new file mode 100644 index 00000000..2cc92e78 --- /dev/null +++ b/spindle/models/cache_test.go @@ -0,0 +1,78 @@ +package models + +import ( + "strings" + "testing" +) + +func TestCacheEntryValidate(t *testing.T) { + valid := CacheEntry{Key: "go-mod-v1", Hash: []string{"go.sum", "sub/dir/package-lock.json"}, Paths: []string{"/workspace/go/pkg/mod", "/root/.cache"}} + if err := valid.Validate(); err != nil { + t.Fatalf("valid entry: %v", err) + } + + cases := []struct { + name string + entry CacheEntry + want string + }{ + {"empty key", CacheEntry{Key: "", Paths: []string{"/x"}}, "invalid key"}, + {"key with slash", CacheEntry{Key: "a/b", Paths: []string{"/x"}}, "invalid key"}, + {"no paths", CacheEntry{Key: "ok"}, "no paths"}, + {"empty path", CacheEntry{Key: "ok", Paths: []string{""}}, "empty path"}, + {"path with space", CacheEntry{Key: "ok", Paths: []string{"/my dir"}}, "unsupported characters"}, + {"path with quote", CacheEntry{Key: "ok", Paths: []string{"/x'$(rm)"}}, "unsupported characters"}, + {"path with shell syntax", CacheEntry{Key: "ok", Paths: []string{"/x;touch-pwn"}}, "unsupported characters"}, + {"path traversal", CacheEntry{Key: "ok", Paths: []string{"/x/../y"}}, ".."}, + {"duplicate path", CacheEntry{Key: "ok", Paths: []string{"/x", "/x"}}, "duplicate"}, + {"level too high", CacheEntry{Key: "ok", Paths: []string{"/x"}, CompressionLevel: 20}, "out of range"}, + {"negative level", CacheEntry{Key: "ok", Paths: []string{"/x"}, CompressionLevel: -1}, "out of range"}, + {"bad when", CacheEntry{Key: "ok", Paths: []string{"/x"}, When: "sometimes"}, "not one of"}, + {"absolute hash path", CacheEntry{Key: "ok", Hash: []string{"/go.sum"}, Paths: []string{"/x"}}, "not repo-relative"}, + {"empty hash path", CacheEntry{Key: "ok", Hash: []string{""}, Paths: []string{"/x"}}, "not repo-relative"}, + {"hash path traversal", CacheEntry{Key: "ok", Hash: []string{"../secret"}, Paths: []string{"/x"}}, "not repo-relative"}, + {"hash path inner traversal", CacheEntry{Key: "ok", Hash: []string{"a/../../b"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with colon", CacheEntry{Key: "ok", Hash: []string{"rev:go.sum"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with space", CacheEntry{Key: "ok", Hash: []string{"my lock"}, Paths: []string{"/x"}}, "unsupported characters"}, + {"hash path with glob", CacheEntry{Key: "ok", Hash: []string{"*.sum"}, Paths: []string{"/x"}}, "unsupported characters"}, + } + for _, tc := range cases { + err := tc.entry.Validate() + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: got %v, want error containing %q", tc.name, err, tc.want) + } + } + + ok := CacheEntry{Key: "go-mod", Hash: []string{"go.sum"}, Paths: []string{"node_modules", "/root/.cache"}, CompressionLevel: 19, When: "always"} + if err := ok.Validate(); err != nil { + t.Errorf("relative and absolute paths should validate, got %v", err) + } +} + +func TestParseCacheEntries(t *testing.T) { + entries, err := ParseCacheEntries(` +cache: + - key: go-mod + hash: [go.sum] + paths: [/root/go/pkg/mod] + compression-level: 7 + when: always +`) + if err != nil { + t.Fatalf("ParseCacheEntries: %v", err) + } + if len(entries) != 1 || entries[0].Key != "go-mod" || entries[0].CompressionLevel != 7 || entries[0].When != "always" { + t.Fatalf("parsed entries = %+v", entries) + } + + _, err = ParseCacheEntries(` +cache: + - key: deps + paths: [/deps] + - key: deps + paths: [/other] +`) + if err == nil || !strings.Contains(err.Error(), "duplicate key") { + t.Fatalf("duplicate cache keys error = %v", err) + } +} diff --git a/spindle/models/clone.go b/spindle/models/clone.go index d9d54389..1f6be1cb 100644 --- a/spindle/models/clone.go +++ b/spindle/models/clone.go @@ -47,7 +47,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe return CloneStep{} } - commitSHA, err := extractCommitSHA(tr) + commitSHA, err := ExtractCommitSHA(tr) if err != nil { return CloneStep{ kind: StepKindSystem, @@ -83,8 +83,7 @@ func BuildCloneStep(twf tangled.Pipeline_Workflow, tr tangled.Pipeline_TriggerMe } } -// extractCommitSHA extracts the commit SHA from trigger metadata based on trigger type -func extractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { +func ExtractCommitSHA(tr tangled.Pipeline_TriggerMetadata) (string, error) { switch workflow.TriggerKind(tr.Kind) { case workflow.TriggerKindPush: if tr.Push == nil { diff --git a/spindle/models/pipeline.go b/spindle/models/pipeline.go index e7a49fe5..fcfe84e8 100644 --- a/spindle/models/pipeline.go +++ b/spindle/models/pipeline.go @@ -1,12 +1,36 @@ package models -import "github.com/bluesky-social/indigo/atproto/syntax" +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + + "tangled.org/core/api/tangled" +) type Pipeline struct { RepoDid syntax.DID Workflows map[Engine][]Workflow // whether the code being ran was checked out from RepoDid itself TrustedSource bool + // used to resolve cache hash files against the checkout the workflow builds + TriggerMetadata *tangled.Pipeline_TriggerMetadata +} + +func TrustedPipelineSource(metadata *tangled.Pipeline_TriggerMetadata, target string) bool { + targetDid, err := syntax.ParseDID(target) + if err != nil || metadata == nil || metadata.Repo == nil || metadata.Repo.RepoDid == nil { + return false + } + checkoutDid, err := syntax.ParseDID(*metadata.Repo.RepoDid) + if err != nil || checkoutDid != targetDid { + return false + } + if metadata.SourceRepo != nil { + sourceDid, err := syntax.ParseDID(*metadata.SourceRepo) + if err != nil || sourceDid != checkoutDid { + return false + } + } + return true } type Step interface { @@ -25,11 +49,15 @@ const ( ) type Workflow struct { - Steps []Step - Name string - Data any - Environment map[string]string - OwnerDID string - RepoDID string - RunID string + Steps []Step + Name string + Data any + Environment map[string]string + OwnerDID string + RepoDID string + RunID string + Caches []CacheEntry + CacheBindings []CacheBinding + CacheNamespace string + Engine string } diff --git a/spindle/models/pipeline_test.go b/spindle/models/pipeline_test.go new file mode 100644 index 00000000..608de864 --- /dev/null +++ b/spindle/models/pipeline_test.go @@ -0,0 +1,34 @@ +package models + +import ( + "testing" + + "tangled.org/core/api/tangled" +) + +func TestTrustedPipelineSourceRequiresConsistentCheckoutMetadata(t *testing.T) { + target := "did:web:target.example" + fork := "did:web:fork.example" + for _, tc := range []struct { + name string + repo *string + source *string + trusted bool + }{ + {name: "target checkout", repo: &target, trusted: true}, + {name: "consistent target source", repo: &target, source: &target, trusted: true}, + {name: "fork checkout", repo: &fork, source: &fork}, + {name: "contradictory source", repo: &fork, source: &target}, + {name: "missing checkout", source: &target}, + } { + t.Run(tc.name, func(t *testing.T) { + metadata := &tangled.Pipeline_TriggerMetadata{SourceRepo: tc.source} + if tc.repo != nil { + metadata.Repo = &tangled.Pipeline_TriggerRepo{RepoDid: tc.repo} + } + if got := TrustedPipelineSource(metadata, target); got != tc.trusted { + t.Fatalf("TrustedPipelineSource() = %t, want %t", got, tc.trusted) + } + }) + } +} diff --git a/spindle/server.go b/spindle/server.go index dd2d0bc1..98b653a0 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -52,6 +52,7 @@ import ( "tangled.org/core/spindle/observability" "tangled.org/core/spindle/quota" "tangled.org/core/spindle/secrets" + "tangled.org/core/spindle/storage" "tangled.org/core/spindle/xrpc" "tangled.org/core/tid" "tangled.org/core/workflow" @@ -94,6 +95,7 @@ type Spindle struct { res *idresolver.Resolver verify repoverify.Verifier vault secrets.Manager + cache storage.Storage motd []byte motdMu sync.RWMutex rootCtx context.Context @@ -107,6 +109,23 @@ type Spindle struct { exec executorClient } +func newCacheStore(ctx context.Context, cfg *config.Config) (storage.Storage, error) { + switch cfg.Cache.Backend { + case "": + return nil, nil + case "disk": + dir := cfg.Cache.DiskDir + if dir == "" { + dir = filepath.Join(filepath.Dir(cfg.Server.DBPath), "cache") + } + return storage.NewDisk(dir) + case "s3": + return storage.NewUnversionedS3(ctx, cfg.Cache.S3Bucket, cfg.Cache.S3Prefix) + default: + return nil, fmt.Errorf("storage: unknown backend %q", cfg.Cache.Backend) + } +} + // New creates a new Spindle server with the provided configuration and engines. func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]models.Engine, qm *quota.Manager) (*Spindle, error) { logger := log.FromContext(ctx) @@ -132,6 +151,14 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m return nil, fmt.Errorf("failed to run startup migrations: %w", err) } + cacheStore, err := newCacheStore(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("failed to setup cache storage: %w", err) + } + if cacheStore != nil { + logger.Info("cache storage enabled", "backend", cfg.Cache.Backend, "storeID", cfg.Cache.StoreID) + } + spindle := &Spindle{ db: d, l: logger, @@ -139,6 +166,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m metrics: metrics, engs: engines, cfg: cfg, + cache: cacheStore, motd: defaultMotd, rootCtx: lifecycleCtx, rootCancel: cancelLifecycle, @@ -161,6 +189,9 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m if cfg.LegacyS3.LogBucket != "" { logger.Warn("SPINDLE_S3_LOG_BUCKET is deprecated; use SPINDLE_ARTIFACT_STORES_S3_BUCKET") } + if cfg.Role != config.RoleExecutor { + engine.StartCachePruner(ctx, logger, d, cacheStore, cfg.Cache.Retention, cfg.Cache.PruneInterval) + } if cfg.Role == config.RoleStandalone { spindle.reader = stores } else { @@ -308,6 +339,7 @@ func New(ctx context.Context, cfg *config.Config, d *db.DB, engines map[string]m cfg.Server.Tap.AdminPassword = pw logger.Info("embedded tap: using random admin password") } + spindle.tap = NewTapClient(spindle) keepLifecycle = true @@ -620,6 +652,8 @@ func Run(ctx context.Context) error { ReconnectGrace: cfg.Mill.ReconnectGrace, CancelAckTimeout: cfg.Mill.CancelAckTimeout, CancelTeardownTimeout: cfg.Mill.CancelTeardownTimeout, + CacheStoreID: cfg.Cache.StoreID, + CacheMaxBytesPerOwner: cfg.Cache.MaxBytesPerOwner, }) engines = map[string]models.Engine{ "nixery": mill.NewEngine("nixery", m), @@ -692,6 +726,7 @@ func Run(ctx context.Context) error { // the engines built above hold the mill, but the mill's db and // notifier only exist after New, so attach them here m.Attach(s.DB(), s.Notifier(), qm) + m.AttachCache(s.cache) s.mill = m s.mill.RegisterMetrics(s.metrics) if err := m.RestoreState(); err != nil { @@ -706,7 +741,7 @@ func Run(ctx context.Context) error { } } if cfg.Role == config.RoleExecutor { - s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store) + s.exec, err = executor.New(cfg, engines, s.DB(), s.Notifier(), log.SubLogger(logger, "executor"), s.store, s.cache) if err != nil { return err } @@ -1355,11 +1390,6 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { } pipelineEnv := models.PipelineEnvVarsForSource(job.Tpl.TriggerMetadata, pipelineId, job.SourceRepo) - trustedSource := true - if tm := job.Tpl.TriggerMetadata; tm != nil && tm.SourceRepo != nil && - *tm.SourceRepo != "" && *tm.SourceRepo != job.RepoDid { - trustedSource = false - } initTpl := job.Tpl if job.SourceRepo != nil && job.Tpl.TriggerMetadata != nil { @@ -1367,6 +1397,7 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { tm.Repo = job.SourceRepo initTpl.TriggerMetadata = &tm } + trustedSource := models.TrustedPipelineSource(initTpl.TriggerMetadata, job.RepoDid) workflows := make(map[models.Engine][]models.Workflow) for _, w := range job.Tpl.Workflows { @@ -1411,13 +1442,20 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { ewf.Environment = make(map[string]string) } maps.Copy(ewf.Environment, pipelineEnv) + + ewf.Engine = w.Engine + ewf.RepoDID = job.RepoDid + if !trustedSource { + ewf.Caches = nil + } workflows[eng] = append(workflows[eng], *ewf) } - engine.StartWorkflows(l, s.vault, s.cfg, s.qm, s.stores, s.db, s.n, jobCtx, &models.Pipeline{ - RepoDid: syntax.DID(job.RepoDid), - Workflows: workflows, - TrustedSource: trustedSource, + engine.StartWorkflows(l, s.vault, s.cfg, s.qm, s.stores, s.db, s.n, s.cache, nil, jobCtx, &models.Pipeline{ + RepoDid: syntax.DID(job.RepoDid), + Workflows: workflows, + TrustedSource: trustedSource, + TriggerMetadata: job.Tpl.TriggerMetadata, }, pipelineId) } diff --git a/spindle/storage/disk.go b/spindle/storage/disk.go index 670a58b6..a851fa3f 100644 --- a/spindle/storage/disk.go +++ b/spindle/storage/disk.go @@ -76,6 +76,52 @@ func (d *Disk) Put(_ context.Context, key string, r io.Reader) error { return nil } +func (d *Disk) Stat(_ context.Context, key string) (int64, error) { + p, err := d.path(key) + if err != nil { + return 0, err + } + info, err := os.Stat(p) + if err != nil { + if os.IsNotExist(err) { + return 0, ErrNotExist + } + return 0, fmt.Errorf("storage: stat %q: %w", key, err) + } + return info.Size(), nil +} + +func (d *Disk) PutIfAbsent(_ context.Context, key string, r io.Reader) error { + p, err := d.path(key) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-*") + if err != nil { + return fmt.Errorf("storage: put %q: %w", key, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if _, err := io.Copy(tmp, r); err != nil { + _ = tmp.Close() + return fmt.Errorf("storage: put %q: %w", key, err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return fmt.Errorf("storage: put %q: %w", key, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("storage: put %q: %w", key, err) + } + if err := os.Link(tmpName, p); err != nil { + return fmt.Errorf("storage: put %q: %w", key, err) + } + return nil +} + func (d *Disk) Delete(_ context.Context, key string) error { p, err := d.path(key) if err != nil { diff --git a/spindle/storage/s3.go b/spindle/storage/s3.go index ab28ba12..bf1b4148 100644 --- a/spindle/storage/s3.go +++ b/spindle/storage/s3.go @@ -7,6 +7,7 @@ import ( "io" "strings" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" ) @@ -81,6 +82,49 @@ func (s *S3) Put(ctx context.Context, key string, r io.Reader) error { return nil } +type s3HeadClient interface { + HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) +} + +func (s *S3) Stat(ctx context.Context, key string) (int64, error) { + full, err := s.fullKey(key) + if err != nil { + return 0, err + } + if client, ok := s.client.(s3HeadClient); ok { + res, err := client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &s.bucket, Key: &full}) + if err != nil { + return 0, fmt.Errorf("storage: stat %q: %w", key, err) + } + if res.ContentLength == nil { + return 0, fmt.Errorf("storage: stat %q: missing content length", key) + } + return *res.ContentLength, nil + } + + r, err := s.Get(ctx, key) + if err != nil { + return 0, err + } + defer r.Close() + n, err := io.Copy(io.Discard, r) + if err != nil { + return 0, fmt.Errorf("storage: stat %q: %w", key, err) + } + return n, nil +} + +func (s *S3) PutIfAbsent(ctx context.Context, key string, r io.Reader) error { + full, err := s.fullKey(key) + if err != nil { + return err + } + if _, err := s.client.PutObject(ctx, &s3.PutObjectInput{Bucket: &s.bucket, Key: &full, Body: r, IfNoneMatch: aws.String("*")}); err != nil { + return fmt.Errorf("storage: put %q: %w", key, err) + } + return nil +} + func (s *S3) Delete(ctx context.Context, key string) error { full, err := s.fullKey(key) if err != nil { diff --git a/spindle/storage/storage.go b/spindle/storage/storage.go index dcf8b9f2..37688e2c 100644 --- a/spindle/storage/storage.go +++ b/spindle/storage/storage.go @@ -17,6 +17,17 @@ type Storage interface { Delete(ctx context.Context, key string) error } +// StatStorage provides the exact stored object size without downloading it. +type StatStorage interface { + Stat(ctx context.Context, key string) (int64, error) +} + +// ConditionalStorage writes only when key does not already exist. +type ConditionalStorage interface { + Storage + PutIfAbsent(ctx context.Context, key string, r io.Reader) error +} + var keyRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:/@%+=-]{0,511}$`) func ValidateKey(key string) error { diff --git a/spindle/storage/storage_test.go b/spindle/storage/storage_test.go index 3a6b2036..362dcd80 100644 --- a/spindle/storage/storage_test.go +++ b/spindle/storage/storage_test.go @@ -45,6 +45,7 @@ func (m *mockS3Client) DeleteObject(_ context.Context, params *s3.DeleteObjectIn delete(m.objects, *params.Bucket+"/"+*params.Key) return &s3.DeleteObjectOutput{}, nil } + func TestValidateKey(t *testing.T) { valid := []string{ "did:plc:xyz123/go-mod-v1", diff --git a/spindle/storage/unversioned_s3.go b/spindle/storage/unversioned_s3.go new file mode 100644 index 00000000..7263a918 --- /dev/null +++ b/spindle/storage/unversioned_s3.go @@ -0,0 +1,30 @@ +package storage + +import ( + "context" + "fmt" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +func NewUnversionedS3(ctx context.Context, bucket, prefix string) (*S3, error) { + if bucket == "" { + return nil, fmt.Errorf("storage: s3 backend requires a bucket") + } + sdkConfig, err := awsconfig.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("storage: load s3 config: %w", err) + } + client := s3.NewFromConfig(sdkConfig) + versioning, err := client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{ + Bucket: &bucket, + }) + if err != nil { + return nil, fmt.Errorf("storage: check s3 bucket versioning: %w", err) + } + if versioning.Status != "" { + return nil, fmt.Errorf("storage: s3 cache bucket must not use versioning") + } + return NewS3(client, bucket, prefix) +}