diff --git a/.gitattributes b/.gitattributes index b359f8668..9ad90d864 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,4 +2,5 @@ api/tangled/** linguist-generated -diff api/tangled/*_ext.go -linguist-generated diff flake.lock -diff web/src/lib/api/lexicons/** linguist-generated -diff +spindle/mill/proto/gen/** linguist-generated -diff Cargo.lock -diff diff --git a/shuttle/src/gen/file_descriptor_set.bin b/shuttle/src/gen/file_descriptor_set.bin index 6d4b1ec4f8947843596476ee0fe454fd4823d932..837b4f756e4d71aa625a18e873863ff3b6171fae 100644 GIT binary patch delta 9044 zcmbQRUHHv5;f5B*7N!>FEi9>18C$leO=W3hW_&mO#SE66RvKK~1(^k@IhlE>@tG+S ztO|@8oKEb!JXngD1Q>(3U~--*Lh@YPB_%oW#i_~pc`3ybY}0SfWKm#bpZ*I(aZHz< z#bUzd#lpcUz_s0f77L3g*Z*)Xb|F>j zMpBMYc~({-W@EUqV2&hAicyH!7;Y>pvrs*=G2C1>Rv~5+xIQ)z2d0mWQHa?Du8)mb zh}i_Ljh$79*%Ypg9mIjTl$}wC*_2ZlYB@Wz5VNV6I2RWeGYbo+5VIK*FP9_O@0<)m z%x2s?5Otgkj3_)NBpwq7s}QpV69-oqILJ7Jm@TGz?O+M1=U^0Kw%`CKgs97Dom@u$x#|8HJb~IYdFKSXh}DScRCKpn=WG!~o(raPe@}vx4Lpg_xb7 zVZh47z%0bK}=Z7g|7Gic5faWure`o2r;|L`$2ud%+by%z$C=%s^D7> z)y%>s#OwxkISUIb6N?aw8ObrVkix9Jih(bN8Agd6whmt)uZzd0z6r&KcH#{g=nT43W;X%pDA;j!mFYOL> zC@UyxL7HXUpqfG1+6Qhfn-H@PlP64yQHa@xQyj))7Gn01f<+%2hY+)mj0ea zvhGmTplt68OEG#(>_W`GOjWjc2yXUS%}$JLz7F5i;0Czh$WCAWV-7f7Pb07 z4nD97HgKvAf;y26l&XW6AX%9WoC$+CAw>flI8g`jgEJ@#8xtte2E(;63NZ&WK@uh# zIMD`k!pvu67Ge$-6awchb|IE9hImFUu6izJ7El2d#sNv`>|kevL(O9cIU}42QUJ1p zoe>T#0NKIL2!|Gc>|kd^z_l?7F-I^#3P5(SGa{e`AUhMt84+TT;AUs42W6f}xIQK! z=1918pu7_aw+@tdBH`9CvkEas!L>1iIFQ`J&cw_p#2m#5$@A2&Kno;xCQ!1Cffh*YOspLBLd-GH5(&hG)M7Ev5{aD& zlx$<+=CTPf$1;J-TNZXEP;C^;3G)~mvk-Hv47j{yVP^s*%Qz->aLK|BN|kYtLK`H- zD8wAc0Sha3L1rQ5I9WNa04^p__D^ESoc?_uOF(@R2P6@3fYU=V)CnA*^pFho8V5K% zBtyN%0ZtFeP_JnYmCqsP(;z4|t4D}fY6C;Zda|#m|*ohpVm`;KEjDv}hO{ktZ zg&UIiIl##$MF`?i4sh~GfqIk!T=b^G9mgcZoC-}n984fHQ=zGc16=Z^LQ@Y16Dakh z!L>09F{eQjD+f6Bq;b~6d=E-KY0%Wl!30V^>Co)X0cxwHGiicT00%hrq_aZ`Rt|9L zNmti^%CWKtF=xR2$_nCW!=%`Rm@~K`xq$s zs9hi)#I8bdJ1%uDCKhfXmQsd_=`RnmsMnW5D|v3PRb^1CxItExL9OBjTU7?NiW_WI z8PqCnuvKMHt3W)6Rb|k0#|kf| z_2PD1++3h^z*@)9$OyKO71Z{qV+FS%SQ){l)-x%9G7g9*#9Yq=u46zPCL!i}Wff3$ z&dLb#4od?g6R6e20^$mRc#1GdCJHHd`?#FYcJVVPKXh1j|o8746@a&dDpv+yzs)w6Una)3R~!pp?KCdAUi$Ry0g z$HfBT3bFMtGW9dEf=mMO89~w_QZOzvh$}0{#m~hGlH~v?m+xm};S%EF01LEp3V_5E z`WczI1i3h2VoX9Ty^JhkTtZx2EcLwLW~l%RgAhv}BMUFoY(`cgmI;hZ99%qLmoS2P z&M-+Zmy-|1Wd_L#3cAg-_z)J8@~6NY61qlhC^1rw_f%S1*dc97xqEWAuiV6GTU zmJ!6|5QK4=L0oV$W@6zJVw=LqFdZ6ve2hXYQ(&RT2g(>MQ(-(u3t2-A2Zsm~NJ!X3(*sLAKPWvc zU}S=%2Yyhs%CdkFnjZK;^(xB(MiE#LF|!G=EM#Qj1*IhxekNuxPaYZ~pgg>gkq?q# z_(3f*mW7P6;7rWI52{;P79sq^A_U@j)I;@x^7JA`PM9lLm_gE#Frz?Cmqm=y?%*ip zhqTXFK`Lb2p!!)^g;*9NOo#A1VUmm>NlsYIu`+|WQqmyVdKP{rRt^wf#sg|Pv~$6- zm{Hapq=bbZ)J$Vp!pOu1^){Ojh^GmYWCU^9!NoF2Ei;I#4r&{KDtNZ#j0|fbIfxY$ zK+73f!JcGg1}CWc6^u-ZAY(y%A(j=4OllxwK|Cf9S49=dWn>g$S;@!*YS^)`GJ}%D zN=7Dd$b)!HAW2A2voeF@a22d(UDlcz{s>2n$rZp0keS-TJj2jYF3sFpo9b} zc?G~ZZ3CnHW@t_W34n9j21bR=(3}PpV-jN72+L_uAyC1)36{|Wm>BC>g;=)0(v1Kp zShp}TS%JLEA^^^5TNpXHVO(aAT0VXlmjlEVummR?0VY^EvxQO60;&R3@@|FYGyzaN zY=sqK0^ms6%2>|LrK}l^JBPXOx6aeSNZJ=zd$;HGXD8#mlkzp?*qm;D5^c|;JlTu7O%;_8HHF5fkF#h%?W|i&mmZF34u%M zL$Kfy0+-Z>V8I3ALW1iMD7Zi^5+M!-A+{roOeYzcA+~WaFbc68fkl-N2LqE3%TY!a z1Mqkbh}+J|2oZslszOXmY(gx@7+F9qUKSQ1Py#&0$O3N5fq0A{Nda&R48&yyaUtab zix3m2q(9Ec0_uc-v@#2^9A{(!7e*jSMi5sBT!w(SU|C4RltqY%h0RWgoKemCkhVsCoo5gf+FP!BNN1tqF^^aVdR7qS)yP!KVcMt zH0wpdiT)`g6KGhBg+&yU=$}GsKM;>mh~+6GzYG^Is0%I%E_I)Rk{`s$pj7`1*1i!1 zwQrt*sz#_JqY%q8So=m4T&6w)RZ9?A(5T;YMkYvJ69v_c&l#D(6(@@*X!L;PIimoi zEhGvWK47VT&Zr2kI$1=SSlNVFULdSx1@pj_J4lid#DzAiL_uu|mKO+X+1P|wUNSOq zgHx|4D8s)5wGtq1Vq*kJ@<3W|qD*YeAg+WYIOEidGO@D>vAjZr6g!v)$)}=B?2I5T zD5SVpSwxxGnL%7=@gT~?!6wA=8sS?GFc0Qi4n`0c?pqFK5Et%QPBtNyH;hd6Feh_@ zxsYN=lnGRXy+<~xfnrQUJ0=G zM47mlL0oBYhm%E=iJMJ`5uz@v;fAd_;sXFPH~%aXlzhc^N@`ewb0b z%pfk@$$V@=ET0g8%m?N{3SChqPppk<5pN!BRJqx%k^a~^n zQOqR7@(Y@GSXr1rIrtYeuduR!+pE7}!wM{*EczSdB#5n`_Udm&=%@kEMh`z{}~zjm>@$1VvIs8{~0+T1&$cFd~RTZ_RqyY9&cb` z3I?YYF>t6fFmXbHQw&^?H86=m;$IA0J~uE)1c57Q5VxI^Pl%;~NwPi=TuVTOK*Kdm z7LZ;%REQa*%q0M73nQBlOCuAsb1KHfD8$yt#MHtB8LSWk2X`ZrAS5-2fs4dOCg@;= z7^tbi*2pB^!UP$t0133$gGMVFnG{->AfpvvLC{naOA`}xv;rywYBM)8K?f_u!2QQo zL|}jheVCX6q0R(FbSo3I-6;kxUR#-<-A6HSqqdbvEC3W3^(-JhI5b+B#QmXx!2}w= z02^uMfGor;#L~*-^A9F>sPy`4!2(ntkyA%?dz2}8y%#F#)G)eey1^`NmHF;Ewwg9+M}5d#ejvUD&($4$h* zWqk(|Kcw*?2JWbKFbP81G9WI*^bRH=NLxmX2~-<&GBJsOhcLvL*o4?RnV5RvsRUFX zbTWy8iXT=`_=75hP9{Yq@EC>|6B`Fey>c%+nSdt5gh65|z3@~57Gn}(>0)9Lh#4rPz z|HMIKUQE!D8F6s_n+OUtP?(8>^4~;Q{u2l1zlpH?Cl1bk6JhyJ9Gw3q!tx)83r=wp znIys4ubu@e1R4~B% z2|C~-4w~y=nSyW_vk-`956%nX;KFhW6SSoy4$ccxn4m2saVBPtdXQ=nJ8(_|@gXjo z!Xyf5T#JMA!c>ImEJ7?(nV=(4;^4e6l?gf$B@WIDQxOgW4cbg)f{sLq!v}4qGC@Zs z#hF-H>xEdRA-v5B=0RF3;^4e6jfowS_{G6_VHy*(J17nwl9`Th7@H8wbcDk|>3lk@ z6cPuwd#1xmA#qSLXPM3<1}bb7I9Wh^$QaplCUM6kwt}Mkl6>P!-FEi9>18CPsio66G2%=m2jiy16CErhsOvh(swBv=&~H8_L7 zY#|{omZI#`N(nZI7>F$-&&6F*k`rH?nw+1PQY^tf{q{^21xAkPe?S!HblF)fCfmbj zv9O48t&ZSg7h+{#n8`GKK{JcU_R94vLTrpy+k3XM#4*+zGjVVQaj~$lFtG?R8#4(> zxhfQ77Nq86=A|lRrsyb?WTYyjR-`7EmgE;H6oG6jE-5NaE-8k}BqoDs!Om;>gW;3`xc4i@FGq^quRv~6{xIPXL2WBJ(qY$$>r!dq44rU={b1`u!kCRo1 z*^-HaD-7&8P7udY%2S~zwIIJZvn0Q$QlTUxu|#3I-3}J}dQL_mW=l?3z;H4PF z3NhO=NkbJffj9~?~#v4g*#u251;Ki`zlXWn>j%c41-z zIgEuBY)10WJ=ZMvit)VIdZGEpKM1T`V9GCLv}I7C|l{E^e3_7CsgsW=|1? zdQ?GHA!bh{ePl6K7FHoZRSG4rK*JEl9JB8&oqWxBJ4)WfNldW%7hcF$yvJazZj3D-#>D5VNlo zEc)0ugqVG0JfOz1L5=m5b%*K&l?HyW6r;z)F2wA|qy>{>6k_&cSB3GIg_!*`G`ZBc zm{{0^Sb`Wrr>pE?QL7K)-~+2*1E=a>s1w;hsXCYml9k!OnJ}1>2d0o&h&h-aoIzRG zm_Ug(1g?!yh&hA_k}%o8i8h22W^g_xt^+L%EcNN!2q zd3Gjdkemd}2cQHP1J}kP#9SZ4WDgEjb|w}^A?6rPNU*Xqu`mlU#|Xm$g@r?iIY!hD zoDSHTK=}Yvdd7&^LiK`@Z7keeRw3qCCQymQ%EHbBD!O8!1rj?GDA~qB3nX?XR*rfh z=2&Qn1mZ$U2 z0L64F)Mp$_jBG;n%&FXv#LodvKB+fvAlnVAMn zJsjYYHw~J4IG8}GCmpVhS%^6unpio&sVAMY9_D*c@=1rLRt_dm^2vZ^cMecXCxb~7 zoB}w&sV9RSQm}G>Q%{Dv22_rfMTj{Q?pIb2M;j)^Cd8b{4Jmv$z-c8@LJiWM-~cC< zOie9tj^z|$$zjNY)P|sXK8FJmKAhkP%7uD{6BI$Y&)KL-BOYgDh65h0wx*3v5>r)GjWNT}4p4xWIN5LG9uK z+f@X$iwkU55!5aa4`Nr5xE+@|7ZVG&5K9?D<@6H=S=8&xpp`r~*s5}6urq6*&g20* zvj*x+5D((antE|NE^aPRI$*75Xkr9g$O>wE)U%3#YH3zRkf|&Uj7*>w6AOqd1mdZ} zB$+^5Wfdrwkx_`Hk&y}1@B*oY@DyQ^Odv^lkU>nWOhRlej0|1%jEr2|T+E=x4@(Op zt1wt86WGpHMkY}4!oteL#301d%E+VylVk#M6(GhkfkKg`jS-v;L24O=Ks-69T1F-i zS4I}fWnvU!X=eoIJ&;-^p?VM(F3SXxm6m}jW@Z#(=|C9F4CcWMW@Z9$;RdrX3bAxD zGJ&dbkijfqo&wY$7A6o^4%CcgV&N5H>tSS=4DmNhJujmWOAjLl*yAj`Obl#7EWM0O z!d!e@EFi8BTQ4Ki1V&bnNgzHWNLoY+#$^U^W#zc|xmZE693bWL6Bt>zgt$1s0_~gv zATfmrjLckuT%0g5CLxwSMiwzHq3O?#u<+IQGqUhPjbvmMVwuRu#KFY_b^s%o=M0ks zb2<57TxO80pb(790pbcfK`mm0G+tOHGKx4tRWPv%u}orQVuu>e1m=lx)q|RYyr2S< zWfCKYAWShcNE)1gnOOLQ*rqZv%z%a(AEOY`h03y!Q3Mu5%xpp|ix`NlsYIu`+|WQqmyVdKP{rRt^wf#sg|Pv{%8h zgi+QVq=bbZ)IMWb%E-h9&O!W4Y(gNOCQOnM#AOE;#vrxKAg(&7Q2?sh*;X(ztb=4A zR!{(~V1yR&%-{r7zmkzj5o9bYGbjhGWMonU84Kbu39+nXRDl%v%-~pG#mEF|(Xp^H zgObE5Mka8`gLq6rEUTbF&B_dp!_~0*fEg6Qt3kH#a#giA6w& zZ38324v5ED1Q>-_Ho!^_0dPj!2q|vsSy%)>Ic*~&(-vq>695OyMn-6PD*&oeSvG=v z2P$s`z&ULrqx=?VP6G*mbJ|8mg)PvW1{GrxV%Y@CX;2|hM%xU_XaY=(^{hfHTVd%& z02HiS8JVm=-enO0hxt}UPHq^N8KjnvAI9YXaRn^F$wq()R?KW=6tsY<02RF3U^z_y z6c5{AWtadslD09{b3n>*0dP*+#wZHO0Rl`+96~JH7{wul7>L`>2^uQd#wY=E9W$#C z%XWm*nZZ1mc2H8=&d3QV5(U6{aXTm*YjQEM2nw<7W@OmM$S5VPFn!8t7UlZgj2sYO z34%-YJup)RL1}UiBa;KzLxSLl*#k3G5F9akK&FCf5N{h(&dUPcZ`B_Rkd*!D6Ci@@}92(j#C6t#t#4hn95 zCXkSr4VN<)6N``#+d)Q#)6*NzusGEpWaNNU4nklL9fEmC2;`wdj7*Ri5(0bZ5UeZ} z0(n2(cYyWIDyj46%)afl-L%C@iXkI2f3O zSdKBW7=Q1zZ?`BpE?mA#fQ2;(}!%%~2L1CKfh3A(oQ} zYgxcNn6;oXF|h~>v7KXNxH!G;9E)cCIYwyS5C+HLc~~$DgW~W! zBaZ_q+qlu?M~CM5MSvxu?_vE5>1dc?@WCCJ6jBFf&*$-)2^0)>VsxR`$e%k82d zUp--Df)sb6ObqpmLM%^U*;EwlttYTHoG3WhpTZn13W}7cj7$(mih|wzl#vrsWQl^^ z{FG4$(wG+oC;DfMOrY^F78X%ZqJIXh{XjfMA(m&1{4!j;ps{>WaH;zYl>8u02BrGv zu=b58sD1MsR5d~+8HHG$!`e5Z;4<|&s9J)^f`Bt~gmlLBj?t zFBkeH4A_~gzuRyH?h@03LL6SU>)|)73(39mAql6?lkj36#3q_|jFM48x`L0o9@Aj-tSCdBdv;ad(c59V7AMi3Y7TMlLr7w%b3HX)X` zj7;?~Cv$?ikYY%b2~>o=W#oa>W1>u)%ph4wa9fQnYfriTxoDOlSPz?n@x!2JtGrngdC)myB^Gir6F!c5SNV;s+XG?#N~l> zxkQG1RvI((#LWD6dmBl!=cS#Fdo?_3K$gIfYoh z!t)C!sNVR>$PH<=i*ho6B)RLM`GpfyjeKPk1Pz5Tv9buU{a|GH2g&lRptAZ0Y=-yw)=~41d9;MUyvtB!~c(|BY#Dv%ym>Bw*zyl*JVvIs84NM%60!Iv7 zJ~uK!yXRsck2f+g1%uOy7&ufKnK&WADF!ac8kxi(@h=80pBtGZg20tDh}+J|C&bdo zBv~H_t|g#Cpg|fY3rHs(D#Q#@<`Mw4g^^8&rHKjJITd4K6k=;)VrpfA3{i-IgS&}I z5Rw|iz(ryc6Lg3|4Aj(MYhsdbWr7S*fCSp>L1Pq6ObV?`kTD9dAZU(>rI`sdM#06! z0u=(anOm5kV-;fH{$m>=FhD~+OiY1LXM!TSjS1TB6ayEpZA{SaqZqhR+r}go01AwH z77!mC8f{GC{?Nc+0u5h)jWlyW7Gf4+X=8HogW3XW`nDsSz%0bl&cx&alLU=6F+qEK zV&G!Boe9$0V-W*QtFg2*$+&aXg9c_m{B}-I-f3r&h4dxGz&+Itgy}3oEFDZtkS44c zI7xLdL5Cj1z$JMH6Ljc73_Q5r!6XD}q=L8*!#kLSAww5pOrVZxC&=)6&`^&UsEg3a z1Z~TRfkp&bI+>uuCSu^SzLN<$Y$68ksCF_5LfSGQF2wXsCLu^$MvMtm8+0)-iGYVN z#F*HG*t(dQ`rxSqR3CIPiGs5sDE!%&g;=_n6qUeZ7-CFp93b_|eeh%gnh6sIiK+C# zQwdm%Nrn5Jnjd88t!EOIMim7O zt1*G62$@*Kh1e!CG0cQ!M{&>y7!!1$MjV_SCxOBc6olfS>^KRQ9mT=faS|*$ii5M` zBv^J72WQ7g;Otn>#lZsNgHzxnCP_$kgo=R1$Y9wKDg+uAV}fQ!ad4w*GQtUrLM)SE zDNr1o87IS1pg1@)PKKpGaZqMtnam_p4;jq>@gdPT8I}UY!C7ny6Lh>m98>^KVPb+5 zFyi3CataeWq;3)iXR#?v(D5H}(3}U$RD{Erg+M%eaMln9mz`6YAPuK_7IARin92lg zIf;Yw##AN|J8;GWaUl+y$|MSDYm0-k#x#WKEJ7^Pn4kkw;^3??jR`t1B@WIS(+~~= zjp0mVf(}fHGqKczayl!>0O&xaI5=-iM|hi6h-Eqx6QuDX4$d0Wnb;u>adB|gn9c<4 z9g2fTXl5WB#wNrv1K}`GvY!Dfl*GZ!pczai&{9bpl, - #[prost(uint32, tag = "3")] + #[prost(uint32, tag="3")] pub cache_read_proxy_port: u32, - #[prost(uint32, tag = "4")] + #[prost(uint32, tag="4")] pub cache_upload_proxy_port: u32, - #[prost(uint32, tag = "5")] + #[prost(uint32, tag="5")] pub dns_proxy_port: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExecStart { - #[prost(string, repeated, tag = "1")] + #[prost(string, repeated, tag="1")] pub argv: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - #[prost(string, repeated, tag = "2")] + #[prost(string, repeated, tag="2")] pub env: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - #[prost(string, tag = "3")] + #[prost(string, tag="3")] pub cwd: ::prost::alloc::string::String, - #[prost(string, tag = "4")] + #[prost(string, tag="4")] pub user: ::prost::alloc::string::String, - #[prost(uint32, tag = "5")] + #[prost(uint32, tag="5")] pub timeout_seconds: u32, - #[prost(uint32, tag = "6")] + #[prost(uint32, tag="6")] pub stdio_vsock_port: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExecStderr { - #[prost(bytes = "bytes", tag = "1")] + #[prost(bytes="bytes", tag="1")] pub data: ::prost::bytes::Bytes, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ExecExit { - #[prost(int32, tag = "1")] + #[prost(int32, tag="1")] pub exit_code: i32, - #[prost(string, tag = "2")] + #[prost(string, tag="2")] pub error: ::prost::alloc::string::String, /// set when the guest killed the step on its own timeout timer, so the host /// can classify it as a timeout rather than inferring failure from exit_code. - #[prost(bool, tag = "3")] + #[prost(bool, tag="3")] pub timed_out: bool, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ActivateConfig { - #[prost(string, tag = "1")] + #[prost(string, tag="1")] pub config_key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] + #[prost(string, tag="2")] pub base_config_hash: ::prost::alloc::string::String, - #[prost(string, tag = "3")] + #[prost(string, tag="3")] pub user_config: ::prost::alloc::string::String, - #[prost(string, tag = "4")] + #[prost(string, tag="4")] pub toplevel: ::prost::alloc::string::String, - #[prost(uint32, tag = "5")] + #[prost(uint32, tag="5")] pub timeout_seconds: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ActivateConfigResult { - #[prost(string, tag = "1")] + #[prost(string, tag="1")] pub config_key: ::prost::alloc::string::String, - #[prost(string, tag = "2")] + #[prost(string, tag="2")] pub toplevel: ::prost::alloc::string::String, - #[prost(string, tag = "3")] + #[prost(string, tag="3")] pub error: ::prost::alloc::string::String, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct BuiltPaths { - #[prost(string, repeated, tag = "1")] + #[prost(string, repeated, tag="1")] pub paths: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, - #[prost(string, tag = "2")] + #[prost(string, tag="2")] pub reason: ::prost::alloc::string::String, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct CacheDrain { - #[prost(uint32, tag = "1")] + #[prost(uint32, tag="1")] pub timeout_seconds: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct CacheDrainResult { - #[prost(string, tag = "1")] + #[prost(string, tag="1")] pub error: ::prost::alloc::string::String, - #[prost(uint32, tag = "2")] + #[prost(uint32, tag="2")] pub cache_queued: u32, - #[prost(uint32, tag = "3")] + #[prost(uint32, tag="3")] pub cache_active: u32, - #[prost(uint32, tag = "4")] + #[prost(uint32, tag="4")] pub cache_uploaded: u32, - #[prost(uint32, tag = "5")] + #[prost(uint32, tag="5")] pub cache_failed: u32, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] -pub struct Poweroff {} +pub struct Poweroff { +} #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PoweroffResult { - #[prost(string, tag = "1")] + #[prost(string, tag="1")] pub error: ::prost::alloc::string::String, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct OpenDebugShell { - #[prost(uint32, tag = "1")] + #[prost(uint32, tag="1")] pub vsock_port: u32, - #[prost(string, tag = "2")] + #[prost(string, tag="2")] pub term: ::prost::alloc::string::String, - #[prost(uint32, tag = "3")] + #[prost(uint32, tag="3")] pub rows: u32, - #[prost(uint32, tag = "4")] + #[prost(uint32, tag="4")] pub cols: u32, /// the exact process context used to launch the failed workflow step. the /// guest ignores its argv and timeout when constructing the debug shell. - #[prost(message, optional, tag = "5")] + #[prost(message, optional, tag="5")] pub failed_step: ::core::option::Option, /// the same dependency-environment prelude prepended to the failed step. - #[prost(string, tag = "6")] + #[prost(string, tag="6")] pub shell_prelude: ::prost::alloc::string::String, } /// changes meaning based on who sends this: /// guest->host is shell output, host->guest is keyboard input #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct PtyData { - #[prost(bytes = "bytes", tag = "1")] + #[prost(bytes="bytes", tag="1")] pub data: ::prost::bytes::Bytes, } #[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] pub struct PtyResize { - #[prost(uint32, tag = "1")] + #[prost(uint32, tag="1")] pub rows: u32, - #[prost(uint32, tag = "2")] + #[prost(uint32, tag="2")] pub cols: u32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct Message { - #[prost(string, tag = "1")] + #[prost(string, tag="1")] pub id: ::prost::alloc::string::String, - #[prost(message, optional, tag = "2")] + #[prost(message, optional, tag="2")] pub hello: ::core::option::Option, - #[prost(message, optional, tag = "3")] + #[prost(message, optional, tag="3")] pub init: ::core::option::Option, - #[prost(message, optional, tag = "4")] + #[prost(message, optional, tag="4")] pub exec_start: ::core::option::Option, - #[prost(message, optional, tag = "6")] + #[prost(message, optional, tag="6")] pub exec_stderr: ::core::option::Option, - #[prost(message, optional, tag = "7")] + #[prost(message, optional, tag="7")] pub exec_exit: ::core::option::Option, - #[prost(message, optional, tag = "8")] + #[prost(message, optional, tag="8")] pub activate_config: ::core::option::Option, - #[prost(message, optional, tag = "9")] + #[prost(message, optional, tag="9")] pub activate_config_result: ::core::option::Option, - #[prost(message, optional, tag = "10")] + #[prost(message, optional, tag="10")] pub built_paths: ::core::option::Option, - #[prost(message, optional, tag = "11")] + #[prost(message, optional, tag="11")] pub cache_drain: ::core::option::Option, - #[prost(message, optional, tag = "12")] + #[prost(message, optional, tag="12")] pub cache_drain_result: ::core::option::Option, - #[prost(message, optional, tag = "13")] + #[prost(message, optional, tag="13")] pub poweroff: ::core::option::Option, - #[prost(message, optional, tag = "14")] + #[prost(message, optional, tag="14")] pub poweroff_result: ::core::option::Option, - #[prost(message, optional, tag = "15")] + #[prost(message, optional, tag="15")] pub open_debug_shell: ::core::option::Option, - #[prost(message, optional, tag = "16")] + #[prost(message, optional, tag="16")] pub pty_data: ::core::option::Option, - #[prost(message, optional, tag = "17")] + #[prost(message, optional, tag="17")] pub pty_resize: ::core::option::Option, } // @@protoc_insertion_point(module) diff --git a/spindle/db/artifacts.go b/spindle/db/artifacts.go index 389aade68..d217edc29 100644 --- a/spindle/db/artifacts.go +++ b/spindle/db/artifacts.go @@ -21,9 +21,9 @@ func (d *DB) GetFinishedLog(wid models.WorkflowId) (*FinishedLog, error) { err := d.QueryRow( `select lease_id, workflow, ref, hash from mill_artifacts - where knot = ? and rkey = ? and workflow = ? + where pipeline_id = ? and workflow = ? order by id desc limit 1`, - wid.Knot, wid.Rkey, wid.Name, + string(wid.PipelineId), wid.Name, ).Scan(&fl.LeaseID, &fl.Workflow, &fl.Ref, &fl.Hash) if err != nil { return nil, err @@ -36,15 +36,15 @@ func (d *DB) SaveArtifactRef(leaseID, repoDid string, wid models.WorkflowId, ref return err } _, err := d.Exec( - `insert into mill_artifacts (lease_id, repo_did, knot, rkey, workflow, ref, hash) - values (?, ?, ?, ?, ?, ?, ?)`, - leaseID, repoDid, wid.Knot, wid.Rkey, wid.Name, ref, hash, + `insert into mill_artifacts (lease_id, repo_did, pipeline_id, workflow, ref, hash) + values (?, ?, ?, ?, ?, ?)`, + leaseID, repoDid, string(wid.PipelineId), wid.Name, ref, hash, ) return err } func validateWorkflowIdentity(wid models.WorkflowId) error { - if wid.Knot == "" || wid.Rkey == "" || wid.Name == "" { + if wid.PipelineId == "" || wid.Name == "" { return errors.New("incomplete workflow identity") } return nil diff --git a/spindle/db/artifacts_test.go b/spindle/db/artifacts_test.go index 75d2bd4d5..c10c681a8 100644 --- a/spindle/db/artifacts_test.go +++ b/spindle/db/artifacts_test.go @@ -46,7 +46,7 @@ func TestMakeAcceptsPrebackfilledArtifactIdentity(t *testing.T) { } t.Cleanup(func() { d.Close() }) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot", Rkey: "rkey"}, Name: "build.yml"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey"), Name: "build.yml"} log, err := d.GetFinishedLog(wid) if err != nil { t.Fatalf("GetFinishedLog(legacy): %v", err) @@ -55,7 +55,7 @@ func TestMakeAcceptsPrebackfilledArtifactIdentity(t *testing.T) { t.Fatalf("GetFinishedLog(legacy) = %+v", log) } - newWID := models.WorkflowId{PipelineId: models.PipelineId{Knot: "new-knot", Rkey: "new-rkey"}, Name: "build.yml"} + newWID := models.WorkflowId{PipelineId: models.PipelineId("new-rkey"), Name: "build.yml"} if err := d.SaveArtifactRef("new-lease", "did:plc:repo", newWID, "new-ref", "new-hash"); err != nil { t.Fatalf("SaveArtifactRef: %v", err) } @@ -121,9 +121,8 @@ func TestMakeMigratesLegacyMillArtifactsRepoDID(t *testing.T) { func TestGetFinishedLogMatchesPipelineAndWorkflow(t *testing.T) { d := newTestDB(t) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "pipeline-1"}, Name: "build.yml"} - otherPipeline := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "pipeline-2"}, Name: wid.Name} - otherKnot := models.WorkflowId{PipelineId: models.PipelineId{Knot: "other.example.com", Rkey: wid.Rkey}, Name: wid.Name} + wid := models.WorkflowId{PipelineId: models.PipelineId("pipeline-1"), Name: "build.yml"} + otherPipeline := models.WorkflowId{PipelineId: models.PipelineId("pipeline-2"), Name: wid.Name} for _, artifact := range []struct { leaseID string @@ -132,7 +131,6 @@ func TestGetFinishedLogMatchesPipelineAndWorkflow(t *testing.T) { }{ {"wanted-old", wid, "wanted-old.log"}, {"other-pipeline", otherPipeline, "other-pipeline.log"}, - {"other-knot", otherKnot, "other-knot.log"}, {"wanted-new", wid, "wanted-new.log"}, } { if err := d.SaveArtifactRef(artifact.leaseID, "did:plc:repo", artifact.wid, artifact.ref, "hash"); err != nil { @@ -152,9 +150,8 @@ func TestGetFinishedLogMatchesPipelineAndWorkflow(t *testing.T) { func TestFinishedLogRejectsIncompleteWorkflowIdentity(t *testing.T) { d := newTestDB(t) for _, wid := range []models.WorkflowId{ - {PipelineId: models.PipelineId{Rkey: "rkey"}, Name: "build.yml"}, - {PipelineId: models.PipelineId{Knot: "knot"}, Name: "build.yml"}, - {PipelineId: models.PipelineId{Knot: "knot", Rkey: "rkey"}}, + {PipelineId: models.PipelineId(""), Name: "build.yml"}, + {PipelineId: models.PipelineId("rkey")}, } { if _, err := d.GetFinishedLog(wid); err == nil { t.Fatalf("GetFinishedLog(%+v) succeeded", wid) diff --git a/spindle/db/cache_test.go b/spindle/db/cache_test.go index f451a00b9..67b28e09b 100644 --- a/spindle/db/cache_test.go +++ b/spindle/db/cache_test.go @@ -326,14 +326,13 @@ func TestMillCacheCapabilitiesAreLeaseScopedAndConsumable(t *testing.T) { ctx := context.Background() d := newTestDB(t) lease := MillLease{ - LeaseID: "lease-1", - NodeID: "node-1", - Epoch: "epoch-1", - Engine: "microvm", - Knot: "k", - Rkey: "r", - Workflow: "build", - State: "running", + LeaseID: "lease-1", + NodeID: "node-1", + Epoch: "epoch-1", + Engine: "microvm", + PipelineID: "r", + Workflow: "build", + State: "running", } if err := d.SaveMillLease(lease); err != nil { t.Fatal(err) diff --git a/spindle/db/db.go b/spindle/db/db.go index 7a28689c0..1c84da1f6 100644 --- a/spindle/db/db.go +++ b/spindle/db/db.go @@ -13,6 +13,7 @@ import ( "tangled.org/core/api/tangled" "tangled.org/core/log" "tangled.org/core/orm" + "tangled.org/core/spindle/models" "tangled.org/core/sqlite" ) @@ -972,6 +973,98 @@ func runMigrations(_ context.Context, conn *sql.Conn, logger *slog.Logger) error return err } + if err := orm.RunMigration(conn, logger, "pipeline-id-identity", func(tx *sql.Tx) error { + if _, err := tx.Exec(` + create table pipeline_log_renames ( + knot text not null, + pipeline_id text not null, + workflow text not null, + primary key (knot, pipeline_id, workflow) + ); + `); err != nil { + return err + } + if err := stagePipelineLogRenames(tx); err != nil { + return err + } + if err := migratePipelineIdentityColumns(tx); err != nil { + return err + } + if _, err := tx.Exec(` + alter table jobs drop column pipeline_id_knot; + alter table jobs rename column pipeline_id_rkey to pipeline_id; + + alter table mill_leases drop column knot; + alter table mill_leases rename column rkey to pipeline_id; + + drop index if exists idx_mill_artifacts_workflow_identity; + alter table mill_artifacts drop column knot; + alter table mill_artifacts rename column rkey to pipeline_id; + create index idx_mill_artifacts_workflow_identity + on mill_artifacts (pipeline_id, workflow, id desc); + + alter table executor_pending_artifacts drop column knot; + alter table executor_pending_artifacts rename column rkey to pipeline_id; + `); err != nil { + return err + } + return nil + }); err != nil { + return err + } + + if err := orm.RunMigration(conn, logger, "drop-legacy-pipeline-events", func(tx *sql.Tx) error { + _, err := tx.Exec(` + drop table if exists events; + drop table if exists events_legacy; + `) + return err + }); err != nil { + return err + } + + return nil +} + +func tableHasColumn(tx *sql.Tx, table, column string) (bool, error) { + var count int + if err := tx.QueryRow( + `select count(*) from pragma_table_info(?) where name = ?`, + table, column, + ).Scan(&count); err != nil { + return false, err + } + return count != 0, nil +} + +func migratePipelineIdentityColumns(tx *sql.Tx) error { + hasRkey, err := tableHasColumn(tx, "pipelines", "rkey") + if err != nil { + return err + } + if hasRkey { + if _, err := tx.Exec(` + alter table pipelines drop column knot; + alter table pipelines rename column rkey to pipeline_id; + `); err != nil { + return err + } + } + + hasStatusRkey, err := tableHasColumn(tx, "workflow_statuses", "rkey") + if err != nil { + return err + } + if hasStatusRkey { + if _, err := tx.Exec(` + drop index if exists idx_workflow_statuses_lookup; + alter table workflow_statuses rename column rkey to pipeline_id; + create index idx_workflow_statuses_lookup + on workflow_statuses (pipeline_id, workflow, id); + `); err != nil { + return err + } + } return nil } @@ -1083,7 +1176,7 @@ func migratePipelines(tx *sql.Tx, logger *slog.Logger) error { continue } - p, kind := mapToCiPipeline(rkey, time.Unix(0, created), raw) + p, kind := mapToCiPipeline(models.PipelineId(rkey), time.Unix(0, created), raw) payload, err := json.Marshal(p) if err != nil { skipped++ diff --git a/spindle/db/delete_test.go b/spindle/db/delete_test.go index dfff5cb61..554c73bd0 100644 --- a/spindle/db/delete_test.go +++ b/spindle/db/delete_test.go @@ -48,7 +48,7 @@ func TestDeleteReposByDid(t *testing.T) { func TestDeleteJobsByRepo(t *testing.T) { d := newTestDB(t) ctx := context.Background() - pid := models.PipelineId{Knot: "k1", Rkey: "p1"} + pid := models.PipelineId("p1") if err := d.EnqueueJob(ctx, "did:plc:repo1", pid, nil, tangled.Pipeline{}, "", ""); err != nil { t.Fatal(err) } @@ -74,14 +74,14 @@ func TestDeleteJobsByRepo(t *testing.T) { func TestDeleteMillLeasesByRepo(t *testing.T) { d := newTestDB(t) for _, l := range []MillLease{ - {LeaseID: "l1", NodeID: "n1", Epoch: "e1", Engine: "microvm", Knot: "k1", Rkey: "r1", Workflow: "w1", State: "active", RepoDID: "did:plc:repo1"}, - {LeaseID: "l2", NodeID: "n1", Epoch: "e1", Engine: "microvm", Knot: "k1", Rkey: "r1", Workflow: "w2", State: "active", RepoDID: "did:plc:repo1"}, - {LeaseID: "l3", NodeID: "n1", Epoch: "e1", Engine: "microvm", Knot: "k1", Rkey: "r9", Workflow: "w1", State: "active", RepoDID: "did:plc:repo2"}, + {LeaseID: "l1", NodeID: "n1", Epoch: "e1", Engine: "microvm", PipelineID: "r1", Workflow: "w1", State: "active", RepoDID: "did:plc:repo1"}, + {LeaseID: "l2", NodeID: "n1", Epoch: "e1", Engine: "microvm", PipelineID: "r1", Workflow: "w2", State: "active", RepoDID: "did:plc:repo1"}, + {LeaseID: "l3", NodeID: "n1", Epoch: "e1", Engine: "microvm", PipelineID: "r9", Workflow: "w1", State: "active", RepoDID: "did:plc:repo2"}, } { if err := d.SaveMillLease(l); err != nil { t.Fatal(err) } - if _, err := d.Exec(`insert into mill_artifacts (lease_id, repo_did, knot, rkey, workflow, ref, hash) values (?, ?, ?, ?, ?, ?, ?)`, l.LeaseID, l.RepoDID, l.Knot, l.Rkey, l.Workflow, "logs/"+l.LeaseID+".log", "h"); err != nil { + if _, err := d.Exec(`insert into mill_artifacts (lease_id, repo_did, pipeline_id, workflow, ref, hash) values (?, ?, ?, ?, ?, ?)`, l.LeaseID, l.RepoDID, l.PipelineID, l.Workflow, "logs/"+l.LeaseID+".log", "h"); err != nil { t.Fatal(err) } if _, err := d.Exec(`insert into executor_pending_artifacts (lease_id, workflow, status, ref, hash) values (?, ?, 'done', 'r', 'h')`, l.LeaseID, l.Workflow); err != nil { @@ -163,41 +163,41 @@ func TestDeleteQuotaStateForRepo(t *testing.T) { } } -func TestDeleteEventsByRepo(t *testing.T) { +func TestDeletePipelinesByRepo(t *testing.T) { d := newTestDB(t) repo := "did:plc:repo1" other := "did:plc:repo2" - insertPipeline := func(rkey, repoDid, knot string) { + insertPipeline := func(rkey, repoDid string) { p := &tangled.CiPipeline{Id: rkey, Repo: repoDid} payload, err := json.Marshal(p) if err != nil { t.Fatal(err) } - if _, err := d.Exec(`insert into pipelines (rkey, knot, repo_did, commit_sha, kind, payload) values (?, ?, ?, '', '', ?)`, rkey, knot, repoDid, string(payload)); err != nil { + if _, err := d.Exec(`insert into pipelines (pipeline_id, repo_did, commit_sha, kind, payload) values (?, ?, '', '', ?)`, rkey, repoDid, string(payload)); err != nil { t.Fatal(err) } } - insertPipeline("p1", repo, "k1") - insertPipeline("p2", other, "k1") + insertPipeline("p1", repo) + insertPipeline("p2", other) for _, rkey := range []string{"p1", "p2", "p9"} { - if _, err := d.Exec(`insert into workflow_statuses (rkey, workflow, status, created_at) values (?, 'build', 'pending', 'now')`, rkey); err != nil { + if _, err := d.Exec(`insert into workflow_statuses (pipeline_id, workflow, status, created_at) values (?, 'build', 'pending', 'now')`, rkey); err != nil { t.Fatal(err) } } - if err := d.DeleteEventsByRepo(repo, []PipelineKey{{Knot: "k7", Rkey: "p9"}}); err != nil { + if err := d.DeletePipelinesByRepo(repo, []models.PipelineId{"p9"}); err != nil { t.Fatal(err) } var n int - if err := d.QueryRow(`select count(*) from pipelines where rkey = 'p1'`).Scan(&n); err != nil || n != 0 { + if err := d.QueryRow(`select count(*) from pipelines where pipeline_id = 'p1'`).Scan(&n); err != nil || n != 0 { t.Fatalf("expected repo pipeline gone, got %d err %v", n, err) } - if err := d.QueryRow(`select count(*) from workflow_statuses where rkey in ('p1', 'p9')`).Scan(&n); err != nil || n != 0 { + if err := d.QueryRow(`select count(*) from workflow_statuses where pipeline_id in ('p1', 'p9')`).Scan(&n); err != nil || n != 0 { t.Fatalf("expected wiped statuses gone, got %d err %v", n, err) } - if err := d.QueryRow(`select count(*) from pipelines where rkey = 'p2'`).Scan(&n); err != nil || n != 1 { + if err := d.QueryRow(`select count(*) from pipelines where pipeline_id = 'p2'`).Scan(&n); err != nil || n != 1 { t.Fatalf("expected other pipeline to survive, got %d err %v", n, err) } } diff --git a/spindle/db/events.go b/spindle/db/events.go index cd38c2053..f8860cd44 100644 --- a/spindle/db/events.go +++ b/spindle/db/events.go @@ -24,9 +24,9 @@ type StatusRow struct { func insertStatusTx(tx DBTX, wid models.WorkflowId, kind models.StatusKind, workflowError *string, exitCode *int64) error { _, err := tx.Exec( - `insert into workflow_statuses (rkey, workflow, status, error, exit_code, created_at) + `insert into workflow_statuses (pipeline_id, workflow, status, error, exit_code, created_at) values (?, ?, ?, ?, ?, ?)`, - wid.PipelineId.Rkey, wid.Name, string(kind), workflowError, exitCode, + wid.PipelineId, wid.Name, string(kind), workflowError, exitCode, time.Now().Format(time.RFC3339Nano), ) return err @@ -40,7 +40,7 @@ func (d *DB) EventHighWater() (int64, error) { func (d *DB) GetEvents(cursor int64, limit int) ([]StatusRow, error) { rows, err := d.Query( - `select id, rkey, workflow, status, error, exit_code + `select id, pipeline_id, workflow, status, error, exit_code from workflow_statuses where id > ? order by id asc @@ -96,7 +96,7 @@ func (d *DB) CompleteMillLease( } func (d *DB) GetStatus(workflowId models.WorkflowId) (models.StatusKind, error) { - pipelineId := workflowId.PipelineId.Rkey + pipelineId := workflowId.PipelineId var status string err := d.QueryRow( @@ -104,7 +104,7 @@ func (d *DB) GetStatus(workflowId models.WorkflowId) (models.StatusKind, error) select status from workflow_statuses where - rkey = ? + pipeline_id = ? and workflow = ? order by id desc @@ -135,8 +135,8 @@ func workflowStartupDelay(ctx context.Context, q statusQueryer, wid models.Workf min(case when status = 'pending' then created_at end), min(case when status = 'running' then created_at end) from workflow_statuses - where rkey = ? and workflow = ? - `, wid.PipelineId.Rkey, wid.Name).Scan(&pending, &running) + where pipeline_id = ? and workflow = ? + `, string(wid.PipelineId), wid.Name).Scan(&pending, &running) if err != nil { return 0, false, err } @@ -171,9 +171,9 @@ func (tx *EventBatchTx) HasWorkflowStatus(ctx context.Context, wid models.Workfl err := tx.tx.QueryRowContext(ctx, ` select exists( select 1 from workflow_statuses - where rkey = ? and workflow = ? and status = ? + where pipeline_id = ? and workflow = ? and status = ? ) - `, wid.PipelineId.Rkey, wid.Name, status).Scan(&present) + `, string(wid.PipelineId), wid.Name, status).Scan(&present) return present, err } @@ -202,13 +202,12 @@ func (d *DB) StatusTimeout(workflowId models.WorkflowId, n *notifier.Notifier) e } type PipelineWorkflow struct { - Knot string - Rkey string - Name string + PipelineID models.PipelineId + Name string } func (d *DB) ListPipelineWorkflows(repoDid string) ([]PipelineWorkflow, error) { - rows, err := d.Query(`select rkey, knot, payload from pipelines where repo_did = ?`, repoDid) + rows, err := d.Query(`select pipeline_id, payload from pipelines where repo_did = ?`, repoDid) if err != nil { return nil, err } @@ -216,8 +215,8 @@ func (d *DB) ListPipelineWorkflows(repoDid string) ([]PipelineWorkflow, error) { var out []PipelineWorkflow for rows.Next() { - var rkey, knot, raw string - if err := rows.Scan(&rkey, &knot, &raw); err != nil { + var pipelineID, raw string + if err := rows.Scan(&pipelineID, &raw); err != nil { return nil, err } var p tangled.CiPipeline @@ -226,57 +225,51 @@ func (d *DB) ListPipelineWorkflows(repoDid string) ([]PipelineWorkflow, error) { } for _, wf := range p.Workflows { if wf != nil { - out = append(out, PipelineWorkflow{Knot: knot, Rkey: rkey, Name: wf.Name}) + out = append(out, PipelineWorkflow{PipelineID: models.PipelineId(pipelineID), Name: wf.Name}) } } } return out, rows.Err() } -// PipelineKey is one exact pipeline at-uri identity -type PipelineKey struct { - Knot string - Rkey string -} - -func (d *DB) DeleteEventsByRepo(repoDid string, extra []PipelineKey) error { +func (d *DB) DeletePipelinesByRepo(repoDid string, extra []models.PipelineId) error { tx, err := d.Begin() if err != nil { return err } defer tx.Rollback() - rows, err := tx.Query(`select rkey from pipelines where repo_did = ?`, repoDid) + rows, err := tx.Query(`select pipeline_id from pipelines where repo_did = ?`, repoDid) if err != nil { return err } seen := make(map[string]bool) - var rkeys []string - add := func(rkey string) { - if rkey != "" && !seen[rkey] { - seen[rkey] = true - rkeys = append(rkeys, rkey) + var pipelineIDs []string + add := func(pipelineID string) { + if pipelineID != "" && !seen[pipelineID] { + seen[pipelineID] = true + pipelineIDs = append(pipelineIDs, pipelineID) } } for rows.Next() { - var rkey string - if err := rows.Scan(&rkey); err != nil { + var pipelineID string + if err := rows.Scan(&pipelineID); err != nil { rows.Close() return err } - add(rkey) + add(pipelineID) } if err := rows.Err(); err != nil { rows.Close() return err } rows.Close() - for _, key := range extra { - add(key.Rkey) + for _, id := range extra { + add(string(id)) } - for _, rkey := range rkeys { - if _, err := tx.Exec(`delete from workflow_statuses where rkey = ?`, rkey); err != nil { + for _, pipelineID := range pipelineIDs { + if _, err := tx.Exec(`delete from workflow_statuses where pipeline_id = ?`, pipelineID); err != nil { return err } } diff --git a/spindle/db/events_metrics_test.go b/spindle/db/events_metrics_test.go index e73c0aa92..811c4fe5b 100644 --- a/spindle/db/events_metrics_test.go +++ b/spindle/db/events_metrics_test.go @@ -20,7 +20,7 @@ func TestWorkflowStartupDelay(t *testing.T) { n := notifier.New() wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "pipeline"}, + PipelineId: models.PipelineId("pipeline"), Name: "build", } if err := database.StatusPending(wid, &n); err != nil { @@ -59,7 +59,7 @@ func TestWorkflowStartupDelayUsesStatusIndex(t *testing.T) { min(case when status = 'pending' then created_at end), min(case when status = 'running' then created_at end) from workflow_statuses - where rkey = ? and workflow = ? + where pipeline_id = ? and workflow = ? `, "pipeline", "build") if err != nil { t.Fatal(err) diff --git a/spindle/db/jobs.go b/spindle/db/jobs.go index 9b627dc6e..9d2a91768 100644 --- a/spindle/db/jobs.go +++ b/spindle/db/jobs.go @@ -13,15 +13,14 @@ import ( ) type JobRow struct { - Id int64 - RepoDid string - PipelineIdKnot string - PipelineIdRkey string - SourceRepo *tangled.Pipeline_TriggerRepo - Tpl tangled.Pipeline - Traceparent string - Tracestate string - CreatedAtNs int64 + Id int64 + RepoDid string + PipelineId models.PipelineId + SourceRepo *tangled.Pipeline_TriggerRepo + Tpl tangled.Pipeline + Traceparent string + Tracestate string + CreatedAtNs int64 } // the atomic check rejects a banned job before admission @@ -95,10 +94,10 @@ func (d *DB) enqueueJob( } if _, err = tx.ExecContext(ctx, ` insert into jobs ( - repo_did, pipeline_id_knot, pipeline_id_rkey, source_repo, tpl, + repo_did, pipeline_id, source_repo, tpl, traceparent, tracestate, created_at, created_at_ns - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?) - `, repoDid, pipelineID.Knot, pipelineID.Rkey, sourceRepoValue, string(tplJSON), + ) values (?, ?, ?, ?, ?, ?, ?, ?) + `, repoDid, pipelineID, sourceRepoValue, string(tplJSON), traceparent, tracestate, time.Unix(0, createdAtNS).Unix(), createdAtNS, ); err != nil { return err @@ -138,8 +137,8 @@ func (d *DB) DequeueJob(ctx context.Context) (*JobRow, error) { order by id asc limit 1 ) - returning id, repo_did, pipeline_id_knot, pipeline_id_rkey, source_repo, tpl, traceparent, tracestate, created_at_ns - `).Scan(&row.Id, &row.RepoDid, &row.PipelineIdKnot, &row.PipelineIdRkey, &sourceRepoStr, &tplJson, &row.Traceparent, &row.Tracestate, &row.CreatedAtNs) + returning id, repo_did, pipeline_id, source_repo, tpl, traceparent, tracestate, created_at_ns + `).Scan(&row.Id, &row.RepoDid, &row.PipelineId, &sourceRepoStr, &tplJson, &row.Traceparent, &row.Tracestate, &row.CreatedAtNs) if err != nil { if err == sql.ErrNoRows { return nil, nil diff --git a/spindle/db/jobs_test.go b/spindle/db/jobs_test.go index e05a03276..f908f4ab5 100644 --- a/spindle/db/jobs_test.go +++ b/spindle/db/jobs_test.go @@ -20,7 +20,7 @@ func TestJobTraceContextSurvivesEnqueue(t *testing.T) { } t.Cleanup(func() { database.Close() }) - pipelineID := models.PipelineId{Knot: "knot.example.com", Rkey: "pipeline"} + pipelineID := models.PipelineId("pipeline") const traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01" const tracestate = "vendor=value" if err := database.EnqueueJob( @@ -59,7 +59,7 @@ func TestEnqueueJobWithPendingCommitsJobAndStatusesTogether(t *testing.T) { t.Cleanup(func() { database.Close() }) n := notifier.New() - pipelineID := models.PipelineId{Knot: "knot.example.com", Rkey: "pipeline"} + pipelineID := models.PipelineId("pipeline") tpl := tangled.Pipeline{Workflows: []*tangled.Pipeline_Workflow{{Name: "build"}}} if err := database.EnqueueJobWithPending( ctx, diff --git a/spindle/db/log_migration.go b/spindle/db/log_migration.go new file mode 100644 index 000000000..992749524 --- /dev/null +++ b/spindle/db/log_migration.go @@ -0,0 +1,207 @@ +package db + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + + "tangled.org/core/api/tangled" + "tangled.org/core/spindle/models" +) + +type pipelineLogRename struct { + knot string + pipelineID models.PipelineId + workflow string +} + +func stagePipelineLogRenames(tx *sql.Tx) error { + var renames []pipelineLogRename + hasLegacyPipelineID, err := tableHasColumn(tx, "pipelines", "rkey") + if err != nil { + return err + } + if hasLegacyPipelineID { + rows, err := tx.Query(`select rkey, knot, payload from pipelines`) + if err != nil { + return err + } + for rows.Next() { + var rkey, knot, payload string + if err := rows.Scan(&rkey, &knot, &payload); err != nil { + rows.Close() + return err + } + var pipeline tangled.CiPipeline + if err := json.Unmarshal([]byte(payload), &pipeline); err != nil { + rows.Close() + return fmt.Errorf("decode pipeline %s while staging log migration: %w", rkey, err) + } + for _, workflow := range pipeline.Workflows { + if workflow != nil && workflow.Name != "" { + renames = append(renames, pipelineLogRename{knot, models.PipelineId(rkey), workflow.Name}) + } + } + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + } + + rows, err := tx.Query(` + select knot, rkey, workflow from mill_leases + union + select knot, rkey, workflow from mill_artifacts + union + select knot, rkey, workflow from executor_pending_artifacts + `) + if err != nil { + return err + } + for rows.Next() { + var knot, rkey, workflow string + if err := rows.Scan(&knot, &rkey, &workflow); err != nil { + rows.Close() + return err + } + renames = append(renames, pipelineLogRename{knot, models.PipelineId(rkey), workflow}) + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + + for _, rename := range renames { + if _, err := tx.Exec( + `insert or ignore into pipeline_log_renames (knot, pipeline_id, workflow) values (?, ?, ?)`, + rename.knot, rename.pipelineID, rename.workflow, + ); err != nil { + return err + } + } + return nil +} + +func (d *DB) MigratePipelineLogFiles(logDir string) error { + var tableExists int + if err := d.QueryRow(` + select count(*) from sqlite_master + where type = 'table' and name = 'pipeline_log_renames' + `).Scan(&tableExists); err != nil { + return err + } + if tableExists == 0 { + return nil + } + if logDir == "" { + return d.finishPipelineLogMigration() + } + if _, err := os.Stat(logDir); err != nil { + if os.IsNotExist(err) { + return d.finishPipelineLogMigration() + } + return err + } + + rows, err := d.Query(`select knot, pipeline_id, workflow from pipeline_log_renames`) + if err != nil { + return err + } + var renames []pipelineLogRename + for rows.Next() { + var rename pipelineLogRename + if err := rows.Scan(&rename.knot, &rename.pipelineID, &rename.workflow); err != nil { + rows.Close() + return err + } + renames = append(renames, rename) + } + if err := rows.Err(); err != nil { + rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + + type plannedRename struct { + rename pipelineLogRename + oldPath string + newPath string + destinationIsSameFile bool + } + var planned []plannedRename + var stale []pipelineLogRename + destinations := make(map[string]string) + for _, rename := range renames { + oldPath := models.LegacyLogFilePath(logDir, rename.knot, rename.pipelineID, rename.workflow) + newPath := models.LogFilePath(logDir, models.WorkflowId{PipelineId: rename.pipelineID, Name: rename.workflow}) + oldInfo, err := os.Stat(oldPath) + if err != nil { + if os.IsNotExist(err) { + stale = append(stale, rename) + continue + } + return fmt.Errorf("stat legacy pipeline log %s: %w", oldPath, err) + } + if !oldInfo.Mode().IsRegular() { + return fmt.Errorf("legacy pipeline log is not a regular file: %s", oldPath) + } + if previous, ok := destinations[newPath]; ok && previous != oldPath { + return fmt.Errorf("legacy pipeline logs %s and %s both map to %s", previous, oldPath, newPath) + } + destinations[newPath] = oldPath + + sameFile := false + if newInfo, err := os.Stat(newPath); err == nil { + if !os.SameFile(oldInfo, newInfo) { + return fmt.Errorf("refusing to overwrite pipeline log %s while migrating %s", newPath, oldPath) + } + sameFile = true + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat pipeline log %s: %w", newPath, err) + } + planned = append(planned, plannedRename{rename, oldPath, newPath, sameFile}) + } + + for _, rename := range stale { + if err := d.clearPipelineLogRename(rename); err != nil { + return err + } + } + for _, plan := range planned { + if !plan.destinationIsSameFile { + if err := os.Link(plan.oldPath, plan.newPath); err != nil { + return fmt.Errorf("link pipeline log %s to %s: %w", plan.oldPath, plan.newPath, err) + } + } + if err := os.Remove(plan.oldPath); err != nil { + return fmt.Errorf("remove legacy pipeline log %s: %w", plan.oldPath, err) + } + if err := d.clearPipelineLogRename(plan.rename); err != nil { + return err + } + } + return d.finishPipelineLogMigration() +} + +func (d *DB) finishPipelineLogMigration() error { + _, err := d.Exec(`drop table if exists pipeline_log_renames`) + return err +} + +func (d *DB) clearPipelineLogRename(rename pipelineLogRename) error { + _, err := d.Exec( + `delete from pipeline_log_renames where knot = ? and pipeline_id = ? and workflow = ?`, + rename.knot, rename.pipelineID, rename.workflow, + ) + return err +} diff --git a/spindle/db/mill_state.go b/spindle/db/mill_state.go index 99b88be18..36a2ea322 100644 --- a/spindle/db/mill_state.go +++ b/spindle/db/mill_state.go @@ -16,8 +16,7 @@ type MillLease struct { NodeID string Epoch string Engine string - Knot string - Rkey string + PipelineID string Workflow string State string QuotaReservationID string @@ -47,16 +46,16 @@ type OutboxDeletion struct { func (d *DB) SaveMillLease(l MillLease) error { _, err := d.Exec( `insert into mill_leases ( - lease_id, node_id, epoch, engine, knot, rkey, workflow, state, + lease_id, node_id, epoch, engine, pipeline_id, workflow, state, quota_reservation_id, owner_did, repo_did, mill_records_terminal_metrics - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict(lease_id) do update set state = excluded.state, quota_reservation_id = excluded.quota_reservation_id, owner_did = excluded.owner_did, repo_did = excluded.repo_did, mill_records_terminal_metrics = excluded.mill_records_terminal_metrics`, - l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.Knot, l.Rkey, l.Workflow, l.State, + l.LeaseID, l.NodeID, l.Epoch, l.Engine, l.PipelineID, l.Workflow, l.State, l.QuotaReservationID, l.OwnerDID, l.RepoDID, l.MillRecordsTerminalMetrics, ) return err @@ -69,7 +68,7 @@ func (d *DB) DeleteMillLease(leaseID string) error { func (d *DB) ListMillLeases() ([]MillLease, error) { rows, err := d.Query(` - select lease_id, node_id, epoch, engine, knot, rkey, workflow, state, + select lease_id, node_id, epoch, engine, pipeline_id, workflow, state, coalesce(quota_reservation_id, ''), coalesce(owner_did, ''), coalesce(repo_did, ''), mill_records_terminal_metrics from mill_leases @@ -83,7 +82,7 @@ func (d *DB) ListMillLeases() ([]MillLease, error) { for rows.Next() { var l MillLease if err := rows.Scan( - &l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.Knot, &l.Rkey, &l.Workflow, &l.State, + &l.LeaseID, &l.NodeID, &l.Epoch, &l.Engine, &l.PipelineID, &l.Workflow, &l.State, &l.QuotaReservationID, &l.OwnerDID, &l.RepoDID, &l.MillRecordsTerminalMetrics, ); err != nil { return nil, err @@ -341,17 +340,16 @@ func (tx *EventBatchTx) InsertArtifactRef(leaseID, repoDid string, wid models.Wo return err } _, err := tx.tx.Exec( - `insert into mill_artifacts (lease_id, repo_did, knot, rkey, workflow, ref, hash) - values (?, ?, ?, ?, ?, ?, ?)`, - leaseID, repoDid, wid.Knot, wid.Rkey, wid.Name, ref, hash, + `insert into mill_artifacts (lease_id, repo_did, pipeline_id, workflow, ref, hash) + values (?, ?, ?, ?, ?, ?)`, + leaseID, repoDid, string(wid.PipelineId), wid.Name, ref, hash, ) return err } type PendingArtifact struct { LeaseID string - Knot string - Rkey string + PipelineID string Workflow string Status string Error string @@ -373,12 +371,11 @@ func (d *DB) SavePendingArtifact( ) error { _, err := d.Exec( `insert into executor_pending_artifacts ( - lease_id, knot, rkey, workflow, status, error, exit_code, ref, hash, + lease_id, pipeline_id, workflow, status, error, exit_code, ref, hash, failure_class, failure_reason, mill_records_terminal_metrics - ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict(lease_id) do update set - knot = excluded.knot, - rkey = excluded.rkey, + pipeline_id = excluded.pipeline_id, workflow = excluded.workflow, status = excluded.status, error = excluded.error, @@ -388,7 +385,7 @@ func (d *DB) SavePendingArtifact( failure_class = excluded.failure_class, failure_reason = excluded.failure_reason, mill_records_terminal_metrics = excluded.mill_records_terminal_metrics`, - leaseID, wid.Knot, wid.Rkey, wid.Name, status, errStr, exitCode, ref, hash, + leaseID, string(wid.PipelineId), wid.Name, status, errStr, exitCode, ref, hash, failureClass, failureReason, millRecordsTerminalMetrics, ) return err @@ -401,7 +398,7 @@ func (d *DB) RemovePendingArtifact(leaseID string) error { func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { rows, err := d.Query(` - select lease_id, knot, rkey, workflow, status, error, exit_code, ref, hash, + select lease_id, pipeline_id, workflow, status, error, exit_code, ref, hash, failure_class, failure_reason, mill_records_terminal_metrics from executor_pending_artifacts `) @@ -414,7 +411,7 @@ func (d *DB) ListPendingArtifacts() ([]PendingArtifact, error) { for rows.Next() { var p PendingArtifact if err := rows.Scan( - &p.LeaseID, &p.Knot, &p.Rkey, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash, + &p.LeaseID, &p.PipelineID, &p.Workflow, &p.Status, &p.Error, &p.ExitCode, &p.Ref, &p.Hash, &p.FailureClass, &p.FailureReason, &p.MillRecordsTerminalMetrics, ); err != nil { return nil, err diff --git a/spindle/db/mill_state_test.go b/spindle/db/mill_state_test.go index a9549c72e..179e621a5 100644 --- a/spindle/db/mill_state_test.go +++ b/spindle/db/mill_state_test.go @@ -22,8 +22,7 @@ func TestMillLeaseRoundTrip(t *testing.T) { NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "knot.example", - Rkey: "rkey1", + PipelineID: "rkey1", Workflow: "build", State: "reserved", MillRecordsTerminalMetrics: true, @@ -169,7 +168,7 @@ func TestCompleteMillLeaseIsAtomic(t *testing.T) { d := newTestDB(t) lease := MillLease{ LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: "running", + PipelineID: "rkey1", Workflow: "build", State: "running", } if err := d.SaveMillLease(lease); err != nil { t.Fatalf("SaveMillLease: %v", err) @@ -189,7 +188,7 @@ func TestCompleteMillLeaseIsAtomic(t *testing.T) { defer n.Unsubscribe(notifications) err := d.CompleteMillLease( "lease-1", - models.WorkflowId{PipelineId: models.PipelineId{Rkey: "rkey1"}, Name: "build"}, + models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"}, "failed", nil, nil, @@ -219,7 +218,7 @@ func TestCompleteMillLeaseIsAtomic(t *testing.T) { } if err := d.CompleteMillLease( "lease-1", - models.WorkflowId{PipelineId: models.PipelineId{Rkey: "rkey1"}, Name: "build"}, + models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"}, "failed", nil, nil, @@ -252,8 +251,7 @@ func TestRestartPersistence(t *testing.T) { } lease := MillLease{ - LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy", - Knot: "k", Rkey: "r", Workflow: "w", State: "running", + LeaseID: "lease-p", NodeID: "node-p", Epoch: "inc-p", Engine: "dummy", PipelineID: "r", Workflow: "w", State: "running", } if err := d.SaveMillLease(lease); err != nil { t.Fatalf("SaveMillLease: %v", err) @@ -437,8 +435,7 @@ func TestBatchRollback(t *testing.T) { d := newTestDB(t) lease := MillLease{ - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "k", Rkey: "r", Workflow: "w", State: "running", + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", PipelineID: "r", Workflow: "w", State: "running", } if err := d.SaveMillLease(lease); err != nil { t.Fatalf("SaveMillLease: %v", err) @@ -480,8 +477,7 @@ func TestTerminalCursorAtomicity(t *testing.T) { d := newTestDB(t) lease := MillLease{ - LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "k", Rkey: "r", Workflow: "w", State: "running", + LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", PipelineID: "r", Workflow: "w", State: "running", } if err := d.SaveMillLease(lease); err != nil { t.Fatalf("SaveMillLease: %v", err) @@ -604,24 +600,24 @@ func TestPendingArtifactWorkflowIdentityMigration(t *testing.T) { if err != nil { t.Fatal(err) } - if len(rows) != 1 || rows[0].Knot != "" || rows[0].Rkey != "" || rows[0].Workflow != "build" { + if len(rows) != 1 || rows[0].PipelineID != "" || rows[0].Workflow != "build" { t.Fatalf("migrated pending artifact = %+v", rows) } var identityColumns int if err := d.QueryRow(` select count(*) from pragma_table_info('executor_pending_artifacts') - where name in ('knot', 'rkey') + where name = 'pipeline_id' `).Scan(&identityColumns); err != nil { t.Fatal(err) } - if identityColumns != 2 { - t.Fatalf("pending artifact identity columns = %d, want 2", identityColumns) + if identityColumns != 1 { + t.Fatalf("pending artifact identity columns = %d, want 1", identityColumns) } } func TestClearPendingArtifacts(t *testing.T) { d := newTestDB(t) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} if err := d.SavePendingArtifact("lease-1", wid, "success", "", 0, "ref", "sha256:x", "none", "success", true); err != nil { t.Fatalf("SavePendingArtifact: %v", err) } @@ -629,7 +625,7 @@ func TestClearPendingArtifacts(t *testing.T) { if err != nil { t.Fatal(err) } - if len(rows) != 1 || rows[0].Knot != wid.Knot || rows[0].Rkey != wid.Rkey || rows[0].Workflow != wid.Name || + if len(rows) != 1 || rows[0].PipelineID != string(wid.PipelineId) || rows[0].Workflow != wid.Name || rows[0].FailureClass != "none" || rows[0].FailureReason != "success" || !rows[0].MillRecordsTerminalMetrics { t.Fatalf("pending artifact attribution = %+v", rows) } diff --git a/spindle/db/pipelines.go b/spindle/db/pipelines.go index 1e3d8d9cd..cfe921adc 100644 --- a/spindle/db/pipelines.go +++ b/spindle/db/pipelines.go @@ -57,32 +57,7 @@ func (d *DB) QueryPipelines(ctx context.Context, repoDid string, commits []strin if err != nil { return nil, "", 0, err } - - // mapToCiPipeline issues nested queries, so rows must be fully drained and - // released before mapping; the pool is a single connection and would - // otherwise deadlock - var rawRows []struct { - rkey string - eventJson string - created int64 - } - for rows.Next() { - var row struct { - rkey string - eventJson string - created int64 - } - if err := rows.Scan(&row.rkey, &row.eventJson, &row.created); err != nil { - rows.Close() - return nil, "", 0, err - } - rawRows = append(rawRows, row) - } - if err := rows.Err(); err != nil { - rows.Close() - return nil, "", 0, err - } - rows.Close() + defer rows.Close() var pipelines []*tangled.CiPipeline var lastId int64 @@ -107,7 +82,6 @@ func (d *DB) QueryPipelines(ctx context.Context, repoDid string, commits []strin if err := d.applyStatuses(ctx, pipelines); err != nil { return nil, "", 0, err } - } nextCursor := "" if len(pipelines) == limit { @@ -117,49 +91,36 @@ func (d *DB) QueryPipelines(ctx context.Context, repoDid string, commits []strin return pipelines, nextCursor, total, nil } -// GetPipelineWithKnot also returns the knot the pipeline was created on, which -// subscribePipelineLogs needs to locate the workflow log file. -func (d *DB) GetPipelineWithKnot(ctx context.Context, rkey string) (*tangled.CiPipeline, string, error) { - var payload, knot string +func (d *DB) GetPipeline(ctx context.Context, id models.PipelineId) (*tangled.CiPipeline, error) { + var payload string if err := d.QueryRowContext(ctx, - `select payload, knot from pipelines where rkey = ?`, rkey, - ).Scan(&payload, &knot); err != nil { - return nil, "", err + `select payload from pipelines where pipeline_id = ?`, id, + ).Scan(&payload); err != nil { + return nil, err } var p tangled.CiPipeline if err := json.Unmarshal([]byte(payload), &p); err != nil { - return nil, "", err + return nil, err } if err := d.applyStatuses(ctx, []*tangled.CiPipeline{&p}); err != nil { - return nil, "", err + return nil, err } - return &p, knot, nil -} - -func (d *DB) GetPipeline(ctx context.Context, rkey string) (*tangled.CiPipeline, error) { - p, _, err := d.GetPipelineWithKnot(ctx, rkey) - return p, err + return &p, nil } -// mapToCiPipeline converts a compiled legacy pipeline into the sh.tangled.ci.pipeline -// payload we store, plus the columns we index on. workflow statuses in the returned -// payload are always "pending": the read path overwrites them from workflow_statuses, -// so the payload never has to be rewritten as a pipeline progresses. -// -// kind is taken straight from raw.TriggerMetadata.Kind, which already holds the -// workflow.TriggerKind vocabulary the queryPipelines `kinds` param uses. Producing it -// in the same switch as the trigger union keeps the two from disagreeing. -func mapToCiPipeline(rkey string, createdAt time.Time, raw tangled.Pipeline) (*tangled.CiPipeline, workflow.TriggerKind) { +// mapToCiPipeline stores the initial status; reads overlay live workflow_statuses rows. +func mapToCiPipeline(id models.PipelineId, createdAt time.Time, raw tangled.Pipeline) (*tangled.CiPipeline, workflow.TriggerKind) { createdAtStr := createdAt.Format(time.RFC3339) + metadata := raw.TriggerMetadata var repoDidStr string - if raw.TriggerMetadata != nil && raw.TriggerMetadata.Repo != nil { - if rd := raw.TriggerMetadata.Repo.RepoDid; rd != nil && *rd != "" { + if metadata != nil && metadata.Repo != nil { + if rd := metadata.Repo.RepoDid; rd != nil && *rd != "" { repoDidStr = *rd } else { - repoDidStr = raw.TriggerMetadata.Repo.Did + repoDidStr = metadata.Repo.Did } } @@ -167,38 +128,38 @@ func mapToCiPipeline(rkey string, createdAt time.Time, raw tangled.Pipeline) (*t var trigger tangled.CiPipeline_Trigger var kind workflow.TriggerKind - if raw.TriggerMetadata != nil { - kind = workflow.TriggerKind(raw.TriggerMetadata.Kind) + if metadata != nil { + kind = workflow.TriggerKind(metadata.Kind) switch kind { case workflow.TriggerKindPush: - if raw.TriggerMetadata.Push != nil { - commitSha = raw.TriggerMetadata.Push.NewSha + if metadata.Push != nil { + commitSha = metadata.Push.NewSha trigger.CiTrigger_Push = &tangled.CiTrigger_Push{ - NewSha: raw.TriggerMetadata.Push.NewSha, - OldSha: raw.TriggerMetadata.Push.OldSha, - Ref: raw.TriggerMetadata.Push.Ref, + NewSha: metadata.Push.NewSha, + OldSha: metadata.Push.OldSha, + Ref: metadata.Push.Ref, } } case workflow.TriggerKindPullRequest: - if raw.TriggerMetadata.PullRequest != nil { - commitSha = raw.TriggerMetadata.PullRequest.SourceSha + if metadata.PullRequest != nil { + commitSha = metadata.PullRequest.SourceSha trigger.CiTrigger_PullRequest = &tangled.CiTrigger_PullRequest{ - Action: raw.TriggerMetadata.PullRequest.Action, - SourceBranch: &raw.TriggerMetadata.PullRequest.SourceBranch, - SourceRepo: raw.TriggerMetadata.SourceRepo, - SourceSha: raw.TriggerMetadata.PullRequest.SourceSha, - TargetBranch: raw.TriggerMetadata.PullRequest.TargetBranch, - Pull: raw.TriggerMetadata.PullRequest.Pull, + Action: metadata.PullRequest.Action, + SourceBranch: &metadata.PullRequest.SourceBranch, + SourceRepo: metadata.SourceRepo, + SourceSha: metadata.PullRequest.SourceSha, + TargetBranch: metadata.PullRequest.TargetBranch, + Pull: metadata.PullRequest.Pull, } } case workflow.TriggerKindManual: - if raw.TriggerMetadata.Manual != nil { - commitSha = raw.TriggerMetadata.Manual.Sha + if metadata.Manual != nil { + commitSha = metadata.Manual.Sha trigger.CiTrigger_Manual = &tangled.CiTrigger_Manual{ - Inputs: pipelinePairsToCiTriggerPairs(raw.TriggerMetadata.Manual.Inputs), - Ref: raw.TriggerMetadata.Manual.Ref, - Sha: raw.TriggerMetadata.Manual.Sha, - SourceRepo: raw.TriggerMetadata.SourceRepo, + Inputs: pipelinePairsToCiTriggerPairs(metadata.Manual.Inputs), + Ref: metadata.Manual.Ref, + Sha: metadata.Manual.Sha, + SourceRepo: metadata.SourceRepo, } } } @@ -210,24 +171,20 @@ func mapToCiPipeline(rkey string, createdAt time.Time, raw tangled.Pipeline) (*t continue } - // NOTE: workflow statuses will be filled from caller workflows = append(workflows, &tangled.CiPipeline_Workflow{ - Id: wf.Name, - Name: wf.Name, - Status: string(models.StatusKindPending), - StartedAt: nil, - FinishedAt: nil, - Error: nil, + Id: wf.Name, + Name: wf.Name, + Status: string(models.StatusKindPending), }) } var sourceRepo *string - if raw.TriggerMetadata != nil { - sourceRepo = raw.TriggerMetadata.SourceRepo + if metadata != nil { + sourceRepo = metadata.SourceRepo } return &tangled.CiPipeline{ - Id: rkey, + Id: string(id), Commit: commitSha, Repo: repoDidStr, CreatedAt: &createdAtStr, @@ -254,24 +211,20 @@ func pipelinePairsToCiTriggerPairs(inputs []*tangled.Pipeline_Pair) []*tangled.C return pairs } -// knot is stored only because the workflow log path still embeds it; it is not -// part of pipeline identity and nothing resolves or dials it. func (d *DB) CreatePipeline(id models.PipelineId, raw tangled.Pipeline) error { - p, kind := mapToCiPipeline(id.Rkey, time.Now(), raw) + p, kind := mapToCiPipeline(id, time.Now(), raw) payload, err := json.Marshal(p) if err != nil { return err } _, err = d.Exec( - `insert into pipelines (rkey, knot, repo_did, commit_sha, kind, payload) values (?, ?, ?, ?, ?, ?)`, - id.Rkey, id.Knot, p.Repo, p.Commit, string(kind), string(payload), + `insert into pipelines (pipeline_id, repo_did, commit_sha, kind, payload) values (?, ?, ?, ?, ?)`, + id, p.Repo, p.Commit, string(kind), string(payload), ) return err } -// applyStatuses overlays live workflow status onto stored payloads with a single -// query for the whole page, replacing the old per-workflow GetStatus + -// GetWorkflowTimes fan-out. CiPipeline.Id is the rkey, so no parallel slice. +// applyStatuses overlays the latest workflow statuses onto stored pipelines. func (d *DB) applyStatuses(ctx context.Context, pipelines []*tangled.CiPipeline) error { if len(pipelines) == 0 { return nil @@ -306,8 +259,8 @@ func (d *DB) applyStatuses(ctx context.Context, pipelines []*tangled.CiPipeline) } type wfKey struct { - Rkey string - Workflow string + PipelineID string + Workflow string } type wfStatus struct { @@ -317,10 +270,10 @@ type wfStatus struct { FinishedAt *string } -func (d *DB) workflowStatuses(ctx context.Context, rkeys []string) (map[wfKey]wfStatus, error) { - filter := orm.FilterIn("rkey", rkeys) +func (d *DB) workflowStatuses(ctx context.Context, pipelineIDs []string) (map[wfKey]wfStatus, error) { + filter := orm.FilterIn("pipeline_id", pipelineIDs) rows, err := d.QueryContext(ctx, - `select rkey, workflow, status, error, created_at + `select pipeline_id, workflow, status, error, created_at from workflow_statuses where `+filter.Condition()+` order by id asc`, filter.Arg()...) @@ -331,13 +284,13 @@ func (d *DB) workflowStatuses(ctx context.Context, rkeys []string) (map[wfKey]wf out := make(map[wfKey]wfStatus) for rows.Next() { - var rkey, wfName, status, createdAt string + var pipelineID, wfName, status, createdAt string var wfError *string - if err := rows.Scan(&rkey, &wfName, &status, &wfError, &createdAt); err != nil { + if err := rows.Scan(&pipelineID, &wfName, &status, &wfError, &createdAt); err != nil { return nil, err } - k := wfKey{rkey, wfName} + k := wfKey{pipelineID, wfName} st := out[k] // rows arrive in insertion order, so the last one wins for the current status diff --git a/spindle/db/pipelines_test.go b/spindle/db/pipelines_test.go index d2fdebb25..c18688532 100644 --- a/spindle/db/pipelines_test.go +++ b/spindle/db/pipelines_test.go @@ -32,7 +32,7 @@ func seedPipeline(t *testing.T, d *DB, rkey, repoDid, kind string) { TriggerMetadata: tm, Workflows: []*tangled.Pipeline_Workflow{{Name: "ci.yml"}}, } - if err := d.CreatePipeline(models.PipelineId{Knot: "knot.test", Rkey: rkey}, raw); err != nil { + if err := d.CreatePipeline(models.PipelineId(rkey), raw); err != nil { t.Fatalf("seed pipeline %s: %v", rkey, err) } } @@ -126,12 +126,12 @@ func TestQueryPipelines_WorkflowStatuses(t *testing.T) { }, Workflows: []*tangled.Pipeline_Workflow{{Name: "a"}, {Name: "b"}}, } - if err := d.CreatePipeline(models.PipelineId{Knot: "knot.test", Rkey: "pl1"}, raw); err != nil { + if err := d.CreatePipeline(models.PipelineId("pl1"), raw); err != nil { t.Fatalf("CreatePipeline: %v", err) } - widA := models.WorkflowId{PipelineId: models.PipelineId{Rkey: "pl1"}, Name: "a"} - widB := models.WorkflowId{PipelineId: models.PipelineId{Rkey: "pl1"}, Name: "b"} + widA := models.WorkflowId{PipelineId: models.PipelineId("pl1"), Name: "a"} + widB := models.WorkflowId{PipelineId: models.PipelineId("pl1"), Name: "b"} for _, step := range []func() error{ func() error { return d.StatusPending(widA, &n) }, diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go index 1f0de4442..0c5ed99a1 100644 --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -287,7 +287,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q wl := l.With( "owner_did", w.OwnerDID, "repo_did", repoDID, - "pipeline_id", pipelineId.AtUri().String(), + "pipeline_id", pipelineId.String(), "workflow_id", wid.String(), ) @@ -321,7 +321,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q )) if span.IsRecording() { attrs := []attribute.KeyValue{ - attribute.String(observability.PipelineIDKey, pipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, pipelineId.String()), attribute.String(observability.WorkflowIDKey, wid.String()), } if w.OwnerDID != "" { @@ -497,7 +497,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q }() if qm != nil && hasQuotaReporter { - resID := quota.WorkflowReservationID(w.RunID, w.OwnerDID, w.RepoDID, wid.Knot, wid.Rkey, wid.Name) + resID := quota.WorkflowReservationID(w.RunID, w.OwnerDID, w.RepoDID, string(wid.PipelineId), wid.Name) req := quota.ReserveRequest{ ID: resID, Kind: quota.KindWorkflow, @@ -629,7 +629,7 @@ func StartWorkflows(l *slog.Logger, vault secrets.Manager, cfg *config.Config, q if stepSpan.IsRecording() { attrs := []attribute.KeyValue{ attribute.String(observability.WorkflowEngineKey, engName), - attribute.String(observability.PipelineIDKey, pipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, pipelineId.String()), attribute.String(observability.WorkflowIDKey, wid.String()), attribute.Int(observability.StepIndexKey, stepIdx), attribute.String(observability.StepNameKey, step.Name()), diff --git a/spindle/engine/engine_quota_test.go b/spindle/engine/engine_quota_test.go index 7909603a4..5877835b9 100644 --- a/spindle/engine/engine_quota_test.go +++ b/spindle/engine/engine_quota_test.go @@ -99,7 +99,7 @@ func TestStartWorkflows_NoDoubleAcquisitionByNoReporterEngine(t *testing.T) { eng: {wf}, }, } - pipelineId := models.PipelineId{Knot: "knot", Rkey: "rkey"} + pipelineId := models.PipelineId("rkey") StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) @@ -141,7 +141,7 @@ func TestStartWorkflows_QuotaAcquisitionAndIdentity(t *testing.T) { eng: {wf}, }, } - pipelineId := models.PipelineId{Knot: "knot", Rkey: "rkey"} + pipelineId := models.PipelineId("rkey") StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, context.Background(), pipeline, pipelineId) @@ -158,7 +158,7 @@ func TestStartWorkflows_QuotaAcquisitionAndIdentity(t *testing.T) { t.Errorf("unexpected identity in reservation: %+v", req.Identity) } - expectedID := quota.WorkflowReservationID("", "did:web:alice", "did:web:alice/repo", "knot", "rkey", "job1") + expectedID := quota.WorkflowReservationID("", "did:web:alice", "did:web:alice/repo", "rkey", "job1") if req.ID != expectedID { t.Errorf("expected reservation ID %q, got %q", expectedID, req.ID) } @@ -236,7 +236,7 @@ func TestStartWorkflows_QuotaFailureRecordsWorkflowFailure(t *testing.T) { } metrics := observability.NewMetrics() ctx := observability.WithMetrics(context.Background(), metrics) - StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, ctx, pipeline, models.PipelineId{Knot: "knot", Rkey: "rkey"}) + StartWorkflows(logger, nil, cfg, qm, nil, testDB, nil, nil, nil, ctx, pipeline, models.PipelineId("rkey")) families, err := metrics.Registry().Gather() if err != nil { diff --git a/spindle/engine/engine_test.go b/spindle/engine/engine_test.go index 39046dc49..fa60985e4 100644 --- a/spindle/engine/engine_test.go +++ b/spindle/engine/engine_test.go @@ -151,10 +151,7 @@ func TestStartWorkflows_CollisionRejection(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) eng := &mockEngine{} - pipelineId := models.PipelineId{ - Knot: "test-knot", - Rkey: "test-rkey", - } + pipelineId := models.PipelineId("test-rkey") // two names that normalize to the same wid must not both run wfColliding1 := models.Workflow{ @@ -234,10 +231,7 @@ func TestCancelWorkflow_NotOverwritten(t *testing.T) { }, } - pipelineId := models.PipelineId{ - Knot: "test-knot", - Rkey: "test-rkey", - } + pipelineId := models.PipelineId("test-rkey") wid := models.WorkflowId{ PipelineId: pipelineId, @@ -332,7 +326,7 @@ func TestStartWorkflows_FlushesLogBeforeTerminalStatus(t *testing.T) { }, } - pipelineId := models.PipelineId{Knot: "test-knot", Rkey: "test-rkey"} + pipelineId := models.PipelineId("test-rkey") wid := models.WorkflowId{PipelineId: pipelineId, Name: "failed_job"} pipeline := &models.Pipeline{ RepoDid: repoDid, @@ -394,7 +388,7 @@ func TestSetupTimeout_ReportsTimeout(t *testing.T) { }, } - pipelineId := models.PipelineId{Knot: "test-knot", Rkey: "test-rkey"} + pipelineId := models.PipelineId("test-rkey") wid := models.WorkflowId{PipelineId: pipelineId, Name: "timeout_job"} pipeline := &models.Pipeline{ @@ -433,10 +427,7 @@ func TestStartWorkflows_SpanEnrichment(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) eng := &mockEngine{} - pipelineId := models.PipelineId{ - Knot: "test-knot", - Rkey: "test-rkey", - } + pipelineId := models.PipelineId("test-rkey") pipeline := &models.Pipeline{ RepoDid: "did:web:test-repo", @@ -488,7 +479,7 @@ func TestStartWorkflows_SpanEnrichment(t *testing.T) { expectedWfAttrs := map[string]string{ observability.OwnerDIDKey: "did:web:test-owner", observability.RepoDIDKey: "did:web:test-repo", - observability.PipelineIDKey: pipelineId.AtUri().String(), + observability.PipelineIDKey: pipelineId.String(), observability.WorkflowIDKey: wid.String(), } for k, wantVal := range expectedWfAttrs { @@ -505,7 +496,7 @@ func TestStartWorkflows_SpanEnrichment(t *testing.T) { expectedStepAttrs := map[string]string{ observability.OwnerDIDKey: "did:web:test-owner", observability.RepoDIDKey: "did:web:test-repo", - observability.PipelineIDKey: pipelineId.AtUri().String(), + observability.PipelineIDKey: pipelineId.String(), observability.WorkflowIDKey: wid.String(), observability.StepNameKey: "step-1", } @@ -592,10 +583,7 @@ func TestStartWorkflows_ResourceTracking(t *testing.T) { usageOk: true, } - pipelineId := models.PipelineId{ - Knot: "test-knot", - Rkey: "test-rkey", - } + pipelineId := models.PipelineId("test-rkey") pipeline := &models.Pipeline{ RepoDid: "did:web:test-repo", @@ -753,7 +741,7 @@ func TestStartWorkflowsPassesPlannedCacheBindingsToEngine(t *testing.T) { got = append([]models.CacheBinding(nil), workflow.CacheBindings...) }, } - pipelineID := models.PipelineId{Knot: "test-knot", Rkey: "test-rkey"} + pipelineID := models.PipelineId("test-rkey") pipeline := &models.Pipeline{ RepoDid: "did:web:example.com", TrustedSource: true, diff --git a/spindle/engines/microvm/debug_e2e_test.go b/spindle/engines/microvm/debug_e2e_test.go index 10475209d..c2d6b1d5d 100644 --- a/spindle/engines/microvm/debug_e2e_test.go +++ b/spindle/engines/microvm/debug_e2e_test.go @@ -112,7 +112,7 @@ func TestDebugShellStaysOpenWhileIdleE2E(t *testing.T) { } wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "pipeline"}, + PipelineId: models.PipelineId("pipeline"), Name: "debug-idle-e2e.yml", } engine := &Engine{l: logger, debug: make(map[string]debugTarget)} diff --git a/spindle/knotfeed.go b/spindle/knotfeed.go index 46662e8bc..6f674128c 100644 --- a/spindle/knotfeed.go +++ b/spindle/knotfeed.go @@ -177,11 +177,11 @@ func (s *Spindle) handleRefOp(ctx context.Context, knot string, repoDid syntax.D if err != nil { return err } - if pipelineId.Rkey == "" { + if pipelineId == "" { l.Info("no workflow matched 'push' trigger, skipping the event") return nil } - l.Info("pipeline triggered", "pipeline", pipelineId.AtUri()) + l.Info("pipeline triggered", "pipeline", pipelineId) return nil } diff --git a/spindle/logview/logview_test.go b/spindle/logview/logview_test.go index 2b004424a..009770f66 100644 --- a/spindle/logview/logview_test.go +++ b/spindle/logview/logview_test.go @@ -32,8 +32,8 @@ func TestFollowDoesNotUseAnotherPipelineArtifact(t *testing.T) { } t.Cleanup(func() { d.Close() }) - target := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "target"}, Name: "build.yml"} - other := models.WorkflowId{PipelineId: models.PipelineId{Knot: target.Knot, Rkey: "other"}, Name: target.Name} + target := models.WorkflowId{PipelineId: models.PipelineId("target"), Name: "build.yml"} + other := models.WorkflowId{PipelineId: models.PipelineId("other"), Name: target.Name} if err := d.SaveArtifactRef("other-lease", "did:plc:other", other, "other.log", "hash"); err != nil { t.Fatal(err) } @@ -54,8 +54,8 @@ func TestFollowReadsExactPipelineArtifact(t *testing.T) { } t.Cleanup(func() { d.Close() }) - target := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "target"}, Name: "build.yml"} - other := models.WorkflowId{PipelineId: models.PipelineId{Knot: target.Knot, Rkey: "other"}, Name: target.Name} + target := models.WorkflowId{PipelineId: models.PipelineId("target"), Name: "build.yml"} + other := models.WorkflowId{PipelineId: models.PipelineId("other"), Name: target.Name} if err := d.SaveArtifactRef("other-lease", "did:plc:other", other, "other.log", "hash"); err != nil { t.Fatal(err) } diff --git a/spindle/mill/auth_test.go b/spindle/mill/auth_test.go index 2fabe2d8f..5697aaf1c 100644 --- a/spindle/mill/auth_test.go +++ b/spindle/mill/auth_test.go @@ -272,9 +272,9 @@ func TestOnStatusEventOwnership(t *testing.T) { m.Attach(bdb, &n, testQuotaManager(t, bdb)) foreign := newLease("lease-foreign", "node-x", "inc-x", "dummy") - foreign.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "foreign"}, Name: "build"} + foreign.wid = models.WorkflowId{PipelineId: models.PipelineId("foreign"), Name: "build"} owned := newLease("lease-owned", "node-z", "inc-z", "dummy") - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "owned"}, Name: "build"} + owned.wid = models.WorkflowId{PipelineId: models.PipelineId("owned"), Name: "build"} m.mu.Lock() m.leases[foreign.id] = foreign m.leases[owned.id] = owned @@ -432,6 +432,32 @@ func TestAuthLabelEscalation(t *testing.T) { } } +func TestProtocolVersionMismatchClosesConnection(t *testing.T) { + _, _, server, secret := setupTestServer(t, []string{"linux"}) + header := http.Header{} + header.Set("Authorization", "Bearer "+secret) + conn, _, err := websocket.DefaultDialer.Dial("ws"+strings.TrimPrefix(server.URL, "http"), header) + if err != nil { + t.Fatalf("dial failed: %v", err) + } + defer conn.Close() + + stream := millproto.NewWSStream(conn) + enc := millproto.NewEncoder(stream) + if err := enc.Encode(&millproto.Message{Hello: &millv1.Hello{ + ProtocolVersion: millproto.ProtocolVersion - 1, + Arch: "amd64", + Labels: []string{"linux"}, + Epoch: "inc-1", + CacheNamespace: "tar-zstd-v1/linux/amd64", + }}); err != nil { + t.Fatalf("encode hello: %v", err) + } + if _, err := millproto.NewDecoder(stream).Decode(); err == nil { + t.Fatal("expected server to close a stale protocol connection") + } +} + func TestHandshakeTimeoutAndConcurrency(t *testing.T) { _, _, server, secret := setupTestServer(t, []string{"linux"}) wsUrl := "ws" + strings.TrimPrefix(server.URL, "http") diff --git a/spindle/mill/executor/capability_test.go b/spindle/mill/executor/capability_test.go index 37e2c5ffb..e6029f764 100644 --- a/spindle/mill/executor/capability_test.go +++ b/spindle/mill/executor/capability_test.go @@ -50,8 +50,7 @@ func TestHandleReserveRejectsMissingTriggerMetadata(t *testing.T) { TargetEngine: "microvm", RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), - Knot: "k", - Rkey: "r", + PipelineId: "r", }) result := (<-enc.messages).GetReserveResult() @@ -94,8 +93,7 @@ func TestHandleReserveValidatesPlacementBeforeAcquiringSlot(t *testing.T) { TargetEngine: "microvm", RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), - Knot: "k", - Rkey: "r", + PipelineId: "r", RepoDid: "did:web:example.com", }) @@ -181,8 +179,7 @@ func TestHandleReserveKeepsTriggerMetadataForExecution(t *testing.T) { TargetEngine: "microvm", RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), - Knot: "k", - Rkey: "r", + PipelineId: "r", RepoDid: repoDID, }) diff --git a/spindle/mill/executor/executor.go b/spindle/mill/executor/executor.go index 5b0c9822b..eaf3089b8 100644 --- a/spindle/mill/executor/executor.go +++ b/spindle/mill/executor/executor.go @@ -426,7 +426,7 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { span.SetAttributes( attribute.String(observability.LeaseIDKey, rs.GetLeaseId()), attribute.String(observability.ExecutorNodeIDKey, e.nodeID), - attribute.String(observability.PipelineIDKey, (&models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()}).AtUri().String()), + attribute.String(observability.PipelineIDKey, rs.GetPipelineId()), ) } accepted := false @@ -481,7 +481,7 @@ func (e *Executor) handleReserve(ctx context.Context, rs *millv1.ReserveSeat) { reject("bad workflow json", millv1.RejectClass_REJECT_CLASS_INCOMPATIBLE) return } - pipelineId := models.PipelineId{Knot: rs.GetKnot(), Rkey: rs.GetRkey()} + pipelineId := models.PipelineId(rs.GetPipelineId()) wid := models.WorkflowId{PipelineId: pipelineId, Name: twf.Name} if span.IsRecording() { span.SetAttributes(attribute.String(observability.WorkflowIDKey, wid.String())) @@ -658,7 +658,7 @@ func (e *Executor) handleCommit(ctx context.Context, cl *millv1.CommitLease) { attribute.String(observability.WorkflowIDKey, res.wid.String()), attribute.String(observability.LeaseIDKey, res.leaseID), attribute.String(observability.ExecutorNodeIDKey, e.nodeID), - attribute.String(observability.PipelineIDKey, res.wid.PipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, res.wid.PipelineId.String()), } if res.pipeline.RepoDid.String() != "" { attrs = append(attrs, attribute.String(observability.RepoDIDKey, res.pipeline.RepoDid.String())) diff --git a/spindle/mill/executor/observe.go b/spindle/mill/executor/observe.go index b88c54ffe..a82173f5d 100644 --- a/spindle/mill/executor/observe.go +++ b/spindle/mill/executor/observe.go @@ -171,11 +171,11 @@ func (e *Executor) finishJob(res *reservation, st *db.StatusRow) error { return nil } -func (e *Executor) reservationFor(rkey, workflow string) *reservation { +func (e *Executor) reservationFor(pipelineID, workflow string) *reservation { e.mu.Lock() defer e.mu.Unlock() for _, res := range e.active { - if res.wid.PipelineId.Rkey == rkey && res.wid.Name == workflow { + if res.wid.PipelineId == models.PipelineId(pipelineID) && res.wid.Name == workflow { return res } } diff --git a/spindle/mill/executor/outbox.go b/spindle/mill/executor/outbox.go index ac8786b19..24e38828e 100644 --- a/spindle/mill/executor/outbox.go +++ b/spindle/mill/executor/outbox.go @@ -246,7 +246,7 @@ func (e *Executor) recoverPendingArtifacts(parent context.Context) error { logDir = e.cfg.Server.LogDir } wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: p.Knot, Rkey: p.Rkey}, + PipelineId: models.PipelineId(p.PipelineID), Name: p.Workflow, } logPath := models.LogFilePath(logDir, wid) diff --git a/spindle/mill/executor/reserved_test.go b/spindle/mill/executor/reserved_test.go index 946ab7ba9..f51367485 100644 --- a/spindle/mill/executor/reserved_test.go +++ b/spindle/mill/executor/reserved_test.go @@ -186,8 +186,7 @@ func testReserveSeat(t *testing.T, leaseID, engineName string) *millv1.ReserveSe TargetEngine: engineName, RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), - Knot: "knot.example", - Rkey: "rkey", + PipelineId: "pipeline", RepoDid: repoDID, } } @@ -405,7 +404,7 @@ func TestPendingArtifactRecoveryCompletesWithoutDeadlock(t *testing.T) { d := testDB(t) logDir := t.TempDir() wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "3abc"}, + PipelineId: models.PipelineId("3abc"), Name: "build", } logPath := models.LogFilePath(logDir, wid) @@ -478,7 +477,7 @@ func TestPendingArtifactRecoveryCompletesWithoutDeadlock(t *testing.T) { func TestPendingArtifactRecoveryKeepsRowWhenLogIsMissing(t *testing.T) { d := testDB(t) wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "3abc"}, + PipelineId: models.PipelineId("3abc"), Name: "build", } if err := d.SavePendingArtifact( @@ -525,8 +524,8 @@ func TestPendingArtifactRecoveryUsesAggregateDeadline(t *testing.T) { d := testDB(t) logDir := t.TempDir() wids := []models.WorkflowId{ - {PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "3abc"}, Name: "build"}, - {PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "3def"}, Name: "test"}, + {PipelineId: models.PipelineId("3abc"), Name: "build"}, + {PipelineId: models.PipelineId("3def"), Name: "test"}, } for i, wid := range wids { logPath := models.LogFilePath(logDir, wid) @@ -609,7 +608,7 @@ func TestHandleCommitPreservesPreauthorizedSecrets(t *testing.T) { repoDidString := repoDid.String() res := &reservation{ leaseID: "lease-1", - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + wid: models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, @@ -702,7 +701,7 @@ func TestConnectHandshakesWhilePendingArtifactRecoveryIsBlocked(t *testing.T) { logDir := t.TempDir() e.cfg.Server.LogDir = logDir wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "3abc"}, + PipelineId: models.PipelineId("3abc"), Name: "build", } logPath := models.LogFilePath(logDir, wid) @@ -791,7 +790,7 @@ func TestHandleCommitRejectsCachesAndSecretsForForkSource(t *testing.T) { cacheObjectID := "11111111-1111-4111-8111-111111111111" res := &reservation{ leaseID: "lease-1", - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + wid: models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, realEngine: inner, slot: &fakeSlot{}, wf: &models.Workflow{ @@ -943,7 +942,7 @@ func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) { d := testDB(t) res := &reservation{ leaseID: "lease-1", - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + wid: models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, cancelled: true, } e := &Executor{ @@ -957,7 +956,7 @@ func TestFinishJobReportsCancelledReservationAsCancelled(t *testing.T) { } e.finishJob(res, &db.StatusRow{ - Pipeline: res.wid.PipelineId.Rkey, + Pipeline: string(res.wid.PipelineId), Workflow: res.wid.Name, Status: string(models.StatusKindFailed), }) @@ -985,7 +984,7 @@ func TestFinishJobWaitsForEngineCleanup(t *testing.T) { res := &reservation{ leaseID: "lease-1", wid: models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + PipelineId: models.PipelineId("r"), Name: "build", }, runDone: make(chan struct{}), @@ -1003,7 +1002,7 @@ func TestFinishJobWaitsForEngineCleanup(t *testing.T) { done := make(chan error, 1) go func() { done <- e.finishJob(res, &db.StatusRow{ - Pipeline: res.wid.PipelineId.Rkey, + Pipeline: string(res.wid.PipelineId), Workflow: res.wid.Name, Status: string(models.StatusKindSuccess), }) @@ -1112,7 +1111,7 @@ func TestSocketCancellationIndependence(t *testing.T) { } res := &reservation{ leaseID: "lease-1", - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + wid: models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, @@ -1475,6 +1474,7 @@ func TestTimerRace(t *testing.T) { TargetEngine: "microvm", RawWorkflowJson: string(twf), RawPipelineJson: string(tpl), + PipelineId: "pipeline", TtlSeconds: 1, RepoDid: "did:web:example.com", }) @@ -1528,7 +1528,7 @@ func TestStructuredShutdown(t *testing.T) { } res := &reservation{ leaseID: "lease-1", - wid: models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + wid: models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, realEngine: inner, slot: slot, wf: &models.Workflow{Name: "build", Steps: []models.Step{fakeStep{}}}, diff --git a/spindle/mill/integration_test.go b/spindle/mill/integration_test.go index d100d5b5f..c355927ca 100644 --- a/spindle/mill/integration_test.go +++ b/spindle/mill/integration_test.go @@ -77,7 +77,7 @@ func TestEndToEndDummyJob(t *testing.T) { if err != nil { t.Fatalf("InitWorkflow: %v", err) } - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} if err := bdb.StatusPending(wid, &bn); err != nil { t.Fatal(err) } @@ -304,7 +304,7 @@ func TestEndToEndDummyJobUsesRequiredLabelsAcrossExecutors(t *testing.T) { if err != nil { t.Fatalf("InitWorkflow: %v", err) } - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey-arm"}, Name: "build-arm"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey-arm"), Name: "build-arm"} placeCtx, placeCancel := context.WithTimeout(ctx, 10*time.Second) defer placeCancel() @@ -356,7 +356,7 @@ func waitForStatus(t *testing.T, d *db.DB, wid models.WorkflowId, want string) b t.Fatalf("GetEvents: %v", err) } for _, ev := range evs { - if ev.Pipeline == wid.PipelineId.Rkey && ev.Workflow == wid.Name && ev.Status == want { + if ev.Pipeline == string(wid.PipelineId) && ev.Workflow == wid.Name && ev.Status == want { return true } } diff --git a/spindle/mill/mill.go b/spindle/mill/mill.go index 5db59bb67..176161cb7 100644 --- a/spindle/mill/mill.go +++ b/spindle/mill/mill.go @@ -629,7 +629,7 @@ func (m *Mill) quotaRequest(wid models.WorkflowId, wf *models.Workflow, resource m.l.Warn("placing workflow without quota because the pipeline has no repo identity", "workflow", wid.String()) return quota.ReserveRequest{}, false } - resID := quota.WorkflowReservationID(wf.RunID, id.OwnerDID, id.RepoDID, wid.Knot, wid.Rkey, wid.Name) + resID := quota.WorkflowReservationID(wf.RunID, id.OwnerDID, id.RepoDID, string(wid.PipelineId), wid.Name) return quota.ReserveRequest{ ID: resID, Kind: quota.KindWorkflow, @@ -640,11 +640,14 @@ func (m *Mill) quotaRequest(wid models.WorkflowId, wf *models.Workflow, resource } func (m *Mill) quotaIdentity(wf *models.Workflow) (quota.Identity, bool) { - st, ok := wf.Data.(*millWorkflowState) - if !ok || st == nil || st.RawPipeline.TriggerMetadata == nil { + if wf == nil { + return quota.Identity{}, false + } + state, ok := wf.Data.(*millWorkflowState) + if !ok || state == nil || state.RawPipeline.TriggerMetadata == nil { return quota.Identity{}, false } - repo := st.RawPipeline.TriggerMetadata.Repo + repo := state.RawPipeline.TriggerMetadata.Repo if repo == nil { return quota.Identity{}, false } @@ -713,7 +716,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId attribute.String(observability.WorkflowIDKey, wid.String()), attribute.String(observability.LeaseIDKey, leaseID), attribute.String(observability.ExecutorNodeIDKey, sess.nodeID), - attribute.String(observability.PipelineIDKey, wid.PipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, wid.PipelineId.String()), } if ownerDID != "" { attrs = append(attrs, attribute.String(observability.OwnerDIDKey, ownerDID)) @@ -740,8 +743,7 @@ func (m *Mill) bid(ctx context.Context, engineName string, wid models.WorkflowId TargetEngine: engineName, RawPipelineJson: rawPipeline, RawWorkflowJson: rawWorkflow, - Knot: wid.PipelineId.Knot, - Rkey: wid.PipelineId.Rkey, + PipelineId: string(wid.PipelineId), TtlSeconds: uint32(m.cfg.ReconnectGrace / time.Second), Traceparent: traceparent, Tracestate: tracestate, @@ -968,7 +970,7 @@ func (m *Mill) commitAndWait(ctx context.Context, wf *models.Workflow, unlocked attribute.String(observability.WorkflowIDKey, lease.wid.String()), attribute.String(observability.LeaseIDKey, lease.id), attribute.String(observability.ExecutorNodeIDKey, lease.nodeID), - attribute.String(observability.PipelineIDKey, lease.wid.PipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, lease.wid.PipelineId.String()), } if lease.ownerDID != "" { attrs = append(attrs, attribute.String(observability.OwnerDIDKey, lease.ownerDID)) @@ -1674,7 +1676,7 @@ func (m *Mill) onEventBatch(sess *millSession, batch *millv1.EventBatch) error { if !strings.HasPrefix(a.GetHash(), "sha256:") { return batchApply{}, protoErrf("invalid log artifact hash %q", a.GetHash()) } - if lease.wid.Knot == "" || lease.wid.Rkey == "" || lease.wid.Name == "" { + if lease.wid.PipelineId == "" || lease.wid.Name == "" { m.l.Error("skipping log artifact with incomplete workflow identity", "lease", lease.id, "wid", lease.wid) } else { if tx != nil { diff --git a/spindle/mill/mill_test.go b/spindle/mill/mill_test.go index ee650285c..2cb6aeb0b 100644 --- a/spindle/mill/mill_test.go +++ b/spindle/mill/mill_test.go @@ -220,7 +220,7 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { SaveID: saveID, SaveKey: "objects/did:web:example.com/" + saveID, }} - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease := newLease("lease-1", "node-1", "inc-1", "dummy") lease.wid = wid wf.Data.(*millWorkflowState).Lease = lease @@ -296,7 +296,7 @@ func TestCommitRetriesAfterSessionCloseBeforeCommitted(t *testing.T) { func TestDestroyRunningLeaseDoesNotDropCancelledTerminal(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{}) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease := newLease("lease-1", "node-1", "inc-1", "dummy") lease.wid = wid lease.setState(leaseRunning) @@ -323,7 +323,7 @@ func TestPlaceBlocksWhenNoCapacity(t *testing.T) { l := slog.New(slog.NewTextHandler(io.Discard, nil)) m := New(l, Config{}) wf := testWorkflow("build") - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} // with no executors at all, place must block until ctx expires and the user sees pending ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) @@ -405,7 +405,7 @@ func TestPlaceWithMissingRequiredLabelsStaysPendingWithoutReserve(t *testing.T) return nil })) wf := testWorkflowWithRunsOn("build", []string{"linux", "arm64"}) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} ctx, cancel := context.WithTimeout(context.Background(), 120*time.Millisecond) defer cancel() @@ -437,7 +437,7 @@ func TestMaxPendingRejects(t *testing.T) { func TestCancelledRunningLeaseSurvivesReleaseForReconnectReplay(t *testing.T) { m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease := newLease("lease-1", "node-1", "inc-1", "dummy") lease.wid = wid lease.setState(leaseRunning) @@ -579,7 +579,7 @@ func TestSilentSessionReplacementWithoutSnapshotFailsLeaseAtDeadline(t *testing. t.Fatal("first attach rejected") } lease := newLease("lease-1", old.nodeID, old.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -638,7 +638,7 @@ func TestPlaceReleasesRemoteReservationWhenInitialPersistenceFails(t *testing.T) slot, err := m.place( ctx, "dummy", - models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"}, + models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"}, testWorkflow("build"), ) if err == nil { @@ -669,7 +669,7 @@ func TestGapsAndDuplicates(t *testing.T) { m.attachSession(sess) owned := newLease("lease-1", "node-1", "inc-1", "dummy") - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + owned.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} m.mu.Lock() m.leases[owned.id] = owned m.mu.Unlock() @@ -713,7 +713,7 @@ func TestAtomicBatchRollback(t *testing.T) { m.attachSession(sess) owned := newLease("lease-1", "node-1", "inc-1", "dummy") - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + owned.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} m.mu.Lock() m.leases[owned.id] = owned m.mu.Unlock() @@ -787,7 +787,7 @@ func TestCacheUpdatesCommitWithStreamCursor(t *testing.T) { m.attachSession(sess) owned := newLease("lease-1", "node-1", "inc-1", "dummy") owned.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + PipelineId: models.PipelineId("r"), Name: "build", } if err := m.persistLease(owned, leaseRowRunning); err != nil { @@ -857,7 +857,7 @@ func TestCacheUpdateCannotDeleteUnplannedObject(t *testing.T) { m.attachSession(sess) lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") lease.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + PipelineId: models.PipelineId("r"), Name: "build", } if err := m.persistLease(lease, leaseRowRunning); err != nil { @@ -908,7 +908,7 @@ func TestMissingPendingRowQueuesAndDeletesPlannedObject(t *testing.T) { m.attachSession(sess) lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") lease.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, + PipelineId: models.PipelineId("r"), Name: "build", } if err := m.persistLease(lease, leaseRowRunning); err != nil { @@ -947,7 +947,7 @@ func TestTerminalBeforeACK(t *testing.T) { ackSent := make(chan struct{}) sess := newSession("node-1", "inc-1", nil, scriptedEncoder(func(msg *millproto.Message) error { if msg.GetAck() != nil { - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} st, err := bdb.GetStatus(wid) if err != nil || st != "success" { t.Errorf("expected terminal status success at ACK time, got status: %v, err: %v", st, err) @@ -959,7 +959,7 @@ func TestTerminalBeforeACK(t *testing.T) { m.attachSession(sess) owned := newLease("lease-1", "node-1", "inc-1", "dummy") - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + owned.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} m.mu.Lock() m.leases[owned.id] = owned m.mu.Unlock() @@ -993,7 +993,7 @@ func TestTerminalWithIncompleteIdentityAdvancesCursor(t *testing.T) { m.attachSession(sess) lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{Name: "build"} m.mu.Lock() m.leases[lease.id] = lease m.mu.Unlock() @@ -1035,7 +1035,7 @@ func TestExecutorRestartEmptySnapshot(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) lease := newLease("lease-1", "node-1", "inc-old", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} m.mu.Lock() m.leases[lease.id] = lease m.mu.Unlock() @@ -1065,7 +1065,7 @@ func TestExecutorRestartEmptySnapshot(t *testing.T) { func TestReplacementLostBeforeSnapshotFailsOldEpochLease(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) lease := newLease("lease-1", "node-1", "inc-old", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1113,7 +1113,7 @@ func TestClaimedSweep(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) lease := newLease("lease-1", "node-1", "inc-1", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.orphaned = true lease.claimed = false m.mu.Lock() @@ -1155,7 +1155,7 @@ func TestUnacknowledgedCancelClosesSession(t *testing.T) { m.attachSession(sess) lease := newLease("lease-1", "node-1", "inc-1", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) m.mu.Lock() m.leases[lease.id] = lease @@ -1177,7 +1177,7 @@ func TestPendingCancelBlocksPlacementUntilAcknowledged(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute, CancelAckTimeout: time.Minute}) sess := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", "node-1", sess.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) m.mu.Lock() m.leases[lease.id] = lease @@ -1195,7 +1195,7 @@ func TestTerminalSettlesPendingCancelPlacementGate(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute, CancelAckTimeout: time.Minute}) sess := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) m.mu.Lock() m.leases[lease.id] = lease @@ -1222,7 +1222,7 @@ func TestCancelSentAfterTerminalDoesNotLeakPlacementGate(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute, CancelAckTimeout: time.Minute}) sess := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", sess.nodeID, sess.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1250,7 +1250,7 @@ func TestValidSnapshotRecoversAfterProtocolError(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) first := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", first.nodeID, first.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) m.mu.Lock() m.leases[lease.id] = lease @@ -1286,7 +1286,7 @@ func TestReconnectRetriesKeepOriginalFailureDeadline(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) first := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", first.nodeID, first.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1323,7 +1323,7 @@ func TestUnacknowledgedCancelReconnectsReachBoundedTerminal(t *testing.T) { m, _ := restoreTestMill(t, Config{ReconnectGrace: time.Minute, CancelAckTimeout: time.Minute}) first := addCandidateSession(t, m, "node-1", nil, 0, nil) lease := newLease("lease-1", first.nodeID, first.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1394,7 +1394,7 @@ func TestAckedCancelFinishesOnlyItsLeaseAfterTeardownDeadline(t *testing.T) { m.attachSession(sess) lease := newLease("lease-1", "node-1", "inc-1", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1443,7 +1443,7 @@ func TestLateCancelAckTimerKeepsReconnectGrace(t *testing.T) { } lease := newLease("lease-1", "node-1", "inc-1", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persistLease: %v", err) @@ -1483,7 +1483,7 @@ func TestCancelAckDeadlineDoesNotCloseReplacementSession(t *testing.T) { oldSession := newSession("node-1", "inc-1", nil, nopEncoder(), discardLogger()) m.attachSession(oldSession) lease := newLease("lease-1", "node-1", oldSession.epoch, "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} lease.setState(leaseRunning) m.mu.Lock() m.leases[lease.id] = lease @@ -1521,7 +1521,7 @@ func TestCleanupRetry(t *testing.T) { m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) lease := newLease("lease-1", "node-1", "inc-1", "dummy") - lease.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + lease.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} if err := m.persistLease(lease, leaseRowRunning); err != nil { t.Fatalf("persist lease: %v", err) } @@ -1578,7 +1578,7 @@ func TestBidPreservesConsistentTypedIncompatibility(t *testing.T) { }) wid := models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "pipeline"}, + PipelineId: models.PipelineId("pipeline"), Name: "build", } _, err := m.bid(context.Background(), "dummy", wid, testWorkflow("build")) diff --git a/spindle/mill/proto/gen/mill.pb.go b/spindle/mill/proto/gen/mill.pb.go index 387c9d029..2b035a406 100644 --- a/spindle/mill/proto/gen/mill.pb.go +++ b/spindle/mill/proto/gen/mill.pb.go @@ -562,12 +562,10 @@ type ReserveSeat struct { TargetEngine string `protobuf:"bytes,2,opt,name=target_engine,json=targetEngine,proto3" json:"target_engine,omitempty"` RawPipelineJson string `protobuf:"bytes,3,opt,name=raw_pipeline_json,json=rawPipelineJson,proto3" json:"raw_pipeline_json,omitempty"` RawWorkflowJson string `protobuf:"bytes,4,opt,name=raw_workflow_json,json=rawWorkflowJson,proto3" json:"raw_workflow_json,omitempty"` - // pipeline id, the executor reconstructs the exact WorkflowId from it - Knot string `protobuf:"bytes,5,opt,name=knot,proto3" json:"knot,omitempty"` - Rkey string `protobuf:"bytes,6,opt,name=rkey,proto3" json:"rkey,omitempty"` - TtlSeconds uint32 `protobuf:"varint,7,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` - Traceparent string `protobuf:"bytes,8,opt,name=traceparent,proto3" json:"traceparent,omitempty"` - Tracestate string `protobuf:"bytes,9,opt,name=tracestate,proto3" json:"tracestate,omitempty"` + PipelineId string `protobuf:"bytes,5,opt,name=pipeline_id,json=pipelineId,proto3" json:"pipeline_id,omitempty"` + TtlSeconds uint32 `protobuf:"varint,6,opt,name=ttl_seconds,json=ttlSeconds,proto3" json:"ttl_seconds,omitempty"` + Traceparent string `protobuf:"bytes,7,opt,name=traceparent,proto3" json:"traceparent,omitempty"` + Tracestate string `protobuf:"bytes,8,opt,name=tracestate,proto3" json:"tracestate,omitempty"` // repository that owns the pipeline, used to distinguish fork source code RepoDid string `protobuf:"bytes,10,opt,name=repo_did,json=repoDid,proto3" json:"repo_did,omitempty"` unknownFields protoimpl.UnknownFields @@ -632,16 +630,9 @@ func (x *ReserveSeat) GetRawWorkflowJson() string { return "" } -func (x *ReserveSeat) GetKnot() string { +func (x *ReserveSeat) GetPipelineId() string { if x != nil { - return x.Knot - } - return "" -} - -func (x *ReserveSeat) GetRkey() string { - if x != nil { - return x.Rkey + return x.PipelineId } return "" } @@ -2114,22 +2105,23 @@ const file_spindle_mill_v1_mill_proto_rawDesc = "" + "\x10active_lease_ids\x18\x03 \x03(\tR\x0eactiveLeaseIds\x1a_\n" + "\fEnginesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x129\n" + - "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\xe6\x02\n" + + "\x05value\x18\x02 \x01(\v2#.spindle.mill.v1.EngineAvailabilityR\x05value:\x028\x01\"\xee\x02\n" + "\vReserveSeat\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12,\n" + "\rtarget_engine\x18\x02 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\ftargetEngine\x12*\n" + "\x11raw_pipeline_json\x18\x03 \x01(\tR\x0frawPipelineJson\x12*\n" + - "\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12\x12\n" + - "\x04knot\x18\x05 \x01(\tR\x04knot\x12\x12\n" + - "\x04rkey\x18\x06 \x01(\tR\x04rkey\x12\x1f\n" + - "\vttl_seconds\x18\a \x01(\rR\n" + + "\x11raw_workflow_json\x18\x04 \x01(\tR\x0frawWorkflowJson\x12(\n" + + "\vpipeline_id\x18\x05 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\n" + + "pipelineId\x12\x1f\n" + + "\vttl_seconds\x18\x06 \x01(\rR\n" + "ttlSeconds\x12 \n" + - "\vtraceparent\x18\b \x01(\tR\vtraceparent\x12\x1e\n" + + "\vtraceparent\x18\a \x01(\tR\vtraceparent\x12\x1e\n" + "\n" + - "tracestate\x18\t \x01(\tR\n" + + "tracestate\x18\b \x01(\tR\n" + "tracestate\x12\"\n" + "\brepo_did\x18\n" + - " \x01(\tB\a\xbaH\x04r\x02\x10\x01R\arepoDid\"\xa4\x04\n" + + " \x01(\tB\a\xbaH\x04r\x02\x10\x01R\arepoDidJ\x04\b\t\x10\n" + + "\"\xa4\x04\n" + "\rReserveResult\x12\"\n" + "\blease_id\x18\x01 \x01(\tB\a\xbaH\x04r\x02\x10\x01R\aleaseId\x12\x1a\n" + "\baccepted\x18\x02 \x01(\bR\baccepted\x12#\n" + diff --git a/spindle/mill/proto/protocol.go b/spindle/mill/proto/protocol.go index f30eb03a8..a0ea88c13 100644 --- a/spindle/mill/proto/protocol.go +++ b/spindle/mill/proto/protocol.go @@ -18,7 +18,7 @@ import ( const ( // ProtocolVersion is the wire version selected by the current implementation. - ProtocolVersion = 4 + ProtocolVersion = 5 // ProtocolMinVersion and ProtocolMaxVersion delimit versions this endpoint // can negotiate during rolling upgrades. ProtocolMinVersion = ProtocolVersion diff --git a/spindle/mill/proto/protocol_test.go b/spindle/mill/proto/protocol_test.go index 292211a3c..e64d2e8d3 100644 --- a/spindle/mill/proto/protocol_test.go +++ b/spindle/mill/proto/protocol_test.go @@ -19,8 +19,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { LeaseId: "lease-1", TargetEngine: "microvm", RawWorkflowJson: `{"name":"build"}`, - Knot: "knot.example", - Rkey: "abc123", + PipelineId: "abc123", TtlSeconds: 30, RepoDid: "did:web:example.com", }, @@ -37,7 +36,7 @@ func TestEncodeDecodeRoundTrip(t *testing.T) { if rs == nil { t.Fatal("decoded message missing reserve_seat") } - if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.TtlSeconds != 30 || rs.RepoDid != "did:web:example.com" { + if rs.LeaseId != "lease-1" || rs.TargetEngine != "microvm" || rs.PipelineId != "abc123" || rs.TtlSeconds != 30 || rs.RepoDid != "did:web:example.com" { t.Fatalf("round-trip mismatch: %+v", rs) } } @@ -385,3 +384,9 @@ func TestQuotaResourceMapBounds(t *testing.T) { } } } + +func TestProtocolVersion(t *testing.T) { + if ProtocolVersion != 5 { + t.Fatalf("ProtocolVersion = %d, want 5", ProtocolVersion) + } +} diff --git a/spindle/mill/proto/spindle/mill/v1/mill.proto b/spindle/mill/proto/spindle/mill/v1/mill.proto index f59007df6..fafb4ed79 100644 --- a/spindle/mill/proto/spindle/mill/v1/mill.proto +++ b/spindle/mill/proto/spindle/mill/v1/mill.proto @@ -48,12 +48,11 @@ message ReserveSeat { string target_engine = 2 [(buf.validate.field).string.min_len = 1]; string raw_pipeline_json = 3; string raw_workflow_json = 4; - // pipeline id, the executor reconstructs the exact WorkflowId from it - string knot = 5; - string rkey = 6; - uint32 ttl_seconds = 7; - string traceparent = 8; - string tracestate = 9; + string pipeline_id = 5 [(buf.validate.field).string.min_len = 1]; + uint32 ttl_seconds = 6; + string traceparent = 7; + string tracestate = 8; + reserved 9; // repository that owns the pipeline, used to distinguish fork source code string repo_did = 10 [(buf.validate.field).string.min_len = 1]; } diff --git a/spindle/mill/quota_admission_test.go b/spindle/mill/quota_admission_test.go index 093757f0e..9d7e06460 100644 --- a/spindle/mill/quota_admission_test.go +++ b/spindle/mill/quota_admission_test.go @@ -180,7 +180,7 @@ func TestPlaceChargesReportedResourcesToPipelineIdentity(t *testing.T) { m, _ := quotaMill(t, store) addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -222,7 +222,7 @@ func TestPlaceReleasesSeatBeforeWaitingOnQuota(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -259,7 +259,7 @@ func TestPlaceFailsAndReleasesSeatOnPermanentDenial(t *testing.T) { m.RegisterMetrics(metrics) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() slot, err := m.place(ctx, "dummy", wid, testWorkflow("build")) @@ -319,7 +319,7 @@ func TestPlaceRejectsRebidWithDifferentResources(t *testing.T) { return map[string]int64{"workflows": 1, "vcpus": 2, "memory_mib": 2048} }) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() slot, err := m.place(ctx, "dummy", wid, testWorkflow("build")) @@ -347,7 +347,7 @@ func TestSlotReleaseReleasesQuotaExactlyOnce(t *testing.T) { m, _ := quotaMill(t, store) addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -369,7 +369,7 @@ func TestFailedLeaseAfterDisconnectReleasesQuotaOnce(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -406,7 +406,7 @@ func TestCacheReserveChargesTheLeaseSubject(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -508,7 +508,7 @@ func TestCacheReserveReplayRequiresSameResources(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -580,7 +580,7 @@ func TestCacheReserveDuplicateQueuesUntilFirstSettles(t *testing.T) { return candB.sess } placeLease := func(name string) *RemoteLease { - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: name} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: name} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow(name)) if err != nil { t.Fatalf("place %s: %v", name, err) @@ -649,7 +649,7 @@ func TestBidRejectsOutOfRangeReportedResources(t *testing.T) { return map[string]int64{"workflows": -1, "vcpus": 2} }) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) defer cancel() slot, err := m.place(ctx, "dummy", wid, testWorkflow("build")) @@ -675,7 +675,7 @@ func TestLiveQuotaReservationIDsSurviveRestart(t *testing.T) { m, bdb := quotaMill(t, store) addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} if _, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")); err != nil { t.Fatalf("place: %v", err) } @@ -728,7 +728,7 @@ func TestPersistedLeaseCarriesQuotaReservation(t *testing.T) { m, bdb := quotaMill(t, store) addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -757,7 +757,7 @@ func TestUnchargeablePlacementStillRuns(t *testing.T) { // place its work addQuotaCandidate(t, m, "node-a", func(int) map[string]int64 { return nil }) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -783,7 +783,7 @@ func TestCacheReservationDuplicateIDRace(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -860,7 +860,7 @@ func TestCacheReservationCommitAfterTeardown(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -951,7 +951,7 @@ func TestCacheReservationFailedReleaseRetry(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -1017,7 +1017,7 @@ func TestCacheReservationStaleEpochRejected(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -1094,7 +1094,7 @@ func TestCacheReservationTeardownCommitRetry(t *testing.T) { m, _ := quotaMill(t, store) cand := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} slot, err := m.place(context.Background(), "dummy", wid, testWorkflow("build")) if err != nil { t.Fatalf("place: %v", err) @@ -1179,7 +1179,7 @@ func TestBlockedSendDoesNotFreezeMill(t *testing.T) { m, _ := quotaMill(t, store) candA := addQuotaCandidate(t, m, "node-a", fixedReport(2, 2048)) - widA := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkeyA"}, Name: "buildA"} + widA := models.WorkflowId{PipelineId: models.PipelineId("rkeyA"), Name: "buildA"} slotA, err := m.place(context.Background(), "dummy", widA, testWorkflow("buildA")) if err != nil { t.Fatalf("place A: %v", err) @@ -1197,7 +1197,7 @@ func TestBlockedSendDoesNotFreezeMill(t *testing.T) { m.mu.Unlock() candB := addQuotaCandidate(t, m, "node-b", fixedReport(2, 2048)) - widB := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "rkeyB"}, Name: "buildB"} + widB := models.WorkflowId{PipelineId: models.PipelineId("rkeyB"), Name: "buildB"} slotB, err := m.place(context.Background(), "dummy", widB, testWorkflow("buildB")) if err != nil { t.Fatalf("place B: %v", err) diff --git a/spindle/mill/restore.go b/spindle/mill/restore.go index b895d8105..c29486970 100644 --- a/spindle/mill/restore.go +++ b/spindle/mill/restore.go @@ -30,8 +30,7 @@ func (m *Mill) persistLease(lease *RemoteLease, state string) error { NodeID: lease.nodeID, Epoch: lease.epoch, Engine: lease.engine, - Knot: lease.wid.PipelineId.Knot, - Rkey: lease.wid.PipelineId.Rkey, + PipelineID: string(lease.wid.PipelineId), Workflow: lease.wid.Name, State: state, QuotaReservationID: quotaID, @@ -80,7 +79,7 @@ func (m *Mill) RestoreState() error { for _, r := range rows { lease := newLease(r.LeaseID, r.NodeID, r.Epoch, r.Engine) lease.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: r.Knot, Rkey: r.Rkey}, + PipelineId: models.PipelineId(r.PipelineID), Name: r.Workflow, } // the charge outlives the mill process. only the id survives, so diff --git a/spindle/mill/restore_test.go b/spindle/mill/restore_test.go index e483ae8f8..ad7144f0d 100644 --- a/spindle/mill/restore_test.go +++ b/spindle/mill/restore_test.go @@ -43,7 +43,7 @@ func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) { if err := bdb.SaveMillLease(db.MillLease{ LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + PipelineID: "rkey1", Workflow: "build", State: leaseRowRunning, }); err != nil { t.Fatalf("SaveMillLease: %v", err) } @@ -67,7 +67,7 @@ func TestRestoreStateRebuildsLeasesAndCursors(t *testing.T) { if lease.getState() != leaseRunning { t.Fatalf("restored lease state = %v, want leaseRunning", lease.getState()) } - wantWid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} + wantWid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} if lease.wid != wantWid { t.Fatalf("restored lease wid = %+v, want %+v", lease.wid, wantWid) } @@ -80,7 +80,7 @@ func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) if err := bdb.SaveMillLease(db.MillLease{ LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + PipelineID: "rkey1", Workflow: "build", State: leaseRowRunning, }); err != nil { t.Fatalf("SaveMillLease: %v", err) } @@ -114,7 +114,7 @@ func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { }, }) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example", Rkey: "rkey1"}, Name: "build"} + wid := models.WorkflowId{PipelineId: models.PipelineId("rkey1"), Name: "build"} st, err := bdb.GetStatus(wid) if err != nil { t.Fatalf("GetStatus after orphan terminal: %v", err) @@ -137,8 +137,8 @@ func TestOrphanTerminalAuthorsStatusRow(t *testing.T) { func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) { _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) for _, l := range []db.MillLease{ - {LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowRunning}, - {LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", Knot: "k", Rkey: "r2", Workflow: "w", State: leaseRowRunning}, + {LeaseID: "lease-kept", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", PipelineID: "r1", Workflow: "w", State: leaseRowRunning}, + {LeaseID: "lease-gone", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", PipelineID: "r2", Workflow: "w", State: leaseRowRunning}, } { if err := bdb.SaveMillLease(l); err != nil { t.Fatalf("SaveMillLease(%s): %v", l.LeaseID, err) @@ -166,7 +166,7 @@ func TestSnapshotReconciliationFailsDroppedOrphans(t *testing.T) { t.Fatal("reconciliation kept a lease the executor no longer holds") } - st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r2"}, Name: "w"}) + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId("r2"), Name: "w"}) if err != nil { t.Fatalf("GetStatus for dropped orphan: %v", err) } @@ -178,7 +178,7 @@ func TestSnapshotReconciliationPreservesRequestedCancellation(t *testing.T) { m, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) lease := newLease("lease-1", "node-1", "inc-old", "dummy") lease.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, + PipelineId: models.PipelineId("r1"), Name: "w", } lease.setState(leaseRunning) @@ -253,7 +253,7 @@ func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) if err := bdb.SaveMillLease(db.MillLease{ LeaseID: "lease-1", NodeID: "node-absent", Epoch: "inc-absent", Engine: "dummy", - Knot: "k", Rkey: "r1", Workflow: "w", State: leaseRowReserved, + PipelineID: "r1", Workflow: "w", State: leaseRowReserved, }); err != nil { t.Fatalf("SaveMillLease: %v", err) } @@ -267,7 +267,7 @@ func TestSweepFailsOrphansOfAbsentExecutors(t *testing.T) { if still { t.Fatal("sweep kept an orphan whose executor never reconnected") } - st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r1"}, Name: "w"}) + st, err := bdb.GetStatus(models.WorkflowId{PipelineId: models.PipelineId("r1"), Name: "w"}) if err != nil { t.Fatalf("GetStatus after sweep: %v", err) } @@ -285,7 +285,7 @@ func TestAckSeqnoPersistsCursor(t *testing.T) { m.attachSession(sess) owned := newLease("lease-1", "node-1", "inc-1", "dummy") - owned.wid = models.WorkflowId{PipelineId: models.PipelineId{Knot: "k", Rkey: "r"}, Name: "build"} + owned.wid = models.WorkflowId{PipelineId: models.PipelineId("r"), Name: "build"} m.mu.Lock() m.leases[owned.id] = owned m.mu.Unlock() @@ -321,7 +321,7 @@ func TestOrphanTerminalFailureKeepsLeaseAndSeqnoRetryable(t *testing.T) { _, bdb := restoreTestMill(t, Config{ReconnectGrace: time.Minute}) if err := bdb.SaveMillLease(db.MillLease{ LeaseID: "lease-1", NodeID: "node-1", Epoch: "inc-1", Engine: "dummy", - Knot: "knot.example", Rkey: "rkey1", Workflow: "build", State: leaseRowRunning, + PipelineID: "rkey1", Workflow: "build", State: leaseRowRunning, }); err != nil { t.Fatalf("SaveMillLease: %v", err) } @@ -410,7 +410,7 @@ func TestSyntheticTerminalRespectsPersistedMetricAuthority(t *testing.T) { lease := newLease("lease-local-metrics", "node-old", "epoch-old", "dummy") lease.wid = models.WorkflowId{ - PipelineId: models.PipelineId{Knot: "knot.test", Rkey: "pipeline"}, + PipelineId: models.PipelineId("pipeline"), Name: "build", } lease.millRecordsTerminalMetrics = false diff --git a/spindle/models/logger.go b/spindle/models/logger.go index 5406c2387..51f6ce727 100644 --- a/spindle/models/logger.go +++ b/spindle/models/logger.go @@ -49,6 +49,10 @@ func LogFilePath(baseDir string, workflowID WorkflowId) string { return logFilePath } +func LegacyLogFilePath(baseDir, knot string, pipelineID PipelineId, workflow string) string { + return filepath.Join(baseDir, fmt.Sprintf("%s-%s-%s.log", normalize(knot), pipelineID, normalize(workflow))) +} + func (l *FileWorkflowLogger) Close() error { for _, w := range l.dataWriters { if err := w.flush(); err != nil { diff --git a/spindle/models/logger_test.go b/spindle/models/logger_test.go index 79cec1ef6..933320183 100644 --- a/spindle/models/logger_test.go +++ b/spindle/models/logger_test.go @@ -9,7 +9,7 @@ import ( ) func testWorkflowId(name string) WorkflowId { - return WorkflowId{PipelineId: PipelineId{Knot: "knot1", Rkey: "rkey1"}, Name: name} + return WorkflowId{PipelineId: PipelineId("rkey1"), Name: name} } func readDataContents(t *testing.T, path string) []string { diff --git a/spindle/models/models.go b/spindle/models/models.go index 77bbbb025..d5b27925f 100644 --- a/spindle/models/models.go +++ b/spindle/models/models.go @@ -6,8 +6,6 @@ import ( "slices" "time" - "tangled.org/core/api/tangled" - "github.com/bluesky-social/indigo/atproto/syntax" ) @@ -15,14 +13,9 @@ var ( re = regexp.MustCompile(`[^a-zA-Z0-9_.-]`) ) -type PipelineId struct { - Knot string - Rkey string -} +type PipelineId syntax.RecordKey -func (p *PipelineId) AtUri() syntax.ATURI { - return syntax.ATURI(fmt.Sprintf("at://did:web:%s/%s/%s", p.Knot, tangled.PipelineNSID, p.Rkey)) -} +func (p PipelineId) String() string { return string(p) } type WorkflowId struct { PipelineId @@ -30,7 +23,7 @@ type WorkflowId struct { } func (wid WorkflowId) String() string { - return fmt.Sprintf("%s-%s-%s", normalize(wid.PipelineId.Knot), wid.PipelineId.Rkey, normalize(wid.Name)) + return fmt.Sprintf("%s-%s", normalize(string(wid.PipelineId)), normalize(wid.Name)) } func normalize(name string) string { diff --git a/spindle/models/pipeline_env.go b/spindle/models/pipeline_env.go index 235343959..9935f54d2 100644 --- a/spindle/models/pipeline_env.go +++ b/spindle/models/pipeline_env.go @@ -23,7 +23,7 @@ func PipelineEnvVarsForSource(tr *tangled.Pipeline_TriggerMetadata, pipelineId P // standard CI env vars env["CI"] = "true" - env["TANGLED_PIPELINE_ID"] = pipelineId.AtUri().String() + env["TANGLED_PIPELINE_ID"] = pipelineId.String() env["TANGLED_PIPELINE_KIND"] = tr.Kind if tr.SourceRepo != nil && *tr.SourceRepo != "" { diff --git a/spindle/models/pipeline_env_test.go b/spindle/models/pipeline_env_test.go index 54fc5f5ea..fd57b69df 100644 --- a/spindle/models/pipeline_env_test.go +++ b/spindle/models/pipeline_env_test.go @@ -23,10 +23,7 @@ func TestPipelineEnvVars_PushBranch(t *testing.T) { DefaultBranch: "main", }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) // Check standard CI variable @@ -86,10 +83,7 @@ func TestPipelineEnvVars_PushTag(t *testing.T) { RepoDid: sp("did:plc:boltless"), }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) if env["TANGLED_REF"] != "refs/tags/v1.2.3" { @@ -118,10 +112,7 @@ func TestPipelineEnvVars_PullRequest(t *testing.T) { RepoDid: sp("did:plc:boltless"), }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) // Check ref variables for PR @@ -185,10 +176,7 @@ func TestPipelineEnvVars_SourceRepo(t *testing.T) { RepoDid: &sourceRepoDid, DefaultBranch: "feature-branch", } - id := PipelineId{ - Knot: "target.example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVarsForSource(tr, id, sourceRepo) @@ -220,10 +208,7 @@ func TestPipelineEnvVars_ManualWithInputs(t *testing.T) { RepoDid: sp("did:plc:boltless"), }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) // Check manual input variables @@ -261,10 +246,7 @@ func TestPipelineEnvVars_DevMode(t *testing.T) { RepoDid: sp("did:plc:boltless"), }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) expectedURL := "http://localhost:3000/did:plc:boltless" @@ -274,10 +256,7 @@ func TestPipelineEnvVars_DevMode(t *testing.T) { } func TestPipelineEnvVars_NilTrigger(t *testing.T) { - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(nil, id) if env != nil { @@ -296,10 +275,7 @@ func TestPipelineEnvVars_NilPushData(t *testing.T) { RepoDid: sp("did:plc:boltless"), }, } - id := PipelineId{ - Knot: "example.com", - Rkey: "123123", - } + id := PipelineId("123123") env := PipelineEnvVars(tr, id) // Should still have repo variables diff --git a/spindle/observability/metrics.go b/spindle/observability/metrics.go index 9bb2d8e1a..5dddbf44c 100644 --- a/spindle/observability/metrics.go +++ b/spindle/observability/metrics.go @@ -1062,7 +1062,7 @@ func (c *dbCollector) refresh(ctx context.Context) { SELECT status, row_number() OVER ( - PARTITION BY rkey, workflow + PARTITION BY pipeline_id, workflow ORDER BY id DESC ) AS rank FROM workflow_statuses diff --git a/spindle/observability/metrics_test.go b/spindle/observability/metrics_test.go index c230a2a26..06ae550a3 100644 --- a/spindle/observability/metrics_test.go +++ b/spindle/observability/metrics_test.go @@ -925,7 +925,7 @@ func TestDBCollectorUsesLatestNormalizedWorkflowStatus(t *testing.T) { {"p2", "test", "failed"}, {"p1", "build", "success"}, } { - if _, err := database.Exec(`insert into workflow_statuses (rkey, workflow, status, created_at) values (?, ?, ?, ?)`, row.pipeline, row.workflow, row.status, time.Now().Format(time.RFC3339)); err != nil { + if _, err := database.Exec(`insert into workflow_statuses (pipeline_id, workflow, status, created_at) values (?, ?, ?, ?)`, row.pipeline, row.workflow, row.status, time.Now().Format(time.RFC3339)); err != nil { t.Fatal(err) } } diff --git a/spindle/quota/manager_test.go b/spindle/quota/manager_test.go index 61714c9ac..1fb01591b 100644 --- a/spindle/quota/manager_test.go +++ b/spindle/quota/manager_test.go @@ -687,8 +687,8 @@ func TestExternalLimitWake(t *testing.T) { } func TestWorkflowReservationID(t *testing.T) { - id1 := quota.WorkflowReservationID("run-1", "owner", "repo", "knot", "rkey", "name") - id2 := quota.WorkflowReservationID("run-1", "owner", "repo", "knot", "rkey", "name") + id1 := quota.WorkflowReservationID("run-1", "owner", "repo", "rkey", "name") + id2 := quota.WorkflowReservationID("run-1", "owner", "repo", "rkey", "name") if id1 != id2 { t.Errorf("expected deterministic IDs, got %q and %q", id1, id2) } @@ -698,18 +698,18 @@ func TestWorkflowReservationID(t *testing.T) { } // length-prefixing keeps shifted field boundaries distinct - idShift1 := quota.WorkflowReservationID("run-1", "ab", "c", "knot", "rkey", "name") - idShift2 := quota.WorkflowReservationID("run-1", "a", "bc", "knot", "rkey", "name") + idShift1 := quota.WorkflowReservationID("run-1", "ab", "c", "rkey", "name") + idShift2 := quota.WorkflowReservationID("run-1", "a", "bc", "rkey", "name") if idShift1 == idShift2 { t.Errorf("expected different IDs for shifted boundaries, both got %q", idShift1) } - idDiff := quota.WorkflowReservationID("run-1", "owner", "repo", "knot", "rkey", "name-changed") + idDiff := quota.WorkflowReservationID("run-1", "owner", "repo", "rkey", "name-changed") if id1 == idDiff { t.Errorf("expected different IDs on changed argument, both got %q", id1) } - idOtherRun := quota.WorkflowReservationID("run-2", "owner", "repo", "knot", "rkey", "name") + idOtherRun := quota.WorkflowReservationID("run-2", "owner", "repo", "rkey", "name") if id1 == idOtherRun { t.Errorf("expected different IDs for distinct runs, both got %q", id1) } diff --git a/spindle/quota/quota.go b/spindle/quota/quota.go index 3c705c1a7..d3f5d5635 100644 --- a/spindle/quota/quota.go +++ b/spindle/quota/quota.go @@ -190,12 +190,11 @@ func validateResourceName(resource string) error { return nil } -func WorkflowReservationID(runID, owner, repoDid, knot, rkey, name string) string { - raw := fmt.Sprintf("%d:%s:%d:%s:%d:%s:%d:%s:%d:%s:%d:%s", +func WorkflowReservationID(runID, owner, repoDid, rkey, name string) string { + raw := fmt.Sprintf("%d:%s:%d:%s:%d:%s:%d:%s:%d:%s", len(runID), runID, len(owner), owner, len(repoDid), repoDid, - len(knot), knot, len(rkey), rkey, len(name), name, ) diff --git a/spindle/server.go b/spindle/server.go index 7e14bb7f3..a546df305 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -609,6 +609,9 @@ func Run(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to setup db: %w", err) } + if err := d.MigratePipelineLogFiles(cfg.Server.LogDir); err != nil { + return fmt.Errorf("failed to migrate pipeline log files: %w", err) + } var qs *db.QuotaStore var qm *quota.Manager @@ -880,10 +883,10 @@ func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger t rawPipeline, err := s.loadPipeline(ctx, repoCloneUri, repoPath, rev) if err != nil { - return models.PipelineId{}, fmt.Errorf("loading pipeline: %w", err) + return "", fmt.Errorf("loading pipeline: %w", err) } if len(rawPipeline) == 0 { - return models.PipelineId{}, nil + return "", nil } tpl := compiler.Compile(compiler.Parse(rawPipeline)) @@ -899,15 +902,12 @@ func (s *Spindle) runPipeline(ctx context.Context, repoDid syntax.DID, trigger t tpl.Workflows = filterWorkflows(tpl.Workflows, only) } if len(tpl.Workflows) == 0 { - return models.PipelineId{}, nil + return "", nil } - pipelineId := models.PipelineId{ - Knot: trigger.Repo.Knot, - Rkey: tid.TID(), - } + pipelineId := models.PipelineId(tid.TID()) if err := s.db.CreatePipeline(pipelineId, tpl); err != nil { - return models.PipelineId{}, fmt.Errorf("creating pipeline: %w", err) + return "", fmt.Errorf("creating pipeline: %w", err) } err = s.processPipeline(ctx, repoDid, tpl, pipelineId, sourceRepo) return pipelineId, err @@ -932,7 +932,7 @@ func filterWorkflows(workflows []*tangled.Pipeline_Workflow, only []string) []*t // TriggerManual dispatches a pipeline at sha, authorized against and recorded // under repoDid. sourceRepo, pull, and inputs are optional trigger payload. -func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { +func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull xrpc.PullContext, inputs []*tangled.Pipeline_Pair) (models.PipelineId, error) { repo, err := s.db.GetRepoByDid(repoDid) if err != nil { return "", fmt.Errorf("unknown repoDid %s: %w", repoDid, err) @@ -998,10 +998,10 @@ func (s *Spindle) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, re if err != nil { return "", err } - if pipelineId.Rkey == "" { + if pipelineId == "" { return "", xrpc.ErrNoMatchingWorkflows } - return pipelineId.AtUri(), nil + return pipelineId, nil } // sourceInfo is nil when the checkout comes from the target repo. @@ -1190,10 +1190,7 @@ func (s *Spindle) StartJobWorkers(ctx context.Context) { } func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { - pipelineId := models.PipelineId{ - Knot: job.PipelineIdKnot, - Rkey: job.PipelineIdRkey, - } + pipelineId := job.PipelineId repoDID := job.RepoDid if job.SourceRepo != nil && job.SourceRepo.RepoDid != nil && *job.SourceRepo.RepoDid != "" { repoDID = *job.SourceRepo.RepoDid @@ -1212,7 +1209,7 @@ func (s *Spindle) runJob(ctx context.Context, job *db.JobRow) { if span.IsRecording() { attrs := []attribute.KeyValue{ attribute.Int64(observability.JobIDKey, job.Id), - attribute.String(observability.PipelineIDKey, pipelineId.AtUri().String()), + attribute.String(observability.PipelineIDKey, pipelineId.String()), } if repoDID != "" { attrs = append(attrs, attribute.String(observability.RepoDIDKey, repoDID)) diff --git a/spindle/tapclient.go b/spindle/tapclient.go index 66bf8f9b0..64104ba84 100644 --- a/spindle/tapclient.go +++ b/spindle/tapclient.go @@ -703,10 +703,7 @@ func (s *Spindle) triggerPullRequestPipeline(ctx context.Context, l *slog.Logger return nil } - pipelineId := models.PipelineId{ - Knot: tpl.TriggerMetadata.Repo.Knot, - Rkey: tid.TID(), - } + pipelineId := models.PipelineId(tid.TID()) if err := s.db.CreatePipeline(pipelineId, tpl); err != nil { l.Error("failed to create pipeline event", "err", err) return nil diff --git a/spindle/tapclient_test.go b/spindle/tapclient_test.go index 2c0027400..879e8a0a9 100644 --- a/spindle/tapclient_test.go +++ b/spindle/tapclient_test.go @@ -552,7 +552,7 @@ func TestTeardownRepo_RBAC(t *testing.T) { if err := vault.AddSecret(context.Background(), secrets.UnlockedSecret{Key: "api_key", Value: "v", Repo: secrets.RepoIdentifier(repoDid.String()), CreatedBy: ownerDid}); err != nil { t.Fatalf("AddSecret: %v", err) } - if err := d.EnqueueJob(context.Background(), repoDid.String(), models.PipelineId{Knot: "knot.test", Rkey: "p1"}, nil, tangled.Pipeline{}, "", ""); err != nil { + if err := d.EnqueueJob(context.Background(), repoDid.String(), models.PipelineId("p1"), nil, tangled.Pipeline{}, "", ""); err != nil { t.Fatalf("EnqueueJob: %v", err) } diff --git a/spindle/wipe.go b/spindle/wipe.go index edbc498c1..8c25766c3 100644 --- a/spindle/wipe.go +++ b/spindle/wipe.go @@ -34,7 +34,7 @@ func (s *Spindle) WipeRepo(ctx context.Context, repoDid syntax.DID, reason strin } // cancel work before deleting the rows it references - wids, err := s.cancelRepoWorkflows(ctx, repoDid, repos, reason) + wids, err := s.cancelRepoWorkflows(ctx, repoDid, reason) if err != nil { return fmt.Errorf("cancel workflows: %w", err) } @@ -79,17 +79,11 @@ func (s *Spindle) WipeRepo(ctx context.Context, repoDid syntax.DID, reason strin fail("remove repo acl", s.e.RemoveRepo(r.Owner.String(), rbac.ThisServer, repoDid.String())) } - // use the recorded knot and rkey for status events - seen := map[db.PipelineKey]bool{} - var keys []db.PipelineKey + pipelineIDs := make([]models.PipelineId, 0, len(wids)) for _, wid := range wids { - k := db.PipelineKey{Knot: wid.PipelineId.Knot, Rkey: wid.PipelineId.Rkey} - if !seen[k] { - seen[k] = true - keys = append(keys, k) - } + pipelineIDs = append(pipelineIDs, wid.PipelineId) } - fail("delete events", s.db.DeleteEventsByRepo(repoDid.String(), keys)) + fail("delete pipelines", s.db.DeletePipelinesByRepo(repoDid.String(), pipelineIDs)) // keep the repo row when cleanup needs a retry if len(errs) > 0 { @@ -138,11 +132,11 @@ func (s *Spindle) WipeOwner(ctx context.Context, ownerDid syntax.DID, reason str return errors.Join(errs...) } -func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, repos []db.Repo, reason string) ([]models.WorkflowId, error) { +func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, reason string) ([]models.WorkflowId, error) { var wids []models.WorkflowId seen := map[models.WorkflowId]bool{} - add := func(knot, rkey, name string) { - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: knot, Rkey: rkey}, Name: name} + add := func(rkey, name string) { + wid := models.WorkflowId{PipelineId: models.PipelineId(rkey), Name: name} if !seen[wid] { seen[wid] = true wids = append(wids, wid) @@ -154,13 +148,7 @@ func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, r return nil, fmt.Errorf("list pipeline workflows: %w", err) } for _, p := range pipes { - if p.Knot != "" { - add(p.Knot, p.Rkey, p.Name) - continue - } - for _, r := range repos { - add(r.Knot, p.Rkey, p.Name) - } + add(string(p.PipelineID), p.Name) } leases, err := s.db.ListMillLeases() @@ -169,27 +157,26 @@ func (s *Spindle) cancelRepoWorkflows(ctx context.Context, repoDid syntax.DID, r } for _, lease := range leases { if lease.RepoDID == repoDid.String() { - add(lease.Knot, lease.Rkey, lease.Workflow) + add(lease.PipelineID, lease.Workflow) } } var errs []error for _, wid := range wids { st, err := s.db.GetStatus(wid) - if err == nil && st.IsFinish() { - continue - } - if err := s.db.StatusCancelled(wid, reason, -1, s.n); err != nil { - errs = append(errs, err) - } - engine.CancelWorkflow(wid) - for _, eng := range s.engs { - if err := eng.DestroyWorkflow(ctx, wid); err != nil { + if err != nil || !st.IsFinish() { + if err := s.db.StatusCancelled(wid, reason, -1, s.n); err != nil { errs = append(errs, err) } + engine.CancelWorkflow(wid) + for _, eng := range s.engs { + if err := eng.DestroyWorkflow(ctx, wid); err != nil { + errs = append(errs, err) + } + } } if err := os.Remove(models.LogFilePath(s.cfg.Server.LogDir, wid)); err != nil && !os.IsNotExist(err) { - errs = append(errs, fmt.Errorf("remove live log for %s: %w", wid, err)) + errs = append(errs, fmt.Errorf("remove log for %s: %w", wid, err)) } } return wids, errors.Join(errs...) diff --git a/spindle/wipe_test.go b/spindle/wipe_test.go index b83035487..3eb070c3a 100644 --- a/spindle/wipe_test.go +++ b/spindle/wipe_test.go @@ -89,13 +89,13 @@ func seedWipeRepo(t *testing.T, s *Spindle, repoDid, owner syntax.DID) { if err := s.db.AddRepoCollaborator(db.RepoCollaborator{OwnerDid: owner, Rkey: syntax.RecordKey("c" + sfx), Subject: "did:plc:collab", RepoDid: repoDid}); err != nil { t.Fatal(err) } - if err := s.db.EnqueueJob(ctx, repoDid.String(), models.PipelineId{Knot: "knot.example.com", Rkey: "p" + sfx}, nil, tangled.Pipeline{}, "", ""); err != nil { + if err := s.db.EnqueueJob(ctx, repoDid.String(), models.PipelineId("p"+sfx), nil, tangled.Pipeline{}, "", ""); err != nil { t.Fatal(err) } - if err := s.db.SaveMillLease(db.MillLease{LeaseID: "l" + sfx, NodeID: "n1", Epoch: "e1", Engine: "microvm", Knot: "knot.example.com", Rkey: "p" + sfx, Workflow: "w" + sfx, State: "active", RepoDID: repoDid.String()}); err != nil { + if err := s.db.SaveMillLease(db.MillLease{LeaseID: "l" + sfx, NodeID: "n1", Epoch: "e1", Engine: "microvm", PipelineID: "p" + sfx, Workflow: "w" + sfx, State: "active", RepoDID: repoDid.String()}); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`insert into mill_artifacts (lease_id, repo_did, knot, rkey, workflow, ref, hash) values (?, ?, 'knot.example.com', ?, ?, ?, 'h')`, "l"+sfx, repoDid.String(), "p"+sfx, "w"+sfx, "out/l"+sfx+".bin"); err != nil { + if _, err := s.db.Exec(`insert into mill_artifacts (lease_id, repo_did, pipeline_id, workflow, ref, hash) values (?, ?, ?, ?, ?, 'h')`, "l"+sfx, repoDid.String(), "p"+sfx, "w"+sfx, "out/l"+sfx+".bin"); err != nil { t.Fatal(err) } if _, err := s.db.Exec(`insert into quota_allocations (repo_did, resource, kind, key, amount) values (?, 'compute', 'generic', 'k', 1)`, repoDid.String()); err != nil { @@ -122,10 +122,10 @@ func seedWipeRepo(t *testing.T, s *Spindle, repoDid, owner syntax.DID) { if err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`insert into pipelines (rkey, knot, repo_did, commit_sha, kind, payload) values (?, 'knot.example.com', ?, '', '', ?)`, pipeline.Id, repoDid.String(), string(payload)); err != nil { + if _, err := s.db.Exec(`insert into pipelines (pipeline_id, repo_did, commit_sha, kind, payload) values (?, ?, '', '', ?)`, pipeline.Id, repoDid.String(), string(payload)); err != nil { t.Fatal(err) } - if _, err := s.db.Exec(`insert into workflow_statuses (rkey, workflow, status, created_at) values (?, ?, 'pending', 'now')`, pipeline.Id, "w"+sfx); err != nil { + if _, err := s.db.Exec(`insert into workflow_statuses (pipeline_id, workflow, status, created_at) values (?, ?, 'pending', 'now')`, pipeline.Id, "w"+sfx); err != nil { t.Fatal(err) } @@ -262,7 +262,7 @@ func TestWipeRepoRemovesLocalEngineArtifacts(t *testing.T) { seedWipeRepo(t, s, repoDid, owner) - wid := models.WorkflowId{PipelineId: models.PipelineId{Knot: "knot.example.com", Rkey: "p1"}, Name: "w1"} + wid := models.WorkflowId{PipelineId: models.PipelineId("p1"), Name: "w1"} localRef := "logs/" + wid.String() + ".log" if err := s.db.SaveArtifactRef(wid.String(), repoDid.String(), wid, localRef, "h"); err != nil { t.Fatal(err) diff --git a/spindle/xrpc/ci_pipeline_subscribe_logs.go b/spindle/xrpc/ci_pipeline_subscribe_logs.go index b7c7b3126..23cb60ad8 100644 --- a/spindle/xrpc/ci_pipeline_subscribe_logs.go +++ b/spindle/xrpc/ci_pipeline_subscribe_logs.go @@ -11,7 +11,6 @@ import ( "time" "github.com/bluesky-social/indigo/atproto/atclient" - "github.com/bluesky-social/indigo/atproto/syntax" "github.com/gorilla/websocket" "tangled.org/core/api/tangled" "tangled.org/core/spindle/logview" @@ -24,13 +23,14 @@ func (x *Xrpc) HandleCiSubscribePipelineLogs(w http.ResponseWriter, r *http.Requ workflows = r.URL.Query()["workflows"] ) - pipeline, err := syntax.ParseTID(pipelineQuery) - if err != nil { - writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("pipeline parameter invalid: %s", pipelineQuery)}) + // pipeline ids are opaque, any format. WorkflowId.String() normalizes before + // it reaches a filesystem path, so only emptiness is rejected here. + if pipelineQuery == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "pipeline parameter is required"}) return } - x.handleSubscribeLogs(w, r, pipeline, workflows) + x.handleSubscribeLogs(w, r, models.PipelineId(pipelineQuery), workflows) } var wsUpgrader = websocket.Upgrader{ @@ -38,11 +38,11 @@ var wsUpgrader = websocket.Upgrader{ WriteBufferSize: 10_000, } -func (x *Xrpc) handleSubscribeLogs(w http.ResponseWriter, r *http.Request, pipeline syntax.TID, workflows []string) { +func (x *Xrpc) handleSubscribeLogs(w http.ResponseWriter, r *http.Request, pipeline models.PipelineId, workflows []string) { l := x.Logger.With("pipeline", pipeline, "workflows", workflows) - // 1. query the pipeline from database to get the knot. knot is used to locate the workflow log files. - tpl, knot, err := x.Db.GetPipelineWithKnot(r.Context(), pipeline.String()) + // 1. the pipeline supplies the workflow names to stream + tpl, err := x.Db.GetPipeline(r.Context(), pipeline) if err != nil { l.ErrorContext(r.Context(), "failed to find pipeline event", "err", err) writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "NotFound", Message: fmt.Sprintf("pipeline not found: %s", pipeline.String())}) @@ -134,10 +134,7 @@ func (x *Xrpc) handleSubscribeLogs(w http.ResponseWriter, r *http.Request, pipel defer wg.Done() wid := models.WorkflowId{ - PipelineId: models.PipelineId{ - Knot: knot, - Rkey: pipeline.String(), - }, + PipelineId: pipeline, Name: wfName, } diff --git a/spindle/xrpc/ci_pipeline_trigger_pipeline.go b/spindle/xrpc/ci_pipeline_trigger_pipeline.go index d2a8fc389..207875c7a 100644 --- a/spindle/xrpc/ci_pipeline_trigger_pipeline.go +++ b/spindle/xrpc/ci_pipeline_trigger_pipeline.go @@ -110,7 +110,7 @@ func (x *Xrpc) TriggerPipeline(w http.ResponseWriter, r *http.Request) { return } - pipelineAt, err := x.Trigger.TriggerManual(r.Context(), repoDid, sha, ref, input.Workflows, sourceRepo, pull, inputs) + pipelineId, err := x.Trigger.TriggerManual(r.Context(), repoDid, sha, ref, input.Workflows, sourceRepo, pull, inputs) if errors.Is(err, ErrNoMatchingWorkflows) { fail(xrpcerr.GenericError(err)) return @@ -121,7 +121,7 @@ func (x *Xrpc) TriggerPipeline(w http.ResponseWriter, r *http.Request) { } if err := writeJson(w, http.StatusOK, tangled.CiTriggerPipeline_Output{ - Pipeline: pipelineAt.String(), + Pipeline: pipelineId.String(), }); err != nil { l.ErrorContext(r.Context(), "failed to write response", "err", err) } diff --git a/spindle/xrpc/ci_query_pipelines.go b/spindle/xrpc/ci_query_pipelines.go index a7a7a8010..c738a5bc4 100644 --- a/spindle/xrpc/ci_query_pipelines.go +++ b/spindle/xrpc/ci_query_pipelines.go @@ -6,6 +6,7 @@ import ( "strconv" "tangled.org/core/api/tangled" + "tangled.org/core/spindle/models" xrpcerr "tangled.org/core/xrpc/errors" ) @@ -66,7 +67,7 @@ func (x *Xrpc) HandleCiGetPipeline(w http.ResponseWriter, r *http.Request) { return } - p, err := x.Db.GetPipeline(r.Context(), pipeline) + p, err := x.Db.GetPipeline(r.Context(), models.PipelineId(pipeline)) if err != nil { fail(xrpcerr.GenericError(err), http.StatusInternalServerError) return diff --git a/spindle/xrpc/pipeline_cancel_pipeline.go b/spindle/xrpc/pipeline_cancel_pipeline.go index 2a8f6844f..83b662f51 100644 --- a/spindle/xrpc/pipeline_cancel_pipeline.go +++ b/spindle/xrpc/pipeline_cancel_pipeline.go @@ -32,9 +32,10 @@ func (x *Xrpc) CancelPipeline(w http.ResponseWriter, r *http.Request) { return } - pipelineTid, err := syntax.ParseTID(input.Pipeline) - if err != nil { - fail(xrpcerr.GenericError(fmt.Errorf("invalid pipeline TID %q: %w", input.Pipeline, err))) + // pipeline ids are opaque, any format. the GetPipeline + repo-ownership check + // below is the real gate; it also covers existence. + if input.Pipeline == "" { + fail(xrpcerr.GenericError(fmt.Errorf("pipeline is required"))) return } @@ -43,15 +44,9 @@ func (x *Xrpc) CancelPipeline(w http.ResponseWriter, r *http.Request) { fail(xerr) return } - repo, err := x.Db.GetRepoByDid(repoDid) - if err != nil { - fail(xrpcerr.GenericError(fmt.Errorf("failed to get repo: %w", err))) - return - } - // the actor is only authorized against input.Repo, so make sure the // pipeline actually belongs to it before cancelling anything - p, err := x.Db.GetPipeline(r.Context(), pipelineTid.String()) + p, err := x.Db.GetPipeline(r.Context(), models.PipelineId(input.Pipeline)) if err != nil { fail(xrpcerr.GenericError(fmt.Errorf("failed to get pipeline: %w", err))) return @@ -61,11 +56,8 @@ func (x *Xrpc) CancelPipeline(w http.ResponseWriter, r *http.Request) { return } - pipelineId := models.PipelineId{ - Knot: repo.Knot, - Rkey: pipelineTid.String(), - } - l = l.With("input.pipeline", pipelineTid, "input.workflows", input.Workflows) + pipelineId := models.PipelineId(input.Pipeline) + l = l.With("input.pipeline", input.Pipeline, "input.workflows", input.Workflows) workflows := input.Workflows if len(workflows) == 0 { @@ -83,7 +75,7 @@ func (x *Xrpc) CancelPipeline(w http.ResponseWriter, r *http.Request) { for _, wName := range workflows { wid := models.WorkflowId{ PipelineId: pipelineId, - Name: wName, + Name: wName, } l.DebugContext(r.Context(), "cancel pipeline", "wid", wid) diff --git a/spindle/xrpc/xrpc.go b/spindle/xrpc/xrpc.go index b01c703b0..05c01a42a 100644 --- a/spindle/xrpc/xrpc.go +++ b/spindle/xrpc/xrpc.go @@ -46,7 +46,7 @@ func requireSha(sha string) error { // this is to break an import cycle. spindle imports this package for Xrpc, // so this package can't import *spindle.Spindle back. type PipelineTrigger interface { - TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) + TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull PullContext, inputs []*tangled.Pipeline_Pair) (models.PipelineId, error) DescribeWorkflowDefinition(ctx context.Context, repoDid syntax.DID, sha string, sourceRepo syntax.DID) (*tangled.CiDescribeWorkflowDefinition_Output, error) } diff --git a/spindle/xrpc/xrpc_test.go b/spindle/xrpc/xrpc_test.go index 8255ff66a..7778b9464 100644 --- a/spindle/xrpc/xrpc_test.go +++ b/spindle/xrpc/xrpc_test.go @@ -27,9 +27,9 @@ type mockTrigger struct { triggered bool } -func (m *mockTrigger) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull PullContext, inputs []*tangled.Pipeline_Pair) (syntax.ATURI, error) { +func (m *mockTrigger) TriggerManual(ctx context.Context, repoDid syntax.DID, sha, ref string, workflows []string, sourceRepo syntax.DID, pull PullContext, inputs []*tangled.Pipeline_Pair) (models.PipelineId, error) { m.triggered = true - return syntax.ParseATURI("at://did:plc:repoowner/sh.tangled.ci.pipeline/testrkey") + return models.PipelineId("testrkey"), nil } func (m *mockTrigger) DescribeWorkflowDefinition(context.Context, syntax.DID, string, syntax.DID) (*tangled.CiDescribeWorkflowDefinition_Output, error) { @@ -191,7 +191,7 @@ func TestCancelPipeline_RBAC(t *testing.T) { {Name: "test-workflow"}, }, } - err = d.CreatePipeline(models.PipelineId{Knot: "knot.test", Rkey: pipelineTid}, tpl) + err = d.CreatePipeline(models.PipelineId(pipelineTid), tpl) if err != nil { t.Fatalf("CreatePipeline: %v", err) }