From e952dd1d1dafdf3cc85ccb5e8e6e612868017f61 Mon Sep 17 00:00:00 2001 From: jobala Date: Mon, 23 Mar 2026 12:11:28 +0000 Subject: [PATCH] spindle/engine: store workflow logs in s3 --- go.mod | 31 +++++++++++++++++++------------ go.sum | 38 ++++++++++++++++++++++++++++++++++++++ nix/gomod2nix.toml | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------ nix/modules/spindle.nix | 10 ++++++++++ spindle/config/config.go | 5 +++++ spindle/engine/engine.go | 27 +++++++++++++++++++++++++++ spindle/engine/s3.go | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 file(s) changed, 224 insertion(s)(+), 36 deletion(s)(-) diff --git a/go.mod b/go.mod --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ github.com/alecthomas/assert/v2 v2.11.0 github.com/alecthomas/chroma/v2 v2.23.1 github.com/avast/retry-go/v4 v4.6.1 - github.com/aws/aws-sdk-go-v2 v1.41.1 - github.com/aws/aws-sdk-go-v2/credentials v1.19.9 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 github.com/blevesearch/bleve/v2 v2.5.3 github.com/bluekeyes/go-gitdiff v0.8.1 github.com/bluesky-social/indigo v0.0.0-20260220055544-bf41e2ee75ab @@ -70,15 +70,22 @@ github.com/adrg/frontmatter v0.2.0 // indirect github.com/alecthomas/repr v0.5.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect - github.com/aws/smithy-go v1.24.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum --- a/go.sum +++ b/go.sum @@ -27,28 +27,66 @@ github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= github.com/aws/aws-sdk-go-v2/credentials v1.19.9 h1:sWvTKsyrMlJGEuj/WgrwilpoJ6Xa1+KhIpGdzw7mMU8= github.com/aws/aws-sdk-go-v2/credentials v1.19.9/go.mod h1:+J44MBhmfVY/lETFiKI+klz0Vym2aCmIjqgClMmW82w= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= diff --git a/nix/gomod2nix.toml b/nix/gomod2nix.toml --- a/nix/gomod2nix.toml +++ b/nix/gomod2nix.toml @@ -7,6 +7,9 @@ [mod."github.com/Blank-Xu/sql-adapter"] version = "v1.1.1" hash = "sha256-9AiQhXoNPCiViV+p5aa3qGFkYU4rJNbADvNdYGq4GA4=" + [mod."github.com/BurntSushi/toml"] + version = "v0.3.1" + hash = "sha256-Rqak1dE/Aj/+Kx1/pl3Hifgt+Q3OzuZ5fJR+/x3nTbo=" [mod."github.com/Microsoft/go-winio"] version = "v0.6.2" hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU=" @@ -16,6 +19,9 @@ [mod."github.com/RoaringBitmap/roaring/v2"] version = "v2.4.5" hash = "sha256-igWY0S1PTolQkfctYcmVJioJyV1pk2V81X6o6BA1XQA=" + [mod."github.com/adrg/frontmatter"] + version = "v0.2.0" + hash = "sha256-WJsVcdCpkIkjqUz5fJOFStZYwQlrcFzQ6+mZatZiimo=" [mod."github.com/alecthomas/assert/v2"] version = "v2.11.0" hash = "sha256-tDJCDKZ0R4qNA7hgMKWrpDyogt1802LCJDBCExxdqaU=" @@ -33,41 +39,62 @@ version = "v4.6.1" hash = "sha256-PeZc8k4rDV64+k8nZt/oy1YNVbLevltXP3ZD1jf6Z6k=" [mod."github.com/aws/aws-sdk-go-v2"] - version = "v1.41.1" - hash = "sha256-umafTZB+cuy8+Kzpl2WrlygJO3tR3D2WkTC0eTY5G/g=" + version = "v1.41.4" + hash = "sha256-k9xv4f8YPSzZ1yR3/zuyNDGenZKk0DD4lceL713yXtc=" [mod."github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream"] - version = "v1.7.4" - hash = "sha256-ZY/Jn1p0IgDe8MONhp0RFHZmRgTBZZ5ddqXlNWEo7Ys=" + version = "v1.7.7" + hash = "sha256-5hw1g8s21Gx1Q3JtiaVM6q8qGt7DW1YO9U1dEEqRXfE=" + [mod."github.com/aws/aws-sdk-go-v2/config"] + version = "v1.32.12" + hash = "sha256-aTkdSRe8KPmVZdsunU8j/hZQLhGw1ckKpLN/ryRBZM0=" [mod."github.com/aws/aws-sdk-go-v2/credentials"] - version = "v1.19.9" - hash = "sha256-eqM5BmetQ/MrxTwoUCqRpkm5frFAspuq+QWod+5wzrU=" + version = "v1.19.12" + hash = "sha256-xEIT1ARA9RYrQtLZIus71E6niNHIOVM1J7mUnA5AhJQ=" + [mod."github.com/aws/aws-sdk-go-v2/feature/ec2/imds"] + version = "v1.18.20" + hash = "sha256-dCTpdKZheVCSt+R+NnFOnlS0bCt4gPavlDh15Kl/sMQ=" [mod."github.com/aws/aws-sdk-go-v2/internal/configsources"] - version = "v1.4.17" - hash = "sha256-W+d0WDYBrVJxdfRSvoe3cvZYgIgUSVyXsVdQlMcIZvc=" + version = "v1.4.20" + hash = "sha256-aATIk4oLd7aaV66ereBdjINLMDwmIHxu+NNsgKWH1t4=" [mod."github.com/aws/aws-sdk-go-v2/internal/endpoints/v2"] - version = "v2.7.17" - hash = "sha256-dKJx0+K1DKi2LJKQNwnTofZH4GWLDgY6/ZI2XR2oCGI=" + version = "v2.7.20" + hash = "sha256-G6266uj64sgfDTJ9V1UY1sQs3UmryB0CFgxzmbjjChY=" + [mod."github.com/aws/aws-sdk-go-v2/internal/ini"] + version = "v1.8.6" + hash = "sha256-oIRPqu99vnGINAWKnCEytpv7N0gRWO7S72tb1r8oxvk=" [mod."github.com/aws/aws-sdk-go-v2/internal/v4a"] - version = "v1.4.17" - hash = "sha256-PKwNn9nFf+7TqocgQprxP9r1Fs4IxwGLGd05MrsxmLg=" + version = "v1.4.21" + hash = "sha256-Sq4kRaiIFkPqwYnKv1dsxplyX4duipUYGFFT5gzJtJM=" [mod."github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding"] - version = "v1.13.4" - hash = "sha256-Rm6czqOnOULP080D97WQQSqkBhmN6ei1qZaTa51SRj8=" + version = "v1.13.7" + hash = "sha256-AfYJdpmnW01Bk/jfHATlNU6lddjqcigFkHw/zcT9WO4=" [mod."github.com/aws/aws-sdk-go-v2/service/internal/checksum"] - version = "v1.9.8" - hash = "sha256-DZhR0aqHrgAFBGSlnsSQ6XeAJ/q504RG/LBWhtQqRVg=" + version = "v1.9.12" + hash = "sha256-gze5CLp6Cp3X1jdqawWlXG7LToH+L6L/79b8F/AgWIY=" [mod."github.com/aws/aws-sdk-go-v2/service/internal/presigned-url"] - version = "v1.13.17" - hash = "sha256-qS7Db9S/KQ24kvJdL8qH4gnoN116J1ezwOnbovCiFEI=" + version = "v1.13.20" + hash = "sha256-a5TifKunIoqKd2uAceYh6F1LvMHMyEQcWvJf0sxKhPM=" [mod."github.com/aws/aws-sdk-go-v2/service/internal/s3shared"] - version = "v1.19.17" - hash = "sha256-x/Cb4j3HFlg1+U21YCAIhBnfpf1yehJU2Ss/PxamEMI=" + version = "v1.19.20" + hash = "sha256-T+pXMFELor69MTIo67TIkcWWyK+wubweg0gS5I9Dp8A=" [mod."github.com/aws/aws-sdk-go-v2/service/s3"] - version = "v1.96.0" - hash = "sha256-lzAn2KHIkd742mDwEZqGEaIXlEfvltL6HdP+sEQ/8YA=" + version = "v1.97.1" + hash = "sha256-J51tttUIPFVEnHddftY4CJXKe2WLLIQDhJ8eYM8jgts=" + [mod."github.com/aws/aws-sdk-go-v2/service/signin"] + version = "v1.0.8" + hash = "sha256-o4pWg3yMZHxdI94x5Z6qbiRg7gpmzbpJnJWsR1BOc44=" + [mod."github.com/aws/aws-sdk-go-v2/service/sso"] + version = "v1.30.13" + hash = "sha256-V277a0ikm/H0paIeDLtPGEyav2a69Kdb9d5bh+JLAeY=" + [mod."github.com/aws/aws-sdk-go-v2/service/ssooidc"] + version = "v1.35.17" + hash = "sha256-r5V5DoCIR4yzN1Ttg+dIA85GVkWMPgeD6Zu0rWGqNJE=" + [mod."github.com/aws/aws-sdk-go-v2/service/sts"] + version = "v1.41.9" + hash = "sha256-I15uxeoKxDURsZrEVDzCRtVIu/HE756M1Rt7PPpdZ7c=" [mod."github.com/aws/smithy-go"] - version = "v1.24.0" - hash = "sha256-ZPFhf2Yv3BQpUn3cN4wSnoO7uBki8oCisZxL6F09nnE=" + version = "v1.24.2" + hash = "sha256-v0y+Lir61fgdCwdVoca5mK+FcGh9OD3cTEwHIfLytcI=" [mod."github.com/aymanbagabas/go-osc52/v2"] version = "v2.0.1" hash = "sha256-6Bp0jBZ6npvsYcKZGHHIUSVSTAMEyieweAX2YAKDjjg=" @@ -696,6 +723,9 @@ [mod."gopkg.in/warnings.v0"] version = "v0.1.2" hash = "sha256-ATVL9yEmgYbkJ1DkltDGRn/auGAjqGOfjQyBYyUo8s8=" + [mod."gopkg.in/yaml.v2"] + version = "v2.4.0" + hash = "sha256-uVEGglIedjOIGZzHW4YwN1VoRSTK8o0eGZqzd+TNdd0=" [mod."gopkg.in/yaml.v3"] version = "v3.0.1" hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU=" diff --git a/nix/modules/spindle.nix b/nix/modules/spindle.nix --- a/nix/modules/spindle.nix +++ b/nix/modules/spindle.nix @@ -109,6 +109,12 @@ default = "5m"; description = "Timeout for each step of a pipeline"; }; + + logBucket = mkOption { + type = types.str; + default = "tangled-logs"; + description = "S3 bucket for workflow logs"; + }; }; }; }; @@ -138,6 +144,10 @@ "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" + "SPINDLE_NIXERY_PIPELINES_LOG_BUCKET=${cfg.pipelines.logBucket}" + "AWS_ACCESS_KEY_ID=${builtins.getEnv "AWS_ACCESS_KEY_ID"}" + "AWS_SECRET_ACCESS_KEY=${builtins.getEnv "AWS_SECRET_ACCESS_KEY"}" + "AWS_REGION=${builtins.getEnv "AWS_REGION"}" ]; ExecStart = "${cfg.package}/bin/spindle"; Restart = "always"; diff --git a/spindle/config/config.go b/spindle/config/config.go --- a/spindle/config/config.go +++ b/spindle/config/config.go @@ -41,9 +41,14 @@ WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` } +type S3 struct { + LogBucket string `env:"LOG_BUCKET"` +} + type Config struct { Server Server `env:",prefix=SPINDLE_SERVER_"` NixeryPipelines NixeryPipelines `env:",prefix=SPINDLE_NIXERY_PIPELINES_"` + S3 S3 `env:",prefix=SPINDLE_S3_"` } func Load(ctx context.Context) (*Config, error) { diff --git a/spindle/engine/engine.go b/spindle/engine/engine.go --- a/spindle/engine/engine.go +++ b/spindle/engine/engine.go @@ -3,7 +3,9 @@ import ( "context" "errors" + "fmt" "log/slog" + "path/filepath" "sync" securejoin "github.com/cyphar/filepath-securejoin" @@ -49,6 +51,17 @@ PipelineId: pipelineId, Name: w.Name, } + + defer func() { + logBucket := cfg.S3.LogBucket + + if logBucket != "" { + logFile := filepath.Join(cfg.Server.LogDir, fmt.Sprintf("%s.log", wid.String())) + if err := uploadWorkflowLogs(ctx, logFile, "tangled-demo"); err != nil { + l.Error("error uploading logs", "err", err) + } + } + }() wfLogger, err := models.NewFileWorkflowLogger(cfg.Server.LogDir, wid, secretValues) if err != nil { @@ -130,4 +143,18 @@ wg.Wait() l.Info("all workflows completed") +} + +func uploadWorkflowLogs(ctx context.Context, logfile, bucket string) error { + s3, err := NewS3(bucket) + if err != nil { + return fmt.Errorf("error creating s3 client: %w", err) + } + + name := filepath.Join(logfile) + if err := s3.WriteFile(ctx, name); err != nil { + return fmt.Errorf("error saving logs: %w", err) + } + + return nil } diff --git a/spindle/engine/s3.go b/spindle/engine/s3.go new file mode 100644 --- /dev/null +++ b/spindle/engine/s3.go @@ -0,0 +1,71 @@ +package engine + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type S3 struct { + bucket string + client *s3.Client +} + +const BASE_S3_PATH = "spindle/workflows" + +func NewS3(bucket string) (*S3, error) { + ctx := context.Background() + sdkConfig, err := config.LoadDefaultConfig(ctx) + + if err != nil { + return nil, fmt.Errorf("error loading s3 config: %w", err) + } + s3Client := s3.NewFromConfig(sdkConfig) + + return &S3{ + bucket: bucket, + client: s3Client, + }, nil +} + +func (s *S3) WriteFile(ctx context.Context, path string) error { + s3_key := fmt.Sprintf("%s/%s", BASE_S3_PATH, filepath.Base(path)) + + file, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening file %s: %w", path, err) + } + defer file.Close() + + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &s.bucket, + Key: &s3_key, + Body: file, + }) + + if err != nil { + return fmt.Errorf("error writing to s3: %w", err) + } + + return nil +} + +func (s *S3) ReadFile(ctx context.Context, name string) ([]byte, error) { + res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &s.bucket, + Key: aws.String(name), + }) + + if err != nil { + return nil, fmt.Errorf("error reading file %s: %w", name, err) + } + defer res.Body.Close() + + return io.ReadAll(res.Body) +} -- tangled.sh