diff --git a/.tangled/workflows/test.yml b/.tangled/workflows/test.yml index 69a1b62c..a31a616d 100644 --- a/.tangled/workflows/test.yml +++ b/.tangled/workflows/test.yml @@ -8,12 +8,28 @@ dependencies: nixpkgs: - go - gcc + - postgresql + - shadow + - util-linux steps: - name: patch static dir command: | mkdir -p appview/pages/static; touch appview/pages/static/x + - name: start postgres + command: | + set -euo pipefail + mkdir -p /pg/socket + useradd -r -m -d /pg/home pguser + chown -R pguser /pg + uid=$(id -u pguser); gid=$(id -g pguser) + setpriv --reuid "$uid" --regid "$gid" --init-groups -- initdb -D /pg/home/data --auth=trust --username=postgres + setpriv --reuid "$uid" --regid "$gid" --init-groups -- pg_ctl -D /pg/home/data -l /pg/home/pg.log -o "-p 5432 -k /pg/socket -h 127.0.0.1" start + for i in $(seq 1 20); do pg_isready -h 127.0.0.1 -p 5432 -U postgres && break; sleep 0.5; done + pg_isready -h 127.0.0.1 -p 5432 -U postgres + createdb -h 127.0.0.1 -p 5432 -U postgres mirror + - name: run linter environment: CGO_ENABLED: 1 @@ -23,5 +39,6 @@ steps: - name: run all tests environment: CGO_ENABLED: 1 + TEST_POSTGRES_URL: postgresql://postgres@127.0.0.1:5432/mirror?sslmode=disable command: | go test -v ./... diff --git a/go.mod b/go.mod index 1653f3b6..b6aa75a0 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,12 @@ go 1.25.0 require ( github.com/Blank-Xu/sql-adapter v1.1.1 + github.com/adrg/frontmatter v0.2.0 github.com/alecthomas/assert/v2 v2.11.0 github.com/alecthomas/chroma/v2 v2.23.1 github.com/avast/retry-go/v4 v4.6.1 github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 github.com/aws/aws-sdk-go-v2/credentials v1.19.12 github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 github.com/blevesearch/bleve/v2 v2.5.3 @@ -28,7 +30,6 @@ require ( github.com/go-chi/chi/v5 v5.2.0 github.com/go-enry/go-enry/v2 v2.9.2 github.com/go-git/go-git/v5 v5.14.0 - github.com/goki/freetype v1.0.5 github.com/google/uuid v1.6.0 github.com/gorilla/feeds v1.2.0 github.com/gorilla/sessions v1.4.0 @@ -48,6 +49,8 @@ require ( github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef github.com/stretchr/testify v1.11.1 + github.com/testcontainers/testcontainers-go v0.42.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/urfave/cli/v3 v3.6.2 github.com/whyrusleeping/cbor-gen v0.3.1 github.com/yuin/goldmark v1.7.13 @@ -58,21 +61,21 @@ require ( golang.org/x/crypto v0.48.0 golang.org/x/image v0.31.0 golang.org/x/net v0.50.0 + golang.org/x/sync v0.19.0 golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da gopkg.in/yaml.v3 v3.0.1 ) require ( - dario.cat/mergo v1.0.1 // indirect + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect - github.com/adrg/frontmatter v0.2.0 // indirect github.com/alecthomas/repr v0.5.2 // indirect github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect @@ -120,13 +123,16 @@ require ( github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect @@ -137,6 +143,7 @@ require ( github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-redis/cache/v9 v9.0.0 // indirect github.com/go-test/deep v1.1.1 // indirect github.com/goccy/go-json v0.10.5 // indirect @@ -176,15 +183,24 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.54.1 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -206,49 +222,48 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.2.0 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.etcd.io/bbolt v1.4.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect - go.opentelemetry.io/otel v1.40.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect - go.opentelemetry.io/otel/metric v1.40.0 // indirect - go.opentelemetry.io/otel/trace v1.40.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.41.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.12.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gotest.tools/v3 v3.5.2 // indirect lukechampine.com/blake3 v1.4.1 // indirect ) diff --git a/go.sum b/go.sum index aa038d8e..7f215548 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ -dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= -dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Blank-Xu/sql-adapter v1.1.1 h1:+g7QXU9sl/qT6Po97teMpf3GjAO0X9aFaqgSePXvYko= @@ -25,54 +27,32 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= -github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= -github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= -github.com/aws/aws-sdk-go-v2/credentials v1.19.9 h1:sWvTKsyrMlJGEuj/WgrwilpoJ6Xa1+KhIpGdzw7mMU8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.9/go.mod h1:+J44MBhmfVY/lETFiKI+klz0Vym2aCmIjqgClMmW82w= github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= @@ -83,8 +63,6 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= -github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= -github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -191,8 +169,14 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -205,8 +189,6 @@ github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WA github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/did-method-plc/go-didplc v0.0.0-20250716171643-635da8b4e038 h1:AGh+Vn9fXhf9eo8erG1CK4+LACduPo64P1OICQLDv88= -github.com/did-method-plc/go-didplc v0.0.0-20250716171643-635da8b4e038/go.mod h1:ddIXqTTSXWtj5kMsHAPj8SvbIx2GZdAkBFgFa6e6+CM= github.com/did-method-plc/go-didplc v0.2.2 h1:53HFhTT8NCAeFmZ6fdIZCf3PGDvj7A3cDjzOOEqn5XM= github.com/did-method-plc/go-didplc v0.2.2/go.mod h1:bKdJ21irnwNHgVLWWL32zUWqZueXYbJRUcxplZghByo= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= @@ -216,14 +198,16 @@ github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZ github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg= github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -264,6 +248,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -280,8 +266,6 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/goki/freetype v1.0.5 h1:yi2lQeUhXnBgSMqYd0vVmPw6RnnfIeTP3N4uvaJXd7A= -github.com/goki/freetype v1.0.5/go.mod h1:wKmKxddbzKmeci9K96Wknn5kjTWLyfC8tKOqAFbEX8E= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= @@ -307,6 +291,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -333,8 +318,8 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.8 h1:NpbJl/eVbvrGE0MJ6X16X9SAifesl6Fwxg/YmCvubRI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.8/go.mod h1:mi7YA+gCzVem12exXy46ZespvGtX/lZmD/RLnQhVW7U= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -411,8 +396,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -425,8 +410,14 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -435,6 +426,8 @@ github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6T github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk= github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI= +github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o= github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -443,10 +436,22 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -526,6 +531,8 @@ github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4 github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= github.com/posthog/posthog-go v1.5.5 h1:2o3j7IrHbTIfxRtj4MPaXKeimuTYg49onNzNBZbwksM= github.com/posthog/posthog-go v1.5.5/go.mod h1:3RqUmSnPuwmeVj/GYrS75wNGqcAKdpODiwc83xZWgdE= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -553,9 +560,11 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-envconfig v1.1.0 h1:cWZiJxeTm7AlCvzGXrEXaSTCNgip5oJepekh/BOQuog= github.com/sethvargo/go-envconfig v1.1.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= @@ -569,6 +578,8 @@ github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+g github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -579,6 +590,10 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -590,6 +605,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8= github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso= @@ -618,6 +637,8 @@ github.com/yuin/goldmark-emoji v1.0.6 h1:QWfF2FYaXwL74tfGOW5izeiZepUDroDJfWubQI9 github.com/yuin/goldmark-emoji v1.0.6/go.mod h1:ukxJDKFpdFb5x0a5HqbdlcKtebh086iJpI31LTKmWuA= github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc h1:+IAOyRda+RLrxa1WC7umKOZRsGq4QrFFMYApOeHzQwQ= github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc/go.mod h1:ovIvrum6DQJA4QsJSovrkC4saKHQVs7TvcaeO8AIl5I= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab h1:gK9tS6QJw5F0SIhYJnGG2P83kuabOdmWBbSmZhJkz2A= gitlab.com/staticnoise/goldmark-callout v0.0.0-20240609120641-6366b799e4ab/go.mod h1:SPu13/NPe1kMrbGoJldQwqtpNhXsmIuHCfm/aaGjU0c= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= @@ -632,20 +653,20 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= -go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= -go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= -go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= -go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0 h1:wVZXIWjQSeSmMoxF74LzAnpVQOAFDo3pPji9Y4SOFKc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0/go.mod h1:khvBS2IggMFNwZK/6lEeHg/W57h/IX6J4URh57fuI40= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= -go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= -go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -722,17 +743,20 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -748,8 +772,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -842,5 +866,7 @@ gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.4.1 h1:I3Smz7gso8w4/TunLKec6K2fn+kyKtDxr/xcQEN84Wg= lukechampine.com/blake3 v1.4.1/go.mod h1:QFosUxmjB8mnrWFSNwKmvxHpfY72bmD2tQ0kBMM3kwo= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= tangled.sh/oppi.li/go-gitdiff v0.8.2 h1:pASJJNWaFn6EmEIUNNjHZQ3stRu6BqTO2YyjKvTcxIc= tangled.sh/oppi.li/go-gitdiff v0.8.2/go.mod h1:WWAk1Mc6EgWarCrPFO+xeYlujPu98VuLW3Tu+B/85AE= diff --git a/knotmirror/db/db.go b/knotmirror/db/db.go index 22eda56e..f7a8f118 100644 --- a/knotmirror/db/db.go +++ b/knotmirror/db/db.go @@ -68,6 +68,17 @@ func Make(ctx context.Context, dbUrl string, maxConns int) (*sql.DB, error) { constraint hosts_pkey primary key (hostname) ); + create table if not exists resync_buffer ( + did text not null, + rkey text not null, + event_rkey text not null, + db_created_at timestamptz not null default now(), + + constraint resync_buffer_pkey primary key (did, rkey, event_rkey), + constraint resync_buffer_repo_fkey foreign key (did, rkey) + references repos (did, rkey) on delete cascade + ); + create index if not exists idx_repos_aturi on repos (at_uri); create index if not exists idx_repos_db_updated_at on repos (db_updated_at desc); create index if not exists idx_hosts_db_updated_at on hosts (db_updated_at desc); diff --git a/knotmirror/db/repos.go b/knotmirror/db/repos.go index 36313b1e..016c38d7 100644 --- a/knotmirror/db/repos.go +++ b/knotmirror/db/repos.go @@ -54,19 +54,6 @@ func UpsertRepo(ctx context.Context, e *sql.DB, repo *models.Repo) error { return nil } -func UpdateRepoState(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, state models.RepoState) error { - if _, err := e.ExecContext(ctx, - `update repos - set state = $1 - where did = $2 and rkey = $3`, - state, - did, rkey, - ); err != nil { - return fmt.Errorf("updating repo: %w", err) - } - return nil -} - func DeleteRepo(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error { if _, err := e.ExecContext(ctx, `delete from repos where did = $1 and rkey = $2`, diff --git a/knotmirror/db/resync_buffer.go b/knotmirror/db/resync_buffer.go new file mode 100644 index 00000000..4d0d362b --- /dev/null +++ b/knotmirror/db/resync_buffer.go @@ -0,0 +1,217 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotmirror/models" +) + +func BufferEvent(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, eventRkey string) error { + tx, err := e.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var state models.RepoState + if err := tx.QueryRowContext(ctx, + `select state from repos where did = $1 and rkey = $2 for update`, + did, rkey, + ).Scan(&state); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("locking repo: %w", err) + } + if state != models.RepoStateResyncing { + return nil + } + + if _, err := tx.ExecContext(ctx, + `insert into resync_buffer (did, rkey, event_rkey) + values ($1, $2, $3) + on conflict (did, rkey, event_rkey) do nothing`, + did, rkey, eventRkey, + ); err != nil { + return fmt.Errorf("buffering event: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + return nil +} + +func MarkDesync(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error { + tx, err := e.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var state models.RepoState + if err := tx.QueryRowContext(ctx, + `select state from repos where did = $1 and rkey = $2 for update`, + did, rkey, + ).Scan(&state); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("locking repo: %w", err) + } + switch state { + case models.RepoStateActive, models.RepoStateDesynchronized, models.RepoStateError: + default: + return nil + } + + if _, err := tx.ExecContext(ctx, + `update repos set state = $1 where did = $2 and rkey = $3`, + models.RepoStateDesynchronized, did, rkey, + ); err != nil { + return fmt.Errorf("marking desync: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + return nil +} + +func CompleteResync(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey) error { + tx, err := e.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var state models.RepoState + if err := tx.QueryRowContext(ctx, + `select state from repos where did = $1 and rkey = $2 for update`, + did, rkey, + ).Scan(&state); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("locking repo: %w", err) + } + if state != models.RepoStateResyncing { + return nil + } + + res, err := tx.ExecContext(ctx, + `delete from resync_buffer where did = $1 and rkey = $2`, + did, rkey, + ) + if err != nil { + return fmt.Errorf("draining buffer: %w", err) + } + deleted, _ := res.RowsAffected() + + finalState := models.RepoStateActive + if deleted > 0 { + finalState = models.RepoStateDesynchronized + } + + if _, err := tx.ExecContext(ctx, + `update repos + set error_msg = '', + retry_count = 0, + retry_after = 0, + state = $1 + where did = $2 and rkey = $3`, + finalState, did, rkey, + ); err != nil { + return fmt.Errorf("finalising repo: %w", err) + } + + return tx.Commit() +} + +func CancelResync(ctx context.Context, e *sql.DB, aturi syntax.ATURI) error { + tx, err := e.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var did syntax.DID + var rkey syntax.RecordKey + var state models.RepoState + if err := tx.QueryRowContext(ctx, + `select did, rkey, state from repos where at_uri = $1 for update`, + aturi, + ).Scan(&did, &rkey, &state); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("locking repo: %w", err) + } + if state != models.RepoStateResyncing { + return nil + } + + if _, err := tx.ExecContext(ctx, + `delete from resync_buffer where did = $1 and rkey = $2`, + did, rkey, + ); err != nil { + return fmt.Errorf("draining buffer: %w", err) + } + + if _, err := tx.ExecContext(ctx, + `update repos set state = $1 where did = $2 and rkey = $3`, + models.RepoStateSuspended, did, rkey, + ); err != nil { + return fmt.Errorf("suspending repo: %w", err) + } + + return tx.Commit() +} + +func FailResync(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, state models.RepoState, errMsg string, retryCount int, retryAfter int64) error { + tx, err := e.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin tx: %w", err) + } + defer tx.Rollback() + + var current models.RepoState + if err := tx.QueryRowContext(ctx, + `select state from repos where did = $1 and rkey = $2 for update`, + did, rkey, + ).Scan(¤t); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil + } + return fmt.Errorf("locking repo: %w", err) + } + if current != models.RepoStateResyncing { + return nil + } + + if _, err := tx.ExecContext(ctx, + `delete from resync_buffer where did = $1 and rkey = $2`, + did, rkey, + ); err != nil { + return fmt.Errorf("draining buffer: %w", err) + } + + if _, err := tx.ExecContext(ctx, + `update repos + set error_msg = $1, + retry_count = $2, + retry_after = $3, + state = $4 + where did = $5 and rkey = $6`, + errMsg, retryCount, retryAfter, state, + did, rkey, + ); err != nil { + return fmt.Errorf("finalising repo: %w", err) + } + + return tx.Commit() +} diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go index f9236c2e..64992c23 100644 --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -49,6 +49,10 @@ func Run(ctx context.Context, cfg *config.Config) error { } logger.Info(fmt.Sprintf("clearing resyning states: %d records updated", rows)) + if _, err := db.ExecContext(ctx, `delete from resync_buffer`); err != nil { + return fmt.Errorf("clearing resync buffer: %w", err) + } + knotstream := knotstream.NewKnotStream(logger, db, cfg) crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, cfg) @@ -119,13 +123,16 @@ func Run(ctx context.Context, cfg *config.Config) error { } logger.Info("shutting down knotmirror") - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 35*time.Second) defer shutdownCancel() var errs []error if err := knotstream.Shutdown(shutdownCtx); err != nil { errs = append(errs, err) } + if err := resyncer.Shutdown(shutdownCtx); err != nil { + errs = append(errs, err) + } if err := db.Close(); err != nil { errs = append(errs, err) } diff --git a/knotmirror/knotstream/slurper.go b/knotmirror/knotstream/slurper.go index b28ca1dc..f0b344f5 100644 --- a/knotmirror/knotstream/slurper.go +++ b/knotmirror/knotstream/slurper.go @@ -312,9 +312,8 @@ func (s *KnotSlurper) ProcessLegacyGitRefUpdate(ctx context.Context, source stri } l = l.With("repoAt", curr.AtUri()) - // TODO: should plan resync to resyncBuffer on RepoStateResyncing - if curr.State != models.RepoStateActive { - l.Debug("skipping non-active repo") + if curr.State == models.RepoStateSuspended || curr.State == models.RepoStatePending { + l.Debug("skipping repo", "state", curr.State) knotstreamEventsSkipped.Inc() return nil } @@ -325,18 +324,21 @@ func (s *KnotSlurper) ProcessLegacyGitRefUpdate(ctx context.Context, source stri return nil } - // if curr.State == models.RepoStateResyncing { - // firehoseEventsSkipped.Inc() - // return fp.events.addToResyncBuffer(ctx, commit) - // } - - // can't skip anything, update repo state - if err := db.UpdateRepoState(ctx, s.db, curr.Did, curr.Rkey, models.RepoStateDesynchronized); err != nil { - return err + switch curr.State { + case models.RepoStateResyncing: + if err := db.BufferEvent(ctx, s.db, curr.Did, curr.Rkey, evt.Rkey); err != nil { + return fmt.Errorf("buffering event: %w", err) + } + case models.RepoStateActive, models.RepoStateDesynchronized, models.RepoStateError: + if err := db.MarkDesync(ctx, s.db, curr.Did, curr.Rkey); err != nil { + return fmt.Errorf("marking desync: %w", err) + } + default: + knotstreamEventsSkipped.Inc() + return nil } l.Info("event processed", "eventRev", evt.Rkey) - knotstreamEventsProcessed.Inc() return nil } diff --git a/knotmirror/resyncer.go b/knotmirror/resyncer.go index 4357e8a0..e99af5f4 100644 --- a/knotmirror/resyncer.go +++ b/knotmirror/resyncer.go @@ -31,6 +31,8 @@ type Resyncer struct { runningJobs map[syntax.ATURI]context.CancelFunc runningJobsMu sync.Mutex + workersWg sync.WaitGroup + repoFetchTimeout time.Duration manualResyncTimeout time.Duration parallelism int @@ -39,6 +41,8 @@ type Resyncer struct { knotBackoffMu sync.RWMutex } +const finalizationTimeout = 30 * time.Second + func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config.Config) *Resyncer { return &Resyncer{ logger: log.SubLogger(l, "resyncer"), @@ -57,8 +61,32 @@ func NewResyncer(l *slog.Logger, db *sql.DB, gitm GitMirrorManager, cfg *config. } func (r *Resyncer) Start(ctx context.Context) { - for i := 0; i < r.parallelism; i++ { - go r.runResyncWorker(ctx, i) + for i := range r.parallelism { + r.workersWg.Add(1) + go func(id int) { + defer r.workersWg.Done() + r.runResyncWorker(ctx, id) + }(i) + } +} + +func (r *Resyncer) Shutdown(ctx context.Context) error { + r.runningJobsMu.Lock() + for _, cancel := range r.runningJobs { + cancel() + } + r.runningJobsMu.Unlock() + + done := make(chan struct{}) + go func() { + r.workersWg.Wait() + close(done) + }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() } } @@ -71,54 +99,85 @@ func (r *Resyncer) runResyncWorker(ctx context.Context, workerID int) { return default: } - repoAt, found, err := r.claimResyncJob(ctx) + repoAt, jobCtx, found, err := r.claimResyncJob(ctx) if err != nil { + if errors.Is(err, context.Canceled) { + return + } l.Error("failed to claim resync job", "error", err) - time.Sleep(time.Second) + if !sleepOrDone(ctx, time.Second) { + return + } continue } if !found { - time.Sleep(time.Second) + if !sleepOrDone(ctx, time.Second) { + return + } continue } l.Info("processing resync", "aturi", repoAt) - if err := r.resyncRepo(ctx, repoAt); err != nil { + if err := r.resyncRepo(jobCtx, repoAt); err != nil { l.Error("resync failed", "aturi", repoAt, "error", err) } } } -func (r *Resyncer) registerRunning(repo syntax.ATURI, cancel context.CancelFunc) { - r.runningJobsMu.Lock() - defer r.runningJobsMu.Unlock() - - if _, exists := r.runningJobs[repo]; exists { - return +func sleepOrDone(ctx context.Context, d time.Duration) bool { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-ctx.Done(): + return false + case <-t.C: + return true } - r.runningJobs[repo] = cancel } -func (r *Resyncer) unregisterRunning(repo syntax.ATURI) { +func (r *Resyncer) finalizeJob(repo syntax.ATURI) { r.runningJobsMu.Lock() defer r.runningJobsMu.Unlock() - delete(r.runningJobs, repo) + if cancel, ok := r.runningJobs[repo]; ok { + cancel() + delete(r.runningJobs, repo) + } } func (r *Resyncer) CancelResyncJob(repo syntax.ATURI) { - r.runningJobsMu.Lock() - defer r.runningJobsMu.Unlock() + r.claimJobMu.Lock() + defer r.claimJobMu.Unlock() - cancel, ok := r.runningJobs[repo] - if !ok { - return + ctx, cancel := context.WithTimeout(context.Background(), finalizationTimeout) + defer cancel() + if err := db.CancelResync(ctx, r.db, repo); err != nil { + r.logger.Error("failed to suspend repo on cancel", "at_uri", repo, "error", err) } - delete(r.runningJobs, repo) - cancel() + r.finalizeJob(repo) } // TriggerResyncJob manually triggers the resync job func (r *Resyncer) TriggerResyncJob(ctx context.Context, repoAt syntax.ATURI) error { + res, err := r.db.ExecContext(ctx, + `update repos + set state = $1, retry_after = $2 + where at_uri = $3 and state != $4`, + models.RepoStatePending, + int64(-1), + repoAt, + models.RepoStateResyncing, + ) + if err != nil { + return fmt.Errorf("triggering resync: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("triggering resync: %w", err) + } + if n > 0 { + return nil + } + repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) if err != nil { return fmt.Errorf("failed to get repo: %w", err) @@ -126,29 +185,14 @@ func (r *Resyncer) TriggerResyncJob(ctx context.Context, repoAt syntax.ATURI) er if repo == nil { return fmt.Errorf("repo not found: %s", repoAt) } - - if repo.State == models.RepoStateResyncing { - return fmt.Errorf("repo already resyncing") - } - - repo.State = models.RepoStatePending - repo.RetryAfter = -1 // resyncer will prioritize this - - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { - return fmt.Errorf("updating repo state to pending %w", err) - } - return nil + return fmt.Errorf("cannot trigger resync: repo in state %s", repo.State) } -func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, bool, error) { - // use mutex to prevent duplicated jobs +func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, context.Context, bool, error) { r.claimJobMu.Lock() defer r.claimJobMu.Unlock() - var repoAt syntax.ATURI - now := time.Now().Unix() - if err := r.db.QueryRowContext(ctx, - `update repos + const query = `update repos set state = $1 where at_uri = ( select at_uri from repos @@ -160,22 +204,29 @@ func (r *Resyncer) claimResyncJob(ctx context.Context) (syntax.ATURI, bool, erro retry_after limit 1 ) - returning at_uri - `, + returning at_uri` + + var repoAt syntax.ATURI + if err := r.db.QueryRowContext(ctx, query, models.RepoStateResyncing, models.RepoStatePending, models.RepoStateDesynchronized, models.RepoStateError, - now, + time.Now().Unix(), ).Scan(&repoAt); err != nil { if errors.Is(err, sql.ErrNoRows) { - return "", false, nil + return "", nil, false, nil } - return "", false, err + return "", nil, false, err } - return repoAt, true, nil + jobCtx, cancel := context.WithCancel(ctx) + r.runningJobsMu.Lock() + r.runningJobs[repoAt] = cancel + r.runningJobsMu.Unlock() + + return repoAt, jobCtx, true, nil } -func (r *Resyncer) resyncRepo(ctx context.Context, repoAt syntax.ATURI) error { +func (r *Resyncer) resyncRepo(jobCtx context.Context, repoAt syntax.ATURI) error { // ctx, span := tracer.Start(ctx, "resyncRepo") // span.SetAttributes(attribute.String("aturi", repoAt)) // defer span.End() @@ -183,15 +234,13 @@ func (r *Resyncer) resyncRepo(ctx context.Context, repoAt syntax.ATURI) error { resyncsStarted.Inc() startTime := time.Now() - jobCtx, cancel := context.WithCancel(ctx) - r.registerRunning(repoAt, cancel) - defer r.unregisterRunning(repoAt) + defer r.finalizeJob(repoAt) success, err := r.doResync(jobCtx, repoAt) if !success { resyncsFailed.Inc() resyncDuration.Observe(time.Since(startTime).Seconds()) - return r.handleResyncFailure(ctx, repoAt, err) + return r.handleResyncFailure(repoAt, err) } resyncsCompleted.Inc() @@ -199,12 +248,12 @@ func (r *Resyncer) resyncRepo(ctx context.Context, repoAt syntax.ATURI) error { return nil } -func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, error) { +func (r *Resyncer) doResync(jobCtx context.Context, repoAt syntax.ATURI) (bool, error) { // ctx, span := tracer.Start(ctx, "doResync") // span.SetAttributes(attribute.String("aturi", repoAt)) // defer span.End() - repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) + repo, err := db.GetRepoByAtUri(jobCtx, r.db, repoAt) if err != nil { return false, fmt.Errorf("failed to get repo: %w", err) } @@ -222,7 +271,7 @@ func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, err // HACK: check knot reachability with short timeout before running actual fetch. // This is crucial as git-cli doesn't support http connection timeout. // `http.lowSpeedTime` is only applied _after_ the connection. - if err := r.checkKnotReachability(ctx, repo); err != nil { + if err := r.checkKnotReachability(jobCtx, repo); err != nil { if isRateLimitError(err) { r.knotBackoffMu.Lock() r.knotBackoff[repo.KnotDomain] = time.Now().Add(10 * time.Second) @@ -237,7 +286,7 @@ func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, err if repo.RetryAfter == -1 { timeout = r.manualResyncTimeout } - fetchCtx, cancel := context.WithTimeout(ctx, timeout) + fetchCtx, cancel := context.WithTimeout(jobCtx, timeout) defer cancel() if err := r.gitm.Sync(fetchCtx, repo); err != nil { @@ -246,12 +295,10 @@ func (r *Resyncer) doResync(ctx context.Context, repoAt syntax.ATURI) (bool, err // repo.GitRev = // repo.RepoSha = - repo.State = models.RepoStateActive - repo.ErrorMsg = "" - repo.RetryCount = 0 - repo.RetryAfter = 0 - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { - return false, fmt.Errorf("updating repo state to active %w", err) + finCtx, finCancel := context.WithTimeout(context.Background(), finalizationTimeout) + defer finCancel() + if err := db.CompleteResync(finCtx, r.db, repo.Did, repo.Rkey); err != nil { + return false, fmt.Errorf("completing resync: %w", err) } return true, nil } @@ -316,38 +363,48 @@ func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) return nil } -func (r *Resyncer) handleResyncFailure(ctx context.Context, repoAt syntax.ATURI, err error) error { +func (r *Resyncer) handleResyncFailure(repoAt syntax.ATURI, err error) error { r.logger.Debug("handleResyncFailure", "at_uri", repoAt, "err", err) + + if errors.Is(err, context.Canceled) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), finalizationTimeout) + defer cancel() + + repo, getErr := db.GetRepoByAtUri(ctx, r.db, repoAt) + if getErr != nil { + return fmt.Errorf("failed to get repo: %w", getErr) + } + if repo == nil { + return fmt.Errorf("failed to get repo. repo '%s' doesn't exist in db", repoAt) + } + + if repo.State != models.RepoStateResyncing { + return nil + } + var state models.RepoState var errMsg string + var retryCount int + var retryAfter int64 if err == nil { state = models.RepoStateDesynchronized - errMsg = "" + retryCount = repo.RetryCount + retryAfter = time.Now().Add(10 * time.Second).Unix() } else { state = models.RepoStateError errMsg = err.Error() + retryCount = repo.RetryCount + 1 + // start a 1 min & go up to 1 hr between retries + retryAfter = time.Now().Add(backoff(retryCount, 60) * 60).Unix() } - repo, err := db.GetRepoByAtUri(ctx, r.db, repoAt) - if err != nil { - return fmt.Errorf("failed to get repo: %w", err) - } - if repo == nil { - return fmt.Errorf("failed to get repo. repo '%s' doesn't exist in db", repoAt) - } - - // start a 1 min & go up to 1 hr between retries - var retryCount = repo.RetryCount + 1 - var retryAfter = time.Now().Add(backoff(retryCount, 60) * 60).Unix() - // remove null bytes errMsg = strings.ReplaceAll(errMsg, "\x00", "") - repo.State = state - repo.ErrorMsg = errMsg - repo.RetryCount = retryCount - repo.RetryAfter = retryAfter - if err := db.UpsertRepo(ctx, r.db, repo); err != nil { + if err := db.FailResync(ctx, r.db, repo.Did, repo.Rkey, state, errMsg, retryCount, retryAfter); err != nil { return fmt.Errorf("failed to update repo state: %w", err) } return nil diff --git a/knotmirror/resyncer_test.go b/knotmirror/resyncer_test.go new file mode 100644 index 00000000..67fb3219 --- /dev/null +++ b/knotmirror/resyncer_test.go @@ -0,0 +1,294 @@ +package knotmirror_test + +import ( + "context" + "database/sql" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "sync" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" + + "tangled.org/core/api/tangled" + "tangled.org/core/knotmirror" + "tangled.org/core/knotmirror/config" + "tangled.org/core/knotmirror/db" + "tangled.org/core/knotmirror/knotstream" + "tangled.org/core/knotmirror/models" +) + +type blockingGitm struct { + mu sync.Mutex + calls []syntax.ATURI + started chan struct{} + release chan struct{} +} + +func newBlockingGitm() *blockingGitm { + return &blockingGitm{ + started: make(chan struct{}, 16), + release: make(chan struct{}, 16), + } +} + +func (f *blockingGitm) Exist(repo *models.Repo) (bool, error) { return true, nil } +func (f *blockingGitm) RemoteSetUrl(ctx context.Context, repo *models.Repo) error { return nil } +func (f *blockingGitm) Clone(ctx context.Context, repo *models.Repo) error { return nil } +func (f *blockingGitm) Fetch(ctx context.Context, repo *models.Repo) error { return nil } + +func (f *blockingGitm) Sync(ctx context.Context, repo *models.Repo) error { + f.mu.Lock() + f.calls = append(f.calls, repo.AtUri()) + f.mu.Unlock() + select { + case f.started <- struct{}{}: + case <-ctx.Done(): + return ctx.Err() + } + select { + case <-f.release: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (f *blockingGitm) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.calls) +} + +var _ knotmirror.GitMirrorManager = (*blockingGitm)(nil) + +func startPostgres(t *testing.T, ctx context.Context) *sql.DB { + t.Helper() + + if url := os.Getenv("TEST_POSTGRES_URL"); url != "" { + database, err := db.Make(ctx, url, 4) + require.NoError(t, err) + _, err = database.ExecContext(ctx, `truncate table repos, hosts, resync_buffer`) + require.NoError(t, err) + t.Cleanup(func() { + _ = database.Close() + }) + return database + } + + pg, err := postgres.Run(ctx, "docker.io/library/postgres:16-alpine", + postgres.WithDatabase("mirror"), + postgres.WithUsername("tnglr"), + postgres.WithPassword("test"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(60*time.Second), + ), + ) + if err != nil { + t.Skipf("postgres container unavailable, is podman/docker running: %v", err) + } + t.Cleanup(func() { + _ = pg.Terminate(context.Background()) + }) + + url, err := pg.ConnectionString(ctx, "sslmode=disable") + require.NoError(t, err) + + database, err := db.Make(ctx, url, 4) + require.NoError(t, err) + t.Cleanup(func() { + _ = database.Close() + }) + return database +} + +func testLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +func mockKnot(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-git-upload-pack-advertisement") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("001e# service=git-upload-pack\n")) + })) + t.Cleanup(srv.Close) + return srv +} + +func seedActiveRepo(t *testing.T, ctx context.Context, database *sql.DB, knotURL string) *models.Repo { + t.Helper() + repo := &models.Repo{ + Did: syntax.DID("did:plc:testingtestingtestingtest"), + Rkey: syntax.RecordKey("3kaaaaaaaaaaaa"), + Name: "race-repo", + KnotDomain: knotURL, + State: models.RepoStateActive, + } + require.NoError(t, db.UpsertRepo(ctx, database, repo)) + return repo +} + +func mkEvent(rkey string, oldSha, newSha string, repo *models.Repo) *knotstream.LegacyGitEvent { + owner := repo.Did.String() + repoDid := repo.Did.String() + return &knotstream.LegacyGitEvent{ + Rkey: rkey, + Nsid: tangled.GitRefUpdateNSID, + Event: tangled.GitRefUpdate{ + OwnerDid: &owner, + RepoName: repo.Name, + RepoDid: &repoDid, + OldSha: oldSha, + NewSha: newSha, + Ref: "refs/heads/main", + }, + } +} + +func TestEventDuringResyncTriggersSecondSync(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + database := startPostgres(t, ctx) + knotSrv := mockKnot(t) + repo := seedActiveRepo(t, ctx, database, knotSrv.URL) + + gitm := newBlockingGitm() + logger := testLogger() + cfg := &config.Config{ + KnotUseSSL: false, + GitRepoFetchTimeout: 10 * time.Second, + ResyncParallelism: 1, + } + + slurper := knotstream.NewKnotSlurper(logger, database, cfg.Slurper) + resyncer := knotmirror.NewResyncer(logger, database, gitm, cfg) + + eventA := mkEvent("3evtaaaaaaaaaa", "0000000000000000000000000000000000000000", "1111111111111111111111111111111111111111", repo) + require.NoError(t, slurper.ProcessLegacyGitRefUpdate(ctx, "testsrc", eventA)) + + got, err := db.GetRepoByAtUri(ctx, database, repo.AtUri()) + require.NoError(t, err) + require.Equal(t, models.RepoStateDesynchronized, got.State, "event A should mark repo desynchronized") + + resyncer.Start(ctx) + t.Cleanup(func() { + cancel() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + _ = resyncer.Shutdown(shutdownCtx) + }) + + select { + case <-gitm.started: + case <-time.After(10 * time.Second): + t.Fatal("timeout waiting for first Sync to start") + } + + got, err = db.GetRepoByAtUri(ctx, database, repo.AtUri()) + require.NoError(t, err) + require.Equal(t, models.RepoStateResyncing, got.State, "state should be resyncing while fetch is in flight") + + eventB := mkEvent("3evtbbbbbbbbbb", "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", repo) + require.NoError(t, slurper.ProcessLegacyGitRefUpdate(ctx, "testsrc", eventB)) + + gitm.release <- struct{}{} + + select { + case <-gitm.started: + gitm.release <- struct{}{} + case <-time.After(10 * time.Second): + t.Fatalf("event B arriving during resync should trigger a second Sync, got %d total Sync calls", gitm.callCount()) + } + + require.Eventually(t, func() bool { + got, err := db.GetRepoByAtUri(ctx, database, repo.AtUri()) + return err == nil && got.State == models.RepoStateActive + }, 10*time.Second, 50*time.Millisecond, "repo should end in active state") + + require.GreaterOrEqual(t, gitm.callCount(), 2, "expected at least 2 Sync calls") +} + +func TestConcurrentWorkersClaimDistinctRepos(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + database := startPostgres(t, ctx) + knotSrv := mockKnot(t) + + repoA := &models.Repo{ + Did: syntax.DID("did:plc:aaaaaaaaaaaaaaaaaaaaaaaa"), + Rkey: syntax.RecordKey("3kaaaaaaaaaaaa"), + Name: "repo-a", + KnotDomain: knotSrv.URL, + State: models.RepoStateDesynchronized, + } + repoB := &models.Repo{ + Did: syntax.DID("did:plc:bbbbbbbbbbbbbbbbbbbbbbbb"), + Rkey: syntax.RecordKey("3kbbbbbbbbbbbb"), + Name: "repo-b", + KnotDomain: knotSrv.URL, + State: models.RepoStateDesynchronized, + } + require.NoError(t, db.UpsertRepo(ctx, database, repoA)) + require.NoError(t, db.UpsertRepo(ctx, database, repoB)) + + gitm := newBlockingGitm() + logger := testLogger() + cfg := &config.Config{ + KnotUseSSL: false, + GitRepoFetchTimeout: 10 * time.Second, + ResyncParallelism: 2, + } + + resyncer := knotmirror.NewResyncer(logger, database, gitm, cfg) + resyncer.Start(ctx) + t.Cleanup(func() { + cancel() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer shutdownCancel() + _ = resyncer.Shutdown(shutdownCtx) + }) + + for range 2 { + select { + case <-gitm.started: + case <-time.After(10 * time.Second): + t.Fatalf("timeout waiting for concurrent Syncs, got %d calls", gitm.callCount()) + } + } + + gitm.mu.Lock() + claimed := map[syntax.ATURI]int{} + for _, c := range gitm.calls { + claimed[c]++ + } + gitm.mu.Unlock() + require.Len(t, claimed, 2, "two distinct repos should be claimed concurrently") + require.Equal(t, 1, claimed[repoA.AtUri()]) + require.Equal(t, 1, claimed[repoB.AtUri()]) + + gitm.release <- struct{}{} + gitm.release <- struct{}{} + + require.Eventually(t, func() bool { + gotA, errA := db.GetRepoByAtUri(ctx, database, repoA.AtUri()) + gotB, errB := db.GetRepoByAtUri(ctx, database, repoB.AtUri()) + return errA == nil && errB == nil && + gotA != nil && gotB != nil && + gotA.State == models.RepoStateActive && + gotB.State == models.RepoStateActive + }, 10*time.Second, 50*time.Millisecond, "both repos should end in active state") +} diff --git a/knotmirror/xrpc/proxy.go b/knotmirror/xrpc/proxy.go index f63755f2..9fe99683 100644 --- a/knotmirror/xrpc/proxy.go +++ b/knotmirror/xrpc/proxy.go @@ -49,7 +49,12 @@ func (x *Xrpc) resolveKnot(ctx context.Context, repoAt syntax.ATURI) (*knotInfo, if err == nil && repo != nil { if repo.State != models.RepoStatePending && repo.State != models.RepoStateResyncing { go func() { - if err := db.UpdateRepoState(context.Background(), x.db, repo.Did, repo.Rkey, models.RepoStatePending); err != nil { + if _, err := x.db.ExecContext(context.Background(), + `update repos set state = $1 + where did = $2 and rkey = $3 and state not in ($4, $5)`, + models.RepoStatePending, repo.Did, repo.Rkey, + models.RepoStatePending, models.RepoStateResyncing, + ); err != nil { x.logger.Error("failed to mark repo for resync after proxy", "err", err) } }()