Something went wrong. Try again.
Spindle fork for filtered Stream ingestion and bounded PDS reconciliation on a home CI runner
Something went wrong. Try again.
spindle docker-compose.yml
13 kB · 423 lines
YAML
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424services: redis: image: redis:7-alpine restart: unless-stopped networks: [tngl]
postgres: image: postgres:14-alpine restart: unless-stopped environment: POSTGRES_USER: tnglr POSTGRES_PASSWORD: tnglr volumes: - postgres-data:/var/lib/postgresql/data - ./localinfra/postgres-init.sql:/docker-entrypoint-initdb.d/init.sql healthcheck: test: ["CMD-SHELL", "pg_isready -U tnglr -d plc"] interval: 2s timeout: 2s retries: 30 start_period: 5s networks: [tngl]
pds: image: ghcr.io/bluesky-social/pds:0.4.219 restart: unless-stopped env_file: localinfra/pds.env environment: PDS_DID_PLC_URL: http://plc:8080 NODE_EXTRA_CA_CERTS: /caddy-ca/root.crt volumes: - pds-data:/pds - ./localinfra/certs/root.crt:/caddy-ca/root.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:3000/xrpc/_health"] interval: 2s timeout: 2s retries: 30 start_period: 5s depends_on: plc: condition: service_started networks: [tngl]
init-accounts: image: alpine:3.22 restart: "no" env_file: localinfra/pds.env environment: PDS_URL: http://pds:3000 OWNER_USER: alice KNOT_HOSTNAME: knot.tngl.boltless.dev SPINDLE_HOSTNAME: spindle.tngl.boltless.dev volumes: - ./localinfra/scripts/init-accounts.sh:/init.sh:ro - init-state:/shared command: sh -c "apk add --no-cache curl jq >/dev/null && sh /init.sh" depends_on: pds: condition: service_healthy networks: [tngl]
plc: platform: linux/amd64 image: ghcr.io/bluesky-social/did-method-plc:plc-f2ab7516bac5bc0f3f86842fa94e996bd1b3815b restart: unless-stopped environment: DEBUG_MODE: "1" LOG_ENABLED: "true" LOG_LEVEL: debug LOG_DESTINATION: "1" PLC_VERSION: 0.0.1 PORT: "8080" DB_CREDS_JSON: &DB_CREDS_JSON '{"username":"tnglr","password":"tnglr","host":"postgres","port":5432}' DB_MIGRATE_CREDS_JSON: *DB_CREDS_JSON depends_on: [postgres] networks: [tngl]
jetstream: platform: linux/amd64 image: ghcr.io/bluesky-social/jetstream:sha-d5a3b62 restart: unless-stopped environment: JETSTREAM_DATA_DIR: /data JETSTREAM_LIVENESS_TTL: 300s JETSTREAM_WS_URL: wss://pds.tngl.boltless.dev/xrpc/com.atproto.sync.subscribeRepos volumes: - jetstream-data:/data - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: pds: condition: service_healthy networks: [tngl]
knot: build: context: . dockerfile: localinfra/knot.Dockerfile restart: unless-stopped environment: KNOT_SERVER_HOSTNAME: knot.tngl.boltless.dev KNOT_SERVER_LISTEN_ADDR: 0.0.0.0:5555 KNOT_SERVER_INTERNAL_LISTEN_ADDR: 127.0.0.1:5444 KNOT_SERVER_DB_PATH: /home/git/knotserver.db KNOT_SERVER_PLC_URL: https://plc.tngl.boltless.dev KNOT_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe KNOT_SERVER_DEV: "false" KNOT_REPO_SCAN_PATH: /home/git/repositories APPVIEW_ENDPOINT: https://tangled.org KNOT_MIRRORS: https://mirror.tngl.boltless.dev ports: - "2222:22" volumes: - knot-data:/home/git - knot-ssh-keys:/etc/ssh/keys - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:5555/"] interval: 2s timeout: 2s retries: 60 start_period: 30s depends_on: plc: condition: service_started jetstream: condition: service_started knotmirror: condition: service_healthy init-accounts: condition: service_completed_successfully networks: [tngl]
ncps-migrate: image: &ncps-image ghcr.io/kalbasit/ncps:v0.9.4 profiles: ["linux"] restart: "no" command: - /bin/dbmate - --migrations-dir=/share/ncps/db/migrations/sqlite - --no-dump-schema - --url=sqlite:/storage/db.sqlite - up volumes: - ncps-data:/storage networks: [tngl]
ncps: image: *ncps-image profiles: ["linux"] restart: unless-stopped environment: ANALYTICS_REPORTING_ENABLED: "false" command: ["/bin/ncps", "serve", "--config", "/etc/ncps/ncps.yaml"] volumes: - ncps-data:/storage - ./localinfra/ncps.yaml:/etc/ncps/ncps.yaml:ro - ./localinfra/ncps-secret-key:/keys/ncps-secret-key:ro depends_on: ncps-migrate: condition: service_completed_successfully networks: [tngl]
spindle: profiles: ["linux"] build: context: . dockerfile: localinfra/spindle.Dockerfile restart: unless-stopped environment: SPINDLE_SERVER_HOSTNAME: spindle.tngl.boltless.dev SPINDLE_SERVER_LISTEN_ADDR: 0.0.0.0:6555 SPINDLE_SERVER_DB_PATH: /var/lib/spindle/spindle.db SPINDLE_SERVER_PLC_URL: https://plc.tngl.boltless.dev SPINDLE_SERVER_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe SPINDLE_SERVER_DEV: "true" SPINDLE_SERVER_DEV_EXTRA_HOSTS: knot.tngl.boltless.dev,mirror.tngl.boltless.dev SPINDLE_SERVER_TAP_DB_PATH: /var/lib/spindle/tap.db SPINDLE_SERVER_TAP_RELAY_URL: https://pds.tngl.boltless.dev SPINDLE_MICROVM_PIPELINES_IMAGE_DIR: /var/lib/spindle/images SPINDLE_MICROVM_PIPELINES_OVERLAY_DIR: /var/lib/spindle/overlays SPINDLE_MICROVM_PIPELINES_AGENT_PORT: "11240" SPINDLE_S3_LOG_BUCKET: "" SPINDLE_MICROVM_PIPELINES_ENABLE_CGROUPS: "false" # route guest nix substitution + uploads through the local ncps cache. # ncps re-signs on serve with cache.local's key, so the guest trusts the # matching public key below (no signing happens in spindle itself). SPINDLE_NIX_CACHE_READ_URLS: http://ncps:8501 SPINDLE_NIX_CACHE_TRUSTED_PUBLIC_KEYS: cache.local:F7YqpMzuBdILYd/v+wMZN2YKxCzliXQyFmeezOxw7rU= SPINDLE_NIX_CACHE_UPLOAD_URL: http://ncps:8501/upload # these two are required for cgroups, uncomment if testing # privileged: true # cgroup: host devices: - /dev/vsock:/dev/vsock - /dev/kvm:/dev/kvm - /dev/vhost-vsock:/dev/vhost-vsock - /dev/net/tun:/dev/net/tun cap_add: - NET_ADMIN - SYS_ADMIN security_opt: - label=disable - seccomp=unconfined volumes: - spindle-data:/var/lib/spindle - spindle-logs:/var/log/spindle - ./out/localinfra-spindle-images:/var/lib/spindle/images:ro - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:6555/"] interval: 2s timeout: 2s retries: 30 start_period: 5s depends_on: plc: condition: service_started jetstream: condition: service_started init-accounts: condition: service_completed_successfully ncps: condition: service_started networks: [tngl]
knotmirror-tap: image: ghcr.io/bluesky-social/indigo/tap:sha-4f47add43060c27e8a37d9d76482ecddf001fcd8 # 0.1.10 restart: unless-stopped environment: TAP_BIND: ":2480" TAP_PLC_URL: https://plc.tngl.boltless.dev TAP_RELAY_URL: https://pds.tngl.boltless.dev # PDS can be used as basic relay without collectiondir. TAP_DATABASE_URL: postgres://tnglr:tnglr@postgres:5432/mirror_tap?sslmode=disable TAP_COLLECTION_FILTERS: sh.tangled.repo TAP_SIGNAL_COLLECTION: sh.tangled.repo TAP_RESYNC_PARALLELISM: "10" TAP_RETRY_TIMEOUT: 60s volumes: - ./localinfra/certs/root.crt:/etc/ssl/certs/ca-certificates.crt:ro depends_on: postgres: condition: service_started pds: condition: service_healthy networks: [tngl]
knotmirror: build: context: . dockerfile: localinfra/knotmirror.Dockerfile restart: unless-stopped environment: MIRROR_LISTEN: 0.0.0.0:7000 MIRROR_ADMIN_LISTEN: 0.0.0.0:7200 MIRROR_HOSTNAME: mirror.tngl.boltless.dev MIRROR_TAP_URL: http://knotmirror-tap:2480 MIRROR_DB_URL: postgres://tnglr:tnglr@postgres:5432/mirror?sslmode=disable MIRROR_REDIS_ADDR: redis:6379 MIRROR_PLC_URL: https://plc.tngl.boltless.dev MIRROR_GIT_BASEPATH: /data/repos MIRROR_KNOT_USE_SSL: "true" MIRROR_KNOT_SSRF: "true" MIRROR_RESYNC_PARALLELISM: "4" MIRROR_APPVIEW_URL: http://appview:3000 MIRROR_SEARCH_ZOEKT_URL: https://zoekt.tngl.boltless.dev/indexserver volumes: - knotmirror-data:/data - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro healthcheck: test: ["CMD", "wget", "-qO-", "http://localhost:7000/"] interval: 2s timeout: 2s retries: 30 start_period: 5s ports: - "7201:7200" depends_on: postgres: condition: service_started knotmirror-tap: condition: service_started networks: [tngl]
zoekt-webserver: build: context: https://tangled.org/boltless.me/zoekt.git#tngl volumes: - zoekt-index:/data/index - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro command: ["zoekt-webserver", "-index", "/data/index", "-pprof", "-rpc"] networks: [tngl]
zoekt-tngl-indexserver: build: context: . dockerfile: localinfra/zoekt-tngl-indexserver.Dockerfile environment: TANGLED_ZOEKT_INDEX_DIR: /data/index TANGLED_ZOEKT_PLC_URL: https://plc.tngl.boltless.dev TANGLED_ZOEKT_APPVIEW_URL: http://127.0.0.1:3000 volumes: - zoekt-index:/data/index - ./localinfra/certs/root.crt:/etc/ssl/certs/caddy.crt:ro command: ["zoekt-tngl-indexserver", "serve"] networks: [tngl]
tailwind: image: d3fk/tailwindcss:v3 restart: unless-stopped working_dir: /build init: true environment: BROWSERSLIST_IGNORE_OLD_DATA: "true" volumes: - ./tailwind.config.js:/build/tailwind.config.js:ro - ./input.css:/build/input.css:ro - ./appview/pages/templates:/build/appview/pages/templates:ro - ./docs:/build/docs:ro - ./blog/templates:/build/blog/templates:ro - ./blog/posts:/build/blog/posts:ro - ./appview/pages/static:/build/appview/pages/static command: ["-i", "input.css", "-o", "appview/pages/static/tw.css", "--watch=always"] network_mode: none
appview: build: context: . dockerfile: localinfra/appview.Dockerfile restart: unless-stopped environment: TANGLED_DEV: "true" TANGLED_APPVIEW_HOST: 127.0.0.1:3000 TANGLED_DB_PATH: /var/lib/appview/appview.db TANGLED_PLC_URL: https://plc.tngl.boltless.dev TANGLED_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe TANGLED_REDIS_ADDR: redis:6379 TANGLED_KNOTMIRROR_URL: https://mirror.tngl.boltless.dev TANGLED_CODESEARCH_ZOEKT_URL: https://zoekt.tngl.boltless.dev ports: - "3000:3000" volumes: - .:/src:cached - go-cache:/go/cache - go-mod-cache:/go/mod - appview-data:/var/lib/appview - init-state:/shared:ro - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro depends_on: redis: condition: service_started pds: condition: service_healthy init-accounts: condition: service_completed_successfully networks: [tngl]
pdsls: build: context: . dockerfile: localinfra/pdsls.Dockerfile restart: unless-stopped ports: - "8081:80" depends_on: init-accounts: condition: service_completed_successfully networks: [tngl]
caddy: image: caddy:2-alpine restart: unless-stopped ports: - "80:80" - "443:443" volumes: - ./localinfra/Caddyfile:/etc/caddy/Caddyfile - ./localinfra/certs:/etc/caddy/certs:ro - caddy-data:/data networks: tngl: aliases: - plc.tngl.boltless.dev - pds.tngl.boltless.dev - alice.pds.tngl.boltless.dev - bob.pds.tngl.boltless.dev - jetstream.tngl.boltless.dev - knot.tngl.boltless.dev - spindle.tngl.boltless.dev - tngl.boltless.dev - mirror.tngl.boltless.dev - zoekt.tngl.boltless.dev - pdsls.tngl.boltless.dev
volumes: caddy-data: postgres-data: pds-data: jetstream-data: knot-data: knot-ssh-keys: knotmirror-data: zoekt-index: spindle-data: spindle-logs: ncps-data: init-state: go-cache: go-mod-cache: appview-data:
networks: tngl: driver: bridge # Public-looking subnet so SSRF checks see container IPs as "public". # RFC1918 + doc/benchmark ranges are blocklisted; 11.x is unrouted on # the public internet, so it passes the check and won't collide. ipam: config: - subnet: 11.0.0.0/24