diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..45fb491 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +# Keep the build context small and cache-friendly. +.git +.claude +tidepool +e2e/ +tests/ +tasks/ +*.md +docker-compose*.yml +Dockerfile +.github/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d497928 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,97 @@ +# Tidepool CI. +# +# unit: always — build, vet, lexicon-manifest check, full unit/integration +# suite against a real postgres service (Coves convention: tests hit real +# infrastructure; PLC-dependent identity tests self-skip when no local PLC +# directory is reachable, and NOTHING ever touches the public +# plc.directory). +# +# e2e: on demand — the full docker-compose stack (Lemmy built from source in +# debug mode, did:plc directory, Jetstream). Heavy (the Lemmy debug build +# is a full Rust compile), so it runs only via workflow_dispatch or the +# `run-e2e` PR label. + +name: ci + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + +jobs: + unit: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_DB: tidepool_test + POSTGRES_USER: tidepool_test + POSTGRES_PASSWORD: tidepool_test + ports: + - 5443:5432 + options: >- + --health-cmd "pg_isready -U tidepool_test -d tidepool_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Build + run: go build ./... + - name: Vet + run: go vet ./... && go vet -tags e2e ./tests/e2e/... + - name: Lexicon manifest check + # No Coves checkout in CI: the script verifies the committed + # manifest and skips the upstream drift comparison gracefully. + run: ./scripts/check-lexicons.sh + - name: Unit & integration tests + env: + TIDEPOOL_TEST_DATABASE_URL: postgres://tidepool_test:tidepool_test@localhost:5443/tidepool_test?sslmode=disable + run: go test ./... + + e2e: + # Heavy and on-demand: manual dispatch, or label a PR `run-e2e`. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'run-e2e')) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + # COLD-BUILD RISK: every run compiles Lemmy from source (a full debug + # Rust build) because BuildKit cache mounts do not persist on GitHub + # runners — plausibly 30-60+ min on a 2-core runner, plus serious disk + # pressure from the cargo target dir. If this starts timing out, + # mitigations (in preference order): publish a prebuilt lemmy-debug + # image to GHCR and pull it here, or wire docker/build-push-action's + # buildx `gha` cache backend. Tracked in FOLLOWUPS.md ("CI"). + - name: Reclaim runner disk (drop preinstalled images) + run: docker system prune -af || true + # Split up/test/down (rather than `make e2e`) so a failure leaves the + # stack alive long enough to dump its logs. `make e2e-up` passes + # --wait-timeout 600 so a crash-looping service fails the step (with + # logs below) instead of wedging until the job-level timeout cancels + # the run — a cancellation would skip failure()-gated steps. + - name: Start stack (builds Lemmy from source — slow first time) + run: make e2e-up + - name: Run e2e suite + run: make e2e-test + - name: Dump stack logs + # always(): a wedged `up` that exhausts timeout-minutes gets the job + # CANCELLED, and cancellation skips failure()-gated steps. Cheap and + # ||-guarded, so unconditional is safe. + if: always() + run: docker compose -f docker-compose.e2e.yml logs --tail 200 || true + - name: Tear down + if: always() + run: make e2e-down diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cbd7858 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# Tidepool — multi-stage build, small static final image. +# +# The binary is fully static (CGO_ENABLED=0; lib/pq is pure Go) and goose +# migrations are embedded, so the final stage needs nothing but CA certs +# (outbound https in production) and busybox wget for compose healthchecks. +# Runs as a non-root user; docker's default ip_unprivileged_port_start=0 +# lets it bind :80 (the e2e harness serves the bridge portless at +# http://tidepool/ so AP object ids stay clean hostnames). + +FROM golang:1.25-alpine AS build + +WORKDIR /src + +# Dependency layer first so code changes don't re-download modules. +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download + +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o /out/tidepool ./cmd/tidepool + +FROM alpine:3.21 + +RUN apk add --no-cache ca-certificates \ + && adduser -D -u 1000 tidepool + +COPY --from=build /out/tidepool /usr/local/bin/tidepool + +USER tidepool + +EXPOSE 80 +ENTRYPOINT ["tidepool"] diff --git a/LOOP_STATE.md b/LOOP_STATE.md index 4ad09cb..906ccf5 100644 --- a/LOOP_STATE.md +++ b/LOOP_STATE.md @@ -13,7 +13,8 @@ update this file → schedule next. Stop the loop when every task is `done`. | 5 | 05-materializer | done | (see git log) | 8 reviewers; fixes: id-authority binding, Note-root panic, create-after-delete, nobridge scrub, embedded-actor trust, byte caps, uri scheme, Group-type check + 9 regression tests | | 6 | 06-ingestion | done | (see git log) | 8 reviewers (5 Claude + codex/gemini; glm wandered, no JSON); fixes: announced-Delete/Undo scoped to announcer authority (+actor-delete only self), bare Update{Person/Group} no-mint gate, announce content community-authority check, Undo{Delete} restore compensation, handleAccept pending-only, queue lease fencing token + shutdown-cancel handling + processed/poisoned exclusivity, backfillReplies tombstone check, truncation leaves resumable, activityID rand-fail propagates + 14 regression tests | | 7 | 07-vote-aggregates | done | (see git log) | 6/8 reviewers (Gemini perm-denied, glm watchdog-killed); fixes: announced-vote subject↔community binding (post mapping-DID / comment reply.root), bare Undo{Like} signer binding, RetractVote id-targeted undo, dup-id 0/0 aggregate-row leak, seeder zero-clobber presence check, limiter sweep-throttle + 50k fail-closed cap, at-uri validation + ~20 regression tests | -| 8 | 08-e2e-harness | pending | | | +| 8 | 08-e2e-harness (infra: Dockerfile, compose, Lemmy federation, Makefile, CI, lexicon-sync) | done | (see git log) | 7/7 reviewers (4 Claude + codex/gemini/glm, first full external panel since 03); fixes: PRODUCTION https→http redirect-downgrade guard (codex unique catch), webfinger fallback narrowed to transport failures + both-legs errors + 4 tests, minter PDS-endpoint scheme threading, PLC image commit-pin, --wait-timeout + CI logs if:always() + Makefile up-failure cleanup/teardown-status, loopback-only host binds, check-lexicons fail-open holes, sync-lexicons bridge-nesting guard, 2 false compose-header claims rewritten (invented env var, wrong --wait semantics) | +| 9 | 08-e2e-harness (tests: tests/e2e helpers + 7 scenarios, FOLLOWUPS.md, README) | in-progress | | impl done in loop 8's agent run; both pre-fix and post-fix `make e2e` passed (8/8 scenarios) | Statuses: pending → in-progress → review → done (or blocked: ). diff --git a/Makefile b/Makefile index 18bf02f..a87ee68 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help build run test test-db-up test-db-down db-migrate db-migrate-down dev-up dev-down plc-up plc-down jetstream-up jetstream-down lint fmt fmt-check clean +.PHONY: help build run test test-db-up test-db-down db-migrate db-migrate-down dev-up dev-down plc-up plc-down jetstream-up jetstream-down lint fmt fmt-check clean e2e e2e-up e2e-test e2e-logs e2e-down check-lexicons .DEFAULT_GOAL := help @@ -9,6 +9,7 @@ YELLOW := \033[33m RED := \033[31m COMPOSE := docker compose -f docker-compose.dev.yml +E2E_COMPOSE := docker compose -f docker-compose.e2e.yml DEV_DATABASE_URL ?= postgres://tidepool:tidepool@localhost:5442/tidepool_dev?sslmode=disable TEST_DATABASE_URL ?= postgres://tidepool_test:tidepool_test@localhost:5443/tidepool_test?sslmode=disable @@ -92,8 +93,47 @@ test: test-db-up ## Run the test suite against real postgres (migrations run in- @TIDEPOOL_TEST_DATABASE_URL="$(TEST_DATABASE_URL)" go test ./... @echo "$(GREEN)✓ Tests complete$(RESET)" +##@ End-to-end (real Lemmy + PLC + Jetstream) + +e2e: ## Full e2e run: build + start the stack, run the suite, tear down (-v) + @echo "$(GREEN)Starting e2e stack (first run builds Lemmy from source — grab a coffee)...$(RESET)" + @up_status=0; $(E2E_COMPOSE) up -d --build --wait --wait-timeout 600 || up_status=$$?; \ + if [ $$up_status -ne 0 ]; then \ + echo "$(RED)✗ e2e stack failed to come up (exit $$up_status); recent logs:$(RESET)" >&2; \ + $(E2E_COMPOSE) logs --tail=100 || true; \ + echo "$(YELLOW)Tearing down half-started e2e stack...$(RESET)"; \ + $(E2E_COMPOSE) down -v --remove-orphans || true; \ + exit $$up_status; \ + fi; \ + echo "$(GREEN)Stack healthy; running e2e suite...$(RESET)"; \ + status=0; go test -tags e2e -count=1 -timeout 20m ./tests/e2e/... || status=$$?; \ + echo "$(YELLOW)Tearing down e2e stack...$(RESET)"; \ + down_status=0; $(E2E_COMPOSE) down -v --remove-orphans || down_status=$$?; \ + if [ $$down_status -ne 0 ]; then \ + echo "$(RED)✗ WARNING: e2e teardown failed (exit $$down_status) — the stack may still be running; try 'make e2e-down'$(RESET)" >&2; \ + fi; \ + if [ $$status -ne 0 ]; then exit $$status; fi; \ + exit $$down_status + +e2e-up: ## Start the e2e stack and leave it running (for iterating on tests) + @$(E2E_COMPOSE) up -d --build --wait --wait-timeout 600 + @echo "$(GREEN)✓ e2e stack up: tidepool 127.0.0.1:8092, lemmy 127.0.0.1:8541, jetstream 127.0.0.1:6028$(RESET)" + +e2e-test: ## Run the e2e suite against an already-running stack (make e2e-up) + @go test -tags e2e -count=1 -v -timeout 20m ./tests/e2e/... + +e2e-logs: ## Tail all e2e stack logs + @$(E2E_COMPOSE) logs -f + +e2e-down: ## Tear down the e2e stack and its volumes + @$(E2E_COMPOSE) down -v --remove-orphans + @echo "$(GREEN)✓ e2e stack removed$(RESET)" + ##@ Code Quality +check-lexicons: ## Verify vendored lexicons match the manifest (and ~/Code/coves when present) + @./scripts/check-lexicons.sh + fmt: ## Format all Go code @echo "$(GREEN)Formatting Go code...$(RESET)" @gofmt -w ./cmd ./internal diff --git a/cmd/tidepool/main.go b/cmd/tidepool/main.go index 6d70239..ae1f793 100644 --- a/cmd/tidepool/main.go +++ b/cmd/tidepool/main.go @@ -141,7 +141,7 @@ func run(logger *slog.Logger) error { // the durable work queue, activity dispatch into the materializer, the // community Follow lifecycle, outbox backfill, and consent enforcement. serviceKeys := store.NewServiceKeys(database) - serviceActor, err := ap.LoadOrCreateServiceActor(ctx, serviceKeys, cfg.BridgeHostname) + serviceActor, err := ap.LoadOrCreateServiceActor(ctx, serviceKeys, cfg.BridgeHostname, cfg.BridgeScheme) if err != nil { return err } @@ -158,6 +158,7 @@ func run(logger *slog.Logger) error { minter, err := identity.NewMinter(identity.MinterOptions{ PLCDirectoryURL: cfg.PLCDirectoryURL, BridgeHostname: cfg.BridgeHostname, + BridgeScheme: cfg.BridgeScheme, RotationKey: rotationKey, Custodian: custodian, Actors: actors, diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..7ff60c1 --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,215 @@ +# Tidepool end-to-end stack: a real Lemmy federating with the bridge, a real +# did:plc directory backing DID minting, and a real Jetstream decoding the +# bridge's subscribeRepos firehose. `make e2e` builds it, waits for health, +# runs tests/e2e (tagged `e2e`) from the host, and tears it down. +# +# ── How Lemmy↔Tidepool federation works over plain HTTP ────────────────── +# Lemmy only accepts http:// federation URLs in a DEBUG build (a compile-time +# property — see e2e/lemmy/Dockerfile), so the harness builds Lemmy from +# source in debug mode instead of pulling a release image. Both sides then +# live portless on :80 under their compose hostnames: +# +# http://lemmy/... Lemmy's AP ids (hostname "lemmy", tls_enabled false) +# http://tidepool/... the bridge's AP ids (BRIDGE_SCHEME=http, dev-only) +# +# Portless matters twice: Lemmy production builds reject explicit ports in +# ids (moot here, but debug keeps the same shape), and Tidepool's bridged +# handles are DNS labels under BRIDGE_HOSTNAME, so the hostname must not +# carry a port. Non-root processes can bind :80 because docker defaults +# net.ipv4.ip_unprivileged_port_start=0 in containers. +# +# No TLS/caddy layer is involved. The release-image alternative (caddy with +# an internal CA, portless https hostnames via network aliases, and the CA +# trusted inside every container) was rejected: release builds have NO +# runtime override for the local-IP/http/port federation checks — they are +# compile-time debug_assertions in the activitypub-federation crate, which +# is exactly why the debug build is required. On top of that, Lemmy's TLS +# trust store is not reliably extensible at runtime, and the proxy adds +# nothing for a local harness. +# +# Host ports (loopback-only — the stack carries an admin token and +# ALLOW_PRIVATE_FETCH=1, so it must not be reachable from the local +# network; offset from the dev stack so both can run): +# 127.0.0.1:8092 Tidepool HTTP (admin API, XRPC, healthz) +# 127.0.0.1:8541 Lemmy HTTP API (8541 nods to lemmy_alpha in upstream's federation compose) +# 127.0.0.1:6028 Jetstream WebSocket (/subscribe) +# +# LOCAL-ONLY: nothing here talks to plc.directory, public relays, or public +# Lemmy instances. All egress stays on the compose network (image pulls and +# build-time package/source downloads aside). + +name: tidepool-e2e + +services: + # ── Tidepool ──────────────────────────────────────────────────────────── + tidepool-postgres: + image: postgres:16 + environment: + POSTGRES_DB: tidepool + POSTGRES_USER: tidepool + POSTGRES_PASSWORD: tidepool + networks: [tidepool-e2e] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U tidepool -d tidepool"] + interval: 2s + timeout: 5s + retries: 30 + + tidepool: + build: . + environment: + # development: migrations-on-start, StrictValidation (lexicon failures + # fail closed — exactly what a conformance harness wants), and the two + # local-federation relaxations below are dev-gated in config. + ENVIRONMENT: development + DATABASE_URL: postgres://tidepool:tidepool@tidepool-postgres:5432/tidepool?sslmode=disable + LISTEN_ADDR: ":80" + BRIDGE_HOSTNAME: tidepool + # Plain-HTTP AP ids so a debug-mode Lemmy can fetch/deliver to us. + BRIDGE_SCHEME: http + # The compose network is private address space; the SSRF guard must let + # the bridge fetch lemmy/plc (dev-only, refused in production). + ALLOW_PRIVATE_FETCH: "1" + PLC_DIRECTORY_URL: http://plc:3000 + ADMIN_TOKEN: e2e-admin-token + SEED_COUNTS_FROM_API: "1" + LOG_LEVEL: debug + ports: + - "127.0.0.1:${TIDEPOOL_E2E_PORT:-8092}:80" + networks: [tidepool-e2e] + depends_on: + tidepool-postgres: + condition: service_healthy + plc: + condition: service_healthy + healthcheck: + # /healthz pings the database, so healthy == migrated and serving. + test: ["CMD", "wget", "--spider", "-q", "http://localhost/healthz"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 20s + + # ── did:plc directory ─────────────────────────────────────────────────── + plc-postgres: + image: postgres:16 + environment: + POSTGRES_DB: plc + POSTGRES_USER: plc + POSTGRES_PASSWORD: plc + networks: [tidepool-e2e] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U plc -d plc"] + interval: 2s + timeout: 5s + retries: 30 + + plc: + build: e2e/plc + environment: + DATABASE_URL: postgresql://plc:plc@plc-postgres:5432/plc?sslmode=disable + DEBUG_MODE: "1" + LOG_ENABLED: "true" + LOG_LEVEL: info + LOG_DESTINATION: "1" + NODE_ENV: development + PORT: 3000 + networks: [tidepool-e2e] + depends_on: + plc-postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/_health"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 30s + + # ── Jetstream (consumes the bridge's firehose) ────────────────────────── + jetstream: + image: ghcr.io/bluesky-social/jetstream:sha-306e463693365e21a5ffd3ec051a5a7920000214 + # Jetstream EXITS when its upstream websocket dies ("shutting down on + # events kill"), which is exactly what the restart scenario provokes by + # bouncing tidepool. Docker revives it; it reconnects with the cursor + # persisted in /data (container-local, survives restarts) and resumes + # replay from the bridge without gaps or duplicates. + restart: unless-stopped + environment: + JETSTREAM_WS_URL: ws://tidepool:80/xrpc/com.atproto.sync.subscribeRepos + JETSTREAM_LISTEN_ADDR: ":6018" + JETSTREAM_METRICS_LISTEN_ADDR: ":6019" + JETSTREAM_DATA_DIR: /data + JETSTREAM_EVENT_TTL: 24h + # The bridge is quiet between test steps; don't self-restart on idle. + JETSTREAM_LIVENESS_TTL: 24h + LOG_LEVEL: debug + ports: + - "127.0.0.1:${JETSTREAM_E2E_PORT:-6028}:6018" + networks: [tidepool-e2e] + depends_on: + tidepool: + condition: service_healthy + # No wget/curl in the image, but it is debian-based with bash — a + # /dev/tcp probe of the subscribe port is a real readiness gate. Without + # a healthcheck, `up --wait` would pass the moment the container is + # running, before Jetstream actually accepts subscribers. + healthcheck: + test: ["CMD", "bash", "-c", "exec 3<>/dev/tcp/localhost/6018"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 10s + + # ── Lemmy ─────────────────────────────────────────────────────────────── + lemmy-postgres: + image: postgres:16 + environment: + POSTGRES_DB: lemmy + POSTGRES_USER: lemmy + POSTGRES_PASSWORD: lemmy + networks: [tidepool-e2e] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U lemmy -d lemmy"] + interval: 2s + timeout: 5s + retries: 30 + + pictrs: + # Lemmy refuses to boot without a pictrs URL configured; 0.5 is the + # pairing for lemmy 0.19 (multi-arch, runs fine on arm64). + image: asonix/pictrs:0.5 + environment: + PICTRS__SERVER__API_KEY: e2e-pictrs-key + networks: [tidepool-e2e] + + lemmy: + build: e2e/lemmy + environment: + RUST_LOG: "warn,lemmy_server=debug,lemmy_apub=debug,lemmy_federate=debug,activitypub_federation=debug" + RUST_BACKTRACE: "1" + LEMMY_CONFIG_LOCATION: /config/config.hjson + # Without this the persistent federation queue delays deliveries by + # 30s–5min; Lemmy's own api_tests set it too. + LEMMY_TEST_FAST_FEDERATION: "1" + volumes: + - ./e2e/lemmy/lemmy.hjson:/config/config.hjson:ro + ports: + - "127.0.0.1:${LEMMY_E2E_PORT:-8541}:80" + networks: [tidepool-e2e] + depends_on: + lemmy-postgres: + condition: service_healthy + pictrs: + condition: service_started + healthcheck: + # /api/v3/site answers once migrations + setup (admin/site creation) ran. + test: ["CMD", "curl", "-sf", "http://localhost/api/v3/site"] + interval: 2s + timeout: 5s + retries: 60 + start_period: 60s + +networks: + tidepool-e2e: + driver: bridge + name: tidepool-e2e-network diff --git a/e2e/lemmy/Dockerfile b/e2e/lemmy/Dockerfile new file mode 100644 index 0000000..e426d99 --- /dev/null +++ b/e2e/lemmy/Dockerfile @@ -0,0 +1,64 @@ +# Lemmy built from source in DEBUG mode — the key trick that makes +# Lemmy↔Tidepool federation work over plain HTTP inside one compose network. +# +# Why a source build: whether Lemmy accepts http:// federation URLs is a +# COMPILE-TIME property, not configuration. Lemmy constructs its federation +# config with `.debug(cfg!(debug_assertions))` (src/lib.rs on 0.19.x; +# crates/server/src/lib.rs on main) and the activitypub-federation crate only +# sets `allow_http_urls` from that debug flag. Every published +# dessalines/lemmy image is a release build, which rejects http:// object IDs +# ("Http urls are only allowed in debug mode"), rejects explicit ports in +# URLs, and refuses private/local IPs — with NO runtime override for any of +# the three. A debug build accepts all three, which is exactly what +# LemmyNet's own docker/federation compose relies on (it builds from source +# with RUST_RELEASE_MODE=debug). +# +# The alternative (release image + caddy internal-CA TLS, portless https +# hostnames, custom CA trust inside the Lemmy container) was considered and +# rejected: Lemmy's reqwest TLS backend does not reliably honor a mounted +# CA, and the extra proxy indirection buys nothing for a local-only harness. +# +# Upstream's docker/Dockerfile cross-compiles arm64 via an amd64-hosted +# toolchain image (Rosetta emulation on Apple Silicon — very slow), so this +# Dockerfile builds natively for whatever platform docker runs on instead. +# First build is a full Rust compile (~15 min); the target-dir cache mount +# and the clone layer make rebuilds cheap. + +FROM rust:1.81-bookworm AS builder + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libssl-dev libpq-dev pkg-config git \ + && rm -rf /var/lib/apt/lists/* + +# Pin the Lemmy release. lemmy-translations is a git submodule required to +# compile (crates/utils embeds it). +ARG LEMMY_VERSION=0.19.19 +RUN git clone --depth 1 --branch ${LEMMY_VERSION} https://github.com/LemmyNet/lemmy.git /lemmy \ + && cd /lemmy \ + && git submodule update --init --recursive --depth 1 + +WORKDIR /lemmy + +# Debug build (no --release) — see header comment. The cache mount keeps the +# cargo target dir across image rebuilds. +RUN --mount=type=cache,target=/lemmy/target,sharing=locked \ + cargo build \ + && cp target/debug/lemmy_server /usr/local/bin/lemmy_server + +FROM debian:bookworm-slim + +# Runtime deps mirror upstream's runner stage: libssl/libpq are linked by the +# binary, ca-certificates for any https egress, curl for compose healthchecks. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libssl3 libpq5 ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/local/bin/lemmy_server /usr/local/bin/lemmy_server + +RUN useradd -m -u 1000 lemmy +USER lemmy + +ENTRYPOINT ["lemmy_server"] +# lemmy.hjson binds port 80 (portless federation ids — see header comment). +EXPOSE 80 +STOPSIGNAL SIGTERM diff --git a/e2e/lemmy/lemmy.hjson b/e2e/lemmy/lemmy.hjson new file mode 100644 index 0000000..24d5178 --- /dev/null +++ b/e2e/lemmy/lemmy.hjson @@ -0,0 +1,25 @@ +# Lemmy config for the Tidepool e2e harness (see e2e/lemmy/Dockerfile for +# why Lemmy is a debug build). Modeled on LemmyNet's own docker/federation +# hjson files: a plain-HTTP instance addressed by its compose hostname. +{ + # The compose service name — this is the authority baked into every AP id + # Lemmy emits (http://lemmy/c/testing, http://lemmy/u/alice, ...). + # Portless + tls_enabled:false + debug build = plain-HTTP federation. + hostname: "lemmy" + bind: "0.0.0.0" + port: 80 + tls_enabled: false + setup: { + # Auto-created on first boot (min 10 chars for the password). + admin_username: "admin" + admin_password: "lemmylemmy" + site_name: "tidepool-e2e" + } + database: { + uri: "postgres://lemmy:lemmy@lemmy-postgres:5432/lemmy" + } + pictrs: { + url: "http://pictrs:8080/" + api_key: "e2e-pictrs-key" + } +} diff --git a/e2e/plc/Dockerfile b/e2e/plc/Dockerfile new file mode 100644 index 0000000..17aa7f4 --- /dev/null +++ b/e2e/plc/Dockerfile @@ -0,0 +1,32 @@ +# Local did:plc directory (the real did-method-plc server) baked into an +# image at build time. The Coves/Tidepool dev composes clone + build the +# repo inside a node container on first START (cached in a volume); for the +# e2e stack that would either survive `down -v` poorly or rebuild for +# minutes on every run, so the clone/build moves into the image where the +# docker layer cache owns it. +# +# The clone is pinned to PLC_COMMIT so local (layer-cached) and CI (cold) +# builds compile the same source. To bump: take the SHA from +# git ls-remote https://github.com/did-method-plc/did-method-plc.git main +# update PLC_COMMIT below, and rebuild +# (docker compose -f docker-compose.e2e.yml build plc). + +FROM node:18-alpine + +RUN apk add --no-cache git python3 make g++ + +ARG PLC_COMMIT=2ed82a5ccf1b424aa5e2f6c5b461dc0ee133278b + +WORKDIR /app +RUN git init -q . \ + && git remote add origin https://github.com/did-method-plc/did-method-plc.git \ + && git fetch --depth 1 origin "${PLC_COMMIT}" \ + && git checkout -q FETCH_HEAD \ + && yarn install --frozen-lockfile \ + && yarn build + +WORKDIR /app/packages/server + +ENV PORT=3000 +EXPOSE 3000 +CMD ["yarn", "start"] diff --git a/go.mod b/go.mod index 47ba61e..6898229 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.7 require ( github.com/bluesky-social/indigo v0.0.0-20260202181658-ea3d39eec464 github.com/go-chi/chi/v5 v5.3.1 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/websocket v1.5.3 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-ipld-format v0.6.0 diff --git a/go.sum b/go.sum index 0359a60..c51b1c9 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= diff --git a/internal/ap/client.go b/internal/ap/client.go index 5a33664..5aaf3b7 100644 --- a/internal/ap/client.go +++ b/internal/ap/client.go @@ -496,7 +496,11 @@ func (c *Client) fetchMediaOnce(ctx context.Context, iri string, maxBytes int64) if err != nil { return nil, "", false, fmt.Errorf("ap: GET %s: bad redirect location %q: %w", target, location, err) } - target = req.URL.ResolveReference(next).String() + resolved := req.URL.ResolveReference(next) + if err := c.checkRedirectScheme(req.URL, resolved); err != nil { + return nil, "", false, fmt.Errorf("ap: GET %s: %w", target, err) + } + target = resolved.String() continue } @@ -822,7 +826,11 @@ func (c *Client) getOnce(ctx context.Context, iri string, mode fetchMode) (body if err != nil { return nil, false, fmt.Errorf("ap: GET %s: bad redirect location %q: %w", target, location, err) } - target = req.URL.ResolveReference(next).String() + resolved := req.URL.ResolveReference(next) + if err := c.checkRedirectScheme(req.URL, resolved); err != nil { + return nil, false, fmt.Errorf("ap: GET %s: %w", target, err) + } + target = resolved.String() continue } @@ -889,6 +897,24 @@ func (c *Client) backoff(ctx context.Context, attempt int) error { return c.sleep(ctx, delay) } +// checkRedirectScheme rejects a redirect hop that downgrades https to http. +// In production every fetch starts (and must stay) https; following an +// http Location from an https response would move the exchange to plaintext +// (MITM/downgrade vector). The only exception is the dev/e2e relaxation +// (ALLOW_PRIVATE_FETCH → guard.allowPrivate), where plain-HTTP peers on the +// compose network are expected. Same-scheme hops and http→https upgrades +// are always allowed. +func (c *Client) checkRedirectScheme(from, to *url.URL) error { + if c.guard.allowPrivate { + return nil + } + if from.Scheme == "https" && to.Scheme == "http" { + return errors.NewValidationError("redirect", + fmt.Sprintf("redirect to %q downgrades https to http", to.String())) + } + return nil +} + // waitForHost validates the URL against the egress guard (scheme, userinfo, // IP literals) and applies the per-host rate limit. The resolved-IP guard // runs later at dial time (defeating DNS rebinding). diff --git a/internal/ap/client_test.go b/internal/ap/client_test.go index 589941d..22471f1 100644 --- a/internal/ap/client_test.go +++ b/internal/ap/client_test.go @@ -213,6 +213,112 @@ func TestFetchObject_FollowsRedirectsWithFreshSignatures(t *testing.T) { assert.Equal(t, "https://x.example/new", obj.ID) } +// scriptedTransport is a RoundTripper answering from a script instead of the +// network: no dial ever happens, so egress-guard-ON behavior can be tested +// against fake public hostnames (guard-on clients cannot hit 127.0.0.1 +// httptest servers). Every request URL is recorded. +type scriptedTransport struct { + handler func(req *http.Request) (*http.Response, error) + + mu sync.Mutex + requests []string +} + +func (s *scriptedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + s.mu.Lock() + s.requests = append(s.requests, req.URL.String()) + s.mu.Unlock() + return s.handler(req) +} + +// seen returns the URLs of all requests issued so far. +func (s *scriptedTransport) seen() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.requests...) +} + +// scriptedResponse builds a minimal *http.Response for scriptedTransport. +func scriptedResponse(status int, body string) *http.Response { + return &http.Response{ + StatusCode: status, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(body)), + } +} + +// TestRedirect_DowngradeRejectedWhenGuardOn: with the egress guard ON +// (production posture) an https fetch that 302s to an http:// URL must fail +// with the downgrade validation error BEFORE any plaintext request is +// issued — for both the object path (getOnce) and the media path +// (fetchMediaOnce). +func TestRedirect_DowngradeRejectedWhenGuardOn(t *testing.T) { + newGuardedClient := func(transport *scriptedTransport) *Client { + c := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + HTTPClient: &http.Client{Transport: transport}, + MaxAttempts: 1, + // AllowPrivateAddresses deliberately false: guard on. + }) + c.sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } + return c + } + newDowngradeTransport := func() *scriptedTransport { + return &scriptedTransport{handler: func(req *http.Request) (*http.Response, error) { + if req.URL.Scheme != "https" { + t.Errorf("plaintext request issued to %s", req.URL) + return nil, fmt.Errorf("unexpected plaintext request") + } + resp := scriptedResponse(http.StatusFound, "") + resp.Header.Set("Location", "http://remote.test/object") + return resp, nil + }} + } + + t.Run("object", func(t *testing.T) { + transport := newDowngradeTransport() + _, _, err := newGuardedClient(transport).getOnce( + context.Background(), "https://remote.test/object", fetchModeObject) + require.Error(t, err) + assert.True(t, errors.IsValidation(err), "downgrade must be a typed validation error, got %v", err) + assert.Contains(t, err.Error(), "downgrades https to http") + require.Len(t, transport.seen(), 1, "the http hop must never be requested") + assert.True(t, strings.HasPrefix(transport.seen()[0], "https://")) + }) + + t.Run("media", func(t *testing.T) { + transport := newDowngradeTransport() + _, _, _, err := newGuardedClient(transport).fetchMediaOnce( + context.Background(), "https://remote.test/img.png", 1<<20) + require.Error(t, err) + assert.True(t, errors.IsValidation(err), "downgrade must be a typed validation error, got %v", err) + assert.Contains(t, err.Error(), "downgrades https to http") + require.Len(t, transport.seen(), 1, "the http hop must never be requested") + assert.True(t, strings.HasPrefix(transport.seen()[0], "https://")) + }) +} + +// TestRedirect_DowngradeFollowedWhenGuardRelaxed: under the dev/e2e +// relaxation (AllowPrivateAddresses, ALLOW_PRIVATE_FETCH) plain-HTTP peers +// are expected, so an https→http redirect IS followed. +func TestRedirect_DowngradeFollowedWhenGuardRelaxed(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/downgraded"}`)) + })) + defer httpServer.Close() + + tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, httpServer.URL+"/object", http.StatusFound) + })) + defer tlsServer.Close() + + client := newTestClient(t, ClientOptions{HTTPClient: tlsServer.Client()}) + obj, err := client.FetchObject(context.Background(), tlsServer.URL+"/old") + require.NoError(t, err) + assert.Equal(t, "https://x.example/downgraded", obj.ID, + "with the guard relaxed the http redirect target must be fetched") +} + func TestFetchActor_RejectsNonActors(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(loadFixture(t, "page_lemmy_world.json")) diff --git a/internal/ap/service_actor.go b/internal/ap/service_actor.go index e3037ff..fa9566c 100644 --- a/internal/ap/service_actor.go +++ b/internal/ap/service_actor.go @@ -22,27 +22,52 @@ const ServiceActorPath = "/actor" // core AS2 plus the security vocabulary that defines publicKey. const serviceActorContext = `["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"]` -// ServiceActor is the bridge's own AP identity: an Application actor whose +// ServiceActor is the bridge's own AP identity: a Service actor whose // RSA key signs every outbound request (fetches and Follows). This is the // AP-side interop key — entirely distinct from the atproto secp256k1 repo // keys task 03 mints. type ServiceActor struct { - // ID is the actor's canonical id, https://{hostname}/actor. + // ID is the actor's canonical id, {scheme}://{hostname}/actor. ID string // Hostname is the bridge's public hostname (config.BridgeHostname). Hostname string + // Scheme is the URL scheme the bridge's own AP URLs are built with + // (config.BridgeScheme). "https" everywhere real; "http" exists for the + // local e2e harness, where a debug-mode Lemmy federates with the bridge + // over plain HTTP inside one compose network. Empty means "https". + Scheme string // Key is the actor's RSA private key. Key *rsa.PrivateKey } +// BaseURL is the origin the bridge's own AP URLs live under, +// e.g. "https://bridge.example". Defensive default: an unset Scheme (a +// hand-built test literal) renders https, never a schemeless URL. +func (a *ServiceActor) BaseURL() string { + scheme := a.Scheme + if scheme == "" { + scheme = "https" + } + return scheme + "://" + a.Hostname +} + // LoadOrCreateServiceActor returns the bridge's service actor, loading its // RSA key from the service_keys store or generating and persisting one on // first run. Losing a concurrent-bootstrap insert race falls back to the // winner's key, so every process converges on the same keypair. -func LoadOrCreateServiceActor(ctx context.Context, keys store.ServiceKeys, hostname string) (*ServiceActor, error) { +// scheme is the URL scheme for the actor's own URLs ("https", or "http" for +// the local e2e harness); empty defaults to https. +func LoadOrCreateServiceActor(ctx context.Context, keys store.ServiceKeys, hostname, scheme string) (*ServiceActor, error) { if hostname == "" { return nil, errors.NewValidationError("hostname", "must not be empty") } + switch scheme { + case "": + scheme = "https" + case "http", "https": + default: + return nil, errors.NewValidationError("scheme", fmt.Sprintf("must be http or https, got %q", scheme)) + } stored, err := keys.Get(ctx, ServiceKeyName) switch { @@ -77,8 +102,9 @@ func LoadOrCreateServiceActor(ctx context.Context, keys store.ServiceKeys, hostn } return &ServiceActor{ - ID: "https://" + hostname + ServiceActorPath, + ID: scheme + "://" + hostname + ServiceActorPath, Hostname: hostname, + Scheme: scheme, Key: key, }, nil } @@ -89,16 +115,16 @@ func LoadOrCreateServiceActor(ctx context.Context, keys store.ServiceKeys, hostn func (a *ServiceActor) KeyID() string { return a.ID + "#main-key" } // InboxURL is the actor's inbox (served by task 06). -func (a *ServiceActor) InboxURL() string { return "https://" + a.Hostname + "/inbox" } +func (a *ServiceActor) InboxURL() string { return a.BaseURL() + "/inbox" } // OutboxURL is the actor's outbox. -func (a *ServiceActor) OutboxURL() string { return "https://" + a.Hostname + "/outbox" } +func (a *ServiceActor) OutboxURL() string { return a.BaseURL() + "/outbox" } // Signer returns a request signer using the actor's key. func (a *ServiceActor) Signer() *Signer { return NewSigner(a.KeyID(), a.Key) } -// Document builds the Application actor document served at -// https://{hostname}/actor. Lemmy requires the publicKey block (RSA, SPKI +// Document builds the Service actor document served at +// {scheme}://{hostname}/actor. Lemmy requires the publicKey block (RSA, SPKI // PEM) to accept our signed requests. func (a *ServiceActor) Document() (*Object, error) { publicPEM, err := EncodePublicKeyPEM(&a.Key.PublicKey) @@ -106,12 +132,18 @@ func (a *ServiceActor) Document() (*Object, error) { return nil, err } return &Object{ - Context: json.RawMessage(serviceActorContext), - ID: a.ID, - Type: TypeApplication, + Context: json.RawMessage(serviceActorContext), + ID: a.ID, + // Service, not Application: Lemmy deserializes a Follow's actor as its + // Person protocol type, whose kind enum is Person|Service|Organization + // (crates/apub .../protocol/.../person.rs, 0.19 and main alike) — an + // Application actor fails deserialization and the Follow is dropped. + // Service is the standard AS2 type for bots/bridges and every other + // platform accepts it. + Type: TypeService, PreferredUsername: a.Hostname, Name: "Tidepool bridge", - Summary: "Bridges threadiverse communities into atproto. https://" + a.Hostname, + Summary: "Bridges threadiverse communities into atproto. " + a.BaseURL(), Inbox: a.InboxURL(), Outbox: a.OutboxURL(), PublicKey: &PublicKey{ diff --git a/internal/ap/service_actor_test.go b/internal/ap/service_actor_test.go index 2a6f2cf..49b76e1 100644 --- a/internal/ap/service_actor_test.go +++ b/internal/ap/service_actor_test.go @@ -54,7 +54,7 @@ func TestLoadOrCreateServiceActor_GeneratesThenLoads(t *testing.T) { keys := newFakeServiceKeys() ctx := context.Background() - first, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + first, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example", "") require.NoError(t, err) assert.Equal(t, "https://bridge.example/actor", first.ID) assert.Equal(t, "https://bridge.example/actor#main-key", first.KeyID()) @@ -69,7 +69,7 @@ func TestLoadOrCreateServiceActor_GeneratesThenLoads(t *testing.T) { assert.True(t, first.Key.Equal(storedKey)) // A second bootstrap loads the same key instead of generating a new one. - second, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + second, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example", "") require.NoError(t, err) assert.True(t, first.Key.Equal(second.Key), "restarts must reuse the persisted key") } @@ -92,14 +92,14 @@ func TestLoadOrCreateServiceActor_LosesBootstrapRace(t *testing.T) { keys.mu.Unlock() } - actor, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + actor, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example", "") require.NoError(t, err) assert.True(t, winnerKey.Equal(actor.Key), "losing the create race must converge on the winner's key") } func TestLoadOrCreateServiceActor_RequiresHostname(t *testing.T) { - _, err := LoadOrCreateServiceActor(context.Background(), newFakeServiceKeys(), "") + _, err := LoadOrCreateServiceActor(context.Background(), newFakeServiceKeys(), "", "") require.Error(t, err) assert.True(t, errors.IsValidation(err)) } @@ -109,14 +109,14 @@ func TestLoadOrCreateServiceActor_CorruptKey(t *testing.T) { _, err := keys.Create(context.Background(), ServiceKeyName, []byte("not a pem")) require.NoError(t, err) - _, err = LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + _, err = LoadOrCreateServiceActor(context.Background(), keys, "bridge.example", "") require.Error(t, err) assert.Contains(t, err.Error(), "corrupt") } func TestServiceActorDocument(t *testing.T) { keys := newFakeServiceKeys() - actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example", "") require.NoError(t, err) docJSON, err := actor.DocumentJSON() @@ -127,7 +127,7 @@ func TestServiceActorDocument(t *testing.T) { doc, err := ParseObject(docJSON) require.NoError(t, err) assert.True(t, doc.IsActor()) - assert.Equal(t, TypeApplication, doc.Type) + assert.Equal(t, TypeService, doc.Type) assert.Equal(t, "https://bridge.example/actor", doc.ID) assert.Equal(t, "https://bridge.example/inbox", doc.Inbox) assert.NotEmpty(t, doc.PreferredUsername, "webfinger reverse resolution needs preferredUsername") @@ -147,7 +147,7 @@ func TestServiceActorDocument(t *testing.T) { func TestServiceActorSigner_RoundTrip(t *testing.T) { keys := newFakeServiceKeys() - actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example", "") require.NoError(t, err) // Resolve the verification key exactly the way a remote instance would: @@ -164,3 +164,29 @@ func TestServiceActorSigner_RoundTrip(t *testing.T) { require.NoError(t, err) assert.Equal(t, actor.ID, signerActorID) } + +func TestLoadOrCreateServiceActor_HTTPScheme(t *testing.T) { + actor, err := LoadOrCreateServiceActor(context.Background(), newFakeServiceKeys(), "tidepool", "http") + require.NoError(t, err) + assert.Equal(t, "http://tidepool/actor", actor.ID) + assert.Equal(t, "http://tidepool/inbox", actor.InboxURL()) + assert.Equal(t, "http://tidepool/outbox", actor.OutboxURL()) + assert.Equal(t, "http://tidepool", actor.BaseURL()) + + doc, err := actor.Document() + require.NoError(t, err) + assert.Equal(t, "http://tidepool/actor", doc.ID) + assert.Equal(t, "http://tidepool/inbox", doc.Inbox) +} + +func TestLoadOrCreateServiceActor_RejectsBadScheme(t *testing.T) { + _, err := LoadOrCreateServiceActor(context.Background(), newFakeServiceKeys(), "bridge.example", "gopher") + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestServiceActor_BaseURLDefaultsToHTTPS(t *testing.T) { + actor := &ServiceActor{Hostname: "bridge.example"} + assert.Equal(t, "https://bridge.example", actor.BaseURL(), + "hand-built literals without a scheme must render https, never a schemeless URL") +} diff --git a/internal/ap/webfinger.go b/internal/ap/webfinger.go index 42dec5f..83bb85f 100644 --- a/internal/ap/webfinger.go +++ b/internal/ap/webfinger.go @@ -3,6 +3,7 @@ package ap import ( "context" "encoding/json" + stderrors "errors" "fmt" "net/url" "strings" @@ -62,6 +63,32 @@ func (c *Client) ResolveHandle(ctx context.Context, handle string) (actorURL str webfingerURL := fmt.Sprintf("https://%s/.well-known/webfinger?%s", host, query.Encode()) body, err := c.getDedupedMode(ctx, webfingerURL, fetchModeWebFinger) + if err != nil && c.guard.allowPrivate && isTransportFailure(err) { + // Local-federation fallback: when the SSRF guard is relaxed + // (ALLOW_PRIVATE_FETCH, dev/e2e only) the peer may be a plain-HTTP + // instance on the compose network (debug-mode Lemmy has no TLS + // listener at all), so an https attempt that failed at the transport + // level (the server never answered HTTP) is retried over http. + // Semantic answers are NOT retried: a 404 said the account doesn't + // exist, a 401/403 must stay distinguishable (Cloudflare, + // defederation), and a 410 is a tombstone. Never reachable in + // production: the guard is forced on there, so allowPrivate is + // always false. + httpURL := fmt.Sprintf("http://%s/.well-known/webfinger?%s", host, query.Encode()) + httpBody, httpErr := c.getDedupedMode(ctx, httpURL, fetchModeWebFinger) + switch { + case httpErr == nil: + body, err = httpBody, nil + case errors.IsNotFound(httpErr): + // The http listener answered and said the account doesn't + // exist: adopt that as the result. + err = httpErr + default: + // Surface both legs — the https transport failure alone would + // hide why the fallback didn't save the lookup. + err = fmt.Errorf("ap: webfinger https: %v; http fallback: %w", err, httpErr) + } + } if err != nil { // Only a genuine 404 means "no such account". A 401/403 (Cloudflare, a // defederating instance) is surfaced as-is so the caller can tell a @@ -105,6 +132,18 @@ func (c *Client) ResolveHandle(ctx context.Context, handle string) (actorURL str return "", errors.NewNotFoundError("webfinger self link", handle) } +// isTransportFailure reports whether a fetch error means the server never +// gave a semantic HTTP answer (dial/TLS/read failure) — as opposed to a +// mapped status (404 → NotFound, 410 → Tombstoned) or a preserved HTTPError +// (401/403 in webfinger mode, 5xx, ...). Only transport failures may trigger +// the dev-only https→http webfinger fallback. +func isTransportFailure(err error) bool { + var httpErr HTTPError + return !stderrors.As(err, &httpErr) && + !errors.IsNotFound(err) && + !errors.IsTombstoned(err) +} + // hrefAuthorityMatches reports whether an href's authority (host:port) equals // the queried WebFinger host. The comparison is case-insensitive. func hrefAuthorityMatches(href, host string) bool { diff --git a/internal/ap/webfinger_test.go b/internal/ap/webfinger_test.go index 5af6a51..7ae9448 100644 --- a/internal/ap/webfinger_test.go +++ b/internal/ap/webfinger_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -161,6 +162,132 @@ func TestResolveHandle_UnknownAccount(t *testing.T) { assert.True(t, errors.IsNotFound(err)) } +// TestResolveHandle_HTTPFallbackOnTransportFailure: with the guard relaxed +// (dev/e2e), an https leg that fails at the transport level falls back to +// http. The server is a PLAIN-http httptest server, so the https attempt +// dies in the TLS handshake — the handler is only ever reached by the http +// leg, mirroring the real debug-mode-Lemmy (no TLS listener) case. +func TestResolveHandle_HTTPFallbackOnTransportFailure(t *testing.T) { + var hits atomic.Int32 + var host string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + require.Equal(t, "/.well-known/webfinger", r.URL.Path) + _, _ = fmt.Fprintf(w, `{"subject":"acct:alice@%s","links":[ + {"rel":"self","type":"application/activity+json","href":"http://%s/u/alice"} + ]}`, host, host) + })) + t.Cleanup(server.Close) + host = strings.TrimPrefix(server.URL, "http://") + + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + MaxAttempts: 1, + AllowPrivateAddresses: true, + }) + + actorURL, err := client.ResolveHandle(context.Background(), "alice@"+host) + require.NoError(t, err, "the http fallback must rescue a transport-level https failure") + assert.Equal(t, "http://"+host+"/u/alice", actorURL) + assert.Equal(t, int32(1), hits.Load(), + "exactly one request must reach the handler: the http leg (https dies in the TLS handshake)") +} + +// TestResolveHandle_NoHTTPFallbackOnSemanticError: even with the guard +// relaxed, an https server that ANSWERED (404 missing account, 403 +// Cloudflare/defederation) must not trigger the http fallback — those +// semantics are deliberately preserved. +func TestResolveHandle_NoHTTPFallbackOnSemanticError(t *testing.T) { + cases := []struct { + name string + status int + check func(t *testing.T, err error) + }{ + {"404 stays not-found", http.StatusNotFound, func(t *testing.T, err error) { + assert.True(t, errors.IsNotFound(err), "https 404 must surface as IsNotFound, got %v", err) + }}, + {"403 stays HTTPError", http.StatusForbidden, func(t *testing.T, err error) { + assert.False(t, errors.IsNotFound(err)) + var httpErr HTTPError + require.ErrorAs(t, err, &httpErr, "the 403 must stay distinguishable") + assert.Equal(t, http.StatusForbidden, httpErr.StatusCode) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + transport := &scriptedTransport{handler: func(req *http.Request) (*http.Response, error) { + require.Equal(t, "https", req.URL.Scheme, + "a semantic https answer must not trigger an http request") + return scriptedResponse(tc.status, ""), nil + }} + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + HTTPClient: &http.Client{Transport: transport}, + MaxAttempts: 1, + AllowPrivateAddresses: true, // fallback armed; it still must not fire + }) + + _, err := client.ResolveHandle(context.Background(), "alice@lemmy.test") + require.Error(t, err) + tc.check(t, err) + seen := transport.seen() + require.Len(t, seen, 1, "no second (http) request may be issued") + assert.True(t, strings.HasPrefix(seen[0], "https://"), "request was %s", seen[0]) + }) + } +} + +// TestResolveHandle_NoHTTPFallbackWhenGuardOn: production posture +// (AllowPrivateAddresses=false) never falls back to http, even for a +// transport-level https failure. Scripted transport: nothing is dialed. +func TestResolveHandle_NoHTTPFallbackWhenGuardOn(t *testing.T) { + transport := &scriptedTransport{handler: func(req *http.Request) (*http.Response, error) { + require.Equal(t, "https", req.URL.Scheme, + "a guard-on client must never issue a plaintext webfinger request") + return nil, fmt.Errorf("simulated TLS handshake failure") + }} + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + HTTPClient: &http.Client{Transport: transport}, + MaxAttempts: 1, + // AllowPrivateAddresses deliberately false: guard on. + }) + + _, err := client.ResolveHandle(context.Background(), "alice@lemmy.test") + require.Error(t, err) + seen := transport.seen() + require.Len(t, seen, 1, "the https failure must be terminal: no http fallback attempt") + assert.True(t, strings.HasPrefix(seen[0], "https://"), "request was %s", seen[0]) +} + +// TestResolveHandle_FallbackFailureSurfacesBothLegs: when the http fallback +// itself fails with a non-404, neither leg's error may be discarded — the +// https transport failure is reported alongside the fallback's HTTPError +// (which stays errors.As-able). +func TestResolveHandle_FallbackFailureSurfacesBothLegs(t *testing.T) { + transport := &scriptedTransport{handler: func(req *http.Request) (*http.Response, error) { + if req.URL.Scheme == "https" { + return nil, fmt.Errorf("simulated TLS handshake failure") + } + return scriptedResponse(http.StatusInternalServerError, ""), nil + }} + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + HTTPClient: &http.Client{Transport: transport}, + MaxAttempts: 1, + AllowPrivateAddresses: true, + }) + + _, err := client.ResolveHandle(context.Background(), "alice@lemmy.test") + require.Error(t, err) + assert.Contains(t, err.Error(), "http fallback") + assert.Contains(t, err.Error(), "simulated TLS handshake failure", + "the https leg's transport error must stay visible") + var httpErr HTTPError + require.ErrorAs(t, err, &httpErr, "the fallback's HTTPError must be wrapped, not discarded") + assert.Equal(t, http.StatusInternalServerError, httpErr.StatusCode) +} + func TestResolveHandle_PrefersActivityJSONLink(t *testing.T) { var host string _, client, h := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { diff --git a/internal/config/config.go b/internal/config/config.go index 850f4fb..1f84e18 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -31,6 +31,12 @@ type Config struct { // BridgeHostname is the public domain the bridge is served from, // e.g. "tidepool.example". Used for WebFinger, actor IDs, and handles. BridgeHostname string + // BridgeScheme is the URL scheme the bridge's own AP URLs (service + // actor id, inbox, activity ids, nodeinfo) are built with. BRIDGE_SCHEME, + // default "https". "http" is only accepted in development — it exists for + // the local e2e harness, where a debug-mode Lemmy federates with the + // bridge over plain HTTP inside one compose network. + BridgeScheme string // PLCDirectoryURL is the did:plc directory used to mint and resolve DIDs. // The dev default is a LOCAL directory (docker compose `plc` profile); // production points at https://plc.directory. Nothing ever falls back to @@ -138,6 +144,23 @@ func Load(logger *slog.Logger) (*Config, error) { return nil, err } + // The bridge's own URL scheme. https everywhere real; http exists so the + // e2e harness can federate with a debug-mode Lemmy over plain HTTP, and is + // refused outside development (like ALLOW_PRIVATE_FETCH). + cfg.BridgeScheme = os.Getenv("BRIDGE_SCHEME") + switch cfg.BridgeScheme { + case "": + cfg.BridgeScheme = "https" + case "https": + case "http": + if !isDevelopment { + return nil, fmt.Errorf("config: BRIDGE_SCHEME=http must not be set in production") + } + logger.Warn("BRIDGE_SCHEME=http: bridge AP URLs are plain HTTP (local federation only)") + default: + return nil, fmt.Errorf("config: BRIDGE_SCHEME must be http or https, got %q", cfg.BridgeScheme) + } + // The dev-default KEK is fixed and public (sha256 of a known string): // fine for local development, catastrophic in production, hence the // required-in-production rule shared with every other stringVar. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 787f8dc..e40a6d9 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -19,7 +19,7 @@ func clearConfigEnv(t *testing.T) { "ENVIRONMENT", "DATABASE_URL", "LISTEN_ADDR", "BRIDGE_HOSTNAME", "PLC_DIRECTORY_URL", "BRIDGE_SERVICE_DID", "USER_AGENT", "BRIDGE_KEK", "ADMIN_TOKEN", "BACKFILL_MAX_POSTS", "MINT_RATE_PER_MINUTE", - "MINT_BURST", "INGEST_WORKERS", + "MINT_BURST", "INGEST_WORKERS", "BRIDGE_SCHEME", } { t.Setenv(name, "") } @@ -138,3 +138,38 @@ func TestLoad_RejectsUnknownEnvironment(t *testing.T) { _, err := Load(discardLogger()) require.Error(t, err) } + +func TestLoad_BridgeScheme(t *testing.T) { + clearConfigEnv(t) + + cfg, err := Load(discardLogger()) + require.NoError(t, err) + assert.Equal(t, "https", cfg.BridgeScheme, "default scheme is https") + + t.Setenv("BRIDGE_SCHEME", "http") + cfg, err = Load(discardLogger()) + require.NoError(t, err) + assert.Equal(t, "http", cfg.BridgeScheme, "http is allowed in development") + + t.Setenv("BRIDGE_SCHEME", "gopher") + _, err = Load(discardLogger()) + require.Error(t, err, "unknown schemes are rejected") +} + +func TestLoad_BridgeSchemeHTTPRefusedInProduction(t *testing.T) { + clearConfigEnv(t) + t.Setenv("ENVIRONMENT", EnvironmentProduction) + t.Setenv("DATABASE_URL", "postgres://prod/db") + t.Setenv("LISTEN_ADDR", ":8080") + t.Setenv("BRIDGE_HOSTNAME", "tidepool.example") + t.Setenv("PLC_DIRECTORY_URL", "https://plc.directory") + t.Setenv("BRIDGE_KEK", "sfDrM4bIeCJp01ZBTArLPJXNQlD7pcYFsod2An6UAF0=") + t.Setenv("ADMIN_TOKEN", "prod-admin-token") + t.Setenv("BRIDGE_SCHEME", "http") + + _, err := Load(discardLogger()) + require.Error(t, err) + assert.Contains(t, err.Error(), "BRIDGE_SCHEME") + assert.Contains(t, err.Error(), "production", + "the refusal must come from the production branch, not generic scheme validation") +} diff --git a/internal/identity/minter.go b/internal/identity/minter.go index f4a104a..5061d8b 100644 --- a/internal/identity/minter.go +++ b/internal/identity/minter.go @@ -67,12 +67,15 @@ type Identity struct { type Minter struct { plcURL string bridgeHostname string - rotationKey *atcrypto.PrivateKeyK256 - custodian *Custodian - actors store.BridgedActors - httpClient *http.Client - userAgent string - logger *slog.Logger + // bridgeScheme is the URL scheme of the advertised PDS endpoint + // ("https" everywhere real; "http" only under the local e2e harness). + bridgeScheme string + rotationKey *atcrypto.PrivateKeyK256 + custodian *Custodian + actors store.BridgedActors + httpClient *http.Client + userAgent string + logger *slog.Logger } // MinterOptions configures NewMinter. All fields except HTTPClient are @@ -84,6 +87,11 @@ type MinterOptions struct { // BridgeHostname anchors the handle space and the PDS endpoint // (config.BridgeHostname). BridgeHostname string + // BridgeScheme is the URL scheme of the PDS endpoint advertised in + // minted DID documents (config.BridgeScheme). "https" everywhere real; + // "http" exists for the local e2e harness. Empty means "https" — the + // same defensive default as ap.ServiceActor.BaseURL. + BridgeScheme string // RotationKey is the bridge escrow rotation key // (LoadOrCreateRotationKey). RotationKey *atcrypto.PrivateKeyK256 @@ -113,6 +121,15 @@ func NewMinter(opts MinterOptions) (*Minter, error) { if opts.BridgeHostname == "" { return nil, errors.NewValidationError("bridge_hostname", "must not be empty") } + scheme := opts.BridgeScheme + switch scheme { + case "": + scheme = "https" + case "http", "https": + default: + return nil, errors.NewValidationError("bridge_scheme", + fmt.Sprintf("must be http or https, got %q", opts.BridgeScheme)) + } if opts.RotationKey == nil { return nil, errors.NewValidationError("rotation_key", "must not be nil") } @@ -137,6 +154,7 @@ func NewMinter(opts MinterOptions) (*Minter, error) { return &Minter{ plcURL: strings.TrimRight(opts.PLCDirectoryURL, "/"), bridgeHostname: strings.ToLower(opts.BridgeHostname), + bridgeScheme: scheme, rotationKey: opts.RotationKey, custodian: opts.Custodian, actors: opts.Actors, @@ -185,7 +203,7 @@ func (m *Minter) MintActor(ctx context.Context, req MintRequest) (*Identity, err } op, err := m.signGenesisOp(genesisOperation( - rotationPub.DIDKey(), signingPub.DIDKey(), handle, "https://"+m.bridgeHostname)) + rotationPub.DIDKey(), signingPub.DIDKey(), handle, m.pdsEndpoint())) if err != nil { return nil, err } @@ -221,6 +239,12 @@ func (m *Minter) MintActor(ctx context.Context, req MintRequest) (*Identity, err }, nil } +// pdsEndpoint is the PDS service endpoint advertised in every minted DID +// document: the bridge itself, under the configured scheme. +func (m *Minter) pdsEndpoint() string { + return m.bridgeScheme + "://" + m.bridgeHostname +} + // availableHandle builds the bridged handle and suffixes it (-2, -3, ...) // until it does not collide with an already-assigned handle. func (m *Minter) availableHandle(ctx context.Context, username, instance string) (string, error) { diff --git a/internal/identity/minter_test.go b/internal/identity/minter_test.go index d2dbb6e..697c090 100644 --- a/internal/identity/minter_test.go +++ b/internal/identity/minter_test.go @@ -112,6 +112,12 @@ func TestGenesisOpEncoding(t *testing.T) { op := genesisOperation("did:key:zRotation", "did:key:zSigning", "technology.lemmy-world.tidepool.example", "https://tidepool.example") + services, ok := op["services"].(map[string]any) + require.True(t, ok) + pds, ok := services["atproto_pds"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "https://tidepool.example", pds["endpoint"], + "the op must advertise the endpoint exactly as passed (scheme included)") op["sig"] = "fakesig" did1, err := didForOperation(op) require.NoError(t, err) @@ -346,3 +352,44 @@ func TestNewMinter_RequiresExplicitConfig(t *testing.T) { assert.True(t, errors.IsValidation(err)) assert.NotContains(t, err.Error(), "plc.directory") } + +// TestNewMinter_PDSEndpointScheme pins how BridgeScheme threads into the PDS +// endpoint advertised in minted DID documents: empty defaults to https (the +// same defensive default as ap.ServiceActor.BaseURL), http is honored (local +// e2e harness, BRIDGE_SCHEME=http), anything else is rejected. Needs no PLC +// or postgres: nothing is dialed. +func TestNewMinter_PDSEndpointScheme(t *testing.T) { + rotationKey, err := atcrypto.GeneratePrivateKeyK256() + require.NoError(t, err) + base := MinterOptions{ + PLCDirectoryURL: "https://plc.invalid", + BridgeHostname: "Bridge.Example", + RotationKey: rotationKey, + Custodian: testCustodian(t), + Actors: struct{ store.BridgedActors }{}, + HTTPClient: http.DefaultClient, + } + + cases := []struct { + scheme string + want string + wantErr bool + }{ + {scheme: "", want: "https://bridge.example"}, + {scheme: "https", want: "https://bridge.example"}, + {scheme: "http", want: "http://bridge.example"}, + {scheme: "gopher", wantErr: true}, + } + for _, tc := range cases { + opts := base + opts.BridgeScheme = tc.scheme + m, err := NewMinter(opts) + if tc.wantErr { + require.Error(t, err, "scheme %q", tc.scheme) + assert.True(t, errors.IsValidation(err), "scheme %q must be a validation error", tc.scheme) + continue + } + require.NoError(t, err, "scheme %q", tc.scheme) + assert.Equal(t, tc.want, m.pdsEndpoint(), "scheme %q", tc.scheme) + } +} diff --git a/internal/ingest/follow.go b/internal/ingest/follow.go index acb2c91..6649b0a 100644 --- a/internal/ingest/follow.go +++ b/internal/ingest/follow.go @@ -380,7 +380,7 @@ func (a *Admin) activityID(kind string) (string, error) { if _, err := rand.Read(buf[:]); err != nil { return "", fmt.Errorf("ingest: mint activity id: %w", err) } - return fmt.Sprintf("https://%s/activities/%s/%s", a.service.Hostname, kind, hex.EncodeToString(buf[:])), nil + return fmt.Sprintf("%s/activities/%s/%s", a.service.BaseURL(), kind, hex.EncodeToString(buf[:])), nil } // writeJSON writes a JSON response body. diff --git a/internal/ingest/inbox.go b/internal/ingest/inbox.go index 0378e8b..e7fab70 100644 --- a/internal/ingest/inbox.go +++ b/internal/ingest/inbox.go @@ -182,7 +182,7 @@ func orderingKeyFor(activity *ap.Object, boundActor string) string { return boundActor } -// handleActor serves the bridge's Application actor document (Lemmy fetches +// handleActor serves the bridge's Service actor document (Lemmy fetches // it to validate our Follow signatures). func (ib *Inbox) handleActor(w http.ResponseWriter, _ *http.Request) { doc, err := ib.service.DocumentJSON() @@ -224,7 +224,7 @@ func (ib *Inbox) handleNodeInfoDiscovery(w http.ResponseWriter, _ *http.Request) _ = json.NewEncoder(w).Encode(map[string]any{ "links": []any{map[string]any{ "rel": "http://nodeinfo.diaspora.software/ns/schema/2.0", - "href": "https://" + ib.service.Hostname + "/nodeinfo/2.0", + "href": ib.service.BaseURL() + "/nodeinfo/2.0", }}, }) } diff --git a/internal/ingest/inbox_test.go b/internal/ingest/inbox_test.go index a7b4c11..67d76fd 100644 --- a/internal/ingest/inbox_test.go +++ b/internal/ingest/inbox_test.go @@ -207,7 +207,7 @@ func TestServiceActorEndpoints(t *testing.T) { actor, err := ap.ParseObject(rec.Body.Bytes()) require.NoError(t, err) assert.Equal(t, h.service.ID, actor.ID) - assert.Equal(t, ap.TypeApplication, actor.Type) + assert.Equal(t, ap.TypeService, actor.Type) require.NotNil(t, actor.PublicKey) assert.Equal(t, h.service.KeyID(), actor.PublicKey.ID) assert.NotEmpty(t, actor.PublicKey.PublicKeyPem) @@ -231,6 +231,8 @@ func TestServiceActorEndpoints(t *testing.T) { } require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &discovery)) require.NotEmpty(t, discovery.Links) + assert.Equal(t, "https://"+bridgeHost+"/nodeinfo/2.0", discovery.Links[0].Href, + "discovery href must be the BaseURL()-derived nodeinfo route") rec = get("/nodeinfo/2.0") require.Equal(t, http.StatusOK, rec.Code) diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go index 5bb5236..6cbb22c 100644 --- a/internal/ingest/ingest_test.go +++ b/internal/ingest/ingest_test.go @@ -530,6 +530,8 @@ func (h *harness) subscribeTechnology() *remoteActor { require.Equal(h.t, ap.TypeFollow, follow.Type) require.Equal(h.t, h.service.ID, follow.Actor.ID) require.Equal(h.t, groupID, follow.Object.ID) + require.True(h.t, strings.HasPrefix(follow.ID, "https://"+bridgeHost+"/activities/follow/"), + "follow activity id %q must live under the bridge's BaseURL", follow.ID) // Lemmy answers with Accept{Follow}, signed by the community. status := h.deliver(group, map[string]any{ diff --git a/lexicons/MANIFEST.sha256 b/lexicons/MANIFEST.sha256 new file mode 100644 index 0000000..395aad6 --- /dev/null +++ b/lexicons/MANIFEST.sha256 @@ -0,0 +1,73 @@ +a9fc756ecba06f930a1788577b9f67ffaf9e7e287fcff1e019497730ff16bdc3 com/atproto/label/defs.json +2d27ecc2a09a86b5065c36e36b987ef726a814023c5e198a1eda8e4c283714e0 com/atproto/repo/strongRef.json +ea71c00445b75f14638c95d77e990daff3ec72a28eb9cd700ad03bb37754178c social/coves/actor/block.json +14395780834c57e49b0834b1907b309c213a9539952a86adf485d843f5d256d6 social/coves/actor/defs.json +7a61da4dfe453345f7353454b448f07db03e87ec381138a71a9533d3d532cba8 social/coves/actor/getComments.json +1cd0164cbf446c53c63c838745efef69b71b2e58a4e1292df9db70df80ebe01d social/coves/actor/getPosts.json +1df537f5e659c09a109233d1acafbfec1195e5bd168f581a9e6fd20d61a3490d social/coves/actor/getProfile.json +f4f7d4cbf5963fa9bd574af2373a62a6ed665965698ad3d443341067c1193204 social/coves/actor/profile.json +559f7d5c498407dfe171eeb847442aee2b894919660ac8c16ba7564d7e4f5abb social/coves/actor/signup.json +034873e4e786105f2755bb6e98ecb33381b2805ce1757683f7b58d2f82058158 social/coves/actor/updateProfile.json +515160f8f83251c009fb6998604e29b12337ed33cb18282e8d7c345039b44ce0 social/coves/aggregator/authorization.json +35853ccc526b245e09364f778bd696771542c8e3e188f7ae5289279ba1da89ab social/coves/aggregator/createApiKey.json +7ed89fa725f1cc74944b8ed6be765e9da65abd9bbcdd90ce41b613f2b08e3f00 social/coves/aggregator/defs.json +692918b0689ecbdd4dc21b355995f4c7449ab79457fedb82e483ba299b571c66 social/coves/aggregator/disable.json +f5db9e63ee5f7143eab4d6d80b5a005414125b3c6eebc9bdaa310f126df80960 social/coves/aggregator/enable.json +1746d594f645cf64aea15ad12a0d0f8cd343b613bdf885fed366933467e26a97 social/coves/aggregator/getApiKey.json +5b7bc57ed5ec1c0b310e81c3cd92ea34d26ec184ec862edb6d9d5ed882cd7bc2 social/coves/aggregator/getAuthorizations.json +acc0984b504e8979db4193d0ad9f093a5fb5792d980fe463459fbac91f6fbb9a social/coves/aggregator/getServices.json +4b842cb6b09df9871d0d514373dec8b3ea70a673f068f9c4e5952ebe8750acbf social/coves/aggregator/listForCommunity.json +a4ad2cdb63a07658d2a54d1bc9a766e1bc99d9ab1229df248a9ca8dd2de3905b social/coves/aggregator/register.json +d3e077e34c9b9ccd8a8892148fe4ce7ab1de65873250a8906aec510d040bfa4c social/coves/aggregator/revokeApiKey.json +0ca7c339793fc8063a312bf39619983d98414fba498b05fee92c135c7ebea0fd social/coves/aggregator/service.json +d688327a711491aeb58b75187f468f213d270adad0c0ca59b96fbef0229cd3f4 social/coves/aggregator/updateConfig.json +020b4a33837455e17e1b0e258304f11241931d14929be099f33ca67a88fc2f49 social/coves/bridge/getVoteAggregates.json +88fb6259698d0097995200ed5d3a7887165f9cd29907c1c60880150c6568aebf social/coves/community/block.json +b8ae0bf6561a53785970b3da1db9007a07ecb8a1f4db527c5461f8ca80da8124 social/coves/community/comment.json +4ff4e1b35004b4757f25778922ec948fb7abaff87c7c46b46e1dadfdc7ab5217 social/coves/community/comment/create.json +153c147d2d307182e91e880c5ce8209090247de1ecefed6b3b776502a23991fe social/coves/community/comment/defs.json +50e2528f308e045b003e6c7d05bdadb9b04d6f4b43b249db8539af4a842b4447 social/coves/community/comment/delete.json +fbef83f475cc78f1ac7a3ce7eab2841b58db51143fe37f8d00883b612e707256 social/coves/community/comment/getComments.json +42e85f1294bba8a5a4805bccb41cc0143d1948b449b07de73926cb70b1f9cee6 social/coves/community/comment/update.json +ea5cac9a27aca28f818d87a90a01e9564920de9e96e6f3606d71daa0ea6294a8 social/coves/community/create.json +64ae5882324d7fe5573e8e0f130b84ac6c67a8fde0c8f47d3741ccce179e234d social/coves/community/defs.json +965b99bbc37ebe3686bd1f5244c3cefca831455e5e6ff7c9de0e1f0556d5af27 social/coves/community/get.json +69efc066cb3c589eb14e0a496d0b5e6a816f12ca086fc701b94ceb8a55e52bed social/coves/community/getMembers.json +c51d12359017474850b5b9bb682154f3c58ebd96d5975ba5103853bc9fa640d2 social/coves/community/getSubscribers.json +4f5fc7cf0aa4b8f17205a28664acbc6b03d398850b336f64434ba113939ae47c social/coves/community/list.json +78608976d210bdd66eb7005e60e99de3394d2c73129cd9a8d914a1bd798b80e4 social/coves/community/moderator.json +f412d89df7ad9cd3afc3cbd35b23b6a77abc857f713e0e56545b543c2e5794c8 social/coves/community/post.json +a93fda24dd3c895283a2ddf5b4b4021061271a81ce2d338cba615bd77d89c04b social/coves/community/post/create.json +e233f344054fb75b2d28744ff8975c560aa29210c652cc4b59de635eb637695b social/coves/community/post/delete.json +02811d7e45ce19ea52b774dcfeba2475d14266a0386780e8d98cc3064d2a1770 social/coves/community/post/get.json +9ea7aba6041acf28e96769b4006c09ea14d5153303fffe3ebc78f889c14ae58f social/coves/community/post/search.json +755643fcce528a4680272911a90f360a7b3eb6253272ff4cb948b5be2bdc4038 social/coves/community/post/update.json +b0d6b7dfc28a9efad6f4c7023fcab909ff420b807491a92f6c6770c79ed69706 social/coves/community/profile.json +fd3c8d491b4097e4d886dbf5dad695e10c75ae3336c869c31b19d040ccf54360 social/coves/community/rules.json +a2ccfb1207088f6dc09e8f2462c89108f104a61e5870382268b3e64b7dd15535 social/coves/community/search.json +ee2c0abaa92ae411ade480c38ffbf6b7ed97eb5e8d27c202cc5d05bbbed825f0 social/coves/community/subscribe.json +b981a408a1acce35783c387a35d7ee99985664d52e29a204bd5cf8bbd2b2741b social/coves/community/subscription.json +2ff387f14368acc38ccb4cf7a281a37a593e8552499ace6098266b80576055e7 social/coves/community/unsubscribe.json +7a88c19870745951419b1026164718e96d9fc8d37a06f16e3e5ea9418516a47b social/coves/community/update.json +d7dd43774cb3d1f813ec6c2e8023e26263a386b213473d8b793b7cfc0cd0adff social/coves/community/wiki.json +8161d253f96b96ab03fcfc210a0a7167943136a9b623e3e4888a18898fe07b5f social/coves/embed/external.json +63fd0777aabeaeafe5347aed875edee224ecb72f153042d2d2c900a71dd8c850 social/coves/embed/images.json +cbba7f146da90e335349e5cb93199d35b8aeac391511a07b2108a51e0545d75f social/coves/embed/post.json +8c4494b55bdb1cc6080407b6e2820f68fc0596fc08cc83b3f68455d38ca4a094 social/coves/embed/video.json +113c652062ce52ee91d4c84aad37a999dae140181fe6edf3f0f41d763feee37e social/coves/feed/defs.json +1f41725f8b7eb54db891ad6e64aee6f2fe8712a7fb12a3e598d24c3472a38d4b social/coves/feed/getAll.json +756eada7383f692eb1df0d6ce054fc01271b4e6d0e7b69734a5952644aadf42f social/coves/feed/getCommunity.json +7d9e45e7654ad7276bf33e4187230299a0c795e6def1e377cf8c2385c79aa58e social/coves/feed/getDiscover.json +41d08276ce1fb8740d0b1ec4d99b0b1ea8b2c296f532e4fe8e4eb04c611a81d4 social/coves/feed/getTimeline.json +6d3d3ba6f0d64997abff0724f9f0dceef5e2eadc39260553d4ae6a8e05325edd social/coves/feed/vote.json +7d3d332472324233b1f4547f05bf4f88bb419ba0fc7dcf17f3a0de3069b0a004 social/coves/feed/vote/create.json +3faff528d5126e0300c62d7806d0976fa29d0ec4cef060e08d6af4a166f442e6 social/coves/feed/vote/delete.json +a6c7a26e8ee6c47a63c5d1f1b398ff9960c48e459c7613be744ad81e02aa1c72 social/coves/moderation/ban.json +c9c6c99a99ae029968c0e07421ccf0a077bc936c605c55bc8b5c7464172ee4b3 social/coves/moderation/banUser.json +28fa8b7fb472ad8dd786325d7b542e34af6ae29d1bc9b95824823eb80abdd9e2 social/coves/moderation/getBanStatus.json +4250f121a4ebd3ccc1de4cf061d6717a73c6bdad399a64388fa5b64c9b560b97 social/coves/moderation/listBans.json +c9de6b733b4e0e5273c52fc800db4539961d426397f359827c571366eb6045ec social/coves/moderation/ruleProposal.json +c1f68117c971732632f471be4f20fd211610dbe681791742846bb55f769d12ed social/coves/moderation/tribunalVote.json +9a5aba0061350e6458a87f3de8d4b1d86b04ef61da7e315b8cb7edad84c5523b social/coves/moderation/unbanUser.json +2ccd0a9647d43ed048a315bf4c639c09b79120c204f82610894a3c25cb7c4fac social/coves/moderation/vote.json +9d4b4c380874cdcee7aa71711d5a333629ef7340dc8ef28fe2ce11a1b3e38082 social/coves/richtext/facet.json diff --git a/scripts/check-lexicons.sh b/scripts/check-lexicons.sh new file mode 100755 index 0000000..23c8853 --- /dev/null +++ b/scripts/check-lexicons.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Verifies the vendored lexicons are in sync. +# +# Two layers: +# 1. Manifest check (always): every vendored lexicon file must match +# lexicons/MANIFEST.sha256, and no unlisted json files may exist. The +# manifest is (re)written by scripts/sync-lexicons.sh, so this catches +# hand-edited vendored files and half-done syncs — including in CI, +# where no Coves checkout exists. +# 2. Upstream drift check (when a Coves checkout is available, default +# ~/Code/coves or $1): the vendored tree must byte-match what +# sync-lexicons.sh would copy today. Skipped gracefully when the +# checkout is absent (CI). +# +# Exit non-zero on any drift. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +DST="$ROOT/lexicons" +COVES="${1:-$HOME/Code/coves}" + +# ── 1. Manifest check ────────────────────────────────────────────────────── +if [ ! -f "$DST/MANIFEST.sha256" ]; then + echo "error: $DST/MANIFEST.sha256 missing — run scripts/sync-lexicons.sh" >&2 + exit 1 +fi + +cd "$DST" +if ! shasum -a 256 --check --quiet MANIFEST.sha256; then + echo "error: vendored lexicons do not match MANIFEST.sha256 — run scripts/sync-lexicons.sh" >&2 + exit 1 +fi +# Files present but not listed (a hash check alone can't see additions). +listed=$(awk '{print $2}' MANIFEST.sha256 | sort) +actual=$(find social com -name '*.json' | sort) +if [ "$listed" != "$actual" ]; then + echo "error: vendored lexicon file set differs from MANIFEST.sha256:" >&2 + diff <(echo "$listed") <(echo "$actual") >&2 || true + echo "run scripts/sync-lexicons.sh" >&2 + exit 1 +fi +echo "ok: vendored lexicons match MANIFEST.sha256 ($(echo "$listed" | wc -l | tr -d ' ') files)" + +# ── 2. Upstream drift check ──────────────────────────────────────────────── +SRC="$COVES/internal/atproto/lexicon" +# Only a truly-absent checkout is a skip; a Coves checkout that exists but +# is unreadable or missing its lexicon dir is an error, not silence. +if [ ! -e "$COVES" ]; then + echo "skip: Coves checkout not found at $COVES — upstream drift not checked (fine in CI)" + exit 0 +fi +if [ ! -d "$SRC" ] || [ ! -r "$SRC" ] || [ ! -x "$SRC" ]; then + echo "error: Coves checkout exists at $COVES but $SRC is not a readable directory" >&2 + exit 1 +fi +if [ ! -d "$SRC/social/coves" ]; then + echo "error: $SRC/social/coves missing — $COVES does not look like a Coves checkout" >&2 + exit 1 +fi + +drift=0 +# Everything sync-lexicons.sh copies must byte-match — including the +# vendored com/atproto files (strongRef, label defs), which are manually +# curated cherry-picks: divergence of a vendored com/ file is drift, but a +# NEW upstream com/ file is not (sync-lexicons.sh would not copy it). +# Lexicons under social/coves/bridge/ are Tidepool-owned (the vote-aggregate +# side channel) and intentionally absent from Coves. +while IFS= read -r rel; do + case "$rel" in + social/coves/bridge/*) continue ;; + social/coves/*) src_file="$SRC/$rel" ;; + com/*) src_file="$SRC/$rel" ;; + *) continue ;; + esac + if [ ! -f "$src_file" ]; then + echo "drift: $rel is vendored but no longer exists in Coves" >&2 + drift=1 + elif ! cmp -s "$src_file" "$rel"; then + echo "drift: $rel differs from Coves" >&2 + drift=1 + fi +done <<<"$actual" + +# New upstream files sync would pick up (social/coves only — new com/ files +# are deliberately not flagged, see above). The find output is captured up +# front so a find failure aborts the script (set -e) instead of silently +# feeding an empty loop via process substitution. +upstream=$(find "$SRC/social/coves" -name '*.json' | sort) +while IFS= read -r src_file; do + [ -n "$src_file" ] || continue + rel="${src_file#"$SRC/"}" + if [ ! -f "$rel" ]; then + echo "drift: Coves has new lexicon $rel not vendored yet" >&2 + drift=1 + fi +done <<<"$upstream" + +if [ "$drift" -ne 0 ]; then + echo "error: vendored lexicons drifted from $SRC — run scripts/sync-lexicons.sh" >&2 + exit 1 +fi +echo "ok: vendored lexicons in sync with $SRC" diff --git a/scripts/sync-lexicons.sh b/scripts/sync-lexicons.sh index e947973..d4db362 100755 --- a/scripts/sync-lexicons.sh +++ b/scripts/sync-lexicons.sh @@ -20,6 +20,15 @@ if [ ! -d "$SRC" ]; then exit 1 fi +# Tidepool OWNS lexicons under social/coves/bridge/ (the vote-aggregate +# side channel, task 07) — they do not exist in Coves and must survive a +# re-sync. +BRIDGE_TMP="$(mktemp -d)" +trap 'rm -rf "$BRIDGE_TMP"' EXIT +if [ -d "$DST/social/coves/bridge" ]; then + cp -R "$DST/social/coves/bridge" "$BRIDGE_TMP/bridge" +fi + rm -rf "$DST/social" "$DST/com" # Everything under social/coves (records Tidepool emits plus every ref @@ -28,9 +37,27 @@ mkdir -p "$DST/social" rsync -a --include='*/' --include='*.json' --exclude='*' \ "$SRC/social/coves/" "$DST/social/coves/" +# Restore the bridge-owned lexicons. If upstream Coves ever ships its own +# social/coves/bridge/, the rsync above will have created the destination +# and a blind cp -R would nest bridge/bridge/ and silently shadow the +# Tidepool-owned lexicons — that namespace collision needs a human. +if [ -d "$BRIDGE_TMP/bridge" ]; then + if [ -e "$DST/social/coves/bridge" ]; then + echo "error: upstream Coves now ships social/coves/bridge/ — namespace collision" >&2 + echo "with the Tidepool-owned bridge lexicons. Resolve manually (rename one side" >&2 + echo "or reconcile the schemas); the pre-sync bridge files are preserved in" >&2 + echo "$BRIDGE_TMP (removed on exit)." >&2 + exit 1 + fi + cp -R "$BRIDGE_TMP/bridge" "$DST/social/coves/bridge" +fi + # The com.atproto refs the social.coves records use. mkdir -p "$DST/com/atproto/repo" "$DST/com/atproto/label" cp "$SRC/com/atproto/repo/strongRef.json" "$DST/com/atproto/repo/strongRef.json" cp "$SRC/com/atproto/label/defs.json" "$DST/com/atproto/label/defs.json" +# Refresh the manifest scripts/check-lexicons.sh verifies (CI drift guard). +(cd "$DST" && find social com -name '*.json' | sort | xargs shasum -a 256 > MANIFEST.sha256) + echo "synced lexicons from $SRC to $DST"