From 35b5fca5adeaf401c84dd54591cd0dec013f3572 Mon Sep 17 00:00:00 2001 From: Eli Mallon Date: Mon, 12 Oct 2026 00:31:43 +0000 Subject: [PATCH] Add user-scoped upstream station mode for Desktop nodes --- .claude/skills/streamplace-docker/SKILL.md | 6 +- .github/workflows/build.yaml | 57 ++- .maestro/README.md | 70 +++ .maestro/logged-in/upstream-station.yaml | 62 +++ go.mod | 4 +- go.sum | 8 +- hack/e2e-local.sh | 39 +- hack/e2e-web-local.sh | 10 +- hack/lib/e2e-harness.sh | 30 +- hack/maestro-config.sh | 13 +- js/app/app.config.ts | 6 +- js/app/features/bluesky/oauthClient.tsx | 143 ++---- js/app/features/bluesky/oauthResolver.tsx | 54 +-- js/app/store/slices/blueskySlice.ts | 8 - js/desktop/README.md | 26 ++ js/desktop/src/index.ts | 26 +- js/desktop/src/node.ts | 12 +- js/desktop/src/tests/server-restart-test.ts | 1 + js/desktop/src/tests/test-runner.ts | 1 + .../installing/downloading-streamplace.md | 42 ++ js/e2e-web/README.md | 54 +++ .../flows/desktop-upstream-station.spec.ts | 147 +++++++ js/e2e-web/flows/login.ts | 52 ++- js/e2e-web/flows/upstream-station.spec.ts | 411 ++++++++++++++++++ js/e2e-web/global-setup.ts | 43 +- js/e2e-web/server-setup.ts | 2 +- js/web/src/lib/oauth-resolver.ts | 111 +---- js/web/src/lib/oauth.ts | 127 ++---- pkg/api/feeds.go | 2 +- pkg/atproto/atproto.go | 70 ++- pkg/atproto/backfill_walk.go | 37 +- pkg/atproto/firehose.go | 74 ++-- pkg/atproto/firehose_moq.go | 26 +- pkg/atproto/headcheck.go | 9 + pkg/atproto/indexing_scope.go | 64 +++ pkg/atproto/indexing_scope_test.go | 248 +++++++++++ pkg/atproto/label_verification.go | 6 + pkg/atproto/labeler_firehose.go | 12 + pkg/atproto/sweep.go | 19 + pkg/atproto/sync.go | 3 + pkg/atproto/upstream_firehose.go | 72 +++ pkg/atproto/upstream_station.go | 44 ++ pkg/atproto/upstream_station_test.go | 76 ++++ pkg/atproto/verification.go | 6 + pkg/cmd/e2e.go | 146 ++++--- pkg/cmd/e2e_https.go | 105 ++++- pkg/cmd/e2e_upstream.go | 289 ++++++++++++ pkg/cmd/streamplace.go | 82 +++- pkg/config/config.go | 48 +- pkg/config/config_check_test.go | 9 + pkg/spmetrics/spmetrics.go | 8 +- pkg/spxrpc/com_atproto_repo.go | 34 +- pkg/spxrpc/com_atproto_repo_test.go | 159 +++++-- pkg/spxrpc/pds_auth_test.go | 2 +- pkg/spxrpc/place_stream_game.go | 2 +- pkg/spxrpc/spxrpc.go | 6 +- pkg/statedb/indexing_users.go | 38 ++ pkg/statedb/indexing_users_test.go | 79 ++++ pkg/statedb/oauth_session.go | 10 +- pkg/statedb/oauth_session_test.go | 37 ++ util/streamplace-dev.sh | 2 +- 61 files changed, 2799 insertions(+), 590 deletions(-) create mode 100644 .maestro/logged-in/upstream-station.yaml create mode 100644 js/e2e-web/flows/desktop-upstream-station.spec.ts create mode 100644 js/e2e-web/flows/upstream-station.spec.ts create mode 100644 pkg/atproto/indexing_scope.go create mode 100644 pkg/atproto/indexing_scope_test.go create mode 100644 pkg/atproto/upstream_firehose.go create mode 100644 pkg/atproto/upstream_station.go create mode 100644 pkg/atproto/upstream_station_test.go create mode 100644 pkg/cmd/e2e_upstream.go create mode 100644 pkg/statedb/indexing_users.go create mode 100644 pkg/statedb/indexing_users_test.go diff --git a/.claude/skills/streamplace-docker/SKILL.md b/.claude/skills/streamplace-docker/SKILL.md index a613afbc..9d7e239e 100644 --- a/.claude/skills/streamplace-docker/SKILL.md +++ b/.claude/skills/streamplace-docker/SKILL.md @@ -155,9 +155,9 @@ specific command. Do not assume a sibling exists or modify its artifacts. ## Isolated scratch node `make dev` builds binaries; it does **not** start a server. The generated -`build-linux-amd64/streamplace` launcher assigns a frontend proxy to port 38081, -so an environment override cannot disable it. For a scratch node serving the -embedded bundle, invoke `libstreamplace` directly. +`build-linux-amd64/streamplace` launcher defaults to a frontend proxy on port 38081. Set `SP_DEV_FRONTEND_PROXY=` to serve the embedded bundle instead, or +invoke `libstreamplace` directly as below. Rebuild with `make app && make dev` +after frontend changes. Enter the container from the host: diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 5ae19d68..b045ec49 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -125,10 +125,13 @@ jobs: # Reuse the binary built by linux-amd64 instead of provisioning another # frontend and native build just to run Playwright. e2e-web: - name: e2e-web + name: e2e-web (${{ matrix.mode }}) runs-on: ubuntu-latest timeout-minutes: 20 needs: linux-amd64 + strategy: + matrix: + mode: [standalone, upstream] steps: - name: Check out code uses: actions/checkout@v4.1.7 @@ -145,6 +148,7 @@ jobs: run: npm install -g corepack@latest && corepack enable && pnpm install --filter @streamplace/e2e-web --filter @atcute/internal-dev-env + --filter streamplace-desktop --filter @streamplace/brand --frozen-lockfile - name: Download streamplace binary @@ -165,14 +169,34 @@ jobs: "PLAYWRIGHT_BROWSERS_PATH=$RUNNER_TEMP/playwright-browsers" pnpm --filter @streamplace/e2e-web install-browser + - name: Install atmoq relay fixture + if: matrix.mode == 'upstream' + run: | + curl -fsSL -o "$RUNNER_TEMP/atmoq.tar.gz" https://github.com/streamplace/atmoq/releases/download/v0.1.1/atmoq-v0.1.1-x86_64-unknown-linux-gnu.tar.gz + echo "513ea3c7e0702b4d6ede99c416fee6faca361df4bceeeaefd24edaffc6c59d29 $RUNNER_TEMP/atmoq.tar.gz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/atmoq-bin" + tar -xzf "$RUNNER_TEMP/atmoq.tar.gz" -C "$RUNNER_TEMP/atmoq-bin" + echo "$RUNNER_TEMP/atmoq-bin" >> "$GITHUB_PATH" + - name: Build Desktop e2e entry point + if: matrix.mode == 'upstream' + run: | + sudo apt-get update && sudo apt-get install -y xvfb + pnpm --filter streamplace-desktop package - name: Run web e2e tests - run: hack/e2e-web-local.sh + run: | + if [ "${{ matrix.mode }}" = upstream ]; then + E2E_DESKTOP_MAIN="$PWD/js/desktop/.webpack/x64/main/index.js" xvfb-run -a hack/e2e-web-local.sh --upstream-station -- upstream-station.spec.ts + else + hack/e2e-web-local.sh + fi - name: Upload e2e-web artifacts if: failure() uses: actions/upload-artifact@v4 with: - name: e2e-web-${{ github.event.pull_request.head.sha || github.sha }} + name: + e2e-web-${{ matrix.mode }}-${{ github.event.pull_request.head.sha || + github.sha }} path: | js/e2e-web/test-results/ js/e2e-web/playwright-report/ @@ -227,6 +251,13 @@ jobs: echo "3440825f514f537c6a96bcf5de995780c2a4a7f83a43208fdc95d4f1fecfad3b /tmp/maestro.zip" | sha256sum -c - unzip -q /tmp/maestro.zip -d "$HOME/.maestro-dist" echo "$HOME/.maestro-dist/maestro/bin" >> $GITHUB_PATH + - name: Install atmoq relay fixture + run: | + curl -fsSL -o "$RUNNER_TEMP/atmoq.tar.gz" https://github.com/streamplace/atmoq/releases/download/v0.1.1/atmoq-v0.1.1-x86_64-unknown-linux-gnu.tar.gz + echo "513ea3c7e0702b4d6ede99c416fee6faca361df4bceeeaefd24edaffc6c59d29 $RUNNER_TEMP/atmoq.tar.gz" | sha256sum -c - + mkdir -p "$RUNNER_TEMP/atmoq-bin" + tar -xzf "$RUNNER_TEMP/atmoq.tar.gz" -C "$RUNNER_TEMP/atmoq-bin" + echo "$RUNNER_TEMP/atmoq-bin" >> "$GITHUB_PATH" - uses: actions/cache@v4 id: avd-cache @@ -274,6 +305,26 @@ jobs: script: E2E_HARNESS_LOG=$PWD/.maestro/artifacts/harness.log bash hack/e2e-local.sh android + - name: Run upstream station Maestro flow + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + target: google_apis + arch: x86_64 + profile: pixel_6 + force-avd-creation: false + cores: 4 + ram-size: 6144M + heap-size: 576M + emulator-options: + -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio + -no-boot-anim -camera-back none + disable-animations: true + script: + E2E_ARTIFACTS=$PWD/.maestro/artifacts/upstream + E2E_HARNESS_LOG=$PWD/.maestro/artifacts/upstream/harness.log + E2E_FLOWS=logged-in/upstream-station.yaml bash hack/e2e-local.sh + android --upstream-station - name: upload maestro artifacts if: always() diff --git a/.maestro/README.md b/.maestro/README.md index e4479621..e50a4bee 100644 --- a/.maestro/README.md +++ b/.maestro/README.md @@ -142,6 +142,76 @@ serves HTTPS on 127.0.0.1:443 on that machine. Only set `E2E_HTTPS_PORT` when the redirect is in place: the app and PDS still use `https://` URLs with no port. +## Upstream-station native flow + +The default single-node plan excludes flows tagged `upstream-station`. Opt in +with the real released atmoq v0.1.1 executable (or a locked build of that exact +release if its prebuilt glibc requirement is newer than the host): + +```bash +make android-e2e +SP_E2E_ATMOQ_BIN=/path/to/released/atmoq \ + hack/e2e-local.sh android --upstream-station +# Explicit subset, including server setup and real OAuth prerequisites: +SP_E2E_ATMOQ_BIN=/path/to/released/atmoq \ + E2E_FLOWS=.maestro/logged-in/upstream-station.yaml \ + hack/e2e-local.sh android --upstream-station +``` + +The runner selects the local HTTP `SERVER_URL`, not the HTTPS upstream station. +It creates an owned `adb reverse tcp: tcp:` mapping and +removes it at teardown. Using `10.0.2.2` as the node URL would change the +DPoP-bound issuer/callback, so the app keeps `http://127.0.0.1:`. +The E2E-only network security plugin allows cleartext solely for `127.0.0.1` +and `localhost`; ordinary releases and all other E2E destinations remain +HTTPS-only. The station/PDS still use genuine TLS and the installed throwaway +CA. The harness prints local/station/MoQ URLs, and uses actual per-DID QUIC +subscriptions rather than a mock firehose. + +`setup/oauth-login.yaml` completes the local → HTTPS station → PDS consent +chain. `logged-in/upstream-station.yaml` performs an authenticated chat write, +kills/relaunches the app, checks indexed history rather than an optimistic +bubble, and writes again with the restored OAuth session. It is platform-neutral +and uses the same semantic selectors on Android and iOS. Browser coverage adds +transport audits, renewal, the actual Go-node restart and final-user revocation +checks; native coverage is login/write/app-relaunch, not a claim those lower-level +assertions were run on a device. + +### iOS Simulator manual recipe (macOS required) + +The Linux runner cannot provision an iOS Simulator. On a macOS development +machine, build/install the current app on a booted simulator, pre-grant its +notification permission, and start the same harness from the checkout root: + +```bash +# Keep this shell alive; its EXIT trap owns the harness and the ephemeral CA. +export SP_E2E_UPSTREAM_STATION=true +export SP_E2E_ATMOQ_BIN=/path/to/released/atmoq +export SP_E2E_APP_BUNDLE_ID= +. hack/lib/e2e-harness.sh +e2e_harness_start +xcrun simctl keychain booted add-root-cert "$E2E_TLS_CA" +SP_E2E_UPSTREAM_STATION=true bash hack/maestro-config.sh \ + .maestro/logged-in/upstream-station.yaml > /tmp/streamplace-ios-upstream.yaml +maestro --device test \ + --config /tmp/streamplace-ios-upstream.yaml \ + -e APP_ID="$SP_E2E_APP_BUNDLE_ID" -e SERVER_URL="$SERVER_URL" \ + -e ACCOUNT_HANDLE="$ACCOUNT_HANDLE" -e ACCOUNT_PASSWORD="$ACCOUNT_PASSWORD" \ + -e E2E_UPSTREAM_STATION=true .maestro +``` + +Simulator networking must also resolve the harness's `plc.directory` to this +run's local PLC TLS front end, not the public directory: arrange a temporary +local DNS/hosts override or a simulator CONNECT proxy using `E2E_PROXY_URL`, +and remove only that override afterward. The station/PDS/account DNS names +already need to resolve to host loopback as described above. Trust the CA in +the simulator, not the macOS system keychain. Confirm the installed build's +generated `Info.plist` permits local-network transport with +`NSAllowsLocalNetworking=true`; do not enable blanket arbitrary HTTP loads. +Remove the temporary workspace file and the simulator's test CA after the run. +These are execution prerequisites, not +evidence that an iOS run has occurred. + ## Notes / gotchas - **Other adb devices:** maestro lists no Android devices at all ("Device diff --git a/.maestro/logged-in/upstream-station.yaml b/.maestro/logged-in/upstream-station.yaml new file mode 100644 index 00000000..0c13a69b --- /dev/null +++ b/.maestro/logged-in/upstream-station.yaml @@ -0,0 +1,62 @@ +appId: ${APP_ID} +tags: + - upstream-station +--- +# The generated plan first runs server-setup and the real OAuth login flow. +# SERVER_URL is the local HTTP loopback node, not the HTTPS station. Android +# uses adb reverse; iOS Simulator reaches host loopback directly. Both hop +# through the station's HTTPS OAuth and the PDS's genuine consent pages. +- assertTrue: ${E2E_UPSTREAM_STATION == "true"} +- stopApp +- launchApp +- extendedWaitUntil: + visible: + id: "home-stream-card" + timeout: 60000 +- tapOn: + id: "home-stream-card" +- extendedWaitUntil: + visible: + text: ".*e2e test stream.*" + timeout: 30000 +- tapOn: + id: "chat-input" +- inputText: "upstream-station native write" +- pressKey: Enter +- extendedWaitUntil: + visible: + text: ".*: upstream-station native write" + timeout: 30000 +- takeScreenshot: upstream-station-write +# Kill the app process, not merely its navigation. The reloaded history is +# served by the local index from the selected user's MoQ track, not the +# optimistic chat bubble. The persisted OAuth client must still write. +- stopApp +- launchApp +- extendedWaitUntil: + visible: + id: "home-stream-card" + timeout: 60000 +- tapOn: + id: "home-stream-card" +- extendedWaitUntil: + visible: + text: ".*: upstream-station native write" + timeout: 30000 +- tapOn: + id: "chat-input" +- inputText: "upstream-station native write after relaunch" +- pressKey: Enter +- extendedWaitUntil: + visible: + text: ".*: upstream-station native write after relaunch" + timeout: 30000 +- stopApp +- launchApp +- tapOn: + id: "home-stream-card" +- extendedWaitUntil: + visible: + text: ".*: upstream-station native write after relaunch" + timeout: 30000 +- takeScreenshot: upstream-station-history-after-relaunch diff --git a/go.mod b/go.mod index d0bfaec2..7b859e1e 100644 --- a/go.mod +++ b/go.mod @@ -73,10 +73,10 @@ require ( github.com/slok/go-http-metrics v0.13.0 github.com/starttoaster/prometheus-exporter-scraper v0.0.1 github.com/streamplace/atmoq/go v0.1.2-0.20261010034308-a0232526686a - github.com/streamplace/atproto-oauth-golang v0.0.0-20260413212710-98956064d06c + github.com/streamplace/atproto-oauth-golang v0.0.0-20260908211910-e9bac3356167 github.com/streamplace/glex v0.0.0-20260820164827-814f46540f22 github.com/streamplace/muxl/go v0.3.6 - github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b + github.com/streamplace/oatproxy v0.0.0-20261012002509-80a9af303a32 github.com/stretchr/testify v1.12.1 github.com/tdewolff/canvas v0.0.0-20250728095813-50d4cb1eee71 github.com/tus/tusd/v2 v2.8.0 diff --git a/go.sum b/go.sum index e483a259..7fa7c310 100644 --- a/go.sum +++ b/go.sum @@ -1383,8 +1383,8 @@ github.com/stbenjam/no-sprintf-host-port v0.2.0 h1:i8pxvGrt1+4G0czLr/WnmyH7zbZ8B github.com/stbenjam/no-sprintf-host-port v0.2.0/go.mod h1:eL0bQ9PasS0hsyTyfTjjG+E80QIyPnBVQbYZyv20Jfk= github.com/streamplace/atmoq/go v0.1.2-0.20261010034308-a0232526686a h1:nIBskQcIHlWkbJCx1PinrhUaSmHubwHw20AssaW6d1M= github.com/streamplace/atmoq/go v0.1.2-0.20261010034308-a0232526686a/go.mod h1:LaO0uknwuCep6sVq83dn9nXvkKdv8TrVnGvUQWDUbNw= -github.com/streamplace/atproto-oauth-golang v0.0.0-20260413212710-98956064d06c h1:IzEPU2O4iL58Nb7aw+7lB9ttnesEwOVVE5oV9NEXemM= -github.com/streamplace/atproto-oauth-golang v0.0.0-20260413212710-98956064d06c/go.mod h1:9LlKkqciiO5lRfbX0n4Wn5KNY9nvFb4R3by8FdW2TWc= +github.com/streamplace/atproto-oauth-golang v0.0.0-20260908211910-e9bac3356167 h1:yfEfsKpEgkXO+GEHnNrS3SeIeXC3+Ay0eytsazzKlVc= +github.com/streamplace/atproto-oauth-golang v0.0.0-20260908211910-e9bac3356167/go.mod h1:9LlKkqciiO5lRfbX0n4Wn5KNY9nvFb4R3by8FdW2TWc= github.com/streamplace/glex v0.0.0-20260820164827-814f46540f22 h1:yy0iaLxWsgdacyeukLFAWMJ6KXRCCZ4IZiv5iODBKGg= github.com/streamplace/glex v0.0.0-20260820164827-814f46540f22/go.mod h1:LRaoeSMvSgOrhFX8s7ygjRlyka7wXdDa1s7JJ9o1IzY= github.com/streamplace/go-dpop v0.0.0-20250510031900-c897158a8ad4 h1:L1fS4HJSaAyNnkwfuZubgfeZy8rkWmA0cMtH5Z0HqNc= @@ -1393,8 +1393,8 @@ github.com/streamplace/indigo v0.0.0-20260218231908-939cdaf0c507 h1:e8M3qPLr37Nx github.com/streamplace/indigo v0.0.0-20260218231908-939cdaf0c507/go.mod h1:Pm2I1+iDXn/hLbF7XCg/DsZi6uDCiOo7hZGWprSM7k0= github.com/streamplace/muxl/go v0.3.6 h1:lyWnY15nft/0GK3LBA4OkD2aHrLV9RRqI+YcDRPFk3g= github.com/streamplace/muxl/go v0.3.6/go.mod h1:aCyYTW3o6c1Kush9UJ/Yv6EYMUbj8l8GTD7cHKcSxw8= -github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b h1:eWbwCtBbMyrDTHLYIold07OR2hmvzXsbAUxi57ElMLk= -github.com/streamplace/oatproxy v0.0.0-20260710202406-60d97b9d780b/go.mod h1:wpY+T/wE00jrUhgh2dKXbbE91D36u86KGlENK/hWFkE= +github.com/streamplace/oatproxy v0.0.0-20261012002509-80a9af303a32 h1:rZpc/Z3xCx2ooa6ypoUGEPUxifZRBprqmOt5kP0lhJk= +github.com/streamplace/oatproxy v0.0.0-20261012002509-80a9af303a32/go.mod h1:WRKRYf1OcBCa/vWJSJOxabqMIPelwktWcT6WkTj56Hk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/hack/e2e-local.sh b/hack/e2e-local.sh index 993655b5..06e743e0 100755 --- a/hack/e2e-local.sh +++ b/hack/e2e-local.sh @@ -4,7 +4,7 @@ # harness on a running Android emulator — the mobile counterpart to # hack/e2e-web-local.sh. # -# hack/e2e-local.sh android [-- ] +# hack/e2e-local.sh android [--upstream-station] [-- ] # # Prerequisites (build once, or again when app or native code changes): # * The harness binary: make dev @@ -36,6 +36,10 @@ REPO="$PWD" PLATFORM="${1:-android}" [ $# -gt 0 ] && shift +if [ "${1:-}" = "--upstream-station" ]; then + export SP_E2E_UPSTREAM_STATION=true + shift +fi [ "${1:-}" = "--" ] && shift case "$PLATFORM" in android) ;; @@ -99,13 +103,14 @@ if [ -z "${SERVER_HTTPS_URL:-}" ]; then exit 1 fi -# The emulator reaches the host's loopback at 10.0.2.2. plc.directory -# comes along because the app resolves did:plc there itself. +# Default to the emulator's host alias. An unprivileged host using a high +# HTTPS port can instead map the fixture through adb reverse on port 443. +# E2E_ANDROID_HOST_ADDR then selects device loopback without changing issuers. hosts="127.0.0.1 localhost ::1 ip6-localhost" for name in "$E2E_HTTPS_STATION_HOSTNAME" "$E2E_HTTPS_PDS_HOSTNAME" "$ACCOUNT_HANDLE" plc.directory; do hosts="$hosts -10.0.2.2 $name" +${E2E_ANDROID_HOST_ADDR:-10.0.2.2} $name" done printf '%s\n' "$hosts" | adb_ shell 'while grep -q " /system/etc/hosts " /proc/mounts; do umount /system/etc/hosts; done cat > /data/local/tmp/sp-e2e-hosts @@ -137,8 +142,25 @@ adb_ shell pm grant "$APP_ID" android.permission.POST_NOTIFICATIONS 2>/dev/null # Chrome reads its command line and the trust store at startup adb_ shell am force-stop com.android.chrome || true -MAESTRO_ARGS=(-e APP_ID="$APP_ID" -e SERVER_URL="$SERVER_HTTPS_URL" - -e ACCOUNT_HANDLE="$ACCOUNT_HANDLE" -e ACCOUNT_PASSWORD="$ACCOUNT_PASSWORD") + +APP_SERVER_URL="$SERVER_HTTPS_URL" +if [ "${E2E_UPSTREAM_STATION:-}" = true ]; then + # Keep the issuer and callback loopback names identical on host and device. + # Merely substituting 10.0.2.2 would change the DPoP-bound request target. + local_port="${SERVER_URL##*:}" + adb_ reverse "tcp:$local_port" "tcp:$local_port" + APP_SERVER_URL="$SERVER_URL" +fi +if [ "${E2E_UPSTREAM_STATION:-}" = true ]; then + e2e_android_stop() { + adb_ reverse --remove "tcp:$local_port" 2>/dev/null || true + e2e_harness_stop + } + trap e2e_android_stop EXIT INT TERM +fi +MAESTRO_ARGS=(-e APP_ID="$APP_ID" -e SERVER_URL="$APP_SERVER_URL" + -e ACCOUNT_HANDLE="$ACCOUNT_HANDLE" -e ACCOUNT_PASSWORD="$ACCOUNT_PASSWORD" + -e E2E_UPSTREAM_STATION="${E2E_UPSTREAM_STATION:-false}") # --- run the flows --------------------------------------------------------- # takeScreenshot paths are relative to maestro's cwd, so run from the @@ -148,6 +170,11 @@ mkdir -p "$ARTIFACTS" ARTIFACTS="$(cd "$ARTIFACTS" && pwd)" # Maestro's flowsOrder accepts names, not phase globs. Generate its order # from the selected files so new scenarios never edit a shared manifest. +if [ "${E2E_UPSTREAM_STATION:-}" = true ] && [ -z "${E2E_FLOWS:-}" ]; then + # Upstream local nodes have no logged-out feed before successful local OAuth. + # The generated plan includes server setup and OAuth before this scenario. + E2E_FLOWS=".maestro/logged-in/upstream-station.yaml" +fi bash "$REPO/hack/maestro-config.sh" ${E2E_FLOWS:-} > "$ARTIFACTS/workspace.yaml" cd "$ARTIFACTS" MAESTRO_CLI_NO_ANALYTICS=1 "$MAESTRO" --device "$DEVICE" test \ diff --git a/hack/e2e-web-local.sh b/hack/e2e-web-local.sh index 4eab53f2..7edbaa31 100755 --- a/hack/e2e-web-local.sh +++ b/hack/e2e-web-local.sh @@ -3,7 +3,7 @@ # Run the headless-web e2e suite (Playwright) against a self-contained # `streamplace e2e` harness — the web counterpart to hack/e2e-local.sh. # -# hack/e2e-web-local.sh [-- ] +# hack/e2e-web-local.sh [--upstream-station] [-- ] # # Prerequisites: # * A streamplace binary with the embedded web bundle (`make dev`). @@ -24,9 +24,17 @@ set -euo pipefail cd "$(dirname "$0")/.." REPO="$PWD" +if [ "${1:-}" = "--upstream-station" ]; then + export SP_E2E_UPSTREAM_STATION=true + shift +fi +[ "${1:-}" != "--" ] || shift # shellcheck source=hack/lib/e2e-harness.sh . hack/lib/e2e-harness.sh e2e_harness_start +if [ "${E2E_UPSTREAM_STATION:-}" = true ] && [ $# -eq 0 ]; then + set -- upstream-station.spec.ts +fi # --- run the flows --------------------------------------------------------- # No `exec` here: that would replace this shell and discard the EXIT trap, diff --git a/hack/lib/e2e-harness.sh b/hack/lib/e2e-harness.sh index 8854eba8..2cc8b2ee 100644 --- a/hack/lib/e2e-harness.sh +++ b/hack/lib/e2e-harness.sh @@ -14,6 +14,11 @@ # SERVER_HTTPS_URL, PDS_HTTPS_URL, # E2E_PROXY_URL, E2E_TLS_SPKI and E2E_TLS_CA. An EXIT trap stops it; a caller # with its own EXIT work calls e2e_harness_stop from its trap instead. +# Upstream mode (SP_E2E_UPSTREAM_STATION=true) additionally exports the local +# node SERVER_URL, UPSTREAM_STATION_URL, ATMOQ_RELAY_URL and private fixture +# paths used by the scoped-indexing/lifecycle tests. SP_E2E_ATMOQ_BIN selects +# the real released bridge executable. E2E_CONTROL_FIFO feeds harness stdin; +# writing restart-local restarts only this run's local node. # # HTTPS mode (for the OAuth flows) is on by default and needs permission to # bind 127.0.0.1:443 (root, as in the build container). The harness serves the @@ -67,10 +72,20 @@ e2e_harness_start() { --https-station-hostname "$E2E_HTTPS_STATION_HOSTNAME" --https-port "${E2E_HTTPS_PORT:-443}") fi + local input=/dev/null + if [ "${SP_E2E_UPSTREAM_STATION:-false}" = true ]; then + E2E_CONTROL_DIR="$(mktemp -d)" + E2E_CONTROL_FIFO="$E2E_CONTROL_DIR/control" + mkfifo "$E2E_CONTROL_FIFO" + # Hold both ends open so the harness scanner survives between commands. + exec 9<>"$E2E_CONTROL_FIFO" + input="$E2E_CONTROL_FIFO" + export E2E_CONTROL_FIFO + fi if command -v setsid >/dev/null 2>&1; then - setsid "$BUILDDIR/streamplace" "${args[@]}" > "$E2E_ENVFILE" 2> "$E2E_LOGFILE" & + setsid "$BUILDDIR/streamplace" "${args[@]}" < "$input" > "$E2E_ENVFILE" 2> "$E2E_LOGFILE" & else - "$BUILDDIR/streamplace" "${args[@]}" > "$E2E_ENVFILE" 2> "$E2E_LOGFILE" & + "$BUILDDIR/streamplace" "${args[@]}" < "$input" > "$E2E_ENVFILE" 2> "$E2E_LOGFILE" & fi E2E_HARNESS_PID=$! trap e2e_harness_stop EXIT INT TERM @@ -85,6 +100,12 @@ e2e_harness_start() { export E2E_FIXTURE_MP4 # only set in HTTPS mode export SERVER_HTTPS_URL PDS_HTTPS_URL E2E_PROXY_URL E2E_TLS_SPKI E2E_TLS_CA + export E2E_UPSTREAM_STATION UPSTREAM_STATION_URL ATMOQ_RELAY_URL E2E_ATMOQ_LOG + export E2E_LOCAL_DATA_DIR E2E_LOCAL_PDS_AUDIT E2E_STATION_DATA_DIR + export UNRELATED_ACCOUNT_DID UNRELATED_ACCOUNT_HANDLE UNRELATED_ACCESS_JWT E2E_PDS_INTERNAL_URL + if [ "${E2E_UPSTREAM_STATION:-}" = true ]; then + echo "upstream topology: local=$SERVER_URL station=$UPSTREAM_STATION_URL relay=$ATMOQ_RELAY_URL" + fi echo "harness up: SERVER_URL=$SERVER_URL ACCOUNT_HANDLE=$ACCOUNT_HANDLE${SERVER_HTTPS_URL:+ SERVER_HTTPS_URL=$SERVER_HTTPS_URL}" } @@ -100,5 +121,10 @@ e2e_harness_stop() { done rm -f "$E2E_ENVFILE" [ -n "$E2E_KEEP_LOG" ] || rm -f "$E2E_LOGFILE" + if [ -n "${E2E_CONTROL_DIR:-}" ]; then + exec 9>&- + rm -rf "$E2E_CONTROL_DIR" + E2E_CONTROL_DIR= + fi E2E_HARNESS_PID= } diff --git a/hack/maestro-config.sh b/hack/maestro-config.sh index 23d67ef5..9273eb9b 100755 --- a/hack/maestro-config.sh +++ b/hack/maestro-config.sh @@ -8,7 +8,14 @@ cd "$REPO/.maestro" shopt -s nullglob logged_out=(logged-out/*.yaml) -logged_in=(logged-in/*.yaml) +logged_in=() +for file in logged-in/*.yaml; do + # Opt-in topologies cannot run against the default single-node harness. + if grep -q '^ - upstream-station$' "$file" && [ "${SP_E2E_UPSTREAM_STATION:-false}" != true ]; then + continue + fi + logged_in+=("$file") +done if [ $# -gt 0 ]; then logged_out=() logged_in=() @@ -16,6 +23,10 @@ if [ $# -gt 0 ]; then file="${file#"$REPO/"}" file="${file#.maestro/}" [ -f "$file" ] || { echo "missing Maestro flow: $file" >&2; exit 1; } + if grep -q '^ - upstream-station$' "$file" && [ "${SP_E2E_UPSTREAM_STATION:-false}" != true ]; then + echo "upstream-station flow requires SP_E2E_UPSTREAM_STATION=true" >&2 + exit 1 + fi case "$file" in logged-out/*.yaml) logged_out+=("$file") ;; logged-in/*.yaml) logged_in+=("$file") ;; diff --git a/js/app/app.config.ts b/js/app/app.config.ts index 139c68fa..032733de 100644 --- a/js/app/app.config.ts +++ b/js/app/app.config.ts @@ -52,7 +52,7 @@ const withAndroidProfileable = (config) => { // the device's user store, where hack/e2e-local.sh installs the e2e harness's // throwaway CA so the app can reach its HTTPS station and PDS. Never ship // this — it lets whoever can add a user CA intercept the app's traffic. -// Cleartext stays off, as in a release build (targetSdk 28+ default). +// Only loopback HTTP is allowed, for the upstream-mode local OAuth proxy. const withE2EUserTrust: ConfigPlugin = (config) => { config = withDangerousMod(config, [ "android", @@ -72,6 +72,10 @@ const withE2EUserTrust: ConfigPlugin = (config) => { + + 127.0.0.1 + localhost + `, ); diff --git a/js/app/features/bluesky/oauthClient.tsx b/js/app/features/bluesky/oauthClient.tsx index 2bce3e0b..8ccd571e 100644 --- a/js/app/features/bluesky/oauthClient.tsx +++ b/js/app/features/bluesky/oauthClient.tsx @@ -5,7 +5,7 @@ import { clientMetadataSchema, OAuthClient, } from "./oauthClientImport"; -import { StreamplaceOAuthResolver } from "./oauthResolver"; +import { routeOAuthThroughStreamplace } from "./oauthResolver"; export default async function createOAuthClient( streamplaceUrl: string, @@ -15,63 +15,55 @@ export default async function createOAuthClient( throw new Error("streamplaceUrl is required"); } - // Will be set after we create the custom resolver - let customResolver: StreamplaceOAuthResolver | null = null; + let getLoginHint: (() => string | null) | null = null; let meta: ClientMetadata; - const redirectURI = - Platform.OS === "web" - ? `${streamplaceUrl}/login` - : `${streamplaceUrl}/api/app-return`; + const nodeUrl = new URL(streamplaceUrl); + const isLoopback = + nodeUrl.protocol === "http:" && + (nodeUrl.hostname === "localhost" || + nodeUrl.hostname === "127.0.0.1" || + nodeUrl.hostname === "[::1]"); + const callbackUrl = new URL(nodeUrl.toString()); + if (isLoopback && callbackUrl.hostname === "localhost") { + callbackUrl.hostname = "127.0.0.1"; + } + if (Platform.OS === "web") { + callbackUrl.pathname = "/login"; + } else if (isLoopback) { + const scheme = Constants.expoConfig?.scheme; + if (!scheme) { + throw new Error("unable to resolve scheme for oauth redirect"); + } + callbackUrl.pathname = `/app-return/${scheme}`; + } else { + callbackUrl.pathname = "/api/app-return"; + } + const redirectURI = callbackUrl.toString(); const res = await fetch( `${streamplaceUrl}/oauth/downstream/client-metadata.json?redirect_uri=${encodeURIComponent(redirectURI)}`, ); meta = await res.json(); + // Legacy public dev nodes advertise a HTTPS redirect wrapper even for their + // HTTP loopback client. New local nodes advertise the actual redirect. if ( - streamplaceUrl.startsWith("http://localhost") || - streamplaceUrl.startsWith("http://127.0.0.1") + isLoopback && + meta.client_id?.startsWith("http://localhost?") && + !meta.redirect_uris?.some((uri) => uri === redirectURI) ) { if (!meta.scope) { throw new Error("meta.scope is required"); } - const isWeb = Platform.OS === "web"; - const u = new URL(streamplaceUrl); - let hostname = u.hostname; - if (hostname == "localhost") { - hostname = "127.0.0.1"; - } - let redirect = `${u.protocol}//${hostname}`; - if (u.port !== "") { - redirect = `${redirect}:${u.port}`; - } - if (isWeb) { - redirect = `${redirect}/login`; - } else { - const scheme = Constants.expoConfig?.scheme; - if (!scheme) { - throw new Error("unable to resolve scheme for oauth redirect"); - } - redirect = `${redirect}/app-return/${scheme}`; - } const queryParams = new URLSearchParams(); queryParams.set("scope", meta.scope); - queryParams.set("redirect_uri", redirect); + queryParams.set("redirect_uri", redirectURI); meta = { + ...meta, client_id: `http://localhost?${queryParams.toString()}`, - redirect_uris: [redirect as any], - scope: meta.scope, - token_endpoint_auth_method: "none", - client_name: "Loopback client", - response_types: ["code"], - grant_types: ["authorization_code", "refresh_token"], - // > There is a special exception for the localhost development workflow [ ... ] - // > These clients use web URLs, but have application_type set to native in the generated client metadata. + redirect_uris: [redirectURI as ClientMetadata["redirect_uris"][number]], application_type: "native", - dpop_bound_access_tokens: true, - subject_type: "public", - authorization_signed_response_alg: "ES256", }; } try { @@ -92,11 +84,11 @@ export default async function createOAuthClient( // Add login_hint parameter to PAR requests if ( - customResolver && + getLoginHint && request.url.includes("/oauth/par") && request.method === "POST" ) { - const resourceServer = customResolver.getCurrentResourceServer(); + const resourceServer = getLoginHint(); if (resourceServer) { const clonedRequest = request.clone(); const body = await clonedRequest.text(); @@ -110,63 +102,9 @@ export default async function createOAuthClient( } } - if (streamplaceUrl.startsWith("http://127.0.0.1")) { - // everything other than PDS resolution gets rewritten to the host - if ( - request.url.includes("plc.directory") || - request.url.endsWith("did.json") || - request.url.endsWith("/.well-known/oauth-protected-resource") || - request.url.endsWith("/.well-known/oauth-authorization-server") - ) { - return fetch(request, init) as any; - } - const newUrl = new URL(request.url.toString()); - newUrl.protocol = "http:"; - newUrl.host = "127.0.0.1:38080"; - let newRequest: Request; - if (request.method === "POST") { - const data = await request.blob(); - newRequest = new Request(newUrl.toString(), { - body: data, - method: "POST", - headers: request.headers, - }); - } else if (request.method === "GET") { - newRequest = new Request(newUrl.toString(), { - method: "GET", - headers: request.headers, - }); - } else { - throw new Error("Unsupported method: " + request.method); - } - return fetch(newRequest) as any; - } else { - // Lie to the oauth client and use our upstream server instead - if ( - request.url.includes("plc.directory") || - request.url.endsWith("did.json") - ) { - const res = await fetch(request, init); - if (!res.ok) { - return res; - } - const data = await res.json(); - const service = data.service.find( - (s: any) => s.id === "#atproto_pds", - ); - if (!service) { - return res; - } - service.serviceEndpoint = streamplaceUrl; - return new Response(JSON.stringify(data), { - status: res.status, - headers: res.headers, - }); - } else { - return fetch(request, init); - } - } + return fetch(request, init); }, + allowHttp: isLoopback, handleResolver: streamplaceUrl, responseMode: "query", // or "fragment" (frontend only) or "form_post" (backend only) @@ -175,15 +113,10 @@ export default async function createOAuthClient( clientMetadata: meta, }); - // Replace the default OAuth resolver with our custom one - customResolver = new StreamplaceOAuthResolver( + getLoginHint = routeOAuthThroughStreamplace( + client.oauthResolver, streamplaceUrl, - client.oauthResolver.identityResolver, - client.oauthResolver.protectedResourceMetadataResolver, - client.oauthResolver.authorizationServerMetadataResolver, ); - // @ts-ignore override readonly property - client.oauthResolver = customResolver; return client; } diff --git a/js/app/features/bluesky/oauthResolver.tsx b/js/app/features/bluesky/oauthResolver.tsx index 54addc14..871b7cae 100644 --- a/js/app/features/bluesky/oauthResolver.tsx +++ b/js/app/features/bluesky/oauthResolver.tsx @@ -1,40 +1,24 @@ -import type { OAuthAuthorizationServerMetadata } from "@atproto/oauth-client"; -import { - OAuthResolver, - ResolveOAuthOptions, -} from "@atproto/oauth-client/dist/oauth-resolver"; +import type { OAuthResolver } from "@atproto/oauth-client/dist/oauth-resolver"; -export class StreamplaceOAuthResolver extends OAuthResolver { - private currentResourceServer: string | null = null; - - constructor( - private streamplaceUrl: string, - ...args: ConstructorParameters - ) { - super(...args); - } - - async resolveFromService( - input: string, - options?: ResolveOAuthOptions, - ): Promise<{ - metadata: OAuthAuthorizationServerMetadata; - }> { - // Input is the resource server URL (e.g., https://selfhosted.social) - // Store it for use in login_hint - this.currentResourceServer = input; - - // Always fetch metadata from our backend - // The issuer will be our backend, not the resource server - const metadata = await this.getResourceServerMetadata( - this.streamplaceUrl, +// The SDK's server factory captures this object during construction. Configure +// its methods in place so authorization, callbacks and refresh share routing. +export function routeOAuthThroughStreamplace( + resolver: OAuthResolver, + streamplaceUrl: string, +): () => string | null { + let loginHint: string | null = null; + resolver.resolveFromService = async (input, options) => { + loginHint = input; + const metadata = await resolver.getResourceServerMetadata( + streamplaceUrl, options, ); - return { metadata }; - } - - getCurrentResourceServer(): string | null { - return this.currentResourceServer; - } + }; + resolver.resolveFromIdentity = async (input, options) => { + const identityInfo = await resolver.resolveIdentity(input, options); + const { metadata } = await resolver.resolveFromService(input, options); + return { identityInfo, metadata, pds: new URL(streamplaceUrl) }; + }; + return () => loginHint; } diff --git a/js/app/store/slices/blueskySlice.ts b/js/app/store/slices/blueskySlice.ts index 03e37dff..7bf77bce 100644 --- a/js/app/store/slices/blueskySlice.ts +++ b/js/app/store/slices/blueskySlice.ts @@ -372,14 +372,6 @@ export const createBlueskySlice: StateCreator< } } const u = await updatedState.client.authorize(handle, authorizeOptions); - if ( - typeof document !== "undefined" && - document.location.href.startsWith("http://127.0.0.1") - ) { - const hostUrl = new URL(document.location.href); - u.host = hostUrl.host; - u.protocol = hostUrl.protocol; - } console.log("Opening link"); await openLoginLink(u.toString()); // cheeky 500ms delay so you don't see the text flash back diff --git a/js/desktop/README.md b/js/desktop/README.md index 583199e3..8b6075fb 100644 --- a/js/desktop/README.md +++ b/js/desktop/README.md @@ -10,3 +10,29 @@ Boot up Streamplace Desktop for development. Two environment variable options: - `AQD_NODE_FRONTEND`: (default `false`): Use frontend bundled with the Streamplace node instead of the dev server on `http://localhost:8081`. + +## Upstream station mode + +Desktop starts its bundled node with `SP_UPSTREAM_STATION=did:web:stream.place`. +Set `SP_UPSTREAM_STATION` to another station DID to override it, or explicitly +set it to an empty string to run the standalone server mode. + +The station must advertise both `#streamplace` (an HTTPS origin) and `#atmoq` +(a raw-QUIC MoQ origin) in its DID document. Missing or invalid advertisements +stop startup; Desktop never falls back to the full public firehose. + +The local node subscribes to exact-DID tracks only for successfully authenticated +local users. Startup, sweeps, history backfill and event handling use the same +persisted OAuth eligibility; previously indexed repos do not authorize new +scraping. Logout removes eligibility when the last local grant ends. Existing +indexed records remain on disk, but no longer receive updates. + +OAuth has three independent DPoP sessions: frontend → loopback node → HTTPS +station → PDS. Only the station needs a public HTTPS callback and retains the +long-lived PDS session. The local node's issuer and frontend callback use its +actual loopback port; `SP_HTTP_ADDR` and `SP_HTTP_INTERNAL_ADDR` can override +the Desktop listener defaults. + +For development with the embedded frontend, set `AQD_NODE_FRONTEND=true` and +`SP_DEV_FRONTEND_PROXY=` after `make app && make dev`. +Auto-update behavior is unchanged. diff --git a/js/desktop/src/index.ts b/js/desktop/src/index.ts index b0164e07..6215991d 100644 --- a/js/desktop/src/index.ts +++ b/js/desktop/src/index.ts @@ -12,7 +12,11 @@ import { makeWindow } from "./window"; if (require("electron-squirrel-startup")) { app.quit(); } else { + // Electron leaves Chromium switches and the app entry in process.argv. + // Keep parsing our own options without rejecting those Electron arguments. const { values: args, positionals } = parseArgs({ + strict: false, + allowPositionals: true, options: { path: { type: "string", @@ -31,6 +35,18 @@ if (require("electron-squirrel-startup")) { }, }, }); + const path = args.path; + const testsToRun = args["tests-to-run"]; + const testDuration = args["self-test-duration"]; + const selfTest = args["self-test"]; + if ( + typeof path !== "string" || + typeof testsToRun !== "string" || + typeof testDuration !== "string" || + (selfTest !== undefined && typeof selfTest !== "boolean") + ) { + throw new TypeError("Invalid Streamplace Desktop command-line options"); + } const env = getEnv(); console.log( "starting with: ", @@ -50,13 +66,13 @@ if (require("electron-squirrel-startup")) { SP_ADMIN_ACCOUNT: account.address.toLowerCase(), SP_ALLOWED_STREAMS: account.address.toLowerCase(), }; - if (args["self-test"]) { + if (selfTest) { app.on("window-all-closed", () => { // need to override this to prevent the app from quitting }); const success = await runTests( - args["tests-to-run"].split(","), - args["self-test-duration"], + testsToRun.split(","), + testDuration, privateKey, ); console.log("tests finished"); @@ -104,9 +120,9 @@ if (require("electron-squirrel-startup")) { let startPath; if (nodeFrontend) { - startPath = `${loadAddr}${args.path}`; + startPath = `${loadAddr}${path}`; } else { - startPath = `http://127.0.0.1:38081${args.path}`; + startPath = `http://127.0.0.1:38081${path}`; } console.log(`opening ${startPath}`); mainWindow.loadURL(startPath); diff --git a/js/desktop/src/node.ts b/js/desktop/src/node.ts index 7d95b282..87e8bff3 100644 --- a/js/desktop/src/node.ts +++ b/js/desktop/src/node.ts @@ -40,12 +40,20 @@ export default async function makeNode(opts: { autoQuit: boolean; }) { const exe = await findExe(); - const addr = opts.env.SP_HTTP_ADDR ?? "127.0.0.1:38082"; - const internalAddr = opts.env.SP_HTTP_INTERNAL_ADDR ?? "127.0.0.1:39092"; + const addr = + opts.env.SP_HTTP_ADDR ?? process.env.SP_HTTP_ADDR ?? "127.0.0.1:38082"; + const internalAddr = + opts.env.SP_HTTP_INTERNAL_ADDR ?? + process.env.SP_HTTP_INTERNAL_ADDR ?? + "127.0.0.1:39092"; const proc = spawn(exe, [], { stdio: "inherit", env: { ...process.env, + SP_UPSTREAM_STATION: + opts.env.SP_UPSTREAM_STATION ?? + process.env.SP_UPSTREAM_STATION ?? + "did:web:stream.place", SP_HTTP_ADDR: addr, SP_HTTP_INTERNAL_ADDR: internalAddr, ...opts.env, diff --git a/js/desktop/src/tests/server-restart-test.ts b/js/desktop/src/tests/server-restart-test.ts index 7d5df7de..720fc484 100644 --- a/js/desktop/src/tests/server-restart-test.ts +++ b/js/desktop/src/tests/server-restart-test.ts @@ -33,6 +33,7 @@ export const serverRestartTest: E2ETest = { SP_DATA_DIR: tmpDir, SP_TEST_STREAM: "true", SP_NO_FIREHOSE: "true", + SP_UPSTREAM_STATION: "", SP_WIDE_OPEN: "true", }; let { proc } = await makeNode({ diff --git a/js/desktop/src/tests/test-runner.ts b/js/desktop/src/tests/test-runner.ts index 157b182e..dc4cd685 100644 --- a/js/desktop/src/tests/test-runner.ts +++ b/js/desktop/src/tests/test-runner.ts @@ -72,6 +72,7 @@ export default async function runTests( SP_RTMP_ADDR: `127.0.0.1:${randomPort()}`, SP_DATA_DIR: tmpDir, SP_NO_FIREHOSE: "true", + SP_UPSTREAM_STATION: "", }; } if (test.setup) { diff --git a/js/docs/src/content/docs/guides/installing/downloading-streamplace.md b/js/docs/src/content/docs/guides/installing/downloading-streamplace.md index 82d85087..11578915 100644 --- a/js/docs/src/content/docs/guides/installing/downloading-streamplace.md +++ b/js/docs/src/content/docs/guides/installing/downloading-streamplace.md @@ -91,3 +91,45 @@ host networking. So that command would look something like: Binaries for all platforms are available to download from [our GitLab server](https://git.stream.place/streamplace/streamplace/-/releases). + +## Local node with an upstream station + +```shell +streamplace --http-addr=127.0.0.1:38080 \ + --upstream-station=did:web:stream.place +``` + +`SP_UPSTREAM_STATION` is the environment equivalent. Without it, normal server +indexing and relay selection remain unchanged. With it, the node discovers the +station's HTTPS OAuth origin and MoQ firehose from its DID document. There is no +fallback to `bsky.network` or the local PDS firehose. + +The local node indexes only users with a successfully completed, non-revoked +local OAuth grant. Pending logins, unrelated followed users and repos left over +from a previous standalone run do not permit PDS scraping. Per-DID subscriptions +start at login and end when the last eligible grant ends. Reconnects reconcile +the same scoped users with their actual PDS; historical records remain on disk +after logout. + +Local HTTP OAuth requires a loopback listener. The local node delegates to the +station using its own DPoP key; the station independently authenticates to the +PDS using its public HTTPS callback. No public callback or development-only +OAuth flag is needed on the local node. + +### Preparing an upstream station + +Run an `atmoq serve` bridge against the station's upstream atproto relay, with +a valid TLS certificate and reachable QUIC UDP port. Advertise its origin: + +```ini +SP_ATMOQ_RELAY=moqt://firehose.example.com:443 +``` + +The station's DID document then includes an `#atmoq` service of type +`AtprotoMoQRelay`, alongside the existing `#streamplace` HTTPS origin. Clients +fail startup if either advertisement is absent or malformed. Use a DNS name +matching the certificate: released atmoq v0.1.1 requires TLS SNI for raw MoQ, +which Go does not send for numeric-IP URLs. + +Deploy the station's chained-OAuth support and MoQ advertisement before +releasing Desktop clients that default to `did:web:stream.place`. diff --git a/js/e2e-web/README.md b/js/e2e-web/README.md index 40d5118b..052b476d 100644 --- a/js/e2e-web/README.md +++ b/js/e2e-web/README.md @@ -126,6 +126,60 @@ popout tests establish their own login using the HTTPS URLs. Run with `E2E_HTTPS_PDS_HOSTNAME=` to skip HTTPS; those tests then skip themselves. Details are in `pkg/cmd/e2e_https.go`. +## Upstream-station topology + +This is opt-in; the ordinary single-node suite is unchanged. Install the real +released **atmoq v0.1.1** binary and verify its release checksum. It must run +on the builder's libc; if the prebuilt binary requires a newer glibc, build +that exact release with its locked Cargo dependencies in an isolated directory, +not by changing a sibling checkout. Set `SP_E2E_ATMOQ_BIN` to that executable +when it is not on `PATH`. + +```bash +# Inside the pinned builder, after make app && make dev: +SP_E2E_ATMOQ_BIN=/path/to/released/atmoq \ + hack/e2e-web-local.sh --upstream-station -- upstream-station.spec.ts +# Equivalent flag source: +SP_E2E_UPSTREAM_STATION=true SP_E2E_ATMOQ_BIN=/path/to/released/atmoq \ + hack/e2e-web-local.sh upstream-station.spec.ts +``` + +The harness starts the existing PDS/PLC, an HTTPS upstream station, a separate +HTTP loopback node configured with `SP_UPSTREAM_STATION=did:web:`, +and `atmoq serve` bridging the PDS's actual WebSocket firehose to QUIC. +The station advertises the bridge through `SP_ATMOQ_RELAY`. QUIC verifies the +throwaway CA and the leaf's loopback IP SAN; there is no insecure TLS override, +mock frame source or fake protocol server. The bridge uses a one-frame group +for sparse per-user traffic. URLs printed as evidence are `SERVER_URL`, +`UPSTREAM_STATION_URL` and `ATMOQ_RELAY_URL`. + +The browser completes local → station → PDS login and consent, writes through +both independently DPoP-bound proxies, and verifies the exact returned record +URI appears in the local durable index after initial login sync. It checks the +real indexed PDS remains the PDS, while the local OAuth session targets the +station. A separately created, followed-but-unrelated account emits real PDS +commits before and after login. The local transport audit checks it is neither +scraped nor enumerated, and the local DB checks it is not indexed. Released +relay logs must show the user's DID track and no aggregate `atproto` track: +bandwidth selection comes from these per-DID subscriptions, not from merely +replacing WebSocket with MoQ. The bridge itself still ingests its PDS firehose. + +Renewal coverage uses the actual browser's persisted DPoP key and local +`/oauth/token`. Only the stored access-token renewal timestamps are advanced +to make genuine station/PDS refresh deterministic; keys, tokens and protocol +responses are never faked. A harness-stdin `restart-local` command restarts the +real Go node with its on-disk sessions and acknowledges readiness in a private +file. Authenticated writes and indexed history must survive that restart. +Real browser logout revokes the local downstream session and stops the final +local DID subscription without destroying the station's confidential PDS grant. + +Prerequisites additionally include Python 3 with SQLite support (for fixture +inspection), a POSIX shell with `mkfifo`, free loopback HTTPS/TCP/UDP ports, and +the HTTPS DNS/trust prerequisites above. The FIFO, child process groups, TLS +keys, temporary databases and relay store are owned by and removed with the run. +Browser traces may contain temporary test-account credentials; treat the test +artifacts as sensitive and do not reuse these accounts outside the harness. + ## Run it locally ```bash diff --git a/js/e2e-web/flows/desktop-upstream-station.spec.ts b/js/e2e-web/flows/desktop-upstream-station.spec.ts new file mode 100644 index 00000000..7680ce76 --- /dev/null +++ b/js/e2e-web/flows/desktop-upstream-station.spec.ts @@ -0,0 +1,147 @@ +import { + _electron, + expect, + test, + type ElectronApplication, +} from "@playwright/test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { loginThroughPds } from "./login"; + +const MAIN = process.env.E2E_DESKTOP_MAIN; +test.skip( + !MAIN || !process.env.E2E_UPSTREAM_STATION, + "requires built Electron Desktop and upstream harness", +); + +async function freePort(): Promise { + const server = createServer(); + await new Promise((done) => server.listen(0, "127.0.0.1", done)); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("expected a TCP listener address"); + } + const port = address.port; + await new Promise((done, reject) => + server.close((error) => (error ? reject(error) : done())), + ); + return port; +} + +// Exercise the actual Electron entry point and its bundled-node launcher, not a +// copied spawn configuration. Mobile counterparts live in upstream-station.yaml. +test("desktop-upstream-station: login and writes survive closing Desktop", async () => { + test.setTimeout(180_000); + const desktopDir = resolve(__dirname, "../../desktop"); + const localPort = await freePort(); + const localHost = `127.0.0.1:${localPort}`; + const nodeURL = `http://${localHost}`; + const state = await mkdtemp(join(tmpdir(), "streamplace-desktop-upstream-")); + const env: Record = Object.fromEntries( + Object.entries(process.env).filter( + ([key, value]) => !key.startsWith("SP_") && value !== undefined, + ), + ) as Record; + Object.assign(env, { + WEBPACK_SERVE: "true", + AQD_SKIP_NODE: "false", + AQD_NODE_FRONTEND: "true", + AQD_NO_UPDATE: "true", + SP_UPSTREAM_STATION: `did:web:${new URL(process.env.UPSTREAM_STATION_URL!).hostname}`, + SP_HTTP_ADDR: localHost, + SP_HTTP_INTERNAL_ADDR: `127.0.0.1:${await freePort()}`, + SP_RTMP_ADDR: `127.0.0.1:${await freePort()}`, + SP_DATA_DIR: join(state, "node"), + SP_BROADCASTER_HOST: localHost, + SP_DEV_FRONTEND_PROXY: "", + SP_DEV_PUBLIC_OAUTH: "false", + SP_TRUST_PRIVATE_NETWORK: "true", + SSL_CERT_FILE: process.env.E2E_TLS_CA!, + HTTPS_PROXY: process.env.E2E_PROXY_URL!, + }); + const args = [ + "--no-sandbox", + `--user-data-dir=${join(state, "electron")}`, + `--proxy-server=${process.env.E2E_PROXY_URL}`, + "--proxy-bypass-list=127.0.0.1;localhost", + `--ignore-certificate-errors-spki-list=${process.env.E2E_TLS_SPKI}`, + resolve(MAIN!), + ]; + const launch = async (): Promise => { + const launched = await _electron.launch({ + executablePath: createRequire(join(desktopDir, "package.json"))( + "electron", + ) as string, + args, + cwd: desktopDir, + env, + }); + // Include the real launcher/node's startup errors in CI failure output. + launched + .process() + .stdout?.on("data", (chunk) => process.stdout.write(chunk)); + launched + .process() + .stderr?.on("data", (chunk) => process.stderr.write(chunk)); + return launched; + }; + let app: ElectronApplication | undefined; + try { + app = await launch(); + let page = await app.firstWindow(); + await expect(page).toHaveURL(nodeURL + "/", { timeout: 60_000 }); + await loginThroughPds(page, nodeURL); + await page.goto(nodeURL + "/settings/privacy"); + const toggleName = `Allow ${localHost} to record your livestream for debugging and improving the service`; + const toggle = page.getByRole("switch", { name: toggleName }); + await expect(toggle).toBeVisible(); + const initial = (await toggle.getAttribute("aria-checked")) === "true"; + const saved = page.waitForResponse( + (response) => + new URL(response.url()).origin === nodeURL && + new URL(response.url()).pathname === "/xrpc/com.atproto.repo.putRecord", + ); + await toggle.click(); + expect((await saved).ok()).toBe(true); + await expect(toggle).toHaveAttribute("aria-checked", String(!initial)); + // Read the actual PDS record: an optimistic toggle alone proves no write. + const recordURL = new URL( + "/xrpc/com.atproto.repo.getRecord", + process.env.PDS_HTTPS_URL!, + ); + recordURL.search = new URLSearchParams({ + repo: process.env.ACCOUNT_DID!, + collection: "place.stream.server.settings", + rkey: localHost, + }).toString(); + const persisted = await page.request.get(recordURL.toString(), { + ignoreHTTPSErrors: true, + }); + expect(persisted.ok()).toBe(true); + expect((await persisted.json()).value.debugRecording).toBe(!initial); + await app.close(); + app = undefined; + app = await launch(); + page = await app.firstWindow(); + await page.goto(nodeURL + "/settings/privacy"); + await expect( + page.getByRole("switch", { name: toggleName }), + ).toHaveAttribute("aria-checked", String(!initial)); + const restoredWrite = page.waitForResponse( + (response) => + new URL(response.url()).origin === nodeURL && + new URL(response.url()).pathname === "/xrpc/com.atproto.repo.putRecord", + ); + await page.getByRole("switch", { name: toggleName }).click(); + expect((await restoredWrite).ok()).toBe(true); + await expect( + page.getByRole("switch", { name: toggleName }), + ).toHaveAttribute("aria-checked", String(initial)); + } finally { + await app?.close(); + await rm(state, { recursive: true, force: true }); + } +}); diff --git a/js/e2e-web/flows/login.ts b/js/e2e-web/flows/login.ts index e37e72e7..355ba2a0 100644 --- a/js/e2e-web/flows/login.ts +++ b/js/e2e-web/flows/login.ts @@ -1,24 +1,28 @@ import { expect, type Page } from "@playwright/test"; import { pointAppAtNode } from "../server-setup"; -// Log in the way a user does: the app's atproto OAuth client talks to the -// node's OAuth proxy (oatproxy), which sends the browser to the PDS's own -// sign-in and consent pages and trades the result for a session. Every hop is -// real https — the harness serves the node and PDS on hostnames that resolve to -// 127.0.0.1, with a throwaway CA (see pkg/cmd/e2e_https.go) — so this runs the -// production OAuth code, not the http://127.0.0.1 development shortcut. -// -// Needs the harness's HTTPS mode (hack/e2e-web-local.sh turns it on); callers -// gate on SERVER_HTTPS_URL themselves and skip when it is unset. -export async function loginThroughPds(page: Page): Promise { - const httpsUrl = process.env.SERVER_HTTPS_URL!; +// Real OAuth login: baseline uses the HTTPS node; upstream mode starts at the +// HTTP loopback node (or an explicit Desktop node URL), continues through the +// HTTPS station, and reaches the PDS's genuine sign-in and consent pages. +// The throwaway CA is scoped to this harness. No fake tokens or Public-mode +// production security relaxations are used for the chained loopback flow. +// Callers require HTTPS PDS metadata and gate their chosen topology themselves. +export async function loginThroughPds( + page: Page, + nodeURL?: string, +): Promise { + const httpsUrl = + nodeURL ?? + (process.env.E2E_UPSTREAM_STATION + ? process.env.SERVER_URL! + : process.env.SERVER_HTTPS_URL!); const pdsUrl = process.env.PDS_HTTPS_URL!; const handle = process.env.ACCOUNT_HANDLE!; const password = process.env.ACCOUNT_PASSWORD!; const appOrigin = new URL(httpsUrl).origin; const pdsOrigin = new URL(pdsUrl).origin; - // This origin is new to the browser, so point the app at the node again. + // Idempotent on an already configured origin, including Desktop localhost. await pointAppAtNode(page, httpsUrl); await page.goto(`${httpsUrl}/login`); @@ -27,11 +31,25 @@ export async function loginThroughPds(page: Page): Promise { .first(); await expect(handleField).toBeVisible({ timeout: 30_000 }); await handleField.fill(handle); - await page.getByTestId("login-submit").click(); - - // oatproxy hands the browser to the PDS's authorization UI; the handle comes - // along as login_hint, so only the password is asked for - await page.waitForURL((u) => u.origin === pdsOrigin, { timeout: 60_000 }); + const stationAuthorization = process.env.E2E_UPSTREAM_STATION + ? page.waitForRequest( + (request) => + new URL(request.url()).origin === + new URL(process.env.UPSTREAM_STATION_URL!).origin && + new URL(request.url()).pathname === "/oauth/authorize", + { timeout: 60_000 }, + ) + : undefined; + // Handle both waits together so an early login failure cannot leave a + // dangling request promise that hides the original error during teardown. + await Promise.all([ + stationAuthorization, + (async () => { + await page.getByTestId("login-submit").click(); + // The handle travels as login_hint; only the password is asked for. + await page.waitForURL((u) => u.origin === pdsOrigin, { timeout: 60_000 }); + })(), + ]); const passwordField = page.locator('input[type="password"]'); await expect(passwordField).toBeVisible({ timeout: 30_000 }); await passwordField.fill(password); diff --git a/js/e2e-web/flows/upstream-station.spec.ts b/js/e2e-web/flows/upstream-station.spec.ts new file mode 100644 index 00000000..003167f0 --- /dev/null +++ b/js/e2e-web/flows/upstream-station.spec.ts @@ -0,0 +1,411 @@ +import { expect, test, type Page } from "@playwright/test"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { stripVTControlCharacters } from "node:util"; +import { loginThroughPds } from "./login"; + +// Semantic native counterpart: .maestro/logged-in/upstream-station.yaml. +// All three servers and the MoQ bridge are real harness processes. SQLite +// observation is read-only except a renewal timestamp fixture; genuine keys, +// grants and responses are unchanged. No production endpoints or mocks. +test.skip(!process.env.E2E_UPSTREAM_STATION, "requires upstream harness mode"); + +const LOCAL = process.env.SERVER_URL!; +const STATION = process.env.UPSTREAM_STATION_URL!; +const DID = process.env.ACCOUNT_DID!; +const UNRELATED = process.env.UNRELATED_ACCOUNT_DID!; + +type Session = { + downstream_dpop_jkt: string; + upstream_dpop_private_jwk: string; + upstream_access_token: string; + downstream_refresh_token: string; + upstream_auth_server_issuer: string; + pds_url: string; + downstream_revoked_at: string | null; +}; + +function query( + node: "local" | "station", + database: "index" | "state", + sql: string, + params: unknown[] = [], +): T[] { + const dir = + process.env[ + node === "local" ? "E2E_LOCAL_DATA_DIR" : "E2E_STATION_DATA_DIR" + ]!; + return JSON.parse( + execFileSync( + "python3", + [ + "-c", + ` +import glob, json, sqlite3, sys +root, kind, sql, params = sys.argv[1:] +paths = glob.glob(root + '/index/index_*.sqlite') if kind == 'index' else [root + '/state.sqlite'] +if len(paths) != 1: + raise RuntimeError('expected exactly one harness database: ' + repr(paths)) +conn = sqlite3.connect('file:' + paths[0] + '?mode=ro', uri=True, timeout=10) +conn.row_factory = sqlite3.Row +print(json.dumps([dict(row) for row in conn.execute(sql, json.loads(params))])) +`, + dir, + database, + sql, + JSON.stringify(params), + ], + { encoding: "utf8" }, + ), + ); +} + +function sessions(node: "local" | "station", jkt: string): Session[] { + return query( + node, + "state", + `SELECT downstream_dpop_jkt, upstream_dpop_private_jwk, + upstream_access_token, downstream_refresh_token, upstream_auth_server_issuer, pds_url, downstream_revoked_at + FROM oauth_sessions WHERE repo_did = ? AND revoked_at IS NULL AND downstream_dpop_jkt = ?`, + [DID, jkt], + ); +} + +function thumbprint(jwk: { + kty: string; + crv: string; + x: string; + y: string; +}): string { + return createHash("sha256") + .update(JSON.stringify({ crv: jwk.crv, kty: jwk.kty, x: jwk.x, y: jwk.y })) + .digest("base64url"); +} + +type PdsRequest = { method: string; path: string; query: string }; +function pdsAudit(): PdsRequest[] { + return readFileSync(process.env.E2E_LOCAL_PDS_AUDIT!, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} +function repoReads(): number { + return pdsAudit().filter((entry) => + /\/(com\.atproto\.sync\.(getRepo|getLatestCommit)|com\.atproto\.repo\.listRecords)$/.test( + entry.path, + ), + ).length; +} + +async function sendChat( + page: Page, + message: string, +): Promise<{ uri: string; jkt: string }> { + const created = page.waitForResponse( + (response) => + response.request().method() === "POST" && + new URL(response.url()).origin === LOCAL && + new URL(response.url()).pathname === + "/xrpc/com.atproto.repo.createRecord" && + response.request().postData()?.includes(message) === true, + ); + const input = page.getByPlaceholder("Type a message..."); + await input.fill(message); + await input.press("Enter"); + const response = await created; + expect( + response.ok(), + `authenticated local write returned ${response.status()}`, + ).toBe(true); + const dpop = (await response.request().allHeaders())["dpop"]; + expect(dpop).toBeTruthy(); + const header = JSON.parse( + Buffer.from(dpop.split(".")[0], "base64url").toString(), + ); + const payload = JSON.parse( + Buffer.from(dpop.split(".")[1], "base64url").toString(), + ); + expect(payload.htu).toBe(`${LOCAL}/xrpc/com.atproto.repo.createRecord`); + const { uri } = await response.json(); + expect(uri).toMatch( + new RegExp(`^at://${DID}/place\\.stream\\.chat\\.message/`), + ); + // Do not accept the optimistic bubble: the exact PDS-created URI must arrive + // in the local durable index, after the initial OAuth repo sync. + await expect + .poll( + () => + query<{ uri: string }>( + "local", + "index", + "SELECT uri FROM chat_messages WHERE uri = ?", + [uri], + ).length, + ) + .toBe(1); + return { uri, jkt: thumbprint(header.jwk) }; +} + +async function refreshAtLocal(page: Page): Promise { + // Expire only the SDK's persisted timestamp, then let the real client renew + // and persist its rotated tokens. Hand-signing a refresh behind the SDK would + // leave its logout holding a stale refresh token. + await page.evaluate(async (did) => { + const db = await new Promise((resolve, reject) => { + const opened = indexedDB.open("@atproto-oauth-client"); + opened.onsuccess = () => resolve(opened.result); + opened.onerror = () => reject(opened.error); + }); + try { + await new Promise((resolve, reject) => { + const transaction = db.transaction("session", "readwrite"); + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error); + transaction.onabort = () => reject(transaction.error); + const cursor = transaction.objectStore("session").openCursor(); + cursor.onsuccess = () => { + const row = cursor.result; + if (!row) return; + const item = row.value; + if (item.value.tokenSet.sub === did) { + item.value.tokenSet.expires_at = new Date(0).toISOString(); + row.update(item); + } + row.continue(); + }; + }); + } finally { + db.close(); + } + }, DID); + const refreshed = page.waitForResponse( + (response) => + response.url() === `${LOCAL}/oauth/token` && + response.request().postData()?.includes("grant_type=refresh_token") === + true && + response.ok(), + ); + await page.reload(); + const body = await (await refreshed).json(); + expect(body.sub).toBe(DID); + expect(body.token_type).toBe("DPoP"); + expect(body.access_token).toBeTruthy(); +} + +test("upstream-station: local login, isolated DPoP hops, MoQ indexing, refresh and revoke", async ({ + page, + request, +}) => { + test.setTimeout(300_000); + await loginThroughPds(page); + await page.goto(`${LOCAL}/`); + await page.getByTestId("home-stream-card").first().click(); + await expect( + page.getByText("Now streaming - e2e test stream").first(), + ).toBeVisible(); + await expect + .poll( + () => + query<{ pds: string; version: string }>( + "local", + "index", + "SELECT pds, version FROM repos WHERE did = ?", + [DID], + )[0]?.version, + ) + .toBeTruthy(); + const readsBefore = repoReads(); + const message = `upstream MoQ ${Date.now()}`; + const { uri, jkt } = await sendChat(page, message); + expect( + repoReads(), + "live MoQ commit must not fall back to a PDS repo scrape", + ).toBe(readsBefore); + expect( + query<{ pds: string }>( + "local", + "index", + "SELECT pds FROM repos WHERE did = ?", + [DID], + )[0].pds, + ).toBe(process.env.PDS_HTTPS_URL); + + const localSession = sessions("local", jkt)[0]; + expect(localSession).toBeDefined(); + expect(localSession.pds_url).toBe(STATION); + expect(localSession.upstream_auth_server_issuer).toBe(STATION); + const stationJkt = thumbprint( + JSON.parse(localSession.upstream_dpop_private_jwk), + ); + expect(stationJkt).not.toBe(jkt); + const stationSession = sessions("station", stationJkt)[0]; + expect(stationSession).toBeDefined(); + expect(stationSession.pds_url).toBe(process.env.PDS_HTTPS_URL); + const pdsJkt = thumbprint( + JSON.parse(stationSession.upstream_dpop_private_jwk), + ); + expect(pdsJkt).not.toBe(stationJkt); + expect(pdsJkt).not.toBe(jkt); + const localTokenBefore = createHash("sha256") + .update(localSession.upstream_access_token) + .digest("hex"); + const stationTokenBefore = createHash("sha256") + .update(stationSession.upstream_access_token) + .digest("hex"); + // Deterministic renewal timing only: the proxies still perform genuine, + // separately DPoP-bound refresh exchanges with the station and the PDS. + for (const [dir, key] of [ + [process.env.E2E_LOCAL_DATA_DIR!, jkt], + [process.env.E2E_STATION_DATA_DIR!, stationJkt], + ]) { + execFileSync("python3", [ + "-c", + ` +import datetime, sqlite3, sys +conn = sqlite3.connect(sys.argv[1] + '/state.sqlite', timeout=10) +expired = (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=1)).isoformat() +conn.execute('UPDATE oauth_sessions SET upstream_access_token_exp = ? WHERE downstream_dpop_jkt = ?', (expired, sys.argv[2])) +conn.commit() +`, + dir, + key, + ]); + } + await refreshAtLocal(page); + await sendChat(page, `write after real chained refresh ${Date.now()}`); + await expect + .poll(() => + createHash("sha256") + .update(sessions("local", jkt)[0].upstream_access_token) + .digest("hex"), + ) + .not.toBe(localTokenBefore); + await expect + .poll(() => + createHash("sha256") + .update(sessions("station", stationJkt)[0].upstream_access_token) + .digest("hex"), + ) + .not.toBe(stationTokenBefore); + + // Publish an unrelated commit AFTER local OAuth, through its real PDS login. + const unrelatedResponse = await request.post( + `${process.env.E2E_PDS_INTERNAL_URL}/xrpc/com.atproto.repo.createRecord`, + { + headers: { Authorization: `Bearer ${process.env.UNRELATED_ACCESS_JWT}` }, + data: { + repo: UNRELATED, + collection: "place.stream.livestream", + record: { + $type: "place.stream.livestream", + createdAt: new Date().toISOString(), + title: `unrelated after login ${Date.now()}`, + }, + }, + }, + ); + expect(unrelatedResponse.ok()).toBe(true); + // A subsequent selected-user event gives the real bridge time to process + // both commits, without using a sleep as the indexing assertion. + await sendChat(page, `selected user after unrelated ${Date.now()}`); + assertScopedIndex(); + + await page.reload(); + await expect(page.getByText(message).first()).toBeVisible(); + expect( + query<{ uri: string }>( + "local", + "index", + "SELECT uri FROM chat_messages WHERE uri = ?", + [uri], + ), + ).toHaveLength(1); + await sendChat(page, `upstream write after reload ${Date.now()}`); + + // Restart the actual local Go process, preserving its genuine on-disk + // sessions. Control belongs to streamplace e2e stdin, not production HTTP. + const restartFile = join(process.env.E2E_LOCAL_DATA_DIR!, "restart.json"); + const generation = existsSync(restartFile) + ? JSON.parse(readFileSync(restartFile, "utf8")).generation + : 0; + writeFileSync(process.env.E2E_CONTROL_FIFO!, "restart-local\n"); + await expect + .poll( + () => + existsSync(restartFile) + ? JSON.parse(readFileSync(restartFile, "utf8")).generation + : 0, + { timeout: 120_000 }, + ) + .toBe(generation + 1); + expect(JSON.parse(readFileSync(restartFile, "utf8")).error).toBeUndefined(); + await page.reload(); + await expect(page.getByText(message).first()).toBeVisible(); + await sendChat(page, `upstream write after node restart ${Date.now()}`); + + // signOut sends a real DPoP-bound /oauth/revoke. Local downstream revocation + // must not expose or destroy the station's long-lived confidential PDS grant. + await page.goto(`${LOCAL}/settings/account`); + const cancellationsBefore = stripVTControlCharacters( + readFileSync(process.env.E2E_ATMOQ_LOG!, "utf8"), + ) + .split("\n") + .filter( + (line) => + line.includes(`track=${DID}`) && line.includes("subscribed cancelled"), + ).length; + const revoked = page.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url() === `${LOCAL}/oauth/revoke` && + response.ok(), + ); + await page.getByText("Log out", { exact: true }).last().click(); + expect((await revoked).ok()).toBe(true); + await expect + .poll(() => sessions("local", jkt)[0]?.downstream_revoked_at) + .toBeTruthy(); + expect(sessions("station", stationJkt)[0]?.downstream_revoked_at).toBeNull(); + await expect + .poll( + () => + stripVTControlCharacters( + readFileSync(process.env.E2E_ATMOQ_LOG!, "utf8"), + ) + .split("\n") + .filter( + (line) => + line.includes(`track=${DID}`) && + line.includes("subscribed cancelled"), + ).length, + ) + .toBeGreaterThan(cancellationsBefore); + assertScopedIndex(); +}); + +function assertScopedIndex(): void { + expect( + query("local", "index", "SELECT did FROM repos WHERE did = ?", [UNRELATED]), + ).toEqual([]); + for (const entry of pdsAudit()) { + expect(entry.path, "local node must not enumerate PDS accounts").not.toBe( + "/xrpc/com.atproto.sync.listRepos", + ); + expect( + entry.path, + "local node must not fall back to PDS WebSocket", + ).not.toBe("/xrpc/com.atproto.sync.subscribeRepos"); + const values: string[] = []; + new URLSearchParams(entry.query).forEach((value) => values.push(value)); + expect(values).not.toContain(UNRELATED); + expect(values).not.toContain(process.env.UNRELATED_ACCOUNT_HANDLE); + } + const relayLog = stripVTControlCharacters( + readFileSync(process.env.E2E_ATMOQ_LOG!, "utf8"), + ); + expect(relayLog).toContain(`track=${DID}`); + expect(relayLog).not.toMatch(/\btrack=(?:"atproto"|atproto)(?:\s|$)/); + expect(relayLog).not.toContain(`track=${UNRELATED}`); +} diff --git a/js/e2e-web/global-setup.ts b/js/e2e-web/global-setup.ts index de473f97..59d7ca71 100644 --- a/js/e2e-web/global-setup.ts +++ b/js/e2e-web/global-setup.ts @@ -1,4 +1,5 @@ import { chromium, expect, type FullConfig } from "@playwright/test"; +import { loginThroughPds } from "./flows/login"; import { pointAppAtNode } from "./server-setup"; import { STORAGE_STATE } from "./storage"; @@ -11,10 +12,30 @@ export default async function globalSetup(_config: FullConfig) { throw new Error("SERVER_URL is not set — start the harness first"); } - const browser = await chromium.launch(); - const page = await browser.newPage(); + const browser = await chromium.launch({ + args: process.env.E2E_TLS_SPKI + ? [`--ignore-certificate-errors-spki-list=${process.env.E2E_TLS_SPKI}`] + : [], + }); + const page = await browser.newPage({ + ...(process.env.E2E_PROXY_URL + ? { + proxy: { + server: process.env.E2E_PROXY_URL, + bypass: "127.0.0.1,localhost", + }, + } + : {}), + }); try { await pointAppAtNode(page, SERVER_URL); + if (process.env.E2E_UPSTREAM_STATION) { + // Keep scenario browser contexts unauthenticated. The actual completed + // bootstrap session remains persisted on the local node and authorizes + // its first repo sync and per-DID MoQ subscription. + await page.context().storageState({ path: STORAGE_STATE }); + await loginThroughPds(page); + } // The harness WHIP stream takes a few seconds after boot to ingest its // first segments and register as live. The home feed fetches getLiveUsers @@ -29,8 +50,24 @@ export default async function globalSetup(_config: FullConfig) { await expect(page.getByTestId("home-stream-card").first()).toBeVisible({ timeout: 30_000, }); + if (process.env.E2E_UPSTREAM_STATION) { + // Bootstrap grants are not an invisible second authorized local user. + // Leave scenarios a configured, logged-out browser and no active local + // subscription, so their final-user revoke actually stops the DID track. + await page.goto(`${SERVER_URL}/settings/account`); + const revoked = page.waitForResponse( + (response) => + response.request().method() === "POST" && + response.url() === `${SERVER_URL}/oauth/revoke` && + response.ok(), + ); + await page.getByText("Log out", { exact: true }).last().click(); + await revoked; + } - await page.context().storageState({ path: STORAGE_STATE }); + if (!process.env.E2E_UPSTREAM_STATION) { + await page.context().storageState({ path: STORAGE_STATE }); + } } finally { await browser.close(); } diff --git a/js/e2e-web/server-setup.ts b/js/e2e-web/server-setup.ts index ec39cfe5..7a9a8c43 100644 --- a/js/e2e-web/server-setup.ts +++ b/js/e2e-web/server-setup.ts @@ -12,7 +12,6 @@ export async function pointAppAtNode(page: Page, url: string) { // toggle "use custom node" on, which reveals the URL field + save button const toggle = page.getByTestId("settings-use-custom-node"); await expect(toggle).toBeVisible({ timeout: 60_000 }); - await toggle.click(); // the react-native-web TextInput renders as an ; fill it const urlField = page @@ -20,6 +19,7 @@ export async function pointAppAtNode(page: Page, url: string) { '[data-testid="settings-custom-node-url"], [data-testid="settings-custom-node-url"] input', ) .first(); + if (!(await urlField.isVisible())) await toggle.click(); await expect(urlField).toBeVisible(); await urlField.fill(url); diff --git a/js/web/src/lib/oauth-resolver.ts b/js/web/src/lib/oauth-resolver.ts index 86f9a865..2cf35d46 100644 --- a/js/web/src/lib/oauth-resolver.ts +++ b/js/web/src/lib/oauth-resolver.ts @@ -1,91 +1,24 @@ -// Duck-typed OAuthResolver that routes through the Streamplace backend. -// -// Why not extend the real `OAuthResolver`? @atproto/oauth-client 0.7.x -// restricts its `exports` to the package root, so deep-importing -// `dist/oauth-resolver` (the only place `OAuthResolver` is exported) is -// blocked. The main entry doesn't re-export it. Adding a direct -// `@atproto/oauth-client` dep just to extend one class felt heavy, so we -// mirror the public surface here. -// -// Surface we need to satisfy (per @atproto/oauth-client 0.7.x): -// - resolve(input) high-level dispatcher; called by OAuthClient.authorize -// - resolveFromIdentity(input) called by OAuthServerAgent.verifyIssuer -// - resolveFromService(input) the original override target -// - getCurrentResourceServer() used by oauth.ts to inject login_hint on PAR -// -// All three resolve* methods return streamplace's OAuth server metadata -// regardless of input. The PAR / authorize flow injects `login_hint: input` -// (or the resolved PDS URL) so the streamplace backend can identify the -// user. The fetch hijack in oauth.ts rewrites the PDS service endpoint of -// the DID document response to streamplace, so a future base-class-style -// resolveFromIdentity would also land on streamplace's metadata. - -export interface OAuthResolverShape { - resolve( - input: string, - options?: Record, - ): Promise<{ metadata: Record }>; - resolveFromService( - input: string, - options?: Record, - ): Promise<{ metadata: Record }>; - resolveFromIdentity( - input: string, - options?: Record, - ): Promise<{ metadata: Record; pds: URL }>; - getCurrentResourceServer(): string | null; -} - -export class StreamplaceOAuthResolver { - private currentResourceServer: string | null = null; - - constructor(private streamplaceUrl: string) {} - - // High-level dispatcher. The base class branches on /^https?:\/\// - // (URLs → resolveFromService, handles → resolveFromIdentity). Both - // paths land on streamplace's metadata for us, so we just call - // resolveFromService unconditionally. - async resolve( - input: string, - options?: Record, - ): Promise<{ metadata: Record }> { - return this.resolveFromService(input, options); - } - - async resolveFromIdentity( - _input: string, - _options?: Record, - ): Promise<{ metadata: Record; pds: URL }> { - // Returns streamplace's metadata plus a pds URL pointing at the - // streamplace backend. The verifyIssuer check in - // OAuthServerAgent uses resolved.pds.href; pointing it at - // streamplace matches the client-side issuer, so the check - // passes. Real identity resolution happens on the streamplace - // backend during the PAR / callback roundtrip. - const { metadata } = await this.resolveFromService(_input, _options); - return { metadata, pds: new URL(this.streamplaceUrl) }; - } - - async resolveFromService( - input: string, - _options?: Record, - ): Promise<{ metadata: Record }> { - this.currentResourceServer = input; - - const metadataUrl = `${this.streamplaceUrl}/.well-known/oauth-authorization-server`; - const res = await fetch(metadataUrl, { - headers: { accept: "application/json" }, - }); - if (!res.ok) { - throw new Error( - `Failed to fetch authorization-server metadata from ${metadataUrl}: ${res.status}`, - ); - } - const metadata = await res.json(); +import type { BrowserOAuthClient } from "@atproto/oauth-client-browser"; + +// The SDK's server factory captures this object during construction. Configure +// its methods in place so authorization, callbacks and refresh share routing. +export function routeOAuthThroughStreamplace( + resolver: BrowserOAuthClient["oauthResolver"], + streamplaceUrl: string, +): () => string | null { + let loginHint: string | null = null; + resolver.resolveFromService = async (input, options) => { + loginHint = input; + const metadata = await resolver.getResourceServerMetadata( + streamplaceUrl, + options, + ); return { metadata }; - } - - getCurrentResourceServer(): string | null { - return this.currentResourceServer; - } + }; + resolver.resolveFromIdentity = async (input, options) => { + const identityInfo = await resolver.resolveIdentity(input, options); + const { metadata } = await resolver.resolveFromService(input, options); + return { identityInfo, metadata, pds: new URL(streamplaceUrl) }; + }; + return () => loginHint; } diff --git a/js/web/src/lib/oauth.ts b/js/web/src/lib/oauth.ts index 483ba51e..12de53dd 100644 --- a/js/web/src/lib/oauth.ts +++ b/js/web/src/lib/oauth.ts @@ -1,12 +1,11 @@ -// Builds an ATProto BrowserOAuthClient with downstream-OAuth support. -// Wraps fetch() to handle loopback rewrites and DID-doc endpoint rewriting. +// Builds an ATProto BrowserOAuthClient routed through the selected node. import { BrowserOAuthClient, type ClientMetadata, clientMetadataSchema, } from "@atproto/oauth-client-browser"; -import { StreamplaceOAuthResolver } from "./oauth-resolver"; +import { routeOAuthThroughStreamplace } from "./oauth-resolver"; import { getCallbackUrl } from "./streamplace-callback"; export default async function createOAuthClient( @@ -16,50 +15,46 @@ export default async function createOAuthClient( throw new Error("streamplaceUrl is required"); } - let customResolver: StreamplaceOAuthResolver | null = null; + let getLoginHint: (() => string | null) | null = null; let meta: ClientMetadata; - const redirectURI = `${streamplaceUrl}${getCallbackUrl()}`; + const nodeUrl = new URL(streamplaceUrl); + const isLoopback = + nodeUrl.protocol === "http:" && + (nodeUrl.hostname === "localhost" || + nodeUrl.hostname === "127.0.0.1" || + nodeUrl.hostname === "[::1]"); + const callbackUrl = new URL(nodeUrl.toString()); + if (isLoopback && callbackUrl.hostname === "localhost") { + callbackUrl.hostname = "127.0.0.1"; + } + callbackUrl.pathname = getCallbackUrl(); + const redirectURI = callbackUrl.toString(); const res = await fetch( `${streamplaceUrl}/oauth/downstream/client-metadata.json?redirect_uri=${encodeURIComponent(redirectURI)}`, ); meta = await res.json(); + // Legacy public dev nodes advertise a HTTPS redirect wrapper even for their + // HTTP loopback client. New local nodes advertise the actual redirect. if ( - streamplaceUrl.startsWith("http://localhost") || - streamplaceUrl.startsWith("http://127.0.0.1") + isLoopback && + meta.client_id?.startsWith("http://localhost?") && + !meta.redirect_uris?.some((uri) => uri === redirectURI) ) { if (!meta.scope) { throw new Error("meta.scope is required"); } - const u = new URL(streamplaceUrl); - let hostname = u.hostname; - if (hostname === "localhost") { - hostname = "127.0.0.1"; - } - let redirect = `${u.protocol}//${hostname}`; - if (u.port !== "") { - redirect = `${redirect}:${u.port}`; - } - redirect = `${redirect}${getCallbackUrl()}`; - const queryParams = new URLSearchParams(); queryParams.set("scope", meta.scope); - queryParams.set("redirect_uri", redirect); + queryParams.set("redirect_uri", redirectURI); meta = { + ...meta, client_id: `http://localhost?${queryParams.toString()}`, - redirect_uris: [redirect as any], - scope: meta.scope, - token_endpoint_auth_method: "none", - client_name: "Loopback client", - response_types: ["code"], - grant_types: ["authorization_code", "refresh_token"], + redirect_uris: [redirectURI as ClientMetadata["redirect_uris"][number]], application_type: "native", - dpop_bound_access_tokens: true, - subject_type: "public", - authorization_signed_response_alg: "ES256", }; } @@ -79,11 +74,11 @@ export default async function createOAuthClient( } if ( - customResolver && + getLoginHint && request.url.includes("/oauth/par") && request.method === "POST" ) { - const resourceServer = customResolver.getCurrentResourceServer(); + const resourceServer = getLoginHint(); if (resourceServer) { const clonedRequest = request.clone(); const body = await clonedRequest.text(); @@ -97,78 +92,18 @@ export default async function createOAuthClient( } } - if (streamplaceUrl.startsWith("http://127.0.0.1")) { - if ( - request.url.includes("plc.directory") || - request.url.endsWith("did.json") || - request.url.endsWith("/.well-known/oauth-protected-resource") || - request.url.endsWith("/.well-known/oauth-authorization-server") - ) { - return fetch(request, init); - } - const newUrl = new URL(request.url.toString()); - newUrl.protocol = "http:"; - newUrl.host = "127.0.0.1:38080"; - let newRequest: Request; - if (request.method === "POST") { - const data = await request.blob(); - newRequest = new Request(newUrl.toString(), { - body: data, - method: "POST", - headers: request.headers, - }); - } else if (request.method === "GET") { - newRequest = new Request(newUrl.toString(), { - method: "GET", - headers: request.headers, - }); - } else { - throw new Error("Unsupported method: " + request.method); - } - return fetch(newRequest); - } else { - if ( - request.url.includes("plc.directory") || - request.url.endsWith("did.json") - ) { - const res = await fetch(request, init); - if (!res.ok) { - return res; - } - const data = await res.json(); - const service = data.service?.find( - (s: { id: string; serviceEndpoint: string }) => - s.id === "#atproto_pds", - ); - if (!service) { - return res; - } - service.serviceEndpoint = streamplaceUrl; - return new Response(JSON.stringify(data), { - status: res.status, - headers: res.headers, - }); - } else { - return fetch(request, init); - } - } + return fetch(request, init); }, + allowHttp: isLoopback, handleResolver: streamplaceUrl, responseMode: "query", clientMetadata: meta, }); - // Replace the default resolver with our downstream-OAuth resolver. - customResolver = new StreamplaceOAuthResolver(streamplaceUrl); - // The library types oauthResolver as a readonly OAuthResolver, but at - // runtime it's a plain property. We duck-type our resolver to match - // the shape the library needs. Cast through unknown to bypass the - // type mismatch (OAuthResolver vs StreamplaceOAuthResolver). - if ("oauthResolver" in client) { - ( - client as unknown as { oauthResolver: StreamplaceOAuthResolver } - ).oauthResolver = customResolver; - } + getLoginHint = routeOAuthThroughStreamplace( + client.oauthResolver, + streamplaceUrl, + ); return client; } diff --git a/pkg/api/feeds.go b/pkg/api/feeds.go index f00e5ff1..814b36dc 100644 --- a/pkg/api/feeds.go +++ b/pkg/api/feeds.go @@ -22,7 +22,7 @@ func (a *StreamplaceAPI) HandleDidJSON(ctx context.Context) httprouter.Handle { host = a.CLI.BroadcasterHost pubKey = atproto.LexiconPubMultibase } - didJSON := atproto.DIDDoc(host, pubKey) + didJSON := atproto.DIDDoc(host, pubKey, a.CLI.AtmoqRelay) w.WriteHeader(200) w.Header().Set("Content-Type", "application/json") bs, err := json.Marshal(didJSON) diff --git a/pkg/atproto/atproto.go b/pkg/atproto/atproto.go index 57d27c8e..07d9e5fd 100644 --- a/pkg/atproto/atproto.go +++ b/pkg/atproto/atproto.go @@ -22,6 +22,10 @@ var SyncGetRepo = comatproto.SyncGetRepo func (atsync *ATProtoSynchronizer) SyncBlueskyRepoCached(ctx context.Context, handle string) (*model.Repo, error) { ctx, span := otel.Tracer("signer").Start(ctx, "SyncBlueskyRepoCached") defer span.End() + handle, err := atsync.indexingDID(ctx, handle) + if err != nil { + return nil, err + } repo, err := atsync.Model.GetRepoByHandleOrDID(handle) if err != nil { return nil, fmt.Errorf("failed to get repo for %s: %w", handle, err) @@ -47,6 +51,10 @@ func (atsync *ATProtoSynchronizer) SyncBlueskyRepoCached(ctx context.Context, ha } func (atsync *ATProtoSynchronizer) SyncBlueskyRepo(ctx context.Context, handle string, mod model.Model) (*model.Repo, error) { + handle, err := atsync.indexingDID(ctx, handle) + if err != nil { + return nil, err + } ident, err := atsync.resolveIdent(ctx, handle, true) if err != nil { return nil, fmt.Errorf("failed to resolve Bluesky handle %s: %w", handle, err) @@ -71,6 +79,9 @@ func (atsync *ATProtoSynchronizer) SyncBlueskyRepo(ctx context.Context, handle s handleLock := handleLocks.GetLock(ident.DID.String()) handleLock.Lock() defer handleLock.Unlock() + if _, err := atsync.indexingDID(ctx, ident.DID.String()); err != nil { + return nil, err + } ctx = markWalking(ctx, ident.DID.String()) // Tell re-entrant callers (handleCreateUpdate syncs the repos it sees @@ -140,6 +151,9 @@ func (atsync *ATProtoSynchronizer) SyncBlueskyRepo(ctx context.Context, handle s } return nil, err } + if _, err := atsync.indexingDID(ctx, ident.DID.String()); err != nil { + return nil, err + } // A completed backfill proves the account is fine, so Status goes back to // empty -- UpdateRepo writes every column, so this happens by construction. @@ -189,6 +203,9 @@ func (atsync *ATProtoSynchronizer) SyncBlueskyRepo(ctx context.Context, handle s // served -- and stays out of the wedge path -- for the entire time its history // is being filled in. func (atsync *ATProtoSynchronizer) DeepenRepo(ctx context.Context, did string) (bool, string, error) { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return false, "", err + } repo, err := atsync.Model.GetRepo(did) if err != nil { return false, "", fmt.Errorf("failed to get repo for %s: %w", did, err) @@ -216,6 +233,9 @@ func (atsync *ATProtoSynchronizer) DeepenRepo(ctx context.Context, did string) ( handleLock := handleLocks.GetLock(did) handleLock.Lock() defer handleLock.Unlock() + if _, err := atsync.indexingDID(ctx, did); err != nil { + return false, "", err + } defer markSyncInFlight(did)() ctx = markWalking(ctx, did) @@ -249,6 +269,9 @@ func (atsync *ATProtoSynchronizer) DeepenRepo(ctx context.Context, did string) ( } return false, "", err } + if _, err := atsync.indexingDID(ctx, did); err != nil { + return false, "", err + } applied, err := atsync.Model.AdvanceRepoBackfill(ctx, did, rev, root, window.Lo, window.Genesis) if err != nil { @@ -317,6 +340,9 @@ func syncInFlight(did string) bool { } func (atsync *ATProtoSynchronizer) RefreshIdentity(ctx context.Context, did string) (*identity.Identity, error) { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return nil, err + } id, err := atsync.resolveIdent(ctx, did, false) if err != nil { return nil, fmt.Errorf("failed to resolve ident: %w", err) @@ -456,7 +482,31 @@ func (atsync *ATProtoSynchronizer) IdentityDirectory() identity.Directory { return atsync.directory(true) } -func DIDDoc(host string, pubMultibase string) map[string]any { +func DIDDoc(host string, pubMultibase string, atmoqRelay string) map[string]any { + services := []map[string]any{ + { + "id": "#bsky_fg", + "type": "BskyFeedGenerator", + "serviceEndpoint": fmt.Sprintf("https://%s", host), + }, + { + "id": "#atproto_pds", + "type": "AtprotoPersonalDataServer", + "serviceEndpoint": fmt.Sprintf("https://%s", host), + }, + { + "id": "#" + StreamplaceServiceID, + "type": "StreamplaceServer", + "serviceEndpoint": fmt.Sprintf("https://%s", host), + }, + } + if atmoqRelay != "" { + services = append(services, map[string]any{ + "id": "#" + AtmoqServiceID, + "type": "AtprotoMoQRelay", + "serviceEndpoint": atmoqRelay, + }) + } return map[string]any{ "@context": []string{ "https://www.w3.org/ns/did/v1", @@ -465,23 +515,7 @@ func DIDDoc(host string, pubMultibase string) map[string]any { }, "id": fmt.Sprintf("did:web:%s", host), "alsoKnownAs": []string{}, - "service": []map[string]any{ - { - "id": "#bsky_fg", - "type": "BskyFeedGenerator", - "serviceEndpoint": fmt.Sprintf("https://%s", host), - }, - { - "id": "#atproto_pds", - "type": "AtprotoPersonalDataServer", - "serviceEndpoint": fmt.Sprintf("https://%s", host), - }, - { - "id": "#" + StreamplaceServiceID, - "type": "StreamplaceServer", - "serviceEndpoint": fmt.Sprintf("https://%s", host), - }, - }, + "service": services, "verificationMethod": []map[string]any{ { "id": fmt.Sprintf("did:web:%s#atproto", host), diff --git a/pkg/atproto/backfill_walk.go b/pkg/atproto/backfill_walk.go index d6b89eef..04e03295 100644 --- a/pkg/atproto/backfill_walk.go +++ b/pkg/atproto/backfill_walk.go @@ -270,6 +270,9 @@ func (atsync *ATProtoSynchronizer) backfillRepo(ctx context.Context, ident *iden // every record in range to the same indexing path the firehose uses. func (atsync *ATProtoSynchronizer) walkBackfill(ctx context.Context, ident *identity.Identity, xrpcc *xrpc.Client, ranges []reposync.KeyRange) (string, string, error) { did := ident.DID.String() + if _, err := atsync.indexingDID(ctx, did); err != nil { + return "", "", err + } // The *uncached* directory on purpose: a signing key cached from before a // rotation would fail commit verification, and backfills are rare enough @@ -281,18 +284,31 @@ func (atsync *ATProtoSynchronizer) walkBackfill(ctx context.Context, ident *iden // a client with the hint-capturing transport installed, which is what // [SyncHTTPClient] is for -- callers build xrpcc with it. retry := reposync.RetryPolicy{Hints: pdsBackoffHints} + var authorize func(context.Context) error + if atsync.upstreamMode() { + authorize = func(ctx context.Context) error { + _, err := atsync.indexingDID(ctx, did) + return err + } + } fetcher := &reposync.CachedFetcher{ // Bounded lifetime: one cache per backfill, so the head fetch and the // walk share blocks without holding a repo in memory afterwards. Cache: reposync.NewMemoryBlockCache(), Inner: &pdsLockedFetcher{ - lock: pdsLocks.GetLock(ident.PDSEndpoint()), - inner: &reposync.XRPCBlockFetcher{Client: xrpcc, DID: did, Retry: retry}, + lock: pdsLocks.GetLock(ident.PDSEndpoint()), + inner: &reposync.XRPCBlockFetcher{Client: xrpcc, DID: did, Retry: retry}, + authorize: authorize, }, } fetchHead := func(ctx context.Context) (*reposync.Head, error) { + if authorize != nil { + if err := authorize(ctx); err != nil { + return nil, err + } + } head, err := reposync.FetchVerifiedHead(ctx, xrpcc, fetcher, dir, did, retry) if err != nil { return nil, fmt.Errorf("failed to fetch verified head for %s from PDS %s: %w", did, xrpcc.Host, err) @@ -315,6 +331,9 @@ func (atsync *ATProtoSynchronizer) walkBackfill(ctx context.Context, ident *iden bs := rec err = atsync.handleCreateUpdate(ctx, did, rkey, &bs, rcid.String(), nsid, false, true, head.Rev) if err != nil { + if errors.Is(err, ErrRepoNotLocal) { + return err + } log.Warn(ctx, "failed to handle create update", "err", err) // invalid CBOR and stuff should get ignored, so we don't return } @@ -429,6 +448,10 @@ func (atsync *ATProtoSynchronizer) legacyBackfill(ctx context.Context, ident *id rev := "" pdsLock := pdsLocks.GetLock(ident.PDSEndpoint()) pdsLock.Lock() + if _, err := atsync.indexingDID(ctx, ident.DID.String()); err != nil { + pdsLock.Unlock() + return "", err + } repoBytes, err := SyncGetRepo(ctx, xrpcc, ident.DID.String(), rev) pdsLock.Unlock() if err != nil { @@ -525,8 +548,9 @@ func newSyncHTTPClient() *http.Client { // the host itself asked for ([pdsBackoffHints]) rather than a guess. // reposync.DefaultRetryMaxDelay is what keeps that pause bounded. type pdsLockedFetcher struct { - lock *sync.Mutex - inner reposync.BlockFetcher + lock *sync.Mutex + inner reposync.BlockFetcher + authorize func(context.Context) error } var _ reposync.BlockFetcher = (*pdsLockedFetcher)(nil) @@ -534,6 +558,11 @@ var _ reposync.BlockFetcher = (*pdsLockedFetcher)(nil) func (f *pdsLockedFetcher) GetBlocks(ctx context.Context, cids []cid.Cid) (map[cid.Cid][]byte, error) { f.lock.Lock() defer f.lock.Unlock() + if f.authorize != nil { + if err := f.authorize(ctx); err != nil { + return nil, err + } + } return f.inner.GetBlocks(ctx, cids) } diff --git a/pkg/atproto/firehose.go b/pkg/atproto/firehose.go index d80233e5..2b3cee65 100644 --- a/pkg/atproto/firehose.go +++ b/pkg/atproto/firehose.go @@ -58,6 +58,7 @@ type ATProtoSynchronizer struct { CachedPLCDirectory identity.Directory dirMu sync.Mutex OATProxy *oatproxy.OATProxy + UpstreamStation *UpstreamStation // firehose liveness, written from every relay consumer concurrently // (unix nanos). @@ -99,6 +100,12 @@ func sinceNanos(ns int64) time.Duration { // ourselves when we actually have an HTTP listener to dial — i.e. never in // tests that run without the server. func (atsync *ATProtoSynchronizer) relayHosts() []string { + if atsync.CLI.UpstreamStation != "" { + if atsync.UpstreamStation == nil { + return nil + } + return []string{atsync.UpstreamStation.FirehoseURL} + } seen := map[string]struct{}{} var hosts []string add := func(h string) { @@ -160,6 +167,9 @@ func (atsync *ATProtoSynchronizer) StartFirehose(ctx context.Context) error { atsync.identityDedup = newFirehoseDeduper(dedupWindow) atsync.markSeen() atsync.markEvent(time.Now()) + if atsync.upstreamMode() { + return atsync.startUpstreamFirehose(ctx) + } log.Log(ctx, "starting firehose consumers", "relays", relays) @@ -168,7 +178,7 @@ func (atsync *ATProtoSynchronizer) StartFirehose(ctx context.Context) error { relay := relay spmetrics.TouchRelay(relay, relayProtocol(relay)) g.Go(func() error { - atsync.consumeRelay(ctx, relay) + atsync.consumeRelay(ctx, relay, "") return nil }) } @@ -183,28 +193,31 @@ func (atsync *ATProtoSynchronizer) StartFirehose(ctx context.Context) error { // capped exponential backoff. It never returns an error: an unhealthy relay // must not disturb the others (or crash the node), so it just loops until ctx // is cancelled. -func (atsync *ATProtoSynchronizer) consumeRelay(ctx context.Context, relay string) { - ctx = log.WithLogValues(ctx, "relay", relay) - - cursor := atsync.newRelayCursor(ctx, relay) - // Persist progress on a timer and once more on the way out, so a restart - // resumes near where we left off rather than re-tailing from live. - flushDone := make(chan struct{}) - go func() { - defer close(flushDone) - ticker := time.NewTicker(cursorFlushInterval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - cursor.flush(ctx) - return - case <-ticker.C: - cursor.flush(ctx) +func (atsync *ATProtoSynchronizer) consumeRelay(ctx context.Context, relay, track string) { + ctx = log.WithLogValues(ctx, "relay", relay, "track", track) + + // On-demand DID tracks reset their group sequences when no subscribers + // remain. Only aggregate tracks have durable replay cursors. + cursor := &relayCursor{host: relay} + if track == "" { + cursor = atsync.newRelayCursor(ctx, relay) + flushDone := make(chan struct{}) + go func() { + defer close(flushDone) + ticker := time.NewTicker(cursorFlushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + cursor.flush(ctx) + return + case <-ticker.C: + cursor.flush(ctx) + } } - } - }() - defer func() { <-flushDone }() + }() + defer func() { <-flushDone }() + } const ( minBackoff = time.Second @@ -216,7 +229,12 @@ func (atsync *ATProtoSynchronizer) consumeRelay(ctx context.Context, relay strin return } start := time.Now() - err := atsync.connectRelay(ctx, relay, cursor) + var err error + if track == "" { + err = atsync.connectRelay(ctx, relay, cursor) + } else { + err = atsync.connectRelayMoq(ctx, relay, track, cursor) + } if ctx.Err() != nil { return } @@ -300,7 +318,7 @@ func (atsync *ATProtoSynchronizer) connectRelay(ctx context.Context, relay strin // handlers, backoff — is shared, so this is just a transport swap. switch u.Scheme { case "moqt", "moql", "moq", "moqs": - return atsync.connectRelayMoq(ctx, relay, cursor) + return atsync.connectRelayMoq(ctx, relay, "atproto", cursor) } u.Path = "xrpc/com.atproto.sync.subscribeRepos" if seq, ok := cursor.param(); ok { @@ -580,6 +598,9 @@ func commitHasIndexedOps(evt *indigoatproto.SyncSubscribeRepos_Commit) bool { } func (atsync *ATProtoSynchronizer) handleCommitEventOps(ctx context.Context, evt *indigoatproto.SyncSubscribeRepos_Commit) { + if !atsync.indexEventAllowed(ctx, evt.Repo) { + return + } ctx = log.WithLogValues(ctx, "event", "commit", "did", evt.Repo, "rev", evt.Rev, "seq", fmt.Sprintf("%d", evt.Seq), "func", "handleCommitEventOps") if evt.TooBig { @@ -916,7 +937,7 @@ func (atsync *ATProtoSynchronizer) handleIndexedOps(ctx context.Context, evt *in // It is called from the commit path with the row that path already loaded, so // the common case costs one comparison and no query at all. func (atsync *ATProtoSynchronizer) reviveRepo(ctx context.Context, r *model.Repo) { - if !r.TerminalStatus() { + if r == nil || !r.TerminalStatus() { return } log.Log(ctx, "repo committed while parked, clearing terminal status", "did", r.DID, "handle", r.Handle, "status", r.Status) @@ -928,6 +949,9 @@ func (atsync *ATProtoSynchronizer) reviveRepo(ctx context.Context, r *model.Repo } func (atsync *ATProtoSynchronizer) handleIdentityEventOps(ctx context.Context, evt *indigoatproto.SyncSubscribeRepos_Identity) { + if !atsync.indexEventAllowed(ctx, evt.Did) { + return + } handle := "" if evt.Handle != nil { handle = *evt.Handle diff --git a/pkg/atproto/firehose_moq.go b/pkg/atproto/firehose_moq.go index fcb8e741..e4554cb6 100644 --- a/pkg/atproto/firehose_moq.go +++ b/pkg/atproto/firehose_moq.go @@ -34,7 +34,7 @@ const moqResumeProbeTimeout = 30 * time.Second // connect it tails the live edge. Gaps past the relay's retained window are // covered by cross-relay redelivery and the idempotent handlers, exactly as for // the best-effort WebSocket cursor. -func (atsync *ATProtoSynchronizer) connectRelayMoq(ctx context.Context, relay string, cursor *relayCursor) error { +func (atsync *ATProtoSynchronizer) connectRelayMoq(ctx context.Context, relay, track string, cursor *relayCursor) error { streamCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -54,10 +54,13 @@ func (atsync *ATProtoSynchronizer) connectRelayMoq(ctx context.Context, relay st // the only caller, so groupStart below already reflects that decision. var sub *atmoq.Subscription resumedFrom, resumed := cursor.groupStart() + if track != atmoq.DefaultTrack { + resumed = false + } if resumed { - sub, err = sess.SubscribeFrom(streamCtx, atmoq.DefaultBroadcast, atmoq.DefaultTrack, resumedFrom) + sub, err = sess.SubscribeFrom(streamCtx, atmoq.DefaultBroadcast, track, resumedFrom) } else { - sub, err = sess.Subscribe(streamCtx, atmoq.DefaultBroadcast, atmoq.DefaultTrack) + sub, err = sess.Subscribe(streamCtx, atmoq.DefaultBroadcast, track) } if err != nil { return fmt.Errorf("subscribing to moq firehose: %w", err) @@ -65,14 +68,25 @@ func (atsync *ATProtoSynchronizer) connectRelayMoq(ctx context.Context, relay st defer sub.Close() protocol := relayProtocol(relay) - spmetrics.FirehoseRelaysConnected.WithLabelValues(relay, protocol).Set(1) - defer spmetrics.FirehoseRelaysConnected.WithLabelValues(relay, protocol).Set(0) + spmetrics.FirehoseRelaysConnected.WithLabelValues(relay, protocol).Inc() + defer spmetrics.FirehoseRelaysConnected.WithLabelValues(relay, protocol).Dec() rsc := atsync.repoStreamCallbacks(ctx, relay, cursor, cancel) scheduler := parallel.NewScheduler(10, 100, relay, rsc.EventHandler) defer scheduler.Shutdown() - log.Log(ctx, "connected to relay firehose (moq)", "version", sess.Version()) + log.Log(ctx, "connected to relay firehose (moq)", "version", sess.Version(), "track", track) + if track != atmoq.DefaultTrack { + // Reconcile after subscribing, so changes made while this sparse track + // was absent cannot fall between the PDS snapshot and live delivery. + // The consumer context survives reconnects; a dropped connection must + // not abort the gap repair it triggered. + go func() { + atsync.sweepOnce(ctx, func(sweepCtx context.Context) error { + return atsync.sweep(sweepCtx, false) + }) + }() + } for { // The relay accepts a SubscribeFrom for any group, including one past // its live edge that it will never produce (a stale or corrupted diff --git a/pkg/atproto/headcheck.go b/pkg/atproto/headcheck.go index 5541e38c..4b944696 100644 --- a/pkg/atproto/headcheck.go +++ b/pkg/atproto/headcheck.go @@ -21,6 +21,9 @@ import ( // the same backoff memory as every other sync request, so a pass over thousands // of repos is as polite to a host as a backfill is. func (atsync *ATProtoSynchronizer) headRev(ctx context.Context, did string) (string, *identity.Identity, error) { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return "", nil, err + } ident, err := atsync.resolveIdent(ctx, did, true) if err != nil { return "", nil, fmt.Errorf("failed to resolve %s: %w", did, err) @@ -34,6 +37,9 @@ func (atsync *ATProtoSynchronizer) headRev(ctx context.Context, did string) (str lock := pdsLocks.GetLock(host) lock.Lock() defer lock.Unlock() + if _, err := atsync.indexingDID(ctx, did); err != nil { + return "", nil, err + } latest, err := reposync.LatestCommit(ctx, xrpcc, did, reposync.RetryPolicy{Hints: pdsBackoffHints}) if err != nil { return "", ident, err @@ -59,6 +65,9 @@ func (atsync *ATProtoSynchronizer) refreshDriftedIdentity(ctx context.Context, r if ident == nil || (ident.Handle.String() == repo.Handle && ident.PDSEndpoint() == repo.PDS) { return } + if !atsync.indexEventAllowed(ctx, repo.DID) { + return + } fresh, err := atsync.resolveIdent(ctx, repo.DID, false) if err != nil { log.Warn(ctx, "failed to confirm drifted identity", "did", repo.DID, "handle", repo.Handle, "err", err) diff --git a/pkg/atproto/indexing_scope.go b/pkg/atproto/indexing_scope.go new file mode 100644 index 00000000..280a3289 --- /dev/null +++ b/pkg/atproto/indexing_scope.go @@ -0,0 +1,64 @@ +package atproto + +import ( + "context" + "errors" + "fmt" + "strings" + + "stream.place/streamplace/pkg/log" +) + +// ErrRepoNotLocal prevents upstream-mode discovery or scraping of accounts +// that have not successfully authenticated to this node. +var ErrRepoNotLocal = errors.New("repo has no authenticated local session") + +func (atsync *ATProtoSynchronizer) upstreamMode() bool { + return atsync.UpstreamStation != nil || (atsync.CLI != nil && atsync.CLI.UpstreamStation != "") +} + +// indexingDID authorizes before identity resolution or a cache hit. Resolve +// local handles to the persisted DID so a handle reassignment cannot authorize +// scraping a different remote account. +func (atsync *ATProtoSynchronizer) indexingDID(ctx context.Context, identifier string) (string, error) { + if !atsync.upstreamMode() { + return identifier, nil + } + if atsync.StatefulDB == nil { + return "", fmt.Errorf("%w: %s", ErrRepoNotLocal, identifier) + } + did, err := atsync.StatefulDB.IndexingDID(ctx, identifier) + if err != nil { + return "", fmt.Errorf("failed to check local OAuth session: %w", err) + } + // Identity events can rename an already indexed user before the OAuth + // session's cached handle changes. The index may supply the DID, but only + // the persisted session can authorize it. + if did == "" && !strings.HasPrefix(identifier, "did:") && atsync.Model != nil { + repo, err := atsync.Model.GetRepoByHandleOrDID(identifier) + if err != nil { + return "", fmt.Errorf("failed to look up local handle: %w", err) + } + if repo != nil { + did, err = atsync.StatefulDB.IndexingDID(ctx, repo.DID) + if err != nil { + return "", fmt.Errorf("failed to check local OAuth session: %w", err) + } + } + } + if did == "" { + return "", fmt.Errorf("%w: %s", ErrRepoNotLocal, identifier) + } + return did, nil +} + +func (atsync *ATProtoSynchronizer) indexEventAllowed(ctx context.Context, did string) bool { + _, err := atsync.indexingDID(ctx, did) + if err == nil { + return true + } + if !errors.Is(err, ErrRepoNotLocal) { + log.Error(ctx, "failed to authorize indexed event", "did", did, "err", err) + } + return false +} diff --git a/pkg/atproto/indexing_scope_test.go b/pkg/atproto/indexing_scope_test.go new file mode 100644 index 00000000..2bb2427e --- /dev/null +++ b/pkg/atproto/indexing_scope_test.go @@ -0,0 +1,248 @@ +package atproto + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "sync/atomic" + "testing" + "time" + + indigoatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/repo" + glex "github.com/streamplace/glex/runtime" + "github.com/streamplace/oatproxy/pkg/oatproxy" + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/comatproto" + "stream.place/streamplace/pkg/config" + "stream.place/streamplace/pkg/devenv" + "stream.place/streamplace/pkg/model" + "stream.place/streamplace/pkg/placestream" + "stream.place/streamplace/pkg/statedb" +) + +func indexingSession(did, handle string) *oatproxy.OAuthSession { + expires := time.Now().UTC().Add(time.Hour) + return &oatproxy.OAuthSession{ + DID: did, Handle: handle, DownstreamDPoPJKT: "jkt-" + did, + DownstreamAccessToken: "local", UpstreamAccessToken: "station", + UpstreamAccessTokenExp: &expires, UpstreamRefreshToken: "refresh", + } +} + +func TestUpstreamIndexingRejectsUnrelatedNetworkWork(t *testing.T) { + ctx := t.Context() + atsync, mod, _ := offlineSynchronizer(t) + atsync.CLI.UpstreamStation = "did:web:station.test" + var requests atomic.Int64 + pds := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusBadRequest) + })) + defer pds.Close() + oldClient := SyncHTTPClient + SyncHTTPClient = pds.Client() + t.Cleanup(func() { SyncHTTPClient = oldClient }) + const did = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: syntax.DID(did), Handle: syntax.Handle("unrelated.test"), + Services: map[string]identity.ServiceEndpoint{ + "atproto_pds": {Type: "AtprotoPersonalDataServer", URL: pds.URL}, + "atproto_labeler": {Type: "AtprotoLabeler", URL: pds.URL}, + }, + }) + atsync.PLCDirectory, atsync.CachedPLCDirectory = &dir, &dir + // This repo was indexed before switching from station/server mode. A + // cached row, even a complete one, does not make its owner a local user. + require.NoError(t, mod.UpdateRepo(&model.Repo{ + DID: did, Handle: "unrelated.test", PDS: pds.URL, + Version: "3lprev0000000", BackfillFloor: "3lprev0000000", + })) + require.NoError(t, atsync.StatefulDB.AddRepo(did)) + operations := []struct { + name string + run func() error + }{ + {"explicit DID", func() error { _, err := atsync.SyncBlueskyRepo(ctx, did, mod); return err }}, + {"explicit handle", func() error { _, err := atsync.SyncBlueskyRepo(ctx, "unrelated.test", mod); return err }}, + {"cached DID", func() error { _, err := atsync.SyncBlueskyRepoCached(ctx, did); return err }}, + {"cached handle", func() error { _, err := atsync.SyncBlueskyRepoCached(ctx, "unrelated.test"); return err }}, + {"history", func() error { _, _, err := atsync.DeepenRepo(ctx, did); return err }}, + {"head", func() error { _, _, err := atsync.headRev(ctx, did); return err }}, + {"identity", func() error { _, err := atsync.RefreshIdentity(ctx, did); return err }}, + {"verifier seed", func() error { return atsync.seedVerifier(ctx, did) }}, + {"label seed", func() error { return atsync.seedLabels(ctx, did, []string{"verified"}) }}, + {"label firehose", func() error { return atsync.StartLabelerFirehoseRetry(ctx, did) }}, + {"record event", func() error { + return atsync.handleCreateUpdate(ctx, did, syntax.RecordKey("key"), nil, "", syntax.NSID("place.stream.chat.message"), false, false, "") + }}, + } + for _, op := range operations { + t.Run(op.name, func(t *testing.T) { + require.ErrorIs(t, op.run(), ErrRepoNotLocal) + require.Zero(t, requests.Load(), "unauthorized work must never reach the PDS") + }) + } + // A filtered firehose commit would ordinarily advance the old repo's rev; + // unrelated events must not mutate its index or schedule repair work. + atsync.handleCommitEventOps(ctx, commitEvent(did, "3lprev0000000", "3lprev0000001")) + atsync.handleIdentityEventOps(ctx, &indigoatproto.SyncSubscribeRepos_Identity{Did: did}) + stored, err := mod.GetRepo(did) + require.NoError(t, err) + require.Equal(t, "3lprev0000000", stored.Version) + candidates, err := atsync.sweepCandidates(ctx) + require.NoError(t, err) + require.Empty(t, candidates) + items, _, err := atsync.deepenCandidates(ctx) + require.NoError(t, err) + require.Empty(t, items) + atsync.feedUnresolved(ctx, []sweepItem{{DID: did}}, func(sweepItem) { t.Error("unauthorized candidate was scheduled") }) + require.NoError(t, atsync.Sweep(ctx)) + require.Zero(t, requests.Load()) + + // Persisting the completed login immediately changes the same gates, + // without updating an in-memory allowlist or an indexed repo set. + session := indexingSession(did, "unrelated.test") + require.NoError(t, atsync.StatefulDB.CreateOAuthSession(session.DownstreamDPoPJKT, session)) + _, err = atsync.SyncBlueskyRepoCached(ctx, "unrelated.test") + require.NoError(t, err) + require.NoError(t, mod.UpdateRepoIdentity(did, "renamed.test", pds.URL)) + _, err = atsync.SyncBlueskyRepoCached(ctx, "renamed.test") + require.NoError(t, err, "an indexed local user's handle may change before their OAuth session's handle") + _, _, err = atsync.headRev(ctx, did) + require.Error(t, err, "the test PDS intentionally rejects the request") + require.NotErrorIs(t, err, ErrRepoNotLocal) + require.EqualValues(t, 1, requests.Load(), "an authorized head check reaches the real HTTP server") + candidates, err = atsync.sweepCandidates(ctx) + require.NoError(t, err) + require.Equal(t, []string{did}, candidates) + now := time.Now().UTC() + require.NoError(t, atsync.StatefulDB.DB.Model(&oatproxy.OAuthSession{}). + Where("repo_did = ?", did).Update("downstream_revoked_at", now).Error) + _, err = atsync.SyncBlueskyRepoCached(ctx, did) + require.ErrorIs(t, err, ErrRepoNotLocal) + candidates, err = atsync.sweepCandidates(ctx) + require.NoError(t, err) + require.Empty(t, candidates) + require.EqualValues(t, 1, requests.Load()) +} + +func TestUpstreamIndexingEligibilitySurvivesRestart(t *testing.T) { + ctx := context.Background() + cli := &config.CLI{ + DBURL: "sqlite://" + filepath.Join(t.TempDir(), "state.sqlite"), + UpstreamStation: "did:web:station.test", + } + mod, err := model.MakeDB(":memory:") + require.NoError(t, err) + state, err := statedb.MakeDB(ctx, cli, nil, mod) + require.NoError(t, err) + const local = "did:plc:aaaaaaaaaaaaaaaaaaaaaaaa" + const unrelated = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + session := indexingSession(local, "local.test") + // Pending PAR/authorization alone cannot let a public caller scrape a DID. + session.DownstreamAccessToken = "" + session.DownstreamAuthorizationCode = "pending-code" + require.NoError(t, state.CreateOAuthSession(session.DownstreamDPoPJKT, session)) + atsync := &ATProtoSynchronizer{CLI: cli, StatefulDB: state, Model: mod} + _, err = atsync.indexingDID(ctx, local) + require.ErrorIs(t, err, ErrRepoNotLocal) + // The successful local token exchange persists before its initial sync. + session.DownstreamAccessToken = "local-token" + require.NoError(t, state.UpdateOAuthSession(session.DownstreamDPoPJKT, session)) + require.NoError(t, state.AddRepo(unrelated)) + require.NoError(t, mod.UpdateRepo(syncedRepo(unrelated, "3lprev0000000"))) + sqlDB, err := state.DB.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + state, err = statedb.MakeDB(ctx, cli, nil, mod) + require.NoError(t, err) + sqlDB, err = state.DB.DB() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + atsync = &ATProtoSynchronizer{CLI: cli, StatefulDB: state, Model: mod} + did, err := atsync.indexingDID(ctx, "local.test") + require.NoError(t, err) + require.Equal(t, local, did) + candidates, err := atsync.sweepCandidates(ctx) + require.NoError(t, err) + require.Equal(t, []string{local}, candidates, "a persisted completed login is boot-eligible even if its initial sync never ran") + _, err = atsync.indexingDID(ctx, unrelated) + require.ErrorIs(t, err, ErrRepoNotLocal, "old server-mode index rows must not survive as indexing permissions") +} + +type indexingRequestCounter struct { + base http.RoundTripper + requests atomic.Int64 +} + +func (c *indexingRequestCounter) RoundTrip(req *http.Request) (*http.Response, error) { + c.requests.Add(1) + return c.base.RoundTrip(req) +} + +// A real PDS CAR carrying a chat record from an unrelated account must never +// cause discovery. A completed local session, in contrast, enables a real +// verified backfill without replacing the indexed repo's actual PDS endpoint. +func TestUpstreamIndexingRealPDSAndUnrelatedFirehose(t *testing.T) { + dev := devenv.WithDevEnv(t) + ctx := t.Context() + atsync, mod := backfillTestSynchronizer(t, dev) + atsync.CLI.UpstreamStation = "did:web:station.test" + local, outsider := dev.CreateAccount(t), dev.CreateAccount(t) + createBackfillRecord(t, local, "place.stream.chat.message", "", chatMessageRecord(local.DID, "local history")) + path := createBackfillRecord(t, outsider, "place.stream.chat.message", "", chatMessageRecord(local.DID, "unrelated sender")) + blocks, err := comatproto.SyncGetRepo(ctx, outsider.XRPC, outsider.DID, "") + require.NoError(t, err) + car, err := repo.ReadRepoFromCar(ctx, bytes.NewReader(blocks)) + require.NoError(t, err) + recordCID, _, err := car.GetRecordBytes(ctx, path) + require.NoError(t, err) + wire := &comatproto.SyncSubscribeRepos_Commit{ + Repo: outsider.DID, Blocks: blocks, Rev: car.SignedCommit().Rev, + Commit: glex.Link(recordCID), Time: time.Now().UTC().Format(time.RFC3339), + Ops: []comatproto.SyncSubscribeRepos_RepoOp{{ + Action: "create", Path: path, Cid: glex.Link(recordCID), + }}, + } + var buf bytes.Buffer + require.NoError(t, wire.MarshalCBOR(&buf)) + var evt indigoatproto.SyncSubscribeRepos_Commit + require.NoError(t, evt.UnmarshalCBOR(bytes.NewReader(buf.Bytes()))) + + counter := &indexingRequestCounter{base: SyncHTTPClient.Transport} + SyncHTTPClient.Transport = counter + t.Cleanup(func() { SyncHTTPClient.Transport = counter.base }) + atsync.handleCommitEventOps(ctx, &evt) + outsiderRepo, err := mod.GetRepo(outsider.DID) + require.NoError(t, err) + require.Nil(t, outsiderRepo) + require.Zero(t, counter.requests.Load()) + _, err = atsync.SyncBlueskyRepoCached(ctx, local.DID) + require.ErrorIs(t, err, ErrRepoNotLocal) + require.Zero(t, counter.requests.Load()) + + session := indexingSession(local.DID, local.Handle) + session.PDSUrl = "https://station.test" + require.NoError(t, atsync.StatefulDB.CreateOAuthSession(session.DownstreamDPoPJKT, session)) + indexed, err := atsync.SyncBlueskyRepoCached(ctx, local.DID) + require.NoError(t, err) + require.NotEmpty(t, indexed.Version) + require.Equal(t, dev.PDSURL, indexed.PDS) + require.Positive(t, counter.requests.Load(), "completed local login permits actual PDS sync requests") + before := counter.requests.Load() + atsync.handleCommitEventOps(ctx, &evt) + outsiderRepo, err = mod.GetRepo(outsider.DID) + require.NoError(t, err) + require.Nil(t, outsiderRepo) + require.Equal(t, before, counter.requests.Load()) + messages, err := mod.MostRecentChatMessages(local.DID, time.Time{}) + require.NoError(t, err) + require.Len(t, messages, 1) + require.Equal(t, "local history", messages[0].Record.Val.(*placestream.ChatMessage).Text) +} diff --git a/pkg/atproto/label_verification.go b/pkg/atproto/label_verification.go index 517ba911..b6e631ac 100644 --- a/pkg/atproto/label_verification.go +++ b/pkg/atproto/label_verification.go @@ -86,6 +86,9 @@ type queryLabelsOut struct { } func (atsync *ATProtoSynchronizer) seedLabels(ctx context.Context, labeler string, patterns []string) error { + if _, err := atsync.indexingDID(ctx, labeler); err != nil { + return err + } ident, err := atsync.resolveIdent(ctx, labeler, true) if err != nil { return err @@ -114,6 +117,9 @@ func (atsync *ATProtoSynchronizer) seedLabels(ctx context.Context, labeler strin } added, removed := 0, 0 for { + if _, err := atsync.indexingDID(ctx, labeler); err != nil { + return err + } params := map[string]any{ // Account labels only: DIDs, not records. "uriPatterns": []string{"did:*"}, diff --git a/pkg/atproto/labeler_firehose.go b/pkg/atproto/labeler_firehose.go index dfcc4c05..aa0cbc73 100644 --- a/pkg/atproto/labeler_firehose.go +++ b/pkg/atproto/labeler_firehose.go @@ -24,6 +24,9 @@ import ( ) func (atsync *ATProtoSynchronizer) StartLabelerFirehose(ctx context.Context, did string) error { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } // The series exists at 0 from here, so a labeler that never connects // shows as 0 rather than as nothing. spmetrics.LabelerFirehosesConnected.WithLabelValues(did) @@ -34,6 +37,9 @@ func (atsync *ATProtoSynchronizer) StartLabelerFirehose(ctx context.Context, did if ctx.Err() != nil { return nil } + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } err := atsync.StartLabelerFirehoseRetry(ctx, did) if err != nil { log.Error(ctx, "firehose error", "err", err) @@ -59,6 +65,9 @@ func (atsync *ATProtoSynchronizer) StartLabelerFirehose(ctx context.Context, did } func (atsync *ATProtoSynchronizer) StartLabelerFirehoseRetry(ctx context.Context, did string) error { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } ctx = log.WithLogValues(ctx, "func", "StartLabelerFirehose") ident, err := atsync.resolveIdent(ctx, did, true) @@ -118,6 +127,9 @@ func (atsync *ATProtoSynchronizer) StartLabelerFirehoseRetry(ctx context.Context defer spmetrics.LabelerFirehosesConnected.WithLabelValues(did).Dec() rsc := &events.RepoStreamCallbacks{ LabelLabels: func(evt *indigoatproto.LabelSubscribeLabels_Labels) error { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } err = atsync.Model.UpdateLabelerCursor(did, evt.Seq) if err != nil { log.Error(ctx, "failed to update labeler cursor", "err", err) diff --git a/pkg/atproto/sweep.go b/pkg/atproto/sweep.go index 117a5854..2987617c 100644 --- a/pkg/atproto/sweep.go +++ b/pkg/atproto/sweep.go @@ -134,6 +134,9 @@ func (atsync *ATProtoSynchronizer) feedUnresolved(ctx context.Context, items []s g.SetLimit(identityResolveConcurrency) for _, item := range items { g.Go(func() error { + if !atsync.indexEventAllowed(gctx, item.DID) { + return nil + } // On cancellation, still hand the item over (with a lane of its // own): the scheduler's workers notice the dead context themselves, // and every item accounted for exactly once is the simpler @@ -696,6 +699,22 @@ func (atsync *ATProtoSynchronizer) sweepCandidates(ctx context.Context) ([]strin for _, did := range terminalDIDs { terminal[did] = struct{}{} } + if atsync.upstreamMode() { + if atsync.StatefulDB == nil { + return nil, fmt.Errorf("cannot list local OAuth users without the state database") + } + dids, err := atsync.StatefulDB.IndexingDIDs(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list local OAuth users: %w", err) + } + eligible := dids[:0] + for _, did := range dids { + if _, parked := terminal[did]; !parked { + eligible = append(eligible, did) + } + } + return eligible, nil + } var allDIDs []string skipped := 0 diff --git a/pkg/atproto/sync.go b/pkg/atproto/sync.go index da1c4033..e3ee31a8 100644 --- a/pkg/atproto/sync.go +++ b/pkg/atproto/sync.go @@ -45,6 +45,9 @@ type moderationPermissionViewEvent struct { // (bus fanout, notification tasks) has to be skipped when nothing changed. The // model layer signals that with [model.ErrAlreadyIndexed]; see pkg/model/indexed.go. func (atsync *ATProtoSynchronizer) handleCreateUpdate(ctx context.Context, userDID string, rkey syntax.RecordKey, recCBOR *[]byte, cid string, collection syntax.NSID, isUpdate bool, isFirstSync bool, repoRev string) error { + if _, err := atsync.indexingDID(ctx, userDID); err != nil { + return err + } ctx = log.WithLogValues(ctx, "func", "handleCreateUpdate", "userDID", userDID, "rkey", rkey.String(), "cid", cid, "collection", collection.String()) now := time.Now() r, err := atsync.Model.GetRepo(userDID) diff --git a/pkg/atproto/upstream_firehose.go b/pkg/atproto/upstream_firehose.go new file mode 100644 index 00000000..5773f3de --- /dev/null +++ b/pkg/atproto/upstream_firehose.go @@ -0,0 +1,72 @@ +package atproto + +import ( + "context" + "fmt" + "sync" + "time" + + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/spmetrics" +) + +const OAuthSessionTopic = "atproto:oauth-sessions" + +// The set of users comes from persisted successful local logins. The map here +// owns only live subscriptions, never authorization or an independent allowlist. +func (atsync *ATProtoSynchronizer) startUpstreamFirehose(ctx context.Context) error { + if atsync.UpstreamStation == nil { + return fmt.Errorf("upstream station has not been resolved") + } + relay := atsync.UpstreamStation.FirehoseURL + spmetrics.TouchRelay(relay, "moq") + changes := atsync.Bus.Subscribe(OAuthSessionTopic) + defer atsync.Bus.Unsubscribe(OAuthSessionTopic, changes) + // Login and revocation wake immediately; the timer also observes session + // expiry and changes persisted by another process sharing the state DB. + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + active := make(map[string]context.CancelFunc) + var workers sync.WaitGroup + defer func() { + for _, cancel := range active { + cancel() + } + workers.Wait() + }() + for { + dids, err := atsync.StatefulDB.IndexingDIDs(ctx) + if err != nil { + return fmt.Errorf("load upstream subscriptions: %w", err) + } + wanted := make(map[string]struct{}, len(dids)) + for _, did := range dids { + wanted[did] = struct{}{} + } + for did, cancel := range active { + if _, ok := wanted[did]; !ok { + cancel() + delete(active, did) + } + } + for _, did := range dids { + if _, ok := active[did]; ok { + continue + } + userCtx, cancel := context.WithCancel(ctx) + active[did] = cancel + workers.Add(1) + go func() { + defer workers.Done() + atsync.consumeRelay(userCtx, relay, did) + }() + log.Log(ctx, "subscribing to upstream user", "did", did, "relay", relay) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-changes: + case <-ticker.C: + } + } +} diff --git a/pkg/atproto/upstream_station.go b/pkg/atproto/upstream_station.go new file mode 100644 index 00000000..bb055235 --- /dev/null +++ b/pkg/atproto/upstream_station.go @@ -0,0 +1,44 @@ +package atproto + +import ( + "context" + "fmt" + "net/url" + + "github.com/bluesky-social/indigo/atproto/syntax" + "stream.place/streamplace/pkg/config" +) + +const AtmoqServiceID = "atmoq" + +// UpstreamStation is resolved once before any relay or OAuth session is opened. +type UpstreamStation struct { + DID string + URL string + FirehoseURL string +} + +func ResolveUpstreamStation(ctx context.Context, did string) (*UpstreamStation, error) { + parsed, err := syntax.ParseDID(did) + if err != nil { + return nil, fmt.Errorf("parse upstream station DID: %w", err) + } + ident, err := CustomDirectory("").LookupDID(ctx, parsed) + if err != nil { + return nil, fmt.Errorf("resolve upstream station %s: %w", did, err) + } + stationURL := ident.GetServiceEndpoint(StreamplaceServiceID) + u, err := url.Parse(stationURL) + if err != nil || u.Scheme != "https" || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return nil, fmt.Errorf("upstream station %s must advertise an HTTPS origin in #%s", did, StreamplaceServiceID) + } + firehoseURL := ident.GetServiceEndpoint(AtmoqServiceID) + if err := config.ValidateMoQOrigin(firehoseURL); err != nil { + return nil, fmt.Errorf("upstream station %s service #%s: %w", did, AtmoqServiceID, err) + } + return &UpstreamStation{ + DID: did, + URL: u.Scheme + "://" + u.Host, + FirehoseURL: firehoseURL, + }, nil +} diff --git a/pkg/atproto/upstream_station_test.go b/pkg/atproto/upstream_station_test.go new file mode 100644 index 00000000..9384f3d8 --- /dev/null +++ b/pkg/atproto/upstream_station_test.go @@ -0,0 +1,76 @@ +package atproto + +import ( + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/require" + "stream.place/streamplace/pkg/aqhttp" + "stream.place/streamplace/pkg/config" +) + +func TestResolveUpstreamStation(t *testing.T) { + var stationURL, firehoseURL string + var advertiseRelay = true + station := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/.well-known/did.json" { + t.Errorf("unexpected discovery request: %s", r.URL.Path) + http.NotFound(w, r) + return + } + relay := firehoseURL + if !advertiseRelay { + relay = "" + } + doc := DIDDoc("station.streamplace.team", "zQ3shRvJrnKp4SWvoxkFyZqijahHcmvJLdViKzkwQKj7cWxVm", relay) + if err := json.NewEncoder(w).Encode(doc); err != nil { + t.Error(err) + } + })) + defer station.Close() + stationURL = "https://station.streamplace.team" + firehoseURL = "moqt://127.0.0.1:4443" + original := aqhttp.Client + client := station.Client() + transport := client.Transport.(*http.Transport).Clone() + target, err := url.Parse(station.URL) + require.NoError(t, err) + transport.TLSClientConfig.ServerName = target.Hostname() + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, target.Host) + } + client.Transport = transport + aqhttp.Client = *client + t.Cleanup(func() { aqhttp.Client = original }) + did := "did:web:station.streamplace.team" + + resolved, err := ResolveUpstreamStation(context.Background(), did) + require.NoError(t, err) + require.Equal(t, stationURL, resolved.URL) + require.Equal(t, firehoseURL, resolved.FirehoseURL) + + // A missing/bad advertisement must not quietly select the global firehose. + advertiseRelay = false + _, err = ResolveUpstreamStation(context.Background(), did) + require.Error(t, err) + advertiseRelay = true + firehoseURL = "wss://bsky.network" + _, err = ResolveUpstreamStation(context.Background(), did) + require.Error(t, err) +} + +func TestUpstreamRelayNeverFallsBack(t *testing.T) { + atsync := &ATProtoSynchronizer{CLI: &config.CLI{ + UpstreamStation: "did:web:station.example", + RelayHost: "wss://bsky.network", + HTTPAddr: "127.0.0.1:38080", + }} + require.Empty(t, atsync.relayHosts()) + atsync.UpstreamStation = &UpstreamStation{FirehoseURL: "moqt://station.example"} + require.Equal(t, []string{"moqt://station.example"}, atsync.relayHosts()) +} diff --git a/pkg/atproto/verification.go b/pkg/atproto/verification.go index 36d42604..f4998b34 100644 --- a/pkg/atproto/verification.go +++ b/pkg/atproto/verification.go @@ -340,6 +340,9 @@ type listRecordsOut struct { } func (atsync *ATProtoSynchronizer) seedVerifier(ctx context.Context, did string) error { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } ident, err := atsync.resolveIdent(ctx, did, true) if err != nil { return err @@ -351,6 +354,9 @@ func (atsync *ATProtoSynchronizer) seedVerifier(ctx context.Context, did string) cursor := "" total := 0 for { + if _, err := atsync.indexingDID(ctx, did); err != nil { + return err + } params := map[string]any{ "repo": did, "collection": constants.APP_BSKY_GRAPH_VERIFICATION, diff --git a/pkg/cmd/e2e.go b/pkg/cmd/e2e.go index dd6693cc..87cd3f59 100644 --- a/pkg/cmd/e2e.go +++ b/pkg/cmd/e2e.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "net" - "net/http" "net/url" "os" "os/exec" @@ -24,6 +23,7 @@ import ( glex "github.com/streamplace/glex/runtime" urfavecli "github.com/urfave/cli/v3" "golang.org/x/sync/errgroup" + "stream.place/streamplace/pkg/appbsky" "stream.place/streamplace/pkg/aqhttp" "stream.place/streamplace/pkg/atproto" spcomatproto "stream.place/streamplace/pkg/comatproto" @@ -78,6 +78,17 @@ func makeE2eCommand(build *config.BuildFlags) *urfavecli.Command { Value: "js/dev-env/run.mjs", Sources: urfavecli.EnvVars("SP_DEV_ENV_MJS"), }, + &urfavecli.BoolFlag{ + Name: "upstream-station", + Usage: "start an HTTPS upstream station, real atmoq bridge and loopback local node (requires HTTPS hostnames and atmoq on PATH)", + Sources: urfavecli.EnvVars("SP_E2E_UPSTREAM_STATION"), + }, + &urfavecli.StringFlag{ + Name: "atmoq-bin", + Usage: "path to the released atmoq binary used in upstream mode", + Value: "atmoq", + Sources: urfavecli.EnvVars("SP_E2E_ATMOQ_BIN"), + }, // Together these turn on locally trusted HTTPS, so atproto OAuth // works end to end; see e2e_https.go. &urfavecli.StringFlag{ @@ -110,7 +121,10 @@ func makeE2eCommand(build *config.BuildFlags) *urfavecli.Command { if (pdsHost == "") != (stationHost == "") { return errors.New("--https-pds-hostname and --https-station-hostname go together") } - return runE2E(ctx, cmd.String("dev-env"), pdsHost, stationHost, int(cmd.Int("https-port")), cmd.String("app-bundle-id")) + if cmd.Bool("upstream-station") && pdsHost == "" { + return errors.New("--upstream-station requires the HTTPS PDS and station hostnames") + } + return runE2E(ctx, cmd.String("dev-env"), pdsHost, stationHost, int(cmd.Int("https-port")), cmd.String("app-bundle-id"), cmd.Bool("upstream-station"), cmd.String("atmoq-bin")) }, } } @@ -150,14 +164,7 @@ func freeUDPPort() (int, error) { // PLC, with extra on top of the configuration every harness node shares. func startNode(ctx context.Context, self string, env e2eDevEnv, dataDir string, extra []string) (*exec.Cmd, error) { nodeCmd := exec.CommandContext(ctx, self) - // Inherit the parent environment (dev builds need LD_LIBRARY_PATH etc.) - // but strip any SP_ vars so the node only gets our config. - for _, kv := range os.Environ() { - if !strings.HasPrefix(kv, "SP_") { - nodeCmd.Env = append(nodeCmd.Env, kv) - } - } - nodeCmd.Env = append(nodeCmd.Env, + nodeCmd.Env = append(e2eNodeEnv(), fmt.Sprintf("SP_RELAY_HOST=%s", strings.ReplaceAll(env.PDSURL, "http://", "ws://")), fmt.Sprintf("SP_PLC_URL=%s", env.PLCURL), fmt.Sprintf("SP_DATA_DIR=%s", dataDir), @@ -175,43 +182,7 @@ func startNode(ctx context.Context, self string, env e2eDevEnv, dataDir string, return nodeCmd, nil } -// waitForNode waits for the node at httpAddr to answer /api/healthz. Bounded, -// so a node that never comes up reports why instead of hanging the harness -// (and `make provision`) with no output forever. -func waitForNode(ctx context.Context, httpAddr string) error { - healthURL := fmt.Sprintf("http://%s/api/healthz", httpAddr) - httpClient := &http.Client{Timeout: 2 * time.Second} - deadline := time.Now().Add(nodeReadyTimeout) - var lastErr error - for { - if err := ctx.Err(); err != nil { - return err - } - resp, err := httpClient.Get(healthURL) - switch { - case err != nil: - lastErr = err - case resp.StatusCode == http.StatusOK: - // Close every response, not just the happy one: a body left open - // holds its connection out of the pool, so a node answering - // non-200 exhausts them. - resp.Body.Close() - return nil - default: - lastErr = fmt.Errorf("GET %s: %s", healthURL, resp.Status) - resp.Body.Close() - } - if time.Now().After(deadline) { - if lastErr == nil { - lastErr = errors.New("no response") - } - return fmt.Errorf("node was not ready after %s: %w", nodeReadyTimeout, lastErr) - } - time.Sleep(200 * time.Millisecond) - } -} - -func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost string, httpsPort int, appBundleID string) error { +func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost string, httpsPort int, appBundleID string, upstream bool, atmoqBin string) error { // Ctrl-C / SIGTERM must unwind through the normal path, or none of the // teardown below runs: Go's default handling exits immediately, which left // the dev-env node, the forked node and the temp data dir behind. @@ -242,10 +213,11 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri if err != nil { return fmt.Errorf("dev-env stdout pipe: %w", err) } + setNodeProcessGroup(devEnvCmd) if err := devEnvCmd.Start(); err != nil { return fmt.Errorf("start dev-env: %w", err) } - defer devEnvCmd.Process.Kill() //nolint:errcheck + defer stopE2EProcess(ctx, devEnvCmd, "") // The PDS logs JSON to stdout too when LOG_ENABLED is set, so skip lines // until the one that carries the URLs. @@ -337,6 +309,16 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri tlsEnv.Serve(ctx, pdsURL.Host, plcURL.Host, httpAddr) } + var bridgeURL string + if upstream { + bridge, relayURL, err := startE2EBridge(ctx, tlsEnv, env.PDSURL, atmoqBin) + if err != nil { + return err + } + defer stopE2EProcess(ctx, bridge, "") + bridgeURL = relayURL + } + // Fork ourselves as a Streamplace server node. self, err := os.Executable() if err != nil { @@ -399,6 +381,9 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri if tlsEnv != nil { nodeEnv = append(nodeEnv, tlsEnv.NodeEnv()...) } + if upstream { + nodeEnv = append(nodeEnv, "SP_ATMOQ_RELAY="+bridgeURL) + } if appBundleID != "" { nodeEnv = append(nodeEnv, "SP_APP_BUNDLE_ID="+appBundleID) } @@ -408,9 +393,9 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri } // Kill the node *and* the ingest workers it detaches; the plain // Process.Kill that used to be here left those behind. - defer killNode(ctx, nodeCmd, out.Did) - if err := waitForNode(ctx, httpAddr); err != nil { - return err + defer stopE2EProcess(ctx, nodeCmd, out.Did) + if err := waitE2EHealth(ctx, fmt.Sprintf("http://%s/api/healthz", httpAddr)); err != nil { + return fmt.Errorf("station node: %w", err) } // A second node, a peer that syndicates whatever the first ingests: it @@ -457,8 +442,8 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri if err != nil { return fmt.Errorf("peer: %w", err) } - defer killNode(ctx, peerCmd, out.Did) - if err := waitForNode(ctx, peerAddr); err != nil { + defer stopE2EProcess(ctx, peerCmd, out.Did) + if err := waitE2EHealth(ctx, fmt.Sprintf("http://%s/api/healthz", peerAddr)); err != nil { return fmt.Errorf("peer: %w", err) } @@ -484,11 +469,34 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri if err != nil { return fmt.Errorf("push node: %w", err) } - defer killNode(ctx, pushCmd, out.Did) - if err := waitForNode(ctx, pushAddr); err != nil { + defer stopE2EProcess(ctx, pushCmd, out.Did) + if err := waitE2EHealth(ctx, fmt.Sprintf("http://%s/api/healthz", pushAddr)); err != nil { return fmt.Errorf("push node: %w", err) } + // Only the local node's own completed OAuth sessions authorize its indexing; + // do not seed a local session here. Its ingest loop retries until login. + var local *e2eLocalNode + var unrelated *e2eUnrelatedAccount + if upstream { + local, err = startE2ELocalNode(ctx, self, tlsEnv, env, out.Did, appBundleID) + if err != nil { + return err + } + defer local.Close(ctx, out.Did) + unrelated, err = createE2EUnrelatedAccount(ctx, env.PDSURL, handleDomain) + if err != nil { + return err + } + if _, err := createRecord(ctx, xrpcc, "app.bsky.graph.follow", out.Did, &appbsky.GraphFollow{ + LexiconTypeID: "app.bsky.graph.follow", + CreatedAt: time.Now().UTC().Format(util.ISO8601), + Subject: unrelated.did, + }); err != nil { + return fmt.Errorf("seed follow to unrelated account: %w", err) + } + } + // Register a stream key for the test account. priv, pub, err := spkey.GenerateStreamKeyForDID(out.Did) if err != nil { @@ -552,6 +560,10 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri // Stream a test fixture in a loop so it outlasts any Maestro test run. fixture := remote.RemoteFixture("3188c071b354f2e548d7f2d332699758e8e3ab1600280e5b07cb67eedc64f274/BigBuckBunny_1sGOP_240p30_NoBframes.mp4") + ingestAddr := httpAddr + if local != nil { + ingestAddr = local.addr + } g, streamCtx := errgroup.WithContext(ctx) g.Go(func() error { for { @@ -563,7 +575,7 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri whip := &WHIPClient{ StreamKey: priv, File: fixture, - Endpoint: fmt.Sprintf("http://%s", httpAddr), + Endpoint: fmt.Sprintf("http://%s", ingestAddr), Count: 1, } if err := whip.WHIP(streamCtx); err != nil && streamCtx.Err() == nil { @@ -582,16 +594,26 @@ func runE2E(ctx context.Context, devEnvPath, httpsPDSHost, httpsStationHost stri // poll for SERVER_URL and then read the whole file. E2E_FIXTURE_MP4 is // the local file the ingest loop is streaming, which tests upload through // the app to get themselves a playable VOD. + serverAddr := httpAddr + if local != nil { + serverAddr = local.addr + } vars := fmt.Sprintf("SERVER_URL=http://%s\nSERVER2_URL=http://%s\nSERVER3_URL=http://%s\nACCOUNT_HANDLE=%s\nACCOUNT_DID=%s\nACCOUNT_PASSWORD=%s\nVIDEO_URI=%s\nE2E_FIXTURE_MP4=%s\n", - httpAddr, peerAddr, pushAddr, out.Handle, out.Did, password, videoURI, fixture) + serverAddr, peerAddr, pushAddr, out.Handle, out.Did, password, videoURI, fixture) if tlsEnv != nil { - // The same node over HTTPS at its public name, plus what clients - // need to reach and trust it (see e2e_https.go): a browser pins the - // leaf by SPKI, a device installs the CA. - vars += fmt.Sprintf("SERVER_HTTPS_URL=%s\nPDS_HTTPS_URL=%s\nE2E_PROXY_URL=%s\nE2E_TLS_SPKI=%s\nE2E_TLS_CA=%s\n", - tlsEnv.StationURL(), tlsEnv.PDSURL(), tlsEnv.ProxyURL(), tlsEnv.spki, tlsEnv.caPath) + vars += fmt.Sprintf("PDS_HTTPS_URL=%s\nE2E_PROXY_URL=%s\nE2E_TLS_SPKI=%s\nE2E_TLS_CA=%s\n", + tlsEnv.PDSURL(), tlsEnv.ProxyURL(), tlsEnv.spki, tlsEnv.caPath) + if local != nil { + vars += fmt.Sprintf("E2E_UPSTREAM_STATION=true\nUPSTREAM_STATION_URL=%s\nATMOQ_RELAY_URL=%s\nE2E_ATMOQ_LOG=%s\nE2E_LOCAL_DATA_DIR=%s\nE2E_STATION_DATA_DIR=%s\nE2E_LOCAL_PDS_AUDIT=%s\nUNRELATED_ACCOUNT_DID=%s\nUNRELATED_ACCOUNT_HANDLE=%s\nUNRELATED_ACCESS_JWT=%s\nE2E_PDS_INTERNAL_URL=%s\n", + tlsEnv.StationURL(), bridgeURL, filepath.Join(tlsEnv.dir, "atmoq.log"), local.dataDir, dataDir, tlsEnv.auditPath, unrelated.did, unrelated.handle, unrelated.accessJWT, env.PDSURL) + } else { + vars += fmt.Sprintf("SERVER_HTTPS_URL=%s\n", tlsEnv.StationURL()) + } } fmt.Print(vars) + if local != nil { + go local.Control(ctx, out.Did) + } <-ctx.Done() return g.Wait() diff --git a/pkg/cmd/e2e_https.go b/pkg/cmd/e2e_https.go index b0eb8f2a..9853a9de 100644 --- a/pkg/cmd/e2e_https.go +++ b/pkg/cmd/e2e_https.go @@ -11,6 +11,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/base64" + "encoding/json" "encoding/pem" "errors" "fmt" @@ -18,6 +19,8 @@ import ( "math/big" "net" "net/http" + "net/http/httputil" + "net/url" "os" "path/filepath" "runtime" @@ -64,11 +67,16 @@ type e2eHTTPS struct { dir string caPath string trustDir string + leafPath string + keyPath string + auditPath string cert tls.Certificate spki string - frontLn net.Listener // 127.0.0.1:443, or whatever loopback 443 redirects to - proxyLn net.Listener // CONNECT proxy + frontLn net.Listener // 127.0.0.1:443, or whatever loopback 443 redirects to + proxyLn net.Listener // CONNECT proxy + localProxyLn net.Listener + localPDSLn net.Listener } const e2ePLCHost = "plc.directory" @@ -108,7 +116,7 @@ func newE2EHTTPS(pdsHost, stationHost string, port int) (*e2eHTTPS, error) { } func (h *e2eHTTPS) Close() { - for _, ln := range []net.Listener{h.frontLn, h.proxyLn} { + for _, ln := range []net.Listener{h.frontLn, h.proxyLn, h.localProxyLn, h.localPDSLn} { if ln != nil { ln.Close() //nolint:errcheck } @@ -209,6 +217,7 @@ func (h *e2eHTTPS) mintCerts() error { SerialNumber: randomSerial(), Subject: pkix.Name{CommonName: h.pdsHost}, DNSNames: []string{h.pdsHost, "*." + h.pdsHost, h.stationHost, e2ePLCHost}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(7 * 24 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, @@ -354,3 +363,93 @@ func pipeConns(a, b net.Conn) { io.Copy(b, a) //nolint:errcheck once.Do(closeBoth) } + +// BridgeTLS writes the leaf and key only for a real atmoq serve child. Its +// station DNS SAN permits verified TLS with the SNI required by raw MoQ. +func (h *e2eHTTPS) BridgeTLS() error { + // atmoq watches the certificate directory. Keep logs and its cursor outside + // it so their writes do not trigger a certificate-reload feedback loop. + dir := filepath.Join(h.dir, "atmoq-tls") + if err := os.Mkdir(dir, 0o700); err != nil { + return err + } + h.leafPath = filepath.Join(dir, "cert.pem") + h.keyPath = filepath.Join(dir, "key.pem") + if err := os.WriteFile(h.leafPath, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: h.cert.Certificate[0]}), 0o600); err != nil { + return err + } + keyDER, err := x509.MarshalPKCS8PrivateKey(h.cert.PrivateKey) + if err != nil { + return err + } + return os.WriteFile(h.keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600) +} + +// StartLocalProxy forwards genuine local-node PDS requests and records their +// method/path/query. The upstream station uses the original proxy, so its +// ordinary global indexing cannot mask forbidden local account scraping. +// This is harness-only transport observation, not a production test endpoint. +func (h *e2eHTTPS) StartLocalProxy(ctx context.Context, pdsURL string) (string, error) { + target, err := url.Parse(pdsURL) + if err != nil { + return "", err + } + h.auditPath = filepath.Join(h.dir, "local-pds-requests.jsonl") + audit, err := os.OpenFile(h.auditPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return "", err + } + h.localPDSLn, err = net.Listen("tcp", "127.0.0.1:0") + if err != nil { + audit.Close() //nolint:errcheck + return "", err + } + h.localProxyLn, err = net.Listen("tcp", "127.0.0.1:0") + if err != nil { + audit.Close() //nolint:errcheck + return "", err + } + reverse := httputil.NewSingleHostReverseProxy(target) + var mu sync.Mutex + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entry := struct { + Method string `json:"method"` + Path string `json:"path"` + Query string `json:"query"` + }{r.Method, r.URL.Path, r.URL.RawQuery} + mu.Lock() + err := json.NewEncoder(audit).Encode(entry) + mu.Unlock() + if err != nil { + log.Error(ctx, "write local PDS audit", "err", err) + http.Error(w, "harness audit failed", http.StatusInternalServerError) + return + } + reverse.ServeHTTP(w, r) + }) + srv := &http.Server{Handler: handler, ReadHeaderTimeout: 10 * time.Second} + go func() { + err := srv.Serve(tls.NewListener(h.localPDSLn, &tls.Config{Certificates: []tls.Certificate{h.cert}, MinVersion: tls.VersionTLS12})) + if err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, net.ErrClosed) && ctx.Err() == nil { + log.Error(ctx, "local PDS observer failed", "err", err) + } + }() + go func() { + <-ctx.Done() + srv.Close() //nolint:errcheck + audit.Close() //nolint:errcheck + }() + go serveConnectProxy(ctx, h.localProxyLn, func(hostport string) string { + host, port, err := net.SplitHostPort(hostport) + if err == nil && port == "443" { + switch { + case host == h.pdsHost || strings.HasSuffix(host, "."+h.pdsHost): + return h.localPDSLn.Addr().String() + case host == h.stationHost || host == e2ePLCHost: + return h.frontLn.Addr().String() + } + } + return hostport + }) + return "http://" + h.localProxyLn.Addr().String(), nil +} diff --git a/pkg/cmd/e2e_upstream.go b/pkg/cmd/e2e_upstream.go new file mode 100644 index 00000000..9ee9480a --- /dev/null +++ b/pkg/cmd/e2e_upstream.go @@ -0,0 +1,289 @@ +package cmd + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" + "github.com/google/uuid" + atmoq "github.com/streamplace/atmoq/go" + "stream.place/streamplace/pkg/aqhttp" + "stream.place/streamplace/pkg/log" + "stream.place/streamplace/pkg/placestream" +) + +// Every child gets a process group, including the PDS and the real MoQ bridge. +// Reap it after teardown; exec.CommandContext alone only kills the direct child. +func stopE2EProcess(ctx context.Context, cmd *exec.Cmd, did string) { + killNode(context.WithoutCancel(ctx), cmd, did) + _ = cmd.Wait() //nolint:errcheck // killed during normal harness teardown +} + +func e2eNodeEnv() []string { + var env []string + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "SP_") { + env = append(env, kv) + } + } + return env +} + +func waitE2EHealth(ctx context.Context, healthURL string) error { + client := &http.Client{Timeout: 2 * time.Second} + deadline := time.Now().Add(nodeReadyTimeout) + var lastErr error + for { + if err := ctx.Err(); err != nil { + return err + } + resp, err := client.Get(healthURL) + if err != nil { + lastErr = err + } else { + resp.Body.Close() + if resp.StatusCode == http.StatusOK { + return nil + } + lastErr = fmt.Errorf("GET %s: %s", healthURL, resp.Status) + } + if time.Now().After(deadline) { + return fmt.Errorf("service was not ready after %s: %w", nodeReadyTimeout, lastErr) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } +} + +func startE2EBridge(ctx context.Context, tlsEnv *e2eHTTPS, pdsURL, binary string) (*exec.Cmd, string, error) { + // QUIC binds UDP, not TCP. Reserve an unused UDP port before handing it to + // the released bridge, rather than guessing from the node's TCP ports. + if err := tlsEnv.BridgeTLS(); err != nil { + return nil, "", fmt.Errorf("write bridge TLS: %w", err) + } + ln, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + return nil, "", err + } + addr := ln.LocalAddr().String() + ln.Close() //nolint:errcheck + wsURL := strings.Replace(pdsURL, "http://", "ws://", 1) + cmd := exec.CommandContext(ctx, binary, "serve", + "--relay-host", wsURL, + "--server-bind", addr, + "--tls-cert", tlsEnv.leafPath, + "--tls-key", tlsEnv.keyPath, + "--web-bind", "", "--web-tls-bind", "", + "--group-store-dir", filepath.Join(tlsEnv.dir, "atmoq-store"), + "--cursor-file", filepath.Join(tlsEnv.dir, "atmoq-cursor"), + "--group-size", "1") + bridgeLog, err := os.OpenFile(filepath.Join(tlsEnv.dir, "atmoq.log"), os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return nil, "", err + } + go func() { + <-ctx.Done() + bridgeLog.Close() //nolint:errcheck + }() + cmd.Env = append(os.Environ(), "RUST_LOG=info") + cmd.Stdout = io.MultiWriter(os.Stderr, bridgeLog) + cmd.Stderr = cmd.Stdout + setNodeProcessGroup(cmd) + if err := cmd.Start(); err != nil { + return nil, "", fmt.Errorf("start released atmoq bridge (%s): %w", binary, err) + } + // Raw MoQ on the released Rust server requires SNI; Go correctly omits + // SNI for an IP literal. Dial the verified fixture DNS name instead. + _, port, _ := net.SplitHostPort(addr) + relayURL := "moqt://" + net.JoinHostPort(tlsEnv.stationHost, port) + caPEM, err := os.ReadFile(tlsEnv.caPath) + if err != nil { + stopE2EProcess(ctx, cmd, "") + return nil, "", err + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(caPEM) { + stopE2EProcess(ctx, cmd, "") + return nil, "", fmt.Errorf("parse bridge CA") + } + deadline := time.Now().Add(nodeReadyTimeout) + for { + dialCtx, cancel := context.WithTimeout(ctx, time.Second) + session, err := atmoq.Dial(dialCtx, relayURL, &atmoq.Options{TLSConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS13}}) + cancel() + if err == nil { + session.Close() + return cmd, relayURL, nil + } + if ctx.Err() != nil || time.Now().After(deadline) { + stopE2EProcess(ctx, cmd, "") + return nil, "", fmt.Errorf("dial real atmoq bridge: %w", err) + } + } +} + +type e2eLocalNode struct { + mu sync.Mutex + cmd *exec.Cmd + addr string + dataDir string +} + +func (n *e2eLocalNode) Close(ctx context.Context, did string) { + n.mu.Lock() + defer n.mu.Unlock() + stopE2EProcess(ctx, n.cmd, did) + os.RemoveAll(n.dataDir) //nolint:errcheck +} + +// Control reads the harness's own stdin, never the node's HTTP listener. +// Restart retains ports, data and genuine OAuth credentials, then acknowledges +// readiness in a run-private file for the browser lifecycle scenario. +func (n *e2eLocalNode) Control(ctx context.Context, did string) { + scanner := bufio.NewScanner(os.Stdin) + generation := 0 + for scanner.Scan() { + if scanner.Text() != "restart-local" { + continue + } + n.mu.Lock() + old := n.cmd + stopE2EProcess(ctx, old, did) + n.cmd = exec.CommandContext(ctx, old.Path, old.Args[1:]...) + n.cmd.Env = old.Env + n.cmd.Stdout = old.Stdout + n.cmd.Stderr = old.Stderr + setNodeProcessGroup(n.cmd) + err := n.cmd.Start() + if err == nil { + err = waitE2EHealth(ctx, "http://"+n.addr+"/api/healthz") + } + n.mu.Unlock() + generation++ + result := struct { + Generation int `json:"generation"` + Error string `json:"error,omitempty"` + }{Generation: generation} + if err != nil { + result.Error = err.Error() + } + data, err := json.Marshal(result) + if err == nil { + tmp := filepath.Join(n.dataDir, "restart.json.tmp") + err = os.WriteFile(tmp, data, 0o600) + if err == nil { + err = os.Rename(tmp, filepath.Join(n.dataDir, "restart.json")) + } + } + if err != nil { + log.Error(ctx, "write harness restart acknowledgment", "err", err) + } + if ctx.Err() != nil { + return + } + } +} + +func startE2ELocalNode(ctx context.Context, self string, tlsEnv *e2eHTTPS, env e2eDevEnv, did, appBundleID string) (*e2eLocalNode, error) { + ports := make([]int, 3) + for i := range ports { + port, err := freePort() + if err != nil { + return nil, err + } + ports[i] = port + } + proxyURL, err := tlsEnv.StartLocalProxy(ctx, env.PDSURL) + if err != nil { + return nil, fmt.Errorf("start local PDS observer: %w", err) + } + dir, err := os.MkdirTemp("", "streamplace-e2e-local-*") + if err != nil { + return nil, err + } + n := &e2eLocalNode{addr: fmt.Sprintf("127.0.0.1:%d", ports[0]), dataDir: dir} + n.cmd = exec.CommandContext(ctx, self) + n.cmd.Env = append(e2eNodeEnv(), + "SP_HTTP_ADDR="+n.addr, + fmt.Sprintf("SP_HTTP_INTERNAL_ADDR=127.0.0.1:%d", ports[1]), + fmt.Sprintf("SP_RTMP_ADDR=127.0.0.1:%d", ports[2]), + "SP_DATA_DIR="+dir, + "SP_BROADCASTER_HOST="+n.addr, + "SP_WEBSOCKET_URL=ws://"+n.addr, + "SP_PLC_URL="+env.PLCURL, + "SP_UPSTREAM_STATION=did:web:"+tlsEnv.stationHost, + "SP_TRUST_PRIVATE_NETWORK=true", + "SP_ADMIN_DIDS="+did, + "SP_STREAM_SESSION_TIMEOUT=30s", + "SSL_CERT_DIR="+tlsEnv.trustDir, + "HTTPS_PROXY="+proxyURL) + if appBundleID != "" { + n.cmd.Env = append(n.cmd.Env, "SP_APP_BUNDLE_ID="+appBundleID) + } + n.cmd.Stdout = os.Stderr + n.cmd.Stderr = os.Stderr + setNodeProcessGroup(n.cmd) + if err := n.cmd.Start(); err != nil { + os.RemoveAll(dir) //nolint:errcheck + return nil, fmt.Errorf("start loopback upstream client node: %w", err) + } + if err := waitE2EHealth(ctx, "http://"+n.addr+"/api/healthz"); err != nil { + n.Close(ctx, did) + return nil, err + } + return n, nil +} + +type e2eUnrelatedAccount struct { + did string + handle string + accessJWT string +} + +func createE2EUnrelatedAccount(ctx context.Context, pdsURL, handleDomain string) (*e2eUnrelatedAccount, error) { + id, err := uuid.NewRandom() + if err != nil { + return nil, err + } + handle := "unrelated-" + id.String()[:8] + "." + handleDomain + password := "test" + email := handle + "@example.com" + client := &xrpc.Client{Host: pdsURL, Client: &aqhttp.TrustedClient} + account, err := comatproto.ServerCreateAccount(ctx, client, &comatproto.ServerCreateAccount_Input{Handle: handle, Email: &email, Password: &password}) + if err != nil { + return nil, fmt.Errorf("create unrelated account: %w", err) + } + session, err := comatproto.ServerCreateSession(ctx, client, &comatproto.ServerCreateSession_Input{Identifier: handle, Password: password}) + if err != nil { + return nil, fmt.Errorf("create unrelated session: %w", err) + } + client.Auth = &xrpc.AuthInfo{Did: account.Did, AccessJwt: session.AccessJwt} + // This produces a real unrelated firehose commit before local OAuth login. + // The browser spec emits another after login using this account's PDS JWT. + if _, err := createRecord(ctx, client, "place.stream.livestream", account.Did, &placestream.Livestream{ + LexiconTypeID: "place.stream.livestream", + CreatedAt: time.Now().UTC().Format(util.ISO8601), + Title: "unrelated e2e stream must not be indexed locally", + }); err != nil { + return nil, fmt.Errorf("seed unrelated record: %w", err) + } + return &e2eUnrelatedAccount{did: account.Did, handle: handle, accessJWT: session.AccessJwt}, nil +} diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 7d86ee72..3d69fb4f 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -279,12 +279,21 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu cli.ServiceAuthKey = serviceAuthKey b := bus.NewBus() + var upstreamStation *atproto.UpstreamStation + if cli.UpstreamStation != "" { + upstreamStation, err = atproto.ResolveUpstreamStation(ctx, cli.UpstreamStation) + if err != nil { + return err + } + log.Log(ctx, "using upstream station", "did", upstreamStation.DID, "url", upstreamStation.URL, "firehose", upstreamStation.FirehoseURL) + } atsync := &atproto.ATProtoSynchronizer{ - CLI: cli, - Model: mod, - StatefulDB: state, - Noter: noter, - Bus: b, + CLI: cli, + Model: mod, + StatefulDB: state, + Noter: noter, + Bus: b, + UpstreamStation: upstreamStation, } mm, err := media.MakeMediaManager(ctx, cli, signer, mod, b, atsync, ldb) if err != nil { @@ -320,12 +329,17 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu var clientMetadata *oatproxy.OAuthClientMetadata var host string - if cli.PublicOAuth { + var oauthBaseURL, oauthUpstreamURL string + if upstreamStation != nil { + oauthUpstreamURL = upstreamStation.URL + } + if cli.PublicOAuth || upstreamStation != nil { u, err := url.Parse(cli.OwnPublicURL()) if err != nil { return err } host = u.Host + oauthBaseURL = cli.OwnPublicURL() clientMetadata = &oatproxy.OAuthClientMetadata{ Scope: atproto.OAuthString, ClientName: "Streamplace", @@ -349,14 +363,44 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu op := oatproxy.New(&oatproxy.Config{ Host: host, CreateOAuthSession: state.CreateOAuthSession, - UpdateOAuthSession: state.UpdateOAuthSession, - GetOAuthSession: state.LoadOAuthSession, - Lock: state.GetNamedLock, - Scope: atproto.OAuthString, - UpstreamJWK: cli.JWK, - DownstreamJWK: cli.AccessJWK, - ClientMetadata: clientMetadata, - Public: cli.PublicOAuth, + UpdateOAuthSession: func(id string, session *oatproxy.OAuthSession) error { + if upstreamStation == nil { + return state.UpdateOAuthSession(id, session) + } + previous, err := state.LoadOAuthSession(id) + if err != nil { + return err + } + if err := state.UpdateOAuthSession(id, session); err != nil { + return err + } + if previous != nil && ((previous.RevokedAt == nil && session.RevokedAt != nil) || + (previous.DownstreamRevokedAt == nil && session.DownstreamRevokedAt != nil)) { + b.Publish(atproto.OAuthSessionTopic, nil) + } + if session.Status() == oatproxy.OAuthSessionStateReady && + (previous == nil || previous.DownstreamAccessToken == "") { + b.Publish(atproto.OAuthSessionTopic, nil) + // Persistence opens the user-scoped indexing gate. Never sync while + // the OAuth library may hold this session's refresh lock. + did := session.DID + go func() { + if _, err := atsync.SyncBlueskyRepoCached(ctx, did); err != nil { + log.Warn(ctx, "failed to sync newly authenticated repo", "did", did, "err", err) + } + }() + } + return nil + }, + GetOAuthSession: state.LoadOAuthSession, + Lock: state.GetNamedLock, + Scope: atproto.OAuthString, + UpstreamJWK: cli.JWK, + DownstreamJWK: cli.AccessJWK, + ClientMetadata: clientMetadata, + Public: cli.PublicOAuth, + BaseURL: oauthBaseURL, + UpstreamURL: oauthUpstreamURL, }) state.OATProxy = op // The synchronizer uses stored sessions to act as a streamer (e.g. ending @@ -679,10 +723,12 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu } }() } - for _, labeler := range cli.Labelers { - group.Go(func() error { - return atsync.StartLabelerFirehose(ctx, labeler) - }) + if upstreamStation == nil { + for _, labeler := range cli.Labelers { + group.Go(func() error { + return atsync.StartLabelerFirehose(ctx, labeler) + }) + } } group.Go(func() error { diff --git a/pkg/config/config.go b/pkg/config/config.go index 059e5949..dcca0ada 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "net" + "net/url" "os" "path/filepath" "runtime" @@ -21,6 +22,7 @@ import ( "math/rand/v2" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/livepeer/go-livepeer/cmd/livepeer/starter" "github.com/lmittmann/tint" @@ -102,6 +104,8 @@ type CLI struct { PKCS11KeypairID string StreamerName string RelayHost string + UpstreamStation string + AtmoqRelay string Debug map[string]map[string]int AllowedStreams []string WideOpen bool @@ -681,6 +685,18 @@ func (cli *CLI) NewCommand(name string) *urfavecli.Command { Destination: &cli.RelayHost, Sources: urfavecli.EnvVars("SP_RELAY_HOST"), }, + &urfavecli.StringFlag{ + Name: "upstream-station", + Usage: "DID of an upstream Streamplace station; consume its MoQ firehose, delegate OAuth to it, and scrape only locally authenticated users", + Destination: &cli.UpstreamStation, + Sources: urfavecli.EnvVars("SP_UPSTREAM_STATION"), + }, + &urfavecli.StringFlag{ + Name: "atmoq-relay", + Usage: "MoQ firehose URL to advertise in this station's DID document for upstream-station clients", + Destination: &cli.AtmoqRelay, + Sources: urfavecli.EnvVars("SP_ATMOQ_RELAY"), + }, &urfavecli.StringFlag{ Name: "color", Usage: "'true' to enable colorized logging, 'false' to disable", @@ -1388,7 +1404,7 @@ func (cli *CLI) OwnPublicURL() string { host, port, _ := net.SplitHostPort(addr) ip := net.ParseIP(host) - if host == "" || ip.IsUnspecified() { + if host == "" || host == "localhost" || ip.IsUnspecified() { host = "127.0.0.1" } return fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(host, port)) @@ -1531,9 +1547,39 @@ func (cli *CLI) CheckConfig() error { if err := cli.validateLiveCDN(); err != nil { errs = append(errs, err) } + if cli.UpstreamStation != "" { + if _, err := syntax.ParseDID(cli.UpstreamStation); err != nil { + errs = append(errs, fmt.Errorf("--upstream-station: %w", err)) + } + } + if (cli.UpstreamStation != "" || cli.PublicOAuth) && !cli.Secure { + origin, err := url.Parse(cli.OwnPublicURL()) + if err != nil || !net.ParseIP(origin.Hostname()).IsLoopback() { + errs = append(errs, fmt.Errorf("local OAuth requires a loopback --http-addr or --secure")) + } + } + if cli.AtmoqRelay != "" { + if err := ValidateMoQOrigin(cli.AtmoqRelay); err != nil { + errs = append(errs, fmt.Errorf("--atmoq-relay: %w", err)) + } + } return errors.Join(errs...) } +// ValidateMoQOrigin rejects endpoints the raw-QUIC firehose consumer cannot use. +func ValidateMoQOrigin(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { + return fmt.Errorf("invalid MoQ firehose origin %q", raw) + } + switch u.Scheme { + case "moqt", "moql", "moq", "moqs": + return nil + default: + return fmt.Errorf("firehose %q must use a MoQ transport", raw) + } +} + // PrepareConfig applies the defaults and process-level setup that used to be // interleaved with the checks in Validate: data-dir substitution, SQL logging // verbosity, host fallbacks, the firebase service account file, and the diff --git a/pkg/config/config_check_test.go b/pkg/config/config_check_test.go index bcfa1f1c..55e350cd 100644 --- a/pkg/config/config_check_test.go +++ b/pkg/config/config_check_test.go @@ -39,6 +39,9 @@ func TestCheckConfig(t *testing.T) { bad(CLI{DataDir: t.TempDir(), DuplicateMistTest: true, RTMPServerAddon: "127.0.0.1:31935"}, "--secure") ok(CLI{DataDir: t.TempDir(), DuplicateMistTest: true, Secure: true, RTMPServerAddon: "127.0.0.1:31935"}) + bad(CLI{DataDir: t.TempDir(), UpstreamStation: "did:web:station.test", HTTPAddr: "192.0.2.1:38080"}, + "loopback --http-addr") + ok(CLI{DataDir: t.TempDir(), UpstreamStation: "did:web:station.test", HTTPAddr: "localhost:38080"}) bad(CLI{}, "data-dir") bad(CLI{DataDir: t.TempDir(), LivepeerGateway: true, LivepeerGatewayURL: "http://127.0.0.1:8935"}, "livepeer-gateway") @@ -73,6 +76,12 @@ func TestCheckConfig(t *testing.T) { require.Contains(t, err.Error(), "vod-cdn-url") } +func TestLocalOAuthCanonicalOrigin(t *testing.T) { + cli := CLI{HTTPAddr: "localhost:38080", UpstreamStation: "did:web:station.test"} + require.Equal(t, "http://127.0.0.1:38080", cli.OwnPublicURL(), + "localhost must produce the numeric loopback issuer accepted by public OAuth clients") +} + // TestCheckConfigSigningKey proves that a broken --signing-key file — a // classic restart-crashloop cause — is caught by the checks, not discovered // mid-boot by the app-updates path that reads it. diff --git a/pkg/spmetrics/spmetrics.go b/pkg/spmetrics/spmetrics.go index 5df6ccf2..d97472a5 100644 --- a/pkg/spmetrics/spmetrics.go +++ b/pkg/spmetrics/spmetrics.go @@ -162,13 +162,11 @@ var LabelerFirehosesConnected = promauto.NewGaugeVec(prometheus.GaugeOpts{ Help: "number of currently connected labeler firehoses", }, []string{"labeler"}) -// FirehoseRelaysConnected is 1 while a relay's firehose is connected and 0 -// while it is reconnecting, labeled by relay host and protocol ("websocket" or -// "moq"). With multi-relay support this shows at a glance how many of the -// configured relays are currently feeding us. +// FirehoseRelaysConnected counts active firehose connections per relay and +// protocol. Upstream mode opens one sparse MoQ connection per local user. var FirehoseRelaysConnected = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "streamplace_firehose_relays_connected", - Help: "1 if the relay's firehose is currently connected, else 0", + Help: "Number of active firehose connections to the relay", }, []string{"relay", "protocol"}) // FirehoseEventsReceivedTotal counts every event a relay delivered, before diff --git a/pkg/spxrpc/com_atproto_repo.go b/pkg/spxrpc/com_atproto_repo.go index 6a1c8b23..57fdca43 100644 --- a/pkg/spxrpc/com_atproto_repo.go +++ b/pkg/spxrpc/com_atproto_repo.go @@ -7,37 +7,37 @@ import ( "fmt" "io" "net/http" - "strings" glex "github.com/streamplace/glex/runtime" "stream.place/streamplace/pkg/comatproto" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/xrpc" "github.com/labstack/echo/v4" "github.com/streamplace/oatproxy/pkg/oatproxy" "go.opentelemetry.io/otel" - "stream.place/streamplace/pkg/aqhttp" "stream.place/streamplace/pkg/atproto" "stream.place/streamplace/pkg/log" ) -func resolveRepoService(ctx context.Context, repo string) (string, string, string, error) { - did := repo - var err error - if !strings.HasPrefix(repo, "did:") { - did, err = oatproxy.ResolveHandleWithClient(ctx, repo, &aqhttp.Client) - if err != nil { - return "", "", "", fmt.Errorf("failed to resolve handle %q: %w", repo, err) - } +func (s *Server) resolveRepoService(ctx context.Context, repo string) (string, string, string, error) { + id, err := syntax.ParseAtIdentifier(repo) + if err != nil { + return "", "", "", fmt.Errorf("invalid repository identifier %q: %w", repo, err) } - - service, handle, err := oatproxy.ResolveServiceWithClient(ctx, did, &aqhttp.Client) + if s.identityDirectory == nil { + return "", "", "", fmt.Errorf("identity directory not configured") + } + ident, err := s.identityDirectory().Lookup(ctx, *id) if err != nil { - return "", "", "", fmt.Errorf("failed to resolve service for did %q: %w", did, err) + return "", "", "", fmt.Errorf("failed to resolve repository %q: %w", repo, err) } - - return did, service, handle, nil + service := ident.PDSEndpoint() + if service == "" { + return "", "", "", fmt.Errorf("no PDS service for did %q", ident.DID.String()) + } + return ident.DID.String(), service, ident.Handle.String(), nil } var maxBlobSize int64 = 1024 * 1024 * 10 // 10MB @@ -97,7 +97,7 @@ func (s *Server) handleComAtprotoRepoDescribeRepo(ctx context.Context, repo stri if err != nil { return nil, fmt.Errorf("list server repo collections: %w", err) } - didDoc, err := glex.Unknown(atproto.DIDDoc(s.cli.ServerHost, atproto.ServerPubMultibase)) + didDoc, err := glex.Unknown(atproto.DIDDoc(s.cli.ServerHost, atproto.ServerPubMultibase, s.cli.AtmoqRelay)) if err != nil { return nil, fmt.Errorf("marshal did doc: %w", err) } @@ -110,7 +110,7 @@ func (s *Server) handleComAtprotoRepoDescribeRepo(ctx context.Context, repo stri }, nil } - didDoc, err := glex.Unknown(atproto.DIDDoc(s.cli.BroadcasterHost, atproto.LexiconPubMultibase)) + didDoc, err := glex.Unknown(atproto.DIDDoc(s.cli.BroadcasterHost, atproto.LexiconPubMultibase, s.cli.AtmoqRelay)) if err != nil { return nil, fmt.Errorf("marshal did doc: %w", err) } diff --git a/pkg/spxrpc/com_atproto_repo_test.go b/pkg/spxrpc/com_atproto_repo_test.go index ebadc566..4db23a0a 100644 --- a/pkg/spxrpc/com_atproto_repo_test.go +++ b/pkg/spxrpc/com_atproto_repo_test.go @@ -3,7 +3,6 @@ package spxrpc import ( "context" "encoding/json" - "net" "net/http" "net/http/httptest" "net/url" @@ -11,6 +10,8 @@ import ( "testing" "time" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/labstack/echo/v4" "github.com/stretchr/testify/require" "stream.place/streamplace/pkg/aqhttp" @@ -18,27 +19,22 @@ import ( ) // A former user's DID still points at a PDS hostname now serving Streamplace. -// Exercise DID resolution and the real HTTP proxy, not a mocked upstream call. +// Exercise the configured identity directory and the real HTTP proxy. func TestRepoProxyLoop(t *testing.T) { for _, method := range []string{"getRecord", "listRecords", "describeRepo"} { t.Run(method, func(t *testing.T) { - s := &Server{cli: &config.CLI{ServerHost: "node.example", BroadcasterHost: "node.example"}} + dir := identity.NewMockDirectory() + s := &Server{ + cli: &config.CLI{ServerHost: "node.example", BroadcasterHost: "node.example"}, + identityDirectory: func() identity.Directory { return &dir }, + } e := echo.New() e.Use(s.ContextPreservingMiddleware()) require.NoError(t, s.RegisterHandlersComatproto(e)) - var did, service string + const did = "did:plc:ho26ynsdw2ey7l56xx4owjzd" var requests atomic.Int32 upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/"+did { - _ = json.NewEncoder(w).Encode(map[string]any{ - "id": did, - "service": []map[string]string{{ - "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": service, - }}, - }) - return - } // Bound the broken implementation so a regression never exhausts // sockets or leaves recursive requests running after the test. if requests.Add(1) > 4 { @@ -48,8 +44,14 @@ func TestRepoProxyLoop(t *testing.T) { e.ServeHTTP(w, r) })) defer upstream.Close() - service = upstream.URL - did = "did:plc:ho26ynsdw2ey7l56xx4owjzd" + service := upstream.URL + dir.Insert(identity.Identity{ + DID: syntax.DID(did), + Handle: syntax.HandleInvalid, + Services: map[string]identity.ServiceEndpoint{ + "atproto_pds": {Type: "AtprotoPersonalDataServer", URL: service}, + }, + }) repoTestClients(t, upstream) params := url.Values{"repo": {did}, "collection": {"app.bsky.actor.profile"}, "rkey": {"self"}} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -69,27 +71,16 @@ func TestRepoProxyLoop(t *testing.T) { // every marked request at ingress would break reads between Streamplace nodes. func TestRepoProxyLocalRecord(t *testing.T) { var e *echo.Echo - var did, service string upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/.well-known/did.json" { - _ = json.NewEncoder(w).Encode(map[string]any{ - "id": did, - "service": []map[string]string{{ - "id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": service, - }}, - }) - return - } e.ServeHTTP(w, r) })) defer upstream.Close() - service = upstream.URL + service := upstream.URL cli, router := newSyncTestNode(t, "node.example", "node.example") e = router - did = cli.ServerDID() + did := cli.ServerDID() commitTestRecord(t, cli, "place.stream.live.viewerCount", "streamer") - repoTestClients(t, upstream) params := url.Values{"repo": {did}, "collection": {"place.stream.live.viewerCount"}, "rkey": {"streamer"}} req, err := http.NewRequest(http.MethodGet, service+"/xrpc/com.atproto.repo.getRecord?"+params.Encode(), nil) require.NoError(t, err) @@ -109,21 +100,109 @@ func TestRepoProxyLocalRecord(t *testing.T) { require.Equal(t, 7, record.Value.Count) } +func TestRepoProxyConfiguredDirectory(t *testing.T) { + const did = "did:plc:ho26ynsdw2ey7l56xx4owjzd" + const recordURI = "at://" + did + "/app.bsky.actor.profile/self" + var service string + var identityRequests, recordRequests atomic.Int32 + upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/" + did: + identityRequests.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": did, + "service": []map[string]string{ + {"id": "#atproto_pds", "type": "AtprotoPersonalDataServer", "serviceEndpoint": service}, + {"id": "#streamplace", "type": "Streamplace", "serviceEndpoint": "https://station.example"}, + }, + }) + case "/xrpc/com.atproto.repo.getRecord": + recordRequests.Add(1) + require.Equal(t, did, r.URL.Query().Get("repo")) + require.Equal(t, "app.bsky.actor.profile", r.URL.Query().Get("collection")) + require.Equal(t, "self", r.URL.Query().Get("rkey")) + _ = json.NewEncoder(w).Encode(map[string]any{ + "uri": recordURI, + "value": map[string]string{ + "$type": "app.bsky.actor.profile", "displayName": "Configured directory", + }, + }) + default: + http.NotFound(w, r) + } + })) + defer upstream.Close() + service = upstream.URL + repoTestClients(t, upstream) + dir := &identity.BaseDirectory{PLCURL: upstream.URL, HTTPClient: *upstream.Client()} + s := &Server{ + cli: &config.CLI{ServerHost: "station.example", BroadcasterHost: "node.example"}, + identityDirectory: func() identity.Directory { return dir }, + } + + record, err := s.handleComAtprotoRepoGetRecord(context.Background(), "", "app.bsky.actor.profile", did, "self") + require.NoError(t, err) + require.Equal(t, recordURI, record.Uri) + require.EqualValues(t, 1, identityRequests.Load(), "DID resolution must use the configured PLC directory") + require.EqualValues(t, 1, recordRequests.Load(), "the account PDS, not the station service, must serve the record") +} + +func TestResolveRepoServiceConfiguredIdentity(t *testing.T) { + const did = "did:plc:ho26ynsdw2ey7l56xx4owjzd" + const handle = "account.example" + const pds = "https://pds.example" + dir := identity.NewMockDirectory() + dir.Insert(identity.Identity{ + DID: syntax.DID(did), + Handle: syntax.Handle(handle), + Services: map[string]identity.ServiceEndpoint{ + "atproto_pds": {Type: "AtprotoPersonalDataServer", URL: pds}, + }, + }) + s := &Server{ + cli: &config.CLI{ServerHost: "station.example", BroadcasterHost: "node.example"}, + identityDirectory: func() identity.Directory { return &dir }, + } + for _, repo := range []string{did, handle} { + t.Run(repo, func(t *testing.T) { + resolvedDID, service, resolvedHandle, err := s.resolveRepoService(context.Background(), repo) + require.NoError(t, err) + require.Equal(t, did, resolvedDID) + require.Equal(t, pds, service) + require.Equal(t, handle, resolvedHandle) + local, service, err := s.isLocalPDS(context.Background(), repo) + require.NoError(t, err) + require.False(t, local, "the station must not treat an upstream account's PDS as a local repository") + require.Equal(t, pds, service) + }) + } +} + +func TestIsLocalPDSOwnDIDsWithoutLookup(t *testing.T) { + for _, serverHost := range []string{"", "node.example", "station.example"} { + t.Run(serverHost, func(t *testing.T) { + s := &Server{ + cli: &config.CLI{ServerHost: serverHost, BroadcasterHost: "node.example"}, + identityDirectory: func() identity.Directory { + t.Fatal("own repository routing must not consult the identity directory") + return nil + }, + } + for _, did := range []string{s.cli.ServerDID(), s.cli.BroadcasterDID()} { + local, service, err := s.isLocalPDS(context.Background(), did) + require.NoError(t, err) + require.True(t, local) + require.Empty(t, service) + } + }) + } +} + func repoTestClients(t *testing.T, upstream *httptest.Server) { t.Helper() - originalClient, originalDefault := aqhttp.Client, http.DefaultClient + originalClient := aqhttp.Client aqhttp.Client = *upstream.Client() - // oatproxy's service resolver currently uses http.DefaultClient rather - // than its client argument. Route its real HTTPS lookups to this fixture. - transport := upstream.Client().Transport.(*http.Transport).Clone() - transport.TLSClientConfig = transport.TLSClientConfig.Clone() - transport.TLSClientConfig.ServerName = "127.0.0.1" - transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { - return (&net.Dialer{}).DialContext(ctx, network, upstream.Listener.Addr().String()) - } - http.DefaultClient = &http.Client{Transport: transport, Timeout: 5 * time.Second} t.Cleanup(func() { - aqhttp.Client, http.DefaultClient = originalClient, originalDefault - transport.CloseIdleConnections() + aqhttp.Client = originalClient }) } diff --git a/pkg/spxrpc/pds_auth_test.go b/pkg/spxrpc/pds_auth_test.go index 4aca77c5..029768f2 100644 --- a/pkg/spxrpc/pds_auth_test.go +++ b/pkg/spxrpc/pds_auth_test.go @@ -425,7 +425,7 @@ func TestAuthMiddlewareStack(t *testing.T) { } func TestDIDDocAdvertisesStreamplaceService(t *testing.T) { - doc := atproto.DIDDoc(pdsAuthTestHost, "zQ3shRvJrnKp4SWvoxkFyZqijahHcmvJLdViKzkwQKj7cWxVm") + doc := atproto.DIDDoc(pdsAuthTestHost, "zQ3shRvJrnKp4SWvoxkFyZqijahHcmvJLdViKzkwQKj7cWxVm", "") services, ok := doc["service"].([]map[string]any) require.True(t, ok) require.Contains(t, services, map[string]any{ diff --git a/pkg/spxrpc/place_stream_game.go b/pkg/spxrpc/place_stream_game.go index b9921a3d..da44b3c9 100644 --- a/pkg/spxrpc/place_stream_game.go +++ b/pkg/spxrpc/place_stream_game.go @@ -95,7 +95,7 @@ func (s *Server) handlePlaceStreamGameGetGame(ctx context.Context, uri string) ( did, collection, rkey := parts[0], parts[1], parts[2] // resolve the DID to get the PDS - did, svc, _, err := resolveRepoService(ctx, did) + did, svc, _, err := s.resolveRepoService(ctx, did) if err != nil { return nil, echo.NewHTTPError(http.StatusBadRequest, "failed to resolve DID") } diff --git a/pkg/spxrpc/spxrpc.go b/pkg/spxrpc/spxrpc.go index 0953a116..6230b15a 100644 --- a/pkg/spxrpc/spxrpc.go +++ b/pkg/spxrpc/spxrpc.go @@ -168,7 +168,11 @@ func NewServer(ctx context.Context, cli *config.CLI, model model.Model, stateful } func (s *Server) isLocalPDS(ctx context.Context, repo string) (bool, string, error) { - did, svc, _, err := resolveRepoService(ctx, repo) + // Our own repositories are local even when their DID documents cannot be fetched. + if repo == s.cli.BroadcasterDID() || repo == s.cli.ServerDID() { + return true, "", nil + } + did, svc, _, err := s.resolveRepoService(ctx, repo) if err != nil { return false, "", fmt.Errorf("resolveRepoService: %w", err) } diff --git a/pkg/statedb/indexing_users.go b/pkg/statedb/indexing_users.go new file mode 100644 index 00000000..6cd10a85 --- /dev/null +++ b/pkg/statedb/indexing_users.go @@ -0,0 +1,38 @@ +package statedb + +import ( + "context" + "time" + + "github.com/streamplace/oatproxy/pkg/oatproxy" + "gorm.io/gorm" +) + +// indexingSessions is the persisted set of successfully authenticated local +// users. Pending OAuth flows and either kind of revocation grant no indexing +// permission. An expired access token remains usable only with a refresh token; +// OAuth records refresh failures by revoking the session. +func (state *StatefulDB) indexingSessions(ctx context.Context) *gorm.DB { + return state.DB.WithContext(ctx).Model(&oatproxy.OAuthSession{}). + Where("repo_did <> ? AND downstream_access_token <> ?", "", ""). + Where("revoked_at IS NULL AND downstream_revoked_at IS NULL"). + Where("upstream_refresh_token <> ? OR (upstream_access_token <> ? AND upstream_access_token_exp > ?)", "", "", time.Now().UTC()) +} + +// IndexingDID resolves a local session's DID or handle without discovering any +// remote account. An empty DID means no eligible session exists. +func (state *StatefulDB) IndexingDID(ctx context.Context, identifier string) (string, error) { + var did string + err := state.indexingSessions(ctx). + Select("repo_did").Where("repo_did = ? OR handle = ?", identifier, identifier). + Order("updated_at DESC").Limit(1).Scan(&did).Error + return did, err +} + +// IndexingDIDs derives boot/sweep eligibility from sessions, not the set of +// repos indexed in a previous server-mode run. No separate allowlist is kept. +func (state *StatefulDB) IndexingDIDs(ctx context.Context) ([]string, error) { + var dids []string + err := state.indexingSessions(ctx).Distinct("repo_did").Order("repo_did").Pluck("repo_did", &dids).Error + return dids, err +} diff --git a/pkg/statedb/indexing_users_test.go b/pkg/statedb/indexing_users_test.go new file mode 100644 index 00000000..b97652a7 --- /dev/null +++ b/pkg/statedb/indexing_users_test.go @@ -0,0 +1,79 @@ +package statedb + +import ( + "testing" + "time" + + "github.com/streamplace/oatproxy/pkg/oatproxy" + "github.com/stretchr/testify/require" +) + +func TestIndexingUsersRequireCompletedUsableSessions(t *testing.T) { + WithAllDatabases(t, func(state *StatefulDB) { + ctx := t.Context() + now := time.Now().UTC() + future, past := now.Add(time.Hour), now.Add(-time.Hour) + cases := []struct { + name string + change func(*oatproxy.OAuthSession) + eligible bool + }{ + {"ready", func(*oatproxy.OAuthSession) {}, true}, + {"pending", func(s *oatproxy.OAuthSession) { s.DownstreamAccessToken = ""; s.DownstreamAuthorizationCode = "code" }, false}, + {"upstream revoked", func(s *oatproxy.OAuthSession) { s.RevokedAt = &now }, false}, + {"downstream logged out", func(s *oatproxy.OAuthSession) { s.DownstreamRevokedAt = &now }, false}, + {"expired without refresh", func(s *oatproxy.OAuthSession) { s.UpstreamAccessTokenExp = &past; s.UpstreamRefreshToken = "" }, false}, + {"expired refreshable", func(s *oatproxy.OAuthSession) { s.UpstreamAccessTokenExp = &past }, true}, + {"unexpired without refresh", func(s *oatproxy.OAuthSession) { s.UpstreamRefreshToken = "" }, true}, + {"missing credentials", func(s *oatproxy.OAuthSession) { s.UpstreamAccessToken = ""; s.UpstreamRefreshToken = "" }, false}, + {"missing expiry without refresh", func(s *oatproxy.OAuthSession) { s.UpstreamAccessTokenExp = nil; s.UpstreamRefreshToken = "" }, false}, + {"logout before completion", func(s *oatproxy.OAuthSession) { s.DownstreamAccessToken = ""; s.DownstreamRevokedAt = &now }, false}, + } + var expected []string + for _, tc := range cases { + session := &oatproxy.OAuthSession{ + DID: "did:plc:" + tc.name, Handle: tc.name + ".test", + DownstreamDPoPJKT: tc.name, DownstreamAccessToken: "downstream", + UpstreamAccessToken: "upstream", UpstreamAccessTokenExp: &future, + UpstreamRefreshToken: "refresh", + } + tc.change(session) + require.NoError(t, state.CreateOAuthSession(session.DownstreamDPoPJKT, session)) + for _, identifier := range []string{session.DID, session.Handle} { + did, err := state.IndexingDID(ctx, identifier) + require.NoError(t, err, tc.name) + if tc.eligible { + require.Equal(t, session.DID, did, tc.name) + } else { + require.Empty(t, did, tc.name) + } + } + if tc.eligible { + expected = append(expected, session.DID) + } + } + dids, err := state.IndexingDIDs(ctx) + require.NoError(t, err) + require.ElementsMatch(t, expected, dids) + + // One revoked device cannot hide another completed local login; two + // devices for one user must not schedule two sweeps of that repo. + require.NoError(t, state.CreateOAuthSession("second-device", &oatproxy.OAuthSession{ + DID: "did:plc:ready", DownstreamDPoPJKT: "second-device", + DownstreamAccessToken: "second", UpstreamRefreshToken: "refresh", + })) + dids, err = state.IndexingDIDs(ctx) + require.NoError(t, err) + require.ElementsMatch(t, expected, dids) + require.NoError(t, state.DB.Model(&oatproxy.OAuthSession{}). + Where("downstream_dpop_jkt = ?", "ready").Update("downstream_revoked_at", now).Error) + did, err := state.IndexingDID(ctx, "did:plc:ready") + require.NoError(t, err) + require.Equal(t, "did:plc:ready", did) + require.NoError(t, state.DB.Model(&oatproxy.OAuthSession{}). + Where("downstream_dpop_jkt = ?", "second-device").Update("revoked_at", now).Error) + did, err = state.IndexingDID(ctx, "did:plc:ready") + require.NoError(t, err) + require.Empty(t, did) + }) +} diff --git a/pkg/statedb/oauth_session.go b/pkg/statedb/oauth_session.go index 21750124..c7bb4e4c 100644 --- a/pkg/statedb/oauth_session.go +++ b/pkg/statedb/oauth_session.go @@ -20,7 +20,12 @@ func (state *StatefulDB) LoadOAuthSession(id string) (*oatproxy.OAuthSession, er } return nil, err } - if session.Status() != oatproxy.OAuthSessionStateReady || session.Status() != oatproxy.OAuthSessionStateReadyUpstream { + if session.Status() != oatproxy.OAuthSessionStateReady && session.Status() != oatproxy.OAuthSessionStateReadyUpstream { + return &session, nil + } + // A delegated session is bound to the station, not the user's actual PDS. + // The real PDS remains authoritative in the indexed repo. + if state.CLI != nil && state.CLI.UpstreamStation != "" { return &session, nil } r, err := state.model.GetRepo(session.DID) @@ -28,7 +33,8 @@ func (state *StatefulDB) LoadOAuthSession(id string) (*oatproxy.OAuthSession, er return nil, fmt.Errorf("failed to get repo: %w", err) } if r == nil { - return nil, fmt.Errorf("repo not found even though we have a valid session!? repodid=%s", session.DID) + // Login is persisted before the asynchronous initial repo sync. + return &session, nil } if r.PDS != session.PDSUrl { return nil, fmt.Errorf("pds mismatch (old: %s, new: %s): please log in again", session.PDSUrl, r.PDS) diff --git a/pkg/statedb/oauth_session_test.go b/pkg/statedb/oauth_session_test.go index 5708ad1d..60892987 100644 --- a/pkg/statedb/oauth_session_test.go +++ b/pkg/statedb/oauth_session_test.go @@ -7,6 +7,7 @@ import ( "github.com/streamplace/oatproxy/pkg/oatproxy" "github.com/stretchr/testify/require" "gorm.io/gorm" + "stream.place/streamplace/pkg/model" ) // literals instead of the pkg/atproto constants: that package imports @@ -83,3 +84,39 @@ func TestGetSessionByDIDWithScope(t *testing.T) { require.Equal(t, "jkt-legacy", got.DownstreamDPoPJKT) }) } + +func TestLoadOAuthSessionReadyPDSBinding(t *testing.T) { + WithAllDatabases(t, func(state *StatefulDB) { + const did = "did:plc:oauth-ready-binding" + require.NoError(t, state.model.UpdateRepo(&model.Repo{DID: did, PDS: "https://actual-pds.example"})) + ready := &oatproxy.OAuthSession{DID: did, DownstreamDPoPJKT: "ready-binding", DownstreamAccessToken: "access", PDSUrl: "https://old-pds.example"} + require.NoError(t, state.CreateOAuthSession(ready.DownstreamDPoPJKT, ready)) + _, err := state.LoadOAuthSession(ready.DownstreamDPoPJKT) + require.ErrorContains(t, err, "pds mismatch", "ready sessions must execute the previously unreachable PDS check") + now := time.Now() + ready.DownstreamRevokedAt = &now + require.NoError(t, state.UpdateOAuthSession(ready.DownstreamDPoPJKT, ready)) + _, err = state.LoadOAuthSession(ready.DownstreamDPoPJKT) + require.ErrorContains(t, err, "pds mismatch", "retained upstream sessions are also PDS-bound") + state.CLI.UpstreamStation = "did:web:station.example" + delegated, err := state.LoadOAuthSession(ready.DownstreamDPoPJKT) + require.NoError(t, err) + require.Equal(t, ready.PDSUrl, delegated.PDSUrl, "station delegation does not overwrite indexed identity") + repo, err := state.model.GetRepo(did) + require.NoError(t, err) + require.Equal(t, "https://actual-pds.example", repo.PDS) + }) +} + +func TestLoadOAuthSessionBeforeInitialSync(t *testing.T) { + WithAllDatabases(t, func(state *StatefulDB) { + session := &oatproxy.OAuthSession{DID: "did:plc:new-oauth-user", DownstreamDPoPJKT: "initial-sync", DownstreamAccessToken: "access", PDSUrl: "https://pds.example"} + require.NoError(t, state.CreateOAuthSession(session.DownstreamDPoPJKT, session)) + loaded, err := state.LoadOAuthSession(session.DownstreamDPoPJKT) + require.NoError(t, err, "successful login is persisted before initial asynchronous indexing") + require.Equal(t, session.DID, loaded.DID) + missing, err := state.LoadOAuthSession("never-created") + require.NoError(t, err) + require.Nil(t, missing) + }) +} diff --git a/util/streamplace-dev.sh b/util/streamplace-dev.sh index 3760e2c2..020cc104 100755 --- a/util/streamplace-dev.sh +++ b/util/streamplace-dev.sh @@ -8,6 +8,6 @@ set -euo pipefail SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) STREAMPLACE_DEV_VERSION=$(go run ./pkg/config/git/git.go -v) \ LD_LIBRARY_PATH="$SCRIPT_DIR/lib" \ -SP_DEV_FRONTEND_PROXY="http://127.0.0.1:38081" \ +SP_DEV_FRONTEND_PROXY="${SP_DEV_FRONTEND_PROXY-http://127.0.0.1:38081}" \ SP_DEV_PUBLIC_OAUTH="${SP_DEV_PUBLIC_OAUTH:-true}" \ exec "$SCRIPT_DIR/libstreamplace" "$@" -- 2.51.2