From 8827d30bb74e9490d664581cd179ed05122ea4de Mon Sep 17 00:00:00 2001 From: Bretton Date: Thu, 20 Aug 2026 00:58:02 -0700 Subject: [PATCH] fix(security): enforce SSRF-safe outbound HTTP Centralize attacker-influenced egress behind a DNS-rebinding-resistant transport and fail closed on non-global destinations. This closes gaps across OAuth discovery, identity resolution, PDS calls, Jetstream fetches, aggregator registration, image and blob retrieval, profile backfill, rematerialization, and link unfurling. Changes: - validate DNS answers and socket destinations against one address policy - guard all Indigo OAuth clients and enforce redirect, timeout, and body limits - block local-use NAT64 and other non-global special-purpose ranges - keep private-host access behind explicit development-only wiring - add guard tests and a hard CI audit for unguarded client construction - publish stable SSRF and vulnerability-reporting documentation - refresh vulnerable dependencies and supported container toolchains Co-Authored-By: Claude Opus 5 (1M context) --- Dockerfile | 4 +- Makefile | 5 +- SECURITY.md | 28 + aggregators/kagi-news/requirements.txt | 6 +- .../reddit-highlights/requirements.txt | 6 +- cmd/backfill-profiles/main.go | 53 +- cmd/backfill-profiles/main_test.go | 206 ++ cmd/reindex-votes/main.go | 32 + .../allow_private_hosts_test.go | 49 + cmd/rematerialize-posts/main.go | 81 +- cmd/server/allow_private_hosts_test.go | 81 + cmd/server/consumers.go | 35 +- cmd/server/pds.go | 2 +- cmd/server/routes.go | 7 +- cmd/server/service_jwt_identity_test.go | 199 ++ cmd/server/wiring.go | 190 +- cmd/validate-live/main.go | 2 +- docker-compose.ci.yml | 2 +- docker-compose.dev.yml | 2 +- docker/ci-runner/Dockerfile | 2 +- docker/plc/Dockerfile | 2 +- docs/SSRF_SECURITY.md | 153 ++ go.mod | 60 +- go.sum | 738 ++++++- internal/api/handlers/aggregator/register.go | 198 +- .../aggregator/register_domain_test.go | 230 +++ .../aggregator/register_guard_test.go | 562 +++++ .../aggregator/register_seam_export_test.go | 95 + .../api/handlers/aggregator/register_test.go | 164 +- .../aggregator/register_users_row_test.go | 12 +- .../imageproxy/avatar_serving_test.go | 18 +- .../imageproxy/blocked_status_test.go | 88 + internal/api/handlers/imageproxy/handler.go | 16 +- .../handlers/imageproxy/proxy_serving_test.go | 9 +- .../imageproxy/roundtrip_serving_test.go | 11 +- internal/api/handlers/post/harness_test.go | 1 + internal/api/routes/aggregator.go | 9 +- internal/atproto/identity/factory.go | 153 +- .../atproto/identity/factory_seam_test.go | 131 ++ internal/atproto/identity/factory_test.go | 22 +- .../atproto/identity/resolver_guard_test.go | 360 ++++ .../jetstream/acceptance_consumer_test.go | 3 +- .../jetstream/admission_durability_test.go | 3 +- internal/atproto/jetstream/authorpost.go | 79 +- .../atproto/jetstream/authorpost_ssrf_test.go | 265 +++ .../atproto/jetstream/community_consumer.go | 264 ++- .../community_consumer_seam_export_test.go | 82 + .../jetstream/community_consumer_ssrf_test.go | 721 +++++++ .../direct_fetch_verification_test.go | 7 +- internal/atproto/oauth/client.go | 78 +- internal/atproto/oauth/client_guard_test.go | 577 ++++++ internal/atproto/oauth/dev_auth_resolver.go | 2 +- internal/atproto/oauth/dev_resolver.go | 2 +- internal/atproto/oauth/transport.go | 953 ++++++++- .../oauth/transport_blocked_error_test.go | 182 ++ .../atproto/oauth/transport_body_cap_test.go | 564 +++++ .../oauth/transport_cap_bounds_test.go | 162 ++ .../oauth/transport_capped_body_test.go | 243 +++ .../atproto/oauth/transport_context_test.go | 87 + .../oauth/transport_dial_errors_test.go | 102 + .../transport_dial_timeout_aggregate_test.go | 229 ++ .../oauth/transport_dial_timeout_test.go | 107 + .../oauth/transport_embedded_ipv4_test.go | 30 +- .../oauth/transport_empty_answer_test.go | 92 + .../oauth/transport_gate_helper_test.go | 212 ++ .../atproto/oauth/transport_idn_host_test.go | 607 ++++++ .../oauth/transport_ip_literal_test.go | 201 ++ .../oauth/transport_private_option_test.go | 250 +++ .../oauth/transport_refusal_sentinel_test.go | 135 ++ .../oauth/transport_request_body_test.go | 176 ++ .../oauth/transport_reserved_ranges_test.go | 62 + .../oauth/transport_response_cap_test.go | 223 ++ .../atproto/oauth/transport_revetting_test.go | 33 +- internal/atproto/oauth/transport_test.go | 4 +- .../atproto/oauth/transport_toctou_test.go | 7 +- .../transport_unspecified_address_test.go | 53 +- .../oauth/transport_zoned_literal_test.go | 168 ++ internal/atproto/pds/applywrites_test.go | 7 +- internal/atproto/pds/factory.go | 191 +- internal/atproto/pds/factory_guard_test.go | 560 +++++ .../blobs/blob_upload_integration_test.go | 28 +- internal/core/blobs/service.go | 92 +- internal/core/blobs/upload_guard_test.go | 300 +++ internal/core/blueskypost/fetcher.go | 2 +- internal/core/comments/comment_write_test.go | 16 +- internal/core/communities/pds_client.go | 188 ++ .../core/communities/pds_client_guard_test.go | 315 +++ internal/core/communities/pds_provisioning.go | 32 +- .../communities/provisioner_failure_test.go | 14 +- internal/core/communities/service.go | 21 +- .../communities/service_provisioning_test.go | 7 +- .../core/communities/service_readpath_test.go | 3 +- .../communities/service_validation_test.go | 5 +- .../service_writeforward_failures_test.go | 6 +- internal/core/communities/token_refresh.go | 31 +- internal/core/imageproxy/errors.go | 15 + internal/core/imageproxy/fetcher.go | 180 +- .../core/imageproxy/fetcher_guard_test.go | 562 +++++ internal/core/imageproxy/fetcher_test.go | 40 +- internal/core/posts/community_repo_factory.go | 16 +- .../core/posts/community_repo_factory_test.go | 9 +- internal/core/posts/engine_contract_test.go | 6 +- internal/core/posts/rematerialize.go | 99 +- .../posts/rematerialize_blob_guard_test.go | 341 +++ .../posts/rematerialize_blob_overrun_test.go | 138 ++ .../posts/rematerialize_blob_seam_test.go | 39 + .../core/posts/rematerialize_blob_test.go | 16 +- .../core/posts/rematerialize_outer_test.go | 29 +- internal/core/posts/service.go | 27 +- .../posts/service_author_posts_query_test.go | 3 +- .../posts/service_create_validation_test.go | 3 +- .../core/posts/service_writeforward_test.go | 22 +- internal/core/unfurl/body_cap_test.go | 245 +++ internal/core/unfurl/circuit_breaker.go | 40 +- .../core/unfurl/circuit_breaker_guard_test.go | 162 ++ internal/core/unfurl/errors.go | 12 + internal/core/unfurl/kagi_test.go | 16 +- internal/core/unfurl/opengraph_test.go | 8 +- .../unfurl/post_unfurl_integration_test.go | 15 + internal/core/unfurl/providers.go | 78 +- internal/core/unfurl/service.go | 90 +- internal/core/unfurl/ssrf_guard_test.go | 560 +++++ .../userblocks/service_writeforward_test.go | 2 +- .../core/users/profile_backfill_cap_test.go | 107 + .../profile_backfill_fallback_guard_test.go | 191 ++ .../core/users/profile_backfill_guard_test.go | 304 +++ .../core/users/profile_backfill_seam_test.go | 45 + internal/core/users/service.go | 114 +- internal/core/users/turnstile.go | 2 +- internal/db/postgres/community_feed_test.go | 1 + internal/notify/telegram/client.go | 2 +- internal/validation/domain.go | 228 ++ internal/validation/domain_test.go | 229 ++ scripts/ci-runner.sh | 14 + scripts/ssrf-audit.sh | 992 +++++++++ tests/audit/ssrf_audit_test.go | 1833 +++++++++++++++++ tests/domaincorpus/corpus.go | 211 ++ tests/fixtures/fixtures.go | 2 +- tests/live/post_unfurl_test.go | 1 + tests/testkit/pds.go | 27 +- tests/testkit/pds_test.go | 40 +- 141 files changed, 19438 insertions(+), 485 deletions(-) create mode 100644 SECURITY.md create mode 100644 cmd/backfill-profiles/main_test.go create mode 100644 cmd/rematerialize-posts/allow_private_hosts_test.go create mode 100644 cmd/server/allow_private_hosts_test.go create mode 100644 cmd/server/service_jwt_identity_test.go create mode 100644 docs/SSRF_SECURITY.md create mode 100644 internal/api/handlers/aggregator/register_domain_test.go create mode 100644 internal/api/handlers/aggregator/register_guard_test.go create mode 100644 internal/api/handlers/aggregator/register_seam_export_test.go create mode 100644 internal/api/handlers/imageproxy/blocked_status_test.go create mode 100644 internal/atproto/identity/factory_seam_test.go create mode 100644 internal/atproto/identity/resolver_guard_test.go create mode 100644 internal/atproto/jetstream/authorpost_ssrf_test.go create mode 100644 internal/atproto/jetstream/community_consumer_seam_export_test.go create mode 100644 internal/atproto/jetstream/community_consumer_ssrf_test.go create mode 100644 internal/atproto/oauth/client_guard_test.go create mode 100644 internal/atproto/oauth/transport_blocked_error_test.go create mode 100644 internal/atproto/oauth/transport_body_cap_test.go create mode 100644 internal/atproto/oauth/transport_cap_bounds_test.go create mode 100644 internal/atproto/oauth/transport_capped_body_test.go create mode 100644 internal/atproto/oauth/transport_context_test.go create mode 100644 internal/atproto/oauth/transport_dial_errors_test.go create mode 100644 internal/atproto/oauth/transport_dial_timeout_aggregate_test.go create mode 100644 internal/atproto/oauth/transport_dial_timeout_test.go create mode 100644 internal/atproto/oauth/transport_empty_answer_test.go create mode 100644 internal/atproto/oauth/transport_gate_helper_test.go create mode 100644 internal/atproto/oauth/transport_idn_host_test.go create mode 100644 internal/atproto/oauth/transport_ip_literal_test.go create mode 100644 internal/atproto/oauth/transport_private_option_test.go create mode 100644 internal/atproto/oauth/transport_refusal_sentinel_test.go create mode 100644 internal/atproto/oauth/transport_request_body_test.go create mode 100644 internal/atproto/oauth/transport_response_cap_test.go create mode 100644 internal/atproto/oauth/transport_zoned_literal_test.go create mode 100644 internal/atproto/pds/factory_guard_test.go create mode 100644 internal/core/blobs/upload_guard_test.go create mode 100644 internal/core/communities/pds_client.go create mode 100644 internal/core/communities/pds_client_guard_test.go create mode 100644 internal/core/imageproxy/fetcher_guard_test.go create mode 100644 internal/core/posts/rematerialize_blob_guard_test.go create mode 100644 internal/core/posts/rematerialize_blob_overrun_test.go create mode 100644 internal/core/posts/rematerialize_blob_seam_test.go create mode 100644 internal/core/unfurl/body_cap_test.go create mode 100644 internal/core/unfurl/circuit_breaker_guard_test.go create mode 100644 internal/core/unfurl/ssrf_guard_test.go create mode 100644 internal/core/users/profile_backfill_cap_test.go create mode 100644 internal/core/users/profile_backfill_fallback_guard_test.go create mode 100644 internal/core/users/profile_backfill_guard_test.go create mode 100644 internal/core/users/profile_backfill_seam_test.go create mode 100644 internal/validation/domain.go create mode 100644 internal/validation/domain_test.go create mode 100755 scripts/ssrf-audit.sh create mode 100644 tests/audit/ssrf_audit_test.go create mode 100644 tests/domaincorpus/corpus.go diff --git a/Dockerfile b/Dockerfile index 7589d2a..7275c27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ # Builds a minimal production image for the Go server # Stage 1: Build -FROM golang:1.25-alpine AS builder +FROM golang:1.26.7-alpine3.24 AS builder # Install build dependencies RUN apk add --no-cache git ca-certificates tzdata @@ -36,7 +36,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${GOARCH} go build \ ./cmd/server # Stage 2: Runtime -FROM alpine:3.19 +FROM alpine:3.24.1 # Install runtime dependencies RUN apk add --no-cache ca-certificates tzdata diff --git a/Makefile b/Makefile index ec41ad0..ae70934 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help dev-up dev-up-otel dev-down dev-logs dev-status dev-reset test test-integration test-e2e test-e2e-dev test-live test-db-prepare test-audit ci ci-clean clean mobile-full-setup +.PHONY: help dev-up dev-up-otel dev-down dev-logs dev-status dev-reset test test-integration test-e2e test-e2e-dev test-live test-db-prepare test-audit ssrf-audit ci ci-clean clean mobile-full-setup # Default target - show help .DEFAULT_GOAL := help @@ -288,6 +288,9 @@ test-db-prepare: ## Create or refresh the template database that testkit.DB clon test-audit: ## Test-suite invariant audit - hard gate, any violation fails (-v for file:line) @./scripts/test-audit.sh +ssrf-audit: ## SSRF guard regression fence over production code - hard gate (-v for file:line) + @./scripts/ssrf-audit.sh + test-db-stop: ## Stop test database @docker-compose -f docker-compose.dev.yml --env-file .env.dev --profile test stop postgres-test @echo "$(GREEN)✓ Test database stopped$(RESET)" diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..4c96621 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,28 @@ +# Security policy + +## Supported versions + +Coves is under active development and does not currently maintain multiple +supported release lines. Security fixes are applied to the latest code on +`main`; operators should update to the newest available revision. + +## Reporting a vulnerability + +Please do not disclose suspected vulnerabilities in a public issue, discussion, +or pull request before maintainers have had an opportunity to investigate. + +Email `support@coves.social` with the subject `Security vulnerability`. Include, +where possible: + +- the affected component and revision; +- the impact and required preconditions; +- reproduction steps or a minimal proof of concept; +- any suggested mitigation; and +- how you would like to be credited. + +Avoid including real user data, production credentials, or unnecessary access +to third-party systems. Maintainers will coordinate validation, remediation, and +appropriate disclosure with the reporter. + +If GitHub private vulnerability reporting is enabled for this repository, the +repository's **Security** tab is also an appropriate private reporting channel. diff --git a/aggregators/kagi-news/requirements.txt b/aggregators/kagi-news/requirements.txt index de24fee..557d85d 100644 --- a/aggregators/kagi-news/requirements.txt +++ b/aggregators/kagi-news/requirements.txt @@ -1,7 +1,7 @@ # Core dependencies feedparser==6.0.11 beautifulsoup4==4.12.3 -requests==2.31.0 +requests==2.33.0 pyyaml==6.0.1 anthropic>=0.40.0 # IDNA2008/UTS-46 host encoding for uri_sanitizer. Also pulled in transitively @@ -10,12 +10,12 @@ anthropic>=0.40.0 idna>=3.4 # Testing -pytest==8.1.1 +pytest==9.0.3 pytest-cov==5.0.0 responses==0.25.0 # Development -black==24.3.0 +black==26.3.1 mypy==1.9.0 types-PyYAML==6.0.12.12 types-requests==2.31.0.20240311 diff --git a/aggregators/reddit-highlights/requirements.txt b/aggregators/reddit-highlights/requirements.txt index 855ab4f..3d0e5a8 100644 --- a/aggregators/reddit-highlights/requirements.txt +++ b/aggregators/reddit-highlights/requirements.txt @@ -1,6 +1,6 @@ # Core dependencies feedparser==6.0.11 -requests==2.31.0 +requests==2.33.0 pyyaml==6.0.1 # IDNA2008/UTS-46 host encoding for uri_sanitizer. Also pulled in transitively # by requests, but declared explicitly because we import it directly — and @@ -8,12 +8,12 @@ pyyaml==6.0.1 idna>=3.4 # Testing -pytest==8.1.1 +pytest==9.0.3 pytest-cov==5.0.0 responses==0.25.0 # Development -black==24.3.0 +black==26.3.1 mypy==1.9.0 types-PyYAML==6.0.12.12 types-requests==2.31.0.20240311 diff --git a/cmd/backfill-profiles/main.go b/cmd/backfill-profiles/main.go index b53415c..953c461 100644 --- a/cmd/backfill-profiles/main.go +++ b/cmd/backfill-profiles/main.go @@ -31,6 +31,7 @@ import ( "sync/atomic" "time" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/core/users" "Coves/internal/db/postgres" @@ -53,6 +54,8 @@ type backfillStats struct { func main() { dryRun := flag.Bool("dry-run", false, "fetch and report what would be updated without writing to the database") concurrency := flag.Int("concurrency", 4, "number of concurrent PDS fetches") + allowPrivateHosts := flag.Bool("allow-private-hosts", false, + "DEV ONLY: dial PDS URLs that resolve to private, loopback or link-local addresses (a local dev stack); never pass this against a real database") flag.Parse() if *concurrency < 1 { @@ -115,7 +118,7 @@ func main() { return } - client := &http.Client{Timeout: 15 * time.Second} + client := newProfileFetchClient(*allowPrivateHosts) var stats backfillStats var wg sync.WaitGroup @@ -147,6 +150,54 @@ func main() { } } +// profileFetchTimeout bounds one getRecord round trip against a user's PDS. +// +// It is this tool's own value and not the AppView's: processUser wraps each +// fetch in a 20s context, and the client's ceiling has always sat below that so +// a stalled PDS is attributed to the fetch rather than to the context. +const profileFetchTimeout = 15 * time.Second + +// newProfileFetchClient builds the client every profile fetch in this job goes +// through, and is the dev gate for this call site. +// +// # WHAT IT IS GUARDING +// +// processUser hands `u.pdsURL` to users.FetchProfileRecord, and that value is +// read straight out of the `users` table — so it is whatever some other +// instance's account data put there, the same attacker-influenced input the +// AppView's own backfill was converted for. Being a CLI makes it worse rather +// than better: it is run by hand against production while an operator reconciles +// an incident, it fetches for every bare user at once, and its failures are one +// log.Printf per user among thousands, so a request to an internal address reads +// as ordinary churn. +// +// # THE FLAG IS THE GATE +// +// DATABASE_URL defaults to the LOCAL DEV database, whose PDS is on loopback, so +// the documented no-environment invocation is exactly the one the guard refuses. +// -allow-private-hosts is what keeps that usage working, and it is the only way +// to open the guard: false yields no options at all, which is +// PrivateAddressOptions' contract. +// +// # THE opts PARAMETER IS THE TEST SEAM +// +// It mirrors users.NewProfileBackfillClient and the other converted sites, and +// exists because a guard test that builds its own client proves only that +// internal/atproto/oauth works. main passes nothing. +// +// # WHY THE CLIENT MOVED OUT OF main() +// +// It was a bare `&http.Client{Timeout: 15 * time.Second}` on a line inside +// main(), which no test can reach — main() opens a database, queries it and +// blocks on a worker pool. This is the smallest extraction that makes the +// construction addressable; main keeps every other line it had. +func newProfileFetchClient(allowPrivateHosts bool, opts ...covesoauth.Option) *http.Client { + client := covesoauth.NewSSRFSafeHTTPClient( + append(covesoauth.PrivateAddressOptions(allowPrivateHosts), opts...)...) + client.Timeout = profileFetchTimeout + return client +} + func updatedVerb(dryRun bool) string { if dryRun { return "would be updated" diff --git a/cmd/backfill-profiles/main_test.go b/cmd/backfill-profiles/main_test.go new file mode 100644 index 0000000..4ba09e7 --- /dev/null +++ b/cmd/backfill-profiles/main_test.go @@ -0,0 +1,206 @@ +package main + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" + "Coves/internal/core/users" +) + +// The reconciliation job's PDS fetch — the same request the AppView's detached +// backfill makes, from a process nothing else is watching. +// +// # WHY A CLI IS NOT A LESSER CALL SITE +// +// `u.pdsURL` is read straight out of the `users` table, so it is the value some +// other instance's account data put there — the same attacker-influenced input +// the AppView's own backfill was converted for. What differs is the +// surroundings, and every difference makes it worse rather than better: +// +// - It is run BY HAND against production, by an operator reconciling a real +// incident, so a request to an internal address goes out from an +// interactive session with a human's attention on the summary line. +// - It fetches for EVERY bare user at once, four at a time — one hostile PDS +// URL in the table is dialled on the next run, whoever triggers it. +// - Its failures are one `log.Printf` per user among thousands, and the +// process exits 1 on any failure, so a refused address reads as ordinary +// churn. +// +// # WHY THE CLIENT MOVED OUT OF main() +// +// It was `&http.Client{Timeout: 15 * time.Second}` on a line inside main(), +// which no test can reach: main() opens a database, queries it and blocks on a +// worker pool. newProfileFetchClient is the smallest extraction that makes the +// construction addressable — it takes the gate boolean main's flag supplies and +// the option seam these tests need, and main keeps every other line it had. + +const ( + backfillCLIDID = "did:plc:backfillcliguard" + + // backfillCLIHost passes every shape check this path applies and is a name + // rather than an address, so classification is the only thing that can refuse + // it. `.example` is reserved by RFC 2606, so nothing resolves it for real if + // the seam is ever bypassed. + backfillCLIHost = "https://user-pds.example" +) + +// countingPDS records whether the job ever reached a listener. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + pds.requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"value":{"displayName":"leaked from an internal endpoint"}}`)) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// TestNewProfileFetchClient_RefusesAPrivatePDSWithoutReachingIt is the binding +// contract for this site. +// +// It drives users.FetchProfileRecord — the call processUser actually makes — so +// what is under test is the client as the job uses it, not the constructor in +// isolation. +func TestNewProfileFetchClient_RefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + _, err := users.FetchProfileRecord(context.Background(), + newProfileFetchClient(false), pds.server.URL, backfillCLIDID) + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. The PDS URL comes straight from the users table, so "+ + "whoever wrote that row chose this address, and the request leaving the process is the "+ + "SSRF whatever comes back", pds.requests.Load()) + + require.Error(t, err, + "the job fetched a loopback address successfully. This runs by hand against production, "+ + "and a dialled internal address arrives as one log line among thousands") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the fetch failed, but not because the guard refused the address. A PDS that simply could "+ + "not be reached fails identically, so without the guard's own identity this assertion "+ + "passes against the current build, where the client is a bare http.Client; got: %v", err) +} + +// TestNewProfileFetchClient_ReachesThePDSWhenTheHatchIsOpen is the other +// direction, and the falsifiability control for the case above. +// +// The hatch is not decoration here: DATABASE_URL defaults to the LOCAL DEV +// database, so running this job against a developer's own stack — where the PDS +// is on loopback — is the documented default invocation. Guarding it without a +// flag would break the only usage that needs no environment at all. +func TestNewProfileFetchClient_ReachesThePDSWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + input, err := users.FetchProfileRecord(context.Background(), + newProfileFetchClient(true), pds.server.URL, backfillCLIDID) + + require.NoErrorf(t, err, + "with -allow-private-hosts the job must reach a loopback PDS, which is what the local dev "+ + "database this tool defaults to is paired with; got: %v", err) + require.NotNil(t, input, "the fixture serves a displayName, so a profile must come back") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestNewProfileFetchClient_PreservesTheConfiguredTimeout guards the setting the +// shared client would otherwise swallow. +// +// NewSSRFSafeHTTPClient ships a 15s ceiling of its own, which happens to equal +// this one TODAY — which is exactly why the assertion is against +// profileFetchTimeout rather than against a literal. The two values are +// independent, and a conversion that drops this one leaves no visible trace +// until the shared default moves and every fetch in this job silently re-times +// with it. +func TestNewProfileFetchClient_PreservesTheConfiguredTimeout(t *testing.T) { + t.Parallel() + + client := newProfileFetchClient(false) + + require.NotNil(t, client, "the gate must return a client") + assert.Equalf(t, profileFetchTimeout, client.Timeout, + "the job's client runs on a %v timeout instead of profileFetchTimeout (%v). processUser "+ + "wraps each fetch in a 20s context, and this value has always sat below it so a stalled "+ + "PDS is attributed to the fetch rather than to the context expiring", + client.Timeout, profileFetchTimeout) +} + +// resolvingProfileFetchClient builds the client the way main does and then +// replaces only its NAME RESOLUTION, so the client under test is the real one. +func resolvingProfileFetchClient(t *testing.T, allowPrivateHosts bool, resolvesTo string) *http.Client { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(resolvesTo) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", resolvesTo) + + return newProfileFetchClient(allowPrivateHosts, + covesoauth.WithHostResolver(func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + })) +} + +// TestNewProfileFetchClient_RefusesAWellFormedHostThatResolvesPrivate is the +// assertion a loopback-literal fixture cannot make: the guard's CLASSIFICATION +// pass, on a name that survives every earlier check. +func TestNewProfileFetchClient_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + client := resolvingProfileFetchClient(t, false, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + _, err := users.FetchProfileRecord(context.Background(), client, backfillCLIHost, backfillCLIDID) + + require.Errorf(t, err, + "%s is a well-formed https host whose DNS answer was 127.0.0.1, and the job fetched it "+ + "anyway. A user's PDS URL is chosen by whoever runs their PDS, and they own the zone", + backfillCLIHost) + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a build where this client was never "+ + "converted looks identical; got: %v", err) +} + +// TestNewProfileFetchClient_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control for the case above. +// +// Identical constructor, identical seam, identical host — only the hatch +// differs. With it open the address is no longer refused, so the request +// proceeds to a dial, which fails because nothing is listening on loopback:443. +// That difference is what pins the refusal above to classification rather than to +// this test being unable to make requests at all. +func TestNewProfileFetchClient_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + client := resolvingProfileFetchClient(t, true, "127.0.0.1") // coves:allow-host-literal: with the hatch open this is dialled and refused by the OS + + _, err := users.FetchProfileRecord(context.Background(), client, backfillCLIHost, backfillCLIDID) + + require.Error(t, err, + "nothing listens on loopback:443, so this fetch must fail — if it succeeded, the seam is "+ + "not answering with the address this test gave it") + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either the gate is not "+ + "reaching the client, or the guarded case above proves nothing: a client that refuses "+ + "every address refuses that case too, for a reason unconnected to classification; got: %v", + err) +} diff --git a/cmd/reindex-votes/main.go b/cmd/reindex-votes/main.go index d9ebccf..71f26b3 100644 --- a/cmd/reindex-votes/main.go +++ b/cmd/reindex-votes/main.go @@ -99,6 +99,30 @@ func main() { } // fetchAllAccountsFromPDS queries the PDS sync API to get all repo DIDs +// +// # WHY THIS FETCH IS NOT GUARDED, AND WHAT WOULD CHANGE THAT +// +// pdsURL is os.Getenv("PDS_URL"), defaulting to this instance's own local dev +// PDS — operator configuration, not data. That is the whole test: every guarded +// site in this remediation dials a host that arrived from a DID document, a +// federated record or a database column, and this one dials the host the person +// running the tool typed. There is no attacker in the path to the ADDRESS. +// +// Guarding it would also break the only invocation that needs no environment at +// all, since the default is loopback and loopback is what the guard refuses. +// +// If pdsURL is ever taken from a record, a database row or a flag whose value +// comes from indexed data, this marker is wrong and the call site needs +// oauth.NewSSRFSafeHTTPClient behind a dev gate, like cmd/backfill-profiles. +// +// # THE SEPARATE DEFECT, RECORDED RATHER THAN QUIETLY FIXED +// +// http.Get is http.DefaultClient, whose Timeout is ZERO — and zero in net/http +// means wait forever. Both loops here page until the cursor runs out, so a PDS +// that accepts the connection and then stops answering hangs this tool with no +// deadline of any kind, and an operator has to notice and kill it. That is a +// real weakness and it is not an SSRF one; it is left here deliberately rather +// than changed under a commit about audit tooling. func fetchAllAccountsFromPDS(pdsURL string) ([]string, error) { // Use com.atproto.sync.listRepos to get all repos on this PDS var allDIDs []string @@ -110,6 +134,7 @@ func fetchAllAccountsFromPDS(pdsURL string) ([]string, error) { reqURL += "&cursor=" + url.QueryEscape(cursor) } + // coves:allow-bare-client: pdsURL is PDS_URL from the environment, so the host is operator config; the DIDs are query parameters only. NO TIMEOUT — see the doc comment. resp, err := http.Get(reqURL) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) @@ -143,6 +168,12 @@ func fetchAllAccountsFromPDS(pdsURL string) ([]string, error) { return allDIDs, nil } +// fetchVotesFromPDS reads one repo's vote records. Same host, same reasoning, +// same separate defect as fetchAllAccountsFromPDS — see its doc comment. +// +// `did` comes from the PDS's own listRepos answer and is url.QueryEscape'd into +// a QUERY PARAMETER. It never reaches the host, so it cannot redirect this +// request anywhere: the address is still PDS_URL and nothing else. func fetchVotesFromPDS(pdsURL, did string) ([]Record, error) { var allRecords []Record cursor := "" @@ -155,6 +186,7 @@ func fetchVotesFromPDS(pdsURL, did string) ([]Record, error) { reqURL += "&cursor=" + url.QueryEscape(cursor) } + // coves:allow-bare-client: the host is PDS_URL from the environment; the DID is an escaped query parameter. NO TIMEOUT — see fetchAllAccountsFromPDS. resp, err := http.Get(reqURL) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) diff --git a/cmd/rematerialize-posts/allow_private_hosts_test.go b/cmd/rematerialize-posts/allow_private_hosts_test.go new file mode 100644 index 0000000..00ca6cb --- /dev/null +++ b/cmd/rematerialize-posts/allow_private_hosts_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "testing" + + "Coves/internal/config" + + "github.com/stretchr/testify/assert" +) + +// The single expression that decides whether this tool's SSRF guard is on. +// +// The cutover tool is the one binary in the tree that DELETES production posts, +// and every outbound call it makes is aimed at an address that came out of the +// database: a community's pds_url column, an author repo's serviceEndpoint. Five +// separate expressions used to spell out whether the guard covering those dials +// was on, and no test evaluated any of them — the same defect cmd/server carried +// at thirteen sites. `.env.ci:140` sets IS_DEV_ENV=true, so the merge gate ran +// the permissive branch at all five. + +// TestAllowPrivateHosts_IsOffUnderAProductionConfig is the binding contract. +// The assert.False is the assertion that fails on inversion. +// +// IS_DEV_ENV is set to the dev value on purpose: the gate must be a property of +// the parsed configuration, never an ambient read. +func TestAllowPrivateHosts_IsOffUnderAProductionConfig(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "true") // the guard must not be ambient + + assert.False(t, allowPrivateHosts(&config.Config{IsDevEnv: false}), + "the SSRF hatch is OPEN under a production config. With this one expression inverted, the "+ + "blob fetch and PDS upload, both community PDS client paths, the blob copy the "+ + "Rematerializer makes from an author repo's serviceEndpoint, and the OAuth client all "+ + "dial private addresses during a run that irreversibly deletes user data — and `make ci` "+ + "cannot see it, because .env.ci:140 sets IS_DEV_ENV=true and runs the other branch") +} + +// TestAllowPrivateHosts_IsOnUnderADevConfig is the falsifiability twin: an +// accessor hardcoded to false passes the test above while making the tool +// unusable against a local stack, whose PDS is on loopback. +func TestAllowPrivateHosts_IsOnUnderADevConfig(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "false") // the hatch must not be ambient either + + assert.True(t, allowPrivateHosts(&config.Config{IsDevEnv: true}), + "the SSRF hatch is SHUT under a dev config, so a rehearsal run against a local stack cannot "+ + "reach its own PDS — which is on loopback, exactly the address class the guard refuses. "+ + "-dry-run is the only way this tool is ever practised before it deletes anything") +} diff --git a/cmd/rematerialize-posts/main.go b/cmd/rematerialize-posts/main.go index 8fa2b15..78f9cbf 100644 --- a/cmd/rematerialize-posts/main.go +++ b/cmd/rematerialize-posts/main.go @@ -163,8 +163,20 @@ func main() { log.Fatalf("rematerialize-posts: %v", err) } - blobService := blobs.NewBlobService(cfg.PDS.URL) - provisioner := communities.NewPDSAccountProvisioner(cfg.Instance.Domain, cfg.PDS.URL) + // Same gate as cmd/server: the blob service's fetch and PDS upload are both + // guarded, and the hatch opens only in dev, where cfg.PDS.URL is a loopback + // address the guard would otherwise refuse. The gate is allowPrivateHosts — + // the IS_DEV_ENV environment variable, not one of this tool's own flags — and + // every other guarded construction here goes through the same accessor, + // including the OAuth client's AllowPrivateIPs. + blobService := blobs.NewBlobService(cfg.PDS.URL, blobs.PrivateHostOptions(allowPrivateHosts(cfg))...) + + // The same gate again, on the xrpc calls this package makes to a community's + // PDS. They used to leave xrpc.Client.Client nil, which makes indigo + // substitute an unguarded util.RobustHTTPClient(). + communityPDSOptions := communities.PrivateHostOptions(allowPrivateHosts(cfg)) + provisioner := communities.NewPDSAccountProvisioner( + cfg.Instance.Domain, cfg.PDS.URL, communityPDSOptions...) communityService := communities.NewCommunityService( postgresRepo.NewCommunityRepository(db), cfg.PDS.URL, @@ -173,13 +185,21 @@ func main() { provisioner, oauthClient, blobService, + communityPDSOptions..., ) // The DIRECT acceptance writer over the production community-repo factory — // the same credential-presence hosting test the acceptance engine uses. The // tool holds this writer and no decider, so it cannot re-run admission. The // same factory is handed to the Rematerializer for the acceptance READ-BACK. - repoFactory := posts.NewCommunityRepoFactory(communityService) + // + // pdsClientOptions is this tool's SSRF dev gate, resolved once and shared by + // both PDS-client paths below: every community PDS URL they dial comes from + // the database, and allowPrivateHosts is what lets a local run reach a + // loopback PDS without opening the guard anywhere else. + pdsClientOptions := pds.PrivateHostOptions(allowPrivateHosts(cfg)) + + repoFactory := posts.NewCommunityRepoFactory(communityService, pdsClientOptions...) writer := posts.NewCommunityRecordWriter(repoFactory, time.Now) authorFactory := posts.NewAuthorRepoFactory(oauthClient.ClientApp, aggregators.DefaultSessionID) @@ -187,17 +207,28 @@ func main() { source := &realLegacySource{ communityFilter: *communityFilter, hostedDIDs: postgresRepo.NewHostedCommunityQuery(db).HostedCommunityDIDs, - openRepo: communityRepoOpener(communityService), + openRepo: communityRepoOpener(communityService, pdsClientOptions...), } progress := newProgressLogger() tool := &posts.Rematerializer{ - Source: source, - Ledger: ledger, - AuthorRepos: authorFactory, - Acceptances: writer, - CommunityRepos: repoFactory, - CommunityScope: *communityFilter, + Source: source, + Ledger: ledger, + AuthorRepos: authorFactory, + Acceptances: writer, + CommunityRepos: repoFactory, + CommunityScope: *communityFilter, + // The blob copy's dev gate, decided HERE rather than inside the state + // machine, exactly as blobs.PrivateHostOptions is decided above. + // + // The Rematerializer's own fallback is guarded with no way to open it, + // because it is what every caller that expressed no opinion degrades to. + // This is the caller that has an opinion: the host it copies from is the + // author repo's serviceEndpoint, which in a local stack is a loopback PDS — + // the address the guard exists to refuse. Passing allowPrivateHosts keeps a + // dev run working and leaves production on the guarded path, and it puts the + // decision where an operator reading the wiring can see which one they got. + Blobs: posts.DefaultRematerializeBlobClient(allowPrivateHosts(cfg)), Progress: progress.log, PerRecordTimeout: perRecordTimeout, AbortOnFallback: !*acceptFallbacks, @@ -469,7 +500,12 @@ func (s *realLegacySource) hostedCommunityDIDs(ctx context.Context) ([]string, e // communityRepoOpener builds the production repo opener: a full PDS client bound // to one community's repo, over freshly-renewed stored credentials. -func communityRepoOpener(creds posts.CommunityCredentialSource) func(context.Context, string) (repoClient, error) { +// +// The options carry the SSRF dev gate, and passing none leaves the client +// guarded: fresh.PDSURL is a per-community database column, so this tool dials +// an address that is data. A local dev run against a loopback PDS is what the +// caller's pds.PrivateHostOptions(allowPrivateHosts(cfg)) opens. +func communityRepoOpener(creds posts.CommunityCredentialSource, opts ...pds.ClientOption) func(context.Context, string) (repoClient, error) { return func(ctx context.Context, did string) (repoClient, error) { community, err := creds.GetByDID(ctx, did) if err != nil { @@ -485,7 +521,7 @@ func communityRepoOpener(creds posts.CommunityCredentialSource) func(context.Con if fresh == nil || fresh.PDSAccessToken == "" { return nil, fmt.Errorf("renewing the credentials of %s: no access token came back", did) } - client, err := pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken) + client, err := pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken, opts...) if err != nil { return nil, err } @@ -700,6 +736,25 @@ func takeAdvisoryLock(ctx context.Context, db *sql.DB) (release func(), err erro }, nil } +// allowPrivateHosts is THE ONE CONVERSION from configuration to SSRF policy in +// this binary, the counterpart of cmd/server's method of the same name. +// +// Every hatch-bearing call site in this tool calls it rather than reading +// cfg.IsDevEnv, so there is exactly one expression whose polarity has to be +// right and exactly one place a test has to reach. The five spellings it +// replaced were evaluated by no test at all, and `.env.ci:140` sets +// IS_DEV_ENV=true, so the merge gate ran the permissive branch at every one of +// them — inverting any single site left `make ci` green. +// +// It takes cfg rather than reading the environment for the reason blobs and +// pds both document: an ambient read makes the guarded branch untestable +// alongside t.Parallel, and it opens every construction in the process at once. +// +// buildOAuthClient's DevMode is deliberately NOT routed through here. It is an +// AUTHENTICATION gate over the same input, and one accessor answering two +// different questions is one that gets changed for the wrong reason. +func allowPrivateHosts(cfg *config.Config) bool { return cfg.IsDevEnv } + // openDatabase opens the AppView Postgres the ledger and community catalogue // live in, with the pool bounds from config. func openDatabase(cfg *config.Config) (*sql.DB, error) { @@ -726,7 +781,7 @@ func buildOAuthClient(cfg *config.Config, db *sql.DB) (*oauth.OAuthClient, error SealSecret: cfg.OAuth.SealSecret, Scopes: oauthScopes(), DevMode: cfg.IsDevEnv, - AllowPrivateIPs: cfg.IsDevEnv, + AllowPrivateIPs: allowPrivateHosts(cfg), PLCURL: cfg.Identity.PLCURL, PDSURL: cfg.PDS.URL, ClientPrivateKeyMultibase: cfg.OAuth.ClientPrivateKeyMultibase, diff --git a/cmd/server/allow_private_hosts_test.go b/cmd/server/allow_private_hosts_test.go new file mode 100644 index 0000000..cd94de9 --- /dev/null +++ b/cmd/server/allow_private_hosts_test.go @@ -0,0 +1,81 @@ +package main + +import ( + "testing" + + "Coves/internal/config" + + "github.com/stretchr/testify/assert" +) + +// The single expression that decides whether this binary's SSRF guard is on. +// +// # WHY THIS FILE EXISTS +// +// Thirteen separate expressions in cmd/server used to spell out the same +// decision — `identity.PrivateHostOptions(a.cfg.IsDevEnv)`, +// `AllowPrivateIPs: a.cfg.IsDevEnv`, `pds.PrivateHostOptions(a.cfg.IsDevEnv)` +// and ten more — and NO TEST ANYWHERE EVALUATED ANY OF THEM. Inverting any one +// of them left `go test ./cmd/server/` green and `make ssrf-audit` at zero, +// because the audit's rule 3 matches only literal hatch spellings +// (`WithPrivateHostsAllowed()`, `PrivateHostOptions(true)`) and a wiring line +// deriving the value from config matches nothing. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// T0+T1+T2 — takes the PERMISSIVE branch at every one of those sites. A green +// merge gate was therefore compatible with every site being unguarded in +// production. Collapsed into application.allowPrivateHosts, there is one +// polarity to get right and this is the one place that ever evaluates it. +// +// # WHY BOTH DIRECTIONS +// +// The guarded direction is the security property. The hatched direction is what +// keeps the guarded one falsifiable: an accessor hardcoded to `false` satisfies +// every assertion below that matters to production while breaking every +// developer's local stack — a dev-breaking but test-passing mutation that only +// the second test can see. + +// TestApplication_AllowPrivateHosts_IsOffUnderAProductionConfig is the binding +// contract, and the assert.False is the assertion that fails on inversion. +// +// The environment is set to the DEV value on purpose. The gate must be a +// property of the parsed configuration and not an ambient read: this same +// process builds productionPLCResolver, which has to stay guarded even in dev, +// so an accessor that consulted os.Getenv would open every construction in the +// binary at once — including the ones that must never open. +func TestApplication_AllowPrivateHosts_IsOffUnderAProductionConfig(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "true") // the guard must not be ambient + + a := &application{cfg: &config.Config{IsDevEnv: false}} + + assert.False(t, a.allowPrivateHosts(), + "the SSRF hatch is OPEN under a production config. Every guarded call site in this binary "+ + "derives from this one expression, so with it inverted the image proxy, the identity "+ + "resolver, the unfurl fetch, the blob fetch and upload, both community PDS client paths, "+ + "the profile backfill, the aggregator registration fetch, the community consumer's "+ + ".well-known fetch, the direct post fetch, the OAuth client and the service-JWT directory "+ + "all dial private addresses in production — and `make ci` cannot see it, because "+ + ".env.ci:140 sets IS_DEV_ENV=true and runs the other branch") +} + +// TestApplication_AllowPrivateHosts_IsOnUnderADevConfig is the falsifiability +// twin. +// +// Without it, `func (a *application) allowPrivateHosts() bool { return false }` +// passes the test above. That mutation is not a security bug — it is the +// opposite — but it silently breaks every local stack: the dev PDS, the dev PLC +// and every fixture origin are on loopback, which is precisely what the guard +// refuses. A gate that can only ever be shut is not a gate. +func TestApplication_AllowPrivateHosts_IsOnUnderADevConfig(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "false") // the hatch must not be ambient either + + a := &application{cfg: &config.Config{IsDevEnv: true}} + + assert.True(t, a.allowPrivateHosts(), + "the SSRF hatch is SHUT under a dev config, so nothing in a local stack is reachable: the "+ + "dev PDS, the dev PLC and every httptest fixture listen on loopback, which is exactly the "+ + "address class the guard exists to refuse. An accessor that returns false unconditionally "+ + "passes the production test above while breaking every developer's machine") +} diff --git a/cmd/server/consumers.go b/cmd/server/consumers.go index 2e29419..65f3145 100644 --- a/cmd/server/consumers.go +++ b/cmd/server/consumers.go @@ -154,11 +154,18 @@ func (a *application) registerFeedConsumers() []feedConsumer { if a.cfg.Instance.SkipDIDWebVerification { slog.Warn("did:web domain verification is DISABLED; this must never be set in production") } + // The SSRF hatch is open only in dev, where a developer's own instance is + // what the .well-known DID document is fetched from. In production that + // fetch dials whatever domain a community record published by anyone on the + // federated network names, from inside this AppView's own network. + communityOpts := append( + []jetstream.CommunityConsumerOption{jetstream.WithCommunityRevGate(a.revGate)}, + jetstream.PrivateHostOptions(a.allowPrivateHosts())...) consumers = append(consumers, feedConsumer{ name: jetstream.ConsumerCommunities, handler: jetstream.NewCommunityEventConsumer( a.communityRepo, a.cfg.Instance.DID, a.cfg.Instance.SkipDIDWebVerification, - a.identityResolver, jetstream.WithCommunityRevGate(a.revGate)), + a.identityResolver, communityOpts...), }) // Posts, in both shapes, plus the community records that decide about @@ -168,15 +175,27 @@ func (a *application) registerFeedConsumers() []feedConsumer { // // The direct fetcher is what makes acceptance-before-post converge without // full relay coverage (PRD §5.4). It dials a PDS named by a DID document - // anyone can publish, so its SSRF guard stays on in production and the - // stood-down constructor is reachable only under IS_DEV_ENV — where the - // hermetic stack's PDS is a private address the guard would otherwise - // refuse. - postFetcher := jetstream.NewDirectPostFetcher(a.identityResolver) - if a.cfg.IsDevEnv { + // anyone can publish, so its SSRF guard stays on in production and the hatch + // opens only under IS_DEV_ENV — where the hermetic stack's PDS is a private + // address the guard would otherwise refuse. + // + // The decision goes through jetstream.PrivatePostFetcherOptions rather than + // the `if` that used to stand here, for the same reason the community + // consumer's gate does nineteen lines above: `.env.ci:140` sets + // IS_DEV_ENV=true, so `make ci` takes the permissive branch and an inline + // conditional in wiring is reachable only by standing up this wiring with a + // production config, which nothing in this tree does. As a pure function the + // branch production actually runs is testable in T0. + // + // The warning stays here, because it is about this process rather than about + // the option — a helper that logged would fire once per test that builds a + // hatched fetcher, and this line has to mean "this server is running + // unguarded". + if a.allowPrivateHosts() { slog.Warn("direct post fetch has SSRF protection DISABLED (IS_DEV_ENV); this must never be set in production") - postFetcher = jetstream.NewDevDirectPostFetcher(a.identityResolver) } + postFetcher := jetstream.NewDirectPostFetcher(a.identityResolver, + jetstream.PrivatePostFetcherOptions(a.allowPrivateHosts())...) consumers = append(consumers, feedConsumer{ name: jetstream.ConsumerPosts, handler: jetstream.NewPostEventConsumer(a.postRepo, a.communityRepo, a.userService, a.db, diff --git a/cmd/server/pds.go b/cmd/server/pds.go index 61d02f4..3ef430e 100644 --- a/cmd/server/pds.go +++ b/cmd/server/pds.go @@ -58,7 +58,7 @@ func authenticateWithPDS(ctx context.Context, pdsURL, handle, password string) ( } req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := http.DefaultClient.Do(req) // coves:allow-bare-client: pdsURL is a.cfg.PDS.URL, operator config; this is the AppView authenticating to its own PDS at startup if err != nil { return "", fmt.Errorf("failed to call PDS: %w", err) } diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 5c5d21d..5f99872 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -7,6 +7,7 @@ import ( "net/http" "time" + "Coves/internal/api/handlers/aggregator" commentsAPI "Coves/internal/api/handlers/comments" "github.com/go-chi/chi/v5" @@ -95,8 +96,12 @@ func registerXRPCRoutes(r chi.Router, app *application) { routes.RegisterActorRoutes(r, app.postService, app.userService, app.voteService, app.blueskyService, app.commentService, app.authMiddleware) + // The registration handler fetches .well-known/atproto-did from a domain an + // unauthenticated caller supplies, so its client is guarded; the hatch is + // for a dev stack whose fixture domains resolve to the machine itself. routes.RegisterAggregatorRoutes(r, app.aggregatorService, app.communityService, - app.userService, app.identityResolver) + app.userService, app.identityResolver, + aggregator.PrivateHostOptions(app.allowPrivateHosts())...) routes.RegisterAggregatorAPIKeyRoutes(r, app.authMiddleware, app.apiKeyService, app.aggregatorService) registerCommentQueryRoute(r, app) diff --git a/cmd/server/service_jwt_identity_test.go b/cmd/server/service_jwt_identity_test.go new file mode 100644 index 0000000..3e912cc --- /dev/null +++ b/cmd/server/service_jwt_identity_test.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The identity directory behind service-JWT validation. +// +// # WHY THIS IS THE MOST CREDENTIAL-FREE SITE OF THE NINE +// +// buildDualAuth hands this directory to indigo's ServiceAuthValidator, which +// resolves a JWT's `iss` DID in order to find the key that would verify the +// signature. That resolution happens BEFORE the credential is trusted, and it +// has to — there is nothing to verify against until the DID document is in +// hand. So an attacker sends a syntactically valid JWT claiming any `iss` they +// like, and the AppView fetches whatever that DID document points at. No token +// needs to be valid. No account needs to exist. Where the image proxy at least +// required a did:plc to have been minted, this one requires nothing at all. +// +// # WHY A PURE FUNCTION RATHER THAN AN `if` IN buildDualAuth +// +// buildDualAuth is a method on *application, so reaching it means standing up +// wiring with a config, a database and an OAuth client — which nothing in this +// tree does. `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` would take the +// permissive branch even if it could. Extracted, the decision is testable +// without any of that, and T0 becomes the one place in the repository where the +// branch production actually runs is evaluated at all. Do not inline it back. +// +// # WHY REACHABILITY IS ASSERTED +// +// Mutation testing produced a guard that classified correctly, emitted +// a byte-identical message, and refused the request AFTER delivering it. For a +// destination a stranger named, the packet leaving IS the SSRF. Both directions +// below stand up a real listener and count what reached it. + +// serviceJWTTestDID is a syntactically valid did:plc. syntax.ParseDID runs +// before any client is touched, so a malformed one would make these tests pass +// for the wrong reason. +const serviceJWTTestDID = "did:plc:z72i7hdynmk6r22z27h6tvur" + +// countingPLC is a PLC directory that answers a DID lookup and records whether +// anything ever reached it. It listens on loopback, which is the address class +// the guard exists to refuse, so its counter doubles as the assertion. +type countingPLC struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPLC(t *testing.T) *countingPLC { + t.Helper() + + plc := &countingPLC{} + plc.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + plc.requests.Add(1) + w.Header().Set("Content-Type", "application/json") + + // No alsoKnownAs: indigo verifies a DECLARED handle bidirectionally over + // DNS and HTTPS, and that lookup would leave the machine. With none + // declared it marks the handle invalid and returns, so this stays local. + _, _ = w.Write([]byte(`{"id":"` + serviceJWTTestDID + `","service":[{"id":"#atproto_pds",` + + `"type":"AtprotoPersonalDataServer","serviceEndpoint":"https://pds.example.invalid"}]}`)) + })) + t.Cleanup(plc.server.Close) + return plc +} + +func resolveThroughDirectory(t *testing.T, plc *countingPLC, allowPrivateHosts bool) error { + t.Helper() + + dir := serviceJWTIdentityDirectory(plc.server.URL, allowPrivateHosts) + require.NotNil(t, dir, "the directory must be built") + + did, err := syntax.ParseDID(serviceJWTTestDID) + require.NoError(t, err, "the test's own DID must parse, or the client is never reached") + + _, err = dir.ResolveDID(context.Background(), did) + return err +} + +// TestServiceJWTIdentityDirectory_GuardedRefusesAPrivatePLCWithoutReachingIt is +// the binding contract: false is the production branch, and it must refuse. +func TestServiceJWTIdentityDirectory_GuardedRefusesAPrivatePLCWithoutReachingIt(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + err := resolveThroughDirectory(t, plc, false) + + require.Errorf(t, err, + "the service-JWT directory resolved a DID against a PLC on loopback with the hatch shut. This "+ + "is the resolution an unauthenticated caller triggers by sending a JWT with an `iss` of "+ + "their choosing, before anything about that JWT has been verified") + assert.Containsf(t, err.Error(), "SSRF blocked", + "the refusal must be the guard's and must say so: indigo wraps this in its own resolution "+ + "error, so the sentence is what an operator has to tell a blocked address from a PLC that "+ + "is merely down; got: %v", err) + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times. The refusal happened, but it happened AFTER the request "+ + "was delivered — which prevents none of the SSRF", plc.requests.Load()) +} + +// TestServiceJWTIdentityDirectory_HatchedReachesAPrivatePLC is the other +// direction. a.cfg.Identity.PLCURL is a local PLC in dev, so this half is what +// keeps service-JWT auth working on a developer's machine — and unlike +// productionPLCResolver, this site genuinely needs it. +func TestServiceJWTIdentityDirectory_HatchedReachesAPrivatePLC(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + err := resolveThroughDirectory(t, plc, true) + + require.NoErrorf(t, err, + "the service-JWT directory could not reach a local PLC with the hatch open. In dev, "+ + "a.cfg.Identity.PLCURL IS a loopback PLC, so without this every aggregator's service JWT "+ + "fails to validate locally; got: %v", err) + assert.Equalf(t, int64(1), plc.requests.Load(), + "the PLC listener was reached %d times rather than once", plc.requests.Load()) +} + +// TestServiceJWTIdentityDirectory_PreservesItsOwnTimeout guards the setting the +// shared client would otherwise swallow. +// +// identityHTTPTimeout is 10s and bounds DID fetches made while validating a +// credential nobody has authenticated yet — so it is a denial-of-service bound +// as much as a latency one. oauth.NewSSRFSafeHTTPClient ships 15s, and silently +// re-timing this as a side effect of an SSRF fix is a second change wearing the +// first one's clothes. +// +// It is also where an ORDERING mistake would show up. indigo's BaseDirectory +// holds an http.Client by VALUE, so the conversion has to dereference the +// pointer the shared constructor returns — and anything set on that client after +// the copy is taken is lost. (The copy itself is safe: an http.Client's +// mutex-bearing state lives behind the Transport pointer.) +func TestServiceJWTIdentityDirectory_PreservesItsOwnTimeout(t *testing.T) { + t.Parallel() + + for name, allowPrivate := range map[string]bool{"guarded": false, "hatched": true} { + t.Run(name, func(t *testing.T) { + t.Parallel() + + dir := serviceJWTIdentityDirectory("https://plc.example.invalid", allowPrivate) + require.NotNil(t, dir) + + assert.Equalf(t, identityHTTPTimeout, dir.HTTPClient.Timeout, + "the %s directory runs on a %v timeout instead of identityHTTPTimeout (%v). This client "+ + "is reached by unauthenticated callers, so its ceiling is what bounds how long one of "+ + "them can hold a request open", name, dir.HTTPClient.Timeout, identityHTTPTimeout) + }) + } +} + +// TestServiceJWTIdentityDirectory_KeepsThePLCURLItWasGiven pins the other half +// of the construction, because a guard that quietly changed where this +// directory points would be a worse bug than the one being fixed: the PLC URL +// is what decides which network's DIDs this instance believes. +func TestServiceJWTIdentityDirectory_KeepsThePLCURLItWasGiven(t *testing.T) { + t.Parallel() + + const plcURL = "https://plc.example.invalid" + + dir := serviceJWTIdentityDirectory(plcURL, false) + require.NotNil(t, dir) + + assert.Equal(t, plcURL, dir.PLCURL, + "the directory must resolve against the configured PLC directory and nothing else") +} + +// TestServiceJWTIdentityDirectory_GuardIsNotAmbient states the invariant cycle 2 +// found the hard way, at this site too. +// +// The decision belongs to the CALLER — buildDualAuth passes a.cfg.IsDevEnv — +// and must never become a read of the environment inside the constructor. This +// process has IS_DEV_ENV set to true and the guarded construction must still +// refuse, because cmd/server also builds a resolver aimed at the public +// directory that has to stay guarded in dev. +func TestServiceJWTIdentityDirectory_GuardIsNotAmbient(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "true") + + plc := newCountingPLC(t) + + err := resolveThroughDirectory(t, plc, false) + + require.Error(t, err, + "the guarded directory reached a loopback PLC while IS_DEV_ENV was true. The hatch must be a "+ + "property of the argument, not of the environment: an ambient read opens every other "+ + "construction in this process at the same time") + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times with IS_DEV_ENV=true", plc.requests.Load()) +} diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index a9eb429..12ac608 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -5,6 +5,7 @@ import ( "Coves/internal/atproto/identity" "Coves/internal/atproto/jetstream" "Coves/internal/atproto/oauth" + "Coves/internal/atproto/pds" "Coves/internal/config" "Coves/internal/core/adminreports" "Coves/internal/core/aggregators" @@ -27,7 +28,6 @@ import ( "database/sql" "fmt" "log/slog" - "net/http" "strings" "sync" "time" @@ -46,9 +46,11 @@ const ( // net for a PLC directory that accepts connections but never answers. identityHTTPTimeout = 10 * time.Second - // profileBackfillTimeout bounds the best-effort fetch of a user's - // social.coves.actor.profile from their PDS during indexing. - profileBackfillTimeout = 10 * time.Second + // (The profile backfill's timeout used to be declared here as well. It now + // lives with the client that carries it, in users.NewProfileBackfillClient, + // because that client also has to re-apply it over the shared SSRF client's + // own ceiling — and a deadline stated in two packages is one that can be + // changed in one of them.) // unfurlTimeout bounds fetching and parsing a link preview target. unfurlTimeout = 10 * time.Second @@ -90,8 +92,8 @@ type application struct { // Repositories reused outside their own service (Jetstream consumers, // route options). - userRepo users.UserRepository - communityRepo communities.Repository + userRepo users.UserRepository + communityRepo communities.Repository // postRepo is held as the CONCRETE repository rather than posts.Repository: // the comment service's PostReader requires the admission-aware // VisibleHeaderView as well, and storing the narrower interface here would @@ -196,15 +198,48 @@ func (a *application) Close() { }) } +// allowPrivateHosts is THE ONE CONVERSION from configuration to SSRF policy in +// this binary. Every hatch-bearing call site below calls this rather than +// reading a.cfg.IsDevEnv, so there is exactly one expression whose polarity has +// to be right, and exactly one place a test has to reach. +// +// It exists because the polarity was previously spelled out thirteen times in +// this package and no test anywhere evaluated any of them. `.env.ci:140` sets +// IS_DEV_ENV=true, so `make ci` — the hermetic merge gate — takes the PERMISSIVE +// branch at every one of those sites; inverting any single spelling left the +// whole gate green. Collapsed here, the production branch is evaluated by +// TestApplication_AllowPrivateHosts_IsOffUnderAProductionConfig, which is the +// only place in the repository that ever runs it. +// +// It deliberately does NOT read the environment. cfg.IsDevEnv is parsed once at +// boot, and an ambient read here would open every construction in this process +// at once — including productionPLCResolver, which has to stay guarded in dev. +// +// NOT every a.cfg.IsDevEnv read belongs here. buildAuth's OAuthConfig.DevMode is +// an AUTHENTICATION gate (cookie Secure flags, redirect-URI validation) that +// happens to share the same input, and the two slog severity switches are about +// what an operator is told rather than about what may be dialled. Folding those +// in would make one accessor answer three different questions. +func (a *application) allowPrivateHosts() bool { return a.cfg.IsDevEnv } + func (a *application) buildIdentity() { - identityConfig := identity.DefaultConfig() + // The hatch, and ONLY here. This resolver follows + // cfg.Identity.ResolverPLCURL, which in dev is a PLC on the developer's own + // machine. productionPLCResolver below is built from the same DefaultConfig + // and must NOT get it — the two resolvers need opposite answers in the same + // process, which is why this is an argument rather than something the + // identity package works out for itself. + identityConfig := identity.DefaultConfig(identity.PrivateHostOptions(a.allowPrivateHosts())...) identityConfig.PLCURL = a.cfg.Identity.ResolverPLCURL if a.cfg.Identity.CacheTTL > 0 { identityConfig.CacheTTL = a.cfg.Identity.CacheTTL } a.identityResolver = identity.NewResolver(a.db, identityConfig) - if a.cfg.IsDevEnv { + // The same expression the hatch above was derived from, not a second read of + // the same config field: a log line describing the SSRF gate has to be unable + // to disagree with the gate itself. + if a.allowPrivateHosts() { slog.Warn("dev mode: identity resolver is using a local PLC directory", "plc_url", identityConfig.PLCURL) } else { @@ -226,7 +261,7 @@ func (a *application) buildAuth() error { // Private IPs are only resolvable in dev, where the PDS and PLC run // on localhost. In production this must stay off: it is what stops // OAuth from being pointed at an internal address. - AllowPrivateIPs: a.cfg.IsDevEnv, + AllowPrivateIPs: a.allowPrivateHosts(), PLCURL: a.cfg.Identity.PLCURL, PDSURL: a.cfg.PDS.URL, // Setting both upgrades this to a confidential client, lifting the @@ -301,7 +336,11 @@ func (a *application) buildServices(ctx context.Context) error { a.cfg.PDS.URL, turnstileVerifier, a.cfg.PDS.AdminPassword, - users.WithProfileBackfill(&http.Client{Timeout: profileBackfillTimeout}), + // The backfill dials a PDS URL taken from the indexed user's record — + // firehose-discovered users bring that value from another instance — so + // the guard stays on in production and the hatch opens only in dev, + // where every PDS a developer indexes against is on loopback. + users.WithProfileBackfill(users.NewProfileBackfillClient(a.allowPrivateHosts())), users.WithInstanceDomain(a.cfg.Instance.Domain), ) @@ -316,17 +355,34 @@ func (a *application) buildServices(ctx context.Context) error { // the fetch: an environment read at the call site would make the guarded // branch untestable alongside t.Parallel and hide the most consequential // input to a security decision from the place that makes it. - blobOptions := []blobs.BlobServiceOption{} - if a.cfg.IsDevEnv { - slog.Warn("dev mode: the blob fetch SSRF guard is disabled; " + - "remote image URLs may resolve to private addresses") - blobOptions = append(blobOptions, blobs.WithPrivateHostsAllowed()) + // + // The decision goes through blobs.PrivateHostOptions rather than an inline + // `if` here, for the reason that helper documents: `.env.ci:140` sets + // IS_DEV_ENV=true, so `make ci` takes the permissive branch, and an inline + // conditional in wiring is reachable only by standing up this wiring with a + // production config — which nothing in this tree does. As a pure function the + // branch production actually runs becomes testable. The warning stays here, + // because it is about this process and not about the option — and it reads the + // SAME accessor the option is derived from, so it cannot drift into describing + // a gate state that is not the one in force. + if a.allowPrivateHosts() { + slog.Warn("dev mode: the blob SSRF guard is disabled on both the remote fetch and the " + + "PDS upload; remote image URLs and PDS URLs may resolve to private addresses") } - blobService := blobs.NewBlobService(a.cfg.PDS.URL, blobOptions...) + blobService := blobs.NewBlobService(a.cfg.PDS.URL, blobs.PrivateHostOptions(a.allowPrivateHosts())...) // V2.0: the PDS generates and manages community DIDs and keys entirely; // Coves performs no cryptography of its own here. - provisioner := communities.NewPDSAccountProvisioner(a.cfg.Instance.Domain, a.cfg.PDS.URL) + // + // The same gate as the blob service above, on the xrpc calls this package + // makes to a community's PDS: account provisioning, token refresh, and the + // password re-auth that carries the community's cleartext password. All three + // used to leave xrpc.Client.Client nil, which makes indigo substitute an + // unguarded util.RobustHTTPClient(). In dev cfg.PDS.URL is a loopback address + // the guard would otherwise refuse. + communityPDSOptions := communities.PrivateHostOptions(a.allowPrivateHosts()) + provisioner := communities.NewPDSAccountProvisioner( + a.cfg.Instance.Domain, a.cfg.PDS.URL, communityPDSOptions...) a.communityService = communities.NewCommunityService( a.communityRepo, a.cfg.PDS.URL, @@ -335,6 +391,7 @@ func (a *application) buildServices(ctx context.Context) error { provisioner, a.oauthClient, blobService, + communityPDSOptions..., ) a.authenticateInstanceWithPDS(ctx) @@ -343,17 +400,34 @@ func (a *application) buildServices(ctx context.Context) error { a.buildDualAuth() - unfurlService := unfurl.NewService( - unfurl.NewRepository(a.db), + // The SSRF hatch is open only in dev, where the links a developer pastes and + // the fixtures the test suite serves both live on the developer's own + // machine. In production this fetch dials whatever address a link pasted into + // a post resolves to — and unlike every other guarded fetch in this tree, it + // hands the response's CONTENT back, so an internal endpoint's page title + // would land in the unfurl cache and then in the post itself. + unfurlOptions := append([]unfurl.ServiceOption{ unfurl.WithTimeout(unfurlTimeout), unfurl.WithUserAgent(unfurlUserAgent), unfurl.WithCacheTTL(unfurlCacheTTL), - ) + }, unfurl.PrivateHostOptions(a.allowPrivateHosts())...) + unfurlService := unfurl.NewService(unfurl.NewRepository(a.db), unfurlOptions...) // Quoted Bluesky posts reference real handles on the production atProto // network, which the dev/test PLC cannot resolve. This resolver is - // therefore always pointed at plc.directory — and is safe to use in dev - // because identity.Resolver only ever issues HTTP GETs. + // therefore always pointed at plc.directory, in every environment. + // + // NO SSRF HATCH, IN DEV EITHER — deliberately, and unlike a.identityResolver + // above. It is aimed at the public directory, so it has no reason to dial a + // private address; and dev is precisely the environment where the loopback + // it would otherwise be allowed to dial is a real PLC, a real Postgres and + // a real PDS. DefaultConfig() with no options is the guarded construction, + // which is why this line looks like it is doing nothing. + // + // (An earlier comment justified this resolver as "safe to use in dev + // because identity.Resolver only ever issues HTTP GETs". That reasoning is + // backwards: only-GETs IS the SSRF primitive. The image-proxy port scanner + // closed in ff901a5 was GET-only.) productionPLCConfig := identity.DefaultConfig() productionPLCConfig.PLCURL = "https://plc.directory" productionPLCResolver := identity.NewResolver(a.db, productionPLCConfig) @@ -384,6 +458,11 @@ func (a *application) buildServices(ctx context.Context) error { a.postRepo, a.communityService, a.aggregatorService, blobService, unfurlService, a.blueskyService, a.cfg.PDS.URL, posts.WithBlockChecker(a.userBlockRepo), + // The SSRF dev gate for the clients this service builds against a + // COMMUNITY's repo, whose PDS URL is a database column. deleteCommunityPost + // is the path that needs it here; the withdrawer's factory carries its own, + // set in buildAcceptanceEngine. + posts.WithPDSClientOptions(pds.PrivateHostOptions(a.allowPrivateHosts())...), // The AUTHOR's own credentials: a browser session when there is one, // and an aggregator's stored tokens when there is not (§4.2 step 3). posts.WithAuthorRepoFactory( @@ -472,7 +551,7 @@ func (a *application) buildDeciderDeps() posts.DeciderDeps { // short-circuits and admits UNLIMITED posts — the same admissions repo the // ingestion consumer writes and the engine settles, so the rows counted as // admitted are the rows the quota meters. - Admissions: a.admissionRepo, + Admissions: a.admissionRepo, Aggregators: a.aggregatorService, Policy: posts.AdmissionPolicy{ Ledger: postgresRepo.NewSubmissionLedger(a.db), @@ -492,7 +571,8 @@ func (a *application) buildDeciderDeps() posts.DeciderDeps { } func (a *application) buildAcceptanceEngine() *posts.AcceptanceEngine { - repoFactory := posts.NewCommunityRepoFactory(a.communityService) + repoFactory := posts.NewCommunityRepoFactory(a.communityService, + pds.PrivateHostOptions(a.allowPrivateHosts())...) a.communityWriter = posts.NewCommunityRecordWriter(repoFactory, time.Now) decider := posts.NewAdmissionEngineDecider(a.buildDeciderDeps()) @@ -577,14 +657,61 @@ func adminReportAlertOptions() ([]adminreports.ServiceOption, error) { }, nil } +// serviceJWTIdentityDirectory builds the identity directory that service-JWT +// validation resolves an inbound `iss` DID through. +// +// # THE MOST CREDENTIAL-FREE FETCH IN THE PROCESS +// +// indigo's ServiceAuthValidator resolves a JWT's `iss` DID in order to find the +// key that would verify its signature — so the resolution happens BEFORE the +// credential is trusted, and it has to: there is nothing to verify against +// until the DID document is in hand. An attacker sends a syntactically valid +// JWT claiming any `iss` they like and the AppView fetches whatever that DID +// document points at. No token needs to be valid and no account needs to exist. +// +// # WHY A FUNCTION AND NOT AN `if` IN buildDualAuth +// +// buildDualAuth is a method on *application, so reaching it means standing up +// wiring with a config, a database and an OAuth client, which nothing in this +// tree does. `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` would take the +// permissive branch even if it could. Extracted, the decision is testable in T0 +// without any of that — and the gate must stay an ARGUMENT rather than an +// ambient read of the environment, because this same process builds +// productionPLCResolver, which has to stay guarded in dev. Do not inline it +// back. +// +// # ORDERING +// +// BaseDirectory.HTTPClient is a VALUE, so the shared constructor's pointer is +// dereferenced here — and the timeout has to be applied BEFORE the copy is +// taken, since anything set afterwards is set on a client nobody holds. +// +// The copy itself is safe, though not for the reason it is tempting to give: +// http.Client carries NO LOCK of its own — its four fields are a Transport +// interface, two funcs and a Duration — so `go vet`'s copylocks has nothing to +// say here and would not have caught a type that did. What makes it safe is +// that the copy shares rather than duplicates: both values hold the same +// *ssrfSafeTransport pointer, so the guard, the resolver and the connection +// pool behind it are one instance. A transport copied BY VALUE would be the +// bug, and it is not what this line does. +// +// identityHTTPTimeout (10s) survives the shared client's own +// 15s ceiling because it bounds a fetch an unauthenticated caller can trigger, +// which makes it a denial-of-service bound and not just a latency one. +func serviceJWTIdentityDirectory(plcURL string, allowPrivateHosts bool) *indigoidentity.BaseDirectory { + client := oauth.NewSSRFSafeHTTPClient(oauth.PrivateAddressOptions(allowPrivateHosts)...) + client.Timeout = identityHTTPTimeout + return &indigoidentity.BaseDirectory{ + PLCURL: plcURL, + HTTPClient: *client, + } +} + // buildDualAuth wires the middleware that accepts all three credential types: // sealed OAuth session tokens (users), PDS-signed service JWTs (aggregators), // and API keys (aggregator bots). func (a *application) buildDualAuth() { - identityDir := &indigoidentity.BaseDirectory{ - PLCURL: a.cfg.Identity.PLCURL, - HTTPClient: http.Client{Timeout: identityHTTPTimeout}, - } + identityDir := serviceJWTIdentityDirectory(a.cfg.Identity.PLCURL, a.allowPrivateHosts()) serviceValidator := &indigoauth.ServiceAuthValidator{ // The instance DID is the audience aggregator JWTs must be issued for. Audience: a.cfg.Instance.DID, @@ -701,7 +828,12 @@ func (a *application) buildImageProxy() error { service, err := imageproxy.NewService( cache, imageproxy.NewProcessor(), - imageproxy.NewPDSFetcher(cfg.FetchTimeout, cfg.MaxSourceSizeMB), + // The SSRF hatch is open only in dev, where the PDS runs on the + // developer's own machine. In production this fetch dials whatever + // address a DID document's serviceEndpoint names, over a public route + // that carries no credential. + imageproxy.NewPDSFetcher(cfg.FetchTimeout, cfg.MaxSourceSizeMB, + imageproxy.PrivateHostOptions(a.allowPrivateHosts())...), cfg, ) if err != nil { diff --git a/cmd/validate-live/main.go b/cmd/validate-live/main.go index dd54866..6adb681 100644 --- a/cmd/validate-live/main.go +++ b/cmd/validate-live/main.go @@ -57,7 +57,7 @@ func main() { log.Fatalf("Failed to load lexicon schemas: %v", err) } - client := &http.Client{Timeout: 30 * time.Second} + client := &http.Client{Timeout: 30 * time.Second} // coves:allow-bare-client: operator-run CLI whose -pds flag the operator types; there is no attacker in this path repos, err := listAllRepos(client, *pdsURL) if err != nil { diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index f6eb694..c0cb30b 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -98,7 +98,7 @@ services: # it exists because a namespace needs a container to belong to and no real # service should be load-bearing for the others' networking. netns: - image: alpine:3.19 + image: alpine:3.24.1 command: ["sleep", "infinity"] init: true healthcheck: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1ecb68c..c59776e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -202,7 +202,7 @@ services: # PLC_DIRECTORY_URL=http://localhost:3002 # IS_DEV_ENV=false # Use production mode but point to local PLC plc-directory: - image: node:18-alpine + image: node:22.23.2-alpine3.24 container_name: coves-dev-plc ports: - "3002:3000" # PLC directory API diff --git a/docker/ci-runner/Dockerfile b/docker/ci-runner/Dockerfile index abb5e4f..1abde39 100644 --- a/docker/ci-runner/Dockerfile +++ b/docker/ci-runner/Dockerfile @@ -6,7 +6,7 @@ # The Go module and build caches are NOT baked in — they are mounted as named # volumes by docker-compose.ci.yml, which is what keeps a warm run fast while # still letting `make ci-clean` discard them. -FROM golang:1.25-alpine +FROM golang:1.26.7-alpine3.24 # git: the go toolchain shells out to it for module operations. # curl: readiness probing from scripts/ci-runner.sh — notably for Jetstream, diff --git a/docker/plc/Dockerfile b/docker/plc/Dockerfile index a084108..7b9d95b 100644 --- a/docker/plc/Dockerfile +++ b/docker/plc/Dockerfile @@ -9,7 +9,7 @@ # CI wipes its volumes every run by design, so paying it per run would add # minutes to every gate. Baking the same steps into an image moves the cost # into Docker's layer cache instead: it is paid again only when PLC_REF changes. -FROM node:18-alpine +FROM node:22.23.2-alpine3.24 # Pinned to a SHA rather than tracking main (which the dev compose does), for # two reasons: a CI run is reproducible, and an upstream PLC commit can never diff --git a/docs/SSRF_SECURITY.md b/docs/SSRF_SECURITY.md new file mode 100644 index 0000000..c79054d --- /dev/null +++ b/docs/SSRF_SECURITY.md @@ -0,0 +1,153 @@ +# SSRF security model + +This document describes the maintained server-side request forgery (SSRF) +boundary in Coves. It is an architecture and contributor guide, not an incident +log. The implementation and its regression tests remain the source of truth. + +Potential security vulnerabilities should be reported privately as described in +[`SECURITY.md`](../SECURITY.md). + +## Threat model + +Coves participates in a federated network and therefore fetches URLs derived +from remote identities, PDS records, OAuth discovery, link previews, image +metadata, and aggregator registrations. A syntactically valid remote record is +not a trust boundary: a remote party may control the hostname, its DNS answers, +redirects, response headers, and response body. + +The shared egress boundary is `NewSSRFSafeHTTPClient` in +`internal/atproto/oauth/transport.go`. Production code that accepts a non-static +destination must use this client or document a narrowly scoped exemption that +is enforced by `scripts/ssrf-audit.sh`. + +## Guard invariants + +The guarded client provides the following properties: + +- Hostnames are normalized consistently with Go's HTTP stack before lookup. +- IP-literal destinations are refused by default. +- DNS is resolved once per request under the request context. Every answer is + classified, and the connection is made directly to an approved address. This + closes the DNS rebinding window between validation and dialing. +- A hostname is refused if any answer is private, local, special-purpose, or + otherwise disallowed. Mixed public/private answer sets fail closed. +- The transport does not use environment-configured HTTP proxies. Adding + `http.ProxyFromEnvironment` would bypass the direct relationship between the + vetted address and the socket destination and must be treated as a security + change. +- Redirects are limited and each destination is processed by the same guard. +- Requests have connection and overall deadlines. Response bodies have a + 32 MiB default cap, with tighter or explicitly larger limits at call sites + that need them. +- Address refusals match `ErrBlockedAddress`; oversized reads match + `ErrResponseTooLarge`. Callers should use `errors.Is` rather than matching + rendered messages. + +Protocol-specific validation remains the caller's responsibility. The generic +client intentionally supports federation endpoints that use non-default ports. +A caller with a narrower protocol contract should apply that contract to the +initial URL and every redirect. + +## Address policy + +The classifier rejects the standard library's loopback, unspecified, private, +link-local, and multicast classes. It also rejects IANA special-purpose ranges +that are not globally reachable, even when a local network happens to route +them. That includes documentation and benchmarking networks, transition +mechanisms that can carry private destinations, and deprecated local-use space. + +Known destination-carrying IPv4-in-IPv6 formats are decoded and the embedded +IPv4 address is classified. The well-known NAT64 prefix `64:ff9b::/96` remains +usable for public IPv4 destinations. + +RFC 8215 local-use NAT64 space (`64:ff9b:1::/48`) is blocked in full. RFC 6052 +allows deployments to choose more-specific Pref64 lengths and layouts, so the +IPv4 position cannot be inferred safely from the address alone. If Coves ever +needs a local-use Pref64, add an explicit operator configuration and decode only +the declared prefix and layout. Do not scan arbitrary IPv6 offsets or weaken +the default block. + +Useful registries and protocol references: + +- [IANA IPv4 Special-Purpose Address Registry](https://www.iana.org/assignments/iana-ipv4-special-registry/) +- [IANA IPv6 Special-Purpose Address Registry](https://www.iana.org/assignments/iana-ipv6-special-registry/) +- [RFC 6052: IPv6 Addressing of IPv4/IPv6 Translators](https://www.rfc-editor.org/rfc/rfc6052) +- [RFC 8215: Local-Use IPv4/IPv6 Translation Prefix](https://www.rfc-editor.org/rfc/rfc8215) + +## OAuth discovery + +Indigo's OAuth application contains three independent HTTP clients: + +1. the main OAuth client; +2. the identity directory client; and +3. the authorization-server metadata resolver. + +`NewOAuthClient` replaces all three with Coves-guarded clients. The metadata +resolver has a ten-second timeout, a 1 MiB response limit, and an HTTPS/no- +explicit-port redirect policy. Construction and end-to-end `StartAuthFlow` +tests cover all three clients, including a private DNS answer that must not +reach a listener and an explicit development-hatch control. + +Password-based PDS login also constructs its API client before calling +`com.atproto.server.createSession`, so the password-bearing request and all +subsequent authenticated requests share the guarded transport. + +## Development hatch + +Private addresses are available only through explicit option functions such as +`WithPrivateAddressesAllowed` and package-specific equivalents. They exist for +local development and hermetic tests, whose services normally listen on +loopback. + +Production wiring must pass no hatch option. Keep the choice local to each +client construction; do not derive it implicitly inside the transport or from +ambient proxy settings. Guard tests should always contain both directions: + +- hatch closed: the private listener is never reached and the error matches + `ErrBlockedAddress`; +- hatch open: the otherwise identical fixture is reached, proving the closed + case is testing classification rather than a broken client. + +## Resolver behavior and hostname syntax + +The transport classifies addresses returned by the resolver, regardless of +which Go resolver implementation produced them. Legacy numeric hostname forms +may be rejected as malformed or may resolve to an IPv4 address depending on the +platform and resolver mode; if they resolve, the resulting address is still +classified before dialing. The production container uses the pure-Go resolver +with `CGO_ENABLED=0`, and the audit pins that build setting to keep production +resolution behavior deterministic. + +Entry points that accept a domain rather than a general URL should additionally +use the positive DNS-name validation in `internal/validation/domain.go`. + +## Adding or changing egress + +When adding a network fetch: + +1. Identify who controls the destination, DNS, redirects, and response body. +2. Use `NewSSRFSafeHTTPClient` before the first request leaves the process. +3. Apply scheme, port, content-type, timeout, and body-size rules for that + protocol. +4. Add a listener-never-reached test for a well-formed hostname resolving to a + private address, plus an explicit hatch-open control where local access is + supported. +5. Run both audit gates and the relevant test tiers. + +Any `coves:allow-*` exemption is a security decision. Keep it next to the +construction it exempts and explain why the destination cannot be controlled by +an untrusted party. + +## Verification + +Run at minimum: + +```sh +make test +make test-audit +make ssrf-audit +``` + +`make ssrf-audit` is a regression tripwire over production egress construction; +it is not a substitute for behavioral tests or code review. Before a release, +run the repository's integration and end-to-end gates as well. diff --git a/go.mod b/go.mod index f418128..aa814d4 100644 --- a/go.mod +++ b/go.mod @@ -1,27 +1,28 @@ module Coves -go 1.25 +go 1.26.0 -toolchain go1.25.1 +toolchain go1.26.7 require ( github.com/bluesky-social/indigo v0.0.0-20260202181658-ea3d39eec464 github.com/disintegration/imaging v1.6.2 - github.com/go-chi/chi/v5 v5.2.1 + github.com/go-chi/chi/v5 v5.3.1 github.com/go-chi/cors v1.2.2 github.com/gorilla/websocket v1.5.3 + github.com/hashicorp/go-retryablehttp v0.7.8 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/lib/pq v1.10.9 github.com/pressly/goose/v3 v3.22.1 github.com/rivo/uniseg v0.4.7 github.com/stretchr/testify v1.11.1 github.com/xeipuuv/gojsonschema v1.2.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 - go.opentelemetry.io/otel v1.39.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 - go.opentelemetry.io/otel/sdk v1.39.0 - golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 - golang.org/x/net v0.47.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 + go.opentelemetry.io/otel v1.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 + go.opentelemetry.io/otel/sdk v1.45.0 + golang.org/x/image v0.45.0 + golang.org/x/net v0.58.0 golang.org/x/time v0.3.0 ) @@ -32,20 +33,19 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-blockservice v0.5.2 // indirect - github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-cid v0.6.1 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect @@ -61,22 +61,22 @@ require ( github.com/ipfs/go-verifcid v0.0.3 // indirect github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipld/go-ipld-prime v0.21.0 // indirect + github.com/ipld/go-ipld-prime v0.23.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mr-tron/base58 v1.2.0 // indirect + github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multibase v0.3.0 // indirect github.com/multiformats/go-multihash v0.2.3 // indirect - github.com/multiformats/go-varint v0.0.7 // indirect + github.com/multiformats/go-varint v0.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f // indirect + github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect @@ -90,22 +90,22 @@ require ( gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.44.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.39.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect - google.golang.org/grpc v1.77.0 // indirect - google.golang.org/protobuf v1.36.10 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/grpc v1.83.1 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect ) diff --git a/go.sum b/go.sum index 62fe454..e883f2f 100644 --- a/go.sum +++ b/go.sum @@ -1,83 +1,265 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bluesky-social/indigo v0.0.0-20260202181658-ea3d39eec464 h1:jL6cPOk1CZ8H06sEn+WFGWufHmqkawsGyDRl+BJhQjs= github.com/bluesky-social/indigo v0.0.0-20260202181658-ea3d39eec464/go.mod h1:VG/LeqLGNI3Ew7lsYixajnZGFfWPv144qbUddh+Oyag= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= +github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/earthboundkid/versioninfo/v2 v2.24.1 h1:SJTMHaoUx3GzjjnUO1QzP3ZXK6Ee/nbWyCm58eY3oUg= github.com/earthboundkid/versioninfo/v2 v2.24.1/go.mod h1:VcWEooDEuyUJnMfbdTh0uFN4cfEIg+kHMuWB2CDCLjw= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8= -github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= +github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= +github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= +github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.5.2 h1:in9Bc+QcXwd1apOVM7Un9t8tixPKdaHQFdLSUM1Xgk8= github.com/ipfs/go-blockservice v0.5.2/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= -github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= -github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cid v0.6.1 h1:T5TnNb08+ueovG76Z5gx1L4Y7QOaGTXHg1F6raWFxIc= +github.com/ipfs/go-cid v0.6.1/go.mod h1:zrY0SwOhjrrIdfPQ/kf+k1sXyJ0QE7cMxfCployLBs0= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= +github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= +github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= +github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= +github.com/ipfs/go-ipfs-pq v0.0.2 h1:e1vOOW6MuOwG2lqxcLA+wEn93i/9laCY8sXAw76jFOY= +github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= +github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= +github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= @@ -95,23 +277,35 @@ github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jq github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-peertaskqueue v0.8.0 h1:JyNO144tfu9bx6Hpo119zvbEL9iQ760FHOiJYsUjqaU= +github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0zs= github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4 h1:oFo19cBmcP0Cmg3XXbrr0V/c+xU9U1huEZp8+OgBzdI= github.com/ipld/go-car v0.6.1-0.20230509095817-92d28eb23ba4/go.mod h1:6nkFF8OmR5wLKBzRKi7/YFJpyYR7+oEn1DX+mMWnlLA= +github.com/ipld/go-car/v2 v2.13.1 h1:KnlrKvEPEzr5IZHKTXLAEub+tPrzeAFQVRlSQvuxBO4= +github.com/ipld/go-car/v2 v2.13.1/go.mod h1:QkdjjFNGit2GIkpQ953KBwowuoukoM75nP/JI1iDJdo= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= -github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= -github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= +github.com/ipld/go-ipld-prime v0.23.0 h1:csqdPZH60BsTC+AZrv7fpa27v+09I/oTqyHYYYE27eE= +github.com/ipld/go-ipld-prime v0.23.0/go.mod h1:46YCFSFNFBJHPjB0pfMuv7Ly7df2eChpkpyPo5SE0bA= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0o= github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= +github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -121,40 +315,100 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= +github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= +github.com/libp2p/go-libp2p v0.22.0 h1:2Tce0kHOp5zASFKJbNzRElvh0iZwdtG5uZheNW8chIw= +github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBxSjRksxxU4= +github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= +github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= +github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.2.0 h1:W6shmB+FeynDrUVl2dgFQvzfBZcXiyqY4VmpQLu9FqU= +github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= +github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= +github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-netroute v0.2.0 h1:0FpsbsvuSnAhXFnCY0VLFbJOzaK0VnP0r1QT/o4nWRE= +github.com/libp2p/go-netroute v0.2.0/go.mod h1:Vio7LTzZ+6hoT4CMZi5/6CpY3Snzh2vgZhWgxMNwlQI= +github.com/libp2p/go-openssl v0.1.0 h1:LBkKEcUv6vtZIQLVTegAil8jbNpJErQ9AnT+bWV+Ooo= +github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= -github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= +github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= -github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multiaddr v0.7.0 h1:gskHcdaCyPtp9XskVwtvEeQOG465sCohbQIirSyqxrc= +github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68= +github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI= +github.com/multiformats/go-multicodec v0.10.0 h1:UpP223cig/Cx8J76jWt91njpK3GTAO1w02sdcjZDSuc= +github.com/multiformats/go-multicodec v0.10.0/go.mod h1:wg88pM+s2kZJEQfRCKBNU+g32F5aWBEjyFHXvZLTcLI= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= -github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= -github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/multiformats/go-multistream v0.3.3 h1:d5PZpjwRgVlbwfdTDjife7XszfZd8KYWfROYFlGcR8o= +github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI= +github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f h1:VXTQfuJj9vKR4TCkEuWIckKvdHFeJH/huIFJ9/cXOB0= -github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f/go.mod h1:/zvteZs/GwLtCgZ4BL6CBsk9IKIlexP43ObX9AxTqTw= +github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a h1:cgqrm0F3zwf9IPzca7xN4w+Zy6MC9ZkPvAC8QEWa/iQ= +github.com/polydawn/refmt v0.89.1-0.20231129105047-37766d95467a/go.mod h1:ocZfO/tLSHqfScRDNTJbAJR1by4D1lewauX9OwTaPuY= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pressly/goose/v3 v3.22.1 h1:2zICEfr1O3yTP9BRZMGPj7qFxQ+ik6yeo+z1LMuioLc= github.com/pressly/goose/v3 v3.22.1/go.mod h1:xtMpbstWyCpyH+0cxLTMCENWBG+0CSxvTsXhW95d5eo= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= @@ -165,31 +419,55 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 h1:RC6RW7j+1+HkWaX/Yh71Ee5ZHaHYt7ZP4sQgUrm6cDU= +github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572/go.mod h1:w0SWMsp6j9O/dk4/ZpIhL+3CkG8ofA2vuv7k+ltqUMc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= +github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e h1:28X54ciEwwUxyHn9yrZfl5ojgF4CBNLWX7LR0rvBkf4= github.com/whyrusleeping/cbor-gen v0.2.1-0.20241030202151-b7a6831be65e/go.mod h1:pM99HXyEbSQHcosHc0iW7YFmwnscr+t9Te4ibko05so= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -198,33 +476,46 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b h1:CzigHMRySiX3drau9C6Q5CAbNIApmLdat5jPMqChvDA= gitlab.com/yawning/secp256k1-voi v0.0.0-20230925100816-f2616030848b/go.mod h1:/y/V339mxv2sZmYYR64O07VuCpdNZqCTwO8ZcouTMI8= gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 h1:qwDnMxjkyLmAFgcfgTnfJrmYKWhHnci3GjDqcZp1M3Q= gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02/go.mod h1:JTnUj0mpYiAsuZLmKjTx/ex3AtMowcCgnE7YNyCEP0I= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 h1:QRefszxJmfPdjXUUm3j6iDzY03mTPXMjqErFqQ67vUg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0/go.mod h1:Tiz03lTBVBrm7eWZBOidzEaYaJa8tjwGUGv6d8mlTyk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0 h1:QBajQ2SrwQijzHyZbQlPsuIzpl/ll8DY6wPWsajeGcI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.45.0/go.mod h1:08ZQLjrPLQ6R4kAXvuOvODEer5Yh4CoFvll5qB2BCI8= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= +go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= +go.opentelemetry.io/otel/sdk/metric v1.45.0 h1:oVFszMfyj1Am6s24Vtc7wBb8BKLcwepJjNEYILuiE3o= +go.opentelemetry.io/otel/sdk/metric v1.45.0/go.mod h1:vUWUxDZvu1WVRj8JA8S0AdhsPrZoDpA2DdZauIh4mDA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -238,92 +529,404 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= -golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= +golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= +golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= -google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= -google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 h1:5D53IMaUuA5InSeMu9eJtlQXS2NxAhyWQvkKEgXZhHI= @@ -338,3 +941,6 @@ modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/api/handlers/aggregator/register.go b/internal/api/handlers/aggregator/register.go index d8a9347..a105786 100644 --- a/internal/api/handlers/aggregator/register.go +++ b/internal/api/handlers/aggregator/register.go @@ -2,7 +2,9 @@ package aggregator import ( "Coves/internal/atproto/identity" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/core/users" + "Coves/internal/validation" "context" "encoding/json" "fmt" @@ -20,24 +22,182 @@ const ( maxWellKnownSize = 4 * 1024 // bytes ) +// ErrDomainInvalid is what a registration whose domain is not a hostname is +// refused with. +// +// IT IS THE SHARED SENTINEL, not a second one. The validator this handler used +// to own now lives in internal/validation, because the community consumer builds +// a URL out of a federated domain exactly the way this handler builds one out of +// a registrant's and could not reach an unexported function in this package — +// which is the only reason that call site stayed open. Re-exporting the value +// rather than declaring a new error keeps `errors.Is` answering the same for +// both call sites: there is one predicate and one identity for its refusal, so a +// test here and a test there cannot come to disagree about what "invalid domain" +// means. +var ErrDomainInvalid = validation.ErrDomainInvalid + // RegisterHandler handles aggregator registration type RegisterHandler struct { userService users.UserService identityResolver identity.Resolver httpClient *http.Client // Allows test injection + + // allowPrivateHosts disables the SSRF guard on the .well-known fetch. + // + // The destination is a domain a stranger POSTs to an unauthenticated route + // (rate-limited 10/10min and nothing else), so in production this must stay + // shut. It is construction state rather than an environment read inside the + // fetch for the reason blobs.blobService.allowPrivateHosts documents: the fixtures that + // exercise this path serve .well-known from httptest, which listens on + // loopback, and Go forbids t.Setenv alongside t.Parallel. + allowPrivateHosts bool + + // transportOptions is the TEST SEAM, carried on the handler so that the + // client the tests exercise is the one NewRegisterHandler builds. + // + // The alternative — a fixture that constructs the handler and then assigns + // registerHTTPClient(...) over its client — is what this field exists to + // delete. That fixture proves registerHTTPClient guards, which is one call + // frame short of the claim: h.allowPrivateHosts is read in exactly one place + // (below), and a build where that line reads a constant `true` instead + // leaves every such test green. Passing the seam THROUGH the constructor is + // what makes the constructor's own wiring the thing under test. + // + // It is UNEXPORTED, and so is the option that sets it: oauth.WithHostResolver + // must not be reachable from any non-test package, which scripts/ssrf-audit.sh + // enforces as a hard gate. See pds.bearerClientConfig.transportOptions, the + // same field for the same reason. + transportOptions []covesoauth.Option +} + +// RegisterHandlerOption configures optional RegisterHandler behaviour. +type RegisterHandlerOption func(*RegisterHandler) + +// WithPrivateHostsAllowed disables the SSRF address guard on the .well-known +// domain-verification fetch. +// +// THE NAME IS THE CONTRACT: production must not call this. It is greppable, +// which is how "which handlers have the guard off" stays an answerable +// question. +func WithPrivateHostsAllowed() RegisterHandlerOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(h *RegisterHandler) { h.allowPrivateHosts = true } +} + +// withTransportOptions passes oauth transport options through to the client +// NewRegisterHandler builds, and is UNEXPORTED so production cannot reach it. +// +// It is not a second hatch: whatever a resolver seam answers is classified by +// the same pass a real DNS answer goes through, and the dial still goes only to +// addresses that survived it. See RegisterHandler.transportOptions, and +// pds/factory.go's withTransportOptions, which is the shape this copies. +func withTransportOptions(opts ...covesoauth.Option) RegisterHandlerOption { + return func(h *RegisterHandler) { h.transportOptions = append(h.transportOptions, opts...) } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to NewRegisterHandler: the hatch when it is set, and +// NOTHING when it is not. +// +// It mirrors oauth.PrivateAddressOptions. `.env.ci:140` sets IS_DEV_ENV=true, +// so `make ci` takes the PERMISSIVE branch at every call site holding such a +// boolean; a unit test against this function is the only place in the +// repository where the branch production actually runs is evaluated. That +// matters especially here because +// RegisterAggregatorRoutes has no config of its own — the alternative to a pure +// function is an inline conditional three call layers away from anything that +// could reach it. Do not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none. +func PrivateHostOptions(allowPrivate bool) []RegisterHandlerOption { + if !allowPrivate { + return nil + } + return []RegisterHandlerOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing } -// NewRegisterHandler creates a new registration handler -func NewRegisterHandler(userService users.UserService, identityResolver identity.Resolver) *RegisterHandler { - return &RegisterHandler{ +// registerHTTPClient builds the client the .well-known domain-verification +// fetch goes through: the SSRF-safe transport of internal/atproto/oauth, which +// resolves the host, refuses private, loopback and link-local addresses, and +// then dials only the address it vetted. +// +// # THE GUARD IS NOT REDUNDANT WITH validation.NormalizeDomain +// +// The validator in front of this refuses every SPELLING of an address — +// 127.0.0.1, localhost, 2130706433, internal-admin — because none is a +// two-label name with an alphabetic TLD. What it cannot see is the DNS answer. +// A well-formed public hostname whose owner points it at 127.0.0.1 passes +// validation completely, and is the cheapest SSRF available here since the +// attacker owns the zone. That input is exactly what this client refuses and +// the validator cannot. +// +// The variadic oauth options exist for the caller's own tests: a hostname +// cannot be made to resolve to a chosen address hermetically, so +// oauth.WithHostResolver is how this package proves ITS wiring rather than +// re-proving oauth's. The seam cannot open the guard — whatever it answers is +// classified by the same pass a real DNS answer goes through. +// +// The 10s ceiling is this handler's own and is re-applied over the shared +// client's 15s. +func registerHTTPClient(allowPrivateHosts bool, opts ...covesoauth.Option) *http.Client { + client := covesoauth.NewSSRFSafeHTTPClient( + append(covesoauth.PrivateAddressOptions(allowPrivateHosts), opts...)...) + client.Timeout = 10 * time.Second + return client +} + +// NewRegisterHandler creates a new registration handler. +// +// THE CLIENT IT BUILDS IS GUARDED UNLESS THE CALLER SAYS OTHERWISE, so that +// forgetting is safe: NewRegisterHandler with no options is what the next caller +// will write. RegisterAggregatorRoutes threads its own variadic registerOpts +// through, which in production is what PrivateHostOptions(false) returns — +// nothing — so the guarded construction is what an empty slice lands on too. +func NewRegisterHandler(userService users.UserService, identityResolver identity.Resolver, + opts ...RegisterHandlerOption) *RegisterHandler { + h := &RegisterHandler{ userService: userService, identityResolver: identityResolver, - httpClient: &http.Client{Timeout: 10 * time.Second}, } + for _, opt := range opts { + opt(h) + } + // After the options, because the hatch and the transport options they may + // have set are what this client is built from. Both are threaded, and the + // second one is why this line is covered at all: the handler's guard tests + // build through this constructor, so changing the argument below to a + // constant fails them. + h.httpClient = registerHTTPClient(h.allowPrivateHosts, h.transportOptions...) + return h } -// SetHTTPClient allows overriding the HTTP client (for testing with self-signed certs) -func (h *RegisterHandler) SetHTTPClient(client *http.Client) { +// setHTTPClient overrides the HTTP client, and is UNEXPORTED because there is no +// safe way for a caller outside this package to use it. +// +// IT REPLACES RATHER THAN WRAPS, DELIBERATELY. Wrapping the guard around +// whatever it is handed reads like the safer design and is not: two fixtures in +// this package (register_test.go's and register_users_row_test.go's, both via +// stubClient) install a transport with a pinned dialer so that a request for a +// made-up domain reaches an httptest listener, and a wrapped guard would either +// refuse that listener's loopback address or fail to resolve the name at all +// under an egress-blocked CI. A third, newRecordingHandler, installs a +// RoundTripper that answers nothing and records what it was asked for. The seam +// has to keep replacing for any of those to work. +// +// AND REPLACING IS EXACTLY WHY IT CANNOT BE EXPORTED. A production caller +// reaching for it silently discards the guarded client NewRegisterHandler built, +// which is the hazard pds/factory.go's withTransportOptions is unexported to +// avoid — so leaving this one exported had the codebase answering the same +// question two opposite ways. An earlier version of this comment conceded the +// footgun and deferred the fix to "a greppable regression fence or a rename"; +// the lowercase name is better than either, because it is the compiler that +// enforces it and there is nothing left to grep for. Every caller is a fixture +// in this package, so it costs them nothing. +// +// TestNoExportedSeamCanReplaceTheGuardedClient pins the property in general — +// nothing exported from this package takes an *http.Client — so the NEXT seam +// somebody adds is covered too. +func (h *RegisterHandler) setHTTPClient(client *http.Client) { h.httpClient = client } @@ -102,6 +262,32 @@ func (h *RegisterHandler) HandleRegister(w http.ResponseWriter, r *http.Request) return } + // The domain must be a hostname before it can be concatenated into a URL, + // and this check has to happen HERE — before any HTTP client is touched. + // + // verifyDomainOwnership builds its URL with + // fmt.Sprintf("https://%s/.well-known/atproto-did", domain), and a string + // parser has no way to know the concatenation was meant to stop at the host: + // `internal-admin/v1/secrets?x=y#` fetches /v1/secrets?x=y from + // internal-admin, because the trailing `#` turns the intended suffix into a + // fragment that is never sent. `evil.com@internal-host` makes evil.com + // userinfo and the internal name the host. So an unauthenticated caller + // chooses the host, the path AND the query of a request this server makes + // from inside its own network. Validating any later — after the request is + // built, or inside verifyDomainOwnership — is too late: by then the caller's + // host has already been resolved and dialled, which is the whole of the SSRF. + // + // The normalized form is what gets used from here on, so that one domain has + // one spelling in the URL, in the logs and in anything that compares them. + normalizedDomain, err := validation.NormalizeDomain(req.Domain) + if err != nil { + log.Printf("Registration refused a domain that is not a hostname for DID %s: %v", req.DID, err) + writeError(w, http.StatusBadRequest, "InvalidDID", + "Domain must be a hostname such as example.com, with no scheme, port, path, query or credentials") + return + } + req.Domain = normalizedDomain + // Verify domain ownership via .well-known if err := h.verifyDomainOwnership(r.Context(), req.DID, req.Domain); err != nil { log.Printf("Domain verification failed for DID %s, domain %s: %v", req.DID, req.Domain, err) diff --git a/internal/api/handlers/aggregator/register_domain_test.go b/internal/api/handlers/aggregator/register_domain_test.go new file mode 100644 index 0000000..106787a --- /dev/null +++ b/internal/api/handlers/aggregator/register_domain_test.go @@ -0,0 +1,230 @@ +package aggregator + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "Coves/internal/atproto/identity" + "Coves/internal/core/users" + "Coves/tests/domaincorpus" + + "github.com/stretchr/testify/require" +) + +// Registration takes a domain from an unauthenticated caller and builds a URL +// out of it — `https://` + the domain + `/.well-known/atproto-did` — and until +// this change the only thing it checked was that the domain was not the empty +// string. String concatenation into a URL is the whole vulnerability: every +// part of a URL that comes after the host can be smuggled in through the host, +// because the parser does not know the concatenation was supposed to stop. +// Three shapes were confirmed against the handler: +// +// internal-admin/v1/secrets?x=y# fetches /v1/secrets?x=y from internal-admin +// evil.com@internal-host fetches from internal-host; evil.com is userinfo +// 127.0.0.1:5432 fetches from a port on the loopback interface +// +// The trailing `#` in the first is what makes this more than an SSRF to one +// fixed path: it turns `/.well-known/atproto-did` into a fragment, which is +// never sent, so the attacker chooses the path AND the query as well as the +// host. Rate limiting is the only other gate and no credential is needed. +// +// The fix these tests pin is a positive allowlist on DNS shape rather than a +// blocklist on addresses; validation.NormalizeDomain's doc comment says why a +// blocklist cannot work here. The tests come in three kinds, and the third is +// the one that would be missing if this were written carelessly: +// +// 1. what the validator refuses, +// 2. what it must go on accepting, +// 3. that the handler consults it BEFORE it touches an HTTP client. +// +// Without (3) an implementation that validates after building and sending the +// request satisfies (1) and (2) completely, while having already resolved and +// dialled the host the attacker named. The refusal is only worth anything if it +// happens first. +// +// # (1) AND (2) HAVE MOVED; (3) IS WHAT STAYS HERE +// +// The validator is being promoted out of this package so the community +// consumer can reach it — its `.well-known/did.json` fetch builds a URL by +// concatenating a federated domain the same way this handler did, and could not +// call `normalizeDomain` because it was unexported here. So the predicate's own +// tables now live in internal/validation, driven from tests/domaincorpus. +// +// What cannot move is (3), because it is a fact about THIS HANDLER: that it +// consults the validator before it reaches for a client. Testing the handler's +// behaviour rather than the private function is also what makes this file +// survive the delegation — it asserts the same thing before and after +// `normalizeDomain` becomes a call into the shared package. + + +// recordingTransport answers nothing and remembers whether it was asked to. +// +// A RoundTripper rather than a dialer, because RoundTrip is the earliest moment +// the handler can reach the network: http.Client.Do calls it before any name is +// resolved, so a call recorded here means the attacker's host was already on its +// way out whether or not a packet followed. Recording the URL as well is what +// makes a failure readable — the message names the request that would have been +// sent, which for the fragment payload is a path nobody wrote down anywhere. +// +// Plain fields, no mutex: http.Client.Do calls RoundTrip on the caller's +// goroutine, and the handler under test is driven synchronously. +type recordingTransport struct { + requested []string +} + +func (r *recordingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + r.requested = append(r.requested, req.URL.String()) + return nil, errors.New("recordingTransport never answers: the handler should not have reached it") +} + +// newRecordingHandler builds the handler with an HTTP client that cannot talk to +// anything and says so afterwards. +func newRecordingHandler(t *testing.T) (*RegisterHandler, *recordingTransport) { + t.Helper() + + transport := &recordingTransport{} + userService := &fakeUserService{registered: map[string]*users.User{}} + resolver := &fakeIdentityResolver{identities: map[string]*identity.Identity{ + registrantDID: {DID: registrantDID, Handle: registrantHandle, PDSURL: registrantPDS}, + }} + + handler := NewRegisterHandler(userService, resolver) + handler.setHTTPClient(&http.Client{Transport: transport}) + return handler, transport +} + +func postRegister(t *testing.T, handler *RegisterHandler, did, domain string) *httptest.ResponseRecorder { + t.Helper() + + body, err := json.Marshal(RegisterRequest{DID: did, Domain: domain}) + require.NoError(t, err, "marshalling the request") + + req := httptest.NewRequest(http.MethodPost, "/xrpc/social.coves.aggregator.register", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + handler.HandleRegister(rec, req) + return rec +} + +// THE ORDERING TEST. +// +// Every case below is one the handler currently sends: each parses as a URL, so +// http.NewRequestWithContext succeeds and Do is reached. That is deliberate — +// an input Go's URL parser rejects would never touch the transport even with no +// validation at all, and would prove nothing here. +// +// What this asserts that no other test in this file can: the refusal happens +// before the client. An implementation that validates the domain after building +// and sending the request passes the whole rejection table and still resolves +// and dials whatever the caller named, which for an SSRF is the entire damage. +func TestRegister_ValidatesTheDomainBeforeTouchingTheNetwork(t *testing.T) { + for _, tt := range domaincorpus.InjectionPayloads() { + t.Run(tt.Name, func(t *testing.T) { + handler, transport := newRecordingHandler(t) + + rec := postRegister(t, handler, registrantDID, tt.Domain) + + require.Emptyf(t, transport.requested, + "the handler asked its HTTP client for %q before refusing the domain %q. "+ + "Validation must run first: by the time the client is called the host has "+ + "been chosen by the caller, and resolving it is already the SSRF", + transport.requested, tt.Domain) + + // The status is a second, weaker signal, and it is here to say which + // mechanism refused. A 401 DomainVerificationFailed means the fetch + // was attempted and failed; only a 400 means the request never left. + require.Equalf(t, http.StatusBadRequest, rec.Code, + "domain %q must be refused as a malformed request, not as a failed verification. Body: %s", + tt.Domain, rec.Body.String()) + var body XRPCError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body), "body is not valid JSON") + require.Equalf(t, "InvalidDID", body.Error, "error code for domain %q", tt.Domain) + }) + } + + // THE CONTROL, and without it every row above is unfalsifiable. + // + // The whole table asserts that a recorder stayed EMPTY. An empty recorder is + // also what you get when the recorder was never installed — and + // setHTTPClient is the only thing that installs it. Mutation-proven: make + // setHTTPClient ignore its argument and every row above stays green, while + // the handler quietly makes real outbound requests through its own default + // client for the whole run. + // + // So one row has to drive a domain that PASSES validation and assert the + // recorder was reached. That is the only assertion in this file that can + // tell "the handler refused before the client" from "the client under + // observation was not the client the handler uses". + t.Run("control: a valid domain does reach the injected client", func(t *testing.T) { + handler, transport := newRecordingHandler(t) + + rec := postRegister(t, handler, registrantDID, stubDomain) + + require.NotEmptyf(t, transport.requested, + "a domain that passes validation (%q) never reached the recording transport. The recorder is "+ + "what every row above asserts on, so if the handler is not using it those rows prove nothing "+ + "— they would pass just as well against a handler that sends every request through a client "+ + "this test cannot see. Either validation is refusing a legitimate hostname, or setHTTPClient "+ + "is not installing the client it is given", + stubDomain) + require.Equalf(t, []string{"https://" + stubDomain + wellKnownPath}, transport.requested, + "the handler asked for %v; the .well-known fetch must go to the domain the caller named, over "+ + "HTTPS, at the well-known path and nothing else", transport.requested) + + // recordingTransport never answers, so verification fails — which is the + // 401 branch, and its presence here proves the request was attempted + // rather than short-circuited. + require.Equalf(t, http.StatusUnauthorized, rec.Code, + "a domain that reached the transport must fail verification (the recorder answers nothing), "+ + "not be refused as malformed. Body: %s", rec.Body.String()) + }) +} + +// The handler normalises before it validates, and that ordering is load-bearing +// in both directions. +// +// `https://example.com` is refused by normalizeDomain — it is not a hostname, +// it is a URL — and yet registration has always accepted it, because the +// handler strips the scheme first. Both facts are true at once and neither is +// an accident: the validator is a predicate about hostnames with no opinion +// about what a client might wrap one in, and the handler is where a small, +// enumerated set of client sloppiness is unwrapped. Nothing dangerous survives +// the unwrapping, because validation still runs on the result — +// `https://evil.com@internal-host` strips to `evil.com@internal-host` and is +// refused. +// +// This is here so that the GREEN change does not delete the trimming as +// redundant. Removing it would break clients that have always been allowed to +// send a scheme, and no other test in the file would notice. +func TestRegister_AcceptsTheClientSloppinessItAlwaysHas(t *testing.T) { + tests := []struct { + name string + domain string + }{ + {name: "a bare hostname", domain: stubDomain}, + {name: "an https scheme the handler strips", domain: "https://" + stubDomain}, + {name: "a trailing slash the handler strips", domain: stubDomain + "/"}, + {name: "surrounding whitespace the handler trims", domain: " " + stubDomain + " "}, + {name: "uppercase, which DNS does not distinguish", domain: "EXAMPLE.COM"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newRegisterFixture(t, wellKnownServing(registrantDID)) + + rec := f.register(t, registrantDID, tt.domain) + + require.Equalf(t, http.StatusOK, rec.Code, + "domain %q used to register successfully and must go on doing so; the new validation "+ + "is meant to refuse hosts the AppView should not fetch, not spellings clients "+ + "have always been allowed to send. Body: %s", + tt.domain, rec.Body.String()) + require.Lenf(t, f.users.created, 1, + "domain %q answered 200 without registering the aggregator", tt.domain) + }) + } +} diff --git a/internal/api/handlers/aggregator/register_guard_test.go b/internal/api/handlers/aggregator/register_guard_test.go new file mode 100644 index 0000000..86af38f --- /dev/null +++ b/internal/api/handlers/aggregator/register_guard_test.go @@ -0,0 +1,562 @@ +package aggregator + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" + "Coves/internal/core/users" + + "Coves/internal/atproto/identity" +) + +// The registration handler's outbound client, and the one assertion that can +// tell it apart from the validator sitting in front of it. +// +// # WHY THIS IS THE HARDEST SITE TO TEST HONESTLY +// +// Two mechanisms refuse this endpoint's hostile inputs, and they overlap almost +// completely. normalizeDomain refuses `127.0.0.1:5432`, `localhost`, +// `2130706433`, `internal-admin` and every other spelling of an address, +// because none of them is a two-label name with an alphabetic TLD. The SSRF +// guard refuses the same destinations one layer down. So a test that asserts +// only "an error came back" passes whether the guarded client is wired or not — +// and register_domain_test.go's whole rejection table is made of inputs the +// validator eats first. +// +// The only input that separates them is a domain that PASSES validation and +// still resolves to a private address: a well-formed public-looking hostname +// whose owner points it at 127.0.0.1. That is not a hypothetical — it is the +// cheapest SSRF an attacker has here, because they control the zone and the +// validator cannot see the DNS answer. It is also, notably, the one input the +// domain-shape validation alone cannot reject. +// +// # THE RESOLVER SEAM +// +// A hostname cannot be made to resolve to a chosen address hermetically: the +// hermetic tiers block egress, and nothing in the tree can write /etc/hosts. +// oauth.WithHostResolver is therefore the seam these tests drive — the same +// field oauth's own tests already set on the transport, exported so a caller +// package can prove ITS wiring rather than re-proving oauth's. +// +// It cannot open the guard. Whatever the seam answers is classified exactly as +// a real DNS answer would be, and the dial still goes only to vetted addresses; +// the seam decides what gets classified, never whether classification happens. +// +// # WHERE THE "NEVER REACHED" ASSERTION LIVES, AND WHERE IT CANNOT +// +// An earlier version of this comment said flatly that this site could not have +// one. That was half right, and the half it got wrong mattered. +// +// It is true of anything driven through verifyDomainOwnership: a +// validation-passing domain has no port, so the URL is always https:///… +// on 443, and a test cannot bind 443. For those tests the CONTROL does the +// equivalent work — same client, same seam, same domain, hatch OPEN, and the +// error becomes a dial failure instead of a classification refusal. That +// difference is what proves the guarded refusal came from classifying the +// address rather than from this test being unable to make requests at all. +// +// It is NOT true of the client itself. TestNewRegisterHandler_DefaultClientIsGuarded +// drives handler.httpClient directly at a host the seam resolves, on a port a +// test IS allowed to bind, and asserts a real request counter stayed at zero — +// with a control proving that same listener is reached when the hatch is open. +// That assertion exists because the fence's old form was a reflect.TypeOf +// comparison, which cannot see a bool field inside oauth's transport and so +// could not fail on polarity at all. + +// privateHostResolver answers every lookup with one address, and records the +// names it was asked about. +func privateHostResolver(t *testing.T, answer string) func(context.Context, string) ([]net.IP, error) { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as public and the test would pass or fail for reasons unconnected + // to its subject. + ip := net.ParseIP(answer) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", answer) + + return func(_ context.Context, _ string) ([]net.IP, error) { + return []net.IP{ip}, nil + } +} + +// guardedRegisterHandler builds the handler the way production does — through +// NewRegisterHandler, from the same allow-private boolean — and replaces only +// its NAME RESOLUTION, by passing the resolver seam as a CONSTRUCTION OPTION. +// +// # THE SEAM GOES THROUGH THE CONSTRUCTOR, AND THAT IS THE POINT +// +// An earlier version of this fixture built the handler exactly as above and then +// assigned `handler.httpClient = registerHTTPClient(allowPrivateHosts, seam)` +// over the result. That second line threw the first one away, and with it the +// only thing these tests are here to prove. h.allowPrivateHosts is read in ONE +// place in this tree — NewRegisterHandler's `registerHTTPClient(h.allowPrivateHosts, +// h.transportOptions...)` — and a build with that argument replaced by a +// constant `true` left every test in this file green, `make ci` green and +// `make ssrf-audit` at zero. What was under test was registerHTTPClient, which +// is one call frame short of the wiring. +// +// So the client under test is now the handler's own, built by the constructor +// production calls, and nothing here overwrites it. Neither setHTTPClient nor a +// direct field assignment appears: both replace the client wholesale, which is +// precisely what makes a test unable to say WHOSE client it exercised. +func guardedRegisterHandler(t *testing.T, allowPrivateHosts bool, resolvesTo string) *RegisterHandler { + t.Helper() + + userService := &fakeUserService{registered: map[string]*users.User{}} + resolver := &fakeIdentityResolver{identities: map[string]*identity.Identity{ + registrantDID: {DID: registrantDID, Handle: registrantHandle, PDSURL: registrantPDS}, + }} + + // PrivateHostOptions is what production passes, so the hatch reaches the + // handler by the production route; the seam is appended to it rather than + // replacing it. + opts := append(PrivateHostOptions(allowPrivateHosts), + withTransportOptions(covesoauth.WithHostResolver(privateHostResolver(t, resolvesTo)))) + + return NewRegisterHandler(userService, resolver, opts...) +} + +// guardTestDomain passes normalizeDomain — two labels, an alphabetic TLD, no +// port, no path, no userinfo — and is exactly what an attacker registers when +// the validator is the only control. `.example` is reserved by RFC 2606, so +// nothing resolves it for real if the seam is ever bypassed. +const guardTestDomain = "aggregator.example" + +// TestVerifyDomainOwnership_RefusesAValidationPassingHostThatResolvesPrivate is +// the assertion this cycle exists for. +// +// The domain is well-formed, so normalizeDomain accepts it and cannot be what +// refuses the request. Only the transport can. Asserting BOTH halves — that the +// guard's sentinel is present and ErrDomainInvalid is not — is what makes the +// two mechanisms separately visible: delete either one and exactly one of these +// two assertions changes. +func TestVerifyDomainOwnership_RefusesAValidationPassingHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + handler := guardedRegisterHandler(t, false, "127.0.0.1") + + err := handler.verifyDomainOwnership(context.Background(), registrantDID, guardTestDomain) + + require.Errorf(t, err, + "%q passed validation and its DNS answer was 127.0.0.1, and the handler fetched it anyway. "+ + "This is the SSRF the domain validator cannot reach: the attacker owns the zone, so the "+ + "name is well-formed and the address is chosen after validation has already run", + guardTestDomain) + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity. Without this, a build where the guarded client "+ + "was never wired looks identical — some other error would still come back, and every "+ + "assertion that only says 'it failed' would still pass; got: %v", err) + + assert.NotErrorIsf(t, err, ErrDomainInvalid, + "the validator refused a hostname it is supposed to accept, which means this test is no longer "+ + "exercising the transport at all: %q is a two-label name with an alphabetic TLD. Fix the "+ + "fixture rather than the assertion — a guard-path test whose input the validator eats is "+ + "the exact false pass this case was written to close; got: %v", guardTestDomain, err) +} + +// TestVerifyDomainOwnership_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control, and without it the test above is unfalsifiable. +// +// Identical handler, identical seam, identical domain — only the hatch differs. +// With it open the address is no longer refused, so the request proceeds to a +// dial, which fails because nothing is listening on loopback:443. The error is +// therefore NOT the guard's. +// +// That is what pins the refusal above to CLASSIFICATION. Without this control, +// a client that could not make any request at all — a broken seam, a transport +// wired to nothing — would satisfy the guarded case just as well. +func TestVerifyDomainOwnership_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + handler := guardedRegisterHandler(t, true, "127.0.0.1") + + err := handler.verifyDomainOwnership(context.Background(), registrantDID, guardTestDomain) + + require.Errorf(t, err, + "nothing listens on loopback:443, so this fetch must fail — if it succeeded, the seam is not "+ + "answering with the address this test gave it") + + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either PrivateHostOptions "+ + "is not reaching the client, or the guarded case above proves nothing: a client that refuses "+ + "every address refuses the guarded case too, for a reason that has nothing to do with "+ + "classification; got: %v", err) +} + +// TestHandleRegister_SeparatesTheTwoRefusalsAtTheHTTPLayer pins the same split +// where an operator and a caller actually see it. +// +// The handler already answers the two mechanisms differently — 400 InvalidDID +// for a malformed domain, 401 DomainVerificationFailed for a fetch that was +// attempted and failed — and that distinction is worth keeping precisely +// because it is the only externally visible evidence of which control fired. +// Unlike the image proxy's status codes, this pair is not an oracle: both +// outcomes are things the caller told the server, and neither reports anything +// about the AppView's network. +func TestHandleRegister_SeparatesTheTwoRefusalsAtTheHTTPLayer(t *testing.T) { + t.Parallel() + + t.Run("a validation-passing host that resolves private is a verification failure", func(t *testing.T) { + t.Parallel() + + handler := guardedRegisterHandler(t, false, "127.0.0.1") + + rec := postRegister(t, handler, registrantDID, guardTestDomain) + + require.Equalf(t, http.StatusUnauthorized, rec.Code, + "a well-formed domain whose address the guard refused must answer as a failed verification, "+ + "not as a malformed request: the caller's input WAS well-formed, and reporting otherwise "+ + "tells them their domain is syntactically wrong when it is not. Body: %s", rec.Body.String()) + + var body XRPCError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body), "body is not valid JSON") + assert.Equal(t, "DomainVerificationFailed", body.Error, + "the error code is the only signal distinguishing the guard's refusal from the validator's") + }) + + t.Run("a malformed domain is still refused by the validator", func(t *testing.T) { + t.Parallel() + + // The guard would refuse this destination too, one layer down — which is + // the whole difficulty. Pinning the validator's own sentinel separately + // is what keeps a future change that deletes normalizeDomain from + // looking like a passing suite: the guard sees an ADDRESS, and + // `internal-admin/v1/secrets?x=y#` is a path injection against whatever + // internal-admin resolves to — not something a classifier can catch. + handler := guardedRegisterHandler(t, false, "127.0.0.1") + + rec := postRegister(t, handler, registrantDID, "internal-admin/v1/secrets?x=y#") + + require.Equalf(t, http.StatusBadRequest, rec.Code, + "a domain that is not a hostname must be refused as malformed, before any client is "+ + "touched. Body: %s", rec.Body.String()) + + var body XRPCError + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body), "body is not valid JSON") + assert.Equal(t, "InvalidDID", body.Error, + "the validator's refusal must stay distinguishable from the guard's") + }) +} + +// countingListener is a server that answers 200 and counts how many requests +// actually arrived. +// +// It is PLAIN HTTP, not TLS, and that is a deliberate narrowing. The claim under +// test is about ADDRESS CLASSIFICATION — did the transport dial, or refuse +// before dialling — and a TLS listener puts a certificate check between the dial +// and the handler. httptest's cert carries SANs [example.com, *.example.com], so +// a guard-test hostname would fail the handshake and the handler would record +// zero requests EVEN WITH THE HATCH OPEN, which quietly turns the control below +// into another vacuous assertion. Plain HTTP removes the only other thing that +// can keep the counter at zero, so a zero means what this test says it means. +type countingListener struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingListener(t *testing.T) *countingListener { + t.Helper() + + listener := &countingListener{} + listener.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + listener.requests.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(registrantDID)) + })) + t.Cleanup(listener.server.Close) + return listener +} + +// port returns the port the listener bound, so a request can name a host the +// seam resolves AND a port a test is allowed to bind. +func (l *countingListener) port(t *testing.T) string { + t.Helper() + + _, port, err := net.SplitHostPort(l.server.Listener.Addr().String()) + require.NoError(t, err, "the listener's address must split into host and port") + return port +} + +// defaultClientRequest drives a handler's OWN client at a host the seam resolves +// to the listener, and reports the error and whether the listener was reached. +// +// It calls handler.httpClient directly rather than verifyDomainOwnership, and +// the reason is the constraint documented at the top of this file: a +// validation-passing domain has no port, so verifyDomainOwnership always builds +// https:///… on 443, and a test cannot bind 443. Naming the port here is +// what makes a reachable listener possible at all — and it costs nothing this +// test was claiming, because the subject is the CLIENT the constructor +// installed, not the URL the handler builds. NormalizeDomain's separate refusal +// of ports is pinned by register_domain_test.go and by the HTTP-layer split +// above. +func defaultClientRequest(t *testing.T, handler *RegisterHandler, listener *countingListener) (reached int64, err error) { + t.Helper() + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, + "http://"+guardTestDomain+":"+listener.port(t)+wellKnownPath, nil) + require.NoError(t, err) + + resp, doErr := handler.httpClient.Do(req) + if doErr == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + return listener.requests.Load(), doErr +} + +// TestNewRegisterHandler_DefaultClientIsGuarded is the fence for every caller +// who never thinks about this. +// +// # WHY THIS TEST IS SHAPED THE WAY IT IS, AND WHAT IT USED TO MEASURE +// +// It used to consist of `reflect.TypeOf(registerHTTPClient(false).Transport)` +// compared against the handler's, plus the timeout. THAT ASSERTION CANNOT FAIL +// ON POLARITY. The hatch is a bool field INSIDE oauth's transport, not a +// different type, so registerHTTPClient(true) and registerHTTPClient(false) +// return an identical reflect.TypeOf — and a build whose constructor hardcoded +// the hatch OPEN passed this test while its name said DefaultClientIsGuarded and +// its comment said "forgetting is safe". A type comparison can see that the +// conversion happened; it can see nothing about which way the switch is set. +// +// So the contract is now carried by REACHABILITY, and the type and timeout +// checks are kept as the cheap secondary they always were. +// +// # THE HATCH OPTION IS ABSENT, WHICH IS THE POINT +// +// The guarded subtest passes NO PrivateHostOptions and NO WithPrivateHostsAllowed +// — only the resolver seam, which sets no policy and cannot open the guard. That +// is bit-for-bit the state a caller who never thought about any of this leaves +// the handler in, and it is the state cmd/server ships. +func TestNewRegisterHandler_DefaultClientIsGuarded(t *testing.T) { + t.Parallel() + + newHandler := func(t *testing.T, listener *countingListener, opts ...RegisterHandlerOption) *RegisterHandler { + t.Helper() + + seam := withTransportOptions(covesoauth.WithHostResolver(privateHostResolver(t, "127.0.0.1"))) + return NewRegisterHandler( + &fakeUserService{registered: map[string]*users.User{}}, + &fakeIdentityResolver{identities: map[string]*identity.Identity{}}, + append(opts, seam)..., + ) + } + + t.Run("the default client refuses a private address without reaching the listener", func(t *testing.T) { + t.Parallel() + + listener := newCountingListener(t) + handler := newHandler(t, listener) // no hatch option: the production default + + reached, err := defaultClientRequest(t, handler, listener) + + // THE REACHABILITY CLAIM COMES FIRST, deliberately. It is the security + // property, and asserting it before the error means a regression reports + // "the listener was reached" rather than the far less useful "an error + // was expected but got nil". + assert.Zerof(t, reached, + "the listener was reached %d time(s), so NewRegisterHandler's own client dialled a host "+ + "whose DNS answer was 127.0.0.1 and delivered the request. Registration is "+ + "unauthenticated and rate-limited only 10/10min, so this client dials whatever "+ + "hostname a stranger posts. This is the assertion the old type comparison could not "+ + "make, and the one that dies if the constructor's hatch argument is ever replaced by "+ + "a constant", reached) + + require.Errorf(t, err, + "the request SUCCEEDED against a private address, so the default client is not guarded "+ + "at all") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a client that simply cannot make requests "+ + "satisfies this case too; got: %v", err) + }) + + t.Run("control: the same listener IS reached with the hatch open", func(t *testing.T) { + t.Parallel() + + listener := newCountingListener(t) + handler := newHandler(t, listener, WithPrivateHostsAllowed()) // coves:allow-ssrf-hatch: the control half; without it the guarded case above is unfalsifiable + + reached, err := defaultClientRequest(t, handler, listener) + + require.NoErrorf(t, err, + "the hatch was open and the request still failed, so the guarded case above proves nothing: "+ + "a client that reaches nothing refuses a private address for reasons unconnected to "+ + "classification; got: %v", err) + assert.Equalf(t, int64(1), reached, + "the listener was reached %d time(s) with the hatch open, want exactly 1. The zero asserted "+ + "above only means something if this listener is reachable when classification permits it", + reached) + }) + + t.Run("secondary: the transport and the timeout are the guarded builder's", func(t *testing.T) { + t.Parallel() + + handler := NewRegisterHandler( + &fakeUserService{registered: map[string]*users.User{}}, + &fakeIdentityResolver{identities: map[string]*identity.Identity{}}, + ) + + require.NotNil(t, handler.httpClient, "the handler must carry an HTTP client") + require.NotNil(t, handler.httpClient.Transport, + "the handler's client uses http.DefaultTransport, which is the unguarded stdlib client") + + // A TYPE COMPARISON AND NOTHING MORE. It catches "the conversion never + // happened"; it is blind to which way the hatch is set, which is why the + // subtests above exist. Kept because the transport type is unexported in + // oauth, so this is the strongest structural claim available from here. + guarded := registerHTTPClient(false) + assert.Equalf(t, reflect.TypeOf(guarded.Transport), reflect.TypeOf(handler.httpClient.Transport), + "the default client's transport is %T, but the guarded builder produces %T", + handler.httpClient.Transport, guarded.Transport) + + assert.Equal(t, 10*time.Second, handler.httpClient.Timeout, + "the handler's own 10s ceiling must survive adopting the shared client, which ships 15s") + }) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// single most important assertion for this call site. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch +// here and everywhere else that holds such a boolean. This function is the one +// place in the repository where the production branch is evaluated, which is why +// it is a pure function rather than an `if` in wiring — and RegisterAggregatorRoutes +// currently has no config access at all, so the alternative would be threading a +// boolean through three call layers to reach an inline conditional CI never runs. +// +// The claim is that there are NO options, not that the options are safe. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). What production gets must be exactly "+ + "NewRegisterHandler's own defaults, with nothing applied on top", len(opts)) +} + +// TestPrivateHostOptions_DisallowedHandlerIsGuarded is the behavioural half: +// zero options has to also MEAN a guarded client. The length check alone would +// still pass if the constructor's default regressed to permissive — the helper +// would be correctly returning nothing, onto a base that no longer refuses. +func TestPrivateHostOptions_DisallowedHandlerIsGuarded(t *testing.T) { + t.Parallel() + + handler := guardedRegisterHandler(t, false, "169.254.169.254") + + err := handler.verifyDomainOwnership(context.Background(), registrantDID, guardTestDomain) + + require.Error(t, err, + "a handler built from PrivateHostOptions(false) fetched from a domain resolving to the cloud "+ + "metadata endpoint. This is the branch production runs and CI never does") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "169.254.169.254 answers credential requests to anything that can reach it, and the refusal "+ + "must come from the guard rather than from the request happening to fail; got: %v", err) +} + +// TestSetHTTPClientSeam_DoesNotSilentlyDowngradeAHandlerThatWasNeverGiven is the +// footgun, stated as the property that can actually be pinned. +// +// setHTTPClient replaces the client wholesale, and three fixtures in this +// package depend on that. Two of them — register_test.go's and +// register_users_row_test.go's, both via stubClient — install a transport with a +// pinned dialer so a request for example.com reaches an httptest listener; +// newRecordingHandler installs a RoundTripper that answers nothing at all. "Wrap +// rather than replace" would break every one of them — a wrapped guard would +// refuse example.com's real address, or fail to resolve it at all under an +// egress-blocked CI — so the seam has to keep replacing. +// +// What can be pinned HERE is that a handler nobody called it on is guarded, +// which is the state every production handler is in. The other half — a +// production caller reaching for the seam at all — is closed by the method being +// unexported, and asserted as a property rather than a name by +// TestNoExportedSeamCanReplaceTheGuardedClient. +func TestSetHTTPClientSeam_DoesNotSilentlyDowngradeAHandlerThatWasNeverGiven(t *testing.T) { + t.Parallel() + + handler := NewRegisterHandler( + &fakeUserService{registered: map[string]*users.User{}}, + &fakeIdentityResolver{identities: map[string]*identity.Identity{}}, + ) + // The client POINTER, not its transport: in a build where the handler's + // client is not yet guarded the transport is nil, and testify's Same refuses + // nil with "both arguments must be pointers" — a failure that reports the + // assertion's own mechanics rather than the property under test. + guardedClient := handler.httpClient + + // A second handler, built identically, is the one that gets overridden. The + // first must be unaffected — a shared or package-level client would make one + // fixture's override everyone's. + other := NewRegisterHandler( + &fakeUserService{registered: map[string]*users.User{}}, + &fakeIdentityResolver{identities: map[string]*identity.Identity{}}, + ) + other.setHTTPClient(&http.Client{Transport: &recordingTransport{}}) + + assert.Samef(t, guardedClient, handler.httpClient, + "overriding one handler's client changed another's. The client must be per-handler state: a "+ + "package-level default would let a single test fixture unguard the production handler for "+ + "the rest of the process") + assert.NotNil(t, other.httpClient.Transport, "the override must install what it was given") +} + +// TestRegisterHTTPClient_HatchIsPerConstruction pins that the builder's argument +// is what decides, at the level the gate helper feeds. +func TestRegisterHTTPClient_HatchIsPerConstruction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + allowPrivate bool + wantBlocked bool + }{ + {name: "guarded refuses a private answer", allowPrivate: false, wantBlocked: true}, + {name: "hatched permits it", allowPrivate: true, wantBlocked: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Loopback rather than an RFC1918 address on purpose: the permissive + // row has to DIAL, and a dial to 10.0.0.1:443 hangs until the client's + // 10s ceiling on a machine with no route to it, while loopback:443 + // answers "connection refused" immediately. Both are private, so the + // guarded row is unaffected by the choice. + client := registerHTTPClient(tt.allowPrivate, + covesoauth.WithHostResolver(privateHostResolver(t, "127.0.0.1"))) + + req, err := http.NewRequestWithContext(context.Background(), + http.MethodGet, "https://"+guardTestDomain+wellKnownPath, nil) + require.NoError(t, err) + + resp, err := client.Do(req) + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "nothing answers at loopback:443, so this request must fail either way") + + if tt.wantBlocked { + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "a client built for the guarded branch must refuse an RFC1918 answer by "+ + "classification; got: %v", err) + return + } + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "a client built with the hatch open must get past classification and fail at the "+ + "dial instead; got: %v", err) + }) + } +} diff --git a/internal/api/handlers/aggregator/register_seam_export_test.go b/internal/api/handlers/aggregator/register_seam_export_test.go new file mode 100644 index 0000000..1de98af --- /dev/null +++ b/internal/api/handlers/aggregator/register_seam_export_test.go @@ -0,0 +1,95 @@ +package aggregator + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNoExportedSeamCanReplaceTheGuardedClient closes the last way a production +// caller in another package can end up with an unguarded registration handler. +// +// # THE CODEBASE WAS MAKING TWO OPPOSITE CALLS ABOUT ONE HAZARD +// +// pds/factory.go's transport seam is UNEXPORTED for exactly this reason, and +// says so: "the resolver seam these tests need must not be reachable from any +// non-test package". SetHTTPClient was the same hazard with the opposite answer +// — an exported method that discards the client NewRegisterHandler built and +// installs whatever it is handed, guard and all. Its own doc comment conceded +// the footgun and deferred the fix. This is the fix. +// +// # UNEXPORTING IS THE WHOLE MECHANISM, AND IT IS ENOUGH +// +// Every caller of the seam is a fixture in THIS package: three install a pinned +// dialer so a made-up domain reaches an httptest listener, which is why the seam +// has to keep REPLACING rather than wrapping — a wrapped guard would refuse that +// listener's loopback address or fail to resolve the name at all under an +// egress-blocked CI. Lowercasing it costs those fixtures nothing and puts the +// hazard out of reach of every package that is not this one, which is where +// production lives. +// +// # WHY THIS IS ASSERTED OVER THE SOURCE +// +// Reflection sees exported methods and not package-level functions, and neither +// is visible to a test that only calls things. Parsing the package's own +// declarations states the property directly — "nothing exported from here takes +// a client" — and it goes on holding for the next seam somebody adds, which is +// the failure mode a test naming one identifier cannot cover. +func TestNoExportedSeamCanReplaceTheGuardedClient(t *testing.T) { + t.Parallel() + + offenders := exportedHTTPClientParameters(t, ".") + + assert.Emptyf(t, offenders, + "these exported declarations take an *http.Client, so any package can hand this one an "+ + "unguarded client and discard the one NewRegisterHandler built: %s. Lowercase them — every "+ + "caller is a fixture in this package, and pds/factory.go's withTransportOptions is the "+ + "shape to copy", strings.Join(offenders, ", ")) +} + +// exportedHTTPClientParameters returns the exported declarations in dir's +// non-test Go files that accept an *http.Client. +func exportedHTTPClientParameters(t *testing.T, dir string) []string { + t.Helper() + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dir, func(info fs.FileInfo) bool { + return !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + require.NoError(t, err, "parsing the package's own sources") + require.NotEmpty(t, pkgs, "the parser found no package, so this test would pass vacuously") + + var offenders []string + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || !fn.Name.IsExported() { + continue + } + for _, param := range fn.Type.Params.List { + star, ok := param.Type.(*ast.StarExpr) + if !ok { + continue + } + sel, ok := star.X.(*ast.SelectorExpr) + if !ok { + continue + } + ident, ok := sel.X.(*ast.Ident) + if !ok || ident.Name != "http" || sel.Sel.Name != "Client" { + continue + } + offenders = append(offenders, fn.Name.Name) + } + } + } + } + return offenders +} diff --git a/internal/api/handlers/aggregator/register_test.go b/internal/api/handlers/aggregator/register_test.go index 7e9722a..5ac554e 100644 --- a/internal/api/handlers/aggregator/register_test.go +++ b/internal/api/handlers/aggregator/register_test.go @@ -8,9 +8,12 @@ import ( "encoding/json" "fmt" "io" + "net" "net/http" "net/http/httptest" + "slices" "strings" + "sync" "testing" "time" ) @@ -45,6 +48,17 @@ const ( registrantPDS = "https://pds.example.com" wellKnownPath = "/.well-known/atproto-did" + + // stubDomain is the name the stub server answers as, and it is not a free + // choice: httptest mints one certificate for every test server in the + // standard library, and that certificate carries the SANs + // [example.com, *.example.com] and nothing else. Pinning the dial (see + // stubClient) gets a request to the stub's listener whatever name it + // carries, but the TLS handshake still checks the name against the + // certificate — a request for sub.example.co.uk against this server dies + // with "certificate is valid for example.com, *.example.com". So the domain + // these tests prove ownership of is example.com, or a single label under it. + stubDomain = "example.com" ) // fakeUserService answers the two calls on the registration path and records @@ -96,9 +110,73 @@ type registerFixture struct { users *fakeUserService resolver *fakeIdentityResolver - // domain is the stub's host:port — the shape a client actually sends, with - // no scheme. + // domain is the hostname a client sends: a name, with no scheme and no + // port, which is the only shape registration accepts. domain string + + // stub records what the .well-known server was actually asked for, which is + // the binding stubClient's pinned dial would otherwise remove entirely. + stub *boundStub +} + +// boundStub is the .well-known server, with the one property the pinned dial +// takes away put back: it answers for ONE name and records every name it is +// asked for. +// +// # WHY THIS IS NOT BOOKKEEPING +// +// Registration's entire authorization rule is "you may register a DID only if +// the domain YOU NAMED publishes it". Every other assertion in this file — the +// 200, the users row, the DID mismatch, the oversized body — is downstream of +// that rule and none of them state it. +// +// The rule used to be pinned for free. The fixtures dialled the stub's own +// address, so a handler that fetched from anywhere else reached nothing and +// every test went red. Pinning the dial to make the fixtures send a hostname +// (see stubClient) removed that binding without replacing it, and httptest's +// certificate finished the job: its SANs are [example.com, *.example.com], so +// the handshake succeeds for any name under example.com. Mutation-proven — make +// verifyDomainOwnership ignore its `domain` argument and fetch +// https://attacker.example.com/.well-known/atproto-did, and all seven tests +// built on this fixture stay green. +// +// Recording the HOST is what closes it. The dial goes wherever the test says, +// but the Host header carries the name the handler chose, and that name is the +// one thing the authorization rule is about. Anything else gets a 404, so the +// mutation above fails loudly rather than verifying a domain nobody claimed. +type boundStub struct { + mu sync.Mutex + // requests holds "host + path" for every request the stub answered, which + // is the whole of what a .well-known fetch is: which domain, and which + // document on it. + requests []string + + // host is the only name this stub answers as. + host string + next http.Handler +} + +func (s *boundStub) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests = append(s.requests, r.Host+r.URL.Path) + s.mu.Unlock() + + // A real domain answers for its own name and nothing else. Returning 404 + // rather than serving the DID anyway is what makes a handler that fetched + // the wrong host fail every ownership check in this file, not merely the + // one test that inspects the recording. + if r.Host != s.host { + http.NotFound(w, r) + return + } + s.next.ServeHTTP(w, r) +} + +// fetched returns what the stub was asked for, in order. +func (s *boundStub) fetched() []string { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.requests) } // newRegisterFixture starts a stub domain serving wellKnown over TLS and builds @@ -106,11 +184,13 @@ type registerFixture struct { func newRegisterFixture(t *testing.T, wellKnown http.Handler) *registerFixture { t.Helper() + bound := &boundStub{host: stubDomain, next: wellKnown} + // TLS rather than plaintext because the handler builds an https:// URL and // refuses to do otherwise — a domain that cannot serve HTTPS cannot prove // ownership. stub.Client() trusts exactly this server's certificate, which // is narrower than disabling verification altogether. - stub := httptest.NewTLSServer(wellKnown) + stub := httptest.NewTLSServer(bound) t.Cleanup(stub.Close) userService := &fakeUserService{registered: map[string]*users.User{}} @@ -127,16 +207,54 @@ func newRegisterFixture(t *testing.T, wellKnown http.Handler) *registerFixture { } handler := NewRegisterHandler(userService, resolver) - handler.SetHTTPClient(stub.Client()) + handler.setHTTPClient(stubClient(t, stub)) return ®isterFixture{ handler: handler, users: userService, resolver: resolver, - domain: strings.TrimPrefix(stub.URL, "https://"), + domain: stubDomain, + stub: bound, } } +// stubClient is stub.Client() with its dial pinned to the stub's own listener, +// so a request for https://example.com/... reaches this server without +// example.com ever being resolved or dialled. +// +// The indirection is new, and it exists because of the validation under test. +// These fixtures used to hand the handler the stub's own address as the domain +// — literally 127.0.0.1:PORT — which registration now refuses as not-a-hostname +// long before an HTTP client is touched, and refusing it is the point. Pinning +// the dial is what lets the tests below go on asserting the .well-known +// contract while sending a domain of the shape a real client sends. +// +// Nothing about what those tests prove changes: the stub still serves over TLS, +// the client still trusts exactly the stub's certificate and no other, and the +// name in the URL is still checked against that certificate. Only the address +// the connection goes to is decided here rather than by a resolver. +func stubClient(t *testing.T, stub *httptest.Server) *http.Client { + t.Helper() + + client := stub.Client() + transport, ok := client.Transport.(*http.Transport) + if !ok { + t.Fatalf("httptest client transport is %T, want *http.Transport — the dial cannot be pinned, "+ + "so every test in this file would be talking to whatever example.com resolves to", client.Transport) + } + + // Taken from the listener rather than written down: an address literal here + // would be both a lie (the port is chosen at listen time) and a test-audit + // violation. + addr := stub.Listener.Addr().String() + transport.DialContext = func(ctx context.Context, network, requested string) (net.Conn, error) { + // requested is "example.com:443", discarded on purpose. + _ = requested + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + return client +} + // register posts a well-formed registration request. func (f *registerFixture) register(t *testing.T, did, domain string) *httptest.ResponseRecorder { t.Helper() @@ -232,6 +350,42 @@ func TestRegister_RegistersAnAggregatorThatProvesItsDomain(t *testing.T) { } } +// TestRegister_ProvesOwnershipOfTheDomainTheCallerNamed states registration's +// authorization rule outright, because no other test in this file does. +// +// The rule is: you may register a DID only if THE DOMAIN YOU NAMED publishes +// it. Every neighbouring test asserts a consequence — a 200, a users row, a 401 +// on a mismatched DID — and every one of those consequences is satisfied by a +// handler that fetched the right document from the wrong domain. That is not a +// hypothetical: the mutation is a one-line change to verifyDomainOwnership, +// dropping its `domain` argument in favour of a fixed +// https://attacker.example.com/.well-known/atproto-did, and before this test it +// left the whole file green. An attacker who controls any host the AppView can +// reach could then register any DID that host publishes. +// +// What makes the assertion possible is boundStub recording the Host header: +// the connection goes wherever the pinned dial sends it, but the Host is the +// name the HANDLER chose, and the rule is about that name alone. +func TestRegister_ProvesOwnershipOfTheDomainTheCallerNamed(t *testing.T) { + f := newRegisterFixture(t, wellKnownServing(registrantDID)) + + rec := f.register(t, registrantDID, f.domain) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200. Body: %s", rec.Code, rec.Body.String()) + } + + fetched := f.stub.fetched() + want := []string{f.domain + wellKnownPath} + if !slices.Equal(fetched, want) { + t.Fatalf("the handler fetched %v, want exactly %v.\n"+ + "Registration's authorization rule is that the DOMAIN THE CALLER NAMED publishes the DID — "+ + "a fetch to any other host proves ownership of something nobody claimed, and a fetch to any "+ + "other path is not the .well-known document at all. The stub's dial is pinned, so the Host "+ + "header is the only place the handler's choice is visible.", + fetched, want) + } +} + // did:web is the other supported method, and it reaches registration by a // different branch of the format check than did:plc. func TestRegister_AcceptsDIDWeb(t *testing.T) { diff --git a/internal/api/handlers/aggregator/register_users_row_test.go b/internal/api/handlers/aggregator/register_users_row_test.go index c4a79a3..0f1d418 100644 --- a/internal/api/handlers/aggregator/register_users_row_test.go +++ b/internal/api/handlers/aggregator/register_users_row_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "strings" "testing" "time" @@ -74,13 +73,16 @@ func TestRegister_WritesTheAggregatorIntoTheUsersTable(t *testing.T) { userService := users.NewUserService(userRepo, resolver, pdsURL, nil, "") handler := NewRegisterHandler(userService, resolver) - handler.SetHTTPClient(stub.Client()) + handler.setHTTPClient(stubClient(t, stub)) body, err := json.Marshal(RegisterRequest{ DID: did, - // The client sends a bare host:port, and the handler is what turns it - // into an https:// URL. - Domain: strings.TrimPrefix(stub.URL, "https://"), + // The client sends a bare hostname, and the handler is what turns it + // into an https:// URL. It cannot send the stub's own address: that is + // 127.0.0.1:PORT, which registration now refuses as not-a-hostname + // before any HTTP client is touched. stubClient pins the dial so this + // name reaches the stub anyway — see register_test.go. + Domain: stubDomain, }) require.NoError(t, err) diff --git a/internal/api/handlers/imageproxy/avatar_serving_test.go b/internal/api/handlers/imageproxy/avatar_serving_test.go index c0da524..de9996a 100644 --- a/internal/api/handlers/imageproxy/avatar_serving_test.go +++ b/internal/api/handlers/imageproxy/avatar_serving_test.go @@ -65,7 +65,18 @@ func provisionCommunityAvatar(t *testing.T, width, height int, fill color.Color) db := testkit.DB(t) endpoints := testkit.Endpoints() - identityConfig := identity.DefaultConfig() + // The hatch, because the PLC this resolver is pointed at is the test stack's + // own and listens on loopback — which is exactly the address class the + // resolver's SSRF guard refuses. It is the same repair the loopback fixtures + // in core/blobs and core/blueskypost carry, and it is a property of THIS + // construction rather than of the environment: cmd/server's + // productionPLCResolver is built from a bare identity.DefaultConfig() and + // stays guarded even in dev. + // + // Nothing here is a test OF the guard. That lives in + // internal/atproto/identity/resolver_guard_test.go, whose configs are built + // without this option and assert the PLC listener is never reached. + identityConfig := identity.DefaultConfig(identity.WithPrivateHostsAllowed()) identityConfig.PLCURL = endpoints.PLC.BaseURL resolver := identity.NewResolver(db, identityConfig) @@ -78,9 +89,10 @@ func provisionCommunityAvatar(t *testing.T, width, height int, fill color.Color) endpoints.PDS.BaseURL, fixtures.InstanceDID(), handleDomain, - communities.NewPDSAccountProvisioner(handleDomain, endpoints.PDS.BaseURL), + communities.NewPDSAccountProvisioner(handleDomain, endpoints.PDS.BaseURL, communities.PrivateHostOptions(true)...), nil, // no PDS client factory: provisioning uses the password session - blobs.NewBlobService(endpoints.PDS.BaseURL), + blobs.NewBlobService(endpoints.PDS.BaseURL, blobs.PrivateHostOptions(true)...), + communities.PrivateHostOptions(true)..., ) // The provisioner prefixes "c-", and a PDS handle's local label is capped diff --git a/internal/api/handlers/imageproxy/blocked_status_test.go b/internal/api/handlers/imageproxy/blocked_status_test.go new file mode 100644 index 0000000..fc9cb1f --- /dev/null +++ b/internal/api/handlers/imageproxy/blocked_status_test.go @@ -0,0 +1,88 @@ +package imageproxy + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "Coves/internal/core/imageproxy" +) + +// The external half of the ErrPDSBlocked contract. +// +// The sentinel exists so that in-process callers can tell "the guard refused +// this destination" from "the fetch failed" — see the fetcher's own guard tests. +// Outside the process it must be invisible. A stranger who names an internal +// address in a DID document has to get back exactly what they would get for a +// PDS that is simply unreachable, because the point of this behavior is +// removing the response-status oracle that currently maps 404 to "the port +// answered", 502 to "refused" and 504 to "filtered". +// +// A new status for blocked addresses would be the same oracle wearing a new +// number, and a strictly better one: it would say "this address is internal" +// rather than merely "something happened here". +// +// handleServiceError's `default` branch serves 500, so an ErrPDSBlocked that +// nobody adds to that switch does not fall back to something harmless — it +// becomes a THIRD distinguishable answer. + +// blockedStatusRequest drives the handler once with a service that fails with +// the given error, and returns the recorder. +func blockedStatusRequest(t *testing.T, serviceErr error) *httptest.ResponseRecorder { + t.Helper() + + handler := NewHandler( + &mockService{ + getImageFunc: func(context.Context, string, string, string, string) ([]byte, error) { + return nil, serviceErr + }, + }, + resolverForPDS("https://pds.example.com"), + ) + + req := createTestRequest(http.MethodGet, "/img/avatar/plain/"+validTestDID+"/"+validTestCID, map[string]string{ + "preset": "avatar", + "did": validTestDID, + "cid": validTestCID, + }) + + w := httptest.NewRecorder() + handler.HandleImage(w, req) + return w +} + +// TestHandler_HandleImage_BlockedIsIndistinguishableFromAFailedFetch asserts the +// two responses are the same response — status and body both. +// +// Asserting 502 alone would not be enough. A body of "blocked: private address" +// under a 502 leaks the classification just as effectively as a distinct status +// would, to anyone reading the response rather than the status line. +func TestHandler_HandleImage_BlockedIsIndistinguishableFromAFailedFetch(t *testing.T) { + t.Parallel() + + blocked := blockedStatusRequest(t, imageproxy.ErrPDSBlocked) + failed := blockedStatusRequest(t, imageproxy.ErrPDSFetchFailed) + + require.Equalf(t, http.StatusBadGateway, blocked.Code, + "a guard refusal was served as %d rather than 502. handleServiceError's default branch is 500, "+ + "so an ErrPDSBlocked that was never added to the switch does not fail safe — it becomes a "+ + "third distinguishable answer, and a stranger probing addresses through a DID document "+ + "reads it as 'this one is internal'. Body: %s", blocked.Code, blocked.Body.String()) + + assert.Equalf(t, failed.Code, blocked.Code, + "a blocked address answers %d while an ordinary fetch failure answers %d. The internal "+ + "distinction is deliberate and must not reach the wire: two statuses is the port-scan "+ + "oracle this behavior exists to remove", blocked.Code, failed.Code) + + assert.Equalf(t, failed.Body.String(), blocked.Body.String(), + "the response bodies differ (%q vs %q). Same status with a different body is the same oracle, "+ + "read one line further down", blocked.Body.String(), failed.Body.String()) + + assert.Equalf(t, "no-store", blocked.Header().Get("Cache-Control"), + "a refusal must stay uncacheable like every other error on this route: it sits behind a CDN "+ + "whose success responses advertise a one-year immutable lifetime") +} diff --git a/internal/api/handlers/imageproxy/handler.go b/internal/api/handlers/imageproxy/handler.go index a56228e..c049d17 100644 --- a/internal/api/handlers/imageproxy/handler.go +++ b/internal/api/handlers/imageproxy/handler.go @@ -151,7 +151,21 @@ func handleServiceError(w http.ResponseWriter, err error) { writeErrorResponse(w, http.StatusNotFound, "blob not found") case errors.Is(err, imageproxy.ErrPDSTimeout): writeErrorResponse(w, http.StatusGatewayTimeout, "request timed out") - case errors.Is(err, imageproxy.ErrPDSFetchFailed): + // ONE BRANCH FOR BOTH, so the status and the body cannot drift apart. A + // guard refusal must be indistinguishable from a PDS that is simply + // unreachable: the endpoint comes from a DID document anyone can mint, and + // this route needs no credential, so a distinct status — or the same status + // with a distinct body, which is the same oracle read one line further + // down — tells a stranger probing addresses which ones are internal. + // + // The internal distinction survives in the log line below and in + // errors.Is at every other caller; only the response is flattened. + case errors.Is(err, imageproxy.ErrPDSFetchFailed), errors.Is(err, imageproxy.ErrPDSBlocked): + if errors.Is(err, imageproxy.ErrPDSBlocked) { + slog.Warn("[IMAGE-PROXY] SSRF guard refused a PDS endpoint", + "error", err, + ) + } writeErrorResponse(w, http.StatusBadGateway, "failed to fetch blob from PDS") case errors.Is(err, imageproxy.ErrInvalidPreset): writeErrorResponse(w, http.StatusBadRequest, "invalid preset") diff --git a/internal/api/handlers/imageproxy/proxy_serving_test.go b/internal/api/handlers/imageproxy/proxy_serving_test.go index 6fa5a81..e2e22f2 100644 --- a/internal/api/handlers/imageproxy/proxy_serving_test.go +++ b/internal/api/handlers/imageproxy/proxy_serving_test.go @@ -124,7 +124,14 @@ func newProxyServer(t *testing.T, resolver identity.Resolver, fetchTimeout time. service, err := imageproxycore.NewService( cache, imageproxycore.NewProcessor(), - imageproxycore.NewPDSFetcher(fetchTimeout, 10), + // The hatch, for the same reason the T0 fixtures in + // core/imageproxy/fetcher_test.go carry it: every PDS these tests point + // at is an httptest server or the CI stack's own PDS, and both listen on + // loopback — which is precisely what the fetcher's SSRF guard refuses. + // Nothing here is a test OF the guard; that lives in + // core/imageproxy/fetcher_guard_test.go, whose fetchers are built + // without this option and assert the listener is never reached. + imageproxycore.NewPDSFetcher(fetchTimeout, 10, imageproxycore.WithPrivateHostsAllowed()), imageproxycore.Config{ Enabled: true, CachePath: cacheDir, diff --git a/internal/api/handlers/imageproxy/roundtrip_serving_test.go b/internal/api/handlers/imageproxy/roundtrip_serving_test.go index 559454e..5b29841 100644 --- a/internal/api/handlers/imageproxy/roundtrip_serving_test.go +++ b/internal/api/handlers/imageproxy/roundtrip_serving_test.go @@ -55,7 +55,11 @@ func TestImageProxy_EmittedURLsAreFetchable(t *testing.T) { db := testkit.DB(t) endpoints := testkit.Endpoints() - identityConfig := identity.DefaultConfig() + // The hatch: the stack's PLC listens on loopback, which the resolver's SSRF + // guard refuses by default. Per-construction, not ambient — see the note in + // avatar_serving_test.go, and the guard's own tests in + // internal/atproto/identity/resolver_guard_test.go. + identityConfig := identity.DefaultConfig(identity.WithPrivateHostsAllowed()) identityConfig.PLCURL = endpoints.PLC.BaseURL // The proxy resolves the community's DID through the stack's PLC directory to // find the blob, so a fabricated DID would never get past resolution. @@ -71,9 +75,10 @@ func TestImageProxy_EmittedURLsAreFetchable(t *testing.T) { endpoints.PDS.BaseURL, fixtures.InstanceDID(), handleDomain, - communities.NewPDSAccountProvisioner(handleDomain, endpoints.PDS.BaseURL), + communities.NewPDSAccountProvisioner(handleDomain, endpoints.PDS.BaseURL, communities.PrivateHostOptions(true)...), nil, // no PDS client factory: provisioning uses the password session - blobs.NewBlobService(endpoints.PDS.BaseURL), + blobs.NewBlobService(endpoints.PDS.BaseURL, blobs.PrivateHostOptions(true)...), + communities.PrivateHostOptions(true)..., ) // The provisioner prefixes "c-", and a PDS handle's local label is capped at diff --git a/internal/api/handlers/post/harness_test.go b/internal/api/handlers/post/harness_test.go index b1956e4..02940f0 100644 --- a/internal/api/handlers/post/harness_test.go +++ b/internal/api/handlers/post/harness_test.go @@ -75,6 +75,7 @@ func newCreateStack(t *testing.T, db *sql.DB) createStack { nil, // no provisioner: no test here creates a community account nil, // no PDS client factory nil, // no blob service + communities.PrivateHostOptions(true)..., ) postService := posts.NewPostService( diff --git a/internal/api/routes/aggregator.go b/internal/api/routes/aggregator.go index 13f4fa0..a6ee2e9 100644 --- a/internal/api/routes/aggregator.go +++ b/internal/api/routes/aggregator.go @@ -15,12 +15,19 @@ import ( // RegisterAggregatorRoutes registers aggregator-related XRPC endpoints // Following Bluesky's pattern for feed generators and labelers +// +// registerOpts is threaded straight through to the registration handler, and +// exists because that handler makes an outbound request to a domain an +// unauthenticated caller supplies. This function has no config of its own, so +// the SSRF gate has to arrive from cmd/server; it is variadic so that the +// guarded construction stays the one a caller gets by writing nothing. func RegisterAggregatorRoutes( r chi.Router, aggregatorService aggregators.Service, communityService communities.Service, userService users.UserService, identityResolver identity.Resolver, + registerOpts ...aggregator.RegisterHandlerOption, ) { // Create query handlers getServicesHandler := aggregator.NewGetServicesHandler(aggregatorService) @@ -28,7 +35,7 @@ func RegisterAggregatorRoutes( listForCommunityHandler := aggregator.NewListForCommunityHandler(aggregatorService, communityService) // Create registration handler - registerHandler := aggregator.NewRegisterHandler(userService, identityResolver) + registerHandler := aggregator.NewRegisterHandler(userService, identityResolver, registerOpts...) // Query endpoints (public - no auth required) // GET /xrpc/social.coves.aggregator.getServices?dids=did:plc:abc,did:plc:def diff --git a/internal/atproto/identity/factory.go b/internal/atproto/identity/factory.go index 3158c3c..c2f4855 100644 --- a/internal/atproto/identity/factory.go +++ b/internal/atproto/identity/factory.go @@ -5,13 +5,122 @@ import ( "net/http" "os" "time" + + covesoauth "Coves/internal/atproto/oauth" ) // Config holds configuration for the identity resolver type Config struct { - HTTPClient *http.Client - PLCURL string - CacheTTL time.Duration + PLCURL string + CacheTTL time.Duration + + // httpClient is the client every resolver built from this config dials + // through. UNEXPORTED, because an exported one is a guard-replacing seam + // with nothing left to grep for: `identity.Config{HTTPClient: unguarded}` + // from any package in the tree, honoured by NewResolver, with no option and + // no coves:allow-ssrf-hatch marker. withHTTPClient is the way in, and + // factory_seam_test.go is the fence. See withHTTPClient. + httpClient *http.Client + + // allowPrivateHosts disables the SSRF guard that refuses private, loopback + // and link-local addresses on the client this config builds. + // + // IT IS PER-CONSTRUCTION, AND THAT IS THE WHOLE DESIGN. cmd/server builds + // two resolvers from this same function with OPPOSITE requirements: + // a.identityResolver points at a PLC that in dev is on loopback and needs + // the hatch there, while productionPLCResolver is always aimed at + // plc.directory and must stay guarded in every environment — dev most of + // all, since a dev machine is exactly where the loopback it would otherwise + // dial is a real PLC, a real Postgres and a real PDS. + // + // So this must never become an ambient read of IS_DEV_ENV inside this + // package. An ambient hatch satisfies every obvious test and silently + // unguards the production resolver; + // TestTwoResolvers_OppositeRequirementsInTheSameProcess is the one + // assertion that notices. + allowPrivateHosts bool +} + +// ConfigOption configures a resolver Config. +type ConfigOption func(*Config) + +// WithPrivateHostsAllowed disables the SSRF address guard on the resolver's +// HTTP client. +// +// THE NAME IS THE CONTRACT: only a caller that knows it is pointing this +// resolver at its own machine may use it. It is greppable, which is how "which +// resolvers have the guard off" stays an answerable question. +func WithPrivateHostsAllowed() ConfigOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(c *Config) { c.allowPrivateHosts = true } +} + +// withHTTPClient substitutes the client a resolver dials through. +// +// # IT IS UNEXPORTED, AND THAT IS THE POINT +// +// This replaces the guard wholesale — not the address policy, which +// WithPrivateHostsAllowed carries and which is greppable so that "which +// resolvers have the guard off" stays an answerable question, but the entire +// vetted transport. There is no policy left to grep for afterwards, so the only +// safe number of packages able to do it is one: this one. +// +// It exists because a guard test that builds its own client proves only that +// internal/atproto/oauth works. The seam has to point at the client production +// actually constructs, and every peer package that needed the same thing kept it +// unexported for the same reason — jetstream's withWellKnownHTTPClient, blobs' +// setHTTPClient, the shared transport's withTransportOptions. +// +// It was an EXPORTED FIELD on Config until factory_seam_test.go removed it: +// `identity.Config{HTTPClient: unguarded}` from any package in the tree, honoured +// by NewResolver, with no option and no marker. +func withHTTPClient(client *http.Client) ConfigOption { + return func(c *Config) { c.httpClient = client } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to DefaultConfig: the hatch when it is set, and NOTHING +// when it is not. +// +// It mirrors oauth.PrivateAddressOptions, and it is a function rather than an +// `if` in cmd/server/wiring.go for the reason documented there: `.env.ci:140` +// sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at every call +// site holding such a boolean. A unit test against this function is the only +// place in the repository where the branch production actually runs is ever +// evaluated. Do not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly DefaultConfig's own +// defaults. +func PrivateHostOptions(allowPrivate bool) []ConfigOption { + if !allowPrivate { + return nil + } + return []ConfigOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + +// resolverHTTPClient builds the client every resolver in this package dials +// through: the SSRF-safe transport of internal/atproto/oauth, which resolves +// the host, refuses private, loopback and link-local addresses, and then dials +// only the address it vetted — closing the check-then-dial window a naive guard +// leaves open. +// +// THIS PACKAGE IS REACHED WITH NO CREDENTIAL. The /img route resolves a DID +// before it fetches anything, and a did:web carries its own hostname — the +// resolver fetches https:///.well-known/did.json, so the destination +// is a string in the URL a stranger typed. Indigo's AllowedTLD check refuses +// eight reserved TLDs — local, arpa, invalid, localhost, internal, example, +// onion and alt — which is a list of NAMES and stops nothing else: a public +// hostname with a 127.0.0.1 A record has a perfectly ordinary TLD, and the TLD +// is the only thing that check looks at. +// +// The 10s timeout is this package's own and predates the guard; the shared +// client ships a 15s ceiling, so it is re-applied here rather than inherited. +// factory_test.go pins it with the reason — an unbounded client here hangs an +// ingestion worker on an unresponsive PDS. +func resolverHTTPClient(allowPrivateHosts bool) *http.Client { + client := covesoauth.NewSSRFSafeHTTPClient(covesoauth.PrivateAddressOptions(allowPrivateHosts)...) + client.Timeout = 10 * time.Second + return client } // DefaultConfig returns a configuration with sensible defaults. @@ -25,16 +134,31 @@ type Config struct { // Callers that deliberately need the production directory - resolving real // Bluesky handles, for instance - must set PLCURL explicitly after calling // this, as cmd/server does for its read-only Bluesky resolver. -func DefaultConfig() Config { +// +// THE CLIENT IT BUILDS IS GUARDED UNLESS THE CALLER SAYS OTHERWISE, so that +// forgetting is safe: DefaultConfig() with no arguments is what the next caller +// will write, and what productionPLCResolver already writes. +func DefaultConfig(opts ...ConfigOption) Config { plcURL := os.Getenv("PLC_DIRECTORY_URL") if plcURL == "" { plcURL = "https://plc.directory" } - return Config{ - PLCURL: plcURL, - CacheTTL: 24 * time.Hour, // Cache for 24 hours - HTTPClient: &http.Client{Timeout: 10 * time.Second}, + config := Config{ + PLCURL: plcURL, + CacheTTL: 24 * time.Hour, // Cache for 24 hours + } + for _, opt := range opts { + opt(&config) + } + // After the options, because the hatch one of them may have set is what this + // client is built from — and only when none of them supplied a client, so + // that withHTTPClient's substitution is not immediately overwritten. The two + // options are exclusive in practice: nothing wants a substituted client and + // an address policy applied to a client it is not using. + if config.httpClient == nil { + config.httpClient = resolverHTTPClient(config.allowPrivateHosts) } + return config } // NewResolver creates a new identity resolver with caching @@ -46,12 +170,19 @@ func NewResolver(db *sql.DB, config Config) Resolver { if config.CacheTTL == 0 { config.CacheTTL = 24 * time.Hour } - if config.HTTPClient == nil { - config.HTTPClient = &http.Client{Timeout: 10 * time.Second} + // GUARDED ON THE SAME TERMS AS DefaultConfig's. `NewResolver(db, + // Config{PLCURL: ...})` is an ordinary spelling in this tree, and a caller + // who writes it has said nothing about private hosts — so they get the + // guard, and the safety of a resolver does not depend on which of two + // equally ordinary constructions its author happened to reach for. A + // caller that does want the hatch sets it through DefaultConfig's option, + // which fills HTTPClient and skips this branch entirely. + if config.httpClient == nil { + config.httpClient = resolverHTTPClient(config.allowPrivateHosts) } // Create base resolver using Indigo - base := newBaseResolver(config.PLCURL, config.HTTPClient) + base := newBaseResolver(config.PLCURL, config.httpClient) // Wrap with caching using PostgreSQL cache := NewPostgresCache(db, config.CacheTTL) diff --git a/internal/atproto/identity/factory_seam_test.go b/internal/atproto/identity/factory_seam_test.go new file mode 100644 index 0000000..d93356e --- /dev/null +++ b/internal/atproto/identity/factory_seam_test.go @@ -0,0 +1,131 @@ +package identity + +import ( + "net/http" + "reflect" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + indigoIdentity "github.com/bluesky-social/indigo/atproto/identity" +) + +// The client-replacing seam on Config, and why it may not be exported. +// +// # WHAT THIS PACKAGE'S GUARD IS WORTH +// +// resolverHTTPClient is the only client a resolver in this package dials +// through, and every constructor routes to it: DefaultConfig fills the field +// after applying options, and NewResolver fills it when a caller left it nil, so +// that `NewResolver(db, Config{PLCURL: ...})` — an ordinary spelling in this +// tree — is guarded rather than accidentally safe. That is a closed set of +// constructions. +// +// An EXPORTED *http.Client field on Config reopens it. Any package in the tree +// can write `identity.Config{HTTPClient: unguarded}` and NewResolver will honour +// it: no option, no coves:allow-ssrf-hatch marker, nothing for the audit to +// grep, and a resolver that fetches whatever a did:web's hostname points at +// with no address check. The hatch this package DOES offer — +// WithPrivateHostsAllowed — is greppable precisely so "which resolvers have the +// guard off" stays an answerable question; an exported field makes the answer +// "unknown". +// +// Every peer package that needed the same test seam made it unexported for this +// reason: jetstream's withWellKnownHTTPClient, blobs' setHTTPClient, the shared +// transport's withTransportOptions. This package was the one that did not. + +// TestConfig_ExposesNoClientReplacingField is the fence, and it is stated over +// the TYPE rather than over any particular field name so that it also catches +// the next one somebody adds. +// +// Reflection is the only way to assert this: the property is "no package outside +// this one can substitute a client", and a test written in this package can +// reach the unexported field regardless. What reflection can see is exactly what +// an outside caller can reach. +func TestConfig_ExposesNoClientReplacingField(t *testing.T) { + t.Parallel() + + configType := reflect.TypeOf(Config{}) + clientTypes := map[reflect.Type]bool{ + reflect.TypeOf(&http.Client{}): true, + reflect.TypeOf(http.Client{}): true, + reflect.TypeOf((*http.RoundTripper)(nil)).Elem(): true, + } + + for i := 0; i < configType.NumField(); i++ { + field := configType.Field(i) + if !field.IsExported() { + continue + } + assert.Falsef(t, clientTypes[field.Type], + "Config.%s is an exported %s, so any package in the tree can hand this resolver an "+ + "UNGUARDED client — no option, no coves:allow-ssrf-hatch marker, nothing to grep. This "+ + "package resolves did:web documents, which means it fetches https:///.well-known/"+ + "did.json where the hostname is a string a stranger typed; the address guard is the only "+ + "thing between that and a loopback PLC, Postgres or PDS on a dev machine. Every peer "+ + "package made the same seam unexported (withWellKnownHTTPClient, setHTTPClient, "+ + "withTransportOptions) — see withHTTPClient in this package", + field.Name, field.Type) + } +} + +// TestWithHTTPClient_IsHonouredByEveryConstructor keeps the seam WORKING once it +// is unexported, because a seam that is merely hidden is one the next author +// re-exports. +// +// Both constructions are covered: through DefaultConfig, where the option runs +// before the client is built, and through NewResolver's own nil branch, which is +// what a hand-built Config degrades to. +func TestWithHTTPClient_IsHonouredByEveryConstructor(t *testing.T) { + t.Parallel() + + // A timeout no default in this package uses, so the assertion cannot be + // satisfied by the guarded client this option is meant to displace: + // resolverHTTPClient's is 10s. + const marker = 3 * time.Second + require.NotEqual(t, marker, 10*time.Second, "the marker must differ from resolverHTTPClient's own timeout") + + t.Run("through DefaultConfig", func(t *testing.T) { + t.Parallel() + + config := DefaultConfig(withHTTPClient(&http.Client{Timeout: marker})) + config.PLCURL = "http://plc.invalid:3002" + config.CacheTTL = 90 * time.Second + + dir := baseDirectoryOf(t, NewResolver(nil, config)) + assert.Equal(t, marker, dir.HTTPClient.Timeout, + "the option was applied but the client it names did not reach the directory. DefaultConfig "+ + "fills the client AFTER options run, so an implementation that overwrites unconditionally "+ + "leaves this seam dead and every test using it silently exercising the default client") + assert.Equal(t, "http://plc.invalid:3002", dir.PLCURL, + "the explicit configuration around the option must survive it") + }) + + t.Run("through NewResolver", func(t *testing.T) { + t.Parallel() + + config := Config{PLCURL: "http://plc.invalid:3002"} + withHTTPClient(&http.Client{Timeout: marker})(&config) + + dir := baseDirectoryOf(t, NewResolver(nil, config)) + assert.Equal(t, marker, dir.HTTPClient.Timeout, + "NewResolver replaced a client the caller supplied. Its nil branch exists to guard callers "+ + "who said NOTHING about a client; one that fires regardless would also overwrite the seam") + }) +} + +// baseDirectoryOf digs out the indigo directory a resolver ended up dialling +// through, which is the only place the client is observable. +func baseDirectoryOf(t *testing.T, resolver Resolver) *indigoIdentity.BaseDirectory { + t.Helper() + + caching, ok := resolver.(*cachingResolver) + require.Truef(t, ok, "NewResolver must return the caching resolver these tests drive, got %T", resolver) + base, ok := caching.base.(*baseResolver) + require.Truef(t, ok, "the caching resolver must wrap the base resolver, got %T", caching.base) + dir, ok := base.directory.(*indigoIdentity.BaseDirectory) + require.Truef(t, ok, "the base resolver must hold an indigo BaseDirectory, got %T", base.directory) + return dir +} diff --git a/internal/atproto/identity/factory_test.go b/internal/atproto/identity/factory_test.go index 73d3852..cf4d7de 100644 --- a/internal/atproto/identity/factory_test.go +++ b/internal/atproto/identity/factory_test.go @@ -34,8 +34,8 @@ func TestDefaultConfig_PrefersTheConfiguredPLCDirectory(t *testing.T) { "the Makefile exports .env.dev, so this is how every locally-built resolver reaches the "+ "stack's own PLC instead of the public one") assert.Equal(t, 24*time.Hour, config.CacheTTL) - require.NotNil(t, config.HTTPClient) - assert.Equal(t, 10*time.Second, config.HTTPClient.Timeout, + require.NotNil(t, config.httpClient) + assert.Equal(t, 10*time.Second, config.httpClient.Timeout, "an unbounded client here hangs an ingestion worker on an unresponsive PDS") } @@ -70,7 +70,7 @@ func TestNewResolver_FillsInMissingConfiguration(t *testing.T) { require.True(t, ok) assert.Equal(t, "https://plc.directory", dir.PLCURL) // coves:allow-public-host: asserting the documented production default, not an endpoint any test dials assert.Equal(t, 10*time.Second, dir.HTTPClient.Timeout, - "a nil HTTPClient in the config must be replaced before it reaches Indigo, which dereferences it") + "a config carrying no client must have one filled in before it reaches Indigo, which dereferences it") cache, ok := caching.cache.(*postgresCache) require.True(t, ok) @@ -82,11 +82,17 @@ func TestNewResolver_FillsInMissingConfiguration(t *testing.T) { func TestNewResolver_HonoursAnExplicitConfiguration(t *testing.T) { t.Parallel() - resolver := NewResolver(nil, Config{ - PLCURL: "http://plc.invalid:3002", - CacheTTL: 90 * time.Second, - HTTPClient: &http.Client{Timeout: 3 * time.Second}, - }) + // The client goes in through withHTTPClient rather than a struct literal: + // the field is unexported so that no package outside this one can replace + // the guarded transport without an option to grep for. See + // factory_seam_test.go. + config := Config{ + PLCURL: "http://plc.invalid:3002", + CacheTTL: 90 * time.Second, + } + withHTTPClient(&http.Client{Timeout: 3 * time.Second})(&config) + + resolver := NewResolver(nil, config) caching := resolver.(*cachingResolver) dir := caching.base.(*baseResolver).directory.(*indigoIdentity.BaseDirectory) diff --git a/internal/atproto/identity/resolver_guard_test.go b/internal/atproto/identity/resolver_guard_test.go new file mode 100644 index 0000000..230f216 --- /dev/null +++ b/internal/atproto/identity/resolver_guard_test.go @@ -0,0 +1,360 @@ +package identity + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The identity resolver's HTTP client, against a PLC directory nobody vetted. +// +// # WHY THIS SITE IS UNAUTHENTICATED +// +// The /img route resolves a DID before it fetches anything, and it does so with +// no credential of any kind. A did:web carries its own hostname — the resolver +// fetches https:///.well-known/did.json — so the destination is a +// string in the URL a stranger typed. Indigo's AllowedTLD check refuses eight +// reserved TLDs — local, arpa, invalid, localhost, internal, example, onion and +// alt — and stops nothing else: a public hostname with a 127.0.0.1 A record has +// a perfectly ordinary TLD, and the TLD is all that check looks at. +// +// # THE TRAP THIS FILE EXISTS FOR: TWO RESOLVERS, OPPOSITE REQUIREMENTS +// +// cmd/server builds two, from the same DefaultConfig, and they need the guard +// set differently: +// +// - a.identityResolver (wiring.go:205) points at cfg.Identity.ResolverPLCURL, +// which in dev is a PLC on loopback. It NEEDS the hatch when IsDevEnv. +// - productionPLCResolver (wiring.go:357) always points at plc.directory, +// because quoted Bluesky posts name real handles the dev PLC cannot resolve. +// It must stay GUARDED even in dev. +// +// So the decision cannot live in this package as an ambient "are we in dev" +// read. It has to be per-construction, and +// TestTwoResolvers_OppositeRequirementsInTheSameProcess is the assertion that +// says so: it builds both in one process with IS_DEV_ENV set and requires them +// to behave differently. A hatch hoisted to package level passes every other +// test in this file and fails that one. +// +// # WHY THESE DRIVE THE BASE RESOLVER +// +// cachingResolver.ResolveDID reads the identity_cache table BEFORE it calls the +// base resolver — the same shape as the image proxy's cache short-circuit, and +// the same false-pass: a warm cache returns a document without a fetch and +// asserts nothing about the guard. Driving newBaseResolver directly removes +// both the cache and the Postgres dependency, which is what keeps these in T0 +// where the guarded branch is the only branch CI ever evaluates. +// +// # WHY REACHABILITY, NOT ONLY THE ERROR +// +// Mutation testing produced a guard that classified correctly, emitted +// a byte-identical message, and refused the request AFTER delivering it. For a +// destination a stranger named, the packet leaving IS the SSRF. Every case here +// stands up a real listener and asserts its handler never ran. + +// testDID is a syntactically valid did:plc. The value is irrelevant — nothing +// below gets far enough for it to matter — but syntax.ParseDID rejects a +// malformed one before any client is used, which would make a guard test pass +// for the wrong reason. +const testDID = "did:plc:z72i7hdynmk6r22z27h6tvur" + +// countingPLC is a PLC directory that answers a DID lookup and records whether +// anything ever reached it. It listens on loopback — the address class the +// guard exists to refuse — so its counter doubles as the assertion. +type countingPLC struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPLC(t *testing.T) *countingPLC { + t.Helper() + + plc := &countingPLC{} + plc.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + plc.requests.Add(1) + w.Header().Set("Content-Type", "application/json") + + // No alsoKnownAs, deliberately. Indigo's LookupDID resolves a DECLARED + // handle over DNS and HTTPS to verify it bidirectionally, and that + // lookup would leave the machine — which the hermetic tiers forbid and + // an egress-blocked CI would fail on. With no handle declared, indigo + // marks it invalid and returns, so this fixture stays local. + _, _ = fmt.Fprintf(w, `{"id":%q,"service":[{"id":"#atproto_pds",`+ + `"type":"AtprotoPersonalDataServer","serviceEndpoint":"https://pds.example.invalid"}]}`, testDID) + })) + t.Cleanup(plc.server.Close) + return plc +} + +// resolveThrough points a base resolver at the listener with the given client +// and asks it to resolve a DID. It returns whatever the resolver returned. +func resolveThrough(t *testing.T, plc *countingPLC, client *http.Client) (*DIDDocument, error) { + t.Helper() + + require.NotNil(t, client, "the config under test produced a nil HTTP client, which indigo dereferences") + return newBaseResolver(plc.server.URL, client).ResolveDID(context.Background(), testDID) +} + +// TestDefaultConfig_ClientRefusesAPrivateAddressWithoutReachingIt is the +// binding contract for every caller who never thinks about this at all. +// +// DefaultConfig() with no arguments is what productionPLCResolver uses, what +// tests/live uses, and what the next caller will use. Its client must be +// guarded, so that forgetting is safe. +// +// IS_DEV_ENV IS SET HERE ON PURPOSE, and it is the most important line in the +// test. The gate belongs to the CALLER, not to this package: a future edit that +// "helpfully" reads the environment inside DefaultConfig would open the hatch +// for productionPLCResolver too, in the one environment where that resolver is +// pointed at the public directory and the machine also runs a PLC, a Postgres +// and a PDS on loopback. This test fails on that edit; nothing else would. +func TestDefaultConfig_ClientRefusesAPrivateAddressWithoutReachingIt(t *testing.T) { + // No t.Parallel: t.Setenv forbids it, and this file's subject is what the + // environment must NOT be able to change. + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("PLC_DIRECTORY_URL", "") + + plc := newCountingPLC(t) + + _, err := resolveThrough(t, plc, DefaultConfig().httpClient) + + require.Errorf(t, err, + "DefaultConfig()'s client resolved a DID against a PLC directory on loopback. This is the "+ + "config every caller gets by writing nothing, and the /img route reaches it with no "+ + "credential at all") + assert.Containsf(t, err.Error(), "SSRF blocked", + "the refusal must be the guard's and must say so: identity's error taxonomy flattens its cause "+ + "into a string, so this sentence is all an operator gets to tell a blocked address from a "+ + "PLC directory that is merely down; got: %v", err) + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times with IS_DEV_ENV=true. Either the client is unguarded, or "+ + "the hatch has been made ambient — and an ambient hatch opens productionPLCResolver as well, "+ + "which is the one resolver that must stay guarded in dev", plc.requests.Load()) +} + +// TestDefaultConfig_HatchedClientReachesALoopbackPLC is the other direction, +// and it is not a nicety: a dev stack runs its PLC on localhost:3002, and the +// Many integration fixtures point a resolver at a loopback +// PLC. Without an injectable allowance the guard takes all of them with it. +func TestDefaultConfig_HatchedClientReachesALoopbackPLC(t *testing.T) { + t.Setenv("PLC_DIRECTORY_URL", "") + + plc := newCountingPLC(t) + + doc, err := resolveThrough(t, plc, DefaultConfig(WithPrivateHostsAllowed()).httpClient) + + require.NoErrorf(t, err, + "a resolver built with WithPrivateHostsAllowed() must reach a loopback PLC; got: %v", err) + require.NotNil(t, doc, "the resolved document must come back") + assert.Equal(t, testDID, doc.DID, "the document must be the one the directory served") + require.Lenf(t, doc.Service, 1, "the PDS service entry must survive resolution: %+v", doc) + assert.Equal(t, "https://pds.example.invalid", doc.Service[0].ServiceEndpoint, + "the PDS endpoint is what every downstream fetch is aimed at") + assert.Equalf(t, int64(1), plc.requests.Load(), + "the PLC listener was reached %d times rather than once", plc.requests.Load()) +} + +// TestTwoResolvers_OppositeRequirementsInTheSameProcess is the trap, stated as +// one test. +// +// Both configs are built in the same process, in the same environment, with +// IS_DEV_ENV=true — the arrangement cmd/server actually produces. The dev +// resolver must reach the loopback PLC and the production-directory resolver +// must refuse it. Any implementation where the hatch is a property of the +// PACKAGE or of the ENVIRONMENT rather than of the individual construction +// makes these two agree, and this is the only test that notices. +// +// The production side is pointed at the loopback listener rather than at +// plc.directory deliberately. What is under test is its CLIENT, and a client +// aimed at a host it will never be allowed to dial proves nothing; aiming it at +// the address the guard is supposed to refuse is what makes the refusal +// observable. Note also that this is precisely how the mistake would present in +// production — the wiring difference between the two resolvers is one line, and +// getting it backwards leaves both green. +func TestTwoResolvers_OppositeRequirementsInTheSameProcess(t *testing.T) { + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("PLC_DIRECTORY_URL", "") + + devPLC := newCountingPLC(t) + productionPLC := newCountingPLC(t) + + // a.identityResolver: dev points it at a local PLC, so it holds the hatch. + devConfig := DefaultConfig(PrivateHostOptions(true)...) + // productionPLCResolver: always the public directory, so it holds nothing. + productionConfig := DefaultConfig() + + _, devErr := resolveThrough(t, devPLC, devConfig.httpClient) + _, productionErr := resolveThrough(t, productionPLC, productionConfig.httpClient) + + require.NoErrorf(t, devErr, + "the dev-configured resolver could not reach its local PLC. Every developer's stack and roughly "+ + "twenty test sites resolve against a PLC on loopback; got: %v", devErr) + assert.Equalf(t, int64(1), devPLC.requests.Load(), + "the dev PLC was reached %d times rather than once", devPLC.requests.Load()) + + require.Errorf(t, productionErr, + "the production-directory resolver reached a PLC on loopback while IS_DEV_ENV was true. Its "+ + "hatch must be closed in EVERY environment: it is the resolver aimed at plc.directory, and "+ + "a dev machine is exactly where the loopback it would otherwise dial is a real PLC, a real "+ + "Postgres and a real PDS") + assert.Zerof(t, productionPLC.requests.Load(), + "the production-directory resolver's listener was reached %d times. If both resolvers behave the "+ + "same way here, the hatch is a property of the package or the environment rather than of the "+ + "construction — which is the one shape of this fix that cannot be right, because the two "+ + "resolvers need opposite answers", productionPLC.requests.Load()) +} + +// TestNewResolver_FallbackClientIsGuarded covers factory.go:50, the OTHER bare +// client in this package. +// +// A caller who passes a Config with no HTTPClient — `NewResolver(db, +// Config{PLCURL: ...})`, which the existing suite shows is a normal thing to +// write — gets one built for them. Guarding DefaultConfig and leaving this +// branch bare would mean the safety of a resolver depends on which of two +// equally ordinary constructions its author happened to use. +// +// It reaches through to the base resolver on purpose: cachingResolver would +// read the identity_cache table first, and this test has no database. That is +// the cache trap, avoided rather than papered over. +func TestNewResolver_FallbackClientIsGuarded(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + resolver := NewResolver(nil, Config{PLCURL: plc.server.URL}) + + caching, ok := resolver.(*cachingResolver) + require.True(t, ok, "NewResolver must return the caching wrapper") + + _, err := caching.base.ResolveDID(context.Background(), testDID) + + require.Errorf(t, err, + "a Config with no HTTPClient produced an unguarded client. factory.go:50 fills the gap for "+ + "every caller who writes Config{PLCURL: ...}, which the rest of this package's tests show "+ + "is the ordinary spelling") + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times through NewResolver's fallback client", plc.requests.Load()) +} + +// TestNewResolver_HonoursAnExplicitlyHatchedConfig pins that a config carrying +// the hatch survives NewResolver, since that is the path cmd/server takes for +// a.identityResolver: DefaultConfig(...), then PLCURL overwritten, then +// NewResolver. +func TestNewResolver_HonoursAnExplicitlyHatchedConfig(t *testing.T) { + t.Setenv("PLC_DIRECTORY_URL", "") + + plc := newCountingPLC(t) + + config := DefaultConfig(WithPrivateHostsAllowed()) + config.PLCURL = plc.server.URL + + resolver := NewResolver(nil, config) + caching, ok := resolver.(*cachingResolver) + require.True(t, ok, "NewResolver must return the caching wrapper") + + doc, err := caching.base.ResolveDID(context.Background(), testDID) + + require.NoErrorf(t, err, + "the hatch did not survive DefaultConfig -> PLCURL override -> NewResolver, which is exactly "+ + "the sequence cmd/server's buildIdentity performs; got: %v", err) + assert.Equal(t, testDID, doc.DID) + assert.Equalf(t, int64(1), plc.requests.Load(), + "the PLC listener was reached %d times rather than once", plc.requests.Load()) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// single most important assertion for this call site. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — T0+T1+T2 — takes the +// PERMISSIVE branch at every site holding such a boolean. The guarded branch is +// evaluated in exactly one place in this repository, and it is this function. +// That is why the gate is a pure function rather than an `if a.cfg.IsDevEnv` in +// cmd/server/wiring.go, and why it must not be inlined back. +// +// The claim is that there are NO options — not that the options are safe. An +// edit that appends a diagnostic option, or returns a one-element slice holding +// a no-op "explicitly deny" closure, keeps every behavioural test green while +// moving the untested branch from "provably applies nothing" to "applies +// something believed harmless". +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). What production gets must be exactly "+ + "DefaultConfig's own defaults, with nothing applied on top", len(opts)) +} + +// TestPrivateHostOptions_DisallowedConfigIsGuarded is the behavioural half of +// the assertion above: zero options has to also MEAN a guarded client. The +// length check alone would still pass if DefaultConfig's own default regressed +// to permissive — the helper would be correctly returning nothing, onto a base +// that no longer refuses anything. +func TestPrivateHostOptions_DisallowedConfigIsGuarded(t *testing.T) { + t.Setenv("PLC_DIRECTORY_URL", "") + + plc := newCountingPLC(t) + + _, err := resolveThrough(t, plc, DefaultConfig(PrivateHostOptions(false)...).httpClient) + + require.Error(t, err, + "a config built from PrivateHostOptions(false) reached a loopback PLC. This is the branch "+ + "production runs and CI never does") + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times, so the packet left the process", plc.requests.Load()) +} + +// TestPrivateHostOptions_AllowedConfigReachesTheListener pins the permissive +// direction through observed behaviour rather than through the shape of the +// slice. A length check here would be worthless: a helper returning some other +// single option satisfies it while leaving every dev stack unable to resolve +// anything. +func TestPrivateHostOptions_AllowedConfigReachesTheListener(t *testing.T) { + t.Setenv("PLC_DIRECTORY_URL", "") + + plc := newCountingPLC(t) + + doc, err := resolveThrough(t, plc, DefaultConfig(PrivateHostOptions(true)...).httpClient) + + require.NoErrorf(t, err, + "a config built from PrivateHostOptions(true) was refused. The permissive branch is what every "+ + "developer and every loopback-PLC fixture in this tree runs; got: %v", err) + assert.Equal(t, testDID, doc.DID) + assert.Equalf(t, int64(1), plc.requests.Load(), + "the PLC listener was reached %d times rather than once", plc.requests.Load()) +} + +// TestDefaultConfig_KeepsItsOwnTimeout guards the setting the shared client +// would otherwise swallow. +// +// oauth.NewSSRFSafeHTTPClient ships a 15s ceiling. This package has always used +// 10s, and factory_test.go pins it with the reason ("an unbounded client here +// hangs an ingestion worker on an unresponsive PDS"). Restated here against the +// hatched construction too, because a conversion that restores the timeout on +// one branch and not the other changes behaviour only in dev, where nobody is +// measuring. +func TestDefaultConfig_KeepsItsOwnTimeout(t *testing.T) { + t.Setenv("PLC_DIRECTORY_URL", "") + + for name, config := range map[string]Config{ + "guarded": DefaultConfig(), + "hatched": DefaultConfig(WithPrivateHostsAllowed()), + } { + t.Run(name, func(t *testing.T) { + require.NotNil(t, config.httpClient, "the config must carry a client") + assert.Equalf(t, 10*time.Second, config.httpClient.Timeout, + "the %s client runs on a %v timeout instead of this package's 10s. The shared SSRF "+ + "client brings a 15s ceiling of its own, so adopting it without re-applying the "+ + "caller's value re-times every identity lookup in the AppView as a side effect of "+ + "an SSRF fix", name, config.httpClient.Timeout) + }) + } +} diff --git a/internal/atproto/jetstream/acceptance_consumer_test.go b/internal/atproto/jetstream/acceptance_consumer_test.go index a60858b..9b285a8 100644 --- a/internal/atproto/jetstream/acceptance_consumer_test.go +++ b/internal/atproto/jetstream/acceptance_consumer_test.go @@ -374,7 +374,8 @@ func newRealRepoFixture(t *testing.T, db *sql.DB) *realRepoFixture { newMockUserService(), db, WithAdmissions(admissions), WithDeletedAccounts(postgres.NewDeletedAccountRepository(db)), - WithPostRecordFetcher(NewDevDirectPostFetcher(pinnedResolver(author.DID, pdsServer.URL()))), + WithPostRecordFetcher(NewDirectPostFetcher(pinnedResolver(author.DID, pdsServer.URL()), + PrivatePostFetcherOptions(true)...)), ) return &realRepoFixture{ diff --git a/internal/atproto/jetstream/admission_durability_test.go b/internal/atproto/jetstream/admission_durability_test.go index 7616550..a635efa 100644 --- a/internal/atproto/jetstream/admission_durability_test.go +++ b/internal/atproto/jetstream/admission_durability_test.go @@ -166,7 +166,8 @@ func TestAdmission_ConvergeMustNotRegressTheEvaluatedCID(t *testing.T) { WithAdmissions(f.admissions), WithDeletedAccounts(postgres.NewDeletedAccountRepository(db)), WithPostRecordFetcher(&racingFetcher{ - inner: NewDevDirectPostFetcher(pinnedResolver(f.author.DID, f.pds.URL())), + inner: NewDirectPostFetcher(pinnedResolver(f.author.DID, f.pds.URL()), + PrivatePostFetcherOptions(true)...), before: func() { require.NoError(t, f.consumer.HandleEvent(context.Background(), pv2Event( f.author.DID, "create", record.RKey, testkit.TID(), newerCID, time.Now().UnixMicro(), diff --git a/internal/atproto/jetstream/authorpost.go b/internal/atproto/jetstream/authorpost.go index 40f5a8b..1dce313 100644 --- a/internal/atproto/jetstream/authorpost.go +++ b/internal/atproto/jetstream/authorpost.go @@ -152,10 +152,12 @@ type DirectPostFetcher struct { resolver identity.Resolver // allowPrivateHosts disables the SSRF protection that blocks private and - // loopback addresses. NEVER set outside tests. It exists for the same reason - // blueskypost's allowPrivateHost does: this package's own tests point the - // fetcher at an httptest server, which necessarily listens on loopback and - // would otherwise be refused by the guard that must stay on in production. + // loopback addresses. Reachable only through withPrivateHostsAllowed, which is + // unexported, so the one production route in is PrivatePostFetcherOptions + // carrying cmd/server's allowPrivateHosts() — false everywhere but dev. It + // exists for the same reason blueskypost's allowPrivateHost does: a dev + // stack's PDS, and this package's own fixtures, listen on loopback, which is + // exactly what the guard that must stay on in production refuses. // // The guard is not decorative here. The DID document that names the PDS is // attacker-controlled — anyone can publish one — so an unguarded fetcher is @@ -192,23 +194,62 @@ const maxFetchedRecordBytes = 1 << 20 // 1 MiB // into the logs, so a hostile host cannot flood them. const maxFetchErrorDetailBytes = 256 -// NewDirectPostFetcher wires the §5.4 fetch. SSRF protection is ON and there is -// no parameter to turn it off: a constructor that accepted a boolean is a -// constructor someone eventually passes true to from production wiring. -func NewDirectPostFetcher(resolver identity.Resolver) *DirectPostFetcher { - return &DirectPostFetcher{resolver: resolver} +// DirectPostFetcherOption configures the §5.4 fetch at construction time. +// +// Construction state, never a read inside the fetch: an environment read at the +// call site would make the guarded branch untestable alongside t.Parallel and +// would hide the most consequential input to a security decision from the place +// that makes it. +type DirectPostFetcherOption func(*DirectPostFetcher) + +// NewDirectPostFetcher wires the §5.4 fetch. SSRF protection is ON unless an +// option stands it down, and it takes no boolean: a constructor that accepted +// one is a constructor someone eventually passes true to from production wiring. +func NewDirectPostFetcher(resolver identity.Resolver, opts ...DirectPostFetcherOption) *DirectPostFetcher { + f := &DirectPostFetcher{resolver: resolver} + for _, opt := range opts { + opt(f) + } + return f +} + +// withPrivateHostsAllowed stands the SSRF guard down on the direct fetch. +// +// UNEXPORTED, AND THAT IS THE WHOLE POINT. It replaced NewDevDirectPostFetcher, +// whose doc comment claimed that "no production wiring can reach it by passing a +// variable that happens to be true" — but that constructor was EXPORTED, so any +// package, cmd/server included, could call it directly and open the hatch. The +// guarantee was prose. This is the type system: outside this package the hatch is +// unreachable, and the only way in is PrivatePostFetcherOptions, whose false +// branch returns nothing. +// +// pds/factory.go's withTransportOptions and this package's own +// withWellKnownHTTPClient are unexported for exactly this hazard. Every +// legitimate caller of the hatch is a fixture in this package, so it costs them +// nothing. +func withPrivateHostsAllowed() DirectPostFetcherOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(f *DirectPostFetcher) { f.allowPrivateHosts = true } } -// NewDevDirectPostFetcher builds a fetcher with the SSRF guard STOOD DOWN, for -// development and hermetic test stacks whose PDS is reachable only on a private -// address. +// PrivatePostFetcherOptions returns the options a caller holding an allow-private +// boolean should pass to NewDirectPostFetcher: the hatch when it is set, and +// NOTHING when it is not. +// +// It mirrors PrivateHostOptions above and oauth.PrivateAddressOptions, and it is +// a function rather than an `if` in cmd/server/consumers.go for the reason +// documented there: `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the +// PERMISSIVE branch at every call site holding such a boolean. A unit test +// against this function is the only place in the repository where the branch +// production actually runs is ever evaluated. Do not inline it back. // -// It is a separate constructor rather than a flag on the safe one so that the -// dangerous choice has to be named at the call site, where a reviewer sees it, -// and so that no production wiring can reach it by passing a variable that -// happens to be true. Its one caller is gated on IS_DEV_ENV. -func NewDevDirectPostFetcher(resolver identity.Resolver) *DirectPostFetcher { - return &DirectPostFetcher{resolver: resolver, allowPrivateHosts: true} +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's own +// defaults. +func PrivatePostFetcherOptions(allowPrivate bool) []DirectPostFetcherOption { + if !allowPrivate { + return nil + } + return []DirectPostFetcherOption{withPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing } // httpClient returns the guarded client, building it once on first use. @@ -226,7 +267,7 @@ func NewDevDirectPostFetcher(resolver identity.Resolver) *DirectPostFetcher { // — costs nothing. func (f *DirectPostFetcher) httpClient() *http.Client { f.clientOnce.Do(func() { - f.client = oauth.NewSSRFSafeHTTPClient(f.allowPrivateHosts) + f.client = oauth.NewSSRFSafeHTTPClient(oauth.PrivateAddressOptions(f.allowPrivateHosts)...) }) return f.client } diff --git a/internal/atproto/jetstream/authorpost_ssrf_test.go b/internal/atproto/jetstream/authorpost_ssrf_test.go new file mode 100644 index 0000000..ac95ba1 --- /dev/null +++ b/internal/atproto/jetstream/authorpost_ssrf_test.go @@ -0,0 +1,265 @@ +package jetstream + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "Coves/internal/atproto/identity" + covesoauth "Coves/internal/atproto/oauth" +) + +// The §5.4 direct post fetch, and the gate that decides whether it is guarded. +// +// # THE CALL SITE +// +// convergeOnAcceptedSubject reaches it when a community acceptance names a post +// this AppView has never indexed. The PDS it dials is resolved from the DID +// document of the repo the ACCEPTANCE named, and anyone on the federated network +// can write an acceptance and publish a DID document — so the destination of +// this request is chosen by a stranger, from inside the AppView's own network, +// next to its Postgres and its PDS. +// +// # WHY THIS FILE IS T0 +// +// httpClient() — the one line that reads f.allowPrivateHosts — has exactly one +// other behavioural proof in the tree, TestDirectPostFetcher_RefusesAPrivateHostByDefault +// at acceptance_consumer_test.go:518. That file is `//go:build integration`, so +// `go test ./...` does not even COMPILE it and the inner loop had no coverage of +// this guard at all. That test is untouched and still the T1 authority; this file +// is its T0 counterpart, plus the first coverage the GATE has ever had. +// +// # WHAT REPLACED NewDevDirectPostFetcher, AND WHY IT IS STRONGER +// +// The hatch used to be a second EXPORTED constructor whose name encoded the +// unsafe choice and whose body hardcoded `allowPrivateHosts: true`. Its own doc +// comment claimed "no production wiring can reach it by passing a variable that +// happens to be true" — but being exported, any package including cmd/server +// could simply call it. The guarantee was prose. +// +// withPrivateHostsAllowed is unexported, so outside this package the hatch is +// unreachable and the only route in is PrivatePostFetcherOptions, whose false +// branch returns nothing. That is the same guarantee enforced by the type system. +// And the BRANCH left cmd/server/consumers.go, which is what §7 requires: +// `.env.ci:140` sets IS_DEV_ENV=true, so the hermetic merge gate only ever ran +// that inline conditional's permissive side. +// +// # WHY THE BEHAVIOURAL ASSERTIONS ARE NOT TYPE-SHAPED +// +// A guarded and a hatched fetcher are the same type — the hatch is a bool field +// inside one struct — so no type check and no reflect.TypeOf can tell them apart. +// Reachability is asserted for a second reason: mutation testing +// produced a guard that classified correctly, emitted a byte-identical message, +// and refused the request AFTER delivering it. For a destination a stranger +// named, the packet leaving IS the SSRF. + +// ssrfTestPostURI is a well-formed at:// record URI. parseRecordURI runs before +// anything else in FetchPost, so a malformed one would make these tests pass for +// the wrong reason — the fetcher would refuse the URI and never reach a client. +const ssrfTestPostURI = "at://did:plc:z72i7hdynmk6r22z27h6tvur/social.coves.community.postv2/3kabcdefghij" + +// ssrfTestAuthorDID is the repo half of the URI above. +const ssrfTestAuthorDID = "did:plc:z72i7hdynmk6r22z27h6tvur" + +// countingPDS is a PDS that answers getRecord and records whether anything ever +// reached it. It listens on loopback, which is the address class the guard +// exists to refuse, so its counter IS the assertion. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + pds.requests.Add(1) + // Not a CAR. The hatched case asserts on the counter, not on success: + // what it has to show is that the request ARRIVED, and building a real + // signed repo here would test repo.ReadRepoFromCar instead of the gate. + // The T1 test that does build a real repo is + // TestDirectFetch_RecomputesTheCIDFromARealRepo. + w.Header().Set("Content-Type", "application/vnd.ipld.car") + _, _ = w.Write([]byte("not a car")) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// resolverFor points the fetcher's DID resolution at the counting PDS. +func resolverFor(pds *countingPDS) identity.Resolver { + return &mockIdentityResolverForUser{identities: map[string]*identity.Identity{ + ssrfTestAuthorDID: {DID: ssrfTestAuthorDID, Handle: "author.test", PDSURL: pds.server.URL}, + }} +} + +// fetchThroughGate drives one fetch through the gate the way +// cmd/server/consumers.go does — NewDirectPostFetcher, from +// PrivatePostFetcherOptions and the same allow-private boolean. +// +// Through the production route on purpose. Setting f.allowPrivateHosts directly, +// or replacing f.client, would prove nothing about the wiring: the community +// consumer's own SSRF file records a mutation where a fixture that replaced the +// client left `newWellKnownClient(true)` — the guard disabled for every +// production consumer — failing zero tests. +func fetchThroughGate(t *testing.T, pds *countingPDS, allowPrivate bool) error { + t.Helper() + + fetcher := NewDirectPostFetcher(resolverFor(pds), PrivatePostFetcherOptions(allowPrivate)...) + + _, err := fetcher.FetchPost(context.Background(), ssrfTestPostURI) + return err +} + +// TestPrivatePostFetcherOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed +// is the §7-standard assertion: isDev=false yields NO options. +// +// The claim is not "the options returned are safe". It is that there are none — +// length zero, nothing applied, the constructor's own defaults left untouched. +func TestPrivatePostFetcherOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivatePostFetcherOptions(false) + + assert.Lenf(t, opts, 0, + "PrivatePostFetcherOptions(false) returned %d option(s). The production branch — the one "+ + "IS_DEV_ENV=true keeps `make ci` from ever evaluating — must contribute nothing at all, "+ + "so that what production gets is exactly the constructor's own defaults", len(opts)) +} + +// TestPrivatePostFetcherOptions_BindTheGateToTheConstructor pins both directions +// through the state the constructor actually ends up in. +// +// A length check on the false branch is worthless alone: a helper returning +// nothing in BOTH directions satisfies it while guaranteeing the hatch can never +// open, and a helper returning the wrong single option satisfies it while +// leaving every developer unable to reach their local PDS. +// +// This asserts a FIELD, not a type — the two directions produce the same type, +// so a type assertion here would be measuring nothing. +func TestPrivatePostFetcherOptions_BindTheGateToTheConstructor(t *testing.T) { + t.Parallel() + + guarded := NewDirectPostFetcher(nil, PrivatePostFetcherOptions(false)...) + assert.False(t, guarded.allowPrivateHosts, + "a fetcher built from PrivatePostFetcherOptions(false) has the SSRF hatch open. This is the "+ + "branch production runs and CI never does") + + hatched := NewDirectPostFetcher(nil, PrivatePostFetcherOptions(true)...) + assert.True(t, hatched.allowPrivateHosts, + "a fetcher built from PrivatePostFetcherOptions(true) is still guarded, so the dev hatch does "+ + "nothing and a local stack's PDS — which is on loopback — cannot be fetched from") +} + +// TestPrivatePostFetcherOptions_GuardedRefusesAPrivatePDSWithoutReachingIt is the +// binding contract, and the behavioural half of the pin: false is the branch +// production runs and the one `make ci` never evaluates. +// +// THE ASSERTIONS THAT GO RED UNDER MUTATION are the ErrBlockedAddress check and +// the zero-requests check. Neither is type-shaped: the first names the mechanism, +// so an unrelated failure cannot satisfy it, and the second observes a real +// listener, so a guard that refuses only after delivering still fails. +func TestPrivatePostFetcherOptions_GuardedRefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + err := fetchThroughGate(t, pds, false) + + require.Error(t, err, + "the direct fetch reached a PDS on loopback with the gate shut. This is the request any "+ + "federated instance triggers by writing an acceptance for a post this AppView has not "+ + "indexed, against a DID document it published itself") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity. Without this, a build where the guarded client "+ + "was never wired looks identical — the fixture serves no valid CAR, so an error comes "+ + "back either way and an assertion that only says 'it failed' would still pass; got: %v", err) + assert.Zerof(t, pds.requests.Load(), + "the PDS listener was reached %d times. The refusal happened, but it happened AFTER the "+ + "request was delivered — which prevents none of the SSRF", pds.requests.Load()) +} + +// TestPrivatePostFetcherOptions_HatchedReachesAPrivatePDS is the falsifiability +// control. +// +// Identical fixture, identical call, only the boolean differs. Without it, a +// fetcher that could make no request at all satisfies the guarded case just as +// well, and the test above would prove nothing about CLASSIFICATION. +// +// It is also the half a developer depends on: the hermetic stack's PDS is a +// private address, so with the gate stuck shut acceptance-before-post never +// converges locally. +func TestPrivatePostFetcherOptions_HatchedReachesAPrivatePDS(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + err := fetchThroughGate(t, pds, true) + + require.Error(t, err, + "the fixture serves nine bytes that are not a CAR, so the fetch must still fail — if it "+ + "succeeded, this test is not reaching the fixture it thinks it is") + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the gate was open and the address was still refused by the guard. Either "+ + "PrivatePostFetcherOptions is not reaching the client, or the guarded case above proves "+ + "nothing: a fetcher that refuses every address refuses the guarded case too, for a reason "+ + "that has nothing to do with classification; got: %v", err) + assert.Equalf(t, int64(1), pds.requests.Load(), + "the PDS listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestNewDirectPostFetcher_RefusesAPrivateHostByDefault is the T0 counterpart of +// acceptance_consumer_test.go:518, which is the only behavioural proof this +// guard had and is `//go:build integration` — so `go test ./...` never compiled +// it and the inner loop could not see a regression here at all. +// +// It passes NO options, so it holds independently of PrivatePostFetcherOptions: +// a caller that expressed no opinion must degrade to the guarded fetcher. The T1 +// test is untouched and remains the authority; this one exists so the failure is +// visible without Postgres. +func TestNewDirectPostFetcher_RefusesAPrivateHostByDefault(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + _, err := NewDirectPostFetcher(resolverFor(pds)).FetchPost(context.Background(), ssrfTestPostURI) + + require.Error(t, err, + "NewDirectPostFetcher with no options reached a PDS on loopback. The PDS this dials is named "+ + "by a DID document anyone can publish, so the default must be the guarded one: every "+ + "caller that expressed no opinion degrades to it") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must be the guard's and must say so; got: %v", err) + assert.Zerof(t, pds.requests.Load(), + "the PDS listener was reached %d times by the default construction", pds.requests.Load()) +} + +// TestPrivatePostFetcherOptions_GuardIsNotAmbient states the invariant at this +// site too. +// +// The decision belongs to the CALLER — cmd/server passes allowPrivateHosts() — +// and must never become a read of the environment inside the gate, the +// constructor or the fetch. This process has IS_DEV_ENV set to true and the +// guarded branch must still refuse, because the same binary builds +// productionPLCResolver, which has to stay guarded in dev. +func TestPrivatePostFetcherOptions_GuardIsNotAmbient(t *testing.T) { + // No t.Parallel: t.Setenv forbids it. + t.Setenv("IS_DEV_ENV", "true") + + pds := newCountingPDS(t) + + err := fetchThroughGate(t, pds, false) + + require.Error(t, err, + "the guarded branch reached a loopback PDS while IS_DEV_ENV was true. The gate must be a "+ + "property of the argument, not of the environment: an ambient read opens every other "+ + "construction in this process at the same time") + assert.Zerof(t, pds.requests.Load(), + "the PDS listener was reached %d times with IS_DEV_ENV=true", pds.requests.Load()) +} diff --git a/internal/atproto/jetstream/community_consumer.go b/internal/atproto/jetstream/community_consumer.go index 0e22938..57f066b 100644 --- a/internal/atproto/jetstream/community_consumer.go +++ b/internal/atproto/jetstream/community_consumer.go @@ -2,9 +2,11 @@ package jetstream import ( "Coves/internal/atproto/identity" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/atproto/utils" "Coves/internal/core/communities" "Coves/internal/core/richtext" + "Coves/internal/validation" "context" "encoding/json" "errors" @@ -31,6 +33,31 @@ type CommunityEventConsumer struct { instanceDID string // DID of this Coves instance skipVerification bool // Skip did:web verification (for dev mode) revGate *RevGate // Optional: cross-feed ordering guard for commit events (nil = ungated) + + // allowPrivateHosts disables the SSRF address guard on the .well-known + // fetch. NEVER set in production: the domain this consumer dials comes off a + // community record published by anyone on the federated network. + allowPrivateHosts bool + + // transportOptions is the TEST SEAM, carried on the consumer so that the + // client the guard tests exercise is the one this constructor builds. + // + // The alternative — a fixture that constructs the consumer and then assigns + // newWellKnownClient(...) over its client — is what this field exists to + // delete, and it is the SECOND time that mistake was made here. The comment + // block above guardedConsumer in community_consumer_ssrf_test.go records the + // first: a client injected through withWellKnownHTTPClient could not see a + // mutation of newWellKnownClient, so the seam was moved onto that function — + // and its one caller, applyWellKnownClient below, was left just as + // uncovered, because the fixture went on to overwrite what the caller built. + // c.allowPrivateHosts is read in exactly one place; threading the seam + // through the constructor is what puts that place under test. + // + // It is UNEXPORTED, and so is the option that sets it: oauth.WithHostResolver + // must not be reachable from any non-test package, which scripts/ssrf-audit.sh + // enforces as a hard gate. See pds.bearerClientConfig.transportOptions, the + // same field for the same reason. + transportOptions []covesoauth.Option } // CommunityConsumerOption configures optional CommunityEventConsumer behaviour. @@ -48,6 +75,159 @@ func WithCommunityRevGate(gate *RevGate) CommunityConsumerOption { } } +// WithPrivateHostsAllowed disables the SSRF address guard on the .well-known +// DID-document fetch. +// +// THE NAME IS THE CONTRACT: production must not call this. cmd/server derives +// the value from config once (the IS_DEV_ENV gate); tests that serve a DID +// document from httptest pass it because loopback is exactly what the guard +// refuses. +func WithPrivateHostsAllowed() CommunityConsumerOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(c *CommunityEventConsumer) { c.allowPrivateHosts = true } +} + +// withTransportOptions passes oauth transport options through to the client +// applyWellKnownClient builds, and is UNEXPORTED so production cannot reach it. +// +// It is not a second hatch: whatever a resolver seam answers is classified by +// the same pass a real DNS answer goes through, and the dial still goes only to +// addresses that survived it. See CommunityEventConsumer.transportOptions, and +// pds/factory.go's withTransportOptions, which is the shape this copies. +// +// Unlike withWellKnownHTTPClient, this does NOT replace the consumer's client — +// it configures the one the constructor builds, which is the whole difference +// between a test that exercises this consumer's wiring and one that exercises a +// client the test assembled itself. +func withTransportOptions(opts ...covesoauth.Option) CommunityConsumerOption { + return func(c *CommunityEventConsumer) { c.transportOptions = append(c.transportOptions, opts...) } +} + +// withWellKnownHTTPClient replaces the client used for the .well-known fetch, +// and is UNEXPORTED because replacing is all it can do. +// +// IT EXISTS FOR TESTS, and it is the narrowest seam that makes the two +// mechanisms separable. The guard classifies ADDRESSES, so proving it requires a +// hostname that passes validation and answers with a private address — and +// there is no hermetic way to make a name resolve to a chosen address without +// oauth.WithHostResolver, which must not appear in production code. Injecting a +// client the test built with that resolver keeps the seam on the test side. +// +// A caller MUST still inject a GUARDED client, or it proves nothing: replacing +// the guarded client with a permissive one is how a fixture repair silently +// deletes the property it was meant to preserve. That sentence used to be the +// whole defence, addressed to a reader the compiler cannot reach, on an EXPORTED +// option guarding a fetch whose domain comes off a community record published by +// anyone federated with this instance. pds/factory.go's withTransportOptions is +// unexported for exactly this hazard, so leaving this one exported had the +// codebase answering one question two opposite ways. The lowercase name is what +// makes "a caller outside this package cannot drop the guard" a fact rather than +// a request; every caller is a fixture in this package, so it costs them +// nothing. +// +// TestNoExportedSeamCanReplaceTheGuardedClient pins the property in general — +// nothing exported from this package takes an *http.Client — so the next seam +// somebody adds is covered too. +func withWellKnownHTTPClient(client *http.Client) CommunityConsumerOption { + return func(c *CommunityEventConsumer) { + if client == nil { + return + } + c.httpClient = client + } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to NewCommunityEventConsumer: the hatch when it is set, +// and NOTHING when it is not. +// +// It mirrors oauth.PrivateAddressOptions, imageproxy.PrivateHostOptions and +// unfurl.PrivateHostOptions, and it is a function rather than an `if` in +// cmd/server/wiring.go for the reason documented there: `.env.ci:140` sets +// IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at every call site +// holding such a boolean. A unit test against this function is the only place in +// the repository where the branch production actually runs is ever evaluated. Do +// not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's own +// defaults. +func PrivateHostOptions(allowPrivate bool) []CommunityConsumerOption { + if !allowPrivate { + return nil + } + return []CommunityConsumerOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + +const ( + // wellKnownTimeout is the ceiling this consumer has always run the + // DID-document fetch under. It is re-applied over the shared SSRF client's + // own 15s; see newWellKnownClient. + wellKnownTimeout = 10 * time.Second + + // wellKnownMaxIdleConnsPerHost is the per-host pool depth this consumer has + // always run with, kept because net/http's default is 2 and a federated + // instance is verified repeatedly. + wellKnownMaxIdleConnsPerHost = 10 +) + +// newWellKnownClient builds the client the DID-document fetch goes through. +// +// The SSRF-safe transport of internal/atproto/oauth resolves the host, refuses +// private, loopback and link-local addresses, and then dials only the address it +// vetted — closing the check-then-dial window a naive guard leaves open. The +// domain it is pointed at comes off a community record published by anyone +// federated with this instance, and the fetch runs from inside the AppView's own +// network, next to its Postgres, its PDS and, in production, a metadata endpoint +// that hands credentials to anything that can reach it. +// +// IT IS HALF OF A TWO-MECHANISM DEFENCE and does not subsume the other half. A +// safe dialler has an opinion about addresses and none about paths, so +// `internal-admin/v1/secrets?x=y#` walks straight past it: that names no private +// address, it names whatever `internal-admin` resolves to, and the trailing `#` +// turns the `.well-known/did.json` suffix into a fragment that is never sent. +// validation.NormalizeDomain, called in verifyDIDDocument BEFORE the URL is +// built, is what closes that. Neither mechanism is redundant with the other. +// +// EVERY SETTING THIS CONSUMER ALREADY HAD IS RE-APPLIED. The shared transport +// carries MaxIdleConns 100 and IdleConnTimeout 90s of its own; the per-host pool +// depth and the 10s ceiling are the two it does not, and both are restored here +// — a firehose path silently re-timed from 10s to the shared client's 15s would +// be a second change wearing an SSRF fix's clothes, the way +// blobs.NewBlobService, imageproxy.NewPDSFetcher and unfurl.NewService all say. +// +// # THE opts PARAMETER IS THE TEST SEAM, AND IT IS WHY THE GUARD IS PROVABLE +// +// It mirrors internal/api/handlers/aggregator's registerHTTPClient exactly, and +// it exists because of a hole a mutation found: with no seam here, the only way +// to test the guard was for a test to build its OWN oauth client with +// WithHostResolver and inject it. That proves internal/atproto/oauth works, +// which the transport tests already prove, and says nothing about this consumer — +// flipping the boolean below to a constant `true` failed no test at all, +// because every input the other tests use is eaten by validation.NormalizeDomain +// one branch earlier and never reaches a transport. +// +// Passing the resolver through HERE means the client under test is the one this +// function builds, from the same allowPrivateHosts boolean production passes. So +// disabling the guard on this line now fails +// TestVerifyDIDDocument_RefusesAValidationPassingHostThatResolvesPrivate. +// +// IT CANNOT OPEN THE GUARD. WithPrivateAddressesAllowed is the only thing that +// does, it is one-way, and it is named at a call site or nowhere; an option +// passed here is classified by the same pass a real DNS answer goes through. +// The seam chooses what gets classified, never whether classification happens. +// +// Production passes nothing. +func newWellKnownClient(allowPrivateHosts bool, opts ...covesoauth.Option) *http.Client { + client := covesoauth.NewSSRFSafeHTTPClient(append( + covesoauth.PrivateAddressOptions(allowPrivateHosts), + append([]covesoauth.Option{ + covesoauth.WithMaxIdleConnsPerHost(wellKnownMaxIdleConnsPerHost), + }, opts...)..., + )...) + client.Timeout = wellKnownTimeout + return client +} + // cachedDIDDoc represents a cached verification result with expiration type cachedDIDDoc struct { expiresAt time.Time // When this cache entry expires @@ -81,20 +261,13 @@ func NewCommunityEventConsumer(repo communities.Repository, instanceDID string, identityResolver: identityResolver, instanceDID: instanceDID, skipVerification: skipVerification, - httpClient: &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - IdleConnTimeout: 90 * time.Second, - }, - }, didCache: cache, wellKnownLimiter: rate.NewLimiter(10, 20), } for _, opt := range opts { opt(fallback) } + applyWellKnownClient(fallback) return fallback } @@ -103,15 +276,6 @@ func NewCommunityEventConsumer(repo communities.Repository, instanceDID string, identityResolver: identityResolver, // Optional - can be nil for tests instanceDID: instanceDID, skipVerification: skipVerification, - // Shared HTTP client with connection pooling for .well-known fetches - httpClient: &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 10, - IdleConnTimeout: 90 * time.Second, - }, - }, // Bounded LRU cache for .well-known verification results (max 1000 entries) // Automatically evicts least-recently-used entries when full didCache: cache, @@ -122,9 +286,36 @@ func NewCommunityEventConsumer(repo communities.Repository, instanceDID string, for _, opt := range opts { opt(consumer) } + applyWellKnownClient(consumer) return consumer } +// applyWellKnownClient gives a consumer its guarded .well-known client, AFTER +// the options have run. +// +// The ordering is the whole of it, in both directions. The hatch is an option, +// so building the client before the loop would read allowPrivateHosts before +// anything could set it and hand every developer a client that refuses their own +// machine. And withWellKnownHTTPClient is also an option, so overwriting +// unconditionally after the loop would throw away the client a test injected — +// which would make every ordering assertion in community_consumer_ssrf_test.go +// pass against a consumer quietly using a client the test cannot see. +// +// Hence the nil check: an injected client wins, and a consumer that was given +// none gets the guarded one. +// +// The transport options are threaded for a third reason, and it is what makes +// this function's own argument observable: the guard tests build a consumer +// through the constructor and pass the resolver seam as an option, so THIS line +// is the one that carries their seam. Replacing c.allowPrivateHosts with a +// constant here now fails them. +func applyWellKnownClient(c *CommunityEventConsumer) { + if c.httpClient != nil { + return + } + c.httpClient = newWellKnownClient(c.allowPrivateHosts, c.transportOptions...) +} + // RevGated reports whether this consumer applies the per-record rev gate (true when a // gate was injected via WithCommunityRevGate). main.go checks this at boot to refuse // multi-feed operation with an ungated consumer. @@ -567,6 +758,45 @@ func (c *CommunityEventConsumer) verifyDIDDocument(ctx context.Context, did, dom return nil } + // SECURITY: the domain came off a community record published by anyone + // federated with this instance, and the URL below is built by concatenating + // it into a string. A URL parser has no way to know the concatenation was + // meant to stop at the host, so every part of a URL that comes after the host + // can be smuggled in through it: + // + // internal-admin/v1/secrets?x=y# fetches /v1/secrets?x=y from internal-admin + // evil.com@internal-host fetches from internal-host; evil.com is userinfo + // 127.0.0.1:5432 fetches from a port on the loopback interface + // + // The trailing `#` in the first is what makes this a full request-forgery + // primitive rather than an SSRF to one fixed path: it turns the + // `.well-known/did.json` suffix into a fragment, which is never sent, so the + // publisher chooses the path AND the query as well as the host. + // + // THE GUARDED CLIENT CANNOT CLOSE THIS. It refuses private ADDRESSES and has + // no opinion about paths, and `internal-admin` is not an address — it is + // whatever this AppView's resolver says it is, which on a split-horizon DNS + // is an ordinary-looking answer. The two mechanisms are halves; see + // newWellKnownClient. + // + // IT RUNS BEFORE THE CACHE AND BEFORE THE RATE LIMITER, not just before the + // URL. The cache is keyed by DID rather than by domain, so a hit would answer + // for a domain nothing ever looked at; and the limiter blocks, which would + // let a flood of malformed records spend the budget that legitimate + // verifications share. + // + // The refusal is NOT cached. It costs one pass over a string to recompute, + // and the cache key is the DID — so caching would let one bad record suppress + // a later good one for the same DID for the full TTL. + normalizedDomain, err := validation.NormalizeDomain(domain) + if err != nil { + return fmt.Errorf("refusing to fetch a DID document for %s: %q is not a hostname: %w", + did, domain, err) + } + // The canonical form from here on, so one domain has one spelling in the URL + // and in the logs. + domain = normalizedDomain + // Check bounded LRU cache first (thread-safe, no locks needed) if cached, ok := c.didCache.Get(did); ok { // Check if cache entry is still valid (not expired) diff --git a/internal/atproto/jetstream/community_consumer_seam_export_test.go b/internal/atproto/jetstream/community_consumer_seam_export_test.go new file mode 100644 index 0000000..5dce134 --- /dev/null +++ b/internal/atproto/jetstream/community_consumer_seam_export_test.go @@ -0,0 +1,82 @@ +package jetstream + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNoExportedSeamCanReplaceTheGuardedClient is the jetstream half of the +// aggregator test of the same name, and the two exist together because the +// codebase was making two opposite calls about one hazard. +// +// pds/factory.go's withTransportOptions is UNEXPORTED and says why: "the +// resolver seam these tests need must not be reachable from any non-test +// package". WithWellKnownHTTPClient was the same seam left exported, and its own +// doc comment carried the whole defence in a sentence addressed to nobody the +// compiler can reach — "a test that uses this MUST still inject a GUARDED +// client". A rule that only a careful reader enforces is not a rule; the +// .well-known fetch it protects takes a domain off a community record published +// by anyone federated with this instance. +// +// Lowercasing costs the fixtures nothing — every caller is in this package — and +// keeps the seam's real property intact: it chooses WHAT gets classified, never +// whether classification happens. +func TestNoExportedSeamCanReplaceTheGuardedClient(t *testing.T) { + t.Parallel() + + offenders := exportedHTTPClientParameters(t, ".") + + assert.Emptyf(t, offenders, + "these exported declarations take an *http.Client, so any package can install an unguarded "+ + "client on this consumer and discard the one newWellKnownClient built: %s. Lowercase them "+ + "— every caller is a fixture in this package, and pds/factory.go's withTransportOptions is "+ + "the shape to copy", strings.Join(offenders, ", ")) +} + +// exportedHTTPClientParameters returns the exported declarations in dir's +// non-test Go files that accept an *http.Client. +func exportedHTTPClientParameters(t *testing.T, dir string) []string { + t.Helper() + + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dir, func(info fs.FileInfo) bool { + return !strings.HasSuffix(info.Name(), "_test.go") + }, 0) + require.NoError(t, err, "parsing the package's own sources") + require.NotEmpty(t, pkgs, "the parser found no package, so this test would pass vacuously") + + var offenders []string + for _, pkg := range pkgs { + for _, file := range pkg.Files { + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || !fn.Name.IsExported() { + continue + } + for _, param := range fn.Type.Params.List { + star, ok := param.Type.(*ast.StarExpr) + if !ok { + continue + } + sel, ok := star.X.(*ast.SelectorExpr) + if !ok { + continue + } + ident, ok := sel.X.(*ast.Ident) + if !ok || ident.Name != "http" || sel.Sel.Name != "Client" { + continue + } + offenders = append(offenders, fn.Name.Name) + } + } + } + } + return offenders +} diff --git a/internal/atproto/jetstream/community_consumer_ssrf_test.go b/internal/atproto/jetstream/community_consumer_ssrf_test.go new file mode 100644 index 0000000..61ae30d --- /dev/null +++ b/internal/atproto/jetstream/community_consumer_ssrf_test.go @@ -0,0 +1,721 @@ +package jetstream + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" + "Coves/internal/validation" + "Coves/tests/domaincorpus" +) + +// The community consumer's DID-document fetch, against a domain a stranger +// published. +// +// # THE CALL SITE +// +// didDocURL := fmt.Sprintf("https://%s/.well-known/did.json", domain) +// +// `domain` is extracted from a community record written to its author's own +// repository and carried to us over the firehose, so anyone federated with this +// instance chooses it. The line is BYTE-IDENTICAL IN SHAPE to the one the +// aggregator registration fix closed, and it stayed open for exactly one reason: +// `normalizeDomain` was unexported in internal/api/handlers/aggregator, so the +// fix was unreachable from this package even for someone who noticed. +// +// # TWO MECHANISMS, AND WHY BOTH ARE NEEDED +// +// A guarded dialler closes the ADDRESS half: it refuses to connect to private, +// loopback and link-local addresses, and it dials only what it vetted. It has no +// opinion whatsoever about paths, and that is the other half: +// +// internal-admin/v1/secrets?x=y# +// +// names no private address. It names whatever `internal-admin` resolves to — +// which on a split-horizon resolver is an ordinary-looking answer — and then +// requests `/v1/secrets?x=y` from it, because the trailing `#` turns the +// `.well-known/did.json` suffix into a fragment that is never sent. So the +// attacker picks the path and the query as well as the host, and a guarded +// client watches it happen. +// +// The validator closes that half. Neither mechanism subsumes the other, which is +// why the tests below assert them SEPARATELY: a validation refusal must carry +// validation.ErrDomainInvalid and NOT the guard's sentinel, and an address +// refusal must carry the guard's sentinel and NOT ErrDomainInvalid. Without that +// separation, deleting either mechanism leaves the suite green — the survivor +// catches most of the same inputs and the assertions cannot tell which one +// fired. + +const ( + // consumerTestDID is the DID a community record claims. + consumerTestDID = "did:web:example.com" + + // consumerTestHandle is the handle the DID document must claim back, and + // consumerTestDomain is the domain half of it. `example.com` is a hostname, + // so it passes validation — which is what makes it usable as the CONTROL in + // every table below. + consumerTestHandle = "example.com" + consumerTestDomain = "example.com" +) + +// recordingWellKnown answers nothing and remembers whether it was asked to. +// +// A RoundTripper rather than a dialer, because RoundTrip is the earliest moment +// the consumer can reach the network: http.Client.Do calls it before any name is +// resolved, so a call recorded here means the attacker's host was already on its +// way out whether or not a packet followed. Recording the URL is what makes a +// failure readable — the message names the request that would have been sent, +// which for the fragment payload is a path nobody wrote down anywhere. +type recordingWellKnown struct { + mu sync.Mutex + requested []string +} + +func (r *recordingWellKnown) RoundTrip(req *http.Request) (*http.Response, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.requested = append(r.requested, req.URL.String()) + return nil, errors.New("recordingWellKnown never answers: the consumer should not have reached it") +} + +func (r *recordingWellKnown) seen() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.requested...) +} + +// newRecordingConsumer builds a consumer whose .well-known client cannot talk to +// anything and says so afterwards. +func newRecordingConsumer(t *testing.T) (*CommunityEventConsumer, *recordingWellKnown) { + t.Helper() + + transport := &recordingWellKnown{} + consumer := NewCommunityEventConsumer( + nil, // repo: verifyDIDDocument never touches it + "did:web:coves.social", + false, // skipVerification MUST be false or verifyDIDDocument returns before doing anything + nil, // identityResolver: unused on this path + withWellKnownHTTPClient(&http.Client{Transport: transport}), + ) + return consumer, transport +} + +// TestVerifyDIDDocument_ValidatesTheDomainBeforeTouchingTheNetwork is the +// binding contract for the half a guarded dialler cannot close. +// +// Every payload is one the consumer currently sends: each parses as a URL once +// concatenated, so http.NewRequestWithContext succeeds and Do is reached. That +// is deliberate — an input Go's URL parser rejects would never touch the +// transport even with no validation at all, and would prove nothing here. +// +// The corpus is shared with the aggregator's identical ordering test so the two +// call sites cannot drift; see tests/domaincorpus. +func TestVerifyDIDDocument_ValidatesTheDomainBeforeTouchingTheNetwork(t *testing.T) { + t.Parallel() + + for _, tt := range domaincorpus.InjectionPayloads() { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + consumer, transport := newRecordingConsumer(t) + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, tt.Domain, consumerTestHandle) + + require.Errorf(t, err, + "the consumer accepted the domain %q and went on to fetch a DID document from it. "+ + "This domain came off a community record published by anyone on the network", + tt.Domain) + + assert.ErrorIsf(t, err, validation.ErrDomainInvalid, + "the fetch failed, but not because the domain was refused as malformed: the error does "+ + "not match validation.ErrDomainInvalid. A guarded client that merely failed to reach "+ + "%q looks identical from here, and it is NOT the same control — the guard sees an "+ + "address, and this payload is a path injection against whatever the host resolves "+ + "to; got: %v", tt.Domain, err) + + assert.Emptyf(t, transport.seen(), + "the consumer asked its HTTP client for %v before refusing the domain %q. Validation "+ + "must run FIRST: by the time the client is called the host has been chosen by a "+ + "stranger, and resolving it is already the SSRF", + transport.seen(), tt.Domain) + }) + } + + // THE CONTROL, and without it every row above is unfalsifiable. + // + // The whole table asserts that a recorder stayed EMPTY. An empty recorder is + // also what you get when the recorder was never installed — and + // withWellKnownHTTPClient is the only thing that installs it. So one case has + // to drive a domain that PASSES validation and assert the recorder WAS + // reached. It is the only assertion in this file that can tell "the consumer + // refused before the client" from "the client under observation was not the + // client the consumer uses". + t.Run("control: a valid domain does reach the injected client", func(t *testing.T) { + t.Parallel() + + consumer, transport := newRecordingConsumer(t) + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, consumerTestDomain, consumerTestHandle) + + require.Errorf(t, err, + "the recorder answers nothing, so a domain that reached it must fail the fetch") + require.NotEmptyf(t, transport.seen(), + "a domain that passes validation (%q) never reached the recording transport. Every row "+ + "above asserts on that recorder, so if the consumer is not using it those rows prove "+ + "nothing — they would pass just as well against a consumer sending every request "+ + "through a client this test cannot see. Either validation is refusing a legitimate "+ + "hostname, or withWellKnownHTTPClient is not installing the client it is given", + consumerTestDomain) + require.Equalf(t, + []string{"https://" + consumerTestDomain + "/.well-known/did.json"}, transport.seen(), + "the consumer asked for %v; the DID-document fetch must go to the domain the record named, "+ + "over HTTPS, at the well-known path and nothing else", transport.seen()) + }) +} + +// TestVerifyDIDDocument_TheTwoMechanismsAreSeparatelyObservable is requirement +// four, and it is the assertion that keeps this fix from collapsing into one +// mechanism. +// +// Most inputs are refused by both. If every test only asserted "an error came +// back", a build with the validator deleted would look identical to a correct +// one — the guard would catch the residue, the suite would stay green, and the +// path-injection half would be wide open. So each subtest names the mechanism it +// expects AND denies the other. +func TestVerifyDIDDocument_TheTwoMechanismsAreSeparatelyObservable(t *testing.T) { + t.Parallel() + + t.Run("a path injection is refused by the validator and not by the guard", func(t *testing.T) { + t.Parallel() + + // A GUARDED client, so both mechanisms are present and the assertion is + // genuinely about which one fired. + consumer := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + withWellKnownHTTPClient(covesoauth.NewSSRFSafeHTTPClient())) + + const payload = "internal-admin/v1/secrets?x=y#" + + err := consumer.verifyDIDDocument(context.Background(), consumerTestDID, payload, consumerTestHandle) + + require.Error(t, err, "a path injection must be refused") + assert.ErrorIsf(t, err, validation.ErrDomainInvalid, + "%q must be refused by the VALIDATOR. The guard cannot refuse it: `internal-admin` is not "+ + "an address, and the payload's damage is the path and query it smuggles past the "+ + "fragment, which a safe dialler has no opinion about; got: %v", payload, err) + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "%q was refused by the address guard rather than the validator. That is the mechanism "+ + "collapse this test exists to catch: it means the refusal depends on what "+ + "`internal-admin` happens to resolve to on this machine, and on a resolver that "+ + "answers with a public address the request goes out; got: %v", payload, err) + }) + +} + +// # WHY THE GUARD NEEDS ITS OWN FIXTURE AND NOT AN INJECTED CLIENT +// +// An earlier version of this file proved the guard by building an oauth client +// with WithHostResolver and injecting it through withWellKnownHTTPClient. That +// was worthless, and a mutation said so: setting PrivateAddressOptions(true) in +// newWellKnownClient — disabling the guard for every production consumer — +// failed ZERO tests. The injected client is not the client the constructor +// builds, so mutating the constructor could not touch it. +// +// The validator shadows the guard completely for every other input in this file: +// `localhost`, `127.0.0.1:5432`, `2130706433` and `internal-admin` are all +// refused by NormalizeDomain one branch before a transport is reached. So the +// ONLY input that separates the two mechanisms is a domain that PASSES +// validation and still resolves to a private address — a well-formed +// public-looking hostname whose owner points it at 127.0.0.1. That is the +// cheapest SSRF available here, because the attacker owns the zone and the +// validator cannot see a DNS answer. +// +// THE FIX FOR THAT WAS ITSELF ONE CALL FRAME SHORT, AND THIS IS THE SECOND +// CORRECTION. Moving the seam onto newWellKnownClient made the guard provable, +// but the fixture went on to assign that client over the one the CONSTRUCTOR +// built — so applyWellKnownClient, the single place c.allowPrivateHosts is ever +// read, stayed uncovered. Mutation confirmed it: `newWellKnownClient(true)` on +// that line left this file green, `make ci` green and `make ssrf-audit` at zero. +// +// These two tests therefore build through NewCommunityEventConsumer, with the +// same allowPrivateHosts boolean production passes, and hand the resolver seam +// to the constructor via withTransportOptions. Nothing overwrites the client +// afterwards. Do not reintroduce an assignment here, in either spelling — a +// fixture that replaces the consumer's client is a fixture that cannot say whose +// client it exercised, which is the failure this comment has now recorded twice. + +// consumerGuardDomain passes validation.NormalizeDomain — two labels, an +// alphabetic TLD, no port, no path, no userinfo — and is exactly what a hostile +// instance publishes when the validator is the only control. `.example` is +// reserved by RFC 2606, so nothing resolves it for real if the seam is ever +// bypassed. +const consumerGuardDomain = "community.example" + +// privateAnsweringResolver answers every lookup with one address. +func privateAnsweringResolver(t *testing.T, answer string) func(context.Context, string) ([]net.IP, error) { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and this test would certify the guard against nothing. + ip := net.ParseIP(answer) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", answer) + + return func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + } +} + +// guardedConsumer builds a consumer the way production does — through +// NewCommunityEventConsumer, from the same allow-private boolean — and replaces +// only its NAME RESOLUTION, by passing the resolver seam as a CONSTRUCTION +// OPTION. +// +// withTransportOptions rather than withWellKnownHTTPClient, and rather than an +// assignment onto the unexported field: both of those REPLACE the consumer's +// client, and a replaced client is one the constructor's wiring never has to +// produce. withTransportOptions configures the client applyWellKnownClient +// builds, so `newWellKnownClient(c.allowPrivateHosts, c.transportOptions...)` is +// the line these tests run through — which is exactly the line that was +// unobservable before. See the comment block above. +func guardedConsumer(t *testing.T, allowPrivateHosts bool, resolvesTo string) *CommunityEventConsumer { + t.Helper() + + // PrivateHostOptions is what production passes, so the hatch reaches the + // consumer by the production route; the seam is appended to it rather than + // replacing it. + opts := append(PrivateHostOptions(allowPrivateHosts), + withTransportOptions(covesoauth.WithHostResolver(privateAnsweringResolver(t, resolvesTo)))) + + return NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, opts...) +} + +// TestVerifyDIDDocument_RefusesAValidationPassingHostThatResolvesPrivate is the +// assertion this cycle exists for. +// +// The domain is well-formed, so NormalizeDomain accepts it and cannot be what +// refuses the request. Only the transport can. Asserting BOTH halves — that the +// guard's sentinel is present and ErrDomainInvalid is not — is what makes the +// two mechanisms separately visible: delete either one and exactly one of these +// two assertions changes. +func TestVerifyDIDDocument_RefusesAValidationPassingHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + consumer := guardedConsumer(t, false, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, consumerGuardDomain, consumerTestHandle) + + require.Errorf(t, err, + "%q passed validation and its DNS answer was 127.0.0.1, and the consumer fetched it anyway. "+ + "This is the SSRF the domain validator cannot reach: the community record's author owns "+ + "the zone, so the name is well-formed and the address is chosen after validation has "+ + "already run", consumerGuardDomain) + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity. Without this, a build where the guarded client "+ + "was never wired looks identical — some other error would still come back, and every "+ + "assertion that only says 'it failed' would still pass; got: %v", err) + + assert.NotErrorIsf(t, err, validation.ErrDomainInvalid, + "the validator refused a hostname it is supposed to accept, which means this test is no "+ + "longer exercising the transport at all: %q is a two-label name with an alphabetic TLD. "+ + "Fix the fixture rather than the assertion — a guard-path test whose input the validator "+ + "eats is the exact false pass this case was written to close; got: %v", + consumerGuardDomain, err) +} + +// TestVerifyDIDDocument_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control, and without it the test above is unfalsifiable. +// +// Identical consumer, identical seam, identical domain — only the hatch differs. +// With it open the address is no longer refused, so the request proceeds to a +// dial, which fails because nothing is listening on loopback:443. The error is +// therefore NOT the guard's. +// +// That is what pins the refusal above to CLASSIFICATION. Without this control, a +// client that could not make any request at all — a broken seam, a transport +// wired to nothing — would satisfy the guarded case just as well. +// +// It is also why there is no "the listener was never reached" assertion in THIS +// test: a validation-passing domain has no port, so verifyDIDDocument always +// builds https:///… on 443, and a test cannot bind 443. This control +// does the equivalent work for anything driven through verifyDIDDocument. +// +// The client itself is a different matter, and does get a real one — +// TestNewCommunityEventConsumer_BuildsAGuardedClientWithItsSettingsPreserved +// drives consumer.httpClient at a port a test may bind and asserts a request +// counter stayed at zero. +func TestVerifyDIDDocument_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + consumer := guardedConsumer(t, true, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; with the hatch open it is dialled and refused by the OS + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, consumerGuardDomain, consumerTestHandle) + + require.Errorf(t, err, + "nothing listens on loopback:443, so this fetch must fail — if it succeeded, the seam is not "+ + "answering with the address this test gave it") + + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either PrivateHostOptions "+ + "is not reaching the client, or the guarded case above proves nothing: a client that "+ + "refuses every address refuses the guarded case too, for a reason that has nothing to do "+ + "with classification; got: %v", err) +} + +// TestVerifyDIDDocument_AcceptsAValidDomainOverTheGuardedPath is the other +// direction: refusing everything is not a fix. +// +// The dial is pinned to a local listener while the URL still names +// `example.com`, the way internal/api/handlers/aggregator's stubClient does it, +// because the validator refuses IP literals INDEPENDENTLY OF THE HATCH — so a +// fixture cannot simply point this call site at `127.0.0.1:PORT` the way the +// unfurl and imageproxy fixtures can. That is worth knowing before writing any +// future fixture for this path. +// +// This case uses a pinned plain client on purpose and proves nothing about the +// guard; the guarded path is the subtest above. What it proves is that a +// legitimate federated instance still verifies. +func TestVerifyDIDDocument_AcceptsAValidDomainOverTheGuardedPath(t *testing.T) { + t.Parallel() + + var requests int64 + var mu sync.Mutex + stub := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests++ + mu.Unlock() + + if r.URL.Path != "/.well-known/did.json" { + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": consumerTestDID, + "alsoKnownAs": []string{"at://" + consumerTestHandle}, + }) + })) + t.Cleanup(stub.Close) + + // stub.Client() trusts exactly this server's certificate, which is narrower + // than disabling verification. Only the ADDRESS is decided here; the name in + // the URL is still checked against the certificate. + client := stub.Client() + transport, ok := client.Transport.(*http.Transport) + require.Truef(t, ok, + "httptest client transport is %T, want *http.Transport — the dial cannot be pinned, so this "+ + "test would be talking to whatever example.com resolves to", client.Transport) + addr := stub.Listener.Addr().String() + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + + consumer := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + withWellKnownHTTPClient(client)) + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, consumerTestDomain, consumerTestHandle) + + require.NoErrorf(t, err, + "a legitimate federated instance failed verification. The validation being added must refuse "+ + "hosts this AppView should not fetch, not the ordinary hostnames every real instance "+ + "runs under; got: %v", err) + + mu.Lock() + defer mu.Unlock() + assert.Equalf(t, int64(1), requests, + "the DID document was fetched %d times rather than once", requests) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// single most important assertion for this call site. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// T0+T1+T2 — runs the PERMISSIVE branch here and at every other site holding +// such a boolean. A green merge gate therefore proves nothing whatsoever about +// whether this consumer is guarded in production. This function is the one place +// in the repository where the production branch is ever evaluated, which is why +// the gate must be a pure function and not an `if cfg.IsDevEnv` in +// cmd/server/wiring.go. +// +// The claim is not "the options returned are safe". It is that there are NONE: +// length zero, nothing applied, the constructor's own defaults left untouched. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). The production branch — the one "+ + "IS_DEV_ENV=true keeps `make ci` from ever evaluating — must contribute nothing at all, "+ + "so that what production gets is exactly the constructor's own defaults", len(opts)) +} + +// TestPrivateHostOptions_BindTheGateToTheConstructor pins the other direction +// through the state the constructor actually ends up in. +// +// A length check on the false branch is worthless on its own: a helper returning +// the wrong single-element slice satisfies it while leaving every developer +// unable to reach anything local, and a helper returning nothing in BOTH +// directions satisfies it while silently guaranteeing the guard can never be +// opened. +func TestPrivateHostOptions_BindTheGateToTheConstructor(t *testing.T) { + t.Parallel() + + guarded := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + PrivateHostOptions(false)...) + assert.False(t, guarded.allowPrivateHosts, + "a consumer built from PrivateHostOptions(false) has the SSRF hatch open. This is the branch "+ + "production runs and CI never does") + + hatched := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + PrivateHostOptions(true)...) + assert.True(t, hatched.allowPrivateHosts, + "a consumer built from PrivateHostOptions(true) is still guarded, so the dev hatch does "+ + "nothing and a developer's local stack cannot be verified against") +} + +// TestNewCommunityEventConsumer_BuildsAGuardedClientWithItsSettingsPreserved is +// the conversion's own fence. +// +// # WHAT CARRIES THE CONTRACT, AND WHAT USED TO +// +// This fence used to be the timeout plus `_, bare := …Transport.(*http.Transport)`. +// THAT TYPE ASSERTION CANNOT FAIL ON POLARITY. The hatch is a bool field INSIDE +// oauth's transport, not a different type, so a consumer built with the guard +// OPEN is just as much "not a bare *http.Transport" as one built with it shut — +// the check sees that the conversion happened and nothing about which way the +// switch is set. A build whose constructor hardcoded the hatch open passed this +// fence unchanged. +// +// So the contract is carried by REACHABILITY below, and the type and timeout +// checks are kept as the cheap secondary they always were. +// +// # THE TIMEOUT +// +// NewSSRFSafeHTTPClient ships a 15s ceiling of its own and this consumer has +// always run on 10s. Adopting the shared client without restoring the caller's +// value LOOSENS every .well-known fetch by five seconds — a change nobody asked +// for, arriving as part of an SSRF fix, on a firehose path where a slow remote +// host holds up event processing. blobs.NewBlobService, +// imageproxy.NewPDSFetcher and unfurl.NewService all restore their own for the +// same reason. +func TestNewCommunityEventConsumer_BuildsAGuardedClientWithItsSettingsPreserved(t *testing.T) { + t.Parallel() + + t.Run("the default client refuses a private address without reaching the listener", func(t *testing.T) { + t.Parallel() + + listener := newCountingWellKnown(t) + // No hatch option: PrivateHostOptions(false) is nil, so this is exactly + // what cmd/server constructs in production, plus a seam that sets no + // policy. + consumer := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + append(PrivateHostOptions(false), + withTransportOptions(covesoauth.WithHostResolver( + privateAnsweringResolver(t, "127.0.0.1"))))...) // coves:allow-host-literal: the address the seam answers with; the guard must refuse it before any dial + + reached, err := wellKnownClientRequest(t, consumer, listener) + + // THE REACHABILITY CLAIM COMES FIRST, deliberately. It is the security + // property, and asserting it before the error means a regression reports + // "the listener was reached" rather than the far less useful "an error + // was expected but got nil". + assert.Zerof(t, reached, + "the listener was reached %d time(s), so the consumer's own .well-known client dialled a "+ + "host whose DNS answer was 127.0.0.1 and delivered the request. That domain comes off "+ + "a community record published by anyone federated with this instance. This is the "+ + "assertion the old *http.Transport type check could not make, and the one that dies "+ + "if applyWellKnownClient's hatch argument is ever replaced by a constant", reached) + + require.Error(t, err, + "the request SUCCEEDED against a private address, so the default client is not guarded "+ + "at all") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a client that simply cannot make requests "+ + "satisfies this case too; got: %v", err) + }) + + t.Run("control: the same listener IS reached with the hatch open", func(t *testing.T) { + t.Parallel() + + listener := newCountingWellKnown(t) + consumer := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + append(PrivateHostOptions(true), // coves:allow-ssrf-hatch: the control half; without it the guarded case above is unfalsifiable + withTransportOptions(covesoauth.WithHostResolver( + privateAnsweringResolver(t, "127.0.0.1"))))...) // coves:allow-host-literal: the address the seam answers with; with the hatch open it must be dialled + + reached, err := wellKnownClientRequest(t, consumer, listener) + + require.NoErrorf(t, err, + "the hatch was open and the request still failed, so the guarded case above proves nothing: "+ + "a client that reaches nothing refuses a private address for reasons unconnected to "+ + "classification; got: %v", err) + assert.Equalf(t, int64(1), reached, + "the listener was reached %d time(s) with the hatch open, want exactly 1. The zero asserted "+ + "above only means something if this listener is reachable when classification permits it", + reached) + }) + + t.Run("secondary: the timeout and the transport are the guarded builder's", func(t *testing.T) { + t.Parallel() + + consumer := NewCommunityEventConsumer(nil, "did:web:coves.social", false, nil, + PrivateHostOptions(false)...) + + require.NotNil(t, consumer.httpClient, "the consumer must hold an HTTP client") + + assert.Equalf(t, 10*time.Second, consumer.httpClient.Timeout, + "the .well-known client runs on a %v timeout instead of the 10s this consumer has always "+ + "used. The shared SSRF client ships a 15s ceiling, so a call site that adopts it without "+ + "re-applying its own value silently re-times every federated verification", + consumer.httpClient.Timeout) + + // A TYPE CHECK AND NOTHING MORE. It catches "the conversion never + // happened"; it is blind to which way the hatch is set, which is why the + // subtests above exist. + _, bare := consumer.httpClient.Transport.(*http.Transport) + assert.Falsef(t, bare, + "the consumer's .well-known client still uses a bare *http.Transport, which resolves and "+ + "dials whatever a federated community record names. It must be the SSRF-safe transport "+ + "from internal/atproto/oauth, which vets the resolved addresses and then dials only "+ + "those — closing the check-then-dial window a naive guard leaves open") + }) +} + +// countingWellKnown is a server that answers a DID document and counts how many +// requests actually arrived. +// +// PLAIN HTTP, not TLS, and deliberately so. The claim is about ADDRESS +// CLASSIFICATION — did the transport dial, or refuse before dialling — and a TLS +// listener puts a certificate check between the dial and the handler. httptest's +// cert carries SANs [example.com, *.example.com], so a guard-test hostname would +// fail the handshake and the handler would record zero requests EVEN WITH THE +// HATCH OPEN, quietly turning the control above into another vacuous assertion. +// Plain HTTP removes the only other thing that can hold the counter at zero. +type countingWellKnown struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingWellKnown(t *testing.T) *countingWellKnown { + t.Helper() + + listener := &countingWellKnown{} + listener.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + listener.requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "id": consumerTestDID, + "alsoKnownAs": []string{"at://" + consumerTestHandle}, + }) + })) + t.Cleanup(listener.server.Close) + return listener +} + +// wellKnownClientRequest drives a consumer's OWN client at a host the seam +// resolves to the listener, and reports the error and whether it was reached. +// +// It calls consumer.httpClient directly rather than verifyDIDDocument, for the +// constraint the guardedConsumer comment block records: a validation-passing +// domain has no port, so verifyDIDDocument always builds https:///… on +// 443, and a test cannot bind 443. Naming the port here is what makes a +// reachable listener possible at all — and it gives up nothing this test claims, +// because the subject is the CLIENT the constructor installed, not the URL +// verifyDIDDocument builds. NormalizeDomain's separate refusal of ports is +// pinned by TestVerifyDIDDocument_RefusesEveryCorpusDomain. +func wellKnownClientRequest(t *testing.T, consumer *CommunityEventConsumer, listener *countingWellKnown) (reached int64, err error) { + t.Helper() + + _, port, err := net.SplitHostPort(listener.server.Listener.Addr().String()) + require.NoError(t, err, "the listener's address must split into host and port") + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, + "http://"+consumerGuardDomain+":"+port+"/.well-known/did.json", nil) + require.NoError(t, err) + + resp, doErr := consumer.httpClient.Do(req) + if doErr == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + return listener.requests.Load(), doErr +} + +// TestVerifyDIDDocument_RefusesEveryCorpusDomain is the wide net. +// +// The ordering test above drives only the payloads that parse into a URL, +// because those are the ones that prove ordering. This one drives the WHOLE +// shared corpus and asserts the same sentinel, so a domain shape added to the +// corpus for the aggregator's sake is automatically a domain shape this call +// site is tested against. That is the entire reason the corpus is a package. +func TestVerifyDIDDocument_RefusesEveryCorpusDomain(t *testing.T) { + t.Parallel() + + for _, tt := range domaincorpus.Invalid() { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + consumer, transport := newRecordingConsumer(t) + + err := consumer.verifyDIDDocument( + context.Background(), consumerTestDID, tt.Domain, consumerTestHandle) + + require.Errorf(t, err, "the consumer accepted the domain %q", tt.Domain) + assert.ErrorIsf(t, err, validation.ErrDomainInvalid, + "domain %q must be refused with validation.ErrDomainInvalid. Some rows in this corpus "+ + "are also refused by net/http or by the guard, and that is exactly why the sentinel "+ + "is asserted rather than the mere presence of an error: an incidental refusal is "+ + "one a dependency upgrade can take away; got: %v", tt.Domain, err) + assert.Emptyf(t, transport.seen(), + "the consumer reached its HTTP client for %v while handling domain %q", + transport.seen(), tt.Domain) + }) + } +} + +// TestVerifyDIDDocument_TheInjectionShapeIsWhatItLooksLike documents, in an +// executable place, the string this whole file exists because of. +// +// It is not testing fmt.Sprintf. It pins the SHAPE that makes the injection +// work, so a reader of a future failure can see in one line why +// `internal-admin/v1/secrets?x=y#` reaches `/v1/secrets?x=y` and why no +// address-level control can prevent it. If this test ever fails, the payload has +// stopped being dangerous and the corpus row can be re-evaluated — which is a +// conclusion nobody should reach by reasoning about it in their head. +func TestVerifyDIDDocument_TheInjectionShapeIsWhatItLooksLike(t *testing.T) { + t.Parallel() + + const payload = "internal-admin/v1/secrets?x=y#" + + parsed, err := url.Parse(fmt.Sprintf("https://%s/.well-known/did.json", payload)) + require.NoError(t, err, "the payload must still parse as a URL, or it would never reach a client") + + assert.Equal(t, "internal-admin", parsed.Host, + "the host the request would go to is the attacker's first label, not a domain they own") + assert.Equal(t, "/v1/secrets", parsed.Path, + "the path is the attacker's, not /.well-known/did.json — the concatenation was supposed to "+ + "decide this and the fragment took it away") + assert.Equal(t, "x=y", parsed.RawQuery, "the query is the attacker's too") + assert.Equal(t, "/.well-known/did.json", parsed.Fragment, + "the suffix this call site appends became a fragment, and fragments are never sent — which "+ + "is why this is a full request-forgery primitive and not an SSRF to one fixed path") +} diff --git a/internal/atproto/jetstream/direct_fetch_verification_test.go b/internal/atproto/jetstream/direct_fetch_verification_test.go index 84c5885..d70dc32 100644 --- a/internal/atproto/jetstream/direct_fetch_verification_test.go +++ b/internal/atproto/jetstream/direct_fetch_verification_test.go @@ -78,7 +78,8 @@ func TestDirectFetch_RecomputesTheCIDFromARealRepo(t *testing.T) { "createdAt": time.Now().UTC().Format(time.RFC3339), }) - fetcher := NewDevDirectPostFetcher(pinnedResolver(author.DID, pdsServer.URL())) + fetcher := NewDirectPostFetcher(pinnedResolver(author.DID, pdsServer.URL()), + PrivatePostFetcherOptions(true)...) consumer := NewPostEventConsumer( postgres.NewPostRepository(db), postgres.NewCommunityRepository(db), newMockUserService(), db, @@ -114,7 +115,7 @@ func TestDirectFetch_UsesSyncGetRecordNotTheJSONEnvelope(t *testing.T) { defer srv.Close() f := newAccFixture(t, db, WithPostRecordFetcher( - NewDevDirectPostFetcher(pinnedResolver(accAuthor, srv.URL)))) + NewDirectPostFetcher(pinnedResolver(accAuthor, srv.URL), PrivatePostFetcherOptions(true)...))) _ = f.consumer.HandleEvent(context.Background(), acceptanceEvent(accCommunity, uri, "bafyreiverifyendpoint", testkit.TID(), time.Now().UnixMicro())) @@ -149,7 +150,7 @@ func TestDirectFetch_RefusesAPDSThatLiesAboutTheCID(t *testing.T) { defer srv.Close() f := newAccFixture(t, db, WithPostRecordFetcher( - NewDevDirectPostFetcher(pinnedResolver(accAuthor, srv.URL)))) + NewDirectPostFetcher(pinnedResolver(accAuthor, srv.URL), PrivatePostFetcherOptions(true)...))) err := f.consumer.HandleEvent(ctx, acceptanceEvent(accCommunity, uri, pinned, testkit.TID(), time.Now().UnixMicro())) diff --git a/internal/atproto/oauth/client.go b/internal/atproto/oauth/client.go index 62f4bdb..443fcc9 100644 --- a/internal/atproto/oauth/client.go +++ b/internal/atproto/oauth/client.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "fmt" "log/slog" + "net/http" "net/url" "time" @@ -36,12 +37,74 @@ type OAuthConfig struct { ClientKeyID string } +// clientOptions is what the clientOption values accumulate into: the extra +// transport options all three of this constructor's HTTP clients are built with. +type clientOptions struct { + // transportOptions is the TEST SEAM, and it is unexported deliberately — + // scripts/ssrf-audit.sh fails the build if the resolver seam it carries is + // reachable from any non-test package. Every peer that needed the same thing + // keeps it unexported for the same reason: pds/factory.go's + // withTransportOptions, identity's withHTTPClient, jetstream's + // withWellKnownHTTPClient, blobs' setHTTPClient. + transportOptions []Option +} + +// clientOption configures the HTTP clients NewOAuthClient builds. +type clientOption func(*clientOptions) + +// withTransportOptions passes extra options to ALL THREE clients this +// constructor builds — the ClientApp's, the OAuth metadata resolver's, and the +// directory's. +// +// IT EXISTS BECAUSE A LOOPBACK FIXTURE PROVES THE WRONG BRANCH. Every address a +// hermetic test can offer is an IP literal, which RoundTrip refuses on SHAPE one +// branch before the address classifier runs — so a literal-only test here would +// leave classification, the check these three clients actually depend on for a +// PDS host that arrives from a DID document, unexercised. Threading +// WithHostResolver through this seam lets a test drive a real HOSTNAME whose +// answer it chooses. +// +// ALL THREE CLIENTS, not one. They are the same construction with the same +// address policy, and a seam that reached only the first would let a test +// certify the ClientApp's client while saying nothing about the directory's. +func withTransportOptions(opts ...Option) clientOption { + return func(c *clientOptions) { c.transportOptions = append(c.transportOptions, opts...) } +} + +const oauthMetadataMaxResponseBytes int64 = 1 << 20 + +// newOAuthResolverHTTPClient keeps Indigo's resolver-specific restrictions on +// top of the shared Coves address guard. Resolver responses are small JSON +// documents, so they get a much tighter cap than general PDS responses. +func newOAuthResolverHTTPClient(opts ...Option) *http.Client { + resolverOptions := append([]Option(nil), opts...) + resolverOptions = append(resolverOptions, WithMaxResponseBytes(oauthMetadataMaxResponseBytes)) + + client := NewSSRFSafeHTTPClient(resolverOptions...) + client.Timeout = 10 * time.Second + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return fmt.Errorf("too many redirects") + } + if req.URL.Scheme != "https" || req.URL.Hostname() == "" || req.URL.Port() != "" { + return fmt.Errorf("OAuth metadata redirect must use HTTPS with no explicit port: %s", req.URL.Redacted()) + } + return nil + } + return client +} + // NewOAuthClient creates a new OAuth client for Coves -func NewOAuthClient(config *OAuthConfig, store oauth.ClientAuthStore) (*OAuthClient, error) { +func NewOAuthClient(config *OAuthConfig, store oauth.ClientAuthStore, opts ...clientOption) (*OAuthClient, error) { if config == nil { return nil, fmt.Errorf("config is required") } + var options clientOptions + for _, opt := range opts { + opt(&options) + } + // PLCURL must be explicit. An empty value used to fall through to indigo's // default directory, which is the production plc.directory - so a test or a // misconfigured deploy that simply forgot the field would silently resolve @@ -133,17 +196,26 @@ func NewOAuthClient(config *OAuthConfig, store oauth.ClientAuthStore) (*OAuthCli // Set user agent clientConfig.UserAgent = "Coves/1.0" + transportOptions := append([]Option(nil), PrivateAddressOptions(config.AllowPrivateIPs)...) + transportOptions = append(transportOptions, options.transportOptions...) // Create the indigo OAuth ClientApp + // coves:allow-bare-client: NewClientApp installs http.DefaultClient (indigo auth/oauth/oauth.go:55); the next statement replaces it with the guarded client clientApp := oauth.NewClientApp(&clientConfig, store) // Override the default HTTP client with our SSRF-safe client // This protects against SSRF attacks via malicious PDS URLs, DID documents, and JWKS URIs - clientApp.Client = NewSSRFSafeHTTPClient(config.AllowPrivateIPs) + clientApp.Client = NewSSRFSafeHTTPClient(transportOptions...) + + // Indigo constructs its OAuth metadata resolver with a third, independent + // HTTP client. Replace it too: StartAuthFlow accepts an attacker-controlled + // https:// authorization-server identifier on a public route, so leaving the + // resolver's default client in place would leave a proxy-aware SSRF path. + clientApp.Resolver.Client = newOAuthResolverHTTPClient(transportOptions...) // Always override the directory so resolution goes to the configured PLC // rather than indigo's default. Use SSRF-safe HTTP client for PLC requests. - httpClient := NewSSRFSafeHTTPClient(config.AllowPrivateIPs) + httpClient := NewSSRFSafeHTTPClient(transportOptions...) baseDir := &identity.BaseDirectory{ PLCURL: config.PLCURL, HTTPClient: *httpClient, diff --git a/internal/atproto/oauth/client_guard_test.go b/internal/atproto/oauth/client_guard_test.go new file mode 100644 index 0000000..9837bfc --- /dev/null +++ b/internal/atproto/oauth/client_guard_test.go @@ -0,0 +1,577 @@ +package oauth + +import ( + "context" + "encoding/base64" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" + + indigooauth "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The three clients NewOAuthClient installs on indigo's ClientApp, and the fact +// that all three are installed BY ASSIGNMENT over working defaults. +// +// # WHY ASSIGNMENT IS THE WHOLE PROBLEM +// +// indigo's NewClientApp returns an app that is already complete: +// +// Client: http.DefaultClient +// Resolver.Client: an Indigo PublicOnlyTransport client +// Dir: identity.DefaultDirectory() +// +// (atproto/auth/oauth/oauth.go:53-57). Our constructor then overwrites all three. So +// the guard here is not a call that fails when it is missing — it is a +// REPLACEMENT, and deleting any assignment leaves a compiling, running, fully +// functional OAuth client with the guard silently gone. There is no nil to +// panic on and no zero value to notice. Before this file, deleting the Client +// assignment failed ZERO tests in this repository; that was established by +// executed mutation, not by reading. +// +// # WHAT IS ON THE OTHER SIDE OF THESE CLIENTS +// +// ClientApp.Client carries every OAuth-authenticated call this AppView makes to +// a user's PDS, ClientApp.Dir resolves the identity records that produce those +// hosts, and ClientApp.Resolver fetches OAuth metadata during StartAuthFlow. The +// last path is publicly reachable and accepts an https:// identifier directly. +// +// # WHY REACHABILITY, NOT ONLY THE ERROR +// +// Mutation testing produced a guard that classified correctly, emitted +// a byte-identical message, and refused the request AFTER delivering it. For a +// destination a stranger named, the packet leaving IS the SSRF. Every case below +// stands up a real listener and asserts its handler never ran. + +// clientGuardTestDID is a syntactically valid did:plc that does not exist on the +// real network. +// +// NOT A REAL DID, DELIBERATELY. The mutation these tests exist to catch — +// deleting `clientApp.Dir = &cacheDir` — hands resolution back to indigo's +// DefaultDirectory, which is pointed at the public plc.directory. A well-known +// DID would then RESOLVE, on any machine with egress, and a test asserting only +// "an error came back" would flip from failing to passing depending on whose +// laptop it ran on. The assertions below are about which listener was reached, +// which does not depend on that; this constant keeps the error path from +// depending on it either. +const clientGuardTestDID = "did:plc:abcdefghijklmnopqrstuvwx" + +// countingHost is an HTTP listener that records whether anything reached it. It +// listens on loopback — the address class the guard exists to refuse — so its +// counter is the assertion. +type countingHost struct { + server *httptest.Server + requests atomic.Int64 +} + +// newCountingPDS answers like a PDS: anything at all, since nothing below +// depends on the body. +func newCountingPDS(t *testing.T) *countingHost { + t.Helper() + return newCountingHost(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + }) +} + +// newCountingPLC answers a DID lookup with a document indigo will accept. +// +// No alsoKnownAs, deliberately. Indigo verifies a DECLARED handle +// bidirectionally over DNS and HTTPS, and that lookup would leave the machine — +// which the hermetic tiers forbid. With none declared it marks the handle +// invalid and returns, so this fixture stays local. +func newCountingPLC(t *testing.T) *countingHost { + t.Helper() + return newCountingHost(t, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = fmt.Fprintf(w, `{"id":%q,"service":[{"id":"#atproto_pds",`+ + `"type":"AtprotoPersonalDataServer","serviceEndpoint":"https://pds.example.invalid"}]}`, + clientGuardTestDID) + }) +} + +func newCountingHost(t *testing.T, handler http.HandlerFunc) *countingHost { + t.Helper() + + host := &countingHost{} + host.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + host.requests.Add(1) + handler(w, r) + })) + t.Cleanup(host.server.Close) + return host +} + +// port is the loopback port this listener is on, so a test can address it by a +// NAME on the same port and have the dial actually arrive. +func (h *countingHost) port(t *testing.T) string { + t.Helper() + + parsed, err := url.Parse(h.server.URL) + require.NoError(t, err, "the httptest server's own URL must parse") + return parsed.Port() +} + +// guardTestConfig is the smallest OAuthConfig NewOAuthClient accepts, in the +// PRODUCTION shape: DevMode off, so nothing about the construction under test is +// a dev-only path. +func guardTestConfig(plcURL string, allowPrivateIPs bool) *OAuthConfig { + return &OAuthConfig{ + PublicURL: "https://coves.example", + SealSecret: base64.StdEncoding.EncodeToString(make([]byte, 32)), + PLCURL: plcURL, + Scopes: []string{"atproto"}, + AllowPrivateIPs: allowPrivateIPs, + } +} + +// resolvesTo builds the seam that makes a HOSTNAME answer with an address the +// test chose, so the address CLASSIFIER runs rather than the shape check that +// refuses IP literals one branch earlier. +func resolvesTo(t *testing.T, addr string) clientOption { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(addr) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", addr) + + return withTransportOptions(WithHostResolver(func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + })) +} + +// TestNewOAuthClient_InstallsGuardedClientsForEveryEgressComponent is the +// construction fence: Indigo creates three independent clients, so checking +// only ClientApp.Client and the directory leaves the metadata resolver behind. +func TestNewOAuthClient_InstallsGuardedClientsForEveryEgressComponent(t *testing.T) { + t.Parallel() + + oauthClient, err := NewOAuthClient( + guardTestConfig("https://plc.example.invalid", false), indigooauth.NewMemStore()) + require.NoError(t, err, "the test's own config must build a client") + + cacheDir, ok := oauthClient.ClientApp.Dir.(*identity.CacheDirectory) + require.True(t, ok, "ClientApp.Dir must be the configured cache directory, got %T", oauthClient.ClientApp.Dir) + baseDir, ok := cacheDir.Inner.(*identity.BaseDirectory) + require.True(t, ok, "the cache must wrap the configured base directory, got %T", cacheDir.Inner) + + clients := []struct { + name string + client *http.Client + }{ + {name: "ClientApp.Client", client: oauthClient.ClientApp.Client}, + {name: "ClientApp.Resolver.Client", client: oauthClient.ClientApp.Resolver.Client}, + {name: "BaseDirectory.HTTPClient", client: &baseDir.HTTPClient}, + } + + for _, candidate := range clients { + candidate := candidate + t.Run(candidate.name, func(t *testing.T) { + require.NotNil(t, candidate.client, "%s must be installed", candidate.name) + transport, ok := candidate.client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "%s uses %T instead of the Coves SSRF transport", candidate.name, candidate.client.Transport) + assert.False(t, transport.allowPrivate, "%s unexpectedly has the private-address hatch open", candidate.name) + assert.Nil(t, transport.base.Proxy, "%s must not honor proxy environment variables", candidate.name) + }) + } + + resolverTransport := oauthClient.ClientApp.Resolver.Client.Transport.(*ssrfSafeTransport) + assert.Equal(t, 10*time.Second, oauthClient.ClientApp.Resolver.Client.Timeout, + "the resolver must retain Indigo's ten-second deadline") + assert.Equal(t, int64(oauthMetadataMaxResponseBytes), resolverTransport.maxResponseBytes, + "OAuth metadata needs its tighter response cap") + assert.NotSame(t, oauthClient.ClientApp.Client, oauthClient.ClientApp.Resolver.Client, + "the resolver must not silently alias the general OAuth client") +} + +func TestOAuthResolverClient_RejectsUnsafeRedirectTargets(t *testing.T) { + t.Parallel() + + client := newOAuthResolverHTTPClient() + require.NotNil(t, client.CheckRedirect, "the metadata client must validate every redirect") + + tests := []struct { + name string + target string + wantErr bool + }{ + {name: "HTTPS without an explicit port", target: "https://auth.example/metadata"}, + {name: "HTTP downgrade", target: "http://auth.example/metadata", wantErr: true}, + {name: "explicit standard port", target: "https://auth.example:443/metadata", wantErr: true}, + {name: "explicit non-standard port", target: "https://auth.example:8443/metadata", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, tt.target, nil) + require.NoError(t, err, "building the redirect request") + + err = client.CheckRedirect(req, []*http.Request{{}}) + if tt.wantErr { + assert.Error(t, err, "unsafe metadata redirect %s was accepted", tt.target) + return + } + assert.NoError(t, err, "safe metadata redirect %s was rejected", tt.target) + }) + } +} + +// routeGuardedClientToTLSServer keeps the guard and resolver intact while +// making every allowed socket land on a hermetic TLS listener. The server's +// client already trusts its generated certificate for example.com. +func routeGuardedClientToTLSServer(t *testing.T, client *http.Client, server *httptest.Server) { + t.Helper() + + guard, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "expected the Coves SSRF transport, got %T", client.Transport) + trustedTransport, ok := server.Client().Transport.(*http.Transport) + require.True(t, ok, "httptest TLS client uses %T, not *http.Transport", server.Client().Transport) + base := trustedTransport.Clone() + dialer := &net.Dialer{} + listenerAddr := server.Listener.Addr().String() + base.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, listenerAddr) + } + guard.base = base +} + +func newOAuthFlowServer(t *testing.T) (*httptest.Server, *atomic.Int64) { + t.Helper() + + const issuer = "https://example.com" + requests := &atomic.Int64{} + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/.well-known/oauth-authorization-server": + _, _ = fmt.Fprintf(w, `{ + "issuer":%q, + "authorization_endpoint":%q, + "token_endpoint":%q, + "response_types_supported":["code"], + "grant_types_supported":["authorization_code","refresh_token"], + "code_challenge_methods_supported":["S256"], + "token_endpoint_auth_methods_supported":["none","private_key_jwt"], + "token_endpoint_auth_signing_alg_values_supported":["ES256"], + "scopes_supported":["atproto"], + "authorization_response_iss_parameter_supported":true, + "require_pushed_authorization_requests":true, + "pushed_authorization_request_endpoint":%q, + "dpop_signing_alg_values_supported":["ES256"], + "client_id_metadata_document_supported":true + }`, + issuer, issuer+"/authorize", issuer+"/token", issuer+"/par") + case "/par": + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"request_uri":"urn:ietf:params:oauth:request_uri:test","expires_in":60}`)) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + return server, requests +} + +// TestOAuthClientApp_StartAuthFlowRefusesPrivateMetadataDNSWithoutReachingIt +// covers the public https:// identifier path end to end, then proves the same +// fixture is reachable only when the explicit dev hatch is open. +func TestOAuthClientApp_StartAuthFlowRefusesPrivateMetadataDNSWithoutReachingIt(t *testing.T) { + t.Parallel() + + const issuer = "https://example.com" + tests := []struct { + name string + allowPrivate bool + wantRequests int64 + }{ + {name: "guarded", allowPrivate: false, wantRequests: 0}, + {name: "hatch open", allowPrivate: true, wantRequests: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server, requests := newOAuthFlowServer(t) + oauthClient, err := NewOAuthClient( + guardTestConfig("https://plc.example.invalid", tt.allowPrivate), indigooauth.NewMemStore(), + resolvesTo(t, "127.0.0.1")) // coves:allow-host-literal: the seam's private DNS answer; production must refuse it before dialing + require.NoError(t, err, "the test's own config must build a client") + + routeGuardedClientToTLSServer(t, oauthClient.ClientApp.Resolver.Client, server) + routeGuardedClientToTLSServer(t, oauthClient.ClientApp.Client, server) + + redirect, flowErr := oauthClient.ClientApp.StartAuthFlow(t.Context(), issuer) + assert.Equal(t, tt.wantRequests, requests.Load(), + "StartAuthFlow reached the private listener %d times", requests.Load()) + + if !tt.allowPrivate { + require.Error(t, flowErr, "private metadata DNS must stop StartAuthFlow") + assert.ErrorIs(t, flowErr, ErrBlockedAddress, + "the failure must come from the Coves address guard; got: %v", flowErr) + return + } + + require.NoError(t, flowErr, "the hatch-open control must complete StartAuthFlow") + assert.Contains(t, redirect, issuer+"/authorize?", + "the control must produce the authorization redirect, got %q", redirect) + }) + } +} + +func TestOAuthResolverClient_CapsMetadataBodies(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + payload := make([]byte, oauthMetadataMaxResponseBytes+1) + payload[0] = '"' + for i := 1; i < len(payload); i++ { + payload[i] = 'a' + } + _, _ = w.Write(payload) + })) + defer server.Close() + + oauthClient, err := NewOAuthClient( + guardTestConfig("https://plc.example.invalid", true), indigooauth.NewMemStore(), + resolvesTo(t, "127.0.0.1")) // coves:allow-host-literal: hatch-open body-cap fixture, routed to the TLS test listener + require.NoError(t, err, "the test's own config must build a client") + routeGuardedClientToTLSServer(t, oauthClient.ClientApp.Resolver.Client, server) + + _, resolveErr := oauthClient.ClientApp.Resolver.ResolveAuthServerMetadata(t.Context(), "https://example.com") + require.Error(t, resolveErr, "metadata larger than the resolver cap must fail") + assert.ErrorIs(t, resolveErr, ErrResponseTooLarge, + "the resolver consumed an oversized document without the metadata cap; got: %v", resolveErr) + assert.Equal(t, int64(1), requests.Load(), "the cap test must reach its server exactly once") +} + +// TestOAuthClientApp_ClientRefusesAPrivatePDSWithoutReachingIt is the binding +// contract for client.go's `clientApp.Client = ...` line. +// +// The listener is a stand-in for whatever shares a network with this AppView — +// its Postgres, its PDS, its Jetstream, a cloud metadata endpoint. Under the +// deletion this client is http.DefaultClient, which reaches every one of them. +func TestOAuthClientApp_ClientRefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + oauthClient, err := NewOAuthClient(guardTestConfig("https://plc.example.invalid", false), nil) + require.NoError(t, err, "the test's own config must build a client") + require.NotNil(t, oauthClient.ClientApp.Client, "the ClientApp must carry an HTTP client") + + resp, getErr := oauthClient.ClientApp.Client.Get(pds.server.URL) + if getErr == nil { + _ = resp.Body.Close() + } + + // THE REACHABILITY CLAIM COMES FIRST, deliberately. It is the security fact; + // the error is only how the caller learns about it. Asserting the error first + // with require would abort on failure and hide whether the request actually + // left the process. + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. ClientApp.Client is what indigo sends every "+ + "OAuth-authenticated PDS call through, and the PDS host comes from the user's DID "+ + "document — so a stranger picks the destination. Deleting the assignment in client.go "+ + "leaves indigo's own default, http.DefaultClient: no guard, and no timeout either", + pds.requests.Load()) + + require.Error(t, getErr, + "ClientApp.Client fetched a loopback address and reported success, which means it is not "+ + "the guarded client this constructor is supposed to install") + + assert.ErrorIsf(t, getErr, ErrBlockedAddress, + "the refusal must carry the guard's own identity. Without it a build where the assignment "+ + "was deleted looks identical — an unreachable address fails too; got: %v", getErr) +} + +// TestOAuthClientApp_ClientRefusesAWellFormedHostThatResolvesPrivate is the +// assertion a loopback-literal fixture cannot make. +// +// The case above is refused on SHAPE — an address written where a hostname +// belongs — one branch before the address classifier runs. That branch is real +// and worth having, but it is not what protects this site: a PDS endpoint out of +// a DID document is a NAME, and its owner controls the zone, so the address is +// decided after every shape check has already passed. This drives that path. +func TestOAuthClientApp_ClientRefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + oauthClient, err := NewOAuthClient( + guardTestConfig("https://plc.example.invalid", false), nil, + resolvesTo(t, "127.0.0.1")) // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + require.NoError(t, err, "the test's own config must build a client") + + // `.example` is reserved by RFC 2606, so if the seam were ever bypassed this + // resolves nowhere rather than reaching a real host. + target := "http://user-pds.example:" + pds.port(t) + "/xrpc/com.atproto.repo.putRecord" + + resp, getErr := oauthClient.ClientApp.Client.Get(target) + if getErr == nil { + _ = resp.Body.Close() + } + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times through a hostname whose DNS answer was 127.0.0.1. The "+ + "port is the loopback listener's own, so this is a request that arrived", pds.requests.Load()) + + require.Error(t, getErr, + "user-pds.example is a well-formed host whose answer was a loopback address, and the request "+ + "went ahead anyway") + + assert.ErrorIsf(t, getErr, ErrBlockedAddress, + "the refusal must be the address classifier's. If this client were indigo's http.DefaultClient "+ + "the seam would not exist at all and the failure would be an ordinary DNS error, which is "+ + "the shape this assertion separates; got: %v", getErr) +} + +// TestOAuthClientApp_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control for the case above. +// +// Identical construction, identical seam, identical host and port — only +// AllowPrivateIPs differs. With the hatch open the address is no longer refused +// and the request ARRIVES at the listener, which is what pins the refusal above +// to classification rather than to this client being unable to make requests at +// all. A client that refused everything would satisfy that test perfectly. +func TestOAuthClientApp_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + oauthClient, err := NewOAuthClient( + guardTestConfig("https://plc.example.invalid", true), nil, + resolvesTo(t, "127.0.0.1")) // coves:allow-host-literal: with the hatch open this is dialled and the listener answers + require.NoError(t, err, "the test's own config must build a client") + + target := "http://user-pds.example:" + pds.port(t) + "/xrpc/com.atproto.repo.putRecord" + + resp, getErr := oauthClient.ClientApp.Client.Get(target) + if getErr == nil { + _ = resp.Body.Close() + } + + require.NoErrorf(t, getErr, + "the hatch is what every dev stack depends on: a client built with AllowPrivateIPs must reach "+ + "a PDS on the developer's own machine; got: %v", getErr) + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once, so the seam is not answering with the "+ + "address this test gave it and the guarded case above proves nothing", pds.requests.Load()) +} + +// TestOAuthClientApp_DirRefusesAPrivatePLCWithoutReachingIt pins the HTTPClient +// field on the BaseDirectory this constructor builds. +// +// THE FIELD IS A VALUE, NOT A POINTER — `HTTPClient http.Client` — so omitting +// it is not a nil dereference and not a compile error. It is a zero-value +// client: Transport nil, which means http.DefaultTransport, and Timeout zero, +// which means wait forever. Both failure modes are invisible at the +// construction site. +func TestOAuthClientApp_DirRefusesAPrivatePLCWithoutReachingIt(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + oauthClient, err := NewOAuthClient(guardTestConfig(plc.server.URL, false), nil) + require.NoError(t, err, "the test's own config must build a client") + require.NotNil(t, oauthClient.ClientApp.Dir, "the ClientApp must carry an identity directory") + + did, err := syntax.ParseDID(clientGuardTestDID) + require.NoError(t, err, "the test's own DID must parse, or no client is ever reached") + + _, lookupErr := oauthClient.ClientApp.Dir.LookupDID(context.Background(), did) + + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times. The directory's HTTPClient field is an http.Client "+ + "VALUE, so deleting it yields a zero-value client that uses http.DefaultTransport and "+ + "never times out — a working directory with no guard on it", plc.requests.Load()) + + require.Error(t, lookupErr, + "the OAuth client's directory resolved a DID against a PLC on loopback") + assert.Containsf(t, lookupErr.Error(), "SSRF blocked", + "the refusal must be the guard's and must say so: indigo wraps this in its own resolution "+ + "error, so the sentence is what tells a blocked address from a PLC that is merely down; "+ + "got: %v", lookupErr) +} + +// TestOAuthClientApp_DirRefusesAWellFormedHostThatResolvesPrivate is the +// classifier-driving twin of the case above, for the same reason its sibling +// exists on the ClientApp's client: a loopback literal is refused on shape. +func TestOAuthClientApp_DirRefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + oauthClient, err := NewOAuthClient( + guardTestConfig("http://plc-directory.example:"+plc.port(t), false), nil, + resolvesTo(t, "127.0.0.1")) // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + require.NoError(t, err, "the test's own config must build a client") + + did, err := syntax.ParseDID(clientGuardTestDID) + require.NoError(t, err, "the test's own DID must parse, or no client is ever reached") + + _, lookupErr := oauthClient.ClientApp.Dir.LookupDID(context.Background(), did) + + assert.Zerof(t, plc.requests.Load(), + "the PLC listener was reached %d times through a hostname whose DNS answer was 127.0.0.1", + plc.requests.Load()) + + require.Error(t, lookupErr, "the directory resolved against a name that answers with a loopback address") + assert.Containsf(t, lookupErr.Error(), "SSRF blocked", + "the refusal must be the address classifier's, not a DNS failure — which is what a directory "+ + "built with a zero-value HTTPClient would produce here, since the seam lives on the client "+ + "this construction is supposed to install; got: %v", lookupErr) +} + +// TestOAuthClientApp_DirIsTheConfiguredDirectory pins the OTHER assignment, +// `clientApp.Dir = &cacheDir`, and it is the only case here that can. +// +// Deleting that line does not produce a broken directory. It produces indigo's +// DefaultDirectory — a real, working, UNGUARDED directory pointed at the public +// plc.directory — so every refusal assertion in this file still passes: nothing +// reaches the test's listener under that mutation either, because resolution +// went somewhere else entirely. Silently resolving against production instead of +// the configured PLC is its own bug, on top of losing the guard. +// +// So this asserts the POSITIVE direction: with the hatch open, the configured +// PLC — and only it — is what answers. The hatch is what makes a loopback +// fixture addressable at all, and it is also the arrangement a developer runs. +func TestOAuthClientApp_DirIsTheConfiguredDirectory(t *testing.T) { + t.Parallel() + + plc := newCountingPLC(t) + + oauthClient, err := NewOAuthClient(guardTestConfig(plc.server.URL, true), nil) + require.NoError(t, err, "the test's own config must build a client") + + did, err := syntax.ParseDID(clientGuardTestDID) + require.NoError(t, err, "the test's own DID must parse, or no client is ever reached") + + ident, lookupErr := oauthClient.ClientApp.Dir.LookupDID(context.Background(), did) + + require.NoErrorf(t, lookupErr, + "the OAuth client's directory could not resolve a DID against the PLC it was configured with. "+ + "In dev that PLC is on loopback, so this is what every local login depends on; got: %v", + lookupErr) + assert.Equalf(t, int64(1), plc.requests.Load(), + "the configured PLC was reached %d times rather than once. Zero means resolution went "+ + "somewhere this test did not configure — which is exactly what deleting `clientApp.Dir = "+ + "&cacheDir` does: indigo's DefaultDirectory answers instead, against the public "+ + "plc.directory, unguarded", plc.requests.Load()) + assert.Equal(t, clientGuardTestDID, ident.DID.String(), + "the identity must be the one the configured directory served") + assert.Equal(t, "https://pds.example.invalid", ident.PDSEndpoint(), + "the PDS endpoint is what every OAuth-authenticated call is subsequently aimed at") +} diff --git a/internal/atproto/oauth/dev_auth_resolver.go b/internal/atproto/oauth/dev_auth_resolver.go index 585fdc6..0b5e42b 100644 --- a/internal/atproto/oauth/dev_auth_resolver.go +++ b/internal/atproto/oauth/dev_auth_resolver.go @@ -34,7 +34,7 @@ type ProtectedResourceMetadata struct { // NewDevAuthResolver creates a resolver that accepts localhost HTTP URLs func NewDevAuthResolver(pdsURL string, allowPrivateIPs bool) *DevAuthResolver { resolver := &DevAuthResolver{ - Client: NewSSRFSafeHTTPClient(allowPrivateIPs), + Client: NewSSRFSafeHTTPClient(PrivateAddressOptions(allowPrivateIPs)...), UserAgent: "Coves/1.0", PDSURL: pdsURL, } diff --git a/internal/atproto/oauth/dev_resolver.go b/internal/atproto/oauth/dev_resolver.go index 5c089ff..c5763ba 100644 --- a/internal/atproto/oauth/dev_resolver.go +++ b/internal/atproto/oauth/dev_resolver.go @@ -25,7 +25,7 @@ type DevHandleResolver struct { func NewDevHandleResolver(pdsURL string, allowPrivateIPs bool) *DevHandleResolver { return &DevHandleResolver{ pdsURL: strings.TrimSuffix(pdsURL, "/"), - httpClient: NewSSRFSafeHTTPClient(allowPrivateIPs), + httpClient: NewSSRFSafeHTTPClient(PrivateAddressOptions(allowPrivateIPs)...), } } diff --git a/internal/atproto/oauth/transport.go b/internal/atproto/oauth/transport.go index 09af26d..7882bf6 100644 --- a/internal/atproto/oauth/transport.go +++ b/internal/atproto/oauth/transport.go @@ -2,32 +2,183 @@ package oauth import ( "context" + "errors" "fmt" + "io" "net" "net/http" + "net/netip" "time" + "unicode" + + "golang.org/x/net/idna" ) +// ErrBlockedAddress is the sentinel every address refusal matches, so a caller +// distinguishes "the guard refused this" from "the network failed" with +// errors.Is rather than by matching on a message. +// +// A resolution failure must NOT match it. The two need opposite handling — a +// DNS hiccup is retryable and unremarkable, a refusal is a security event — so +// wrapping every RoundTrip error in this sentinel would hand every caller a +// signal that means "something went wrong" and nothing more. +var ErrBlockedAddress = errors.New("SSRF blocked") + +// blockedDial builds a refusal from the DIAL path, matching ErrBlockedAddress. +// +// A CONSTRUCTOR RATHER THAN THREE fmt.Errorf CALLS, because all three of those +// refusals used to render "SSRF blocked" and match nothing — the words were +// right and the identity was missing, which is the worst of both: the string +// reads as a block to a human scanning a log, and classifies as an ordinary +// network failure to the code deciding whether to retry. One place to get it +// right is one place for the next refusal added here to get it right too. +// +// The rendering is unchanged: ErrBlockedAddress's own message is "SSRF blocked", +// so "%w: " reproduces the prefix the rest of this package asserts on. +// +// The dial path has no BlockedAddressError of its own on purpose. That type +// carries a host and the address it resolved to, and none of these three +// refusals has resolved anything — they are structural failures of the dial +// itself, not a classification. +func blockedDial(format string, args ...any) error { + return fmt.Errorf("%w: "+format, append([]any{ErrBlockedAddress}, args...)...) +} + +// BlockedAddressError carries the detail behind a refusal — which host, and +// which of its addresses caused it — where errors.As can reach it and a +// rendered message cannot leak it. +// +// The split is the point. An operator debugging a block still has to know which +// answer caused it, especially when a name resolved to several and only one was +// private; a reader of the rendered string must not learn the same thing, +// because error strings travel into HTTP response bodies, shared logs and +// support tickets, and every host that reaches this transport was named by a +// stranger. +type BlockedAddressError struct { + Host string + + // `json:"-"` because encoding/json is the OTHER renderer that reaches this + // field, and it is reached on the same journeys Error() was rewritten for: + // a structured logger handed the error value, an API rendering a failure as + // JSON. Marshalling would put the address back beside the attacker-chosen + // host — the whole mapping primitive in one object. + // + // IT COVERS encoding/json AND NOTHING ELSE. %#v and %+v read the field + // through reflection regardless of tags, so a caller that dumps this struct + // with a formatting verb still sees the address; shutting that route down + // needs the field unexported behind an accessor, which is a larger change + // and is not made here. errors.As remains the intended way to reach it. + IP net.IP `json:"-"` + + // literal separates the two refusals this type carries — an address written + // where a hostname belongs, versus a name that resolved to a blocked one — + // so the rendered sentence is true of the one that actually happened. It is + // unexported because it selects wording rather than being something a caller + // acts on; the fields above are the diagnostic. Its zero value gives the + // resolution wording, which is the case every other construction is. + literal bool +} + +// Error renders the refusal WITHOUT the resolved address. +// +// That address is an internal-network oracle: the attacker supplied the +// hostname — a DID document's serviceEndpoint, an acceptance record's subject, +// a name whose zone they control — so an error that reports what it resolved to +// turns each refusal into a mapping primitive. Point a name at a candidate +// address, read the answer back out of the message, repeat. +// +// The host stays because it is the half of the sentence the attacker already +// supplied, and because it cannot be hidden anyway: http.Client.Do wraps this +// in a *url.Error that embeds the full request URL. +func (e *BlockedAddressError) Error() string { + if e.literal { + return fmt.Sprintf("SSRF blocked: %s is an address written where a hostname belongs", e.Host) + } + return fmt.Sprintf("SSRF blocked: %s resolves to a private or reserved address", e.Host) +} + +// Unwrap makes errors.Is(err, ErrBlockedAddress) hold. The sentinel is the +// checkable identity; this type is the detail behind it. +func (e *BlockedAddressError) Unwrap() error { + return ErrBlockedAddress +} + +// ErrResponseTooLarge is what a read fails with once a response body has +// delivered more than the transport's cap. +// +// IT IS DELIBERATELY NOT io.EOF, and that is the whole point of having a +// sentinel here rather than ending the stream. io.ReadAll treats io.EOF as the +// clean end of a body and returns a nil error, so a cap that reported itself +// that way would hand every caller in this tree a short body indistinguishable +// from a complete one — a parser would read half a document, a size check like +// blobs/service.go's would never fire, and a truncated image would be written +// to a user's PDS as a whole one. Silent truncation is worse than no cap at +// all, because no cap at least fails loudly. +var ErrResponseTooLarge = errors.New("response body exceeds the maximum size") + // ssrfSafeTransport wraps http.Transport to prevent SSRF attacks type ssrfSafeTransport struct { - base *http.Transport - allowPrivate bool // For dev/testing only + base *http.Transport + + // allowPrivate turns the address guard off, for dev and testing only. Set ONLY + // by WithPrivateAddressesAllowed, which only ever opens it — the constructor's + // struct literal leaves it at its zero value, so a client built with no + // options is guarded. Read in two places in RoundTrip; see the option for why + // that matters. + allowPrivate bool + + // maxResponseBytes is how much of a response body a caller may read before + // the read fails. Set by WithMaxResponseBytes; DefaultMaxResponseBytes + // otherwise, applied by the constructor BEFORE the options run so a client + // built without one is capped rather than capped at zero. + maxResponseBytes int64 // lookupIP resolves a hostname. A field so a test can drive the - // check-then-dial window that the guard has to close; nil means net.LookupIP. - lookupIP func(host string) ([]net.IP, error) + // check-then-dial window that the guard has to close; nil means the default + // resolver. + lookupIP func(ctx context.Context, host string) ([]net.IP, error) } // resolveHost is the transport's one name lookup per request. -func (t *ssrfSafeTransport) resolveHost(host string) ([]net.IP, error) { +// +// The lookup runs under the REQUEST'S OWN CONTEXT, which is what makes a +// cancellation mean something here. net.LookupIP takes no context, so a caller +// that gives up — a client that closed the connection, a handler whose deadline +// expired, a shutdown in progress — released nothing: the lookup kept a +// goroutine and a socket alive until the resolver's own unbounded timeout, and +// every fetch site sitting behind a request-scoped context was paying for a +// deadline the resolution never saw. +func (t *ssrfSafeTransport) resolveHost(ctx context.Context, host string) ([]net.IP, error) { if t.lookupIP != nil { - return t.lookupIP(host) + return t.lookupIP(ctx, host) + } + + addrs, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, err + } + + // The IPv6 zone on each answer is DISCARDED, and this is a limitation rather + // than a conversion that loses nothing. Nothing regresses today, because the + // dialler rebuilds its destination with net.JoinHostPort(ip.String(), port) + // and that spelling carries no zone either — so a zoned address was never + // dialled as one. What stays broken is the operator running with the private + // hatch open who points this client at a link-local address needing its + // interface (fe80::1%en0): the zone is gone by the time anything tries to + // connect. Carrying it would mean threading net.IPAddr through the vetted + // addresses and the dial, which no caller has asked for. + ips := make([]net.IP, 0, len(addrs)) + for _, addr := range addrs { + ips = append(ips, addr.IP) } - return net.LookupIP(host) + return ips, nil } // reservedNetworks are the ranges, in BOTH families, that no stdlib predicate -// names. +// names. The default is fail-closed for IANA special-purpose space that is not +// globally reachable: local routing can give nominally unroutable ranges a +// meaning, and an attacker choosing a destination gets to exploit that local +// meaning even when the public Internet does not route the prefix. // // Parsed ONCE, at package scope, because isPrivateIP runs on the per-request hot // path — every resolved address of every outbound call walks this list. @@ -45,11 +196,34 @@ func (t *ssrfSafeTransport) resolveHost(host string) ([]net.IP, error) { // mesh — Tailscale hands out addresses from this block. // - 192.0.0.0/24 is reserved for protocol machinery (DS-Lite's 192.0.0.0/29 // among it), never a destination a caller legitimately asks for. +// - 192.88.99.0/24 is the 6to4 anycast relay, the other half of a mechanism +// whose destination side 2002::/16 already refuses below. Banning one and +// not the other is half a decision: this /24 is the address a host sends +// 6to4 traffic TO, and nothing else answers on it — RFC 7526 deprecated it, +// IANA marks the prefix deprecated, and operators were advised to stop +// originating the route, so there is no legitimate destination left inside +// it. (RFC 7526 does NOT itself withdraw the route, and an earlier draft of +// this comment said it did. Deprecation is not withdrawal; §6 only asks +// current operators to "consider carefully whether the anycast relay can be +// discontinued".) Note which way round the RFC cuts here, +// because it is the reverse of the 2002::/16 case: RFC 7526 deprecates THIS +// prefix by name, so the entry below is the RFC's call, while banning +// 2002::/16 is ours — see embeddedIPv4, which says so at length. // - 198.18.0.0/15 is the benchmarking range, routed internally where it is // routed at all. +// - The three TEST-NET blocks are documentation space. They are not globally +// reachable, but local labs and overlays do route them; caller-supplied URLs +// have no legitimate reason to depend on such deployment-specific routes. // - 240.0.0.0/4 is former class E, and it carries 255.255.255.255 with it: // the all-hosts broadcast, which the stack handles unlike a unicast // destination. +// - 64:ff9b:1::/48 is RFC 8215 local-use translation space. Without an +// explicitly configured Pref64 the IPv4 payload position is unknowable, so +// accepting an unrecognised layout is a NAT64 bypass. The well-known +// globally reachable 64:ff9b::/96 remains decoded in embeddedIPv4. +// - IPv6 discard, dummy, benchmarking, documentation and SRv6 SID ranges are +// non-global and may acquire local routing semantics. Teredo and deprecated +// ORCHID are blocked for the same fail-closed reason. // - 2002::/16 is 6to4, banned outright rather than decoded. Its embedded IPv4 // names the tunnel's gateway, not where the packet ends up — see // embeddedIPv4, which explains why this one prefix is the exception to @@ -67,12 +241,41 @@ var reservedNetworks = []*net.IPNet{ mustParseCIDR("0.0.0.0/8"), mustParseCIDR("100.64.0.0/10"), mustParseCIDR("192.0.0.0/24"), + mustParseCIDR("192.0.2.0/24"), + mustParseCIDR("192.88.99.0/24"), mustParseCIDR("198.18.0.0/15"), + mustParseCIDR("198.51.100.0/24"), + mustParseCIDR("203.0.113.0/24"), mustParseCIDR("240.0.0.0/4"), + mustParseCIDR("64:ff9b:1::/48"), + mustParseCIDR("100::/64"), + mustParseCIDR("100:0:0:1::/64"), + mustParseCIDR("2001::/32"), + mustParseCIDR("2001:2::/48"), + mustParseCIDR("2001:10::/28"), + mustParseCIDR("2001:db8::/32"), mustParseCIDR("2002::/16"), + mustParseCIDR("3fff::/20"), + mustParseCIDR("5f00::/16"), mustParseCIDR("fec0::/10"), } +// 2001::/23 is IANA's IPv6 IETF Protocol Assignments parent reservation. The +// parent is non-global except for the more-specific assignments below, so a +// flat denylist cannot represent it without either failing open on unallocated +// protocol space or blackholing the globally reachable exceptions. +var ietfProtocolAssignments = mustParseCIDR("2001::/23") + +var globallyReachableIETFAssignments = []*net.IPNet{ + mustParseCIDR("2001:1::1/128"), // PCP anycast + mustParseCIDR("2001:1::2/128"), // TURN anycast + mustParseCIDR("2001:1::3/128"), // DNS-SD registration anycast + mustParseCIDR("2001:3::/32"), // AMT + mustParseCIDR("2001:4:112::/48"), + mustParseCIDR("2001:20::/28"), // ORCHIDv2 + mustParseCIDR("2001:30::/28"), // Drone Remote ID DETs +} + // mustParseCIDR panics on a malformed prefix. Its arguments are compile-time // constants in this file, so a failure is a typo caught at startup rather than a // range that silently stops being checked. @@ -85,7 +288,7 @@ func mustParseCIDR(cidr string) *net.IPNet { } // isPrivateIP reports whether an address reaches this host, the operator's own -// network, or something the kernel treats specially. +// network, special-purpose space, or a range that is not globally reachable. // // THE DEFAULT IS THE DANGEROUS DIRECTION. Anything this predicate does not // recognise is treated as public and dialled, and the address space holds far @@ -142,6 +345,19 @@ func isPrivateIP(ip net.IP) bool { } } + // IANA marks the 2001::/23 parent non-global except for a small set of + // explicit assignments. Preserve those public services and fail closed on + // every other address under the parent, including future-looking or locally + // routed space that a static list of today's named children would miss. + if ietfProtocolAssignments.Contains(ip) { + for _, network := range globallyReachableIETFAssignments { + if network.Contains(ip) { + return false + } + } + return true + } + // Last, because everything above answers "which block is this address in" and // some IPv6 forms defeat that question rather than answering it wrongly: the // destination is a four-byte field carried INSIDE the address. @@ -174,8 +390,6 @@ func isPrivateIP(ip net.IP) bool { // - NAT64 well-known prefix, 64:ff9b::/96 (RFC 6052) — IPv4 in the last four // bytes. Purpose-built to mean "this IPv4 host", so a translator on the path // delivers it exactly there. -// - NAT64 local-use prefix, 64:ff9b:1::/48 (RFC 8215) — the same mechanism -// with an operator-chosen prefix, and the same consequence if banned. // - SIIT IPv4-translated, ::ffff:0:0:0/96 (RFC 6052 §2.2) — IPv4 in the last // four bytes, and likewise the destination itself. // - IPv4-compatible IPv6, ::/96 — deprecated. It slips past every range check @@ -205,11 +419,6 @@ func isPrivateIP(ip net.IP) bool { // disabled by default"), 2002::/16's place on the standard bogon lists, and // 6to4 rounding to 0.00% of Google's measured IPv6 traffic. // -// - Teredo — needs a Teredo tunnel on the host. If it is ever added the prefix -// is 2001:0000::/32 and NEVER 2001::/16, which is sixteen bits too wide: -// live atProto PDSes sit at 2001:19f0:7002:191:: and 2001:550:5a00:785b::1, -// and a /16 rule blocks both. They are pinned as allowed rows in the tests. -// // - ISATAP — its 0000:5efe:V4ADDR interface identifier can appear under ANY // unicast /64, so matching it means reading the low bytes of every IPv6 // address rather than recognising a prefix, and it reaches nothing without @@ -239,17 +448,6 @@ func embeddedIPv4(ip net.IP) net.IP { return net.IPv4(ip16[12], ip16[13], ip16[14], ip16[15]) } - // NAT64, local-use prefix: the same four bytes, then 0001, then zeroes. - // - // Only the /96-suffix shape is read. RFC 8215 hands the operator a /48 and - // RFC 6052 then puts the IPv4 at an offset that depends on the prefix length - // they actually deployed, so a 64:ff9b:1:: address with a non-zero middle is - // one whose payload offset this code cannot know — and guessing wrong would - // invent a destination rather than find one. - if hasBytePrefix(ip16, 0x00, 0x64, 0xff, 0x9b, 0x00, 0x01) && isAllZero(ip16[6:12]) { - return net.IPv4(ip16[12], ip16[13], ip16[14], ip16[15]) - } - // SIIT IPv4-translated: eight zero bytes, ffff, two more zero bytes. // // A BYTE PATTERN AND NOT A CIDR ENTRY, deliberately, because the CIDR that @@ -302,6 +500,67 @@ func isAllZero(b []byte) bool { return true } +// asciiHost returns the hostname in the form net/http will actually use, which +// is the only form worth vetting. +// +// # WHY THE GUARD CANNOT SKIP THIS +// +// net/http punycodes a URL's host before that string becomes a dial address, a +// connection-pool key or a TLS ServerName: canonicalAddr calls +// idnaASCIIFromURL, which calls idnaASCII, which is the two lines below in the +// same order (net/http/transport.go and net/http/request.go). A guard that +// resolves req.URL.Hostname() raw is therefore asking about a string nothing +// downstream will ever use. +// +// For an IDN host that is not a subtle mismatch, it is an outage. The +// production build is CGO_ENABLED=0, so the pure-Go resolver answers, and it +// gates on isDomainName — which permits only [A-Za-z0-9._-] and drops any byte +// at or above 0x80 into its default case. So `bücher.example` comes back "no +// such host" without a packet being sent, and every atProto PDS on a non-ASCII +// domain is unreachable through this client at every site that adopted it. It +// fails closed, which is why nothing noticed: no fixture in this tree uses an +// IDN host. +// +// # THE ASCII SHORT-CIRCUIT IS NOT AN OPTIMISATION +// +// It is the half of this function that prevents a worse regression than the one +// it fixes, and net/http has it for the same reason. +// +// idna.Lookup is not a punycode encoder. The profile applies ValidateLabels, +// CheckHyphens and the BidiRule, so it REFUSES ASCII hostnames that Go's own +// resolver resolves and that this AppView reaches today — `_atproto.example.com` +// (UTS#46 disallows U+005F; net.isDomainName permits it deliberately, citing +// SRV-style underscore labels) and `aa--bb.example.com` (hyphens in the third +// and fourth positions, which is ordinary RFC-valid DNS). Running every host +// through ToASCII would trade "IDN hosts are unreachable" for "IDN hosts and a +// family of ASCII hosts are unreachable". +// +// Byte-wise rather than rune-wise, matching net/http's ascii.Is exactly: a +// hostname that is all ASCII is returned untouched, so the ASCII path through +// this transport is byte-identical to what it was before normalization existed. +// That includes case — ToASCII would lowercase `PDS.Example.COM`, and not +// lowercasing it is what keeps the name this guard vets equal to the name +// net/http computes for the pool key. +func asciiHost(host string) (string, error) { + if isASCII(host) { + return host, nil + } + return idna.Lookup.ToASCII(host) +} + +// isASCII reports whether every byte of s is ASCII, which is net/http's +// ascii.Is under a different name — the comparison is against unicode.MaxASCII +// there too, and matching it byte for byte is the point rather than an +// incidental resemblance. +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > unicode.MaxASCII { + return false + } + } + return true +} + // vettedAddrsKeyType keys the addresses RoundTrip approved, so the dialler can // read them off the request's own context. A private type, so nothing outside // this file can plant a value under the same key. @@ -329,10 +588,136 @@ var vettedAddrsKey vettedAddrsKeyType func (t *ssrfSafeTransport) RoundTrip(req *http.Request) (*http.Response, error) { host := req.URL.Hostname() - // A literal address is already the thing that will be dialled, so there is - // no second resolution to defend against — but it still has to pass the - // private check below. - ips, err := t.resolveHost(host) + // THE REQUEST BODY IS CLOSED ON EVERY RETURN, which net/http requires in + // those words: "RoundTrip must always close the body, including on errors". + // http.Client is written against that promise and does not close the body + // itself, so a refusal that returns early leaks whatever the body was + // holding — a file handle, a pipe, a buffer with a finalizer — once per + // refused request. The refusals below are the path a hostile input takes, + // so the leak scales with how well the guard is working. + // + // OWNERSHIP TRANSFERS EXACTLY ONCE, which is why this is a flag rather than + // an unconditional defer. The base transport takes the body over when it is + // called and closes it itself, including on its own errors, so the flag is + // cleared the moment it returns — before the declared-length refusal below, + // which would otherwise close a SECOND time. Once is the contract in both + // directions: a body whose Close is not idempotent reports an error the + // second time. + ownsBody := req.Body != nil + defer func() { + if ownsBody { + _ = req.Body.Close() + } + }() + + // THE HOSTNAME BECOMES ITS A-LABEL BEFORE ANYTHING ELSE LOOKS AT IT. + // + // asciiHost says why the translation has to happen at all. This comment is + // about WHERE it happens, which is a security question and not a tidiness + // one. + // + // BEFORE THE LITERAL CHECK BELOW, because IDNA mapping can PRODUCE an IP + // literal. The Lookup profile maps before it validates, and its mapping + // table folds fullwidth digits (U+FF10..U+FF19) and the ideographic full + // stop (U+3002) onto their ASCII equivalents — so `１２７。０。０。１` is not a + // literal on the way in and is exactly "127.0.0.1" on the way out, verified + // by probe. Normalize after the shape check and the check inspects a string + // that is not yet the address it is about to become. + // + // The row where that has consequences is a PUBLIC one. For the loopback + // spelling, classification is a backstop: the mapped literal resolves to + // itself and is refused a few lines further down, at the wrong layer but + // refused. `８.８.８.８` has no backstop — it maps to a public address that + // classification cannot refuse, and the literal check is the only control + // standing between a caller-supplied address and a destination this AppView + // has no business reaching. + // + // AFTER THE BODY DEFER ABOVE, because a refusal here is a return, and + // RoundTrip must close the request body on every one of them. + // + // NOT GATED ON THE HATCH, unlike both refusals below it, and the reason is + // the same one PrivateAddressOptions exists for: `.env.ci:140` sets + // IS_DEV_ENV=true, so the merge gate runs every call site with allowPrivate + // open. A translation that only happened on the guarded path would be a + // translation CI never exercises in the form production runs. It is also not + // a gate — it decides what gets vetted, never whether vetting happens, which + // is the same division of labour WithHostResolver is safe to export under. + // + // REFUSED RATHER THAN FALLEN BACK ON, which is the one place this diverges + // from net/http on purpose. idnaASCIIFromURL swallows the error and keeps + // the raw host, which is right for a transport whose next step is a dial + // that will simply fail; it is wrong for a guard, whose next step is to make + // a decision about a name. Vetting a string that nothing will dial is how a + // guard approves one host and connects to another. + // + // The refusal is written here rather than through blockedDial: that + // constructor names the DIAL path, and this fires a layer earlier. It is not + // a BlockedAddressError either, because that type's contract is a host AND + // the address it resolved to, and nothing has been resolved — there is no + // address to carry, and its two renderings are both untrue of this case. + // ErrBlockedAddress is the identity a caller matches on, and it is here. + normalized, err := asciiHost(host) + if err != nil { + return nil, fmt.Errorf("%w: %s is not a hostname this transport can resolve: %w", + ErrBlockedAddress, host, err) + } + host = normalized + + // AN ADDRESS WRITTEN WHERE A HOSTNAME BELONGS IS REFUSED OUTRIGHT, before + // anything is resolved. + // + // There is no traffic to lose. A legitimate atProto endpoint is always a + // name: the handle specification forbids IP literals, and a DID document's + // serviceEndpoint is an HTTPS URL with a hostname. So refusing the shape is + // both cheaper and more total than classifying whatever it points at — and + // classification is what cannot help here, since a caller-supplied literal + // may well name a PUBLIC address and pass the check below on its way to a + // destination this AppView has no business reaching. + // + // Before resolution rather than after, because resolving a literal asks a + // question the URL already answered, and the answer would come back from a + // resolver an attacker may influence. + // + // GATED ON THE HATCH, which is not a softening — it is what the hatch means. + // allowPrivate says "this client is pointed at a developer-chosen address", + // and every integration fixture in this tree is served from an httptest + // listener, which is to say from a loopback literal: + // internal/core/blobs/fetch_guard_test.go and + // internal/core/blueskypost/service_test.go both drive THIS client at + // 127.0.0.1:PORT with the hatch open. An ungated check would not merely fail + // those suites, it would leave a dev environment unable to reach anything + // local. + // + // netip.ParseAddr AND NOT net.ParseIP, because net.ParseIP returns nil for + // any address carrying a ZONE — the `%eth0` in `fe80::1%eth0`, naming the + // interface the address is scoped to. url.Parse does understand the form: + // `http://[2600::1%25eth0]/` yields a Hostname of `2600::1%eth0`, + // so a zoned literal was refused as "not a literal", handed to the resolver, + // resolved locally with no DNS involved, its zone silently discarded, and + // dialled. netip.ParseAddr accepts zones and every spelling ParseIP accepts, + // so the switch closes the hole without narrowing anything. + // + // WHAT THIS COVERS: the dotted-quad and bracketed-IPv6 spellings, uppercase + // hex, IPv4-mapped and now zoned forms, which is what the tests pin. + // + // WHAT THIS DOES NOT DO: it does not close the obfuscated encodings. BOTH + // parsers refuse 0x7f.0.0.1, 2130706433, 127.1 and "127.0.0.1." — verified, + // not assumed — so all four are not literals as far as this check is + // concerned and reach the resolver. Resolver implementations differ on + // whether they reject those spellings or normalize them to an address; any + // returned address is still classified before it can be dialled. + if !t.allowPrivate { + if literal, err := netip.ParseAddr(host); err == nil { + // AsSlice, because BlockedAddressError.IP is a net.IP and the + // diagnostic is the whole reason that field exists. The zone is not + // carried across — netip drops it here, as resolveHost drops it + // there — and the address is what an operator reading the block + // needs. + return nil, &BlockedAddressError{Host: host, IP: literal.AsSlice(), literal: true} + } + } + + ips, err := t.resolveHost(req.Context(), host) if err != nil { return nil, fmt.Errorf("failed to resolve host: %w", err) } @@ -344,23 +729,436 @@ func (t *ssrfSafeTransport) RoundTrip(req *http.Request) (*http.Response, error) if !t.allowPrivate { for _, ip := range ips { if isPrivateIP(ip) { - return nil, fmt.Errorf("SSRF blocked: %s resolves to private IP %s", host, ip) + return nil, &BlockedAddressError{Host: host, IP: ip} } } } - return t.base.RoundTrip(req.WithContext(context.WithValue(req.Context(), vettedAddrsKey, ips))) + // coves:allow-bare-client: this IS the guard handing off to its base transport, on the far side of the classification above + resp, err := t.base.RoundTrip(req.WithContext(context.WithValue(req.Context(), vettedAddrsKey, ips))) + + // The handover, and it is cleared on BOTH outcomes: the base transport's own + // contract is the one quoted above, so whether it answered or failed it has + // already closed the body. Anything after this line must not close it again. + ownsBody = false + + if err != nil { + return nil, err + } + + // AN ANNOUNCED length over the cap is refused without reading the body, and + // the response is closed because a refused response still holds a connection. + // + // This is an OPTIMISATION AND NOT THE CONTROL. The header is chosen by the + // same party as the body, so it is a hint at best — and against compression + // it is not even that: http.Transport sends Accept-Encoding: gzip on its own, + // transparently decompresses the reply, and DELETES Content-Length while + // setting ContentLength to -1 when it does. So anyone wanting past this check + // need only enable compression on their server. Do not read the wrapper below + // as redundant with this branch; it is the other way round. + // + // A NEGATIVE LENGTH IS "UNKNOWN", NOT SUSPICIOUS. -1 is what every chunked + // response reports and what every transparently decompressed one reports, so + // refusing on it would refuse a large share of ordinary traffic. Unknown + // means "rely on the wrapper", which is why the comparison is > and not !=. + if resp.ContentLength > t.maxResponseBytes { + _ = resp.Body.Close() + return nil, fmt.Errorf("%w: %d bytes declared, %d allowed", + ErrResponseTooLarge, resp.ContentLength, t.maxResponseBytes) + } + + // The control proper. Wrapping the body the base transport RETURNS is what + // makes the unit DECOMPRESSED bytes: by this point resp.Body is the gzip + // reader's output, so the count is of what io.ReadAll will actually allocate. + // A cap on bytes off the wire would let a thousand-to-one bomb deliver + // gigabytes through a limit it never appeared to exceed. + // + // PER-HOP, NOT CUMULATIVE across a redirect chain, because RoundTrip runs + // once per hop. That is adequate rather than a compromise: http.Client drains + // a redirect body at roughly 2 KB before following it. + resp.Body = newCappedBody(resp.Body, t.maxResponseBytes) + return resp, nil +} + +// newCappedBody wraps body with an allowance that Read can survive. +// +// THE CLAMP LIVES HERE AND IN THE CONSTRUCTOR, at the two boundaries where a +// number becomes an allowance, rather than being re-checked on the per-read hot +// path. What it buys is that `remaining` is positive by construction, so the +// arithmetic below never has to reason about a negative slice bound. +func newCappedBody(body io.ReadCloser, limit int64) *cappedBody { + return &cappedBody{body: body, remaining: clampResponseCap(limit)} +} + +// cappedBody fails the read once the body it wraps has delivered more than +// remaining bytes. +type cappedBody struct { + body io.ReadCloser + remaining int64 + exceeded bool +} + +// Read delivers at most the remaining allowance and then fails. +// +// The mechanism is to READ ONE BYTE PAST THE ALLOWANCE and treat that byte's +// existence as the proof, which is what puts the boundary in the right place: a +// body of exactly the cap yields EOF on the extra read and arrives complete, +// while one byte more is seen and refused. Measuring after the fact — the +// obvious alternative — gets the same boundary and misses the point, since by +// then the whole body is in memory and the cap has protected nothing. +func (b *cappedBody) Read(p []byte) (int, error) { + // Sticky, because the alternative is the truncation this cap exists to + // avoid: a caller that reads again after the failure would otherwise be told + // EOF once the underlying body ran out, and would take the short body it + // already holds for a complete one. + if b.exceeded { + return 0, ErrResponseTooLarge + } + + // The probe is sized under the comparison rather than as `remaining+1` + // computed up front, and that ordering is the whole defence against + // overflow: inside this branch remaining is BELOW len(p), which is an int, + // so remaining+1 cannot exceed MaxInt and cannot wrap. The obvious spelling + // wraps math.MaxInt64 to math.MinInt64 and panics on the slice bound — and + // MaxInt64 is what someone writes for "no limit". + if b.remaining < int64(len(p)) { + probe := b.remaining + 1 + if probe < 0 { + // Unreachable through newCappedBody, which clamps. It stands + // between a hand-built wrapper and a negative slice bound, since a + // panic here takes down whatever goroutine is reading a body. + probe = 0 + } + p = p[:probe] + } + + n, err := b.body.Read(p) + if int64(n) > b.remaining { + // max(0, remaining) and not remaining, because io.Reader FORBIDS a + // negative count and the standard library does not defend against one: + // bytes.Buffer.ReadFrom panics with "reader returned negative count + // from Read". Unreachable through newCappedBody for the same reason as + // above; kept because the consequence of being wrong is a panic in a + // caller doing the most ordinary thing there is with a response body. + delivered := b.remaining + if delivered < 0 { + delivered = 0 + } + + // BOTH FIELDS, because they describe one state. This body will never + // deliver another byte, so an allowance left positive is a count of + // bytes that can never be spent — meaningless today only because Read + // consults exceeded first, which is a read-order coincidence and not an + // invariant. Zeroing it makes (exceeded, remaining>0) unrepresentable + // rather than merely unreached. + b.exceeded = true + b.remaining = 0 + + return int(delivered), ErrResponseTooLarge + } + b.remaining -= int64(n) + return n, err +} + +// Close closes the body underneath. +// +// DELEGATING IS LOAD-BEARING. http.Transport returns a connection to the idle +// pool when its body is closed, so a wrapper that implements Read and forgets +// Close leaks one connection per request while passing every functional test — +// bodies still arrive, caps still fire, and MaxIdleConns describes a pool +// nothing is ever returned to. +func (b *cappedBody) Close() error { + return b.body.Close() +} + +// DefaultMaxResponseBytes is the response-body cap a client gets when no option +// tightens it: 32 MiB, chosen to sit above every caller in this tree (the +// largest is 10 MB) so that adopting the shared client changes no existing +// behavior, while still bounding what a remote host can make this process +// allocate. +// +// A CALLER WITH ITS OWN, LARGER LIMIT MUST RAISE THIS EXPLICITLY. The trap is +// the image proxy: IMAGE_PROXY_MAX_SOURCE_SIZE_MB is operator-configurable, so +// an operator who sets it above 32 would find their own setting silently +// clamped by a constant in another package. Whoever wires that call site onto +// this client has to pass WithMaxResponseBytes from the configured value, the +// way blobs.NewBlobService raises the client timeout back to 30s rather than +// living with the shared default. +const DefaultMaxResponseBytes = 32 << 20 + +// Option configures the transport behind NewSSRFSafeHTTPClient. +type Option func(*ssrfSafeTransport) + +// WithMaxResponseBytes caps how many bytes of a response body a caller can read +// before the read fails with ErrResponseTooLarge, replacing +// DefaultMaxResponseBytes. +// +// A cap of zero or less is not honoured; see clampResponseCap. +func WithMaxResponseBytes(n int64) Option { + return func(t *ssrfSafeTransport) { + t.maxResponseBytes = n + } +} + +// WithMaxIdleConnsPerHost sets how many idle connections the transport keeps +// per destination host, replacing net/http's DefaultMaxIdleConnsPerHost of 2. +// +// IT EXISTS SO A CALL SITE WITH ITS OWN POOL SETTINGS DOES NOT LOSE THEM WHEN +// IT ADOPTS THIS CLIENT. The base transport below already carries the +// MaxIdleConns and IdleConnTimeout every caller in this tree had set, so this +// was the one pool setting a conversion silently changed — the community +// consumer's .well-known fetch ran on 10 and would have dropped to 2 without +// anything failing, which is a throughput regression arriving as part of an SSRF +// fix and attributable to nothing. +// +// A value of zero or less is ignored rather than installed, so a caller that +// passes an unset config field gets net/http's default instead of a transport +// that pools nothing. +func WithMaxIdleConnsPerHost(n int) Option { + return func(t *ssrfSafeTransport) { + if n <= 0 { + return + } + t.base.MaxIdleConnsPerHost = n + } +} + +// WithHostResolver replaces the transport's name lookup. +// +// IT IS A SEAM FOR CALLERS' OWN TESTS, and it exists because there is no +// hermetic way to make a hostname answer with a chosen address: the hermetic +// tiers block egress, and nothing in this tree can write /etc/hosts. Without +// it, a package that wires this client can only prove its guard by naming an IP +// literal — which the guard refuses on SHAPE, one branch earlier than the +// classification most call sites actually depend on. The aggregator's +// registration handler is the case that forced this: its domain validator +// already refuses every IP literal, so a literal-based test there proves +// nothing about the transport at all. +// +// IT CANNOT OPEN THE GUARD, and that is what makes exporting it safe. Whatever +// this function answers is classified by exactly the same pass a real DNS answer +// goes through, and the dial still goes only to addresses that survived it. The +// seam chooses what gets classified; it has no say in whether classification +// happens. A caller that wanted to reach a private address would use +// WithPrivateAddressesAllowed, which says so in its name. +// +// A nil lookup is ignored rather than installed, so a caller that passes one by +// accident gets the real resolver instead of a client that panics on its first +// request. +func WithHostResolver(lookup func(ctx context.Context, host string) ([]net.IP, error)) Option { // coves:allow-dns-seam: the seam's own declaration; a production CALL is what the rule is for + return func(t *ssrfSafeTransport) { + if lookup == nil { + return + } + t.lookupIP = lookup + } +} + +// WithPrivateAddressesAllowed disables the address guard entirely, for a client +// a developer has deliberately pointed at their own machine. +// +// IT EXISTS TO BE READ AT THE CALL SITE. The byte ceiling already had a +// self-documenting name while the setting that turns the guard OFF was an +// unlabelled positional boolean, which is exactly backwards — the more dangerous +// switch was the one wearing no label. The old spelling — the constructor called +// with a bare `true` — said nothing: a reader had to open this file to learn that +// the argument was the difference between a guarded client and an unguarded one. +// +// ONE NAME FOR SEVERAL DECISIONS is what makes a named option worth more here +// than at any other setting on this transport. allowPrivate is read in two +// separate places in RoundTrip — the literal refusal, which covers the dotted- +// quad, bracketed-IPv6 and zoned spellings in one netip.ParseAddr, and the +// classification pass over the resolved answers — so the boolean is one token +// standing for gates whose spelling names none of them. The option names the +// state they share: this client has the hatch open. +// +// It is also the thing a regression fence can find. `true` is not greppable; +// this identifier is, which is how "which call sites disable the guard" stays an +// answerable question as call sites are added. +// +// ONE-WAY, so it composes: it only ever opens the hatch, and no unrelated option +// can close a hatch this one opened, whatever order the options run in. A client +// built with NO options stays guarded because the constructor's struct literal +// does not set allowPrivate at all and its zero value is false — unlike the byte +// cap, which needs an explicit default written there for the same reason: what a +// caller gets by omission has to be the safe value. +func WithPrivateAddressesAllowed() Option { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(t *ssrfSafeTransport) { t.allowPrivate = true } +} + +// PrivateAddressOptions returns the options a caller holding an allow-private +// boolean should pass to NewSSRFSafeHTTPClient: the hatch option when the +// boolean is set, and NOTHING when it is not. +// +// # WHY THIS IS A FUNCTION AND NOT AN `if` IN WIRING +// +// It looks like a conditional worth inlining, and inlining it would delete the +// only test coverage the production branch has. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// running T0+T1+T2 — takes the PERMISSIVE branch at every call site that holds +// such a boolean. A green merge gate therefore says nothing whatsoever about +// whether production is guarded: the guarded branch is evaluated in exactly one +// place in this repository, and that place is a unit test against this function. +// +// An inline `if cfg.IsDevEnv { ... }` in wiring is reachable only by standing up +// that wiring with a production config, and nothing in this tree does that. As a +// pure function the decision is testable without wiring, without a config and +// without an environment, which is the only reason the branch production +// actually runs is tested at all. Do not inline it back. +// +// # FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT +// +// Not "options that are safe" — no options. What a guarded caller gets is +// exactly the constructor's own struct literal, untouched, which is a claim a +// reader can check in one glance rather than one that has to be re-argued +// against whatever the slice holds this month. +// +// So an edit that appends a diagnostic option here, or that returns a +// one-element slice holding a no-op "explicitly deny" closure, is a breaking +// change even though it changes no behaviour: it moves the branch CI never runs +// from "provably applies nothing" to "applies something believed harmless". +// TestPrivateAddressOptions_ReturnsZeroOptionsWhenPrivateAddressesAreDisallowed +// asserts the exact length and will fail on both. That is deliberate; the answer +// is not to relax it. +// +// The slice is built fresh per call rather than shared from a package-level var, +// because callers append to it — the image proxy passes its operator-configured +// WithMaxResponseBytes alongside — and a shared backing array would let two call +// sites write over each other's options. +func PrivateAddressOptions(allowPrivate bool) []Option { + if !allowPrivate { + return nil + } + return []Option{WithPrivateAddressesAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + +// joinDialErrors turns the per-address dial failures into the ONE error the +// dial returns, and it owns both branches so that "what a caller can learn from +// a failed dial" is decided in a single place. +// +// # ONE ERROR IS RETURNED BARE +// +// Joining it would be aggregation with nothing to aggregate, and it costs the +// timeout signal for the reason spelled out at the call site. It also puts one +// more wrapper between a caller and the concrete type it may be matching on, +// for no information gained. +// +// # THE AGGREGATE HAS TO ANSWER THE SAME QUESTIONS ITS MEMBERS DO +// +// errors.Join alone returns an *errors.joinError, which implements Unwrap() +// []error and NOTHING ELSE — so the bare-return fix above preserved Timeout() +// for a host with one address and left it severed for a host with two. An +// ordinary dual-stack host has an A record and an AAAA record, so the +// aggregation branch is the COMMON case, not the corner one, and a caller that +// retries on timeouts stopped seeing them exactly there. +// +// BOTH METHODS OR NEITHER. url.Error.Timeout needs only `interface{ Timeout() +// bool }`, but `if ne, ok := err.(net.Error); ok && ne.Timeout()` — the shape +// retry and circuit-breaker logic is actually written in — needs Temporary() +// too, and an assertion to net.Error fails outright without it. Implementing +// half the interface would fix half the callers and look like it had fixed all +// of them. +// +// # ALL, NOT ANY +// +// The aggregate reports a property only when EVERY member reports it, because +// the caller's question is about the destination as a whole. A host whose IPv6 +// address timed out while its IPv4 address was REFUSED has given a definite +// answer on one of them; calling that a timeout tells a retry loop to keep +// waiting on something that already said no. A member that is not a net.Error +// at all answers false for the same reason — an aggregate cannot claim a +// property of a member that cannot report it. +// +// Inside, membership is tested with errors.As rather than a direct assertion, +// and that asymmetry with our own callers is deliberate: what the dialer hands +// back is a *net.OpError wrapping the real cause, so walking the chain is how +// the members are read honestly. Our callers cannot do that — the stdlib +// asserts directly — which is precisely why this type exists to answer for +// them. +func joinDialErrors(errs []error) error { + if len(errs) == 1 { + return errs[0] + } + return &dialAggregateError{errs: errs, joined: errors.Join(errs...)} +} + +// dialAggregateError is errors.Join's aggregation with net.Error's answers. +type dialAggregateError struct { + errs []error + + // joined carries the message and nothing else. Reusing errors.Join for the + // text keeps the operator-facing output byte-identical to what the plain + // join produced, so this change adds a capability without editing a string + // anyone may be reading in a log. + joined error +} + +func (e *dialAggregateError) Error() string { return e.joined.Error() } + +// Unwrap is the multi-error form, which is what lets errors.Is and errors.As +// walk to every per-address failure. Dropping it would keep the message and +// lose the tree. +func (e *dialAggregateError) Unwrap() []error { return e.errs } + +func (e *dialAggregateError) Timeout() bool { + return e.everyMemberSatisfies(net.Error.Timeout) +} + +func (e *dialAggregateError) Temporary() bool { + return e.everyMemberSatisfies(net.Error.Temporary) +} + +// everyMemberSatisfies reports whether every joined error is a net.Error for +// which want holds. An empty aggregate answers false: joinDialErrors never +// builds one, and "vacuously true" is the wrong default for a question a caller +// acts on. +func (e *dialAggregateError) everyMemberSatisfies(want func(net.Error) bool) bool { + if len(e.errs) == 0 { + return false + } + for _, err := range e.errs { + var netErr net.Error + if !errors.As(err, &netErr) || !want(netErr) { + return false + } + } + return true +} + +// clampResponseCap maps a cap that is not a cap onto the default. +// +// ZERO AND NEGATIVE ARE BOTH ARRIVALS, NOT CHOICES. Zero is what an unset +// struct field, a missing config key or an unparsed environment variable +// arrives as, and honouring it refuses every non-empty response there is — +// which reads at the call site as "this remote host is broken", for every host. +// A negative cap is a typo, and honouring it used to make Read return n = -1. +// +// FALLING BACK TO THE DEFAULT RATHER THAN REFUSING THE OPTION, because a cap is +// a safety limit and the failure mode of getting it wrong should be a working +// client with a conservative bound, not a client that refuses everything or a +// constructor that panics on a value a config file supplied. +// +// There is no upper clamp, and none is needed: cappedBody.Read no longer +// computes remaining+1 anywhere it could overflow, so math.MaxInt64 is an +// ordinary — if useless — allowance rather than a panic. Do not add a magic +// upper bound to "fix" that; fix the arithmetic if it ever regresses. +func clampResponseCap(n int64) int64 { + if n <= 0 { + return DefaultMaxResponseBytes + } + return n } // NewSSRFSafeHTTPClient creates an HTTP client with SSRF protections -func NewSSRFSafeHTTPClient(allowPrivate bool) *http.Client { +func NewSSRFSafeHTTPClient(opts ...Option) *http.Client { dialer := &net.Dialer{ Timeout: 10 * time.Second, KeepAlive: 30 * time.Second, } transport := &ssrfSafeTransport{ - base: &http.Transport{ + base: &http.Transport{ // coves:allow-bare-client: the base transport the guard wraps; its DialContext below only reaches addresses RoundTrip vetted // The dial IGNORES the hostname in addr and connects to an address // RoundTrip already approved, which is what closes the // check-then-dial window. It takes only the port from addr, because @@ -379,31 +1177,102 @@ func NewSSRFSafeHTTPClient(allowPrivate bool) *http.Client { DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { vetted, _ := ctx.Value(vettedAddrsKey).([]net.IP) if len(vetted) == 0 { - return nil, fmt.Errorf("SSRF blocked: refusing to dial %s with no vetted address "+ + return nil, blockedDial("refusing to dial %s with no vetted address "+ "(the SSRF-safe transport was bypassed)", addr) } _, port, err := net.SplitHostPort(addr) if err != nil { - return nil, fmt.Errorf("SSRF blocked: cannot read a port from %q: %w", addr, err) + return nil, blockedDial("cannot read a port from %q: %w", addr, err) } - var lastErr error + // EVERY failure is kept, not just the last one. A host with both + // an A and an AAAA record is the ordinary case, and the two + // commonly fail differently — IPv6 unreachable on a v4-only host, + // connection refused on the v4 address. Keeping one symptom sends + // whoever is debugging a federation failure after a single address + // with nothing to say another was tried at all. + // + // The addresses appear in the joined message because the stdlib + // dial error embeds the one it tried, and naming them here is not + // the oracle the classification refusal above avoids: these are + // addresses ALREADY ACCEPTED, reported to an in-process caller, + // rather than an answer to "what does the name you gave me resolve + // to inside this network". + var dialErrs []error for _, ip := range vetted { conn, dialErr := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) if dialErr == nil { return conn, nil } - lastErr = dialErr + dialErrs = append(dialErrs, dialErr) + } + + // The loop's own contract, made total rather than borrowed from + // twenty lines up. errors.Join of nothing is nil, so falling out of + // an attempt-free loop would return (nil, nil) — a connection that + // does not exist and no error explaining it, which is not a result + // net/http is written to survive. + // + // THIS IS LATENT, NOT LIVE: an empty vetted slice cannot reach here + // through the public API, because the guard above refuses it, so + // the loop always runs at least once today. What this closes is the + // edit that moves or weakens that guard without noticing that the + // loop below was depending on it. + if len(dialErrs) == 0 { + return nil, blockedDial("no vetted address was attempted for %s", addr) } - return nil, lastErr + + // ONE ERROR IS RETURNED BARE. Joining it would be aggregation + // with nothing to aggregate, and it costs the timeout signal: + // errors.Join returns a *errors.joinError, which implements + // Unwrap() []error and nothing else, while url.Error.Timeout — + // the method every caller of this client reaches, since + // http.Client wraps every RoundTrip error in a *url.Error — is a + // DIRECT TYPE ASSERTION rather than errors.As. Same for the + // `if ne, ok := err.(net.Error); ok && ne.Timeout()` that retry + // and circuit-breaker logic is built on. So a join over a lone + // dial failure leaves the information in the chain and out of + // reach, and a caller that cannot tell a timeout from a refusal + // either retries what it should not or gives up on what it + // should. + // + // The join stays for the genuine multi-address case, which is + // what it was added for: a host with both an A and an AAAA + // record commonly fails differently on each, and an operator + // needs both symptoms. joinDialErrors owns both branches. + return nil, joinDialErrors(dialErrs) }, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, }, - allowPrivate: allowPrivate, + // allowPrivate is NOT set here, and its zero value is the whole point: + // a client built with no options is GUARDED. The hatch is reachable only + // through WithPrivateAddressesAllowed, so "this client can reach private + // addresses" is a phrase that appears at the call site or nowhere. + // + // This replaced a positional boolean. `NewSSRFSafeHTTPClient(true)` said + // nothing at a call site about which of the guard's three refusals it was + // switching off, and a reader had to open this file to learn that the + // argument was the difference between a guarded client and an unguarded + // one. + + // Before the options, so a client built without one is capped at the + // default rather than at a zero value — which would refuse every + // response there is. + maxResponseBytes: DefaultMaxResponseBytes, } - return &http.Client{ + for _, opt := range opts { + opt(transport) + } + + // AFTER the options, so a cap outside the usable range is corrected wherever + // it came from. It also keeps the ContentLength comparison in RoundTrip + // sane: that branch reads this field directly, so a zero here would refuse + // every response that declared a length before a byte was read. + transport.maxResponseBytes = clampResponseCap(transport.maxResponseBytes) + + return &http.Client{ // coves:allow-bare-client: this IS the guarded client being constructed; the ssrfSafeTransport below is what makes it safe Timeout: 15 * time.Second, Transport: transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { diff --git a/internal/atproto/oauth/transport_blocked_error_test.go b/internal/atproto/oauth/transport_blocked_error_test.go new file mode 100644 index 0000000..1f37646 --- /dev/null +++ b/internal/atproto/oauth/transport_blocked_error_test.go @@ -0,0 +1,182 @@ +package oauth + +import ( + "encoding/json" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeHTTPClient_RefusalIsTypedAndDoesNotDiscloseTheResolvedAddress +// pins the vocabulary of a refusal: matchable by type, detailed for the +// operator, silent about the address to everyone else. +// +// # WHY THE RESOLVED IP MUST LEAVE THE MESSAGE +// +// Before this change the refusal rendered as "SSRF blocked: %s resolves to +// private IP %s" — this test is what removed the second verb, so the sentence +// below describes what WAS wrong, not what the code does now. +// The second verb is an internal-network oracle. The host in that sentence is +// the attacker's own input — a DID document's serviceEndpoint, a community +// record's domain, a name whose zone they control — so they can ask about any +// name they like and read back which address it resolved to INSIDE our network. +// That turns a refusal into a mapping primitive: point a name at a candidate, +// read the answer out of the error, repeat. Error strings travel further than +// the code that writes them — into an HTTP response body, a shared log, a +// support ticket — and every one of the nine fetch sites about to use this +// client formats errors somewhere. +// +// # WHY errors.Is RATHER THAN A SUBSTRING +// +// The callers being wired up need to tell "the guard refused this" from "the +// network failed", because the two get different HTTP statuses and different +// log levels. Substring matching on a message makes the message an API: it +// cannot be reworded, and a caller that misspells the substring fails open with +// no signal. A sentinel is checkable by the compiler's users and reworded +// freely. +// +// # WHAT IS DELIBERATELY NOT ASSERTED +// +// That the HOSTNAME is absent. http.Client.Do wraps every RoundTrip error in a +// *url.Error whose Error() embeds the full request URL, so the host appears in +// what the caller sees no matter what this transport returns — an assertion to +// the contrary would be unsatisfiable, and an unsatisfiable assertion is one +// somebody eventually weakens. It also protects nothing: the host is the +// attacker's own input, and they already know what they sent. The resolved +// address is the only half of that sentence they did not supply, which is +// exactly why it is the only half that has to go. +func TestSSRFSafeHTTPClient_RefusalIsTypedAndDoesNotDiscloseTheResolvedAddress(t *testing.T) { + t.Parallel() + + const blockedHost = "blocked.test" + + // Checked, not assumed: isPrivateIP(nil) returns false, so a typo'd literal + // here would classify as public, the request would never be refused, and + // this test would fail for a reason that has nothing to do with its subject. + resolved := net.ParseIP("10.99.13.37") + require.NotNil(t, resolved, "the test's own private address must parse") + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{blockedHost: {resolved}}} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + resp, err := client.Get("http://" + blockedHost + "/") + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "a hostname resolving to a private address must be refused") + + // Matchable by identity. Non-fatal so the message assertions below still run + // and one failure reports the whole gap rather than the first step of it. + assert.ErrorIs(t, err, ErrBlockedAddress, + "the refusal does not match ErrBlockedAddress. The nine callers being wired onto this client have to "+ + "separate a guard refusal from a network failure to choose a status code and a log level, and "+ + "substring matching on a message makes the message an API that cannot be reworded and fails open "+ + "when it is misspelled; got: %v", err) + + assert.Contains(t, err.Error(), "SSRF blocked", + "the refusal must keep the prefix that transport_unspecified_address_test.go:78 and the outer "+ + "acceptance contract both assert on, so a transport error cannot be mistaken for a block; got: %v", err) + + assert.NotContains(t, err.Error(), resolved.String(), + "the rendered refusal discloses %s, the address the attacker's own hostname resolved to inside our "+ + "network. They chose the name and can point it anywhere, so an error that answers 'and what did "+ + "that resolve to' is a mapping primitive rather than a diagnostic; got: %v", resolved, err) + + // The detail is not deleted, it is relocated: an operator debugging a block + // still needs to know which address caused it, and errors.As is where that + // now lives. Fatal, because the field assertions below would nil-deref. + var blocked *BlockedAddressError + require.ErrorAs(t, err, &blocked, + "the refusal must carry its detail on a typed error reachable with errors.As. Moving the address off "+ + "the message only works if it stays recoverable somewhere — otherwise this is a genericisation that "+ + "costs the operator the diagnostic; got: %v", err) + + assert.Equal(t, blockedHost, blocked.Host, + "the typed error must name the host that was refused") + assert.True(t, blocked.IP.Equal(resolved), + "the typed error names %s as the blocking address, but the answer that caused the refusal was %s", + blocked.IP, resolved) +} + +// TestBlockedAddressError_MarshalsWithoutTheResolvedAddress closes the second +// route out of this type. +// +// Error() was rewritten to drop the resolved address because error strings +// travel — into HTTP response bodies, shared logs and support tickets. Every one +// of those destinations is also somewhere errors get MARSHALLED rather than +// formatted: a structured logger handed the error value, an API that renders a +// failure as JSON, a handler that dumps a request context. IP is an exported +// net.IP, and net.IP marshals to its own textual form, so `json.Marshal(err)` +// puts back exactly the oracle Error() removed — with the attacker-chosen host +// beside it, which is the whole mapping primitive in one object. +// +// # WHAT THIS DOES AND DOES NOT COVER +// +// It covers encoding/json, which is the reflective renderer this tree actually +// reaches for. It does NOT cover %#v, %+v on a struct-formatting verb, or any +// other reflection-based dumper: those read the field regardless of tags, and +// shutting them out needs the field unexported behind an accessor. That is a +// larger change and is deliberately not made here — so this assertion is scoped +// to what it can honestly claim. +func TestBlockedAddressError_MarshalsWithoutTheResolvedAddress(t *testing.T) { + t.Parallel() + + resolved := net.ParseIP("10.99.13.37") + require.NotNil(t, resolved, "the test's own private address must parse") + + blocked := &BlockedAddressError{Host: "blocked.test", IP: resolved} + + encoded, err := json.Marshal(blocked) + require.NoError(t, err, "the typed refusal must stay marshallable; got: %v", err) + + assert.NotContains(t, string(encoded), resolved.String(), + "json.Marshal of the refusal renders %s, the address the attacker's own hostname resolved to inside "+ + "our network. Error() drops that address precisely because a refusal must not answer 'and what did "+ + "that resolve to' — a struct tag is all that stands between the two renderings, and a structured "+ + "logger handed the error value takes the marshalling one; got: %s", resolved, encoded) + + // The relocation is what makes the omission acceptable: errors.As still + // reaches the address, so the operator keeps the diagnostic. + assert.True(t, blocked.IP.Equal(resolved), + "the field itself must keep the address — hiding it from the renderer is not the same as deleting the "+ + "diagnostic errors.As exists to deliver") +} + +// TestSSRFSafeHTTPClient_ResolutionFailureIsNotABlockedAddress is the fence +// around the sentinel: it must mean "the guard refused this address", not "this +// request failed". +// +// A DNS failure and a refusal need opposite handling — one is retryable and +// unremarkable, the other is a security event worth logging loudly — so the +// cheapest wrong implementation of the cycle above, wrapping every RoundTrip +// error in ErrBlockedAddress, would give every caller a sentinel that says +// nothing. This passes today and must still pass afterwards. +func TestSSRFSafeHTTPClient_ResolutionFailureIsNotABlockedAddress(t *testing.T) { + t.Parallel() + + // An empty answer table, so hostRoutedResolver reports the host as not + // found: a resolution failure, not a classification. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{}} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + resp, err := client.Get("http://unresolvable.test/") + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "a hostname that does not resolve must still fail the request") + + assert.NotErrorIs(t, err, ErrBlockedAddress, + "a name that failed to resolve reports itself as a blocked address. The sentinel then means 'something "+ + "went wrong' rather than 'the guard refused a destination', and a caller using it to decide between "+ + "a retry and a security log gets the wrong answer for every DNS hiccup; got: %v", err) +} diff --git a/internal/atproto/oauth/transport_body_cap_test.go b/internal/atproto/oauth/transport_body_cap_test.go new file mode 100644 index 0000000..57d6008 --- /dev/null +++ b/internal/atproto/oauth/transport_body_cap_test.go @@ -0,0 +1,564 @@ +package oauth + +import ( + "compress/gzip" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The response-size cap, at the unit level. The outer acceptance contract in +// transport_response_cap_test.go pins the headline property — an over-large +// chunked body fails the read rather than truncating — and this file covers the +// shapes around it. +// +// # WHY THE CAP BELONGS TO THE TRANSPORT +// +// Four callers already implement this control themselves, at four different +// limits: blobs at 6 MB, the image proxy at 10 MB by default, unfurl at 10 MB, +// profile backfill at 1 MiB. Nine more fetch sites are about to be wired onto +// this client, and the ones that forget are the ones that matter — an +// unbounded io.ReadAll on an attacker-chosen URL is an out-of-memory the remote +// host triggers whenever it likes. A control every caller must remember is a +// control that will be missing somewhere. +// +// # REDIRECTS ARE PER-HOP, DELIBERATELY +// +// RoundTrip runs once per hop, so every hop's body is wrapped, but the cap does +// not accumulate across a redirect chain. That is adequate rather than a +// compromise: http.Client drains a redirect body at roughly 2 KB before +// following, far under any cap worth setting. There is no test for it here +// because it is an incidental of where the wrapping happens, and pinning an +// incidental is how a test suite acquires assertions nobody can change. + +// capPayload is the size of the over-cap bodies below, and testCap the limit +// they are read through. Small on purpose: the property is a boundary, and a +// test that proves it with 64 KiB proves it exactly as well as one that moves +// 32 MiB, in a fraction of the time. +const ( + testCap = 64 << 10 + capPayload = 1 << 20 +) + +// writeChunks writes n bytes and stops early if the client has gone away, which +// is the expected end of an over-cap response rather than a failure: once the +// cap engages the transport drops the connection and the handler's next write +// fails. +func writeChunks(w io.Writer, n int) { + chunk := make([]byte, 4<<10) + for sent := 0; sent < n; { + // Clamped to the remainder. A loop that always writes a full chunk + // overshoots any n that is not a multiple of the chunk size, which is + // fatal to a boundary test: it would send 4096 bytes for a 4097-byte cap + // and "one byte over" would silently become "one byte under". + if len(chunk) > n-sent { + chunk = chunk[:n-sent] + } + written, err := w.Write(chunk) + if err != nil { + return + } + sent += written + } +} + +// cappedClient is the client under test: the hatch is open because every +// listener here is on loopback, and classification is not what these tests are +// about. +func cappedClient(t *testing.T, max int64) *http.Client { + t.Helper() + return NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed(), WithMaxResponseBytes(max)) +} + +// fetchBody performs the request and reads the body to exhaustion, returning +// whichever error a caller would actually see — the request's or the read's. +// +// The two are deliberately collapsed for the boundary test below, because the +// cap may legitimately be enforced at either point: a Content-Length early-out +// refuses before the body, a body wrapper fails during it. Which mechanism +// catches an over-cap response is the implementer's choice; that it is caught is +// the property. Where the DISTINCTION is the point — the early-out, the lying +// header — the tests below drive Get and ReadAll separately instead. +func fetchBody(t *testing.T, client *http.Client, url string) ([]byte, error) { + t.Helper() + + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + return io.ReadAll(resp.Body) +} + +// TestSSRFSafeHTTPClient_RefusesAnOversizedContentLengthBeforeReading pins the +// cheap early-out: a response that ANNOUNCES more than the cap is refused +// without its body being read at all. +// +// This is an optimisation rather than a security control — the header is chosen +// by the same party as the body, so it is only ever a hint, and the test below +// this one is what pins the control proper. The distinction matters for what is +// asserted: the property here is that the failure arrives from client.Get +// itself, with no response to read, rather than partway through a body that has +// already been streamed into this process. +// +// # THE TRIPWIRE, AND WHY THE ERROR ALONE WAS NOT ENOUGH +// +// This test used to assert only that some error came back, and it passed for +// the wrong reason. Mutation-proven: insert `io.Copy(io.Discard, resp.Body)` +// BEFORE the refusal — so the whole megabyte is read into the process and then +// discarded, which is precisely the failure the test's name rejects — and it +// stayed green. "Refuses before reading" was in the name and nowhere in the +// assertions. +// +// What discriminates is a SERVER-SIDE byte count: the handler cannot finish +// writing a body nobody is reading, because the client drops the connection and +// the next write fails. So a handler that completed all its writes is a handler +// whose body was drained. It is the byte-level analogue of the `reached` +// tripwire in transport_response_cap_test.go, and it is measured at the server +// because the client side of a drained-and-discarded body looks identical to +// one that was never read. +// +// The payload is deliberately far larger than any socket buffer: the handler +// gets to write whatever the kernel absorbs before the close is noticed +// (~250 KB on the machine this was probed on), and the assertion has to sit +// well clear of that number to mean anything. +func TestSSRFSafeHTTPClient_RefusesAnOversizedContentLengthBeforeReading(t *testing.T) { + t.Parallel() + + // Eight megabytes, thirty-odd times what a socket buffer holds. See above. + const declaredPayload = 8 << 20 + + var handlerWrote atomic.Int64 + // Closed when the handler returns, so the count below is read after the + // handler has finished rather than while it is still writing. A channel and + // not a poll loop: the handler's exit is an event, and waiting for an event + // with a timer is the race this suite exists without. + handlerDone := make(chan struct{}) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + defer close(handlerDone) + + w.Header().Set("Content-Length", strconv.Itoa(declaredPayload)) + w.WriteHeader(http.StatusOK) + + chunk := make([]byte, 4<<10) + for sent := 0; sent < declaredPayload; { + if len(chunk) > declaredPayload-sent { + chunk = chunk[:declaredPayload-sent] + } + written, err := w.Write(chunk) + handlerWrote.Add(int64(written)) + if err != nil { + // The expected end: the client refused and hung up. + return + } + sent += written + } + })) + defer server.Close() + + resp, err := cappedClient(t, testCap).Get(server.URL) + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, + "a response announcing %d bytes through a %d-byte cap must be refused by the request itself. Reading "+ + "the body first and checking afterwards is the same as having no cap: the bytes are already in "+ + "memory by the time the check runs", declaredPayload, int64(testCap)) + + // By identity, not by substring. Nothing else in this file asserts the + // sentinel, so a refusal that came back as any other error — a dial failure, + // a parse error, a nil-deref recovered somewhere — would have satisfied the + // require above while meaning something entirely different. + require.ErrorIs(t, err, ErrResponseTooLarge, + "the refusal must be the cap's, matchable by identity: the callers being wired onto this client "+ + "have to tell an over-large body from a transport failure to choose a status code; got: %v", err) + + <-handlerDone + assert.Lessf(t, handlerWrote.Load(), int64(declaredPayload), + "the handler wrote all %d bytes of a body that was supposed to be refused unread. A server cannot "+ + "finish writing to a client that has hung up, so a complete write means this process read and "+ + "discarded the whole body first — which is the same as having no cap at all, and is exactly "+ + "what this test's name says does not happen. Wrote %d bytes.", + declaredPayload, handlerWrote.Load()) +} + +// TestSSRFSafeHTTPClient_DeclaredContentLengthBoundsTheBody is a +// CHARACTERIZATION test, and it passes today. It exists because the early-out +// above is only sound if this holds. +// +// # WHY THIS IS NOT THE "LYING HEADER" TEST IT LOOKS LIKE +// +// The obvious companion to the early-out is a server that declares a small +// Content-Length and then writes far more, proving the header cannot be trusted +// and the body wrapper must catch it. That test cannot be written, because the +// scenario does not exist: net/http's client frames the body by the declared +// length and stops there. Verified by probe against a HIJACKED handler, so the +// Go server's own Content-Length enforcement was out of the way and the bytes on +// the wire were a genuinely lying response — the caller received exactly 128 +// bytes with a nil error while the server wrote a megabyte. +// +// So an understated header cannot overflow a caller, and a test asserting the +// wrapper catches it would be asserting an artifact of the standard library +// rather than anything this package does. +// +// The header is still not the control, but the reason is different and worth +// getting right: it can be ABSENT. A chunked response has no length, and a +// transparently decompressed one has its length deleted by the transport. Those +// are the cases the wrapper exists for, and they are pinned in +// TestSSRFSafeHTTPClient_CapsDecompressedBytesNotCompressedBytes and +// TestSSRFSafeHTTPClient_UnknownLengthIsNeitherTrustedNorRejected. +// +// What this test protects is the assumption underneath the early-out: if the +// client ever stopped honouring the declared length, a response could sail past +// a header check and then deliver more than it promised. That would turn the +// cheap optimisation into a hole, and this is what would notice. +func TestSSRFSafeHTTPClient_DeclaredContentLengthBoundsTheBody(t *testing.T) { + t.Parallel() + + const declared = 128 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Hijacked so the response on the wire really does understate itself. + // Through the normal ResponseWriter the Go SERVER would refuse the + // excess writes, and the test would be pinning the server's behavior + // rather than the client's. + conn, buf, err := w.(http.Hijacker).Hijack() + if err != nil { + return + } + defer func() { _ = conn.Close() }() + + _, _ = buf.WriteString("HTTP/1.1 200 OK\r\nContent-Length: " + strconv.Itoa(declared) + "\r\n\r\n") + writeChunks(buf, capPayload) + _ = buf.Flush() + })) + defer server.Close() + + resp, err := cappedClient(t, testCap).Get(server.URL) + require.NoError(t, err, "a response declaring %d bytes is far under the cap and must not be refused", declared) + defer func() { _ = resp.Body.Close() }() + + body, readErr := io.ReadAll(resp.Body) + + require.NoError(t, readErr, "the declared body is well under the cap and must read cleanly; got: %v", readErr) + assert.Len(t, body, declared, + "the caller received %d bytes from a response that declared %d and then wrote %d. net/http no longer "+ + "frames the body by the declared length, which means a Content-Length early-out can now be walked "+ + "past by a server that understates itself — the cap must not rely on the header alone", + len(body), declared, capPayload) +} + +// TestSSRFSafeHTTPClient_CapsDecompressedBytesNotCompressedBytes is the case +// most easily missed, and the one that quietly disables every other check. +// +// http.Transport adds Accept-Encoding: gzip on its own and transparently +// decompresses the reply. When it does, it DELETES Content-Length and sets +// resp.ContentLength to -1 — so the header early-out above is a no-op against +// any compressed response, and enabling compression is all it takes to walk past +// it. +// +// Decompressed bytes are the only honest unit. They are what io.ReadAll +// allocates, and zeros compress about a thousand to one: a cap counting +// compressed bytes lets a body three orders of magnitude over the limit through, +// which is the classic decompression bomb. +// +// This is also why the wrapper belongs on the response the base transport +// RETURNS — wrapping there wraps the gzip reader's output, and this case then +// costs nothing to satisfy. +func TestSSRFSafeHTTPClient_CapsDecompressedBytesNotCompressedBytes(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + w.WriteHeader(http.StatusOK) + + gz := gzip.NewWriter(w) + writeChunks(gz, capPayload) + _ = gz.Close() + })) + defer server.Close() + + resp, err := cappedClient(t, testCap).Get(server.URL) + require.NoError(t, err, "the request must succeed: compressed, this response is a few kilobytes") + defer func() { _ = resp.Body.Close() }() + + // Precondition, not decoration. If the transport did not decompress this + // transparently the test would be reading gzip bytes and its premise would + // be gone, so it is checked rather than assumed. + require.True(t, resp.Uncompressed, + "http.Transport did not transparently decompress this response, so the test is not exercising the "+ + "decompressed path it was written for") + require.EqualValues(t, -1, resp.ContentLength, + "a transparently decompressed response must report an unknown length, which is precisely why the "+ + "Content-Length early-out cannot be the control here") + + body, readErr := io.ReadAll(resp.Body) + + require.Error(t, readErr, + "%d decompressed bytes were read through a %d-byte cap. The compressed body was a few kilobytes, so a "+ + "cap counting bytes off the wire never fired — and the memory a decompression bomb costs is "+ + "decompressed memory", len(body), int64(testCap)) + assert.LessOrEqual(t, int64(len(body)), int64(testCap), + "the caller obtained %d decompressed bytes through a %d-byte cap", len(body), int64(testCap)) +} + +// TestSSRFSafeHTTPClient_CapBoundary pins the two sizes that decide whether the +// comparison is > or >=. +// +// blobs.(*blobService).FetchImageForURL documents the technique — read +// max+1 bytes, and the presence of that one extra byte is what separates "at the +// limit" from "over it". A cap implemented by measuring after an io.ReadAll gets +// the boundary right and the property wrong, since the whole body is in memory +// by then. +// +// No Content-Length is sent, so the body wrapper is what has to catch the +// over-cap case; the sizes are small enough that Go would otherwise buffer and +// declare a length. +func TestSSRFSafeHTTPClient_CapBoundary(t *testing.T) { + t.Parallel() + + const boundary = 4 << 10 + + tests := []struct { + name string + size int + mustFail bool + assertion string + }{ + { + name: "exactly at the cap", + size: boundary, + mustFail: false, + assertion: "a body of exactly the cap must be readable in full. A cap that refuses its own limit " + + "is off by one in the direction that breaks callers rather than the one that protects them", + }, + { + name: "one byte over the cap", + size: boundary + 1, + mustFail: true, + assertion: "a body one byte over the cap must fail. This is the assertion that distinguishes a real " + + "limit from one that rounds, buffers, or compares after the fact", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Flushing forces a chunked response, so nothing here declares a + // Content-Length and the wrapper is the only thing that can catch + // the over-cap case. + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + writeChunks(w, tt.size) + })) + defer server.Close() + + body, err := fetchBody(t, cappedClient(t, boundary), server.URL) + + if tt.mustFail { + require.Error(t, err, "%s; read %d bytes", tt.assertion, len(body)) + assert.LessOrEqual(t, int64(len(body)), int64(boundary), + "the caller obtained %d bytes through a %d-byte cap", len(body), int64(boundary)) + return + } + require.NoError(t, err, "%s; got: %v", tt.assertion, err) + assert.Len(t, body, tt.size, + "a body at exactly the cap must arrive complete, not one byte short") + }) + } +} + +// TestSSRFSafeHTTPClient_UnknownLengthIsNeitherTrustedNorRejected pins the +// meaning of ContentLength == -1. +// +// It means "unknown", and it arrives from two ordinary places: a chunked +// response, and any response the transport transparently decompressed. So it can +// be read neither as 0 ("nothing to check") nor as a reason to refuse — the +// first waves through every streaming body, the second breaks chunked responses +// and, by way of the gzip case above, a large share of the real internet. +// +// Both directions are pinned because an implementation can get one right and the +// other wrong. +func TestSSRFSafeHTTPClient_UnknownLengthIsNeitherTrustedNorRejected(t *testing.T) { + t.Parallel() + + newChunkedServer := func(size int) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + writeChunks(w, size) + })) + } + + t.Run("an unknown length under the cap is delivered", func(t *testing.T) { + t.Parallel() + + server := newChunkedServer(testCap / 4) + defer server.Close() + + body, err := fetchBody(t, cappedClient(t, testCap), server.URL) + require.NoError(t, err, + "a chunked response well under the cap was refused. ContentLength is -1 for every chunked body and "+ + "every decompressed one, so treating an unknown length as a reason to refuse rejects ordinary "+ + "traffic; got: %v", err) + assert.Len(t, body, testCap/4, "the body must arrive complete") + }) + + t.Run("an unknown length over the cap is stopped", func(t *testing.T) { + t.Parallel() + + server := newChunkedServer(capPayload) + defer server.Close() + + body, err := fetchBody(t, cappedClient(t, testCap), server.URL) + require.Error(t, err, + "a chunked response of %d bytes was read in full through a %d-byte cap. An unknown length read as "+ + "'nothing to check' waves through every streaming body, which is exactly the shape an attacker "+ + "would choose; read %d bytes", capPayload, int64(testCap), len(body)) + }) +} + +// TestSSRFSafeHTTPClient_UnderCapResponseReusesTheConnection pins that the +// wrapper delegates Close to the body it wraps. +// +// A wrapper that implements Read and forgets Close is invisible in every +// functional test — bodies still arrive, caps still fire — and leaks a +// connection per request. http.Transport returns a connection to the pool when +// the body is closed and drained; if the close never reaches it, MaxIdleConns +// and IdleConnTimeout describe a pool nothing is ever returned to, and a busy +// AppView accumulates sockets until it runs out. +// +// Asserted through the server's connection count rather than the wrapper's +// shape, so the implementer keeps room: two sequential requests over one client +// must be observed as ONE connection. +func TestSSRFSafeHTTPClient_UnderCapResponseReusesTheConnection(t *testing.T) { + t.Parallel() + + var connections atomic.Int64 + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + writeChunks(w, 1<<10) + })) + server.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + connections.Add(1) + } + } + server.Start() + defer server.Close() + + client := cappedClient(t, testCap) + + // Sequential, and each body fully read and closed — which is exactly what a + // pooled connection requires, and what makes a missing Close observable. + for i := range 2 { + body, err := fetchBody(t, client, server.URL) + require.NoErrorf(t, err, "request %d must succeed: %d bytes is far under the cap", i+1, 1<<10) + require.Lenf(t, body, 1<<10, "request %d must return the whole body", i+1) + } + + assert.EqualValues(t, 1, connections.Load(), + "the server saw %d connections for two sequential requests. The second did not reuse the first, which "+ + "means the response body never reported itself closed to http.Transport — a capping wrapper that "+ + "implements Read and forgets Close passes every functional test and leaks one connection per "+ + "request", connections.Load()) +} + +// TestSSRFSafeHTTPClient_EmptyBodiesDoNotTripTheCap covers the responses that +// have no body to cap. +// +// Both rows exist because zero is a suspicious-looking number: an implementation +// that treats ContentLength == 0 as "unknown, therefore check harder", or that +// requires at least one byte before deciding a response is well-formed, breaks +// on traffic that is entirely ordinary. +func TestSSRFSafeHTTPClient_EmptyBodiesDoNotTripTheCap(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/no-content" { + w.WriteHeader(http.StatusNoContent) + return + } + w.WriteHeader(http.StatusOK) + writeChunks(w, 1<<10) + })) + // t.Cleanup, NOT defer. This server is shared with PARALLEL subtests, and a + // deferred Close runs when this function returns — which is the moment the + // first subtest calls t.Parallel and pauses, long before any of them make a + // request. The result is a connection-refused failure that looks like a bug + // in the code under test. t.Cleanup runs after the subtests finish. + t.Cleanup(server.Close) + + client := cappedClient(t, testCap) + + t.Run("204 No Content", func(t *testing.T) { + t.Parallel() + + body, err := fetchBody(t, client, server.URL+"/no-content") + require.NoError(t, err, "a 204 carries no body and must not be treated as a capped read; got: %v", err) + assert.Empty(t, body, "a 204 must produce an empty body") + }) + + t.Run("HEAD request", func(t *testing.T) { + t.Parallel() + + resp, err := client.Head(server.URL + "/") + require.NoError(t, err, "a HEAD request must succeed; got: %v", err) + defer func() { _ = resp.Body.Close() }() + + body, readErr := io.ReadAll(resp.Body) + require.NoError(t, readErr, "a HEAD response has no body to read; got: %v", readErr) + assert.Empty(t, body, "a HEAD response must produce an empty body") + }) +} + +// TestSSRFSafeHTTPClient_DefaultCap pins what a caller gets without an option. +// +// The VALUE is asserted against the constant so the number is stated once. 32 +// MiB is chosen to sit above every cap in this tree — the largest is the image +// proxy's 10 MB — so that pointing an existing caller at this client cannot make +// a request that used to work start failing. That cross-package claim is not +// asserted here on purpose: those packages import this one, so a test importing +// them back would be an import cycle. +// +// The behavioral half is the one that catches a real mistake. A cap stored in a +// zero-valued field, because the default was never applied when no option was +// passed, refuses EVERY response — and a test that only ever constructs clients +// with WithMaxResponseBytes, as every other test in this file does, would not +// notice. +func TestSSRFSafeHTTPClient_DefaultCap(t *testing.T) { + t.Parallel() + + assert.EqualValues(t, 32<<20, DefaultMaxResponseBytes, + "the default cap must be 32 MiB: above every caller's own limit in this tree (the largest is 10 MB), "+ + "so adopting this client changes no existing behavior, and still a bound on what a remote host can "+ + "make this process allocate") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + writeChunks(w, 1<<20) + })) + defer server.Close() + + // No WithMaxResponseBytes: the default is what is under test. + body, err := fetchBody(t, NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed()), server.URL) + require.NoError(t, err, + "a 1 MiB response was refused by a client with no cap option, so the default was not applied and the "+ + "limit is sitting at a zero value — which refuses every response there is; got: %v", err) + assert.Len(t, body, 1<<20, "the body must arrive complete under the default cap") +} diff --git a/internal/atproto/oauth/transport_cap_bounds_test.go b/internal/atproto/oauth/transport_cap_bounds_test.go new file mode 100644 index 0000000..3fd2cce --- /dev/null +++ b/internal/atproto/oauth/transport_cap_bounds_test.go @@ -0,0 +1,162 @@ +package oauth + +import ( + "bytes" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// WithMaxResponseBytes takes an int64 and does nothing with it, so three values +// an operator can plausibly pass turn the client into something worse than +// uncapped. +// +// # WHAT EACH ONE DOES TODAY, VERIFIED BY PROBE +// +// WithMaxResponseBytes(0) every non-empty response is refused +// WithMaxResponseBytes(-1) Read returns n = -1 +// WithMaxResponseBytes(math.MaxInt64) Read panics: slice bounds out of range +// +// The negative case is the one that is not merely wrong but ILLEGAL. io.Reader +// forbids a negative count, and the standard library does not defend against +// one: bytes.Buffer.ReadFrom panics with "reader returned negative count from +// Read". So a caller doing the most ordinary thing there is with a response body +// takes a panic out of a configuration typo, on a goroutine it may not own. +// +// MaxInt64 is a typo away from being what someone writes for "no limit", and it +// panics inside cappedBody.Read on `remaining+1` overflowing to MinInt64 and +// being used as a slice bound. +// +// Zero is the value a struct field, a missing config key or an unparsed +// environment variable arrives as. It refuses every response, which reads at the +// call site as "this remote host is broken" — for every host. +// +// # WHAT THIS ASSERTS, AND WHAT IT DELIBERATELY DOES NOT +// +// The property is that a cap outside the usable range does not break the +// client: an ordinary small response still arrives, whole, without a panic and +// without a negative count. The clamp VALUE is left to the implementation — +// asserting "0 becomes DefaultMaxResponseBytes" would pin a number the +// behaviour does not require, and the next person to tune it would have to +// change a test that was never about that. +// +// TestSSRFSafeHTTPClient_DefaultCap and the boundary tests in +// transport_body_cap_test.go remain the fence in the other direction: whatever +// clamping is added must not turn a SENSIBLE cap into a larger one. +func TestWithMaxResponseBytes_ACapOutsideTheUsableRangeDoesNotBreakTheClient(t *testing.T) { + t.Parallel() + + // Deliberately tiny. Any cap a reasonable clamp could produce is above this, + // so the assertion is about the cap being usable at all rather than about + // where it landed. + const payload = 512 + + tests := []struct { + name string + cap int64 + why string + }{ + { + name: "zero", + cap: 0, + why: "zero is what an unset field, a missing config key or an unparsed environment variable " + + "arrives as, and it currently refuses every non-empty response there is", + }, + { + name: "negative", + cap: -1, + why: "a negative cap makes cappedBody.Read return n = -1, which io.Reader forbids outright — " + + "bytes.Buffer.ReadFrom panics on it, so a configuration typo becomes a panic in a caller " + + "doing nothing unusual", + }, + { + name: "math.MaxInt64", + cap: math.MaxInt64, + why: "MaxInt64 is what someone writes for 'no limit'; remaining+1 overflows to MinInt64 and is " + + "used as a slice bound, so the first read panics", + }, + } + + // Both response shapes, because the two are stopped by different mechanisms + // and only one of them reaches cappedBody.Read at all. A declared length is + // refused by the header early-out — which is where a negative or zero cap + // currently fails, before any byte is read — while a chunked response has no + // length to check and goes through the wrapper, which is where the negative + // count and the overflow panic actually live. Testing only the first would + // pin the symptom and miss the io.Reader violation entirely. + shapes := []struct { + name string + chunked bool + }{ + {name: "declared length", chunked: false}, + {name: "chunked, no declared length", chunked: true}, + } + + for _, tt := range tests { + for _, shape := range shapes { + t.Run(tt.name+"/"+shape.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + if shape.chunked { + // Flushing the header commits the response before the + // body is known, so Go frames it chunked and declares + // no length. + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + } + writeChunks(w, payload) + })) + defer server.Close() + + // allowPrivate, because the listener is on loopback and + // classification is not what is under test here. + resp, err := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed(), WithMaxResponseBytes(tt.cap)).Get(server.URL) + require.NoErrorf(t, err, + "a %d-byte response was refused by the request itself under WithMaxResponseBytes(%d): %s; got: %v", + payload, tt.cap, tt.why, err) + defer func() { _ = resp.Body.Close() }() + + n, readErr := readWithoutPanicking(resp.Body) + + require.NoErrorf(t, readErr, + "reading a %d-byte body through WithMaxResponseBytes(%d) failed: %s; got: %v", + payload, tt.cap, tt.why, readErr) + assert.EqualValuesf(t, payload, n, + "the caller obtained %d of %d bytes under WithMaxResponseBytes(%d). A cap outside the "+ + "usable range must fall back to a usable one, not truncate: %s", n, payload, tt.cap, tt.why) + }) + } + } +} + +// readWithoutPanicking drains r through bytes.Buffer.ReadFrom and turns a panic +// into an error. +// +// bytes.Buffer.ReadFrom is the reader chosen on purpose: it is what panics on a +// negative count, so it is the detector for the io.Reader violation this test is +// about, and it is what any caller building a body in memory ends up using. +// +// The recover is not defensive decoration. An unrecovered panic takes the whole +// package's test binary down with it, so the RED run for one row would hide +// every other failure in the package behind a stack trace — and the point of +// this cycle is a readable list of what is broken. +func readWithoutPanicking(r io.Reader) (n int64, err error) { + defer func() { + if p := recover(); p != nil { + n, err = 0, fmt.Errorf("reading the response body PANICKED: %v", p) + } + }() + + var buf bytes.Buffer + n, err = buf.ReadFrom(r) + return n, err +} diff --git a/internal/atproto/oauth/transport_capped_body_test.go b/internal/atproto/oauth/transport_capped_body_test.go new file mode 100644 index 0000000..ccf0b15 --- /dev/null +++ b/internal/atproto/oauth/transport_capped_body_test.go @@ -0,0 +1,243 @@ +package oauth + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// cappedBody at the unit level, for the two properties the end-to-end tests +// structurally cannot see. +// +// transport_body_cap_test.go drives everything through a real listener, which +// is the right level for a cap: it proves what a caller experiences. But two of +// the wrapper's decisions are invisible from there, and both were confirmed +// invisible by mutation: +// +// - CLOSE. Deleting the delegation in cappedBody.Close leaves the whole suite +// green, including TestSSRFSafeHTTPClient_UnderCapResponseReusesTheConnection, +// which was written for exactly this. That test reads both bodies to EOF, +// and http.Transport recycles a connection when the body is EXHAUSTED as +// well as when it is closed — so the connection count it asserts on is +// satisfied by the reads alone. +// - THE STICKY exceeded FLAG. Deleting the branch leaves the suite green, +// because every test stops reading at the first error. A caller that reads +// again is the case the flag exists for, and nothing calls it. +// +// A fake ReadCloser is the only way to observe either. That makes these +// structural tests, which is a cost worth naming: they know cappedBody has a +// Close and a remaining counter, so a rewrite that changes the wrapper's shape +// has to change them too. It is the right trade only because the alternative is +// what the mutations found — no coverage at all. + +// countingReadCloser records how many times it was closed and what Close +// returned, which is the whole of what the delegation test can observe. +type countingReadCloser struct { + io.Reader + closes int + closeErr error +} + +func (c *countingReadCloser) Close() error { + c.closes++ + return c.closeErr +} + +func TestCappedBody_CloseReachesTheBodyUnderneath(t *testing.T) { + t.Parallel() + + t.Run("close is delegated exactly once", func(t *testing.T) { + t.Parallel() + + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, 8))} + body := &cappedBody{body: underlying, remaining: testCap} + + require.NoError(t, body.Close(), "closing an ordinary body must not fail") + + assert.Equal(t, 1, underlying.closes, + "cappedBody.Close closed the body underneath %d times, want 1. http.Transport returns a "+ + "connection to the idle pool when its body reports itself closed, so a wrapper that "+ + "implements Read and swallows Close leaks one connection per request while passing every "+ + "functional test — MaxIdleConns then describes a pool nothing is ever returned to", + underlying.closes) + }) + + t.Run("close is delegated on a body that was never read", func(t *testing.T) { + t.Parallel() + + // The path an early refusal takes: the caller closes without reading a + // byte. Separate from the row above because a delegation that only + // happens once something has been read is not a delegation. + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, 8))} + body := &cappedBody{body: underlying, remaining: testCap} + + require.NoError(t, body.Close(), "closing an unread body must not fail") + assert.Equal(t, 1, underlying.closes, + "a body that was never read must still be closed through to the connection underneath") + }) + + t.Run("close is delegated after the cap has fired", func(t *testing.T) { + t.Parallel() + + // The case that matters most in production: an over-cap response is + // precisely the one whose connection must not be leaked, because it is + // the one an attacker can produce at will. + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, 64))} + body := &cappedBody{body: underlying, remaining: 8} + + _, readErr := io.ReadAll(body) + require.ErrorIs(t, readErr, ErrResponseTooLarge, "the premise: this read must trip the cap") + + require.NoError(t, body.Close(), "closing after the cap fired must not fail") + assert.Equal(t, 1, underlying.closes, + "the connection behind an OVER-CAP response was not closed. That is the response an attacker "+ + "chooses to send, so a leak on this path is one they can repeat until the process runs out "+ + "of sockets") + }) + + t.Run("the underlying close error is not swallowed", func(t *testing.T) { + t.Parallel() + + sentinel := errors.New("the connection underneath failed to close") + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, 8)), closeErr: sentinel} + body := &cappedBody{body: underlying, remaining: testCap} + + assert.ErrorIs(t, body.Close(), sentinel, + "cappedBody.Close discarded the error from the body underneath. A wrapper that always reports "+ + "success hides exactly the failures a caller checking Close is looking for") + }) +} + +// TestCappedBody_TheFailureIsStickyAndIsNeverEOF pins the branch the whole cap +// is built to protect, and the sentinel it is built to report. +// +// # WHY STICKY +// +// A caller that reads again after the cap fires must not be told the stream +// ended cleanly. io.ReadAll treats io.EOF as the end of the body and returns a +// nil error, so a wrapper that failed once and then reported EOF would hand +// back a SHORT BODY WITH NO ERROR — which is the silent truncation this design +// rejected in favour of a sentinel. transport.go's ErrResponseTooLarge comment +// spells out what that costs downstream: a parser reads half a document, a size +// check never fires, a truncated image is written to a user's PDS as a whole +// one. +// +// # WHY NOTHING ELSE COVERS IT +// +// Every other test in the package stops at the first error, because that is +// what io.ReadAll does. The second read is the entire subject here, and +// deleting the sticky branch leaves the rest of the suite green. +func TestCappedBody_TheFailureIsStickyAndIsNeverEOF(t *testing.T) { + t.Parallel() + + const limit = 8 + + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, limit*8))} + body := &cappedBody{body: underlying, remaining: limit} + + // First read: over the cap, so it fails. This much the end-to-end tests + // already cover; it is the premise for what follows. + first := make([]byte, limit*4) + n, err := body.Read(first) + require.ErrorIs(t, err, ErrResponseTooLarge, + "the premise: reading %d bytes through a %d-byte cap must fail with ErrResponseTooLarge, and it "+ + "must be ASSERTABLE BY IDENTITY — nothing else in this package checks that sentinel, so a cap "+ + "that failed with any old error would look identical; got: %v", len(first), limit, err) + require.GreaterOrEqual(t, n, 0, "io.Reader forbids a negative count") + require.LessOrEqual(t, n, limit, "the failing read must not deliver more than the cap") + + // The subject. A caller that reads on — a decoder in a loop, a copy that + // did not check — must be told the same thing again. + second := make([]byte, limit*4) + n, err = body.Read(second) + + assert.Equal(t, 0, n, + "the read after the cap fired returned %d bytes. Once the limit is breached there is nothing more "+ + "this body may deliver", n) + assert.ErrorIs(t, err, ErrResponseTooLarge, + "the read after the cap fired did not report ErrResponseTooLarge; got: %v", err) + assert.NotErrorIs(t, err, io.EOF, + "the read after the cap fired reported io.EOF (possibly wrapped). io.ReadAll treats EOF as the "+ + "clean end of a body and returns a nil error, so a caller that reads again would come away "+ + "with a short body it has no way to know is short — which is the silent truncation this "+ + "sentinel exists to prevent, and is worse than having no cap at all; got: %v", err) + + // And again, because "sticky" means it does not decay. A flag cleared by a + // later read would satisfy both assertions above. + third := make([]byte, limit*4) + n, err = body.Read(third) + assert.Equal(t, 0, n, "the third read returned %d bytes; the failure must not decay", n) + assert.ErrorIs(t, err, ErrResponseTooLarge, + "the third read stopped reporting ErrResponseTooLarge, so the failure is not sticky but merely "+ + "delayed; got: %v", err) +} + +// TestCappedBody_TheAllowanceIsSpentWhenTheCapFires closes the gap between the +// two fields that describe one state. +// +// exceeded and remaining are not independent: once the cap has fired the body +// will never deliver another byte, so an allowance that is still positive +// describes bytes that can never be spent. `(exceeded: true, remaining: 1024)` +// is representable and means nothing, and the only reason it is currently +// harmless is that Read consults exceeded FIRST. That is a read-order +// coincidence, not an invariant — the next edit to this wrapper (a Reset, a +// second reader, a metric that reports the unused allowance) reads remaining +// and gets a number that was true before the refusal and has been stale ever +// since. +// +// Zeroing it alongside the flag makes the illegal state unrepresentable, which +// is the cheaper half of the two ways to fix this. The other is collapsing the +// pair into one field; the flag is kept because ErrResponseTooLarge must stay +// distinguishable from "the allowance happened to land on zero". +func TestCappedBody_TheAllowanceIsSpentWhenTheCapFires(t *testing.T) { + t.Parallel() + + const limit = 8 + + underlying := &countingReadCloser{Reader: bytes.NewReader(make([]byte, limit*8))} + body := &cappedBody{body: underlying, remaining: limit} + + // A read far larger than the allowance, so the refusal happens with plenty + // of it notionally unspent — the shape in which a stale counter is most + // obviously wrong. + _, err := body.Read(make([]byte, limit*4)) + require.ErrorIs(t, err, ErrResponseTooLarge, + "the premise: reading %d bytes through a %d-byte cap must trip the cap; got: %v", + limit*4, limit, err) + + require.True(t, body.exceeded, "the premise: the refusal must have set the sticky flag") + assert.Zero(t, body.remaining, + "the cap fired and left an allowance of %d bytes behind. The two fields describe one state, so a "+ + "positive remaining alongside exceeded is a number about bytes this body will never deliver — "+ + "harmless only for as long as Read keeps checking the flag before the counter, which is an "+ + "ordering coincidence rather than an invariant", body.remaining) +} + +// TestCappedBody_AtTheCapTheBodyArrivesWhole is the boundary at the unit level, +// and it is the fence around every assertion above: a wrapper that failed +// eagerly would satisfy all of them. +// +// The end-to-end boundary test owns the same property through a real listener. +// This one owns it against a reader that cannot be affected by framing, +// buffering or chunk sizes, so a failure here names the wrapper and nothing +// else. +func TestCappedBody_AtTheCapTheBodyArrivesWhole(t *testing.T) { + t.Parallel() + + const limit = 8 + + underlying := &countingReadCloser{Reader: bytes.NewReader(bytes.Repeat([]byte{'x'}, limit))} + body := &cappedBody{body: underlying, remaining: limit} + + got, err := io.ReadAll(body) + + require.NoError(t, err, + "a body of exactly the cap must read cleanly. Reading one byte past the allowance is how the "+ + "boundary is detected, and that extra read must find EOF rather than be mistaken for an "+ + "overflow; got: %v", err) + assert.Len(t, got, limit, "a body at exactly the cap must arrive complete, not one byte short") +} diff --git a/internal/atproto/oauth/transport_context_test.go b/internal/atproto/oauth/transport_context_test.go new file mode 100644 index 0000000..41c6288 --- /dev/null +++ b/internal/atproto/oauth/transport_context_test.go @@ -0,0 +1,87 @@ +package oauth + +import ( + "context" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeHTTPClient_ResolutionHonoursTheRequestContext pins that a +// cancelled request stops the name lookup instead of running it to the +// resolver's own timeout. +// +// # WHAT WAS BROKEN (fixed by the change this test drove) +// +// resolveHost used to call net.LookupIP, which takes no context. So a caller that +// cancels — a user who closed the connection, a handler whose deadline expired, +// a shutdown — releases nothing: the lookup keeps a goroutine and a socket alive +// until the resolver's own unbounded timeout, and every one of the nine fetch +// sites about to use this client sits behind a request-scoped context that means +// nothing to it. CLAUDE.md names a missing context.Context as a red flag for +// exactly this reason. +// +// # WHY THIS DRIVES THE DEFAULT PATH AND NOT THE lookupIP SEAM +// +// The seam is the wrong instrument here, and the distinction matters enough to +// write down. RoundTrip handing req.Context() to resolveHost is PLUMBING — +// whether a fake resolver receives a cancelled context is decided by that one +// argument, so a test built on the seam measures the wiring rather than the +// behavior, and it would pass the moment the signature changed even with +// net.LookupIP still underneath. Driving the real path is what distinguishes +// "the context arrives" from "the context is obeyed", and only the second is the +// property. +// +// # AND WHY THE HOST IS A NAME RATHER THAN AN IP LITERAL +// +// A dotted quad would make this test assert nothing at all. Both the old and the +// new resolver short-circuit a literal and return it BEFORE consulting the +// context, so a cancelled-context test pointed at 127.0.0.1 succeeds either way. +// The host has to be one that reaches the resolver proper. +// +// `cancelled.invalid` is that host, and .invalid is chosen over the .test names +// used elsewhere in this package deliberately: RFC 6761 §6.4 guarantees it is +// never resolvable, where a developer's local dnsmasq may well map *.test to +// loopback. Note what the assertions do NOT depend on, though — once the fix +// lands, a cancelled context means NO DNS QUERY IS MADE AT ALL, so the green +// path touches no resolver. Only the red path performs a lookup, and it fails +// this test on every possible answer to it (NXDOMAIN, timeout, or even a +// successful one), because none of them is a cancellation. +func TestSSRFSafeHTTPClient_ResolutionHonoursTheRequestContext(t *testing.T) { + t.Parallel() + + // No seam installed. transport.lookupIP stays nil so resolveHost takes its + // production path, which is the thing under test. + client := NewSSRFSafeHTTPClient() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://cancelled.invalid/", nil) + require.NoError(t, err, "building the request") + + resp, err := client.Do(req) + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, "a request whose context is already cancelled must fail") + + // The anti-vacuity assertion, and it has to come first because it is what + // makes the next one mean something. "failed to resolve host" is RoundTrip's + // own wrapper, so its presence proves the failure came from THIS transport's + // resolution step rather than from http.Client noticing the dead context and + // short-circuiting before RoundTrip ever ran. Without it, a client-level + // short-circuit would satisfy the cancellation assertion below while the + // lookup stayed exactly as context-blind as it is today. + assert.Contains(t, err.Error(), "failed to resolve host", + "the failure did not come from the transport's own resolution step, so this test cannot say anything "+ + "about whether that step honours the context; got: %v", err) + + assert.ErrorIs(t, err, context.Canceled, + "the resolution of a cancelled request did not report the cancellation — it ran the lookup anyway and "+ + "failed for some unrelated reason. net.LookupIP takes no context, so cancelling a request releases "+ + "nothing and the lookup holds its goroutine until the resolver's own timeout; "+ + "net.DefaultResolver.LookupIPAddr(ctx, host) is the drop-in that observes it. Got: %v", err) +} diff --git a/internal/atproto/oauth/transport_dial_errors_test.go b/internal/atproto/oauth/transport_dial_errors_test.go new file mode 100644 index 0000000..ae1779f --- /dev/null +++ b/internal/atproto/oauth/transport_dial_errors_test.go @@ -0,0 +1,102 @@ +package oauth + +import ( + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeTransport_DialErrorAccountsForEveryVettedAddress pins that a +// request which tried several addresses and failed on all of them says so. +// +// # WHAT AN OPERATOR USED TO LOSE (fixed by the change this test drove) +// +// The dial loop kept only a `lastErr`, so every failure but the final one was +// discarded. A hostname with an A and an AAAA record is the ordinary case, not a +// corner one, and the two commonly fail differently — IPv6 unreachable on a +// v4-only host, connection refused on the v4 address. What reaches the log is +// the second symptom alone, with nothing to say another address was tried at +// all. Someone debugging a federation failure then investigates one address +// while the request also failed against another they cannot see. +// +// # A SECOND, WEAKER REASON, STATED CAREFULLY +// +// `return nil, lastErr` at the end of the loop is only guaranteed non-nil +// because the `len(vetted) == 0` guard above it refuses the empty case first. +// That is a latent fragility in the loop's contract, NOT a live bug: an empty +// vetted slice cannot reach the loop through the public API today, and nothing +// currently returns `(nil, nil)` from the dial. It is worth fixing while this +// code is open because the guard and the loop are separated by twenty lines, and +// an edit to either one that forgets the other would silently produce a dial +// result net/http does not tolerate. Do not read this paragraph as a claim that +// the current code can panic — it cannot. +// +// # WHY NAMING ADDRESSES HERE IS NOT THE ORACLE CYCLE 1 REMOVED +// +// The refusal error deliberately does NOT name the resolved IP, because a +// classification refusal is reachable by a stranger who picked the hostname and +// would learn what it resolved to inside our network. This error is a different +// animal: it reports that addresses we already accepted could not be connected +// to. The dial-path errors at transport.go:382 and :387 name `addr` for the same +// reason and cycle 1 left them untouched on purpose — see the scope note there +// and the assertion at transport_revetting_test.go:252. +func TestSSRFSafeTransport_DialErrorAccountsForEveryVettedAddress(t *testing.T) { + t.Parallel() + + // A port this test owned and then released, rather than a number written + // down: nothing is listening on it, and it cannot collide with a service a + // developer happens to be running. Both dials below therefore fail, which is + // the only thing the assertions may depend on. + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "binding a throwaway listener to claim a port") + _, port, err := net.SplitHostPort(probe.Addr().String()) + require.NoError(t, err, "splitting the throwaway listener address %q", probe.Addr()) + require.NoError(t, probe.Close(), "releasing the claimed port so nothing is listening on it") + + // Both loopback, in the two families, which is what makes them fail for + // PLATFORM-DEPENDENT reasons — refused where the family is available, + // unreachable where it is not. That is deliberate: the assertions below name + // the addresses and never the operating system's wording, because the wording + // differs across platforms and the addresses do not. + // + // 127.0.0.2 was considered and rejected for the same reason in reverse: Linux + // has all of 127.0.0.0/8 on lo where macOS does not bind 127.0.0.2, so the + // two platforms differ in how long the attempt takes as well as what it says. + first := net.ParseIP("127.0.0.1") + second := net.ParseIP("::1") + require.NotNil(t, first, "the test's own first address must parse") + require.NotNil(t, second, "the test's own second address must parse") + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{ + "multi.test": {first, second}, + }} + + // The hatch is OPEN because both addresses are loopback and classification is + // not what is under test here — the dial loop is. And unlike most tests in + // this package, transport.base is deliberately NOT substituted: the loop + // lives inside the base transport built by the constructor, so replacing it + // would replace the unit under test with the test's own fake. + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed()) + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + resp, err := client.Get("http://" + net.JoinHostPort("multi.test", port) + "/") + if err == nil { + _ = resp.Body.Close() + } + require.Error(t, err, + "nothing is listening on the claimed port, so a request to it must fail against both addresses") + + assert.Contains(t, err.Error(), first.String(), + "the error does not mention %s, the FIRST address the dial loop tried. Its failure was overwritten by "+ + "the next attempt's, so an operator reading this sees one symptom and cannot tell that another "+ + "address was tried at all — which is the ordinary case for any host with both an A and an AAAA "+ + "record; got: %v", first, err) + + assert.Contains(t, err.Error(), second.String(), + "the error does not mention %s, the LAST address the dial loop tried. An aggregate that drops the "+ + "final attempt has traded one blind spot for another; got: %v", second, err) +} diff --git a/internal/atproto/oauth/transport_dial_timeout_aggregate_test.go b/internal/atproto/oauth/transport_dial_timeout_aggregate_test.go new file mode 100644 index 0000000..dc9491f --- /dev/null +++ b/internal/atproto/oauth/transport_dial_timeout_aggregate_test.go @@ -0,0 +1,229 @@ +package oauth + +import ( + "context" + "errors" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeTransport_AnAllTimeoutAggregateStillReportsTimeout is the other +// half of TestSSRFSafeTransport_ASingleFailedAddressKeepsItsTimeoutSignal, and +// it covers the case that is COMMON rather than the case that is rare. +// +// # THE SINGLE-ADDRESS FIX LEFT THE ORDINARY HOST BROKEN +// +// Returning a lone dial error bare preserves Timeout() for a host with one +// address. An ordinary dual-stack host has two — an A record and an AAAA record +// — so it takes the errors.Join branch, and *errors.joinError implements +// Unwrap() []error AND NOTHING ELSE. url.Error.Timeout is a direct type +// assertion for `interface{ Timeout() bool }`, and every retry helper in the +// wild is `if ne, ok := err.(net.Error); ok && ne.Timeout()`. So a caller that +// retries on timeouts stops seeing them exactly when a host is dual-stack, +// which is the common case and not the corner one. +// +// # net.Error IS BOTH METHODS OR IT IS NOTHING +// +// Implementing Timeout() alone would satisfy url.Error and still fail every +// `err.(net.Error)` assertion, because net.Error requires Temporary() too. The +// aggregate therefore answers both, and answers them the only way an aggregate +// honestly can: true when EVERY joined error says true. +func TestSSRFSafeTransport_AnAllTimeoutAggregateStillReportsTimeout(t *testing.T) { + t.Parallel() + + // Two loopback addresses, checked rather than assumed. Two is the premise: + // with one the bare-return branch above already handles it. + first := net.ParseIP("127.0.0.1") + second := net.ParseIP("127.0.0.2") + require.NotNil(t, first, "the test's own first address must parse") + require.NotNil(t, second, "the test's own second address must parse") + + // A port claimed and released: well formed, and nothing is listening. + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "binding a throwaway listener to claim a port") + addr := probe.Addr().String() + require.NoError(t, probe.Close(), "releasing the claimed port so nothing is listening on it") + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + // A deadline ALREADY IN THE PAST makes both dials time out deterministically, + // without waiting for a real timeout and without touching the network. + ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Minute)) + defer cancel() + ctx = context.WithValue(ctx, vettedAddrsKey, []net.IP{first, second}) + + conn, err := transport.base.DialContext(ctx, "tcp", addr) + if conn != nil { + _ = conn.Close() + } + require.Error(t, err, "a dial under an expired deadline must fail on both addresses") + + // Asserted as url.Error does it — a direct type assertion, never errors.As — + // because the information IS still in the chain and reachability by the + // mechanism callers use is the entire property under test. + timeouter, ok := err.(interface{ Timeout() bool }) + assert.Truef(t, ok, + "the aggregated dial error is a %T, which does not implement Timeout() bool. A dual-stack host "+ + "(A + AAAA) takes the aggregation branch, so every caller of this client loses the timeout "+ + "signal on the COMMON shape of host while keeping it on the rare one; got: %v", + err, err) + if ok { + assert.Truef(t, timeouter.Timeout(), + "the aggregate implements Timeout() and reports false although EVERY address failed with a "+ + "deadline error. An aggregate over all-timeouts is a timeout; got: %v", err) + } + + // net.Error is the interface retry and circuit-breaker logic asserts on, and + // it is not satisfied by Timeout() alone. + netErr, ok := err.(net.Error) + assert.Truef(t, ok, + "the aggregated dial error is a %T, which does not satisfy net.Error. `if ne, ok := err.(net.Error); "+ + "ok && ne.Timeout()` is the shape retry logic is written in, and it needs Temporary() as well as "+ + "Timeout(); got: %v", err, err) + if ok { + assert.Truef(t, netErr.Timeout(), + "net.Error.Timeout() is false on an aggregate whose every member timed out; got: %v", err) + } + + // Aggregation's own reason for existing must survive the fix: an operator + // debugging a dual-stack host needs BOTH symptoms, so both errors stay + // reachable through the tree. + var opErr *net.OpError + assert.ErrorAsf(t, err, &opErr, + "the underlying *net.OpError is no longer reachable with errors.As, so the fix traded the "+ + "diagnostic for the signal instead of keeping both; got: %v", err) + assert.ErrorIsf(t, err, context.DeadlineExceeded, + "the expired-deadline cause is no longer reachable in the error chain; got: %v", err) + + unwrapped, ok := err.(interface{ Unwrap() []error }) + assert.Truef(t, ok, + "the aggregate no longer implements Unwrap() []error, so errors.Is/As can no longer walk to the "+ + "per-address failures; got: %T", err) + if ok { + assert.Lenf(t, unwrapped.Unwrap(), 2, + "the aggregate must account for EVERY vetted address that was attempted — two here — so an "+ + "operator sees both symptoms rather than whichever failed last; got: %v", err) + } +} + +// TestJoinDialErrors_ClassifiesTheAggregateFromItsMembers pins the classification +// rule itself, at the one place it can be stated without a network: an aggregate +// is a timeout only when EVERY member is one. +// +// # WHY "ALL" AND NOT "ANY" +// +// The caller's question is "should I retry this the way I retry a timeout?", and +// a host whose IPv6 address timed out while its IPv4 address was REFUSED has +// given a definite answer on one of them. Reporting that as a timeout tells a +// retry loop to keep waiting on a destination that already said no. "All" is the +// only reading under which the aggregate's answer is true of the whole thing it +// aggregates. +func TestJoinDialErrors_ClassifiesTheAggregateFromItsMembers(t *testing.T) { + t.Parallel() + + plain := errors.New("connection refused") + + tests := []struct { + name string + errs []error + wantTimeout bool + wantTemporary bool + }{ + { + name: "every member timed out", + errs: []error{stubNetError{timeout: true}, stubNetError{timeout: true}}, + wantTimeout: true, + wantTemporary: false, + }, + { + name: "one member did not time out", + errs: []error{stubNetError{timeout: true}, stubNetError{}}, + wantTimeout: false, + wantTemporary: false, + }, + { + name: "a member that is not a net.Error at all", + // The honest answer is false: an aggregate cannot claim a property + // of a member that cannot report it. + errs: []error{stubNetError{timeout: true}, plain}, + wantTimeout: false, + wantTemporary: false, + }, + { + name: "every member is temporary", + errs: []error{stubNetError{temporary: true}, stubNetError{temporary: true}}, + wantTimeout: false, + wantTemporary: true, + }, + { + name: "a wrapped timeout still counts", + // errors.As, not a direct assertion, INSIDE the aggregate: a + // *net.OpError carrying a timeout is what the dialer actually + // returns, and the members are ours to inspect properly even though + // our callers cannot. + errs: []error{ + &net.OpError{Op: "dial", Err: stubNetError{timeout: true}}, + stubNetError{timeout: true}, + }, + wantTimeout: true, + wantTemporary: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + joined := joinDialErrors(tt.errs) + require.Error(t, joined, "joining %d errors must produce one", len(tt.errs)) + + netErr, ok := joined.(net.Error) + require.Truef(t, ok, + "the aggregate is a %T and does not satisfy net.Error; retry logic discovers timeouts "+ + "through that interface", joined) + + assert.Equalf(t, tt.wantTimeout, netErr.Timeout(), + "Timeout() on an aggregate of %v", tt.errs) + assert.Equalf(t, tt.wantTemporary, netErr.Temporary(), + "Temporary() on an aggregate of %v", tt.errs) + + // Every member stays reachable however the aggregate classifies. + for _, member := range tt.errs { + assert.ErrorIsf(t, joined, member, + "member %v is no longer reachable through the aggregate", member) + } + }) + } +} + +// TestJoinDialErrors_ReturnsALoneErrorBare re-states the single-address contract +// at the function that now owns it, so a refactor of the dial loop cannot lose +// it without a unit test failing. The end-to-end proof stays in +// TestSSRFSafeTransport_ASingleFailedAddressKeepsItsTimeoutSignal. +func TestJoinDialErrors_ReturnsALoneErrorBare(t *testing.T) { + t.Parallel() + + only := stubNetError{timeout: true} + joined := joinDialErrors([]error{only}) + + assert.Equal(t, error(only), joined, + "a single dial failure must be returned EXACTLY as it arrived. Wrapping it is aggregation with "+ + "nothing to aggregate, and any wrapper is one more thing between a caller and the concrete "+ + "error type it may be matching on") +} + +// stubNetError is a net.Error whose two answers are set by the test. +type stubNetError struct { + timeout bool + temporary bool +} + +func (e stubNetError) Error() string { return "stub net error" } +func (e stubNetError) Timeout() bool { return e.timeout } +func (e stubNetError) Temporary() bool { return e.temporary } diff --git a/internal/atproto/oauth/transport_dial_timeout_test.go b/internal/atproto/oauth/transport_dial_timeout_test.go new file mode 100644 index 0000000..d418d31 --- /dev/null +++ b/internal/atproto/oauth/transport_dial_timeout_test.go @@ -0,0 +1,107 @@ +package oauth + +import ( + "context" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeTransport_ASingleFailedAddressKeepsItsTimeoutSignal pins that +// aggregating one error is not aggregating at all. +// +// # WHAT errors.Join COSTS HERE +// +// url.Error.Timeout — the method every caller of this client reaches, because +// http.Client wraps every RoundTrip error in a *url.Error — is implemented as a +// DIRECT TYPE ASSERTION, not errors.As: +// +// func (e *Error) Timeout() bool { +// t, ok := e.Err.(interface{ Timeout() bool }) +// return ok && t.Timeout() +// } +// +// net.Error is discovered the same way at every call site that matters: +// `if ne, ok := err.(net.Error); ok && ne.Timeout()`. errors.Join returns a +// *errors.joinError, which implements Unwrap() []error and nothing else — so +// wrapping a single *net.OpError in one severs Timeout() even though the +// information is still in the chain. Verified: the joined error does not +// implement the interface at all. +// +// # WHY THE SINGLE-ADDRESS CASE IS THE ONE TO FIX +// +// Aggregation exists so an operator debugging a host with both an A and an AAAA +// record can see both failures — that is +// TestSSRFSafeTransport_DialErrorAccountsForEveryVettedAddress, and it must go +// on passing. With ONE address there is nothing to aggregate: the join adds no +// information and costs the timeout signal, which is the signal retry and +// circuit-breaker logic is built on. A caller that cannot tell a timeout from a +// refusal either retries what it should not or gives up on what it should. +func TestSSRFSafeTransport_ASingleFailedAddressKeepsItsTimeoutSignal(t *testing.T) { + t.Parallel() + + // Checked, not assumed. + loopback := net.ParseIP("127.0.0.1") + require.NotNil(t, loopback, "the test's own address must parse") + + // A port claimed and released: well formed, and nothing is listening. + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "binding a throwaway listener to claim a port") + addr := probe.Addr().String() + require.NoError(t, probe.Close(), "releasing the claimed port so nothing is listening on it") + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + // A deadline ALREADY IN THE PAST, which is how a timeout is produced without + // waiting for one. net.Dialer checks the context before it does anything + // else and returns a *net.OpError carrying os.ErrDeadlineExceeded, whose + // Timeout() reports true — the same error shape a real dial timeout + // produces, arrived at deterministically. + ctx, cancel := context.WithDeadline(t.Context(), time.Now().Add(-time.Minute)) + defer cancel() + + // Exactly ONE vetted address. That is the whole premise: with one address + // the join has nothing to combine. + ctx = context.WithValue(ctx, vettedAddrsKey, []net.IP{loopback}) + + conn, err := transport.base.DialContext(ctx, "tcp", addr) + if conn != nil { + _ = conn.Close() + } + require.Error(t, err, "a dial under an expired deadline must fail") + + // Asserted the way url.Error and every retry helper in the wild does it: a + // direct type assertion, not errors.As. Using errors.As here would pass + // against the broken implementation and prove nothing, because the + // information IS still in the chain — what is lost is its reachability by + // the mechanism callers actually use. + timeouter, ok := err.(interface{ Timeout() bool }) + assert.Truef(t, ok, + "the dial error is a %T, which does not implement Timeout() bool. url.Error.Timeout uses a direct "+ + "type assertion rather than errors.As, so every caller of this client — they all go through "+ + "http.Client, which wraps RoundTrip errors in *url.Error — now sees a timeout as an ordinary "+ + "failure. With a single vetted address the aggregation buys nothing and costs exactly this; got: %v", + err, err) + if ok { + assert.True(t, timeouter.Timeout(), + "the dial error implements Timeout() and reports false for a dial that expired against its "+ + "deadline; got: %v", err) + } + + // The detail must survive as well as the signal: a fix that replaced the + // join with a bare "dial failed" string would satisfy the assertion above + // and lose what an operator reads. + var opErr *net.OpError + assert.ErrorAsf(t, err, &opErr, + "the underlying *net.OpError is no longer reachable with errors.As, so the fix traded the timeout "+ + "signal for the diagnostic instead of keeping both; got: %v", err) + assert.ErrorIsf(t, err, context.DeadlineExceeded, + "the expired-deadline cause is no longer reachable in the error chain. net's timeout error reports "+ + "itself as context.DeadlineExceeded, and a fix that flattens the dial failures into a string "+ + "would lose that as well as Timeout(); got: %v", err) +} diff --git a/internal/atproto/oauth/transport_embedded_ipv4_test.go b/internal/atproto/oauth/transport_embedded_ipv4_test.go index 67fe821..0698a5a 100644 --- a/internal/atproto/oauth/transport_embedded_ipv4_test.go +++ b/internal/atproto/oauth/transport_embedded_ipv4_test.go @@ -59,9 +59,10 @@ import ( // ban. That would be an outage. `64:ff9b::/96` is a legitimate connect() // destination: an IPv6-only host with DNS64 reaches every IPv4-only server in the // world through it, so on an IPv6-only deployment a wholesale ban does not block -// some outbound federation, it blocks ALL of it. `64:ff9b:1::/48` (RFC 8215) is -// the same mechanism with a locally-chosen prefix and carries the same -// consequence. +// some outbound federation, it blocks ALL of it. `64:ff9b:1::/48` (RFC 8215) +// differs in the security-relevant way: it is local-use space and RFC 6052 +// permits several payload layouts beneath it. Without a configured Pref64 the +// guard cannot decode it soundly, so the secure default is to block that /48. // // The asymmetry, stated plainly, is the thing to remember about this file: // **6to4 embeds a gateway, so we ban it. NAT64 embeds the destination, so we @@ -69,11 +70,11 @@ import ( // // # WHAT THE ALLOWED ROWS ARE FOR // -// Every remaining allowed row embeds the public 8.8.8.8 under a prefix that is -// still decoded. They are what stops the 6to4 decision from being generalised -// into "ban every prefix": a wholesale ban of NAT64 or SIIT turns all their -// blocked rows green and these red. The blocked rows prove the extraction -// happens; the allowed rows prove it is a decode and not a ban. +// Every remaining allowed row embeds the public 8.8.8.8 under a globally +// reachable prefix that is still decoded. They are what stops the fail-closed +// local-use decision from being generalised into "ban every translation +// prefix": a wholesale ban of well-known NAT64 or SIIT turns all their blocked +// rows green and these red. // // # DELIBERATE EXCLUSIONS // @@ -109,13 +110,16 @@ func TestIsPrivateIP_IPv6FormsEmbeddingIPv4(t *testing.T) { {"NAT64 well-known embedding RFC1918 10/8", "64:ff9b::a00:1", true}, {"NAT64 well-known embedding a public address", "64:ff9b::808:808", false}, - // NAT64 local-use prefix, 64:ff9b:1::/48 (RFC 8215). Same semantics as the - // well-known prefix and a separate range: an implementation matching only - // 64:ff9b::/96 lets every one of these through, and the metadata-service row - // is what that costs. + // NAT64 local-use prefix, 64:ff9b:1::/48 (RFC 8215). RFC 6052 allows + // operators to allocate multiple Pref64 lengths beneath this reservation, + // moving the embedded IPv4 field. With no configured Pref64 there is no + // sound payload offset, so every layout fails closed. {"NAT64 local-use embedding loopback", "64:ff9b:1::7f00:1", true}, {"NAT64 local-use embedding the metadata service", "64:ff9b:1::a9fe:a9fe", true}, - {"NAT64 local-use embedding a public address", "64:ff9b:1::808:808", false}, + {"NAT64 local-use /96 layout with a public payload", "64:ff9b:1::808:808", true}, + {"NAT64 local-use custom /96 embedding loopback", "64:ff9b:1:abcd::7f00:1", true}, + {"NAT64 local-use custom layout with opaque payload", "64:ff9b:1:abcd:1234:5678::1", true}, + {"Just above NAT64 local-use", "64:ff9b:2::1", false}, // SIIT IPv4-translated, ::ffff:0:0:0/96 — bytes 0-7 zero, 8-9 ffff, 10-11 // zero, IPv4 in the last 4. Decoded, not banned: like NAT64, the embedded diff --git a/internal/atproto/oauth/transport_empty_answer_test.go b/internal/atproto/oauth/transport_empty_answer_test.go new file mode 100644 index 0000000..c56872c --- /dev/null +++ b/internal/atproto/oauth/transport_empty_answer_test.go @@ -0,0 +1,92 @@ +package oauth + +import ( + "context" + "net" + "net/http" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeHTTPClient_RefusesAnEmptyLookupAnswer covers RoundTrip's +// `len(ips) == 0` guard, which had none. +// +// # WHY A GUARD WITH NO COVERAGE MATTERS HERE MORE THAN USUAL +// +// It is not an isolated branch. The dial loop twenty lines further down +// documents THIS guard as the reason its own fail-closed check is latent +// ("an empty vetted slice cannot reach here through the public API, because the +// guard above refuses it"). So the two are coupled: delete this guard and an +// empty answer reaches a dial with nothing vetted, where errors.Join of nothing +// would be nil — a connection that does not exist and no error explaining it, +// which is not a result net/http is written to survive. Two branches, one of +// them dead by assumption, and until this test neither had a line of coverage. +// +// # WHERE AN EMPTY, NON-ERROR ANSWER COMES FROM +// +// It is not hypothetical. A resolver answering NOERROR with no records — a +// name with only CNAME or TXT data, a split-horizon server, a filtering +// resolver stripping AAAA — returns success and nothing. net.LookupIPAddr +// normally converts that to an error, but this transport's lookup is a FIELD: +// dev_resolver.go substitutes it, tests substitute it, and any future +// substitution is one `return nil, nil` away from handing RoundTrip an empty +// slice. +// +// # WHAT IS ASSERTED, AND WHAT IS DELIBERATELY NOT +// +// That the request fails and NOTHING IS DIALLED. Not that it matches +// ErrBlockedAddress: an answer with no addresses is a resolution outcome, not a +// classification, and transport_blocked_error_test.go:113 pins the rule that a +// resolution failure must not report itself as a block. Deciding otherwise here +// would quietly widen the sentinel to mean "something went wrong". +func TestSSRFSafeHTTPClient_RefusesAnEmptyLookupAnswer(t *testing.T) { + t.Parallel() + + const host = "answers-nothing.test" + + // A NON-NIL, EMPTY slice with a NIL error: success, with no addresses. A nil + // slice would be the same to len() but this spelling says what is being + // modelled — the resolver answered, and the answer was empty. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{host: {}}} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + // The base transport is substituted by a recording dialler, and that choice + // is what makes this test discriminating. The constructor's own base + // contains the fail-closed guard, so leaving it in place would mean an empty + // answer is refused there instead — and the test would pass with RoundTrip's + // guard deleted. Replacing it removes the second net, so `dialled` reports + // on RoundTrip alone. + var dialled atomic.Bool + transport.base = &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + dialled.Store(true) + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: net.UnknownNetworkError(addr)} + }, + } + + resp, err := client.Get("http://" + host + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, + "a lookup that answered with no addresses must fail the request. There is nothing to connect to and "+ + "nothing to classify, so continuing means dialling a destination the guard never saw") + + assert.False(t, dialled.Load(), + "the transport dialled after a lookup returned no addresses. Nothing was vetted, so whatever the "+ + "dialler connected to was chosen by the address string rather than by the guard — which is the "+ + "check-then-dial window this design closes") + + assert.NotErrorIs(t, err, ErrBlockedAddress, + "an empty lookup answer reported itself as a blocked address. It is a resolution outcome, not a "+ + "classification: transport_blocked_error_test.go pins that the sentinel must mean 'the guard "+ + "refused a destination' and not 'the request failed'; got: %v", err) +} diff --git a/internal/atproto/oauth/transport_gate_helper_test.go b/internal/atproto/oauth/transport_gate_helper_test.go new file mode 100644 index 0000000..49a9ab1 --- /dev/null +++ b/internal/atproto/oauth/transport_gate_helper_test.go @@ -0,0 +1,212 @@ +package oauth + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// PrivateAddressOptions is the gate: the one place in this tree that decides, +// from a boolean a caller is holding, whether a client gets the hatch option or +// nothing at all. Seven production call sites hold such a boolean +// (api.allowPrivateHost, s.allowPrivateHosts, config.AllowPrivateIPs, +// allowPrivateIPs) and each is about to route through here. +// +// # WHY THIS IS A FUNCTION AND NOT AN `if` IN WIRING +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// T0+T1+T2 — runs the PERMISSIVE branch at every one of those call sites. A +// green merge gate therefore cannot prove that production is guarded: the +// guarded branch is evaluated in exactly one place in this repository, and that +// place is this file. +// +// An inline `if cfg.IsDevEnv { ... }` in wiring would be reachable only by +// standing up wiring with a production config, and nothing in this tree does +// that. Pulled out as a pure function, the decision is testable without wiring, +// without a config, and without an environment — which is the only way the +// production branch gets tested at all. + +// TestPrivateAddressOptions_ReturnsZeroOptionsWhenPrivateAddressesAreDisallowed +// is the key production-polarity assertion for this helper. +// +// The claim is not "the returned options are safe". It is that there ARE no +// returned options: length zero, nothing to apply, the constructor's own struct +// literal left untouched. That distinction is what makes the guarantee +// auditable. "Safe options" is a property of whatever the slice happens to hold +// today and has to be re-argued every time the slice changes; "no options" is a +// property a reader can check in one glance and a test can pin exactly. +// +// It is written as an exact length and not as a behavioural check on purpose. A +// later edit that appends a diagnostic option, or that returns a one-element +// slice holding a no-op "explicitly deny" closure, would keep every behavioural +// test in this file green — and would move the production branch from "provably +// applies nothing" to "applies something we believe is harmless", which is a +// different and much weaker claim about the only branch CI never runs. +// +// So: if this assertion is ever in the way, the answer is not to relax it. +func TestPrivateAddressOptions_ReturnsZeroOptionsWhenPrivateAddressesAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateAddressOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateAddressOptions(false) returned %d option(s). The production branch — the one "+ + "IS_DEV_ENV=true keeps `make ci` from ever evaluating — must contribute NOTHING to the "+ + "constructor, so that what production gets is exactly the constructor's own defaults. An "+ + "option here that is believed harmless is not the same guarantee as no option at all, and "+ + "nothing downstream of this line would catch the difference", len(opts)) +} + +// TestPrivateAddressOptions_DisallowedClientIsGuarded is the behavioural half of +// the assertion above: zero options has to also MEAN a guarded client. +// +// The length check alone would still pass if the constructor's defaults ever +// regressed to permissive — the helper would be returning nothing, correctly, +// onto a base that no longer refuses anything. Both halves together say: the +// helper adds nothing, and nothing is what keeps the guard on. +// +// Every row of privateHatchGates runs, because allowPrivate gates three separate +// refusals in RoundTrip (the IP-literal check, the zoned-literal check inside it, +// and the classification pass over the resolved answers). A helper that somehow +// re-opened one of the three would be invisible to a single-row test. +func TestPrivateAddressOptions_DisallowedClientIsGuarded(t *testing.T) { + t.Parallel() + + for _, tt := range privateHatchGates { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newHatchProbe(t, tt.hostname, tt.resolves, PrivateAddressOptions(false)...) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Errorf(t, err, + "GET http://%s/ succeeded on a client built from PrivateAddressOptions(false). %s", + tt.urlHost, tt.why) + assert.ErrorIsf(t, err, ErrBlockedAddress, + "the refusal must be the guard's, matchable by identity: a request that failed for some "+ + "other reason is not the same control and would not hold in production; got: %v", err) + + assert.Zerof(t, probe.invocations.Load(), + "the listener was reached %d times for http://%s/. This probe's dialler sends every "+ + "connection to that one server whatever address it was handed, so any invocation means "+ + "the packet left the transport — and for a destination a stranger named, the packet "+ + "leaving IS the SSRF, whatever error came back afterwards", + probe.invocations.Load(), tt.urlHost) + }) + } +} + +// TestPrivateAddressOptions_AllowedClientOpensEveryGate pins the other +// direction, and pins it through OBSERVED BEHAVIOUR rather than through the +// shape of the returned slice. +// +// A length check here would be worthless: `[]Option{WithMaxResponseBytes(1024)}` +// has length one just as `[]Option{WithPrivateAddressesAllowed()}` does, and a +// helper that returned the wrong option would satisfy it while leaving every dev +// environment and every httptest fixture in this tree unable to reach loopback. +// So the assertion is that a client built from these options actually reaches a +// listener at an address the guard would otherwise refuse. +// +// All three rows again, for the same reason as the guarded direction: the hatch +// has to open all three gates, or the migrated call sites lose part of their dev +// hatch in a way that only shows up on a developer's machine. +func TestPrivateAddressOptions_AllowedClientOpensEveryGate(t *testing.T) { + t.Parallel() + + for _, tt := range privateHatchGates { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newHatchProbe(t, tt.hostname, tt.resolves, PrivateAddressOptions(true)...) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + defer func() { _ = resp.Body.Close() }() + } + + require.NoErrorf(t, err, + "GET http://%s/ was refused on a client built from PrivateAddressOptions(true). The "+ + "permissive branch is what every developer and every fixture in this tree runs, so a "+ + "helper that returns the wrong option — or none — breaks local development everywhere "+ + "at once — %s", tt.urlHost, tt.why) + assert.Equalf(t, http.StatusOK, resp.StatusCode, + "GET http://%s/ reached the listener but did not complete", tt.urlHost) + + assert.Equalf(t, int64(1), probe.invocations.Load(), + "the listener was reached %d times for http://%s/. Every connection this probe makes "+ + "lands there regardless of destination, so anything but exactly one means the request "+ + "never got out of the transport", probe.invocations.Load(), tt.urlHost) + }) + } +} + +// TestPrivateAddressOptions_SpreadsAlongsideOtherOptions pins that the helper's +// result is usable the way call sites will actually use it — spread into a +// constructor that is also passing settings of its own — and that neither +// setting eats the other. +// +// This is not hypothetical composition. The image proxy has an +// operator-configurable size limit (IMAGE_PROXY_MAX_SOURCE_SIZE_MB) that has to +// arrive as WithMaxResponseBytes, and it is one of the call sites holding an +// allow-private boolean. It will pass both. Options are applied by iterating a +// slice of closures over one transport struct, so an implementation that +// returned a whole constructor-worth of options — rather than only the ones its +// own concern owns — would silently reset the cap to the package default and the +// operator's setting would vanish with no error anywhere. +// +// Both effects are asserted in one request: the listener is reached (so the +// hatch is open at an address the guard would otherwise refuse) AND the read +// fails with ErrResponseTooLarge (so the caller's cap, not the 32 MiB default, +// is what bounded it). +func TestPrivateAddressOptions_SpreadsAlongsideOtherOptions(t *testing.T) { + t.Parallel() + + // Comfortably under the body below and far under DefaultMaxResponseBytes, so + // a failure to apply this option cannot be mistaken for the default doing the + // work: at the default, 4 KiB is not remarkable. + const capBytes = 64 + body := bytes.Repeat([]byte("a"), 4096) + + // The private row: the hatch has to be open for this request to leave the + // transport at all, so reaching the listener is itself the evidence that + // PrivateAddressOptions(true) survived being spread next to another option. + const ( + hostname = "private.test" + resolves = "127.0.0.1" + ) + + opts := append(PrivateAddressOptions(true), WithMaxResponseBytes(capBytes)) + probe := newHatchProbeWithBody(t, hostname, resolves, body, opts...) + + resp, err := probe.client.Get("http://" + hostname + "/") + if err == nil { + // The cap has two enforcement points — a declared Content-Length larger + // than the allowance is refused in RoundTrip, and anything else is caught + // by the body wrapper mid-read — and which one fires depends on whether + // the test server chose to declare a length. Reading through covers both + // without pinning an implementation detail neither the caller nor this + // test has an opinion about. + _, err = io.ReadAll(resp.Body) + _ = resp.Body.Close() + } + + assert.Equalf(t, int64(1), probe.invocations.Load(), + "the listener was reached %d times. The address is loopback, so the request only leaves the "+ + "transport with the hatch open: anything but exactly one means WithMaxResponseBytes "+ + "displaced the option PrivateAddressOptions(true) contributed", + probe.invocations.Load()) + + require.ErrorIsf(t, err, ErrResponseTooLarge, + "a %d-byte body came back whole under a %d-byte cap. The caller's cap was passed alongside "+ + "PrivateAddressOptions(true) and has to survive it — an image proxy whose operator raises "+ + "IMAGE_PROXY_MAX_SOURCE_SIZE_MB, or lowers it, gets no error when the setting is silently "+ + "replaced by the package default; got: %v", len(body), capBytes, err) +} diff --git a/internal/atproto/oauth/transport_idn_host_test.go b/internal/atproto/oauth/transport_idn_host_test.go new file mode 100644 index 0000000..7146328 --- /dev/null +++ b/internal/atproto/oauth/transport_idn_host_test.go @@ -0,0 +1,607 @@ +package oauth + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/idna" +) + +// The guard resolves the hostname net/http would NEVER have dialled. +// +// # THE DEFECT +// +// RoundTrip takes req.URL.Hostname() raw and hands that exact string to the +// resolver. net/http does not: canonicalAddr runs the host through +// idnaASCIIFromURL → idna.Lookup.ToASCII before it becomes a dial address, a +// connection-pool key or a TLS ServerName (net/http/transport.go's +// canonicalAddr, and idnaASCII in net/http/request.go). +// +// So for https://bücher.example/xrpc/... the guard asks for the literal UTF-8 +// string. On the PRODUCTION build (CGO_ENABLED=0) the pure-Go resolver refuses +// it inside the process: goLookupIPCNAMEOrder gates on isDomainName, which +// permits only [A-Za-z0-9._-], so a byte ≥ 0x80 falls through to its default +// case. Verified by probe — `lookup bücher.example: no such host` comes back +// with NO server named, while the A-label spelling names the resolver it +// actually queried. +// +// The consequence is an availability regression rather than a bypass: every +// atProto PDS on a non-ASCII domain became unreachable from this AppView at +// every site that adopted this client, and it fails closed, so nothing in CI +// noticed. No fixture in this tree uses an IDN host. +// +// # WHY THE FIX IS ORDERED THE WAY IT IS +// +// Normalization runs BEFORE the IP-literal shape check, and that ordering is +// security-critical rather than tidy — see +// TestSSRFSafeHTTPClient_NormalizesBeforeTheLiteralCheck, which is the row that +// proves it. IDNA does not merely punycode: its mapping table folds fullwidth +// and ideographic forms to ASCII, so a host that is not an IP literal on the way +// in can BE one on the way out. +// +// # WHY THE ASCII SHORT-CIRCUIT IS NOT AN OPTIMISATION +// +// TestSSRFSafeHTTPClient_ASCIIHostsAreUnaffectedByNormalization. The Lookup +// profile applies ValidateLabels, CheckHyphens and the BidiRule, and refuses +// ASCII hostnames that Go's own resolver resolves happily. Running every host +// through ToASCII would trade this availability bug for a wider one. + +// dialRecorder stands in for the base transport and records the address +// net/http computed for the dial. +// +// That address is canonicalAddr's output, which is the SAME string net/http +// uses as the connection-pool key and — with the port stripped — as the TLS +// ServerName. So recording it here is how a test asks "is the name the guard +// vetted the name net/http would have connected to", which is the question the +// whole normalization change exists to answer yes to. +// +// It always refuses the connection, so nothing here can leave the machine even +// if every guard above it were deleted. +type dialRecorder struct { + mu sync.Mutex + addrs []string +} + +func (d *dialRecorder) transport() *http.Transport { + return &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + d.mu.Lock() + d.addrs = append(d.addrs, addr) + d.mu.Unlock() + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: net.UnknownNetworkError(addr)} + }, + } +} + +func (d *dialRecorder) dialled() []string { + d.mu.Lock() + defer d.mu.Unlock() + return slices.Clone(d.addrs) +} + +// dialledHosts returns the recorded addresses with their ports stripped, which +// is the form to compare against what the resolver was asked. +func (d *dialRecorder) dialledHosts(t *testing.T) []string { + t.Helper() + hosts := make([]string, 0, len(d.addrs)) + for _, addr := range d.dialled() { + host, _, err := net.SplitHostPort(addr) + require.NoErrorf(t, err, "net/http handed the dialler %q, which carries no port", addr) + hosts = append(hosts, host) + } + return hosts +} + +// TestSSRFSafeHTTPClient_ResolvesTheALabelOfAnIDNHost is the regression +// reproducing. +// +// The assertion that carries the weight is the one about what the RESOLVER WAS +// ASKED, not the one about whether the request succeeded. A test that only +// checked for an error would pass against the broken transport, since the +// broken transport does produce an error — the wrong one, for the wrong reason, +// at the wrong layer. +func TestSSRFSafeHTTPClient_ResolvesTheALabelOfAnIDNHost(t *testing.T) { + t.Parallel() + + const unicodeHost = "bücher.example" + const aLabel = "xn--bcher-kva.example" + + // The premise, checked rather than assumed: if x/net/idna ever stopped + // producing this A-label the test would be demanding the wrong string, and + // would say so here rather than failing obscurely below. + got, err := idna.Lookup.ToASCII(unicodeHost) + require.NoError(t, err, "the Lookup profile must accept an ordinary IDN host") + require.Equal(t, aLabel, got, "this is the A-label net/http computes for the same host") + + // A public answer, so the request survives classification and the test is + // measuring normalization rather than the guard refusing for its own + // reasons. It answers ONLY for the A-label: a transport that asks for the + // raw UTF-8 string gets a DNS failure, which is exactly what production + // gets. + publicAnswer := net.ParseIP("93.184.216.34") + require.NotNil(t, publicAnswer, "the fixture address must parse; a nil IP classifies as public and greens this test for nothing") + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{aLabel: {publicAnswer}}} + recorder := &dialRecorder{} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + transport.base = recorder.transport() + + resp, err := client.Get("http://" + unicodeHost + "/xrpc/com.atproto.repo.getRecord") + if err == nil { + _ = resp.Body.Close() + } + + // THE DISCRIMINATOR. + assert.Equal(t, []string{aLabel}, resolver.hostsAsked(), + "the guard asked the resolver for the hostname verbatim. net/http punycodes it before the string "+ + "becomes a dial address, so on the production build (CGO_ENABLED=0) the pure-Go resolver refuses the "+ + "raw UTF-8 inside the process and every PDS on a non-ASCII domain is unreachable through this client") + + // And it must have got PAST the guard rather than merely reaching it: a + // normalization that resolved the A-label and then refused it would be the + // same outage wearing a different error. + assert.NotErrorIs(t, err, ErrBlockedAddress, + "an IDN host resolving to a public address must not be refused; got: %v", err) + assert.Equal(t, []string{aLabel}, recorder.dialledHosts(t), + "the request must reach the dial, at the A-label, having been classified on the way") +} + +// TestSSRFSafeHTTPClient_StillRefusesAnIDNHostResolvingToAPrivateAddress is the +// security-critical half: normalization must feed the classifier, never replace +// it. +// +// The listener is REAL and bound, and the assertion is that its handler was +// never invoked. Mutation testing on this package has already produced an +// implementation that classified correctly, rendered a byte-identical error and +// refused the request AFTER delivering it; every message assertion passed +// against it and only "was it actually reached" caught it. +func TestSSRFSafeHTTPClient_StillRefusesAnIDNHostResolvingToAPrivateAddress(t *testing.T) { + t.Parallel() + + const unicodeHost = "bücher.example" + const aLabel = "xn--bcher-kva.example" + + var reached bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached = true + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + listenerHost, listenerPort, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + require.NoError(t, err, "splitting the test server address") + loopback := net.ParseIP(listenerHost) + require.NotNil(t, loopback, "the test server address %q must parse as an IP", listenerHost) + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{aLabel: {loopback}}} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + // The IDN host at the REAL listener's port, so a transport that resolved + // the A-label and failed to classify it would genuinely connect. + resp, err := client.Get("http://" + unicodeHost + ":" + listenerPort + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "an IDN host resolving to loopback must be refused") + assert.ErrorIs(t, err, ErrBlockedAddress, + "the refusal must carry the guard's identity. A resolution failure that happens to stop the request "+ + "is a different control and does not hold once the name resolves; got: %v", err) + + // Proves the refusal is the classifier acting on the NORMALIZED name rather + // than the resolver choking on the raw one — without this the test above + // passes against the unfixed transport for entirely the wrong reason. + assert.Equal(t, []string{aLabel}, resolver.hostsAsked(), + "the classifier must have been handed the A-label") + + assert.False(t, reached, + "the request reached the listener. Normalizing an IDN host must not become a way to reach a private "+ + "address — the whole point of doing it before vetting is that vetting still applies to what it produces") +} + +// TestSSRFSafeHTTPClient_NormalizesBeforeTheLiteralCheck pins the ORDERING, and +// it is the row that makes the ordering a security question rather than a +// stylistic one. +// +// # IDNA MAPPING PRODUCES IP LITERALS +// +// The Lookup profile maps before it validates, and its mapping table folds +// fullwidth digits (U+FF10..U+FF19) and the ideographic full stop (U+3002) onto +// their ASCII equivalents. Verified by probe: +// +// url.Parse("http://１２７。０。０。１:8080/").Hostname() == "１２７。０。０。１" +// netip.ParseAddr of that → error, not a literal +// idna.Lookup.ToASCII of that → "127.0.0.1" +// +// So a host that is NOT an IP literal when the shape check would see it becomes +// one after normalization. Put normalization after the literal check and the +// check inspects a string that is not yet the address it is about to become. +// +// # WHY THE PUBLIC ROW IS THE ONE THAT MATTERS +// +// For the loopback spelling, classification is a backstop: normalize late and +// the resolver is handed "127.0.0.1", which resolves to itself and is refused a +// few lines further down. The wrong answer, at the wrong layer, but refused. +// +// ８.８.８.８ has no backstop. It maps to a PUBLIC address, so classification +// cannot refuse it and the literal check is the only control there is — and the +// literal check exists precisely because a caller-supplied literal naming a +// public address is still a destination this AppView has no business reaching. +// Normalize late and that check is bypassable by respelling the digits. +// +// # THIS IS NOT LIVE TODAY +// +// It is a trap in the FIX, not a pre-existing hole: with no normalization at +// all the raw fullwidth string goes to the resolver and fails closed on both +// resolver modes (verified on pure-Go and cgo). What this test forbids is +// converting that closed failure into an open one while fixing the IDN outage. +func TestSSRFSafeHTTPClient_NormalizesBeforeTheLiteralCheck(t *testing.T) { + t.Parallel() + + // A real listener, addressed in fullwidth digits, so the private row is a + // reachability claim and not a shape claim. + var reached bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached = true + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + _, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) + require.NoError(t, err, "splitting the test server address") + + tests := []struct { + name string + host string + port string + maps string + why string + }{ + { + name: "fullwidth loopback, in front of a real listener", + host: "１２７。０。０。１", + port: port, + maps: "127.0.0.1", + why: "classification would eventually refuse this one, but only after an attacker-supplied address " + + "had been sent to a resolver — the step the literal check exists to skip", + }, + { + name: "fullwidth public literal", + host: "８.８.８.８", + port: "80", + maps: "8.8.8.8", + why: "PUBLIC, so classification cannot refuse it and the literal check is the only control. This is " + + "the row where normalizing after the shape check has consequences", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // The premise: not a literal as it arrives, a literal once mapped. + // If either half stopped holding this test would be pinning + // something that is no longer true. + mapped, mapErr := idna.Lookup.ToASCII(tt.host) + require.NoError(t, mapErr, "IDNA must map %q rather than refuse it", tt.host) + require.Equal(t, tt.maps, mapped, "IDNA must fold %q onto an IP literal", tt.host) + + // An empty answer table: anything that reaches the resolver dies + // there, so the assertion below is about what was ASKED and never + // about whether the request failed. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{}} + recorder := &dialRecorder{} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + transport.base = recorder.transport() + + resp, err := client.Get("http://" + tt.host + ":" + tt.port + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "a fullwidth spelling of an IP literal must be refused") + assert.ErrorIsf(t, err, ErrBlockedAddress, + "the refusal must be the literal check's, matchable by identity. %s; got: %v", tt.why, err) + assert.Containsf(t, err.Error(), "is an address written where a hostname belongs", + "the refusal must be the LITERAL wording, which is what says the check fired before resolution "+ + "rather than the classifier catching it afterwards; got: %v", err) + + // THE DISCRIMINATOR between the two orderings. A transport that + // normalizes after the shape check hands the mapped literal to the + // resolver; one that normalizes before it never asks at all. + assert.Emptyf(t, resolver.hostsAsked(), + "the transport sent %v to the resolver. %q maps to %s, which is an address written where a "+ + "hostname belongs, and it must be refused on shape before anything is resolved", + resolver.hostsAsked(), tt.host, tt.maps) + + assert.Emptyf(t, recorder.dialled(), + "the request reached the dialler at %v; the destination was chosen by whoever supplied the URL", + recorder.dialled()) + }) + } + + assert.False(t, reached, + "a fullwidth spelling of the loopback literal reached the listener") +} + +// TestSSRFSafeHTTPClient_RefusesAHostItCannotNormalize covers the error branch, +// and the thing it pins is WHOSE error comes back. +// +// net/http's own idnaASCIIFromURL swallows the failure and falls back to the raw +// host, which is the right call there and the wrong one here: a guard that +// cannot determine the name it is about to vet must not proceed to vet +// something else. Failing closed is the easy half. The half worth a test is that +// the refusal carries ErrBlockedAddress, so a caller can tell "the guard refused +// this" from "DNS is having a bad day" — the two want opposite handling, and a +// name that cannot be normalized is not a transient condition to retry. +func TestSSRFSafeHTTPClient_RefusesAHostItCannotNormalize(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + host string + why string + }{ + { + name: "a label the CONTEXTJ rules refuse", + host: "a‍b.example", + why: "a ZERO WIDTH JOINER outside the contexts that permit one", + }, + { + name: "a name the BidiRule refuses", + host: "1ا.example", + why: "an ASCII digit leading a right-to-left label", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // The premise: this host is one ToASCII genuinely refuses. Without + // this the test could pass against a transport that never + // normalizes, simply because the name does not resolve. + _, mapErr := idna.Lookup.ToASCII(tt.host) + require.Errorf(t, mapErr, "the Lookup profile must refuse %q (%s), or this row pins nothing", tt.host, tt.why) + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{}} + recorder := &dialRecorder{} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + transport.base = recorder.transport() + + resp, err := client.Get("http://" + tt.host + "/xrpc/x") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "a host that cannot be normalized must be refused") + assert.ErrorIsf(t, err, ErrBlockedAddress, + "the refusal must name the guard that made it. Falling back to the raw host the way "+ + "net/http does would vet a string that is not the name anything would dial, and reporting "+ + "this as a resolution failure tells a caller to retry a name that can never work; got: %v", err) + + // A resolution failure is what this MUST NOT look like, asserted at + // the resolver rather than by matching on the error's shape: if the + // resolver was never asked, the failure cannot have come from it. + assert.Emptyf(t, resolver.hostsAsked(), + "the transport sent %v to the resolver after failing to normalize it", resolver.hostsAsked()) + assert.Emptyf(t, recorder.dialled(), "the request reached the dialler at %v", recorder.dialled()) + }) + } +} + +// TestSSRFSafeHTTPClient_ASCIIHostsAreUnaffectedByNormalization is the test that +// pins the ASCII SHORT-CIRCUIT, and it is the reason the fix is four lines +// rather than two. +// +// # THE TRAP +// +// The obvious implementation runs every host through idna.Lookup.ToASCII. The +// Lookup profile is not a punycode encoder — it applies ValidateLabels, +// CheckHyphens and the BidiRule — so it REFUSES ASCII hostnames that Go's own +// resolver resolves and that a bare http.Client reaches today. Doing that would +// trade an availability bug for a wider one. +// +// net/http does not have this problem because idnaASCII short-circuits on +// ascii.Is before it ever calls ToASCII, and the TODO above that check says in +// as many words that skipping it "may be possible to have two IDNs that appear +// identical to the user where the ASCII-only version causes an error +// downstream". The short-circuit is what keeps the ASCII path byte-identical to +// what this AppView does today. +// +// # THE ROWS ARE REAL, NOT CONSTRUCTED +// +// Both are names Go's pure-Go resolver — the production one — accepts and +// QUERIES. Verified by probe against a live resolver: each came back naming the +// DNS server it was sent to, which is what distinguishes "asked and answered no" +// from "rejected inside the process" the way the raw UTF-8 host is. +// +// - An underscore label. isDomainName permits '_' deliberately, citing +// golang.org/issue/12421 and SRV-style names, and _atproto is the label +// atProto's own handle resolution is built on. UTS#46 disallows U+005F +// outright. +// - Hyphens in the third and fourth positions. Ordinary, RFC-valid, resolvable +// DNS, and exactly the shape CheckHyphens rejects because it is where a +// punycode prefix lives. +// +// The leading- and trailing-hyphen spellings (-foo.example.com, foo-.example.com) +// are ALSO refused by the Lookup profile, and they are deliberately not asserted +// here: isDomainName rejects both as well, so they name nothing production could +// reach and pinning them would be pinning a difference with no consequence. +func TestSSRFSafeHTTPClient_ASCIIHostsAreUnaffectedByNormalization(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + host string + why string + }{ + { + name: "an underscore label", + host: "_atproto.example.com", + why: "UTS#46 disallows U+005F; net.isDomainName permits it on purpose", + }, + { + name: "hyphens in the third and fourth positions", + host: "aa--bb.example.com", + why: "CheckHyphens refuses it; it is ordinary resolvable DNS", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // The premise, and the whole reason the row is here: an + // unconditional ToASCII would refuse this host. + _, mapErr := idna.Lookup.ToASCII(tt.host) + require.Errorf(t, mapErr, + "the Lookup profile must refuse %q (%s). If it stops refusing it, this row no longer pins the "+ + "short-circuit and a different ASCII shape has to be found", tt.host, tt.why) + + publicAnswer := net.ParseIP("93.184.216.34") + require.NotNil(t, publicAnswer, "the fixture address must parse; a nil IP classifies as public") + + // Keyed on the host EXACTLY as written. A transport that normalizes + // this one asks for something else and gets a DNS failure. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{tt.host: {publicAnswer}}} + recorder := &dialRecorder{} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + transport.base = recorder.transport() + + resp, err := client.Get("http://" + tt.host + "/xrpc/x") + if err == nil { + _ = resp.Body.Close() + } + + assert.NotErrorIsf(t, err, ErrBlockedAddress, + "%q was refused by the guard. It is an ASCII hostname this AppView reaches today and Go's "+ + "resolver queries; normalization must not narrow what ASCII names are acceptable, and the "+ + "ASCII short-circuit is what stops it doing so; got: %v", tt.host, err) + + assert.Equalf(t, []string{tt.host}, resolver.hostsAsked(), + "an all-ASCII host must reach the resolver byte-identically. Running it through ToASCII "+ + "rewrites or refuses it — %s", tt.why) + + assert.Equalf(t, []string{tt.host}, recorder.dialledHosts(t), + "%q must still reach the dial", tt.host) + }) + } +} + +// TestSSRFSafeHTTPClient_VetsTheNameNetHTTPWouldDial closes the question the fix +// raises but does not answer on its own: the guard normalizes the host it +// RESOLVES, and rewrites nothing in the URL, so three other consumers of that +// URL compute their own spelling of the host. +// +// - The DIAL ADDRESS is unaffected either way, because this transport's +// DialContext ignores the address it is handed and connects to the vetted +// IPs, taking only the port from it. +// - The CONNECTION-POOL KEY is connectMethodKey.addr, which is +// cm.targetAddr, which is canonicalAddr(url) — punycoded by net/http +// itself. +// - The TLS SERVERNAME is cm.tlsHost(), the same targetAddr with its port +// stripped — likewise punycoded by net/http itself. +// +// All three therefore agree with the guard rather than diverging from it, +// because both sides reach the A-label through the same function with the same +// short-circuit. This test states that as a property rather than as a claim in +// a comment: whatever net/http ends up dialling, it is the name the guard asked +// the resolver about. +// +// The one deliberate divergence is the error branch. net/http's +// idnaASCIIFromURL swallows a ToASCII failure and falls back to the raw host; +// the guard refuses. There is no discrepancy in that case either, for the +// blunt reason that the request never reaches net/http's transport at all — +// TestSSRFSafeHTTPClient_RefusesAHostItCannotNormalize is where that is pinned. +func TestSSRFSafeHTTPClient_VetsTheNameNetHTTPWouldDial(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + host string + }{ + {name: "an IDN host", host: "bücher.example"}, + {name: "an ordinary ASCII host", host: "pds.example.com"}, + {name: "an ASCII host in mixed case", host: "PDS.Example.COM"}, + {name: "a host already in its A-label form", host: "xn--bcher-kva.example"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + publicAnswer := net.ParseIP("93.184.216.34") + require.NotNil(t, publicAnswer, "the fixture address must parse; a nil IP classifies as public") + + // permissiveResolver answers anything, so the request always + // reaches the dial and the two spellings can be compared. What is + // under test is agreement, not reachability. + resolver := &permissiveResolver{answer: publicAnswer} + recorder := &dialRecorder{} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + transport.base = recorder.transport() + + resp, err := client.Get("http://" + tt.host + "/xrpc/x") + if err == nil { + _ = resp.Body.Close() + } + + asked := resolver.hostsAsked() + require.Len(t, asked, 1, "the host must be resolved exactly once per request (err: %v)", err) + + assert.Equal(t, asked, recorder.dialledHosts(t), + "the guard vetted %v while net/http computed %v for the dial address, the connection-pool key "+ + "and the TLS ServerName. A guard that approves one name while the connection is keyed and "+ + "authenticated under another has approved a host the connection never went to", + asked, recorder.dialledHosts(t)) + }) + } +} + +// permissiveResolver answers every name with the same address and records what +// it was asked. It exists because hostRoutedResolver's table is keyed on the +// host, and a test about WHICH host was asked cannot pre-key a table on the +// answer it is trying to discover. +type permissiveResolver struct { + mu sync.Mutex + answer net.IP + asked []string +} + +func (r *permissiveResolver) lookup(_ context.Context, host string) ([]net.IP, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.asked = append(r.asked, host) + return []net.IP{r.answer}, nil +} + +func (r *permissiveResolver) hostsAsked() []string { + r.mu.Lock() + defer r.mu.Unlock() + return slices.Clone(r.asked) +} diff --git a/internal/atproto/oauth/transport_ip_literal_test.go b/internal/atproto/oauth/transport_ip_literal_test.go new file mode 100644 index 0000000..4a2f5cf --- /dev/null +++ b/internal/atproto/oauth/transport_ip_literal_test.go @@ -0,0 +1,201 @@ +package oauth + +import ( + "context" + "net" + "net/http" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ipLiteralHosts are the URL host spellings that must be recognised as +// addresses rather than names. One table, shared by the hatch-closed and +// hatch-open tests below, so the two cannot drift apart — a spelling refused in +// production but also refused in dev is an outage, and the pairing is the point. +// +// urlHost is what goes in the URL (brackets and all, for IPv6); hostname is what +// url.Hostname() yields from it, which is what the check actually sees. +var ipLiteralHosts = []struct { + name string + urlHost string + hostname string +}{ + // A PUBLIC dotted quad, and the row that carries this test. Every other + // spelling here is one the classifier would refuse anyway once it resolved, + // so only this one can distinguish "refused because it is a literal" from + // "refused because of what it points at". The fake dialler ensures the real + // public address is never contacted. + {"public dotted quad", "8.8.8.8", "8.8.8.8"}, + + // Bracketed public IPv6. The brackets are URL syntax + // rather than part of the address: url.Hostname() strips them, so a check + // written against req.URL.Host instead of req.URL.Hostname() sees + // "[2600::1]" and net.ParseIP returns nil for it. + {"bracketed IPv6", "[2600::1]", "2600::1"}, + + // An uppercase-hex IPv4-mapped spelling of 127.0.0.1. net.ParseIP normalises + // it — verified, not assumed — which is exactly why the check must stay a + // ParseIP call. A hand-rolled "does it look like a dotted quad" string test, + // the obvious optimisation for a per-request hot path, misses this and every + // other IPv6 spelling. + {"uppercase-hex mapped loopback", "[::FFFF:7F00:1]", "::FFFF:7F00:1"}, +} + +// literalProbe is a client whose resolver and dialler are both recorded and +// neither of which touches the network. +// +// Both seams are needed, and for different reasons. The RESOLVER records whether +// the transport asked a question it had already been told the answer to — a +// literal is the address, so a lookup is a second decision on an +// attacker-influenced answer. The DIALLER records whether the request got out at +// all, which is what separates a refusal from a connection that merely failed. +type literalProbe struct { + client *http.Client + resolver *hostRoutedResolver + dialled *atomic.Bool +} + +func newLiteralProbe(t *testing.T, allowPrivate bool, hostname string) *literalProbe { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) returns false, so a typo'd literal + // would be classified as public and the row would pass or fail for reasons + // unconnected to its subject. + answer := net.ParseIP(hostname) + require.NotNil(t, answer, "the test's own host %q must parse as an IP address", hostname) + + // The resolver answers the literal with itself, which is what the real + // resolver does for a literal. So if the literal check is missing, the + // request proceeds exactly as it would in production — through + // classification and on to a dial — and the recorders below capture it. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{hostname: {answer}}} + + client := NewSSRFSafeHTTPClient(PrivateAddressOptions(allowPrivate)...) + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + // Substituting the base transport discards the dials-only-vetted-addresses + // property for the duration of these tests, which is safe only because + // TestSSRFTransport_DialsOnlyTheAddressItVetted owns that property outright. + // What it buys is that no packet can leave the machine even when the guard + // is expected NOT to refuse — the hatch-open case below. + var dialled atomic.Bool + transport.base = &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + dialled.Store(true) + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: net.UnknownNetworkError(addr)} + }, + } + + return &literalProbe{client: client, resolver: resolver, dialled: &dialled} +} + +// TestSSRFSafeHTTPClient_RefusesIPLiteralHostsWithTheHatchClosed pins that an +// address written where a name belongs is turned away before anything is +// resolved. +// +// # WHY A LITERAL IS REFUSABLE AT ALL +// +// A legitimate atProto endpoint is always a hostname: the handle specification +// forbids IP literals, and a DID document's serviceEndpoint is an HTTPS URL with +// a hostname. So there is no traffic to lose, and refusing the whole shape is +// cheaper and more total than classifying whatever it points at. +// +// # WHY BEFORE RESOLUTION RATHER THAN AFTER +// +// The dotted-quad row is public, so classification would wave it through — that +// is the case the classifier cannot help with, and it is the reason this check +// exists rather than being another range in the list. For the spellings the +// classifier WOULD catch, refusing first still matters: resolving a literal asks +// a question whose answer the URL already gave, and the answer comes back from a +// resolver an attacker may control. +// +// # WHAT THIS DELIBERATELY DOES NOT COVER +// +// The obfuscated forms — 0x7f.0.0.1, 2130706433, 127.1, 127.0.0.1. — are NOT +// here. net.ParseIP returns nil for all four, so they are not literals as far as +// this check is concerned and they fall through to the resolver. They are +// refused today on both platforms, but by different mechanisms (a cgo build +// resolves them to loopback and IsLoopback catches them; the pure-Go resolver on +// the production build rejects them as malformed hostnames), so a test asserting +// the mechanism would be platform-dependent. In either case a resolved address +// still passes through classification before dialing; the deterministic +// production resolver policy is documented in docs/SSRF_SECURITY.md. +func TestSSRFSafeHTTPClient_RefusesIPLiteralHostsWithTheHatchClosed(t *testing.T) { + t.Parallel() + + for _, tt := range ipLiteralHosts { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newLiteralProbe(t, false, tt.hostname) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "GET http://%s/ must be refused with the hatch closed", tt.urlHost) + + assert.ErrorIs(t, err, ErrBlockedAddress, + "the refusal must carry the guard's sentinel so a caller can tell it from a network failure; "+ + "got: %v", err) + assert.Contains(t, err.Error(), "SSRF blocked", + "the refusal must keep the prefix the rest of this package asserts on; got: %v", err) + + assert.Empty(t, probe.resolver.hostsAsked(), + "the transport resolved %q (hosts asked: %v). The URL already names the address that will be "+ + "dialled, so a lookup here is a second decision taken on an answer the caller does not "+ + "control — the literal has to be refused before resolveHost runs", + tt.hostname, probe.resolver.hostsAsked()) + + assert.False(t, probe.dialled.Load(), + "a connection was attempted to %s. An error returned after the dial is not a refusal: the "+ + "packet has already gone, and for a literal naming a local service that is the whole of "+ + "the SSRF", tt.urlHost) + }) + } +} + +// TestSSRFSafeHTTPClient_AllowsIPLiteralHostsWithTheHatchOpen is the regression +// fence around the test suite itself. +// +// Every integration fixture in this tree is served from an httptest listener, +// which is to say from a loopback IP literal, and two suites drive THIS client +// at one: internal/core/blobs/fetch_guard_test.go via WithPrivateHostsAllowed, +// and internal/core/blueskypost/service_test.go via allowPrivateHost. A literal +// check that ignores allowPrivate does not merely fail those tests — it makes +// the dev environment unable to reach anything local, which is what a dev +// environment is for. +// +// The assertion is that the request reaches the DIALLER, not that it succeeds: +// the fake dialler above always fails, so success is not available and is not +// the property. Getting as far as the dial is proof the guard did not refuse. +func TestSSRFSafeHTTPClient_AllowsIPLiteralHostsWithTheHatchOpen(t *testing.T) { + t.Parallel() + + for _, tt := range ipLiteralHosts { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newLiteralProbe(t, true, tt.hostname) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + _ = resp.Body.Close() + } + + assert.True(t, probe.dialled.Load(), + "GET http://%s/ never reached the dialler with allowPrivate set, so the literal check is "+ + "unconditional rather than gated on the hatch. Every httptest fixture in this tree is "+ + "addressed by IP literal", tt.urlHost) + + assert.NotErrorIs(t, err, ErrBlockedAddress, + "the request was refused by the guard despite the hatch being open; got: %v", err) + }) + } +} diff --git a/internal/atproto/oauth/transport_private_option_test.go b/internal/atproto/oauth/transport_private_option_test.go new file mode 100644 index 0000000..30b80b5 --- /dev/null +++ b/internal/atproto/oauth/transport_private_option_test.go @@ -0,0 +1,250 @@ +package oauth + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// privateHatchGates are the three distinct behaviours the private-address hatch +// controls. ONE TABLE, shared by the option test and the guarded-by-default +// fence below, so the two directions cannot drift: a spelling the hatch opens +// but the default does not close is a hole, and a spelling the default closes +// but the hatch does not open is an outage in every dev environment and every +// httptest fixture in this tree. +// +// They are three behaviours and not three views of one check, which is the +// reason a named option is worth more here than at any other setting on this +// transport. `allowPrivate` gates a literal refusal, a zoned-literal refusal AND +// a classification pass, in three separate places in RoundTrip — so +// `NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed())` is one unlabelled positional bool standing for +// three decisions, none of which its spelling names. +// +// Rows 2 and 3 carry addresses that classify as PUBLIC, which is what makes +// them independent of row 1. Classification would wave both +// through; only the literal check refuses them. If they used loopback instead, +// all three rows would be one behaviour wearing three spellings and the table +// would prove a third of what it claims. +var privateHatchGates = []struct { + name string + // urlHost is what goes in the URL: brackets and the RFC 6874 `%25` zone + // escape included, because that is what an attacker sends and what url.Parse + // is written against. + urlHost string + // hostname is what url.Hostname() yields from urlHost, which is what the + // guard actually inspects — and for the zoned row the two differ in more than + // punctuation. + hostname string + // resolves is the answer the fake resolver gives for hostname. For the two + // literal rows it is the literal itself, which is what a real resolver does + // for a literal — so a missing literal check does not merely fail the row, it + // lets the request proceed exactly as it would in production. + resolves string + why string +}{ + { + name: "a private address reached through a hostname", + urlHost: "private.test", + hostname: "private.test", + resolves: "127.0.0.1", + why: "the classification pass: a name a stranger chose, resolving to an address inside this host. " + + "This is the behaviour the hatch exists for — every httptest fixture in this tree is served " + + "from loopback", + }, + { + name: "an IP literal written where a hostname belongs", + urlHost: "8.8.8.8", + hostname: "8.8.8.8", + resolves: "8.8.8.8", + why: "the literal refusal, which is a SEPARATE gate on the same boolean: this address classifies as " + + "public, so classification cannot account for either direction of this row", + }, + { + name: "a zoned IPv6 literal", + urlHost: "[2600::1%25eth0]", + hostname: "2600::1%eth0", + resolves: "2600::1", + why: "the zoned-literal refusal, the spelling net.ParseIP misses and netip.ParseAddr catches. Also " + + "public space, so again only the literal check moves this row — and a zoned " + + "address is the one form an operator legitimately reaches for with the hatch open, since " + + "fe80::1%en0 needs its interface to be reachable at all", + }, +} + +// hatchProbe is a client every one of whose connections lands on a single +// loopback listener, whatever address it thinks it is dialling. +// +// That is what makes both directions observable from one fixture. "The request +// proceeded" is a 200 from a real server rather than the absence of an error, +// and "the request was refused" is a listener with zero invocations rather than +// a failure that might have come from the network — and since the dialler +// ignores the destination entirely, an invocation count of zero cannot be +// explained by an address that merely happened to be unreachable. +// +// Substituting the base transport discards the dials-only-vetted-addresses +// property for the duration of these tests, which is safe only because +// TestSSRFTransport_DialsOnlyTheAddressItVetted owns that property outright. +// What it buys is that no packet can leave the machine even in the cases where +// the guard is expected NOT to refuse. +type hatchProbe struct { + client *http.Client + resolver *hostRoutedResolver + invocations *atomic.Int64 +} + +func newHatchProbe(t *testing.T, hostname, resolves string, opts ...Option) *hatchProbe { + t.Helper() + + return newHatchProbeWithBody(t, hostname, resolves, nil, opts...) +} + +// newHatchProbeWithBody is the same probe with a reply body, for the callers +// that need the response to be big enough to trip a byte cap. The empty-body +// form above is the common case and stays the one most tests read. +func newHatchProbeWithBody(t *testing.T, hostname, resolves string, body []byte, opts ...Option) *hatchProbe { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) returns false, so a typo'd fixture + // would classify as public and the row would pass or fail for reasons + // unconnected to its subject. + answer := net.ParseIP(resolves) + require.NotNil(t, answer, "the test's own answer %q must parse as an IP address", resolves) + + var invocations atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + invocations.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + })) + t.Cleanup(server.Close) + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{hostname: {answer}}} + + // The boolean stays false in every construction here. The option is the only + // thing that may open the hatch, which is the whole subject of this file. + client := NewSSRFSafeHTTPClient(opts...) + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + serverAddr := server.Listener.Addr().String() + dialer := &net.Dialer{} + transport.base = &http.Transport{ + DialContext: func(ctx context.Context, network, _ string) (net.Conn, error) { + return dialer.DialContext(ctx, network, serverAddr) + }, + } + + return &hatchProbe{client: client, resolver: resolver, invocations: &invocations} +} + +// TestWithPrivateAddressesAllowed_OpensEveryGateTheBooleanOpens pins that the +// named option is equivalent to `allowPrivate = true`, at all three of the +// places that boolean is read. +// +// # WHY A NAMED OPTION AND NOT THE BOOLEAN +// +// `NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed())` reads as nothing at a call site. The reader has +// to open this package to learn that the argument is the difference between a +// guarded client and an unguarded one, and nine call sites are about to be +// written against it. The byte ceiling already got a self-documenting name +// (WithMaxResponseBytes); the setting that disables the guard ENTIRELY did not, +// which is exactly backwards — the more dangerous switch is the one wearing no +// label. +// +// It also has to be greppable. The regression fence needs a way to find +// "this client has the hatch open", and `true` is not a thing you can grep for. +func TestWithPrivateAddressesAllowed_OpensEveryGateTheBooleanOpens(t *testing.T) { + t.Parallel() + + for _, tt := range privateHatchGates { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newHatchProbe(t, tt.hostname, tt.resolves, WithPrivateAddressesAllowed()) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + defer func() { _ = resp.Body.Close() }() + } + + require.NoErrorf(t, err, + "GET http://%s/ was refused with WithPrivateAddressesAllowed() passed. The option has to open "+ + "the same gate the boolean opens, or the seven call sites migrating onto it silently lose "+ + "their dev hatch — %s", tt.urlHost, tt.why) + assert.Equalf(t, http.StatusOK, resp.StatusCode, + "GET http://%s/ reached the listener but did not complete", tt.urlHost) + + assert.Equalf(t, int64(1), probe.invocations.Load(), + "the listener was reached %d times for http://%s/. Every connection this probe makes lands "+ + "there regardless of destination, so anything but exactly one means the request never got "+ + "out of the transport", probe.invocations.Load(), tt.urlHost) + }) + } +} + +// TestNewSSRFSafeHTTPClient_IsGuardedWithoutTheHatchOption is the other half of +// the equivalence, and it is the assertion `make ci` can never make. +// +// `.env.ci:140` sets `IS_DEV_ENV=true`, so every call site about to be built on +// this option runs its PERMISSIVE branch under the merge gate. The guarded +// branch — the one production runs, the one the whole remediation is for — is +// exercised nowhere in CI except here. T0 is not one tier among several for this +// property; it is the only tier that has it. +// +// The two constructions are both needed. "No options at all" is the default +// every un-migrated caller gets and the state a new caller falls into by +// omission. "An unrelated option" is the leak test: options are applied by +// iterating a slice of closures over one struct, so an implementation that +// opened the hatch from the wrong closure — or from the constructor, before the +// options run — would pass the first case and fail the second. +func TestNewSSRFSafeHTTPClient_IsGuardedWithoutTheHatchOption(t *testing.T) { + t.Parallel() + + constructions := []struct { + name string + opts []Option + }{ + {name: "no options at all", opts: nil}, + {name: "an unrelated option", opts: []Option{WithMaxResponseBytes(1024)}}, + } + + for _, construction := range constructions { + t.Run(construction.name, func(t *testing.T) { + t.Parallel() + + for _, tt := range privateHatchGates { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + probe := newHatchProbe(t, tt.hostname, tt.resolves, construction.opts...) + + resp, err := probe.client.Get("http://" + tt.urlHost + "/") + if err == nil { + _ = resp.Body.Close() + } + + require.Errorf(t, err, + "GET http://%s/ succeeded on a client built with %s. %s", + tt.urlHost, construction.name, tt.why) + assert.ErrorIsf(t, err, ErrBlockedAddress, + "the refusal must be the guard's, matchable by identity: a request that failed for some "+ + "other reason is not the same control and would not hold in production; got: %v", err) + + assert.Zerof(t, probe.invocations.Load(), + "the listener was reached %d times for http://%s/. The dialler here sends every "+ + "connection to that one server whatever address it was handed, so any invocation "+ + "means the packet left the transport — and for a destination a stranger named, the "+ + "packet leaving IS the SSRF, whatever error came back afterwards", + probe.invocations.Load(), tt.urlHost) + }) + } + }) + } +} diff --git a/internal/atproto/oauth/transport_refusal_sentinel_test.go b/internal/atproto/oauth/transport_refusal_sentinel_test.go new file mode 100644 index 0000000..269aa80 --- /dev/null +++ b/internal/atproto/oauth/transport_refusal_sentinel_test.go @@ -0,0 +1,135 @@ +package oauth + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ErrBlockedAddress says of itself that it is "the sentinel every address +// refusal matches". Three refusals in the dial path say "SSRF blocked" in their +// message and match nothing. +// +// # WHY A MESSAGE THAT SAYS "SSRF BLOCKED" IS NOT ENOUGH +// +// The sentinel exists precisely so callers stop matching on the message — +// transport_blocked_error_test.go:28 sets out the argument at length, and the +// nine fetch sites being wired onto this client are supposed to use errors.Is +// to choose between a retry and a security log. A refusal that renders the +// right words and fails errors.Is gives those callers the WORST of both: the +// string looks like a block to a human reading a log, and classifies as an +// ordinary network failure to the code deciding what to do about it. +// +// # THE ONE THAT MATTERS MOST +// +// The fail-closed refusal is the guard announcing that it was BYPASSED — +// something reached the base transport without going through RoundTrip, on a +// code path that would otherwise dial any address a caller names. A caller +// doing `if errors.Is(err, ErrBlockedAddress) { alert() } else { retry() }` +// currently retries it. Retrying a bypass is retrying the thing the wrapper +// exists to prevent. +// +// # WHY THIS DRIVES THE BASE TRANSPORT DIRECTLY +// +// Both refusals below are unreachable through client.Get by construction: +// going through the front door is what populates the vetted-address context +// value and what guarantees the dial address is well formed. +// TestSSRFSafeTransport_BaseTransportFailsClosedWhenBypassed established the +// pattern and owns the message assertions; this test owns the identity. +// +// # THE THIRD REFUSAL IS NOT HERE, AND THAT IS A FINDING RATHER THAN AN OMISSION +// +// The dial loop's "no vetted address was attempted for %s" carries the same +// defect and CANNOT be reached from a test. It fires only when the loop body +// never runs, which needs an empty vetted slice, which the guard exercised +// below refuses twenty lines earlier — transport.go says so itself ("THIS IS +// LATENT, NOT LIVE"). There is no seam to drive it through, so an honest test +// cannot exist for it until one is introduced. It must still be fixed: the +// cheapest fix that covers all three is a single constructor for these refusals +// so the sentinel cannot be forgotten by the next one added. +func TestSSRFSafeTransport_EveryDialRefusalMatchesTheSentinel(t *testing.T) { + t.Parallel() + + // A port claimed and released, so the address below is well formed and + // nothing is listening on it. Written down as a literal it would be both a + // guess and a test-audit violation. + probe, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err, "binding a throwaway listener to claim a port") + addr := probe.Addr().String() + require.NoError(t, probe.Close(), "releasing the claimed port so nothing is listening on it") + + loopback := net.ParseIP("127.0.0.1") + require.NotNil(t, loopback, "the test's own address must parse") + + t.Run("the transport was bypassed", func(t *testing.T) { + t.Parallel() + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + // No vetted addresses on the context: the signature of a caller that + // reached the base transport without going through RoundTrip. + conn, err := transport.base.DialContext(t.Context(), "tcp", addr) + if conn != nil { + _ = conn.Close() + } + + require.Error(t, err, "the dial must refuse a destination nothing vetted") + assert.ErrorIs(t, err, ErrBlockedAddress, + "the guard announced its own bypass with an error that does not match ErrBlockedAddress. This is "+ + "the single most important refusal in the package to classify correctly — it means the wrapper "+ + "was skipped on a path that reaches any address a caller names — and a caller branching on "+ + "errors.Is treats it as an ordinary network failure and RETRIES it; got: %v", err) + }) + + t.Run("the dial address carries no port", func(t *testing.T) { + t.Parallel() + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + // Vetted addresses present, so the refusal below can only come from the + // address being unparseable — not from the fail-closed guard above it. + ctx := context.WithValue(t.Context(), vettedAddrsKey, []net.IP{loopback}) + conn, err := transport.base.DialContext(ctx, "tcp", "a-destination-with-no-port") + if conn != nil { + _ = conn.Close() + } + + require.Error(t, err, "the dial must refuse an address it cannot read a port from") + assert.ErrorIs(t, err, ErrBlockedAddress, + "a dial address the guard could not parse produced an error that does not match "+ + "ErrBlockedAddress, while rendering as 'SSRF blocked'. The two halves disagree, and code "+ + "reads the half that is wrong; got: %v", err) + }) + + // The fence, and it is the reason the fix has to be applied per refusal + // rather than by wrapping everything the dial returns. An ordinary failed + // connection is not a block: transport_blocked_error_test.go:113 pins the + // same boundary on the resolution side, and this is its dial-side twin. + t.Run("fence: a connection that simply failed is not a block", func(t *testing.T) { + t.Parallel() + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + ctx := context.WithValue(t.Context(), vettedAddrsKey, []net.IP{loopback}) + conn, err := transport.base.DialContext(ctx, "tcp", addr) + if conn != nil { + _ = conn.Close() + } + + require.Error(t, err, "nothing is listening on the claimed port, so the dial must fail") + assert.NotErrorIs(t, err, ErrBlockedAddress, + "a refused TCP connection to an address the guard ALREADY APPROVED reports itself as a blocked "+ + "address. The sentinel would then mean 'the request failed' rather than 'the guard refused a "+ + "destination', which is exactly the genericisation transport.go's doc comment warns against; "+ + "got: %v", err) + }) +} diff --git a/internal/atproto/oauth/transport_request_body_test.go b/internal/atproto/oauth/transport_request_body_test.go new file mode 100644 index 0000000..e9a4fc0 --- /dev/null +++ b/internal/atproto/oauth/transport_request_body_test.go @@ -0,0 +1,176 @@ +package oauth + +import ( + "bytes" + "context" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// RoundTrip must close the request body on EVERY return, and this transport's +// refusals do not. +// +// # THIS IS A STANDARD LIBRARY CONTRACT, NOT A STYLE PREFERENCE +// +// net/http's RoundTripper documentation states it outright: "RoundTrip must +// always close the body, including on errors". http.Client is written against +// that promise — it does not close the body itself after calling RoundTrip — so +// a RoundTripper that returns early without closing leaks whatever the body was +// holding. For the callers being wired onto this client that is a file handle, +// a pipe, or a buffer with a finalizer, one per refused request. +// +// # WHY THE REFUSAL PATHS ARE EXACTLY THE ONES THAT LEAK +// +// The three returns below all happen BEFORE the base transport is reached, and +// the base transport is what would otherwise have closed the body. They are +// also the paths a hostile input takes: a caller pointed at an attacker-chosen +// URL refuses often, and the leak scales with how well the guard is working. +// +// # WHY THIS DRIVES RoundTrip DIRECTLY RATHER THAN client.Do +// +// Because the contract belongs to RoundTrip. Going through http.Client would +// measure whatever the standard library does around it and could not tell a +// transport that closes the body from one that does not. +func TestSSRFSafeTransport_ClosesTheRequestBodyOnEveryRefusal(t *testing.T) { + t.Parallel() + + // Checked, not assumed: isPrivateIP(nil) returns false, so a typo here would + // classify as public and the private-address row would stop being one. + private := net.ParseIP("10.99.13.37") + require.NotNil(t, private, "the test's own private address must parse") + + tests := []struct { + name string + host string + answers map[string][]net.IP + why string + }{ + { + // A genuinely public address, so only the literal-shape check can + // refuse it before resolution. + name: "an IP literal, refused before resolution", + host: "8.8.8.8", + answers: map[string][]net.IP{}, + why: "the literal check returns before anything else in RoundTrip runs", + }, + { + name: "a hostname that does not resolve", + host: "unresolvable.test", + answers: map[string][]net.IP{}, + why: "the resolution-failure return is taken on every DNS hiccup, not only on hostile input", + }, + { + name: "a hostname resolving to a private address", + host: "blocked.test", + answers: map[string][]net.IP{"blocked.test": {private}}, + why: "the classification refusal is the guard's main path and therefore the most frequent leak", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = (&hostRoutedResolver{answers: tt.answers}).lookup + + // A dialler that records and refuses, so a guard that let any of + // these through fails loudly here instead of opening a socket. + var dialled atomic.Bool + transport.base = &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + dialled.Store(true) + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: net.UnknownNetworkError(addr)} + }, + } + + body := &countingBody{Reader: bytes.NewReader([]byte("a request body"))} + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, "http://"+tt.host+"/", body) + require.NoError(t, err, "building the request") + + resp, err := transport.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "%s must be refused", tt.host) + require.False(t, dialled.Load(), "%s must be refused before any dial", tt.host) + + assert.EqualValues(t, 1, body.closes.Load(), + "RoundTrip returned without closing the request body (Close called %d times). net/http "+ + "documents that a RoundTripper must always close the body, INCLUDING ON ERRORS, and "+ + "http.Client relies on that promise rather than closing it itself — so every refusal on "+ + "this path leaks whatever the body was holding. %s", + body.closes.Load(), tt.why) + }) + } + + // The fourth early return, and it is a FENCE rather than a RED. + // + // By the time an oversized declared length is refused the request has been + // sent, so the base transport has already closed the body — verified by + // probe: exactly one Close today. It is here so that the fix for the three + // rows above is applied to the returns that need it rather than as a blanket + // `defer` on entry, which would close this body a SECOND time. Once is the + // contract, both directions. + t.Run("fence: an oversized declared length closes the body exactly once", func(t *testing.T) { + t.Parallel() + + const declared = 1 << 20 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + w.Header().Set("Content-Length", strconv.Itoa(declared)) + w.WriteHeader(http.StatusOK) + writeChunks(w, declared) + })) + defer server.Close() + + // The hatch is open because the listener is on loopback; the response + // cap is what this row is about. + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed(), WithMaxResponseBytes(testCap)) + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + + body := &countingBody{Reader: bytes.NewReader([]byte("a request body"))} + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, server.URL, body) + require.NoError(t, err, "building the request") + + resp, err := transport.RoundTrip(req) + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "a response declaring %d bytes through a %d-byte cap must be refused", declared, testCap) + assert.EqualValues(t, 1, body.closes.Load(), + "the request body was closed %d times on the declared-length refusal. The base transport has "+ + "already closed it by this point, so closing again here — which is what a blanket defer on "+ + "RoundTrip's entry would do — breaks the contract in the other direction: a body whose Close "+ + "is not idempotent (a file, a pipe) reports an error the second time", + body.closes.Load()) + }) +} + +// countingBody is a request body that counts how many times it was closed. +// +// atomic because http.Transport writes the request on a goroutine of its own, +// so the close in the fence subtest above happens off the test's goroutine. +type countingBody struct { + io.Reader + closes atomic.Int64 +} + +func (b *countingBody) Close() error { + b.closes.Add(1) + return nil +} diff --git a/internal/atproto/oauth/transport_reserved_ranges_test.go b/internal/atproto/oauth/transport_reserved_ranges_test.go index 63904de..094ff05 100644 --- a/internal/atproto/oauth/transport_reserved_ranges_test.go +++ b/internal/atproto/oauth/transport_reserved_ranges_test.go @@ -34,8 +34,20 @@ import ( // - IETF PROTOCOL ASSIGNMENTS (192.0.0.0/24). Reserved for protocol machinery // (DS-Lite's 192.0.0.0/29 among it); nothing here is a destination a caller // legitimately names. +// - 6to4 ANYCAST RELAY (192.88.99.0/24). The relay side of a mechanism whose +// destination side is already refused: reservedNetworks bans 2002::/16 +// outright, and this /24 is where a host sends 6to4 traffic to reach it. +// Note the asymmetry in what RFC 7526 actually says, because the comment in +// transport.go is careful about it and this row is the other half — RFC 7526 +// deprecates THIS prefix, and states verbatim that the IPv6 side is not +// deprecated. The 2002::/16 ban is our policy call; this one is the RFC's. // - BENCHMARKING (198.18.0.0/15). Reserved for device test harnesses and, in // practice, routed internally where it is used at all. +// - IANA NON-GLOBAL SPACE. Documentation, discard, dummy, IPv6 benchmarking, +// local-use NAT64, Teredo, deprecated ORCHID and SRv6 SID prefixes are not +// globally reachable, but IANA explicitly does not promise they are +// unroutable in a local context. Failing open on those ranges lets local +// routing policy become an SSRF bypass. // - RESERVED (240.0.0.0/4) and BROADCAST (255.255.255.255). Former class E and // the all-hosts broadcast; the stack handles both unlike a normal unicast // destination. @@ -112,10 +124,38 @@ func TestIsPrivateIP_ReservedAndUnspecifiedRanges(t *testing.T) { {"IETF protocol assignments first", "192.0.0.0", true}, {"IETF protocol assignments last", "192.0.0.255", true}, + // 6to4 anycast relay 192.88.99.0/24, pinned at both ends. There is + // nothing inside it to reach on purpose: an address here is not a host, it + // is whichever relay router the routing system happens to hand you, and + // the mechanism it serves is one this guard already refuses on the IPv6 + // side. Blocking it costs no legitimate destination. + {"6to4 anycast relay first", "192.88.99.0", true}, + {"6to4 anycast relay last", "192.88.99.255", true}, + // Benchmarking 198.18.0.0/15. {"Benchmarking low edge", "198.18.0.1", true}, {"Benchmarking high edge", "198.19.255.255", true}, + // IPv4 documentation space is non-global and can still be routed locally. + {"TEST-NET-1", "192.0.2.1", true}, + {"TEST-NET-2", "198.51.100.1", true}, + {"TEST-NET-3", "203.0.113.1", true}, + + // IANA IPv6 special-purpose ranges whose Globally Reachable property is + // false (plus Teredo, whose registry value is context-dependent). + {"NAT64 local-use low", "64:ff9b:1::1", true}, + {"NAT64 local-use custom allocation", "64:ff9b:1:abcd::7f00:1", true}, + {"IPv6 discard-only", "100::1", true}, + {"IPv6 dummy", "100:0:0:1::1", true}, + {"Teredo", "2001::1", true}, + {"IETF protocol assignments unallocated child", "2001:5::1", true}, + {"IETF protocol assignments beside an anycast exception", "2001:1::4", true}, + {"IPv6 benchmarking", "2001:2::1", true}, + {"Deprecated ORCHID", "2001:10::1", true}, + {"IPv6 documentation", "2001:db8::1", true}, + {"IPv6 documentation 3fff", "3fff::1", true}, + {"SRv6 SID", "5f00::1", true}, + // Reserved 240.0.0.0/4, pinned at both ends for the same reason. It runs // all the way to the limited broadcast address, which is why there is no // "just above" row: there is no above. Nor is there a meaningful "just @@ -153,8 +193,29 @@ func TestIsPrivateIP_ReservedAndUnspecifiedRanges(t *testing.T) { {"Just above benchmarking", "198.20.0.0", false}, {"Just above IETF protocol assignments", "192.0.1.0", false}, {"Above IETF protocol assignments", "192.0.1.1", false}, + + // Both neighbours of the /24, and they catch different mistakes per this + // file's own doctrine: widen 192.88.99.0/24 by one bit and it normalises + // to 192.88.98.0/23, which swallows the row below it; the row above + // catches a wrong base address or a widening of two bits or more. + {"Just below the 6to4 anycast relay", "192.88.98.255", false}, + {"Just above the 6to4 anycast relay", "192.88.100.0", false}, {"Just below multicast", "223.255.255.255", false}, {"Just below 10/8", "9.255.255.255", false}, + + // More-specific IANA assignments that are globally reachable must remain + // usable when the non-global 2001::/23 parent is blocked. + {"AS112 IPv4", "192.31.196.1", false}, + {"AMT IPv4", "192.52.193.1", false}, + {"Direct AS112 IPv4", "192.175.48.1", false}, + {"PCP anycast IPv6", "2001:1::1", false}, + {"TURN anycast IPv6", "2001:1::2", false}, + {"DNS-SD registration anycast IPv6", "2001:1::3", false}, + {"AMT IPv6", "2001:3::1", false}, + {"AS112 IPv6", "2001:4:112::1", false}, + {"ORCHIDv2", "2001:20::1", false}, + {"Drone DET", "2001:30::1", false}, + {"Direct AS112 IPv6", "2620:4f:8000::1", false}, } for _, tt := range tests { @@ -259,6 +320,7 @@ func TestIsPrivateIP_MappedSpellingsOfReservedRanges(t *testing.T) { {"Mapped zero network", "::ffff:0.0.0.1", true}, {"Mapped CGNAT", "::ffff:100.64.0.1", true}, {"Mapped IETF protocol assignments", "::ffff:192.0.0.1", true}, + {"Mapped 6to4 anycast relay", "::ffff:192.88.99.1", true}, {"Mapped benchmarking", "::ffff:198.18.0.1", true}, {"Mapped reserved former class E", "::ffff:240.0.0.1", true}, {"Mapped limited broadcast", "::ffff:255.255.255.255", true}, diff --git a/internal/atproto/oauth/transport_response_cap_test.go b/internal/atproto/oauth/transport_response_cap_test.go new file mode 100644 index 0000000..c219a7c --- /dev/null +++ b/internal/atproto/oauth/transport_response_cap_test.go @@ -0,0 +1,223 @@ +package oauth + +import ( + "io" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeHTTPClient_ProtectsACallerThatValidatesNothing is the binding +// acceptance contract for the hardened transport. +// +// # THE CALLER THIS IS WRITTEN FOR +// +// Nine fetch sites are about to be pointed at this client, and every one of them +// was written by someone who assumed the URL was trustworthy: no address check, +// no size limit, `io.ReadAll` on whatever comes back. That assumption is false at +// all nine — the URL is a DID document's `serviceEndpoint`, a community record's +// domain, a user's `PDSURL` — and rewriting nine call sites to validate for +// themselves is nine chances to get it wrong and nine places for the next one to +// forget. +// +// So the property is not "the transport offers protections". It is that the +// transport ALONE protects a caller that caps nothing and validates nothing. The +// two subtests are the two halves of that: where the connection is allowed to go, +// and how much is allowed back. +// +// # WHY THIS IS WRITTEN AGAINST THE CONSTRUCTOR +// +// Everything here goes through `NewSSRFSafeHTTPClient` and a real listener, +// never against `isPrivateIP`. A predicate returning true is evidence; a service +// that was never touched, and a read that failed rather than lied, are the +// properties. The unit tests in this package own the classifier's rows — this +// file owns what a caller experiences. +func TestSSRFSafeHTTPClient_ProtectsACallerThatValidatesNothing(t *testing.T) { + t.Parallel() + + // (a) THE CONNECTION. A legitimate atProto endpoint is always a hostname — + // the handle spec forbids IP literals and the DID spec requires an HTTPS URL + // with a hostname — so a caller-supplied URL whose host is a dotted quad is + // never a destination this AppView has business reaching, and refusing it + // outright is cheaper than classifying it. + // + // THE DISCRIMINATOR IS "WAS THE NAME EVER LOOKED UP", NOT "WAS IT REFUSED". + // A loopback literal is already refused today by the classifier, so a test + // that points at 127.0.0.1 and asserts an error proves the PREVIOUS PR and + // nothing about this one. What is new is WHERE the refusal happens: a literal + // must be turned away before `resolveHost` runs at all. + t.Run("an IP-literal URL is refused before the name is ever resolved", func(t *testing.T) { + t.Parallel() + + // A genuinely public address, so the classifier cannot green this case + // and the assertion below can only be satisfied by the literal check. The + // test resolver still maps it to loopback, so no packet can leave. + const publicLiteral = "8.8.8.8" + + t.Run("hatch closed: refused without a lookup", func(t *testing.T) { + t.Parallel() + + var reached atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached.Store(true) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // The listener's own address, read back rather than written down. + // The resolver below answers the public literal with it, which is + // what keeps every packet on loopback AND what arms the handler + // tripwire: a guard that permitted this request would dial the + // vetted address and land on this listener for real. + listenerHost, port, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(t, err, "splitting the test listener address %q", server.Listener.Addr()) + listenerIP := net.ParseIP(listenerHost) + require.NotNil(t, listenerIP, + "the test's own listener address %q must parse. isPrivateIP(nil) returns false, so a nil here "+ + "would be classified as public and this case would assert nothing", listenerHost) + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{ + publicLiteral: {listenerIP}, + }} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + target := "http://" + net.JoinHostPort(publicLiteral, port) + "/" + resp, err := client.Get(target) + if err == nil { + _ = resp.Body.Close() + } + + // The discriminating assertion. Everything below it passes today. + assert.Empty(t, resolver.hostsAsked(), + "the transport sent %q to the resolver (hosts asked: %v). A dotted quad IS the address that "+ + "will be dialled, so resolving it asks a question the URL had already answered — and the "+ + "answer comes back from a resolver the attacker may control, which is one more chance for "+ + "the destination to become something other than what was classified. With the hatch closed "+ + "the literal must be refused before resolveHost runs", + publicLiteral, resolver.hostsAsked()) + + // Standing anti-cheat, not the discriminator: it passes before and + // after. Mutation testing on the previous PR produced an + // implementation that classified correctly, emitted a byte-identical + // error, and refused the request AFTER delivering it — every + // error-message assertion passed against it and only this one caught + // it. + assert.False(t, reached.Load(), + "GET %s was delivered to the listener. A caller that hands this client an attacker-chosen URL "+ + "performs no check of its own, so refusing after the request lands is not a refusal at all — "+ + "the service has already acted on it", target) + + require.Error(t, err, "GET %s must be refused", target) + assert.Contains(t, err.Error(), "SSRF blocked", + "transport_unspecified_address_test.go:78 establishes that prefix as the refusal contract, and "+ + "the literal refusal must keep it so a transport error cannot be mistaken for a block; got: %v", err) + }) + + // The gating, and it is a regression fence rather than decoration. + // internal/core/blobs/fetch_guard_test.go and + // internal/core/blueskypost/service_test.go both point this client at an + // httptest server — which is to say at a loopback IP literal — with the + // hatch open, because every integration fixture in the tree is served + // from loopback. A literal check that is not gated on allowPrivate takes + // both of those suites down with it. + t.Run("hatch open: the literal is dialled", func(t *testing.T) { + t.Parallel() + + var reached atomic.Bool + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached.Store(true) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // server.URL is a real IP literal — httptest listens on loopback — + // which is exactly the shape the case above refuses. + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed()) + resp, err := client.Get(server.URL) + require.NoError(t, err, + "the dev hatch must still reach a loopback listener addressed by IP literal. Every integration "+ + "fixture in this tree is served that way, so a literal check that ignores allowPrivate is an "+ + "outage in the test suite and in dev; got: %v", err) + defer func() { _ = resp.Body.Close() }() + + assert.Equal(t, http.StatusOK, resp.StatusCode, "the hatch-open request must complete normally") + assert.True(t, reached.Load(), + "the request never reached the listener even with allowPrivate set, so the literal refusal is "+ + "unconditional rather than gated on the hatch") + }) + }) + + // (b) THE RESPONSE. A destination that passes the address check still + // chooses how much it sends, and an unbounded `io.ReadAll` at the call site + // is an out-of-memory the remote host triggers at will. + // + // THE FAILURE MODE THIS PINS IS NOT "no cap" — IT IS A CAP THAT TRUNCATES. + // A wrapper that stops at the limit by returning io.EOF reads, to every + // caller in the tree, as a complete body: `io.ReadAll` treats io.EOF as the + // clean end of the stream and returns `err == nil`, so the caller gets a + // short body it has no way to know is short. For the unfurl provider that + // means parsing half a document; for a signature check it means verifying a + // prefix. Silent truncation is worse than no cap, because no cap at least + // fails loudly. + // + // The response is deliberately sent WITHOUT a Content-Length: the header is + // chosen by the same attacker as the body, so the cap that matters is the + // one on the bytes actually read. + t.Run("an over-large body fails the read instead of truncating", func(t *testing.T) { + t.Parallel() + + const ( + maxResponseBytes = 64 << 10 // the cap this client is given + payloadBytes = 1 << 20 // what the server sends: sixteen times the cap + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Chunked, and the write error is the exit condition rather than a + // failure: once the cap engages the client drops the connection, and + // the handler noticing that is the expected end of this response. + chunk := make([]byte, 4<<10) + for sent := 0; sent < payloadBytes; sent += len(chunk) { + if _, err := w.Write(chunk); err != nil { + return + } + } + })) + defer server.Close() + + // allowPrivate, because the listener is on loopback and subtest (a) is + // the proof that a production client would refuse it. What is under test + // here is the cap alone. + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed(), WithMaxResponseBytes(maxResponseBytes)) + + resp, err := client.Get(server.URL) + require.NoError(t, err, "the request itself must succeed — the cap belongs to the body, not the dial") + defer func() { _ = resp.Body.Close() }() + + // io.ReadAll is exactly what the nine call sites do, which is why it is + // what this asserts through. It is also the detector: it swallows io.EOF, + // so a truncating wrapper arrives here as `err == nil` and a short body. + body, readErr := io.ReadAll(resp.Body) + + require.Error(t, readErr, + "io.ReadAll returned no error after reading %d bytes through a %d-byte cap. Either nothing is "+ + "enforcing the cap, or the cap ends the stream cleanly — and a clean end is the worse of the "+ + "two: the caller believes it holds the whole body", len(body), int64(maxResponseBytes)) + assert.NotErrorIs(t, readErr, io.EOF, + "the cap reported itself as io.EOF (possibly wrapped), which every reader in the standard library "+ + "treats as the clean end of the body rather than as a failure. The error must be distinguishable "+ + "from a complete response; got: %v", readErr) + assert.LessOrEqual(t, int64(len(body)), int64(maxResponseBytes), + "the caller obtained %d bytes through a %d-byte cap, so the limit is advisory rather than "+ + "enforced — a remote host still decides how much memory this process allocates", + len(body), int64(maxResponseBytes)) + }) +} diff --git a/internal/atproto/oauth/transport_revetting_test.go b/internal/atproto/oauth/transport_revetting_test.go index 25fb8c6..79d7ace 100644 --- a/internal/atproto/oauth/transport_revetting_test.go +++ b/internal/atproto/oauth/transport_revetting_test.go @@ -38,7 +38,7 @@ type hostRoutedResolver struct { asked []string } -func (r *hostRoutedResolver) lookup(host string) ([]net.IP, error) { +func (r *hostRoutedResolver) lookup(_ context.Context, host string) ([]net.IP, error) { r.mu.Lock() defer r.mu.Unlock() r.asked = append(r.asked, host) @@ -97,7 +97,7 @@ func TestSSRFSafeHTTPClient_RevetsEachRedirectHop(t *testing.T) { "hop2.test": {hop2}, }} - client := NewSSRFSafeHTTPClient(false) + client := NewSSRFSafeHTTPClient() transport, ok := client.Transport.(*ssrfSafeTransport) require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) transport.lookupIP = resolver.lookup @@ -173,7 +173,7 @@ func TestSSRFSafeHTTPClient_RefusesAMixedLookupAnswer(t *testing.T) { resolver := &hostRoutedResolver{answers: map[string][]net.IP{"mixed.test": tt.answer}} - client := NewSSRFSafeHTTPClient(false) + client := NewSSRFSafeHTTPClient() transport, ok := client.Transport.(*ssrfSafeTransport) require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) transport.lookupIP = resolver.lookup @@ -197,9 +197,28 @@ func TestSSRFSafeHTTPClient_RefusesAMixedLookupAnswer(t *testing.T) { require.Error(t, err, "an answer containing a private address must be refused") assert.Contains(t, err.Error(), "SSRF blocked", "the refusal must name the guard that made it; got: %v", err) - assert.Contains(t, err.Error(), private.String(), - "the refusal must name the address that caused it, so an operator reading the log knows which "+ - "of the answers was the problem; got: %v", err) + // THE DIAGNOSTIC MOVED; IT WAS NOT DELETED. This assertion used to + // read `assert.Contains(err.Error(), private.String())`, and the + // reasoning behind it still holds in full: an operator looking at a + // refusal has to be able to tell WHICH of the answers was the + // problem, or a mixed-answer block is indistinguishable from any + // other. What changed is where that detail lives. The rendered + // message reaches places the operator does not control — an HTTP + // response, a shared log — and "which address did this name resolve + // to inside your network" is the one half of that sentence the + // attacker did not already supply. So the address is now reachable + // through errors.As, which serves the operator without answering + // the question to anyone who can read the string. + require.ErrorIs(t, err, ErrBlockedAddress, + "the refusal must be matchable by identity rather than by substring; got: %v", err) + + var blocked *BlockedAddressError + require.ErrorAs(t, err, &blocked, + "the refusal must carry its detail on a typed error an operator can reach; got: %v", err) + assert.True(t, blocked.IP.Equal(private), + "the typed error names %s as the blocking address, but the private answer in this case was %s — "+ + "an operator reading the block still has to know which of the two answers caused it", + blocked.IP, private) assert.False(t, dialled.Load(), "a connection was attempted despite a private address in the answer. The dialler picks which "+ @@ -233,7 +252,7 @@ func TestSSRFSafeHTTPClient_RefusesAMixedLookupAnswer(t *testing.T) { func TestSSRFSafeTransport_BaseTransportFailsClosedWhenBypassed(t *testing.T) { t.Parallel() - client := NewSSRFSafeHTTPClient(false) + client := NewSSRFSafeHTTPClient() transport, ok := client.Transport.(*ssrfSafeTransport) require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) diff --git a/internal/atproto/oauth/transport_test.go b/internal/atproto/oauth/transport_test.go index 032d222..7a28c81 100644 --- a/internal/atproto/oauth/transport_test.go +++ b/internal/atproto/oauth/transport_test.go @@ -74,7 +74,7 @@ func TestNewSSRFSafeHTTPClient(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - client := NewSSRFSafeHTTPClient(tt.allowPrivate) + client := NewSSRFSafeHTTPClient(PrivateAddressOptions(tt.allowPrivate)...) if client == nil { t.Fatal("NewSSRFSafeHTTPClient returned nil") @@ -101,7 +101,7 @@ func TestNewSSRFSafeHTTPClient(t *testing.T) { } func TestSSRFSafeHTTPClient_RedirectLimit(t *testing.T) { - client := NewSSRFSafeHTTPClient(false) + client := NewSSRFSafeHTTPClient() // Simulate checking redirect limit if client.CheckRedirect == nil { diff --git a/internal/atproto/oauth/transport_toctou_test.go b/internal/atproto/oauth/transport_toctou_test.go index a8b5731..b95e11e 100644 --- a/internal/atproto/oauth/transport_toctou_test.go +++ b/internal/atproto/oauth/transport_toctou_test.go @@ -1,6 +1,7 @@ package oauth import ( + "context" "net" "net/http" "net/http/httptest" @@ -41,7 +42,7 @@ type flippingResolver struct { calls int } -func (r *flippingResolver) lookup(string) ([]net.IP, error) { +func (r *flippingResolver) lookup(context.Context, string) ([]net.IP, error) { r.mu.Lock() defer r.mu.Unlock() r.calls++ @@ -86,7 +87,7 @@ func TestSSRFTransport_DialsOnlyTheAddressItVetted(t *testing.T) { // pass against a transport with the bug still in it. Disabling the private // check leaves exactly one thing under test: whether the address that was // vetted is the address that gets dialled. - client := NewSSRFSafeHTTPClient(true) + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed()) transport, ok := client.Transport.(*ssrfSafeTransport) if !ok { t.Fatalf("NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) @@ -124,7 +125,7 @@ func TestSSRFTransport_StillRefusesAPrivateFirstAnswer(t *testing.T) { private: net.ParseIP("169.254.169.254"), } - client := NewSSRFSafeHTTPClient(false) + client := NewSSRFSafeHTTPClient() transport, ok := client.Transport.(*ssrfSafeTransport) if !ok { t.Fatalf("NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) diff --git a/internal/atproto/oauth/transport_unspecified_address_test.go b/internal/atproto/oauth/transport_unspecified_address_test.go index 9840ab7..3b036e9 100644 --- a/internal/atproto/oauth/transport_unspecified_address_test.go +++ b/internal/atproto/oauth/transport_unspecified_address_test.go @@ -41,6 +41,29 @@ import ( // It asserts through NewSSRFSafeHTTPClient rather than against `isPrivateIP`, // because the contract is the client's refusal to CONNECT. A predicate returning // true is evidence; a service that was never touched is the property. +// +// # WHY THE TARGET IS A HOSTNAME AND NOT THE LITERAL 0.0.0.0 +// +// It used to be `http://0.0.0.0:PORT/` — the URL an attacker actually types — +// and that is now the wrong instrument. The transport refuses an IP-literal host +// outright, before it resolves anything, so a literal target would make this +// test pass on that check alone and never reach `isPrivateIP` at all. A green +// that cannot fail when the thing under test breaks is worse than no test: this +// file's whole subject is the CLASSIFIER's handling of 0.0.0.0, so the request +// has to reach the classifier. The host is therefore a name, and the lookup seam +// answers it with 0.0.0.0 — which is also a faithful model of the real attack, +// since a DID document's endpoint is a hostname and its zone is the attacker's. +// +// The literal check has its own coverage and does not need this file's: +// transport_ip_literal_test.go for the unit rows, and the outer acceptance +// contract in transport_response_cap_test.go end to end. +// +// EVERYTHING ELSE IS UNCHANGED, DELIBERATELY. The listener is real and the base +// transport is NOT substituted, so the kernel substitution this test is about +// still happens for real: a guard that classified 0.0.0.0 as public would hand +// that address to the dialler, the kernel would turn it into the local host at +// connect time, and `reached` would flip. That assertion is the point of the +// file and it is not weakened here. func TestSSRFSafeHTTPClient_RefusesTheUnspecifiedAddress(t *testing.T) { t.Parallel() @@ -58,21 +81,35 @@ func TestSSRFSafeHTTPClient_RefusesTheUnspecifiedAddress(t *testing.T) { _, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://")) require.NoError(t, err, "splitting the test server address %q", server.URL) - // The port is the server's; the host is rewritten to the unspecified address. - // Rebuilding the URL rather than hardcoding a port is what keeps the target a - // listener this test owns. - target := "http://" + net.JoinHostPort("0.0.0.0", port) + "/" + // The port is the server's; the host is a name the seam below answers with + // the unspecified address. Rebuilding the URL rather than hardcoding a port + // is what keeps the target a listener this test owns. + target := "http://" + net.JoinHostPort("unspecified.test", port) + "/" + + // Checked, not assumed. A typo here makes net.ParseIP return nil, + // isPrivateIP(nil) returns false, and the address under test silently becomes + // "no address at all" — a fixture that can turn the test into one which + // asserts nothing. + unspecified := net.ParseIP("0.0.0.0") + require.NotNil(t, unspecified, "the test's own unspecified address must parse") + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = (&hostRoutedResolver{answers: map[string][]net.IP{ + "unspecified.test": {unspecified}, + }}).lookup - client := NewSSRFSafeHTTPClient(false) resp, err := client.Get(target) if err == nil { _ = resp.Body.Close() } assert.False(t, reached.Load(), - "GET %s reached the loopback listener: 0.0.0.0 passed the private-address check, and the kernel "+ - "then resolved it to the local host at connect time. The guard classified an address it never "+ - "actually dialled, so an attacker-supplied URL like http://0.0.0.0:5432/ hits a local service", + "GET %s reached the loopback listener: the host resolved to 0.0.0.0, that address passed the "+ + "private-address check, and the kernel then substituted the local host at connect time. The guard "+ + "classified an address it never actually dialled, so an attacker-supplied endpoint resolving to "+ + "0.0.0.0 hits whatever is listening on 127.0.0.1 at that port", target) require.Error(t, err, "GET %s must be refused", target) assert.Contains(t, err.Error(), "SSRF blocked", diff --git a/internal/atproto/oauth/transport_zoned_literal_test.go b/internal/atproto/oauth/transport_zoned_literal_test.go new file mode 100644 index 0000000..02c4b8e --- /dev/null +++ b/internal/atproto/oauth/transport_zoned_literal_test.go @@ -0,0 +1,168 @@ +package oauth + +import ( + "context" + "net" + "net/http" + "net/netip" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestSSRFSafeHTTPClient_RefusesAZonedIPv6LiteralBeforeResolving closes the one +// spelling of an IP literal that walks past the literal check. +// +// # THE HOLE +// +// The check is `net.ParseIP(req.URL.Hostname())`, and net.ParseIP returns nil +// for any address carrying a ZONE — the `%eth0` in `fe80::1%eth0`, which names +// the interface the address is scoped to. url.Parse does understand the form: +// `http://[2600::1%25eth0]/` yields a Hostname of `2600::1%eth0` +// (verified by probe, along with ParseIP returning nil for it). So the address +// is refused as "not a literal", handed to the resolver, resolved locally with +// no DNS involved, its zone silently discarded, and dialled. +// +// # WHAT THIS IS AND IS NOT +// +// It is a hole in the CONTROL, not a live SSRF: an address that would be +// dangerous once the zone is stripped — fe80::1, ::1 — is still caught by +// classification a few lines later, and transport_ip_literal_test.go covers +// those. What gets through the literal check untouched is a PUBLIC-classified +// zoned address, and the literal check exists precisely because classification +// is not the answer for literals: a caller-supplied literal naming a public +// address is still a destination this AppView has no business reaching, and +// that is the case this row demonstrates. +// +// The private row is here too, and it is not redundant: it pins WHERE the +// refusal happens. Today it is refused after a lookup; it must be refused +// before one, for the same reason every other literal is — resolving an address +// asks a question the URL already answered, of a resolver an attacker may +// influence. +// +// # THE FIX +// +// netip.ParseAddr, which accepts zones. The assertion below is deliberately +// about behaviour rather than about which parser is used, but the parser is +// worth naming because there is no way to reach the same result with +// net.ParseIP short of hand-splitting on '%'. +func TestSSRFSafeHTTPClient_RefusesAZonedIPv6LiteralBeforeResolving(t *testing.T) { + t.Parallel() + + // The premise, checked rather than assumed. If netip ever stopped accepting + // zones the fix this test asks for would not exist, and the test would be + // demanding something impossible instead of something undone. + zoned, err := netip.ParseAddr("2600::1%eth0") + require.NoError(t, err, "netip.ParseAddr must accept a zoned address: it is the parser this fix needs") + require.Equal(t, "eth0", zoned.Zone(), "the parsed address must carry its zone") + require.Nil(t, net.ParseIP("2600::1%eth0"), + "net.ParseIP must still return nil for a zoned address — that nil is the whole defect, and if it "+ + "ever stops being nil this test is pinning a bug that no longer exists") + + tests := []struct { + name string + // url carries the RFC 6874 spelling: the zone's '%' is percent-encoded + // as %25 inside the brackets, which is what a URL parser requires and + // what an attacker would send. + url string + why string + }{ + { + name: "a zoned address that classifies as public", + url: "http://[2600::1%25eth0]/", + why: "globally reachable space, which this package classifies as public — so classification cannot " + + "refuse it and only the literal check can. This is the row where the hole has consequences", + }, + { + name: "a zoned link-local address", + url: "http://[fe80::1%25en0]/", + why: "link-local, so classification refuses it today — but only AFTER handing an attacker-supplied " + + "address to a resolver, which is the step the literal check exists to skip", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // An empty answer table: nothing here is resolvable, so a request + // that gets past the literal check dies at resolution rather than + // reaching a socket. The assertion is on what the resolver was + // ASKED, not on whether the request failed. + resolver := &hostRoutedResolver{answers: map[string][]net.IP{}} + + client := NewSSRFSafeHTTPClient() + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + // Belt and braces: a dialler that records and refuses, so nothing + // can leave the machine even if both guards were removed. + var dialled atomic.Bool + transport.base = &http.Transport{ + DialContext: func(_ context.Context, _, addr string) (net.Conn, error) { + dialled.Store(true) + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: net.UnknownNetworkError(addr)} + }, + } + + resp, err := client.Get(tt.url) + if err == nil { + _ = resp.Body.Close() + } + + // THE DISCRIMINATOR. Everything else here passes today. + assert.Emptyf(t, resolver.hostsAsked(), + "the transport sent %v to the resolver. %s is an address written where a hostname belongs, "+ + "and net.ParseIP returning nil for it because of the zone does not make it a name — %s", + resolver.hostsAsked(), tt.url, tt.why) + + require.Error(t, err, "GET %s must be refused", tt.url) + assert.ErrorIsf(t, err, ErrBlockedAddress, + "the refusal must be the literal check's, matchable by identity. A resolution failure that "+ + "happens to stop the request is not the same control and does not hold when the name is "+ + "one the resolver can answer; got: %v", err) + + assert.Falsef(t, dialled.Load(), + "GET %s reached the dialler. A zoned literal is still a literal, and the destination was "+ + "chosen by whoever supplied the URL", tt.url) + }) + } +} + +// The hatch, and the reason it needs its own row: every integration fixture in +// this tree is served from a loopback listener, so a literal check that ignores +// allowPrivate is an outage in dev and in the test suite. A zoned address is +// the one spelling an operator would legitimately reach for with the hatch open +// — fe80::1%en0 needs its interface to be reachable at all — and it must not +// become the one spelling the hatch does not cover. +// +// Asserted at the RESOLVER rather than through a completed request: the zone is +// discarded by resolveHost today (transport.go documents this), so a real dial +// to a link-local address would not work anyway. What must hold is that the +// request is not refused OUT OF HAND when the hatch is open. +func TestSSRFSafeHTTPClient_TheHatchStillAdmitsAZonedLiteral(t *testing.T) { + t.Parallel() + + resolver := &hostRoutedResolver{answers: map[string][]net.IP{}} + + client := NewSSRFSafeHTTPClient(WithPrivateAddressesAllowed()) + transport, ok := client.Transport.(*ssrfSafeTransport) + require.True(t, ok, "NewSSRFSafeHTTPClient must install an ssrfSafeTransport, got %T", client.Transport) + transport.lookupIP = resolver.lookup + + resp, err := client.Get("http://[fe80::1%25en0]/") + if err == nil { + _ = resp.Body.Close() + } + + require.Error(t, err, "the empty resolver answers nothing, so the request fails — but at resolution") + assert.NotErrorIs(t, err, ErrBlockedAddress, + "the zoned literal was refused as a blocked address with the hatch OPEN. allowPrivate means 'this "+ + "client is pointed at a developer-chosen address', and a zoned link-local address is the case "+ + "that most needs it; got: %v", err) + assert.Equal(t, []string{"fe80::1%en0"}, resolver.hostsAsked(), + "with the hatch open the host must reach the resolver exactly as every other literal does") +} diff --git a/internal/atproto/pds/applywrites_test.go b/internal/atproto/pds/applywrites_test.go index c62fdc9..ac534f2 100644 --- a/internal/atproto/pds/applywrites_test.go +++ b/internal/atproto/pds/applywrites_test.go @@ -34,7 +34,12 @@ func newCommitClient(t *testing.T, handler http.HandlerFunc) (CommitClient, func server := httptest.NewServer(handler) - generic, err := NewFromAccessToken(server.URL, applyWritesDID, "test-token") + // The hatch is open because httptest listens on loopback, which is exactly the + // address class the guard refuses. These tests are about applyWrites' WIRE + // FORMAT, not about which addresses may be dialled — factory_guard_test.go + // owns that — so opening it here keeps each test measuring one thing. + generic, err := NewFromAccessToken(server.URL, applyWritesDID, "test-token", + PrivateHostOptions(true)...) if err != nil { server.Close() t.Fatalf("NewFromAccessToken: %v", err) diff --git a/internal/atproto/pds/factory.go b/internal/atproto/pds/factory.go index 660dbac..e5d8351 100644 --- a/internal/atproto/pds/factory.go +++ b/internal/atproto/pds/factory.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "net/http" + "time" covesoauth "Coves/internal/atproto/oauth" + atprotoapi "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/atclient" "github.com/bluesky-social/indigo/atproto/auth/oauth" "github.com/bluesky-social/indigo/atproto/syntax" @@ -18,6 +20,39 @@ import ( // // The oauthClient is used to resume the session and get a properly configured // APIClient that handles DPoP proof generation and nonce rotation automatically. +// +// # IT TAKES NO ClientOption, AND THE REASON IS A CHAIN THIS PACKAGE DOES NOT OWN +// +// The other two constructors build their own HTTP client (see +// newBearerHTTPClient). This one never touches the field: the client arrives +// already installed, copied twice by indigo from the ClientApp the caller hands +// in. Re-read at v0.0.0-20260202181658-ea3d39eec464, the version go.mod pins: +// +// link 1 ours internal/atproto/oauth, in NewOAuthClient +// clientApp.Client = NewSSRFSafeHTTPClient(PrivateAddressOptions(config.AllowPrivateIPs)...) +// link 2 indigo atproto/auth/oauth/oauth.go:200, in ResumeSession +// sess := ClientSession{Client: app.Client, ...} +// link 3 indigo atproto/auth/oauth/session.go:403, in ClientSession.APIClient +// c := atclient.APIClient{Client: sess.Client, ...} +// +// So the OAuth path IS guarded today, and the guard is also correctly gated — +// AllowPrivateIPs at link 1 is the same dev switch PrivateHostOptions carries +// here. +// +// # WHAT THE CHAIN DEGRADES TO, WHICH IS THE PART WORTH KNOWING +// +// Link 1 is an OVERRIDE, not an initialisation: indigo's NewClientApp sets +// `Client: http.DefaultClient` at oauth.go:55. Deleting or skipping our line +// therefore does not produce a nil client that fails loudly — it produces the +// unguarded, un-timed stdlib default, and every OAuth-authenticated PDS call in +// the AppView reverts silently. +// +// Links 2 and 3 are indigo's and can change in a dependency bump; +// factory_guard_test.go fences both. Link 1 is ours and is NOT fenced anywhere +// today — nothing in this tree asserts that clientApp.Client is guarded, so +// deleting that line fails no test. Fencing it means standing up oauth.NewClient +// with a config, which belongs in internal/atproto/oauth's own tests. That is a +// known gap, recorded here rather than assumed away. func NewFromOAuthSession(ctx context.Context, oauthClient *oauth.ClientApp, sessionData *oauth.ClientSessionData) (Client, error) { if oauthClient == nil { return nil, fmt.Errorf("oauthClient is required") @@ -84,7 +119,7 @@ func classifyResumeFailure(err error, did, sessionID string) error { // // Note: This establishes a new session with the PDS. For repeated calls, // consider using NewFromAccessToken if you already have a valid access token. -func NewFromPasswordAuth(ctx context.Context, host, handle, password string) (Client, error) { +func NewFromPasswordAuth(ctx context.Context, host, handle, password string, opts ...ClientOption) (Client, error) { if host == "" { return nil, fmt.Errorf("host is required") } @@ -95,22 +130,37 @@ func NewFromPasswordAuth(ctx context.Context, host, handle, password string) (Cl return nil, fmt.Errorf("password is required") } - // LoginWithPasswordHost creates a session and returns an authenticated APIClient - // This handles the createSession call and Bearer token setup - apiClient, err := atclient.LoginWithPasswordHost(ctx, host, handle, password, "", nil) + // Build the API client before createSession so the password-bearing login + // request uses the same guarded transport as every request after it. Indigo's + // LoginWithPasswordHost cannot be used here because it creates and uses an + // http.DefaultClient internally before returning the APIClient to its caller. + // coves:allow-bare-client: NewAPIClient installs http.DefaultClient (indigo atclient/apiclient.go:43); the next line replaces it before any request is made + apiClient := atclient.NewAPIClient(host) + apiClient.Client = newBearerHTTPClient(opts...) + + session, err := atprotoapi.ServerCreateSession(ctx, apiClient, &atprotoapi.ServerCreateSession_Input{ + Identifier: handle, + Password: password, + }) if err != nil { return nil, fmt.Errorf("failed to login with password: %w", err) } - // Get DID from the authenticated client - did := "" - if apiClient.AccountDID != nil { - did = apiClient.AccountDID.String() + did, err := syntax.ParseDID(session.Did) + if err != nil { + return nil, fmt.Errorf("parsing DID returned by password login: %w", err) } + apiClient.Auth = &atclient.PasswordAuth{Session: atclient.PasswordSessionData{ + AccessToken: session.AccessJwt, + RefreshToken: session.RefreshJwt, + AccountDID: did, + Host: host, + }} + apiClient.AccountDID = &did return &client{ apiClient: apiClient, - did: did, + did: did.String(), host: host, }, nil } @@ -121,7 +171,7 @@ func NewFromPasswordAuth(ctx context.Context, host, handle, password string) (Cl // // WARNING: This creates a client with Bearer auth only. Do NOT use this with // OAuth access tokens - those require DPoP proofs. Use NewFromOAuthSession instead. -func NewFromAccessToken(host, did, accessToken string) (Client, error) { +func NewFromAccessToken(host, did, accessToken string, opts ...ClientOption) (Client, error) { if host == "" { return nil, fmt.Errorf("host is required") } @@ -133,7 +183,9 @@ func NewFromAccessToken(host, did, accessToken string) (Client, error) { } // Create APIClient with Bearer auth + // coves:allow-bare-client: NewAPIClient installs http.DefaultClient (indigo atclient/apiclient.go:43); the next line replaces it before any request is made apiClient := atclient.NewAPIClient(host) + apiClient.Client = newBearerHTTPClient(opts...) apiClient.Auth = &bearerAuth{token: accessToken} return &client{ @@ -143,6 +195,125 @@ func NewFromAccessToken(host, did, accessToken string) (Client, error) { }, nil } +// bearerRequestTimeout bounds a single PDS request made through a Bearer-authed +// client. +// +// The value it replaces is http.DefaultClient's, which is ZERO — and zero in +// net/http means "wait forever". posts/service.go reaches this constructor on a +// request path, so a PDS that accepts a connection and then stops answering used +// to hold that goroutine for the life of the process. 30s matches the longest +// deadline anything else in this tree allows a PDS (blobs' upload POST), because +// this client carries record writes whose size is not bounded as tightly as a +// getRecord's. +const bearerRequestTimeout = 30 * time.Second + +// bearerClientConfig is what the ClientOptions accumulate into. +type bearerClientConfig struct { + // allowPrivateHosts opens the SSRF hatch. NEVER set in production. + allowPrivateHosts bool + + // transportOptions is the TEST SEAM, and it is unexported deliberately: the + // resolver seam these tests need must not be reachable from any non-test + // package, which is the rule the new audit category enforces. + transportOptions []covesoauth.Option +} + +// ClientOption configures a Bearer-authed PDS client. +type ClientOption func(*bearerClientConfig) + +// WithPrivateHostsAllowed disables the SSRF address guard on a Bearer-authed +// PDS client. +// +// THE NAME IS THE CONTRACT: production must not call this. The hosts these +// constructors dial are `community.PDSURL` — a per-community database field — and +// the AppView shares a network with its Postgres, its PDS, its Jetstream and a +// cloud metadata endpoint. Tests that drive a PDS on loopback pass it because +// loopback is exactly what the guard refuses, and a local dev stack runs its PDS +// on the developer's own machine. +func WithPrivateHostsAllowed() ClientOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(c *bearerClientConfig) { c.allowPrivateHosts = true } +} + +// withTransportOptions is the test seam, unexported so production cannot reach +// it. See bearerClientConfig.transportOptions. +func withTransportOptions(opts ...covesoauth.Option) ClientOption { + return func(c *bearerClientConfig) { c.transportOptions = append(c.transportOptions, opts...) } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to these constructors: the hatch when it is set, and +// NOTHING when it is not. +// +// It mirrors oauth.PrivateAddressOptions and the same helper in imageproxy, +// blobs, unfurl and jetstream, and it is a function rather than an `if` at the +// call site for the reason documented there: `.env.ci:140` sets IS_DEV_ENV=true, +// so `make ci` takes the PERMISSIVE branch at every call site holding such a +// boolean. A unit test against this function is the only place in the repository +// where the branch production actually runs is ever evaluated. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's own +// defaults. +func PrivateHostOptions(allowPrivate bool) []ClientOption { + if !allowPrivate { + return nil + } + return []ClientOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + +// newBearerHTTPClient builds the HTTP client the Bearer-authed PDS constructors +// install on their APIClient. +// +// # WHAT IT FIXES +// +// atclient.NewAPIClient leaves APIClient.Client as http.DefaultClient, and +// atclient's apiclient.go substitutes http.DefaultClient again when the field is +// nil — so the unguarded, un-timed default was reached two ways. The field is +// public and documented as customisable, so this is an assignment and not a +// wrapper. +// +// # WHAT THE ADDRESS GUARD IS FOR HERE +// +// The host these constructors dial is `community.PDSURL` — a per-community +// database column, written when a community is created or federated in, and read +// by posts.(*postService).deleteCommunityPost, posts.NewCommunityRepoFactory and +// cmd/rematerialize-posts's communityRepoOpener. deleteCommunityPost reaches this +// constructor on a REQUEST path, so the trigger is an ordinary API call. +// +// This client is Bearer-authed, which makes it the worse half of the two. The +// Authorization header is on the wire before any response exists, so "the +// address was dialled" and "a live PDS credential left the process" are the same +// event. A refused DNS answer costs an attacker a retry; a leaked bearer token +// is not recoverable. +// +// # THE PREVIOUS COMMENT ARGUED THIS COULD NOT BE DONE, AND WAS WRONG +// +// It claimed NewFromAccessToken's shape was pinned by a named function type at +// tests/testkit/pds.go's PasswordAuthFactory, so a variadic option parameter +// would break ~15 integration files. The type is GENERIC over the constructor, +// so widening it carries the option through and every direct call site compiles +// unchanged. What was true in that comment is that the guard cannot be switched +// on without a hatch — every integration and E2E test in this tree drives a PDS +// on loopback through these constructors, which is the address class the guard +// refuses. That is what PrivateHostOptions is for. +// +// # THE TIMEOUT IS THIS SITE'S OWN +// +// bearerRequestTimeout is re-applied over the shared client's 15s ceiling. +// Inheriting the shared value would halve the allowance for every PDS write in +// the AppView as a silent side effect of an SSRF fix. +func newBearerHTTPClient(opts ...ClientOption) *http.Client { + cfg := &bearerClientConfig{} + for _, opt := range opts { + opt(cfg) + } + + client := covesoauth.NewSSRFSafeHTTPClient( + append(covesoauth.PrivateAddressOptions(cfg.allowPrivateHosts), cfg.transportOptions...)...) + client.Timeout = bearerRequestTimeout + return client +} + // bearerAuth implements atclient.AuthMethod for simple Bearer token auth. // This is used for password-based sessions where DPoP is not required. type bearerAuth struct { diff --git a/internal/atproto/pds/factory_guard_test.go b/internal/atproto/pds/factory_guard_test.go new file mode 100644 index 0000000..f1c0529 --- /dev/null +++ b/internal/atproto/pds/factory_guard_test.go @@ -0,0 +1,560 @@ +package pds + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + indigooauth "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/bluesky-social/indigo/atproto/syntax" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The three PDS client entry points reach their HTTP client by different routes: +// +// NewFromOAuthSession sess.APIClient() → Client: sess.Client → app.Client +// NewFromPasswordAuth atclient.NewAPIClient(host) + guarded createSession +// NewFromAccessToken atclient.NewAPIClient(host) +// +// APIClient.Client is a PUBLIC, SETTABLE *http.Client, documented as "May be +// customized after the overall APIClient struct is created; for example to set a +// default request timeout." NewAPIClient initially installs http.DefaultClient, +// so both constructors replace it before their first request. Password login is +// performed through the generated createSession client instead of +// LoginWithPasswordHost, which would make the request before Coves could replace +// its transport. +// +// # WHY THIS FILE WAS REWRITTEN +// +// It previously asserted that the installed client is not `http.DefaultClient` +// and that its Timeout is not zero. Both are true of +// +// &http.Client{Timeout: bearerRequestTimeout} +// +// which is what this package actually built — a client with a deadline and NO +// ADDRESS GUARD — so the file was green while the property its name promised was +// absent. The audit that scans for bare `&http.Client{}` construction found it. +// Everything below asserts refusal and reachability instead, because "which +// client object is installed" is not a security property and "which addresses it +// will dial" is. + +const ( + factoryGuardDID = "did:plc:factoryguard2222222" + + // factoryGuardHost passes every shape check these constructors apply and is a + // name rather than an address, so classification is the only thing that can + // refuse it. `.example` is reserved by RFC 2606, so nothing resolves it for + // real if the seam is ever bypassed. + factoryGuardHost = "https://community-pds.example" +) + +// countingPDS records whether a request ever reached a listener, and what +// credential it carried. +// +// The token is recorded because for these constructors "the listener was +// reached" and "a live PDS credential left the process" are the same event: the +// client is Bearer-authed, so the Authorization header is on the wire before any +// response exists. A blocked request costs an attacker a retry; a leaked bearer +// token is not recoverable. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 + tokens chan string +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{tokens: make(chan string, 8)} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pds.requests.Add(1) + select { + case pds.tokens <- r.Header.Get("Authorization"): + default: + } + w.Header().Set("Content-Type", "application/json") + // Serves both shapes these tests provoke: a createSession response for the + // password login, and a getRecord response for everything after it. + _, _ = w.Write([]byte(`{"did":"` + factoryGuardDID + `","handle":"test.example",` + + `"accessJwt":"access-token","refreshJwt":"refresh-token",` + + `"uri":"at://` + factoryGuardDID + `/app.test/self","cid":"bafyguard",` + + `"value":{"leaked":"from an internal endpoint"}}`)) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// assertNoTokenLeaked fails with the credential named, because the severity of +// this site is not legible from a request count alone. +func (p *countingPDS) assertNoTokenLeaked(t *testing.T) { + t.Helper() + select { + case token := <-p.tokens: + assert.Failf(t, "a PDS access token was handed to the listener", + "the listener received %q. This is credential exfiltration and not only SSRF: the host "+ + "came from a community record, and the guard must refuse it BEFORE the request "+ + "carrying the token is sent", token) + default: + } +} + +// --------------------------------------------------------------------------- +// NewFromAccessToken +// --------------------------------------------------------------------------- + +// TestNewFromAccessToken_RefusesAPrivateHostWithoutReachingIt is the binding +// contract for database-derived PDS destinations. +// +// # WHAT IS BEING DIALLED +// +// `host` is `community.PDSURL` — a per-community database column, written when +// the community is created or federated in, and read by +// posts.(*postService).deleteCommunityPost, posts.NewCommunityRepoFactory and +// cmd/rematerialize-posts's communityRepoOpener. The AppView shares a network +// with its Postgres, its PDS, its Jetstream and a cloud metadata endpoint, and +// deleteCommunityPost reaches this constructor on a REQUEST path, so the trigger +// is an ordinary API call. +func TestNewFromAccessToken_RefusesAPrivateHostWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + c, err := NewFromAccessToken(pds.server.URL, factoryGuardDID, "secret-pds-access-token", + PrivateHostOptions(false)...) + require.NoError(t, err, "constructing a client with valid arguments") + + _, err = c.GetRecord(context.Background(), "app.test", "self") + + // THE REACHABILITY CLAIMS COME FIRST, deliberately. They are the security + // facts; the error is only how the caller learns about them. Asserting the + // error first with require would abort on failure and hide whether the token + // actually left the process. + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. The host is a community's PDSURL — a database column, "+ + "not a constant — and the request leaving the process is the SSRF whatever comes back", + pds.requests.Load()) + pds.assertNoTokenLeaked(t) + + require.Error(t, err, + "a Bearer-authed PDS client read a record from a loopback address and reported success. "+ + "posts.(*postService).deleteCommunityPost reaches this constructor on a request path") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the read failed, but not because the guard refused the address. A PDS that simply could "+ + "not be reached fails identically, so a plain require.Error here cannot tell a refusal "+ + "from an unreachable host — which is how this file was once green while the property its "+ + "name promised was absent, against a build whose client was `&http.Client{Timeout: "+ + "bearerRequestTimeout}`: a deadline and no guard at all; got: %v", err) +} + +// TestNewFromAccessToken_ReachesThePDSWhenTheHatchIsOpen is the other direction, +// and the falsifiability control for the case above. +// +// It is also the property the whole integration tier depends on: every test in +// this tree that writes to the CI stack's PDS goes through this constructor +// against loopback, which is exactly what the guard refuses. +func TestNewFromAccessToken_ReachesThePDSWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + c, err := NewFromAccessToken(pds.server.URL, factoryGuardDID, "secret-pds-access-token", + PrivateHostOptions(true)...) + require.NoError(t, err, "constructing a client with valid arguments") + + record, err := c.GetRecord(context.Background(), "app.test", "self") + + require.NoErrorf(t, err, + "the hatch is what every integration test and every dev stack depends on: a client built "+ + "with PrivateHostOptions(true) must reach a loopback PDS; got: %v", err) + require.NotNil(t, record, "the fixture serves a record, so one must come back") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestNewFromAccessToken_InstallsAClientRatherThanTheStdlibDefault is what this +// file used to assert as its whole contract, kept because it still fences one +// thing the tests above do not. +// +// IT IS NECESSARY AND NOT SUFFICIENT, and the name now says so. A client can +// differ from http.DefaultClient and still dial anything — that is precisely the +// state the audit caught. What this keeps is the NIL route: atclient's +// apiclient.go:142 substitutes http.DefaultClient when Client is nil, so a +// conversion that left the field unset would reach the unguarded default by a +// quieter path than the one everyone reads for. +func TestNewFromAccessToken_InstallsAClientRatherThanTheStdlibDefault(t *testing.T) { + t.Parallel() + + c, err := NewFromAccessToken("https://pds.example", factoryGuardDID, "token") + require.NoError(t, err, "constructing a client with valid arguments") + + concrete, ok := c.(*client) + require.True(t, ok, "NewFromAccessToken must return the concrete *client these tests drive") + require.NotNil(t, concrete.apiClient, "the PDS client must hold an APIClient") + + require.NotNilf(t, concrete.apiClient.Client, + "the APIClient's HTTP client is nil. atclient's apiclient.go:142 substitutes "+ + "http.DefaultClient for a nil client, so leaving it unset reaches the unguarded, "+ + "un-timed default without ever naming it") + assert.NotSamef(t, http.DefaultClient, concrete.apiClient.Client, + "the PDS client is using http.DefaultClient — unguarded, and with no timeout at all") +} + +// TestNewFromAccessToken_PreservesTheRequestTimeout guards the value the shared +// client would otherwise swallow. +// +// NewSSRFSafeHTTPClient ships a 15s ceiling. bearerRequestTimeout is 30s, chosen +// to match the longest deadline anything in this tree allows a PDS because this +// client carries record writes and blob uploads. Adopting the guarded client +// without re-applying it would halve the allowance for every PDS write in the +// AppView, as a silent side effect of an SSRF fix. +func TestNewFromAccessToken_PreservesTheRequestTimeout(t *testing.T) { + t.Parallel() + + c, err := NewFromAccessToken("https://pds.example", factoryGuardDID, "token") + require.NoError(t, err, "constructing a client with valid arguments") + + concrete, ok := c.(*client) + require.True(t, ok, "NewFromAccessToken must return the concrete *client these tests drive") + require.NotNil(t, concrete.apiClient.Client, "the APIClient must hold an HTTP client") + + assert.Equalf(t, bearerRequestTimeout, concrete.apiClient.Client.Timeout, + "the PDS client runs on a %v timeout instead of bearerRequestTimeout (%v). This client "+ + "carries record writes and blob uploads, and no other deadline covers this call site", + concrete.apiClient.Client.Timeout, bearerRequestTimeout) +} + +// --------------------------------------------------------------------------- +// The gate +// --------------------------------------------------------------------------- + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// only place the branch production runs is ever evaluated. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at +// every call site holding such a boolean. A green merge gate therefore says +// nothing about whether production is guarded. +// +// The claim is not "the options returned are safe". It is that there are NONE, so +// what production gets is exactly the constructor's own defaults — a claim a +// reader can check in one glance. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). The production branch must contribute "+ + "nothing at all", len(opts)) +} + +// TestPrivateHostOptions_BindTheGateToTheClient pins the other direction through +// behaviour, so a helper that returns nothing in BOTH directions — which +// satisfies the length check above perfectly — is caught here instead. +func TestPrivateHostOptions_BindTheGateToTheClient(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + c, err := NewFromAccessToken(pds.server.URL, factoryGuardDID, "token", + PrivateHostOptions(true)...) + require.NoError(t, err, "constructing a client with valid arguments") + + _, err = c.GetRecord(context.Background(), "app.test", "self") + + require.NoErrorf(t, err, + "a client built from PrivateHostOptions(true) could not reach a loopback PDS, so the dev "+ + "hatch does nothing and the guarded assertions elsewhere in this file prove only that "+ + "the client refuses everything; got: %v", err) + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// --------------------------------------------------------------------------- +// Classification, through the resolver seam +// --------------------------------------------------------------------------- + +// resolvingAccessTokenClient builds the client the way production does and then +// replaces only its NAME RESOLUTION, so the client under test is the real one. +// +// withTransportOptions is unexported on purpose: the seam must not be reachable +// from any non-test package, which is what the audit's WithHostResolver category +// enforces. +func resolvingAccessTokenClient(t *testing.T, allowPrivateHosts bool, resolvesTo string) Client { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(resolvesTo) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", resolvesTo) + + opts := append(PrivateHostOptions(allowPrivateHosts), + withTransportOptions(covesoauth.WithHostResolver( + func(context.Context, string) ([]net.IP, error) { return []net.IP{ip}, nil }))) + + c, err := NewFromAccessToken(factoryGuardHost, factoryGuardDID, "secret-pds-access-token", opts...) + require.NoError(t, err, "constructing a client with valid arguments") + return c +} + +// TestNewFromAccessToken_RefusesAWellFormedHostThatResolvesPrivate is the +// assertion a loopback-literal fixture cannot make: the guard's CLASSIFICATION +// pass, on a name that survives every earlier check. +// +// It matters here more than at most sites because a community's PDSURL is a +// HOSTNAME in production — `https://pds.somecommunity.example` — never a literal. +// A guard that only refused literals would pass every test above and refuse +// nothing that would actually be attempted. +func TestNewFromAccessToken_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + c := resolvingAccessTokenClient(t, false, "169.254.169.254") + + _, err := c.GetRecord(context.Background(), "app.test", "self") + + require.Errorf(t, err, + "%s is a well-formed https host whose DNS answer was the cloud metadata address, and the "+ + "client read from it anyway. A federated community's PDS URL is chosen by whoever wrote "+ + "the record, and they own the zone", factoryGuardHost) + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a build where this constructor was never "+ + "converted looks identical; got: %v", err) +} + +// TestNewFromAccessToken_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control for the case above. +// +// Identical constructor, identical seam, identical host — only the hatch +// differs. With it open the address is no longer refused, so the request +// proceeds to a dial. That difference is what pins the refusal above to +// classification rather than to this test being unable to make requests at all. +func TestNewFromAccessToken_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + c := resolvingAccessTokenClient(t, true, "127.0.0.1") // coves:allow-host-literal: with the hatch open this is dialled and refused by the OS + + _, err := c.GetRecord(context.Background(), "app.test", "self") + + require.Error(t, err, + "nothing listens on loopback:443, so this read must fail — if it succeeded, the seam is not "+ + "answering with the address this test gave it") + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either PrivateHostOptions "+ + "is not reaching the client, or the guarded case above proves nothing: a client that "+ + "refuses every address refuses that case too, for a reason unconnected to "+ + "classification; got: %v", err) +} + +// --------------------------------------------------------------------------- +// NewFromPasswordAuth +// --------------------------------------------------------------------------- + +// TestNewFromPasswordAuth_RefusesAPrivateHostWithoutReachingIt proves the +// password-bearing createSession request uses the guard too. This is the request +// that Indigo's LoginWithPasswordHost would send through http.DefaultClient. +func TestNewFromPasswordAuth_RefusesAPrivateHostWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + _, err := NewFromPasswordAuth(context.Background(), pds.server.URL, "test.example", "password", + PrivateHostOptions(false)...) + + assert.Zerof(t, pds.requests.Load(), + "the password-bearing login reached the loopback listener %d time(s)", pds.requests.Load()) + require.Error(t, err, "password login to a loopback PDS must be refused") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the login failed, but not because the guard refused the address; got: %v", err) +} + +// TestNewFromPasswordAuth_ReachesThePDSWhenTheHatchIsOpen is the control, and +// the property tests/fixtures and the E2E tier depend on. +func TestNewFromPasswordAuth_ReachesThePDSWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + c, err := NewFromPasswordAuth(context.Background(), pds.server.URL, "test.example", "password", + PrivateHostOptions(true)...) + require.NoErrorf(t, err, "the fake PDS serves a valid createSession response; got: %v", err) + + record, err := c.GetRecord(context.Background(), "app.test", "self") + + require.NoErrorf(t, err, + "with the hatch open a password-authed client must reach a loopback PDS; got: %v", err) + require.NotNil(t, record, "the fixture serves a record, so one must come back") + assert.Equalf(t, int64(2), pds.requests.Load(), + "the fixture must have been reached twice — once by the login, once by the read; got %d", + pds.requests.Load()) +} + +// TestNewFromPasswordAuth_PreservesTheRequestTimeout is the same fence as the +// access-token constructor's, at the other entry point. +func TestNewFromPasswordAuth_PreservesTheRequestTimeout(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + c, err := NewFromPasswordAuth(context.Background(), pds.server.URL, "test.example", "password", + PrivateHostOptions(true)...) + require.NoErrorf(t, err, "the fake PDS serves a valid createSession response; got: %v", err) + + concrete, ok := c.(*client) + require.True(t, ok, "NewFromPasswordAuth must return the concrete *client these tests drive") + require.NotNil(t, concrete.apiClient.Client, "the APIClient must hold an HTTP client") + + assert.Equalf(t, bearerRequestTimeout, concrete.apiClient.Client.Timeout, + "the client runs on a %v timeout instead of bearerRequestTimeout (%v)", + concrete.apiClient.Client.Timeout, bearerRequestTimeout) +} + +// --------------------------------------------------------------------------- +// NewFromOAuthSession — the chain this package does not own +// --------------------------------------------------------------------------- + +// The OAuth path is guarded today by three assignments in two dependencies: +// +// 1. internal/atproto/oauth NewOAuthClient clientApp.Client = NewSSRFSafeHTTPClient(...) +// 2. indigo oauth.go:200 sess := ClientSession{Client: app.Client, ...} +// 3. indigo session.go:403 c := atclient.APIClient{Client: sess.Client, ...} +// +// All three were re-read in the vendored source at +// v0.0.0-20260202181658-ea3d39eec464 and all three hold. Links 2 and 3 are +// indigo's and can vanish in a dependency bump; link 1 is ours. +// +// TWO OF THE THREE ARE FENCED BELOW. Link 1 is not fenced here and cannot be +// from this package — it is an assignment inside NewOAuthClient, reachable only +// by standing up that constructor with a config, which belongs in +// internal/atproto/oauth's own tests. +// +// It is now fenced THERE. An earlier version of this comment ended "today +// nothing in the tree asserts that clientApp.Client is guarded, so deleting that +// line fails no test", which was true when written and is no longer: +// internal/atproto/oauth/client_guard_test.go stands the constructor up and +// asserts the installed client refuses a private address without reaching a live +// listener. Deleting link 1 now fails three tests there. The line is left +// described rather than deleted because the CHAIN is the thing worth reading in +// one place — this file fences links 2 and 3, and a reader needs to know where +// link 1 lives to know the chain is whole. + +// fakeSessionStore is the minimum indigo's ResumeSession needs: a GetSession that +// answers. Every other method panics, which is this repo's convention — a call +// nobody predicted fails immediately rather than returning a zero value that +// quietly changes what the test proves. +type fakeSessionStore struct { + session *indigooauth.ClientSessionData +} + +func (s *fakeSessionStore) GetSession(context.Context, syntax.DID, string) (*indigooauth.ClientSessionData, error) { + return s.session, nil +} + +func (s *fakeSessionStore) SaveSession(context.Context, indigooauth.ClientSessionData) error { + panic("fakeSessionStore: SaveSession is not part of this test's contract") +} + +func (s *fakeSessionStore) DeleteSession(context.Context, syntax.DID, string) error { + panic("fakeSessionStore: DeleteSession is not part of this test's contract") +} + +func (s *fakeSessionStore) GetAuthRequestInfo(context.Context, string) (*indigooauth.AuthRequestData, error) { + panic("fakeSessionStore: GetAuthRequestInfo is not part of this test's contract") +} + +func (s *fakeSessionStore) SaveAuthRequestInfo(context.Context, indigooauth.AuthRequestData) error { + panic("fakeSessionStore: SaveAuthRequestInfo is not part of this test's contract") +} + +func (s *fakeSessionStore) DeleteAuthRequestInfo(context.Context, string) error { + panic("fakeSessionStore: DeleteAuthRequestInfo is not part of this test's contract") +} + +// TestResumeSession_CopiesTheClientAppsClientIntoTheSession fences LINK 2, which +// nothing in this tree previously pinned. +// +// THIS TEST IS A FENCE AND IS EXPECTED TO PASS TODAY. It is here because the +// property is load-bearing and lives in someone else's library: if a dependency +// bump stops ResumeSession copying app.Client, every OAuth-authenticated PDS call +// in the AppView silently reverts to http.DefaultClient — unguarded and with no +// timeout — and no other test in the repository notices. +func TestResumeSession_CopiesTheClientAppsClientIntoTheSession(t *testing.T) { + t.Parallel() + + // A sentinel with a Timeout nothing else in the tree uses, so an assertion on + // identity cannot be satisfied by some other client that happens to exist. + sentinel := &http.Client{Timeout: 41 * time.Second} + + did, err := syntax.ParseDID("did:plc:oauthsessiontest") + require.NoError(t, err, "the test's own DID must parse") + + // ResumeSession's LAST act is to parse the session's DPoP key, so a session + // without one never reaches the assignment under test. Generated rather than + // hard-coded: a real P-256 key is what the parser accepts, and pinning a + // literal would only pin this test to today's encoding. + dpopKey, err := atcrypto.GeneratePrivateKeyP256() + require.NoError(t, err, "the test's own DPoP key must generate") + + app := &indigooauth.ClientApp{ + Client: sentinel, + Config: &indigooauth.ClientConfig{}, + Store: &fakeSessionStore{session: &indigooauth.ClientSessionData{ + AccountDID: did, + SessionID: "session-1", + HostURL: "https://pds.example", + DPoPPrivateKeyMultibase: dpopKey.Multibase(), + }}, + } + + sess, err := app.ResumeSession(context.Background(), did, "session-1") + require.NoErrorf(t, err, "the fake store answers, so the resume must succeed; got: %v", err) + require.NotNil(t, sess, "ResumeSession must return a session") + + assert.Samef(t, sentinel, sess.Client, + "indigo no longer copies ClientApp.Client into the session it builds. That copy is the "+ + "FIRST of the two links that make NewFromOAuthSession guarded — "+ + "internal/atproto/oauth's NewOAuthClient sets ClientApp.Client to a guarded client, and if "+ + "it stops arriving here the guard never reaches the APIClient either") +} + +// TestNewFromOAuthSession_InheritsTheGuardedClientFromTheClientApp fences LINK 3. +// +// It drives indigo's own structs rather than standing up a session store, +// because the link that can break is indigo's: APIClient() is the second copy, +// and a ClientSession literal exercises it directly. Building a real session +// would need a DPoP key and would test the fixture more than the contract. +func TestNewFromOAuthSession_InheritsTheGuardedClientFromTheClientApp(t *testing.T) { + t.Parallel() + + sentinel := &http.Client{Timeout: 41 * time.Second} + + did, err := syntax.ParseDID("did:plc:oauthsessiontest") + require.NoError(t, err, "the test's own DID must parse") + + sess := &indigooauth.ClientSession{ + Client: sentinel, + Config: &indigooauth.ClientConfig{}, + Data: &indigooauth.ClientSessionData{ + AccountDID: did, + HostURL: "https://pds.example", + }, + } + + apiClient := sess.APIClient() + + require.NotNil(t, apiClient, "ClientSession.APIClient must return a client") + assert.Samef(t, sentinel, apiClient.Client, + "indigo no longer threads ClientSession.Client into the APIClient it builds. That chain is "+ + "the ONLY reason NewFromOAuthSession is guarded today: internal/atproto/oauth's NewOAuthClient "+ + "sets ClientApp.Client to a guarded client, ResumeSession copies it into the session, and "+ + "APIClient copies it again. If this assertion fails after a dependency bump, every "+ + "OAuth-authenticated PDS call in the AppView has silently reverted to http.DefaultClient") +} diff --git a/internal/core/blobs/blob_upload_integration_test.go b/internal/core/blobs/blob_upload_integration_test.go index f6d833d..7d2c010 100644 --- a/internal/core/blobs/blob_upload_integration_test.go +++ b/internal/core/blobs/blob_upload_integration_test.go @@ -60,7 +60,19 @@ func TestBlobUpload_E2E_PostWithImages(t *testing.T) { userRepo := postgres.NewUserRepository(db) // Setup services (pdsURL already declared in health check above) - blobService := blobs.NewBlobService(pdsURL) + // + // THE HATCH IS OPEN BECAUSE THE TEST PDS IS ON LOOPBACK, and for no other + // reason. testkit.Endpoints().PDS.BaseURL is the CI stack's PDS on the local + // machine, so its address is exactly the class the upload guard refuses — + // the same reason fetch_guard_test.go opens it for the download half. + // + // This is the honest repair rather than the convenient one. Handing the + // service a client that skips the guarded transport would keep this test + // green while removing what it exercises: the upload below would no longer + // travel the path production travels. The hatch changes ONE decision, the + // address classification, and leaves the vetted dial, the 30s timeout and + // everything else where production has them. + blobService := blobs.NewBlobService(pdsURL, blobs.WithPrivateHostsAllowed()) identityConfig := identity.DefaultConfig() identityResolver := identity.NewResolver(db, identityConfig) userService := users.NewUserService(userRepo, identityResolver, pdsURL, nil, "") @@ -394,7 +406,11 @@ func TestBlobUpload_E2E_CommentWithImage(t *testing.T) { commentRepo := postgres.NewCommentRepository(db) // Setup services (pdsURL already declared in health check above) - blobService := blobs.NewBlobService(pdsURL) + // + // The hatch is open because the test PDS is on loopback; see + // TestBlobUpload_E2E_PostWithImages for why that is the honest repair and + // not a client substitution. + blobService := blobs.NewBlobService(pdsURL, blobs.WithPrivateHostsAllowed()) // Create test author author := fixtures.User(t, db, "commentblob.test", "did:plc:commentblob123") @@ -506,7 +522,11 @@ func TestBlobUpload_PDS_MockServer(t *testing.T) { defer mockPDS.Close() // Create blob service pointing to mock - blobService := blobs.NewBlobService(mockPDS.URL) + // + // The hatch is open because mockPDS is an httptest server on loopback; see + // TestBlobUpload_E2E_PostWithImages for why that is the honest repair and + // not a client substitution. + blobService := blobs.NewBlobService(mockPDS.URL, blobs.WithPrivateHostsAllowed()) // Create test community community := &communities.Community{ @@ -538,7 +558,7 @@ func TestBlobUpload_Validation(t *testing.T) { db := testkit.DB(t) communityRepo := postgres.NewCommunityRepository(db) - blobService := blobs.NewBlobService(testkit.Endpoints().PDS.BaseURL) + blobService := blobs.NewBlobService(testkit.Endpoints().PDS.BaseURL, blobs.PrivateHostOptions(true)...) community := createTestCommunityWithBlobCredentials(t, communityRepo, "validation") ctx := context.Background() diff --git a/internal/core/blobs/service.go b/internal/core/blobs/service.go index 775af20..e220584 100644 --- a/internal/core/blobs/service.go +++ b/internal/core/blobs/service.go @@ -75,6 +75,12 @@ type blobService struct { // assembled per call would be the same code with a per-call chance of being // assembled wrongly. fetchClient *http.Client + + // uploadClient is the client the PDS uploadBlob POST goes through. It is + // SSRF-guarded from the same allowPrivateHosts boolean as fetchClient, and is + // a separate field only because the two carry different options; see + // newBlobUploadClient for why this half is the worse of the two to leave open. + uploadClient *http.Client } // BlobServiceOption configures optional blob service behaviour. @@ -86,10 +92,33 @@ type BlobServiceOption func(*blobService) // the value from config once (the IS_DEV_ENV gate); tests that serve their // fixtures from httptest pass it because loopback is exactly what the guard // refuses. -func WithPrivateHostsAllowed() BlobServiceOption { +func WithPrivateHostsAllowed() BlobServiceOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract return func(s *blobService) { s.allowPrivateHosts = true } } +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to NewBlobService: the hatch when it is set, and NOTHING +// when it is not. +// +// It mirrors oauth.PrivateAddressOptions and the same helper in imageproxy, +// unfurl and jetstream. cmd/server calls THIS rather than writing an inline +// `if a.cfg.IsDevEnv`, which is the one shape this helper exists to replace: +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch, +// and an inline `if` in wiring is reachable only by standing up that wiring with +// a production config — which nothing in this tree does. As a pure function the +// branch production actually runs becomes testable without a config or an +// environment. Do not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's own +// defaults. +func PrivateHostOptions(allowPrivate bool) []BlobServiceOption { + if !allowPrivate { + return nil + } + return []BlobServiceOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + // NewBlobService creates a new blob service func NewBlobService(pdsURL string, opts ...BlobServiceOption) Service { s := &blobService{ @@ -109,11 +138,60 @@ func NewBlobService(pdsURL string, opts ...BlobServiceOption) Service { // allowed: thumbnails come from CDNs that are slow rather than hostile, and // tightening an unrelated timeout while fixing an SSRF hole would be a // second change wearing the first one's clothes. - s.fetchClient = covesoauth.NewSSRFSafeHTTPClient(s.allowPrivateHosts) + s.fetchClient = covesoauth.NewSSRFSafeHTTPClient(covesoauth.PrivateAddressOptions(s.allowPrivateHosts)...) s.fetchClient.Timeout = 30 * time.Second + + // The UPLOAD half's client, built here rather than inline in UploadBlob so + // that both halves of this service are decided in one place. See + // newBlobUploadClient. + s.uploadClient = newBlobUploadClient(s.allowPrivateHosts) return s } +// newBlobUploadClient builds the client the PDS uploadBlob request goes through. +// +// # WHY THIS HALF WAS STILL OPEN +// +// The DOWNLOAD half above has been guarded since before this effort started. +// UploadBlob built a fresh `&http.Client{Timeout: 30 * time.Second}` a hundred +// and forty lines further down, and that asymmetry reads as deliberate until +// someone checks — which is exactly why it survived a security review of the +// file it lives in. +// +// # WHY THE UPLOAD HALF IS THE WORSE OF THE TWO +// +// The download half fetches a URL. This one sends `Authorization: Bearer +// ` to `owner.GetPDSURL()`, which for a federated community is a +// PDS URL carried on a record from another instance. So the primitive here is +// not "make the AppView fetch an internal address" — it is "make the AppView +// POST a live PDS credential to an address of my choosing", with the blob body +// attached. A refused DNS answer costs an attacker nothing to retry; a leaked +// bearer token is not recoverable. +// +// # THE opts PARAMETER IS THE TEST SEAM +// +// It mirrors jetstream's newWellKnownClient and the aggregator's +// registerHTTPClient, and it exists for the reason a mutation found there: a +// guard test that builds its OWN client proves only that internal/atproto/oauth +// works. Passing oauth.WithHostResolver through HERE means the client under test +// is the one this constructor builds, from the same allowPrivateHosts boolean, +// so disabling the guard on this line fails that test. It cannot open the guard. +// Production passes nothing. +// +// # THE TIMEOUT IS THIS SITE'S OWN +// +// 30s, re-applied over the shared client's 15s ceiling exactly as the download +// half does thirty lines up. This POST carries the largest body in the service — +// up to 6 MB of blob — so it is the request here most likely to need the +// headroom, and re-timing it while fixing an SSRF hole would be a second change +// wearing the first one's clothes. +func newBlobUploadClient(allowPrivateHosts bool, opts ...covesoauth.Option) *http.Client { + client := covesoauth.NewSSRFSafeHTTPClient( + append(covesoauth.PrivateAddressOptions(allowPrivateHosts), opts...)...) + client.Timeout = 30 * time.Second + return client +} + // UploadBlobFromURL fetches an image from a URL and uploads it to PDS // Flow: // 1. Fetch image from URL with timeout @@ -252,13 +330,9 @@ func (s *blobService) UploadBlob(ctx context.Context, owner BlobOwner, data []by req.Header.Set("Content-Type", mimeType) req.Header.Set("Authorization", "Bearer "+accessToken) - // Create HTTP client with timeout - client := &http.Client{ - Timeout: 30 * time.Second, - } - - // Execute request - resp, err := client.Do(req) + // Execute request through the service's upload client (see + // newBlobUploadClient). + resp, err := s.uploadClient.Do(req) if err != nil { return nil, fmt.Errorf("PDS request failed: %w", err) } diff --git a/internal/core/blobs/upload_guard_test.go b/internal/core/blobs/upload_guard_test.go new file mode 100644 index 0000000..a96bc4f --- /dev/null +++ b/internal/core/blobs/upload_guard_test.go @@ -0,0 +1,300 @@ +package blobs + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The blob service's UPLOAD half, and the asymmetry that kept it open. +// +// # WHY THIS ONE SURVIVED +// +// NewBlobService builds the DOWNLOAD half's client through the SSRF-safe +// transport, and has since before this effort started. UploadBlob built a fresh +// `&http.Client{Timeout: 30 * time.Second}` a hundred and forty lines further +// down, in the same file, under the same package review. A reader who checks the +// top of the file concludes the service is guarded; a reader who checks the +// bottom concludes it is not; and both stop reading, because the file has +// already answered the question once. +// +// # WHY IT IS THE WORSE HALF +// +// The download half fetches a URL and reads bytes. This half sends +// +// Authorization: Bearer +// +// to `owner.GetPDSURL()` — a PDS URL that for a federated community arrives on a +// record from another instance. So the primitive is not "make the AppView reach +// an internal address", it is "make the AppView POST a live PDS credential to an +// address I chose", with a blob body attached. A blocked fetch costs an attacker +// a retry. A leaked bearer token is not recoverable, and nothing in the response +// has to come back for the attack to have succeeded — which is why every case +// below asserts the listener was NEVER REACHED rather than that an error was +// returned. + +// stubOwner is a BlobOwner pointing at wherever the test says. +type stubOwner struct { + pdsURL string + token string +} + +func (o stubOwner) GetPDSURL() string { return o.pdsURL } +func (o stubOwner) GetPDSAccessToken() string { return o.token } + +// countingPDS records whether anything reached it, and what credential it +// carried. The credential is recorded because "the listener was reached" and +// "the token left the process" are the same event here, and naming it in the +// failure message is what makes the severity legible. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 + tokens chan string +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{tokens: make(chan string, 8)} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + pds.requests.Add(1) + select { + case pds.tokens <- r.Header.Get("Authorization"): + default: + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"blob":{"$type":"blob","ref":{"$link":"bafyupload"},` + + `"mimeType":"image/jpeg","size":3}}`)) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// pngBytes is a payload UploadBlob's MIME and size validation both accept, so a +// refusal below is attributable to the address and to nothing else. +var pngBytes = []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A} + +// TestUploadBlob_RefusesAPrivatePDSWithoutReachingIt is the binding contract for +// this site. +func TestUploadBlob_RefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + service := NewBlobService(pds.server.URL, PrivateHostOptions(false)...) + + _, err := service.UploadBlob(context.Background(), + stubOwner{pdsURL: pds.server.URL, token: "secret-pds-access-token"}, + pngBytes, "image/png") + + // THE REACHABILITY CLAIMS COME FIRST, deliberately. They are the security + // facts; the error is only how the caller learns about them. Asserting the + // error first with require would abort the test on failure and hide whether + // the token actually left the process — which is the one thing a reader of + // this failure most needs to know. + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. For this half of the service the request LEAVING is the "+ + "whole breach: the Authorization header is already on the wire, and no response is "+ + "needed for the token to have been handed over", pds.requests.Load()) + + select { + case token := <-pds.tokens: + assert.Failf(t, "a PDS access token was handed to the listener", + "the listener received %q. This is credential exfiltration and not only SSRF: the "+ + "address came from a federated owner record, and the guard must refuse it BEFORE "+ + "the request carrying the token is sent", token) + default: + } + + require.Error(t, err, + "UploadBlob POSTed a blob and a live PDS bearer token to a loopback address, and reported "+ + "success. The PDS URL comes from the owner record, which for a federated community is "+ + "written by another instance, so this is a stranger choosing where the AppView sends a "+ + "credential") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the upload failed, but not because the guard refused the address. Without the guard's own "+ + "identity in the chain, a build where this half was never converted looks identical — "+ + "an unreachable address fails too; got: %v", err) +} + +// TestUploadBlob_ReachesThePDSWhenTheHatchIsOpen is the other direction, and the +// falsifiability control for the case above: a client that could make no request +// at all would satisfy that test just as well. +func TestUploadBlob_ReachesThePDSWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + service := NewBlobService(pds.server.URL, PrivateHostOptions(true)...) + + ref, err := service.UploadBlob(context.Background(), + stubOwner{pdsURL: pds.server.URL, token: "secret-pds-access-token"}, + pngBytes, "image/png") + + require.NoErrorf(t, err, + "the hatch is what every fixture in this tree and every dev stack depends on: a service "+ + "built with PrivateHostOptions(true) must reach a loopback PDS; got: %v", err) + require.NotNil(t, ref, "a successful upload must return a blob ref") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// gate this site does not currently have. +// +// cmd/server/wiring.go writes `if a.cfg.IsDevEnv { … WithPrivateHostsAllowed() }` +// inline for this service. `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes +// the permissive branch, and an inline `if` in wiring is reachable only by +// standing up that wiring with a production config — which nothing in this tree +// does. So the branch production actually runs is, today, evaluated nowhere. +// +// The claim is not "the options returned are safe". It is that there are NONE. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). The production branch — the one "+ + "IS_DEV_ENV=true keeps `make ci` from ever evaluating — must contribute nothing at all, "+ + "so that what production gets is exactly the constructor's own defaults", len(opts)) +} + +// TestPrivateHostOptions_BindTheGateToTheConstructor pins the other direction +// through the state the constructor ends up in, so a helper that returns nothing +// in BOTH directions — which satisfies the length check above perfectly — is +// caught here instead. +func TestPrivateHostOptions_BindTheGateToTheConstructor(t *testing.T) { + t.Parallel() + + guarded, ok := NewBlobService("https://pds.example", PrivateHostOptions(false)...).(*blobService) + require.True(t, ok, "NewBlobService must return the concrete *blobService these tests drive") + assert.False(t, guarded.allowPrivateHosts, + "a service built from PrivateHostOptions(false) has the SSRF hatch open. This is the branch "+ + "production runs and CI never does") + + hatched, ok := NewBlobService("https://pds.example", PrivateHostOptions(true)...).(*blobService) + require.True(t, ok, "NewBlobService must return the concrete *blobService these tests drive") + assert.True(t, hatched.allowPrivateHosts, + "a service built from PrivateHostOptions(true) is still guarded, so the dev hatch does "+ + "nothing and a developer's local PDS cannot be uploaded to") +} + +// TestUploadBlob_PreservesTheConfiguredTimeout guards the setting the shared +// client would otherwise swallow. +// +// NewSSRFSafeHTTPClient ships a 15s ceiling of its own and this upload has always +// allowed 30s — a blob POST carries up to 6 MB of body, so it is the request in +// this service most likely to need the headroom. NewBlobService already restores +// the download half's for the same reason. +func TestUploadBlob_PreservesTheConfiguredTimeout(t *testing.T) { + t.Parallel() + + service, ok := NewBlobService("https://pds.example").(*blobService) + require.True(t, ok, "NewBlobService must return the concrete *blobService these tests drive") + + require.NotNil(t, service.uploadClient, "the service must hold an upload client") + assert.Equalf(t, 30*time.Second, service.uploadClient.Timeout, + "the upload client runs on a %v timeout instead of the 30s this POST has always allowed. "+ + "The shared SSRF client ships a 15s ceiling, so adopting it without re-applying this "+ + "value silently re-times every blob upload — on the path that carries the largest body "+ + "in the service", service.uploadClient.Timeout) +} + +// # WHY THE GUARD NEEDS ITS OWN FIXTURE HERE TOO +// +// Nothing else in this file separates the guard from the rest of UploadBlob's +// validation, because every address a loopback fixture can offer is refused on +// SHAPE (an IP literal) one branch before classification runs. A mutation that +// disabled classification while leaving the literal check would fail nothing. +// +// So these two drive newBlobUploadClient — the function the constructor calls, +// with the same allowPrivateHosts boolean — and pass the resolver seam through +// it, exactly as jetstream's community-consumer tests do after the same gap was +// found there by mutation. + +// uploadGuardHost passes every shape check UploadBlob applies and is a name +// rather than an address, so classification is the only thing left that can +// refuse it. `.example` is reserved by RFC 2606, so nothing resolves it for real +// if the seam is ever bypassed. +const uploadGuardHost = "https://community-pds.example" + +// resolvingUploadService builds the service the way production does and then +// replaces only its NAME RESOLUTION, so the client under test is the real one. +func resolvingUploadService(t *testing.T, allowPrivateHosts bool, resolvesTo string) *blobService { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(resolvesTo) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", resolvesTo) + + service, ok := NewBlobService(uploadGuardHost, PrivateHostOptions(allowPrivateHosts)...).(*blobService) + require.True(t, ok, "NewBlobService must return the concrete *blobService these tests drive") + + service.uploadClient = newBlobUploadClient(allowPrivateHosts, + covesoauth.WithHostResolver(func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + })) + return service +} + +// TestUploadBlob_RefusesAWellFormedHostThatResolvesPrivate is the assertion a +// literal-shaped fixture cannot make. +func TestUploadBlob_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + service := resolvingUploadService(t, false, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + _, err := service.UploadBlob(context.Background(), + stubOwner{pdsURL: uploadGuardHost, token: "secret-pds-access-token"}, + pngBytes, "image/png") + + require.Errorf(t, err, + "%s is a well-formed https host whose DNS answer was 127.0.0.1, and the upload went ahead. "+ + "A federated community's PDS URL is chosen by whoever wrote the record, and they own "+ + "the zone — so the name looks ordinary and the address is decided after every shape "+ + "check has already passed", uploadGuardHost) + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity. Without it, a build where this half was never "+ + "wired to the guarded client looks identical; got: %v", err) +} + +// TestUploadBlob_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control for the case above. +// +// Identical service, identical seam, identical host — only the hatch differs. +// With it open the address is no longer refused, so the request proceeds to a +// dial, which fails because nothing is listening on loopback:443. The error is +// therefore NOT the guard's, which is what pins the refusal above to +// CLASSIFICATION rather than to this test being unable to make requests at all. +func TestUploadBlob_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + service := resolvingUploadService(t, true, "127.0.0.1") // coves:allow-host-literal: with the hatch open this is dialled and refused by the OS + + _, err := service.UploadBlob(context.Background(), + stubOwner{pdsURL: uploadGuardHost, token: "secret-pds-access-token"}, + pngBytes, "image/png") + + require.Error(t, err, + "nothing listens on loopback:443, so this upload must fail — if it succeeded, the seam is "+ + "not answering with the address this test gave it") + + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either PrivateHostOptions "+ + "is not reaching the client, or the guarded case above proves nothing: a client that "+ + "refuses every address refuses the guarded case too, for a reason that has nothing to "+ + "do with classification; got: %v", err) +} diff --git a/internal/core/blueskypost/fetcher.go b/internal/core/blueskypost/fetcher.go index caef292..412645f 100644 --- a/internal/core/blueskypost/fetcher.go +++ b/internal/core/blueskypost/fetcher.go @@ -160,7 +160,7 @@ type blueskyAPIRecordValue struct { // fetchBlueskyPost fetches a Bluesky post from the public API func fetchBlueskyPost(ctx context.Context, atURI string, timeout time.Duration, api blueskyAPI) (*BlueskyPostResult, error) { // Create SSRF-safe HTTP client - client := oauth.NewSSRFSafeHTTPClient(api.allowPrivateHost) + client := oauth.NewSSRFSafeHTTPClient(oauth.PrivateAddressOptions(api.allowPrivateHost)...) client.Timeout = timeout // Construct API URL diff --git a/internal/core/comments/comment_write_test.go b/internal/core/comments/comment_write_test.go index 36d2af8..d3d3aa1 100644 --- a/internal/core/comments/comment_write_test.go +++ b/internal/core/comments/comment_write_test.go @@ -58,7 +58,7 @@ func TestCommentWrite_CreateTopLevelComment(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } commentService := comments.NewCommentServiceWithPDSFactory( @@ -266,7 +266,7 @@ func TestCommentWrite_CreateNestedReply(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } commentService := comments.NewCommentServiceWithPDSFactory( @@ -406,7 +406,7 @@ func TestCommentWrite_UpdateComment(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } commentService := comments.NewCommentServiceWithPDSFactory( @@ -516,7 +516,7 @@ func TestCommentWrite_DeleteComment(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } commentService := comments.NewCommentServiceWithPDSFactory( @@ -609,7 +609,7 @@ func TestCommentWrite_CannotUpdateOthersComment(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } // Setup service @@ -674,7 +674,7 @@ func TestCommentWrite_CannotDeleteOthersComment(t *testing.T) { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } // Setup service @@ -748,7 +748,7 @@ func TestCommentWrite_ConcurrentModificationDetection(t *testing.T) { if session.HostURL == "" { return nil, fmt.Errorf("session has no host URL") } - return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken) + return pds.NewFromAccessToken(session.HostURL, session.AccountDID.String(), session.AccessToken, pds.PrivateHostOptions(true)...) } commentService := comments.NewCommentServiceWithPDSFactory( @@ -818,7 +818,7 @@ func TestCommentWrite_ConcurrentModificationDetection(t *testing.T) { // Create a PDS client and attempt to update with the stale (original) CID t.Logf("\n🔍 Step 3: Testing concurrent modification detection with stale CID...") - pdsClient, err := pds.NewFromAccessToken(pdsURL, userDID, pdsAccessToken) + pdsClient, err := pds.NewFromAccessToken(pdsURL, userDID, pdsAccessToken, pds.PrivateHostOptions(true)...) if err != nil { t.Fatalf("Failed to create PDS client: %v", err) } diff --git a/internal/core/communities/pds_client.go b/internal/core/communities/pds_client.go new file mode 100644 index 0000000..36b76ba --- /dev/null +++ b/internal/core/communities/pds_client.go @@ -0,0 +1,188 @@ +package communities + +import ( + "context" + "errors" + "log/slog" + "net/http" + "time" + + covesoauth "Coves/internal/atproto/oauth" + + "github.com/bluesky-social/indigo/util" + "github.com/hashicorp/go-retryablehttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +// pdsRequestTimeout is the ceiling every xrpc call in this package has always +// run under, and it is indigo's rather than ours: util.RobustHTTPClient sets +// exactly this on the client it hands back. It is re-applied over the shared +// SSRF client's 15s for the reason blobs/service.go and pds/factory.go re-apply +// theirs — halving the allowance for every community provisioning and token +// refresh would be a second change wearing an SSRF fix's clothes. +// +// It bounds the WHOLE call including retries, again matching indigo: the inner +// client's own Timeout is cleared so three attempts cannot add up to ninety +// seconds. +const pdsRequestTimeout = 30 * time.Second + +// pdsClientConfig is what the options below assemble. +type pdsClientConfig struct { + // allowPrivateHosts opens the SSRF hatch. NEVER set in production. + allowPrivateHosts bool + + // transportOptions is the TEST SEAM, unexported deliberately: the resolver + // seam these tests need must not be reachable from any non-test package. + // pds/factory.go's field of the same name is the shape this copies. + transportOptions []covesoauth.Option +} + +// PDSClientOption configures the HTTP client this package's xrpc calls go +// through. +type PDSClientOption func(*pdsClientConfig) + +// WithPrivateHostsAllowed disables the SSRF address guard on the PDS clients +// this package builds. +// +// THE NAME IS THE CONTRACT: production must not call this. The host these calls +// dial is a community's PDSURL — a per-community database column — and the +// AppView shares a network with its Postgres, its PDS, its Jetstream and, in +// production, a metadata endpoint that hands credentials to anything that can +// reach it. Tests and the local dev stack drive a PDS on loopback, which is +// exactly the address class the guard refuses. +func WithPrivateHostsAllowed() PDSClientOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(c *pdsClientConfig) { c.allowPrivateHosts = true } +} + +// withTransportOptions is the test seam, unexported so production cannot reach +// it. See pdsClientConfig.transportOptions. +func withTransportOptions(opts ...covesoauth.Option) PDSClientOption { + return func(c *pdsClientConfig) { c.transportOptions = append(c.transportOptions, opts...) } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass: the hatch when it is set, and NOTHING when it is not. +// +// It mirrors oauth.PrivateAddressOptions and the same helper in blobs, +// imageproxy, unfurl, jetstream and pds, and it is a function rather than an +// `if` at the call site for the reason documented there: `.env.ci:140` sets +// IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at every call site +// holding such a boolean. A unit test against this function is the only place in +// the repository where the branch production actually runs is ever evaluated. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's own +// defaults. +func PrivateHostOptions(allowPrivate bool) []PDSClientOption { + if !allowPrivate { + return nil + } + return []PDSClientOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + +// newPDSHTTPClient builds the client this package's xrpc.Client values carry. +// +// # WHAT IT FIXES, AND WHY NOTHING HERE LOOKED WRONG +// +// The four call sites in this package build `&xrpc.Client{Host: pdsURL}` and +// leave the optional `.Client` field nil. indigo's getClient() (xrpc/xrpc.go:31) +// then substitutes util.RobustHTTPClient() on EVERY call — so the unguarded +// client is real, is used on every request, and appears in this repository's +// source not at all. That is why an audit sweeping for `&http.Client{` walked +// past all four: there is nothing to grep for. The fix is to stop omitting the +// field. +// +// # WHAT THE ADDRESS GUARD IS FOR HERE +// +// pdsURL is a community's PDSURL, a per-community database column. It is +// operator-pinned today — every community lives on this instance's own PDS — and +// pds_provisioning.go's own doc comment describes V2.1 portability to non-Coves +// PDSs, which is the change that makes this column carry a value somebody else +// chose. Two of the four sites are worse than an ordinary SSRF while they wait: +// refreshPDSToken sends the community's refresh token as the Authorization +// header, and reauthenticateWithPassword POSTs its CLEARTEXT password. The +// credential is on the wire before any response exists, so "the address was +// dialled" and "a live credential left the process" are one event. +// +// # THE RETRY WRAPPER IS PRESERVED, NOT ADDED +// +// util.RobustHTTPClient is not merely an unguarded client — it is an unguarded +// client with three retries, indigo's XRPC retry policy (which treats 429 as +// final), otel instrumentation and a 30s total ceiling. Replacing it with a bare +// guarded client would fix the SSRF hole and silently delete the retries, so one +// transient 5xx would fail a community's provisioning or its token refresh. This +// reproduces RobustHTTPClient exactly, with the ONE substitution that is the +// point: the SSRF-safe transport where cleanhttp's pooled transport used to be. +// +// LAYER ORDER MATTERS AND IS INDIGO'S. otel wraps the transport that dials, so +// a refused dial is still a recorded span; retryablehttp sits outside the client +// entirely, so each attempt is a full guarded round trip — resolved, classified +// and dialled afresh — rather than a retry against an address vetted once. +func newPDSHTTPClient(opts ...PDSClientOption) *http.Client { + cfg := &pdsClientConfig{} + for _, opt := range opts { + opt(cfg) + } + + inner := covesoauth.NewSSRFSafeHTTPClient( + append(covesoauth.PrivateAddressOptions(cfg.allowPrivateHosts), cfg.transportOptions...)...) + inner.Transport = otelhttp.NewTransport(inner.Transport) + + // Cleared so the 30s below is a budget for the whole call rather than for + // each of four attempts, which is how util.RobustHTTPClient spends it. + inner.Timeout = 0 + + // coves:allow-bare-client: NewClient installs cleanhttp.DefaultPooledClient (go-retryablehttp client.go:431); the next line replaces it with the guarded client built above + retryClient := retryablehttp.NewClient() + retryClient.HTTPClient = inner + retryClient.RetryMax = 3 + retryClient.RetryWaitMin = 1 * time.Second + retryClient.RetryWaitMax = 10 * time.Second + retryClient.Logger = retryablehttp.LeveledLogger(pdsRetryLogger{ + inner: slog.Default().With("subsystem", "communities.pdsClient"), + }) + retryClient.CheckRetry = pdsRetryPolicy + + client := retryClient.StandardClient() + client.Timeout = pdsRequestTimeout + return client +} + +// pdsRetryPolicy is indigo's XRPC retry policy with the transport's own +// refusals made FINAL. +// +// retryablehttp's default treats any transport-level error as transient, so +// without this a refused address is dialled again after 1s, 2s and 4s — four +// identical security decisions, seven seconds of latency on a path a request is +// waiting behind, and four log lines suggesting a flaky network where there is a +// deliberate block. A response over the byte cap is the same kind of answer: +// re-fetching it produces the same oversized body. +// +// It returns the error rather than nil so the caller still sees WHY, which is +// what every assertion on ErrBlockedAddress in this package depends on. +// +// Everything else stays indigo's, including its one deliberate departure from +// retryablehttp's default: 429 is NOT retried, so rate limiting is the +// application's decision rather than a wait this client takes on its behalf. +func pdsRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + if errors.Is(err, covesoauth.ErrBlockedAddress) || errors.Is(err, covesoauth.ErrResponseTooLarge) { + return false, err + } + return util.XRPCRetryPolicy(ctx, resp, err) +} + +// pdsRetryLogger adapts slog to retryablehttp's leveled logger. +// +// It exists because indigo's own util.LeveledSlog has an unexported field, so it +// cannot be constructed from here — the type is exported and uninstantiable. The +// level mapping is indigo's: a retryablehttp ERROR is an INTERMEDIATE failure +// that is about to be retried, so logging it at error level would page someone +// about a request that then succeeded. +type pdsRetryLogger struct { + inner *slog.Logger +} + +func (l pdsRetryLogger) Error(msg string, args ...any) { l.inner.Warn(msg, args...) } +func (l pdsRetryLogger) Warn(msg string, args ...any) { l.inner.Warn(msg, args...) } +func (l pdsRetryLogger) Info(msg string, args ...any) { l.inner.Info(msg, args...) } +func (l pdsRetryLogger) Debug(msg string, args ...any) { l.inner.Debug(msg, args...) } diff --git a/internal/core/communities/pds_client_guard_test.go b/internal/core/communities/pds_client_guard_test.go new file mode 100644 index 0000000..2b6bc74 --- /dev/null +++ b/internal/core/communities/pds_client_guard_test.go @@ -0,0 +1,315 @@ +package communities + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + covesoauth "Coves/internal/atproto/oauth" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The four xrpc.Client sites in this package, which were unguarded by OMISSION +// rather than by construction. +// +// # WHY THESE FOUR LOOKED SAFE +// +// Every other site in this remediation held a visible `&http.Client{}`. These +// four hold `&xrpc.Client{Host: pdsURL}` and leave the optional `.Client` field +// nil — and indigo's getClient() (xrpc/xrpc.go:31-36) then substitutes +// util.RobustHTTPClient() on every call. So the unguarded client is real, is +// used on every request, and appears in this repository's source not at all. +// Nothing to grep for is why the audit's `&http.Client{` sweep walked past them. +// +// # TWO OF THEM CARRY LIVE CREDENTIALS TO THE HOST THEY DIAL +// +// refreshPDSToken sends a community's PDSRefreshToken as the Authorization +// header. reauthenticateWithPassword POSTs the community's cleartext PDSPassword +// and PDSEmail. Both are on the wire before any response exists, so "the address +// was dialled" and "a live PDS credential left the process" are the same event. +// A refused address costs an attacker a retry; a leaked password does not come +// back. +// +// # THE HOST IS OPERATOR-PINNED TODAY AND THE FILE PLANS OTHERWISE +// +// pdsURL is fresh.PDSURL, a per-community database column, which today is always +// this instance's own PDS. pds_provisioning.go's own doc comment describes V2.1 +// portability to non-Coves PDSs, and that is the commit where this column starts +// carrying a value someone else chose. Guarding it now costs nothing; guarding it +// then means remembering. + +// recordingPDS is a loopback PDS that records whether it was ever reached. +// +// THE ASSERTION IS "NEVER INVOKED", not "an error came back". A guard that +// refuses AFTER delivering the request is byte-identical from the caller's side +// and useless — that exact mutation survived every error-message assertion in +// an earlier mutation test, and only the reached/not-reached counter caught it. It matters more +// here than anywhere else in this remediation, because what the request carries +// is the credential. +func recordingPDS(t *testing.T) (url string, reached *atomic.Int64) { + t.Helper() + + var hits atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"did":"did:web:pds.example","accessJwt":"a","refreshJwt":"r","handle":"h"}`)) + })) + t.Cleanup(server.Close) + return server.URL, &hits +} + +// resolverOption points the guarded client's NAME RESOLUTION at a chosen +// address, so the classification pass can be driven by a hostname that is +// otherwise well-formed. It cannot open the guard. +func resolverOption(t *testing.T, answer string) PDSClientOption { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd literal would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(answer) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", answer) + + return withTransportOptions(covesoauth.WithHostResolver( + func(context.Context, string) ([]net.IP, error) { return []net.IP{ip}, nil })) +} + +func TestRefreshPDSToken_RefusesAPrivatePDSWithoutSendingTheRefreshToken(t *testing.T) { + t.Parallel() + + pdsURL, reached := recordingPDS(t) + + _, _, err := refreshPDSToken(context.Background(), newPDSHTTPClient(), pdsURL, "the-refresh-token") + + // THE REACHABILITY CLAIM COMES FIRST, and the ordering is load-bearing rather + // than stylistic. require aborts the test, so asserting the error first means + // that under the mutation this test exists to catch — deleting + // `retryClient.HTTPClient = inner`, which leaves retryablehttp's own + // cleanhttp.DefaultPooledClient in place and the request succeeding — the + // failure reads "An error is expected but got nil" and says NOTHING about + // whether the refresh token left the process. That is the one fact a reader of + // this failure most needs. Verified by running that mutation, not assumed. + assert.Zerof(t, reached.Load(), + "the PDS listener was reached %d time(s). refreshPDSToken sends the community's refresh token "+ + "as the Authorization header, so a guard that refuses AFTER the request is indistinguishable "+ + "from no guard: the credential is already gone", reached.Load()) + + require.Error(t, err, "a PDS on loopback must be refused when the hatch is shut") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal does not carry the guard's identity, so a build where xrpc.Client's nil .Client "+ + "field is still falling back to util.RobustHTTPClient() looks the same — some error would "+ + "come back either way; got: %v", err) +} + +// TestRefreshPDSToken_RefusesAWellFormedHostThatResolvesPrivate is the +// assertion a loopback-literal fixture cannot make. +// +// The transport refuses an IP LITERAL on shape, one branch before it classifies +// anything, so the test above passes against an implementation that only checks +// shape. A hostname that resolves privately is the input that reaches +// classification, and it is the cheapest SSRF here once PDSURL carries a value +// somebody else chose: the attacker owns the zone. +func TestRefreshPDSToken_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + client := newPDSHTTPClient(resolverOption(t, "127.0.0.1")) // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + _, _, err := refreshPDSToken(context.Background(), client, + "https://pds.aggregator.example", "the-refresh-token") + + require.Error(t, err, "a well-formed host resolving to loopback must be refused") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "a hostname whose DNS answer was 127.0.0.1 was not refused by CLASSIFICATION. This is the "+ + "input an IP-literal fixture cannot produce and the one an attacker controlling a zone "+ + "actually uses; got: %v", err) +} + +func TestReauthenticateWithPassword_RefusesAPrivatePDSWithoutSendingThePassword(t *testing.T) { + t.Parallel() + + pdsURL, reached := recordingPDS(t) + + _, _, err := reauthenticateWithPassword(context.Background(), newPDSHTTPClient(), + pdsURL, "c-test@example.com", "the-cleartext-password") + + // Reachability first — see the note in the refresh-token case above. + assert.Zerof(t, reached.Load(), + "the PDS listener was reached %d time(s). This call POSTs the community's CLEARTEXT password "+ + "and email in the request body — the worst payload in this package to deliver to an address "+ + "of someone else's choosing", reached.Load()) + + require.Error(t, err, "a PDS on loopback must be refused when the hatch is shut") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal does not carry the guard's identity; got: %v", err) +} + +func TestProvisionCommunityAccount_RefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pdsURL, reached := recordingPDS(t) + + _, err := NewPDSAccountProvisioner("coves.example", pdsURL). + ProvisionCommunityAccount(context.Background(), "guardtest") + + // Reachability first — see the note in the refresh-token case above. + assert.Zerof(t, reached.Load(), + "the PDS listener was reached %d time(s). createAccount POSTs a generated password and the "+ + "community's system email", reached.Load()) + + require.Error(t, err, "a PDS on loopback must be refused when the hatch is shut") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal does not carry the guard's identity; got: %v", err) +} + +func TestFetchPDSDID_RefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pdsURL, reached := recordingPDS(t) + + _, err := FetchPDSDID(context.Background(), pdsURL) + + // Reachability first — see the note in the refresh-token case above. + assert.Zerof(t, reached.Load(), + "the PDS listener was reached %d time(s)", reached.Load()) + + require.Error(t, err, "a PDS on loopback must be refused when the hatch is shut") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal does not carry the guard's identity; got: %v", err) +} + +// TestNewPDSHTTPClient_TheHatchOpensIt is the falsifiability control. Without +// it, a client that could not make any request at all — a transport wired to +// nothing, a fixture that never starts — would satisfy every refusal above. +func TestNewPDSHTTPClient_TheHatchOpensIt(t *testing.T) { + t.Parallel() + + pdsURL, reached := recordingPDS(t) + + _, err := FetchPDSDID(context.Background(), pdsURL, PrivateHostOptions(true)...) + + require.NoError(t, err, "with the hatch open the same loopback PDS must answer") + assert.Positivef(t, reached.Load(), + "the listener was never reached even with the hatch OPEN, so the refusals above prove nothing "+ + "about classification — this client cannot make requests at all") +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// only place in this repository where the branch production actually runs is +// evaluated. `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the +// PERMISSIVE branch at every call site holding such a boolean. +// +// ZERO, not "options that are safe": what production gets must be exactly the +// constructor's own defaults, which is a claim a reader can check in one glance. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + assert.Empty(t, PrivateHostOptions(false), + "PrivateHostOptions(false) returned options. The production branch must apply NOTHING, so that "+ + "a reviewer reading newPDSHTTPClient sees the whole of what production runs") + assert.Len(t, PrivateHostOptions(true), 1, + "PrivateHostOptions(true) must return exactly the hatch") +} + +// TestNewPDSHTTPClient_PreservesTheRobustClientsTimeout pins the ceiling these +// four sites have always run under. +// +// indigo's util.RobustHTTPClient sets 30s. The shared SSRF client ships 15s, so +// adopting it without re-applying would HALVE the allowance for every community +// provisioning and token refresh in the AppView, as a silent side effect of an +// SSRF fix — a second change wearing the first one's clothes. +func TestNewPDSHTTPClient_PreservesTheRobustClientsTimeout(t *testing.T) { + t.Parallel() + + assert.Equalf(t, 30*time.Second, newPDSHTTPClient().Timeout, + "the PDS client runs on a %v ceiling. indigo's util.RobustHTTPClient — what xrpc.Client's nil "+ + ".Client field used to fall back to — allows 30s across all retries, and that is the "+ + "behaviour this conversion has to preserve", newPDSHTTPClient().Timeout) +} + +// TestNewPDSHTTPClient_KeepsTheRetryBehaviourItReplaces is the other half of +// "preserve what was there". +// +// util.RobustHTTPClient retries connection errors and 5xx up to three times. +// Dropping that while fixing an SSRF hole would make a single transient blip +// fail a community's creation or its token refresh — a regression arriving +// inside a security fix and attributable to nothing. +func TestNewPDSHTTPClient_KeepsTheRetryBehaviourItReplaces(t *testing.T) { + t.Parallel() + + // ONE failure, not three: retryablehttp's first backoff is a full second, so + // each extra attempt costs this T0 test a second of wall clock to re-prove + // something the first retry already proved. + var attempts atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if attempts.Add(1) < 2 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"did":"did:web:pds.example"}`)) + })) + t.Cleanup(server.Close) + + did, err := FetchPDSDID(context.Background(), server.URL, PrivateHostOptions(true)...) + + require.NoError(t, err, "a transient 500 must be ridden out, as util.RobustHTTPClient rode it out") + assert.Equal(t, "did:web:pds.example", did) + assert.EqualValuesf(t, 2, attempts.Load(), + "the PDS was called %d time(s). util.RobustHTTPClient retries 5xx, and these four sites have "+ + "always had that; a conversion that silently drops it turns one bad gateway response into a "+ + "failed community provisioning or a failed token refresh", attempts.Load()) +} + +// TestNewPDSHTTPClient_DoesNotRetryTheGuardsRefusal is the other side of keeping +// the retry wrapper: retryablehttp's default treats every transport-level error +// as transient, and a refused address is not one. +// +// Without this the guard's decision is taken four times, 1s + 2s + 4s of backoff +// are spent on a request somebody is waiting behind, and four log lines suggest a +// flaky network where there is a deliberate block. Asserted on the CLOCK because +// the retry attempts are invisible from the outside — the listener is never +// reached, so a counter cannot see them. +func TestNewPDSHTTPClient_DoesNotRetryTheGuardsRefusal(t *testing.T) { + t.Parallel() + + pdsURL, _ := recordingPDS(t) + + start := time.Now() + _, err := FetchPDSDID(context.Background(), pdsURL) + elapsed := time.Since(start) + + require.Error(t, err, "a PDS on loopback must be refused when the hatch is shut") + assert.Lessf(t, elapsed, time.Second, + "the refusal took %v. retryablehttp's default policy re-dials a transport error after 1s, 2s "+ + "and 4s, so a blocked address is being classified four times and answered seven seconds "+ + "late — with four warnings implying a flaky network rather than one saying the address was "+ + "refused", elapsed) +} + +// TestNewCommunityService_BuildsAGuardedPDSClientByDefault pins the wiring, not +// the helper: a service constructed with no options must hold a guarded client, +// because that is what forgetting looks like and forgetting has to be safe. +func TestNewCommunityService_BuildsAGuardedPDSClientByDefault(t *testing.T) { + t.Parallel() + + svc, ok := NewCommunityService(nil, "https://pds.example", "did:web:coves.example", + "coves.example", nil, nil, nil).(*communityService) + require.True(t, ok, "NewCommunityService must return the concrete service these tests drive") + require.NotNil(t, svc.pdsHTTPClient, "the service must hold a PDS client, not nil") + + pdsURL, reached := recordingPDS(t) + _, _, err := refreshPDSToken(context.Background(), svc.pdsHTTPClient, pdsURL, "the-refresh-token") + + // Reachability first — see the note in the refresh-token case above. + assert.Zerof(t, reached.Load(), "the service's client reached the listener %d time(s)", reached.Load()) + + require.Error(t, err, "the service's own client must refuse a loopback PDS") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the service's default PDS client is not the guarded one; got: %v", err) +} diff --git a/internal/core/communities/pds_provisioning.go b/internal/core/communities/pds_provisioning.go index fd86d84..6561db6 100644 --- a/internal/core/communities/pds_provisioning.go +++ b/internal/core/communities/pds_provisioning.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/base64" "fmt" + "net/http" "strings" "github.com/bluesky-social/indigo/api/atproto" @@ -41,13 +42,27 @@ func (c *CommunityPDSAccount) GetPDSAccessToken() string { type PDSAccountProvisioner struct { instanceDomain string pdsURL string // URL to call PDS (e.g., http://localhost:3001) + + // httpClient is the SSRF-guarded client createAccount goes through, built + // ONCE at construction. Per blobs/service.go's fetchClient: the guard's whole + // property is that it resolves the host and dials only the address it vetted, + // so a client assembled per call would be the same code with a per-call chance + // of being assembled wrongly. + httpClient *http.Client } // NewPDSAccountProvisioner creates a new provisioner for V2.0 (PDS-managed keys) -func NewPDSAccountProvisioner(instanceDomain, pdsURL string) *PDSAccountProvisioner { +// +// THE CLIENT IT BUILDS IS GUARDED UNLESS THE CALLER SAYS OTHERWISE, so that +// forgetting is safe: NewPDSAccountProvisioner with no options is what the next +// caller will write. cmd/server and cmd/rematerialize-posts pass +// PrivateHostOptions(cfg.IsDevEnv); tests that drive a PDS on loopback pass +// PrivateHostOptions(true), because loopback is exactly what the guard refuses. +func NewPDSAccountProvisioner(instanceDomain, pdsURL string, opts ...PDSClientOption) *PDSAccountProvisioner { return &PDSAccountProvisioner{ instanceDomain: instanceDomain, pdsURL: pdsURL, + httpClient: newPDSHTTPClient(opts...), } } @@ -102,8 +117,12 @@ func (p *PDSAccountProvisioner) ProvisionCommunityAccount( // 3. Create a DID (did:plc:xxx) // 4. Register DID with PLC directory // 5. Return credentials (DID, handle, tokens) + // Client is set EXPLICITLY. Leaving it nil makes indigo's getClient() + // substitute util.RobustHTTPClient() — no address guard — on a POST carrying + // this community's generated password and system email. See newPDSHTTPClient. client := &xrpc.Client{ - Host: p.pdsURL, + Client: p.httpClient, + Host: p.pdsURL, } emailStr := email @@ -167,9 +186,14 @@ func generateSecurePassword(length int) (string, error) { // FetchPDSDID queries the PDS to get its DID via com.atproto.server.describeServer // This is the proper way to get the PDS DID rather than hardcoding it // Works in both development (did:web:localhost) and production (did:web:pds.example.com) -func FetchPDSDID(ctx context.Context, pdsURL string) (string, error) { +// +// It takes the options rather than a client because it is a standalone function +// with no construction to hang one off. Omitting them yields the GUARDED client, +// which is the branch a caller that forgets lands on. +func FetchPDSDID(ctx context.Context, pdsURL string, opts ...PDSClientOption) (string, error) { client := &xrpc.Client{ - Host: pdsURL, + Client: newPDSHTTPClient(opts...), + Host: pdsURL, } resp, err := comatproto.ServerDescribeServer(ctx, client) diff --git a/internal/core/communities/provisioner_failure_test.go b/internal/core/communities/provisioner_failure_test.go index 5b20cb0..d53b91c 100644 --- a/internal/core/communities/provisioner_failure_test.go +++ b/internal/core/communities/provisioner_failure_test.go @@ -69,7 +69,7 @@ func TestProvisioner_FailsClosedWhenThePDSCannotBeReached(t *testing.T) { "://missing-scheme", "", } { - provisioner := communities.NewPDSAccountProvisioner(instanceDomain, badURL) + provisioner := communities.NewPDSAccountProvisioner(instanceDomain, badURL, communities.PrivateHostOptions(true)...) _, err := provisioner.ProvisionCommunityAccount(ctx, "testcommunity") assert.Errorf(t, err, "provisioning against PDS URL %q must fail", badURL) } @@ -78,7 +78,7 @@ func TestProvisioner_FailsClosedWhenThePDSCannotBeReached(t *testing.T) { t.Run("reports an unreachable PDS", func(t *testing.T) { t.Parallel() - provisioner := communities.NewPDSAccountProvisioner(instanceDomain, unreachableAddress(t)) + provisioner := communities.NewPDSAccountProvisioner(instanceDomain, unreachableAddress(t), communities.PrivateHostOptions(true)...) _, err := provisioner.ProvisionCommunityAccount(ctx, "testcommunity") require.Error(t, err) @@ -100,7 +100,7 @@ func TestProvisioner_FailsClosedWhenThePDSCannotBeReached(t *testing.T) { expired, cancel := context.WithTimeout(ctx, time.Nanosecond) defer cancel() - provisioner := communities.NewPDSAccountProvisioner(instanceDomain, testkit.Endpoints().PDS.BaseURL) + provisioner := communities.NewPDSAccountProvisioner(instanceDomain, testkit.Endpoints().PDS.BaseURL, communities.PrivateHostOptions(true)...) _, err := provisioner.ProvisionCommunityAccount(expired, "testcommunity") require.Error(t, err, "a request with an expired deadline must not reach a live PDS") }) @@ -118,7 +118,7 @@ func TestFetchPDSDID(t *testing.T) { t.Run("reads the DID from a live PDS", func(t *testing.T) { t.Parallel() - did, err := communities.FetchPDSDID(ctx, testkit.Endpoints().PDS.BaseURL) + did, err := communities.FetchPDSDID(ctx, testkit.Endpoints().PDS.BaseURL, communities.PrivateHostOptions(true)...) require.NoError(t, err) assert.NotEmpty(t, did) assert.Contains(t, did, "did:", "com.atproto.server.describeServer must answer with a DID, got %q", did) @@ -128,7 +128,7 @@ func TestFetchPDSDID(t *testing.T) { t.Parallel() for _, badURL := range []string{"not-a-url", "http://", ""} { - _, err := communities.FetchPDSDID(ctx, badURL) + _, err := communities.FetchPDSDID(ctx, badURL, communities.PrivateHostOptions(true)...) assert.Errorf(t, err, "FetchPDSDID must fail for %q rather than return an empty DID", badURL) } }) @@ -137,7 +137,7 @@ func TestFetchPDSDID(t *testing.T) { t.Parallel() address := unreachableAddress(t) - _, err := communities.FetchPDSDID(ctx, address) + _, err := communities.FetchPDSDID(ctx, address, communities.PrivateHostOptions(true)...) require.Error(t, err) assert.ErrorContains(t, err, "failed to describe server") assert.ErrorContains(t, err, address, "the error must name the server that did not answer") @@ -149,7 +149,7 @@ func TestFetchPDSDID(t *testing.T) { expired, cancel := context.WithTimeout(ctx, time.Nanosecond) defer cancel() - _, err := communities.FetchPDSDID(expired, testkit.Endpoints().PDS.BaseURL) + _, err := communities.FetchPDSDID(expired, testkit.Endpoints().PDS.BaseURL, communities.PrivateHostOptions(true)...) require.Error(t, err) }) } diff --git a/internal/core/communities/service.go b/internal/core/communities/service.go index be227f5..4e609dc 100644 --- a/internal/core/communities/service.go +++ b/internal/core/communities/service.go @@ -46,6 +46,13 @@ type communityService struct { oauthClient *oauthclient.OAuthClient pdsClientFactory PDSClientFactory // Optional, for testing. If nil, uses OAuth. + // pdsHTTPClient is the SSRF-guarded client the token-refresh and + // password-reauth xrpc calls go through, built ONCE at construction. Both + // carry a live credential to fresh.PDSURL — a per-community database column — + // so this field is what stands between a credential and an address someone + // else chose. See newPDSHTTPClient. + pdsHTTPClient *http.Client + // Token refresh concurrency control // Each community gets its own mutex to prevent concurrent refresh attempts refreshMutexes map[string]*sync.Mutex @@ -68,12 +75,18 @@ const ( ) // NewCommunityService creates a new community service with OAuth client for user authentication +// +// The variadic PDSClientOptions configure the SSRF posture of the token-refresh +// and password-reauth calls, and omitting them yields the GUARDED client — the +// branch a caller that forgets lands on. cmd/server passes +// PrivateHostOptions(cfg.IsDevEnv). func NewCommunityService( repo Repository, pdsURL, instanceDID, instanceDomain string, provisioner *PDSAccountProvisioner, oauthClient *oauthclient.OAuthClient, blobService blobs.Service, + opts ...PDSClientOption, ) Service { // SECURITY: Basic validation that did:web domain matches configured instanceDomain // This catches honest configuration mistakes but NOT malicious code modifications @@ -97,6 +110,7 @@ func NewCommunityService( provisioner: provisioner, oauthClient: oauthClient, blobService: blobService, + pdsHTTPClient: newPDSHTTPClient(opts...), refreshMutexes: make(map[string]*sync.Mutex), } } @@ -109,6 +123,7 @@ func NewCommunityServiceWithPDSFactory( provisioner *PDSAccountProvisioner, factory PDSClientFactory, blobService blobs.Service, + opts ...PDSClientOption, ) Service { return &communityService{ repo: repo, @@ -118,6 +133,7 @@ func NewCommunityServiceWithPDSFactory( provisioner: provisioner, pdsClientFactory: factory, blobService: blobService, + pdsHTTPClient: newPDSHTTPClient(opts...), refreshMutexes: make(map[string]*sync.Mutex), } } @@ -679,7 +695,7 @@ func (s *communityService) EnsureFreshToken(ctx context.Context, community *Comm log.Printf("[TOKEN-REFRESH] Community: %s, Event: token_refresh_started, Message: Access token expiring soon", fresh.DID) // Attempt token refresh using refresh token - newAccessToken, newRefreshToken, err := refreshPDSToken(ctx, fresh.PDSURL, fresh.PDSRefreshToken) + newAccessToken, newRefreshToken, err := refreshPDSToken(ctx, s.pdsHTTPClient, fresh.PDSURL, fresh.PDSRefreshToken) if err != nil { // Check if refresh token expired (need password fallback) // Match both "ExpiredToken" and "Token has expired" error messages @@ -689,6 +705,7 @@ func (s *communityService) EnsureFreshToken(ctx context.Context, community *Comm // Fallback: Re-authenticate with stored password newAccessToken, newRefreshToken, err = reauthenticateWithPassword( ctx, + s.pdsHTTPClient, fresh.PDSURL, fresh.PDSEmail, fresh.PDSPassword, // Retrieved decrypted from DB @@ -1345,7 +1362,7 @@ func (s *communityService) callPDSWithAuth(ctx context.Context, method, endpoint timeout = 30 * time.Second // Extended timeout for write operations } - client := &http.Client{Timeout: timeout} + client := &http.Client{Timeout: timeout} // coves:allow-bare-client: builds from s.pdsURL, the AppView's own configured PDS, not a caller-supplied endpoint resp, err := client.Do(req) if err != nil { return "", "", fmt.Errorf("failed to call PDS: %w", err) diff --git a/internal/core/communities/service_provisioning_test.go b/internal/core/communities/service_provisioning_test.go index 9758ef5..df92deb 100644 --- a/internal/core/communities/service_provisioning_test.go +++ b/internal/core/communities/service_provisioning_test.go @@ -89,9 +89,10 @@ func newCommunityServiceWithDatabase(t *testing.T) ( pdsServer.URL(), instanceDID, instanceDomain, - communities.NewPDSAccountProvisioner(instanceDomain, pdsServer.URL()), - testkit.PasswordAuthFactory(pds.NewFromAccessToken), - blobs.NewBlobService(pdsServer.URL()), + communities.NewPDSAccountProvisioner(instanceDomain, pdsServer.URL(), communities.PrivateHostOptions(true)...), + testkit.PasswordAuthFactory(pds.NewFromAccessToken, pds.PrivateHostOptions(true)...), + blobs.NewBlobService(pdsServer.URL(), blobs.PrivateHostOptions(true)...), + communities.PrivateHostOptions(true)..., ) return service, repo, pdsServer, db } diff --git a/internal/core/communities/service_readpath_test.go b/internal/core/communities/service_readpath_test.go index 7cd2926..515f8a1 100644 --- a/internal/core/communities/service_readpath_test.go +++ b/internal/core/communities/service_readpath_test.go @@ -60,7 +60,8 @@ func newFakeBackedService(t *testing.T) (communities.Service, *fakeCommunityRepo t.Helper() repo := newFakeCommunityRepo() service := communities.NewCommunityServiceWithPDSFactory( - repo, "http://pds.invalid", fakeInstanceDID, fakeInstanceDomain, nil, nil, nil) + repo, "http://pds.invalid", fakeInstanceDID, fakeInstanceDomain, nil, nil, nil, + communities.PrivateHostOptions(true)...) return service, repo } diff --git a/internal/core/communities/service_validation_test.go b/internal/core/communities/service_validation_test.go index b988f6b..34ea1b4 100644 --- a/internal/core/communities/service_validation_test.go +++ b/internal/core/communities/service_validation_test.go @@ -56,9 +56,10 @@ const reachedProvisioning = "PDS account creation failed" func newValidationService(t *testing.T) (communities.Service, *fakeCommunityRepo) { t.Helper() repo := newFakeCommunityRepo() - provisioner := communities.NewPDSAccountProvisioner(fakeInstanceDomain, unparseablePDSURL) + provisioner := communities.NewPDSAccountProvisioner(fakeInstanceDomain, unparseablePDSURL, communities.PrivateHostOptions(true)...) service := communities.NewCommunityServiceWithPDSFactory( - repo, unparseablePDSURL, fakeInstanceDID, fakeInstanceDomain, provisioner, nil, nil) + repo, unparseablePDSURL, fakeInstanceDID, fakeInstanceDomain, provisioner, nil, nil, + communities.PrivateHostOptions(true)...) return service, repo } diff --git a/internal/core/communities/service_writeforward_failures_test.go b/internal/core/communities/service_writeforward_failures_test.go index 3219ba6..8f89018 100644 --- a/internal/core/communities/service_writeforward_failures_test.go +++ b/internal/core/communities/service_writeforward_failures_test.go @@ -117,7 +117,8 @@ func writeForwardService(t *testing.T) (communities.Service, *fakeCommunityRepo, userPDS := &fakeUserPDS{t: t, did: subscriberDID} service := communities.NewCommunityServiceWithPDSFactory( repo, "http://pds.invalid", fakeInstanceDID, fakeInstanceDomain, nil, - func(context.Context, *oauth.ClientSessionData) (pds.Client, error) { return userPDS, nil }, nil) + func(context.Context, *oauth.ClientSessionData) (pds.Client, error) { return userPDS, nil }, nil, + communities.PrivateHostOptions(true)...) return service, repo, userPDS } @@ -132,7 +133,8 @@ func unbuildableClientService(t *testing.T, cause error) (communities.Service, * }) service := communities.NewCommunityServiceWithPDSFactory( repo, "http://pds.invalid", fakeInstanceDID, fakeInstanceDomain, nil, - func(context.Context, *oauth.ClientSessionData) (pds.Client, error) { return nil, cause }, nil) + func(context.Context, *oauth.ClientSessionData) (pds.Client, error) { return nil, cause }, nil, + communities.PrivateHostOptions(true)...) return service, repo } diff --git a/internal/core/communities/token_refresh.go b/internal/core/communities/token_refresh.go index 55a6508..00f4421 100644 --- a/internal/core/communities/token_refresh.go +++ b/internal/core/communities/token_refresh.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net/http" "strings" "github.com/bluesky-social/indigo/api/atproto" @@ -13,7 +14,17 @@ import ( // refreshPDSToken exchanges a refresh token for new access and refresh tokens // Uses com.atproto.server.refreshSession endpoint via Indigo SDK // CRITICAL: Refresh tokens are single-use - old refresh token is revoked on success -func refreshPDSToken(ctx context.Context, pdsURL, refreshToken string) (newAccessToken, newRefreshToken string, err error) { +// +// httpClient is REQUIRED and is the SSRF guard. Leaving xrpc.Client.Client nil +// makes indigo substitute util.RobustHTTPClient() — unguarded — on a call that +// sends the community's refresh token as the Authorization header. See +// newPDSHTTPClient. +func refreshPDSToken( + ctx context.Context, httpClient *http.Client, pdsURL, refreshToken string, +) (newAccessToken, newRefreshToken string, err error) { + if httpClient == nil { + return "", "", fmt.Errorf("HTTP client is required") + } if pdsURL == "" { return "", "", fmt.Errorf("PDS URL is required") } @@ -26,7 +37,8 @@ func refreshPDSToken(ctx context.Context, pdsURL, refreshToken string) (newAcces // but refreshSession requires the refresh token in that header. // So we put the refresh token in AccessJwt to make it work correctly. client := &xrpc.Client{ - Host: pdsURL, + Client: httpClient, + Host: pdsURL, Auth: &xrpc.AuthInfo{ AccessJwt: refreshToken, // Refresh token goes here (sent as Authorization header) RefreshJwt: refreshToken, // Also set here for completeness @@ -63,7 +75,17 @@ func refreshPDSToken(ctx context.Context, pdsURL, refreshToken string) (newAcces // reauthenticateWithPassword creates a new session using stored credentials // This is the fallback when refresh tokens expire (after ~2 months) // Uses com.atproto.server.createSession endpoint via Indigo SDK -func reauthenticateWithPassword(ctx context.Context, pdsURL, email, password string) (accessToken, refreshToken string, err error) { +// +// httpClient is REQUIRED and is the SSRF guard. This call POSTs the community's +// CLEARTEXT password and system email in the request body, which makes it the +// worst payload in this package to deliver to an address someone else chose. See +// newPDSHTTPClient. +func reauthenticateWithPassword( + ctx context.Context, httpClient *http.Client, pdsURL, email, password string, +) (accessToken, refreshToken string, err error) { + if httpClient == nil { + return "", "", fmt.Errorf("HTTP client is required") + } if pdsURL == "" { return "", "", fmt.Errorf("PDS URL is required") } @@ -76,7 +98,8 @@ func reauthenticateWithPassword(ctx context.Context, pdsURL, email, password str // Create unauthenticated XRPC client client := &xrpc.Client{ - Host: pdsURL, + Client: httpClient, + Host: pdsURL, } // Prepare createSession input diff --git a/internal/core/imageproxy/errors.go b/internal/core/imageproxy/errors.go index 1fcb8cc..79129cb 100644 --- a/internal/core/imageproxy/errors.go +++ b/internal/core/imageproxy/errors.go @@ -15,6 +15,21 @@ var ( // ErrPDSFetchFailed is returned when fetching a blob from a PDS fails for any reason. ErrPDSFetchFailed = errors.New("failed to fetch blob from PDS") + // ErrPDSBlocked is returned when the SSRF guard refuses the PDS URL — its + // address is private, reserved or loopback, or its shape is one this + // service will not dial. + // + // IT IS DISTINCT FROM ErrPDSFetchFailed IN-PROCESS AND IDENTICAL ON THE + // WIRE. A caller that cannot tell a security refusal from a network failure + // logs both the same way and retries both the same way, which is why the + // sentinel exists at all. But the endpoint being refused comes from a DID + // document a stranger minted, so a distinguishable RESPONSE would hand that + // stranger a better oracle than the one this guard removes — it would say + // "this address is internal" rather than merely "something happened here". + // handleServiceError therefore serves it as the same 502, with the same + // body, as an ordinary fetch failure. + ErrPDSBlocked = errors.New("PDS URL refused by the SSRF guard") + // ErrPDSNotFound is returned when the requested blob does not exist on the PDS. ErrPDSNotFound = errors.New("blob not found on PDS") diff --git a/internal/core/imageproxy/fetcher.go b/internal/core/imageproxy/fetcher.go index 852d883..0ee2264 100644 --- a/internal/core/imageproxy/fetcher.go +++ b/internal/core/imageproxy/fetcher.go @@ -3,12 +3,15 @@ package imageproxy import ( "context" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "strings" "time" + + covesoauth "Coves/internal/atproto/oauth" ) // Fetcher defines the interface for fetching blobs from a PDS. @@ -23,38 +26,127 @@ type PDSFetcher struct { client *http.Client timeout time.Duration maxSizeBytes int64 + + // allowPrivateHosts disables the SSRF guard that refuses private, loopback + // and link-local addresses on the PDS fetch. NEVER set in production: the + // address this fetcher dials is the serviceEndpoint of a DID document, + // which anyone can mint for free, and the route in front of it carries no + // credential at all — so the destination is chosen by a stranger, and the + // AppView shares a network with its Postgres, its PDS, its Jetstream and a + // cloud metadata endpoint. + // + // It is construction state rather than an environment read inside Fetch, + // for the reason blobs.blobService.allowPrivateHosts documents: every honest + // test of this fetch serves its PDS from httptest, which listens on loopback, and + // Go's testing package refuses t.Setenv alongside t.Parallel — so an env + // read would make the guarded branch untestable in parallel and force the + // whole package serial. + allowPrivateHosts bool } // DefaultMaxSourceSizeMB is the default maximum source image size if not configured. const DefaultMaxSourceSizeMB = 10 +// PDSFetcherOption configures optional PDSFetcher behaviour. +type PDSFetcherOption func(*PDSFetcher) + +// WithPrivateHostsAllowed disables the SSRF address guard on the PDS fetch. +// +// THE NAME IS THE CONTRACT: production must not call this. cmd/server derives +// the value from config once (the IS_DEV_ENV gate); tests that serve their +// fixtures from httptest pass it because loopback is exactly what the guard +// refuses, and a local dev stack runs its PDS on the developer's own machine. +func WithPrivateHostsAllowed() PDSFetcherOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(f *PDSFetcher) { f.allowPrivateHosts = true } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to NewPDSFetcher: the hatch when it is set, and NOTHING +// when it is not. +// +// It mirrors oauth.PrivateAddressOptions, and it is a function rather than an +// `if` in cmd/server/wiring.go for the reason documented there: `.env.ci:140` +// sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at every call +// site holding such a boolean. A unit test against this function is the only +// place in the repository where the branch production actually runs is ever +// evaluated. Do not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's +// own defaults. +func PrivateHostOptions(allowPrivate bool) []PDSFetcherOption { + if !allowPrivate { + return nil + } + return []PDSFetcherOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + // NewPDSFetcher creates a new PDSFetcher with the specified timeout. // maxSizeMB specifies the maximum allowed image size in megabytes (0 uses default of 10MB). -func NewPDSFetcher(timeout time.Duration, maxSizeMB int) *PDSFetcher { +func NewPDSFetcher(timeout time.Duration, maxSizeMB int, opts ...PDSFetcherOption) *PDSFetcher { if maxSizeMB <= 0 { maxSizeMB = DefaultMaxSourceSizeMB } - return &PDSFetcher{ - client: &http.Client{ - Timeout: timeout, - }, + f := &PDSFetcher{ timeout: timeout, maxSizeBytes: int64(maxSizeMB) * 1024 * 1024, } + for _, opt := range opts { + opt(f) + } + + // The SSRF-safe transport of internal/atproto/oauth: it resolves the host, + // refuses private, loopback and link-local addresses, and then dials only + // the address it vetted — closing the check-then-dial window a naive guard + // leaves open. + // + // THE BYTE CAP IS RAISED FROM THIS FETCHER'S OWN LIMIT rather than left at + // the transport's 32 MiB default, because IMAGE_PROXY_MAX_SOURCE_SIZE_MB is + // operator-configurable: an operator who set it to 64 would otherwise be + // silently clamped by a constant in another package, and would see it as an + // image that will not load and an error blaming the remote host. A config + // value that quietly does not take effect is worse than one that is + // rejected. oauth.DefaultMaxResponseBytes documents this call site by name. + // + // ONE BYTE ABOVE, deliberately. Fetch reads maxSizeBytes+1 through an + // io.LimitReader so that an oversized body is DETECTED rather than + // truncated; a transport cap set to exactly maxSizeBytes would fail that + // probing read and turn every ErrImageTooLarge into a generic fetch + // failure. + clientOpts := append( + covesoauth.PrivateAddressOptions(f.allowPrivateHosts), + covesoauth.WithMaxResponseBytes(f.maxSizeBytes+1), + ) + f.client = covesoauth.NewSSRFSafeHTTPClient(clientOpts...) + + // The shared client ships a 15s ceiling of its own, and this one is + // operator-configured (IMAGE_PROXY_FETCH_TIMEOUT_SECONDS, default 30). It + // is restored the way blobs.NewBlobService restores its own: silently + // re-timing every image fetch would be a second change wearing an SSRF + // fix's clothes. + f.client.Timeout = timeout + return f } // Fetch retrieves a blob from the specified PDS using the com.atproto.sync.getBlob endpoint. // Returns: +// - ErrPDSBlocked if the SSRF guard refuses the endpoint's shape or its address // - ErrPDSNotFound if the blob does not exist (404 response) // - ErrPDSTimeout if the request times out or context is cancelled // - ErrPDSFetchFailed for any other error func (f *PDSFetcher) Fetch(ctx context.Context, pdsURL, did, cid string) ([]byte, error) { // Construct the request URL - endpoint, err := url.Parse(pdsURL) + endpoint, err := parsePDSEndpoint(pdsURL) if err != nil { - return nil, fmt.Errorf("%w: invalid PDS URL: %v", ErrPDSFetchFailed, err) + return nil, err } + + // RawPath goes with Path. url.URL keeps the escaped spelling separately and + // EscapedPath() prefers it when it is a valid encoding of Path — it is not, + // once Path has been replaced, so this is belt-and-braces rather than a + // live hole. It costs one line and removes the need to re-derive that. endpoint.Path = "/xrpc/com.atproto.sync.getBlob" + endpoint.RawPath = "" query := url.Values{} query.Set("did", did) @@ -73,6 +165,25 @@ func (f *PDSFetcher) Fetch(ctx context.Context, pdsURL, did, cid string) ([]byte // Execute the request resp, err := f.client.Do(req) if err != nil { + // THE GUARD'S REFUSAL IS CLASSIFIED FIRST AND ON ITS OWN IDENTITY. + // Falling through to the branches below would map a refused internal + // address onto ErrPDSTimeout (504) or ErrPDSFetchFailed (502) + // depending on how the dial failed — which is the port-scan oracle + // this guard exists to remove, rebuilt one layer up. + if errors.Is(err, covesoauth.ErrBlockedAddress) { + return nil, fmt.Errorf("%w: %v", ErrPDSBlocked, err) + } + + // The transport refuses an ANNOUNCED Content-Length over its cap + // before the body is read, so this is the same condition the + // Content-Length branch below reports — just detected one layer down, + // for a declared length above maxSizeBytes+1. Classifying it the same + // way keeps a 2 MB declaration and a 1 MB+1 declaration from arriving + // as two different errors. + if errors.Is(err, covesoauth.ErrResponseTooLarge) { + return nil, fmt.Errorf("%w: %v", ErrImageTooLarge, err) + } + // Check if the error is due to context cancellation or timeout if ctx.Err() != nil { return nil, fmt.Errorf("%w: %v", ErrPDSTimeout, ctx.Err()) @@ -127,6 +238,61 @@ func (f *PDSFetcher) Fetch(ctx context.Context, pdsURL, did, cid string) ([]byte } } +// parsePDSEndpoint parses a PDS URL and refuses every shape this fetcher will +// not dial, before a packet leaves the process. +// +// # WHY SHAPE IS CHECKED AT ALL, GIVEN THE ADDRESS GUARD +// +// Fetch overwrites .Path, .RawPath and .RawQuery and nothing else, so every +// OTHER component of the caller-supplied URL survives into the request — and the +// caller here is a DID document's serviceEndpoint, which a stranger mints. The +// address guard has no opinion on any of them: +// +// - .User: `https://evil@internal-host` dials internal-host and carries +// userinfo to it, which becomes an Authorization header on the wire. +// - .Fragment: a component of the attacker's string survives a rewrite that +// is supposed to replace the whole request target. +// - .Opaque: url.URL.String() emits Opaque INSTEAD of Path, so the rewrite +// above is discarded entirely and the attacker's own path is requested. +// - .Scheme: file:// and gopher:// are refused today only by accident of what +// http.Transport happens to register, which a future transport option can +// quietly change. A positive allowlist is the control; the stdlib's +// registration table is not. +// +// # REFUSED, NOT SANITISED +// +// Clearing these components would also close the holes, and it would do it +// silently: a serviceEndpoint carrying userinfo is not a URL with one field too +// many, it is evidence that the endpoint is not what this fetcher is for. +// Refusing says so, and says it in one place a reader can check, rather than +// leaving a reader to prove that the list of fields cleared is exhaustive. +func parsePDSEndpoint(pdsURL string) (*url.URL, error) { + endpoint, err := url.Parse(pdsURL) + if err != nil { + return nil, fmt.Errorf("%w: invalid PDS URL: %v", ErrPDSFetchFailed, err) + } + + // url.Parse lower-cases the scheme, so this comparison is already + // case-insensitive over the input. + if endpoint.Scheme != "http" && endpoint.Scheme != "https" { + return nil, fmt.Errorf("%w: PDS URL scheme %q is not http or https", ErrPDSBlocked, endpoint.Scheme) + } + if endpoint.Opaque != "" { + return nil, fmt.Errorf("%w: PDS URL is opaque and has no authority to vet", ErrPDSBlocked) + } + if endpoint.User != nil { + return nil, fmt.Errorf("%w: PDS URL carries userinfo", ErrPDSBlocked) + } + if endpoint.Fragment != "" { + return nil, fmt.Errorf("%w: PDS URL carries a fragment", ErrPDSBlocked) + } + if endpoint.Host == "" { + return nil, fmt.Errorf("%w: PDS URL has no host", ErrPDSBlocked) + } + + return endpoint, nil +} + // pdsErrorResponse represents the error response structure from AT Protocol PDS type pdsErrorResponse struct { Error string `json:"error"` diff --git a/internal/core/imageproxy/fetcher_guard_test.go b/internal/core/imageproxy/fetcher_guard_test.go new file mode 100644 index 0000000..2e08818 --- /dev/null +++ b/internal/core/imageproxy/fetcher_guard_test.go @@ -0,0 +1,562 @@ +package imageproxy + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The image proxy's PDS fetch, against the destination a stranger chose. +// +// # WHY THIS IS THE WORST OF THE NINE SITES +// +// There is no authentication anywhere on the path. The route is public, guarded +// only by a global 100/min/IP limiter, and the address it dials comes from the +// `serviceEndpoint` of a DID document — a did:plc anyone can mint, for free, +// naming any endpoint they like. So the attacker supplies the destination +// directly and pays nothing to do it. +// +// What comes back is a clean port scanner. handler.go:148 maps the fetch errors +// onto three distinct statuses — 404 when the port answered, 502 when the +// connection was refused, 504 when it was filtered — so a stranger reads the +// AppView's internal network topology one request at a time, and the AppView +// shares that network with its Postgres, its PDS, its Jetstream and, in +// production, a metadata endpoint at 169.254.169.254 that hands credentials to +// anything that can reach it. +// +// # WHY THESE TESTS DRIVE THE FETCHER AND NOT THE SERVICE +// +// service.go:85 checks the disk cache BEFORE calling fetcher.Fetch. A guard test +// written against the service that happened to hit a warm cache would return +// bytes, assert nothing, and pass — having never once evaluated the code it +// claims to test. The guard cases below therefore drive PDSFetcher directly. +// The one service-level case at the bottom exists to prove the assembled path is +// closed too, and it starts by asserting its own cache is cold. +// +// # WHY REACHABILITY IS ASSERTED AND NOT ONLY THE ERROR +// +// This project has already been bitten here. Mutation testing +// produced an implementation that classified every address correctly, emitted a +// byte-identical error message, and refused the request AFTER delivering it. +// Every error-message assertion in the suite passed against it. For a +// destination a stranger named, the packet leaving IS the SSRF — whatever error +// comes back afterwards — so each case below stands up a real listener and +// asserts its handler was never invoked. + +// countingPDS is a listener that answers like a PDS and records whether anything +// ever reached it. It listens on loopback, which is exactly the address class +// the guard exists to refuse, so the counter doubles as the assertion. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + pds.requests.Add(1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("blob bytes")) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// TestPDSFetcher_Fetch_RefusesAPrivateAddressWithoutReachingIt is the binding +// contract for image-proxy egress. +// +// The listener is real and the fetcher is the production one — no options, the +// shape wiring builds outside dev. A guarded fetcher must leave the request +// counter at zero. +func TestPDSFetcher_Fetch_RefusesAPrivateAddressWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + fetcher := NewPDSFetcher(5*time.Second, 10) + + _, err := fetcher.Fetch(context.Background(), pds.server.URL, "did:plc:test123", "bafyreicid123") + + require.Errorf(t, err, + "a PDS URL pointing at a loopback address was fetched successfully. The endpoint comes from a "+ + "DID document anyone can mint for free, and this route needs no credential at all, so this "+ + "is a stranger making the AppView dial its own internal network") + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. The refusal happened, but it happened AFTER the request was "+ + "delivered — which prevents none of the SSRF and is precisely the implementation that passed "+ + "every error-message assertion during mutation testing", + pds.requests.Load()) +} + +// TestPDSFetcher_Fetch_BlockedIsItsOwnSentinelAndNotTheOracle pins the internal +// half of the sentinel contract. +// +// A refusal by the guard is not a fetch that failed, and a caller has to be able +// to tell them apart — for logging, for alerting, and so that a future retry or +// circuit-breaker treats "the network hiccuped" and "we refused to make this +// request" differently. That is what ErrPDSBlocked is for. +// +// The two NotErrorIs assertions matter more than the positive one. ErrPDSNotFound +// maps to 404 and ErrPDSTimeout maps to 504 at handler.go:148, so a refusal that +// matched either would hand back exactly the port-scan oracle this guard exists +// to remove — and it would do so while the "is it blocked" assertion above stayed +// green. +func TestPDSFetcher_Fetch_BlockedIsItsOwnSentinelAndNotTheOracle(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + fetcher := NewPDSFetcher(5*time.Second, 10) + + _, err := fetcher.Fetch(context.Background(), pds.server.URL, "did:plc:test123", "bafyreicid123") + require.Error(t, err, "the guarded fetcher must refuse a loopback PDS URL") + + assert.ErrorIsf(t, err, ErrPDSBlocked, + "the refusal must carry its own identity, matchable with errors.Is rather than by reading a "+ + "message: a caller that cannot distinguish a security refusal from a network failure logs "+ + "both the same way and retries both the same way; got: %v", err) + + assert.NotErrorIsf(t, err, ErrPDSNotFound, + "a blocked address classified as ErrPDSNotFound, which handler.go:148 serves as 404. That is "+ + "half the port-scan oracle: 404 tells the stranger who named this address that something "+ + "answered on it; got: %v", err) + + assert.NotErrorIsf(t, err, ErrPDSTimeout, + "a blocked address classified as ErrPDSTimeout, which handler.go:148 serves as 504 — the "+ + "'filtered' half of the port-scan oracle, distinguishable from 502 by anyone probing; got: %v", err) + + assert.NotErrorIsf(t, err, ErrPDSFetchFailed, + "the guard refusal also matches ErrPDSFetchFailed, so the two are indistinguishable in-process. "+ + "They must map to the SAME status externally — see the handler test — but a single sentinel "+ + "covering both means no caller and no log line can ever tell a refused destination from an "+ + "unreachable one; got: %v", err) +} + +// TestPDSFetcher_Fetch_AllowsAPrivateAddressWhenExplicitlyPermitted is the dev +// hatch, and it is not a nicety. +// +// Every honest test of this fetch serves its PDS from httptest, which listens on +// loopback — proxy_serving_test.go and avatar_serving_test.go both do — and a +// local dev stack runs its PDS on the developer's own machine. Without an +// injectable allowance the guard takes all of that with it. +func TestPDSFetcher_Fetch_AllowsAPrivateAddressWhenExplicitlyPermitted(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) + + data, err := fetcher.Fetch(context.Background(), pds.server.URL, "did:plc:test123", "bafyreicid123") + + require.NoErrorf(t, err, + "the hatch is what every fixture in this tree and every dev stack depends on: a fetcher built "+ + "with WithPrivateHostsAllowed() must reach a loopback PDS") + assert.Equal(t, "blob bytes", string(data), "the blob bytes must come back unchanged through the guarded client") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestPDSFetcher_Fetch_RefusesSmuggledURLComponents covers what Fetch does NOT +// currently touch. +// +// Fetch overwrites `.Path` and `.RawQuery` and nothing else, so every other +// component of a caller-supplied URL survives into the request: `.User`, +// `.Fragment`, `.Opaque` and `.Scheme`. A `serviceEndpoint` of +// `https://evil@internal-host` dials internal-host carrying userinfo; an +// `.Opaque` form bypasses the path rewrite entirely, since url.URL.String() +// prefers Opaque over Path; and `file://` or `gopher://` is not a fetch this +// service has any business making. +// +// # EVERY ROW RUNS WITH THE HATCH OPEN, DELIBERATELY +// +// If these ran on a guarded fetcher, the loopback address alone would refuse +// them and every row would pass without the URL ever being inspected. Opening +// the hatch removes the address guard from the picture, so the only thing left +// that can refuse the request is the URL's own shape — which is what these rows +// are about. The row above proves a plain URL at this same listener succeeds +// under the same options, so a refusal here is attributable to the smuggled +// component and to nothing else. +func TestPDSFetcher_Fetch_RefusesSmuggledURLComponents(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + // pdsURL is built from the listener's address where the case needs a + // reachable destination, and is a fixed string where it does not. + pdsURL func(base string) string + why string + }{ + { + name: "userinfo in the authority", + pdsURL: func(base string) string { return "http://evil@" + trimScheme(base) }, + why: "a DID document's serviceEndpoint of https://evil@internal-host dials internal-host " + + "and carries credentials to it; url.Parse puts `evil` in .User, which Fetch never clears", + }, + { + name: "a fragment on the endpoint", + pdsURL: func(base string) string { return base + "/#fragment" }, + why: "Fetch overwrites .Path and .RawQuery but leaves .Fragment, so a component of the " + + "attacker's string survives the rewrite that is supposed to replace the whole request target", + }, + { + name: "an opaque URL", + pdsURL: func(string) string { return "http:internal-host/v1/secrets" }, + why: "url.URL.String() emits .Opaque INSTEAD of .Path when it is set, so the path rewrite " + + "Fetch performs is discarded and the attacker's own path is what gets requested", + }, + { + name: "the file scheme", + pdsURL: func(string) string { return "file:///etc/passwd" }, + why: "there must be a positive http/https allowlist. The stdlib refuses this scheme today " + + "by accident of what http.Transport registers, which is a different mechanism that a " + + "future transport option could quietly change", + }, + { + name: "the gopher scheme", + pdsURL: func(string) string { return "gopher://internal-host:70/" }, + why: "gopher is the classic SSRF protocol-smuggling scheme, and the same positive allowlist " + + "is what refuses it rather than a registration detail of the stdlib transport", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + // The hatch is open: the address guard is out of the way, so the URL's + // shape is the only thing that can refuse this. + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) + + _, err := fetcher.Fetch(context.Background(), tt.pdsURL(pds.server.URL), "did:plc:test123", "bafyreicid123") + + require.Errorf(t, err, + "the PDS URL was accepted with the hatch open, so nothing inspected its shape — %s", tt.why) + + assert.ErrorIsf(t, err, ErrPDSBlocked, + "the refusal must be the guard's own sentinel and not an incidental failure from "+ + "somewhere in net/http. An error that arrives by accident is one a dependency "+ + "upgrade can take away, and it maps to a status nobody chose — %s; got: %v", tt.why, err) + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. A URL this malformed must be refused before a "+ + "packet leaves the process — %s", pds.requests.Load(), tt.why) + }) + } +} + +// trimScheme strips the leading http:// from an httptest URL so a case can +// rebuild the authority with something smuggled in front of it. +func trimScheme(url string) string { + const prefix = "http://" + if len(url) > len(prefix) && url[:len(prefix)] == prefix { + return url[len(prefix):] + } + return url +} + +// TestPDSFetcher_PreservesTheConfiguredTimeout guards the setting the shared +// client would otherwise swallow. +// +// NewSSRFSafeHTTPClient returns a client with its own 15s ceiling, and +// IMAGE_PROXY_FETCH_TIMEOUT_SECONDS is an operator setting that defaults to 30. +// Adopting the shared client without restoring the caller's own value silently +// re-times every image fetch in production — a change nobody asked for, arriving +// as part of an SSRF fix. blobs.NewBlobService raises its own back to 30s for +// exactly this reason. +func TestPDSFetcher_PreservesTheConfiguredTimeout(t *testing.T) { + t.Parallel() + + const configured = 27 * time.Second + + fetcher := NewPDSFetcher(configured, 10) + + require.NotNil(t, fetcher.client, "the fetcher must hold an HTTP client") + assert.Equalf(t, configured, fetcher.client.Timeout, + "the fetcher's client runs on a %v timeout instead of the configured %v. The shared SSRF client "+ + "ships a 15s ceiling of its own, so a call site that adopts it without re-applying its own "+ + "value hands operators a setting that no longer does anything", + fetcher.client.Timeout, configured) +} + +// TestPDSFetcher_Fetch_HonoursASizeLimitAboveTheTransportDefault covers the +// configuration clamp regression. +// +// The shared transport now caps response bodies at DefaultMaxResponseBytes +// (32 MiB) unless a caller raises it. IMAGE_PROXY_MAX_SOURCE_SIZE_MB is +// operator-configurable and defaults to 10, so nothing breaks today — and that +// is the danger. An operator who raises it to 64 gets a fetcher that still +// refuses at 32 MiB, with the failure surfacing as an image that will not load +// and an error that blames the remote host. A config value that silently does +// not take effect is worse than one that is rejected. +// +// So the conversion has to pass WithMaxResponseBytes explicitly from the +// configured size rather than inheriting the package default. The body below is +// past the transport's default and under this fetcher's own limit, which is the +// only window where the two can be told apart. +func TestPDSFetcher_Fetch_HonoursASizeLimitAboveTheTransportDefault(t *testing.T) { + t.Parallel() + + // One KiB past the transport's default, so a fetcher that inherited the + // default fails and a fetcher carrying its own 64 MiB limit does not. + const bodySize = covesoauth.DefaultMaxResponseBytes + 1024 + + var requests atomic.Int64 + chunk := make([]byte, 1<<20) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusOK) + for sent := 0; sent < bodySize; { + end := len(chunk) + if remaining := bodySize - sent; remaining < end { + end = remaining + } + n, err := w.Write(chunk[:end]) + sent += n + if err != nil { + return + } + } + })) + t.Cleanup(server.Close) + + // 64 MiB configured — comfortably above the transport's 32 MiB default and + // above the body, so the only thing that can refuse this response is a cap + // the conversion failed to raise. + fetcher := NewPDSFetcher(30*time.Second, 64, WithPrivateHostsAllowed()) + + data, err := fetcher.Fetch(context.Background(), server.URL, "did:plc:test123", "bafyreicid123") + + require.NoErrorf(t, err, + "a %d-byte blob was refused by a fetcher configured for 64 MiB. The limit that stopped it is "+ + "the shared transport's 32 MiB default, applied from another package — so an operator who "+ + "raises IMAGE_PROXY_MAX_SOURCE_SIZE_MB above 32 gets no error, no warning, and a setting "+ + "that quietly does nothing; got: %v", bodySize, err) + assert.Lenf(t, data, bodySize, + "the blob came back truncated: %d bytes of %d. A short body that arrives without an error is "+ + "worse than a refusal — the processor would encode whatever partial image it got", len(data), bodySize) + assert.Equalf(t, int64(1), requests.Load(), "the listener was reached %d times rather than once", requests.Load()) +} + +// TestPDSFetcher_Fetch_ImageTooLarge_IsStillTheFetchersOwnLimit exists because +// the conversion moved a limit without moving an assertion. +// +// The transport now carries a cap of its own, set to maxSizeBytes+1, and it +// refuses a DECLARED Content-Length above that before Fetch ever sees the +// response. TestPDSFetcher_Fetch_ImageTooLarge_ContentLength declares 2 MiB +// against a 1 MiB fetcher, so it is now satisfied one layer down: its assertion +// (ErrImageTooLarge) still holds, but the branch it used to cover — +// `resp.ContentLength > f.maxSizeBytes` in Fetch — no longer runs for that +// input. Nobody would notice, because the test is still green. +// +// That branch is now reachable through a ONE-BYTE WINDOW: a declared length must +// exceed maxSizeBytes to fail it, and must not exceed maxSizeBytes+1 or the +// transport takes it first. maxSizeBytes+1 is the only value that lands there, +// and it is what the first case below declares. +// +// The mechanism is identified by MESSAGE rather than by errors.Is, which is not +// a stylistic choice: fetcher.go:184 wraps oauth.ErrResponseTooLarge with %v +// rather than %w, so the transport's identity is not in the chain and both +// layers arrive as a bare ErrImageTooLarge. That is worth knowing on its own — +// nothing downstream can tell "the image is too big for us" from "the transport +// refused to hand it over" — and until it changes, the rendered sentence is the +// only thing that distinguishes them. +func TestPDSFetcher_Fetch_ImageTooLarge_IsStillTheFetchersOwnLimit(t *testing.T) { + t.Parallel() + + // One MiB, matching the fetcher below, spelled out because both cases are + // positioned relative to it by single bytes. + const maxSizeBytes = 1 << 20 + + t.Run("a declared length in the one-byte window the transport leaves", func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", strconv.Itoa(maxSizeBytes+1)) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + fetcher := NewPDSFetcher(5*time.Second, 1, WithPrivateHostsAllowed()) + + _, err := fetcher.Fetch(context.Background(), server.URL, "did:plc:test123", "bafyreicid123") + + require.ErrorIsf(t, err, ErrImageTooLarge, + "a declared length of %d against a %d-byte limit must be refused; got: %v", + maxSizeBytes+1, maxSizeBytes, err) + assert.Containsf(t, err.Error(), "content length", + "the refusal came from the transport's cap rather than from Fetch's own Content-Length "+ + "check, which means that branch is now dead code and no test covers it. This is the "+ + "only input that can reach it — one byte lower and it is within the limit, one byte "+ + "higher and the transport refuses first; got: %v", err) + }) + + t.Run("a streamed body with no declared length", func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(make([]byte, 2*maxSizeBytes)) + })) + t.Cleanup(server.Close) + + fetcher := NewPDSFetcher(5*time.Second, 1, WithPrivateHostsAllowed()) + + _, err := fetcher.Fetch(context.Background(), server.URL, "did:plc:test123", "bafyreicid123") + + require.ErrorIsf(t, err, ErrImageTooLarge, + "a %d-byte body against a %d-byte limit must be refused; got: %v", 2*maxSizeBytes, maxSizeBytes, err) + assert.Containsf(t, err.Error(), "response body exceeds maximum", + "this is the case the transport's cap must NOT shadow. Fetch reads maxSizeBytes+1 through "+ + "an io.LimitReader precisely so an oversized body is detected rather than truncated, and "+ + "the transport's allowance is one byte above that so the probing read survives. A cap set "+ + "to maxSizeBytes exactly would turn this into a generic read failure — the same refusal, "+ + "reported as something the remote host did wrong; got: %v", err) + }) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// single most important assertion for this call site. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// T0+T1+T2 — runs the PERMISSIVE branch here and at every other site holding +// such a boolean. A green merge gate therefore proves nothing whatsoever about +// whether the image proxy is guarded in production. This function is the one +// place in the repository where the production branch is ever evaluated, which +// is why the gate must be a pure function and not an `if cfg.IsDevEnv` in +// cmd/server/wiring.go. +// +// The claim is not "the options returned are safe". It is that there are NONE: +// length zero, nothing applied, the constructor's own defaults left untouched. +// An edit that appends a diagnostic option, or returns a one-element slice +// holding a no-op "explicitly deny" closure, keeps every behavioural test green +// while moving the untested branch from "provably applies nothing" to "applies +// something believed harmless". If this assertion is ever in the way, the answer +// is not to relax it. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). The production branch — the one IS_DEV_ENV=true "+ + "keeps `make ci` from ever evaluating — must contribute nothing at all, so that what "+ + "production gets is exactly the constructor's own defaults", len(opts)) +} + +// TestPrivateHostOptions_DisallowedFetcherIsGuarded is the behavioural half of +// the assertion above: zero options has to also MEAN a guarded fetcher. +// +// The length check alone would still pass if the constructor's own default ever +// regressed to permissive — the helper would correctly be returning nothing, +// onto a base that no longer refuses anything. +func TestPrivateHostOptions_DisallowedFetcherIsGuarded(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + fetcher := NewPDSFetcher(5*time.Second, 10, PrivateHostOptions(false)...) + + _, err := fetcher.Fetch(context.Background(), pds.server.URL, "did:plc:test123", "bafyreicid123") + + require.Error(t, err, + "a fetcher built from PrivateHostOptions(false) reached a loopback PDS. This is the branch "+ + "production runs and CI never does") + assert.ErrorIsf(t, err, ErrPDSBlocked, + "the refusal must be the guard's, matchable by identity — a request that failed for some other "+ + "reason is not the same control and would not hold in production; got: %v", err) + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times, so the packet left the process", pds.requests.Load()) +} + +// TestPrivateHostOptions_AllowedFetcherReachesTheListener pins the other +// direction through observed behaviour rather than through the shape of the +// slice. +// +// A length check here would be worthless: a helper returning the wrong +// single-element slice satisfies it while leaving every fixture in this tree +// unable to reach loopback. +func TestPrivateHostOptions_AllowedFetcherReachesTheListener(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + fetcher := NewPDSFetcher(5*time.Second, 10, PrivateHostOptions(true)...) + + data, err := fetcher.Fetch(context.Background(), pds.server.URL, "did:plc:test123", "bafyreicid123") + + require.NoErrorf(t, err, + "a fetcher built from PrivateHostOptions(true) was refused. The permissive branch is what every "+ + "developer and every fixture in this tree runs, so a helper that returns the wrong option — "+ + "or none — breaks local development everywhere at once; got: %v", err) + assert.Equal(t, "blob bytes", string(data), "the blob must come back through the permissive fetcher") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestImageProxyService_GetImage_RefusesAPrivateAddressOnAColdCache proves the +// assembled path is closed, not just the fetcher in isolation. +// +// # THE CACHE TRAP +// +// service.go:85 reads the disk cache before it calls fetcher.Fetch. A test that +// inherited a warm cache — a shared directory, a fixture written by a sibling +// test, a second call in the same function — returns bytes from disk and asserts +// nothing about the guard. So this one starts by asserting its own cache is +// cold, in the same terms the service uses, and only then drives GetImage. +func TestImageProxyService_GetImage_RefusesAPrivateAddressOnAColdCache(t *testing.T) { + t.Parallel() + + const ( + preset = "avatar" + did = "did:plc:test123" + cid = "bafyreicid123" + ) + + pds := newCountingPDS(t) + + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err, "creating the disk cache this test owns") + + // The cache is cold, asserted rather than assumed: everything below is + // meaningless if service.go:85 can answer from disk. + _, found, err := cache.Get(preset, did, cid) + require.NoError(t, err, "reading a fresh cache must not error") + require.False(t, found, + "this test's cache already holds an entry for (%s, %s, %s), so GetImage would return it without "+ + "ever calling the fetcher and this whole case would pass without exercising the guard", + preset, did, cid) + + service, err := NewService(cache, NewProcessor(), NewPDSFetcher(5*time.Second, 10), Config{ + Enabled: true, + CachePath: t.TempDir(), + CacheMaxGB: 1, + FetchTimeout: 5 * time.Second, + MaxSourceSizeMB: 10, + }) + require.NoError(t, err, "creating the image proxy service") + + _, err = service.GetImage(context.Background(), preset, did, cid, pds.server.URL) + + require.Error(t, err, + "the assembled image proxy fetched a blob from a loopback PDS URL. This is the shape of the "+ + "production request: a public route, no credential, and an endpoint taken from a DID "+ + "document a stranger minted") + assert.ErrorIsf(t, err, ErrPDSBlocked, + "the service must pass the guard's refusal through unchanged so the handler can map it; got: %v", err) + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times through the service", pds.requests.Load()) +} diff --git a/internal/core/imageproxy/fetcher_test.go b/internal/core/imageproxy/fetcher_test.go index 7bfa537..49e8c6f 100644 --- a/internal/core/imageproxy/fetcher_test.go +++ b/internal/core/imageproxy/fetcher_test.go @@ -9,6 +9,26 @@ import ( "time" ) +// EVERY FETCHER BELOW IS BUILT WITH THE HATCH OPEN, AND THAT IS A FIXTURE +// REPAIR RATHER THAN A CHANGE OF SUBJECT. +// +// These are the fetcher's ordinary behaviours — status mapping, timeouts, URL +// construction, the size limit — and every one of them serves its PDS from +// httptest, which listens on loopback. Loopback is exactly what the SSRF guard +// refuses, so once NewPDSFetcher started returning a guarded client all ten +// failed identically, at the guard, before reaching the behaviour under test. +// WithPrivateHostsAllowed() puts the address guard back out of the way; not one +// assertion in this file was weakened, and the same repair is what +// blobs/fetch_guard_test.go:128 and blueskypost/service_test.go:62 did. +// +// WHAT THIS COSTS, STATED PLAINLY: with the hatch open none of these tests says +// anything about the guard. They are not supposed to. The guard is bound in +// fetcher_guard_test.go, whose cases build the fetcher WITHOUT the hatch and +// assert the listener is never reached — so if a repair here ever leaked into +// those, they would go green while the endpoint was wide open. That is the +// failure mode this comment exists to name, and the reason the hatch appears +// only in fixtures whose subject is something else entirely. + func TestPDSFetcher_Fetch_Success(t *testing.T) { // Setup test server that returns blob data expectedData := []byte("test image data") @@ -28,7 +48,7 @@ func TestPDSFetcher_Fetch_Success(t *testing.T) { })) defer server.Close() - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx := context.Background() data, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -46,7 +66,7 @@ func TestPDSFetcher_Fetch_NotFound(t *testing.T) { })) defer server.Close() - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -72,7 +92,7 @@ func TestPDSFetcher_Fetch_Timeout(t *testing.T) { defer close(release) // Use a very short timeout - fetcher := NewPDSFetcher(50*time.Millisecond, 10) + fetcher := NewPDSFetcher(50*time.Millisecond, 10, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -82,7 +102,7 @@ func TestPDSFetcher_Fetch_Timeout(t *testing.T) { } func TestPDSFetcher_Fetch_NetworkError(t *testing.T) { - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx := context.Background() // Port 99999 is outside the valid range, so this is a malformed address @@ -109,7 +129,7 @@ func TestPDSFetcher_Fetch_ContextCancellation(t *testing.T) { defer server.Close() defer close(release) - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx, cancel := context.WithCancel(context.Background()) // Cancel the context immediately @@ -131,7 +151,7 @@ func TestPDSFetcher_Fetch_ServerError(t *testing.T) { })) defer server.Close() - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -149,7 +169,7 @@ func TestPDSFetcher_Fetch_URLConstruction(t *testing.T) { })) defer server.Close() - fetcher := NewPDSFetcher(5*time.Second, 10) + fetcher := NewPDSFetcher(5*time.Second, 10, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:abc123", "bafyreicid456") @@ -174,7 +194,7 @@ func TestPDSFetcher_Fetch_ImageTooLarge_ContentLength(t *testing.T) { defer server.Close() // Use 1MB max size - fetcher := NewPDSFetcher(5*time.Second, 1) + fetcher := NewPDSFetcher(5*time.Second, 1, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -193,7 +213,7 @@ func TestPDSFetcher_Fetch_ImageTooLarge_StreamingBody(t *testing.T) { defer server.Close() // Use 1MB max size - fetcher := NewPDSFetcher(5*time.Second, 1) + fetcher := NewPDSFetcher(5*time.Second, 1, WithPrivateHostsAllowed()) ctx := context.Background() _, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") @@ -215,7 +235,7 @@ func TestPDSFetcher_Fetch_SizeWithinLimit(t *testing.T) { defer server.Close() // Use 1MB max size - fetcher := NewPDSFetcher(5*time.Second, 1) + fetcher := NewPDSFetcher(5*time.Second, 1, WithPrivateHostsAllowed()) ctx := context.Background() data, err := fetcher.Fetch(ctx, server.URL, "did:plc:test123", "bafyreicid123") diff --git a/internal/core/posts/community_repo_factory.go b/internal/core/posts/community_repo_factory.go index 4a1eca8..b5e87f7 100644 --- a/internal/core/posts/community_repo_factory.go +++ b/internal/core/posts/community_repo_factory.go @@ -81,7 +81,19 @@ func (r credentialRefresher) RefreshCommunityCredentials(ctx context.Context, co // refresh token — which happens exactly once, when it provisioned the account // through social.coves.community.create — or it does not. That is the honest // question, and it is the only one this factory asks. -func NewCommunityRepoFactory(source CommunityCredentialSource) CommunityRepoFactory { +// +// # THE OPTIONS ARE THE SSRF DEV GATE, AND OMITTING THEM IS THE SAFE DIRECTION +// +// fresh.PDSURL is a per-community database column, so the address this factory +// dials is data rather than configuration — which is why the client it builds is +// address-guarded (see pds.newBearerHTTPClient). Passing nothing yields the +// guarded client, so a caller that forgets the gate gets the strict behaviour +// and finds out; the reverse default would be an unguarded client nobody +// notices. Production wiring passes pds.PrivateHostOptions(cfg.IsDevEnv)..., +// and tests driving the CI stack's loopback PDS pass +// pds.PrivateHostOptions(true)... because loopback is exactly what the guard +// refuses. +func NewCommunityRepoFactory(source CommunityCredentialSource, opts ...pds.ClientOption) CommunityRepoFactory { return func(ctx context.Context, communityDID string) (CommunityRepo, error) { community, err := source.GetByDID(ctx, communityDID) if err != nil { @@ -120,7 +132,7 @@ func NewCommunityRepoFactory(source CommunityCredentialSource) CommunityRepoFact return nil, fmt.Errorf("refreshing the credentials of %s: no access token came back", communityDID) } - client, err := pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken) + client, err := pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken, opts...) if err != nil { return nil, fmt.Errorf("building a PDS client for %s: %w", communityDID, err) } diff --git a/internal/core/posts/community_repo_factory_test.go b/internal/core/posts/community_repo_factory_test.go index 52f626c..065f87e 100644 --- a/internal/core/posts/community_repo_factory_test.go +++ b/internal/core/posts/community_repo_factory_test.go @@ -6,6 +6,7 @@ import ( "context" "testing" + "Coves/internal/atproto/pds" "Coves/internal/core/posts" "Coves/tests/fixtures" "Coves/tests/testkit" @@ -43,7 +44,13 @@ func TestCommunityRepoFactory_OpensAHostedCommunitysRepo(t *testing.T) { t.Parallel() fixture := newPostFixture(t) - factory := posts.NewCommunityRepoFactory(fixture.communityService) + + // The hatch, and this is the ONLY test in this file that needs it: it is the + // only one that gets far enough to open a repo, which means dialling the CI + // stack's PDS on loopback. The two refusal cases below deliberately keep the + // GUARDED spelling — they must fail before any client is built, so if either + // ever started dialling, an SSRF refusal is exactly the loud failure wanted. + factory := posts.NewCommunityRepoFactory(fixture.communityService, pds.PrivateHostOptions(true)...) repo, err := factory(context.Background(), fixture.community.DID) require.NoError(t, err, "the AppView provisioned this community's account, so it holds its credentials") diff --git a/internal/core/posts/engine_contract_test.go b/internal/core/posts/engine_contract_test.go index a5cef13..c16124d 100644 --- a/internal/core/posts/engine_contract_test.go +++ b/internal/core/posts/engine_contract_test.go @@ -96,7 +96,7 @@ func newEngineFixture(t *testing.T) *engineFixture { base := newPostFixture(t) account := base.communityAccount(t) - generic, err := pds.NewFromAccessToken(base.pds.URL(), account.DID, account.AccessToken) + generic, err := pds.NewFromAccessToken(base.pds.URL(), account.DID, account.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) repo, ok := generic.(pds.CommitClient) @@ -602,7 +602,7 @@ func (r *racingRepo) PutRecordWithCommit(ctx context.Context, collection, rkey s func (f *engineFixture) racingWriter(t *testing.T, fn func()) posts.CommunityRecordWriter { t.Helper() - generic, err := pds.NewFromAccessToken(f.pds.URL(), f.communityAt.DID, f.communityAt.AccessToken) + generic, err := pds.NewFromAccessToken(f.pds.URL(), f.communityAt.DID, f.communityAt.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) repo, ok := generic.(pds.CommitClient) require.True(t, ok) @@ -676,7 +676,7 @@ func TestEngine_RemovalCommitLosesItsSwapCommitAndConverges(t *testing.T) { otherPost := f.publishPost(t, "an unrelated post whose acceptance advances the head") - generic, err := pds.NewFromAccessToken(f.pds.URL(), f.communityAt.DID, f.communityAt.AccessToken) + generic, err := pds.NewFromAccessToken(f.pds.URL(), f.communityAt.DID, f.communityAt.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) commitClient, ok := generic.(pds.CommitClient) require.True(t, ok) diff --git a/internal/core/posts/rematerialize.go b/internal/core/posts/rematerialize.go index 84db331..fd102a7 100644 --- a/internal/core/posts/rematerialize.go +++ b/internal/core/posts/rematerialize.go @@ -14,6 +14,7 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/core/blobs" ) @@ -1070,11 +1071,18 @@ func (r *Rematerializer) report(p RematerializeProgress) { } // blobClient returns the injected blob client, or the bounded HTTP default. +// +// THE DEFAULT IS GUARDED, WITH NO WAY TO OPEN IT FROM HERE. The host it dials +// comes from the author repo's HostURL — a DID document's serviceEndpoint — so +// the fallback has to be the safe one; a caller that genuinely needs the dev +// hatch (cmd/rematerialize-posts against a local PDS) sets Blobs explicitly from +// DefaultRematerializeBlobClient(cfg.IsDevEnv), which is a decision visible at +// the wiring rather than a boolean threaded through the state machine. func (r *Rematerializer) blobClient() RematerializeBlobClient { if r.Blobs != nil { return r.Blobs } - return DefaultRematerializeBlobClient() + return DefaultRematerializeBlobClient(false) } // maxRematerializeBlobBytes caps a single blob copy. It is generous — larger than @@ -1286,16 +1294,91 @@ type httpRematerializeBlobClient struct { } // DefaultRematerializeBlobClient is the production blob client, over an HTTP -// client with its OWN timeout. +// client with its OWN timeout and an ADDRESS GUARD, and it is the dev gate for +// this call site. // // http.DefaultClient has none, and a batch tool that hangs on a half-open socket // is a batch tool the operator must kill and re-run — mid-migration, without -// knowing where it stopped. -func DefaultRematerializeBlobClient() RematerializeBlobClient { - return newRematerializeBlobClient( - &http.Client{Timeout: rematerializeBlobFetchTimeout}, - maxRematerializeBlobBytes, - ) +// knowing where it stopped. "Which client may this tool dial with" has a second +// half, and the guard is it. THE TWO METHODS DIAL DIFFERENT HOSTS, and both are +// data rather than config: +// +// - Fetch takes communityHostURL's answer — hostURLOf on the COMMUNITY's repo, +// which NewCommunityRepoFactory builds from the community's PDSURL database +// column, written when a community is created or federated in. A federated +// community's PDSURL is whatever the remote instance said it was. +// - Present takes repoHostURL's answer — hostURLOf on the AUTHOR's repo, which +// is the serviceEndpoint of the author's DID document, and minting a did:plc +// with any endpoint is free. +// +// Either way it is the same attacker-chosen input class as the image proxy's +// pdsURL, arriving by a route that reads like internal plumbing. That this is an +// operator-run batch tool does not shrink the exposure: an attacker plants the +// endpoint and waits for the migration, which runs once, by hand, with thousands +// of records scrolling past and a refused address looking exactly like a dialled +// one. +// +// # BOTH OF THIS SITE'S OWN LIMITS ARE RE-APPLIED, AND ONE OF THEM IS A TRAP +// +// rematerializeBlobFetchTimeout is two MINUTES against the shared client's 15s, +// because a single blob may be 100 MiB — inheriting the shared ceiling would not +// hurry those copies, it would fail them. +// +// maxRematerializeBlobBytes is 100 MiB against oauth.DefaultMaxResponseBytes's +// 32 MiB — SMALLER — so unlike every other conversion in this remediation, +// adopting the shared client here TIGHTENS the limit unless WithMaxResponseBytes +// raises it, and the failure would arrive as an error on a blob between the two +// numbers, after the postv2 is written and before the legacy record is deleted. +// The transport cap and c.maxBytes are separate controls: the transport refuses +// an announced Content-Length before a byte of body is read, so setting only the +// field would leave a client whose cap says 100 MiB unable to receive 40. +// +// # THE DEV GATE IS THE ONLY THING A CALLER MAY SAY +// +// It takes the gate and nothing else, so "only allowPrivateHosts opens the +// guard" is a fact about the signature rather than a description of current call +// sites. It used to take `opts ...covesoauth.Option` as a test seam — an +// EXPORTED type, and covesoauth.WithPrivateAddressesAllowed() is an exported +// option, so any package in the tree could open the guard on the tool that +// copies blobs from whatever host a federated community's PDSURL or an author's +// DID document names, with nothing for the audit to grep. The seam is real and +// still needed (a guard test that builds its own client proves only that +// internal/atproto/oauth works), so it lives on +// newGuardedRematerializeBlobClient, which is unexported and reachable only from +// this package's own tests. users.NewProfileBackfillClient is the same shape. +func DefaultRematerializeBlobClient(allowPrivateHosts bool) RematerializeBlobClient { + return newGuardedRematerializeBlobClient(allowPrivateHosts, maxRematerializeBlobBytes) +} + +// newGuardedRematerializeBlobClient builds the production client at a copy cap +// the caller names, so the RELATIONSHIP between the two caps can be driven by a +// test without moving 100 MiB through one. +// +// # THE TRANSPORT CAP IS ONE BYTE ABOVE THE COPY CAP, DELIBERATELY +// +// Fetch reads maxBytes+1 through an io.LimitReader because io.ReadAll cannot +// tell "the body ended" from "the limit was reached" — the extra byte's +// existence IS the overrun signal. A transport cap set to exactly maxBytes +// clips that byte, so `len(data) > c.maxBytes` becomes unreachable in +// production and every overrun surfaces as a generic ErrResponseTooLarge +// naming a limit no operator configured, instead of the error explaining that a +// truncated copy is DIFFERENT bytes under a DIFFERENT CID. Not a hole — the +// oversized blob is refused either way — but the wrong message, during a +// one-shot migration that has already written the postv2 and is about to delete +// the only intact copy. imageproxy/fetcher.go:118 is the same trap, sprung the +// same way. +// +// The two caps are separate controls and both are needed: the transport refuses +// an announced Content-Length before a byte of body is read, while c.maxBytes is +// what a chunked body — which reports no length at all — is measured against. +func newGuardedRematerializeBlobClient( + allowPrivateHosts bool, maxBytes int, opts ...covesoauth.Option, +) *httpRematerializeBlobClient { + options := append(covesoauth.PrivateAddressOptions(allowPrivateHosts), + covesoauth.WithMaxResponseBytes(int64(maxBytes)+1)) + client := covesoauth.NewSSRFSafeHTTPClient(append(options, opts...)...) + client.Timeout = rematerializeBlobFetchTimeout + return newRematerializeBlobClient(client, maxBytes) } // newRematerializeBlobClient is the constructor the tests use to shrink the cap. diff --git a/internal/core/posts/rematerialize_blob_guard_test.go b/internal/core/posts/rematerialize_blob_guard_test.go new file mode 100644 index 0000000..ce90f3f --- /dev/null +++ b/internal/core/posts/rematerialize_blob_guard_test.go @@ -0,0 +1,341 @@ +package posts + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The re-materialization tool's blob copy, and the input nobody reads as +// attacker-controlled because of where it arrives from. +// +// # WHERE THE HOST COMES FROM +// +// Fetch and Present both take `host`, and every production caller gets it from +// hostURLOf(repo) — HostURL() on the author's repo, which is the +// serviceEndpoint of that author's DID document. Anyone can mint a DID document +// and put any URL in it. So this is the same input class as the image proxy's +// pdsURL and the profile backfill's PDS URL, arriving by a route that reads like +// internal plumbing: a repo handle the tool already opened, not a string off the +// wire. +// +// # WHY THE DELAY MAKES IT WORSE, NOT BETTER +// +// This is an operator-run batch tool, so an attacker cannot fire it. They do not +// need to. They plant the serviceEndpoint and wait — the migration runs once, by +// hand, during a maintenance window, at whatever hour the operator picked, with +// -yes already passed and 4,131 records scrolling past. A refused address and a +// dialled one look the same in that output. Nothing about the timing reduces the +// exposure; it only guarantees nobody is watching the request that matters. +// +// # WHAT THE COMMENT AT THE CONSTRUCTOR ALREADY SAYS +// +// DefaultRematerializeBlobClient documents why http.DefaultClient was rejected: +// no timeout, so a half-open socket hangs the whole run. That is one half of +// "which client may this tool dial with". The address guard is the other half, +// and it was missing from the same three lines that argued the first one. + +const ( + // rematerializeGuardDID and rematerializeGuardCID are shaped like the real + // thing so HydrateBlobURL builds a URL; neither participates in the refusal. + rematerializeGuardDID = "did:plc:rematerializeguard22222" + rematerializeGuardCID = "bafkreirematerializeguard" + + // rematerializeGuardHost passes every check this path applies — HydrateBlobURL + // only refuses empty strings — and is a NAME rather than an address, so + // classification is the only thing that can refuse it. `.example` is reserved + // by RFC 2606, so nothing resolves it for real if the seam is ever bypassed. + rematerializeGuardHost = "https://author-pds.example" +) + +// countingBlobHost records whether the copy ever reached a listener. +type countingBlobHost struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingBlobHost(t *testing.T) *countingBlobHost { + t.Helper() + + host := &countingBlobHost{} + host.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + host.requests.Add(1) + _, _ = w.Write([]byte("blob bytes from an internal endpoint")) + })) + t.Cleanup(host.server.Close) + return host +} + +// TestDefaultRematerializeBlobClient_FetchRefusesAPrivateHostWithoutReachingIt +// is the binding contract for the copy leg. +func TestDefaultRematerializeBlobClient_FetchRefusesAPrivateHostWithoutReachingIt(t *testing.T) { + t.Parallel() + + host := newCountingBlobHost(t) + + _, err := DefaultRematerializeBlobClient(false). + Fetch(context.Background(), host.server.URL, rematerializeGuardDID, rematerializeGuardCID) + + assert.Zerof(t, host.requests.Load(), + "the listener was reached %d times. The host is the author repo's HostURL — a DID "+ + "document's serviceEndpoint, which anyone can mint — so the request leaving the process "+ + "is the SSRF whatever comes back", host.requests.Load()) + + require.Error(t, err, + "the blob copy fetched a loopback address successfully. This runs inside a maintenance "+ + "window with thousands of records scrolling past, so a dialled internal address is "+ + "indistinguishable from a refused one in the tool's output") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the fetch failed, but not because the guard refused the address. A host that simply could "+ + "not be reached fails identically, so without the guard's own identity this assertion "+ + "passes against the current build, where the client is a bare http.Client; got: %v", err) +} + +// TestDefaultRematerializeBlobClient_PresentRefusesAPrivateHostWithoutReachingIt +// covers the OTHER method, which is not a duplicate of the one above. +// +// Present is the probe the tool consults before deleting the community's copy of +// a blob, and its contract is that a transport failure is an ERROR and never a +// false. A guard bolted onto Fetch alone would leave the probe dialling; a guard +// whose refusal Present collapsed into "absent" would be worse than no guard at +// all, because "absent" is the answer that licenses a delete. +func TestDefaultRematerializeBlobClient_PresentRefusesAPrivateHostWithoutReachingIt(t *testing.T) { + t.Parallel() + + host := newCountingBlobHost(t) + + present, err := DefaultRematerializeBlobClient(false). + Present(context.Background(), host.server.URL, rematerializeGuardDID, rematerializeGuardCID) + + assert.Zerof(t, host.requests.Load(), + "the presence probe reached the listener %d times", host.requests.Load()) + + require.Error(t, err, + "the presence probe dialled a loopback address. This probe decides whether it is safe to "+ + "delete the only surviving copy of a blob") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity; got: %v", err) + assert.Falsef(t, present, + "a refused address was reported as PRESENT. Present is what licenses the delete of the "+ + "community's copy, so a refusal must never read as 'the bytes are there'") +} + +// TestDefaultRematerializeBlobClient_ReachesTheHostWhenTheHatchIsOpen is the +// other direction, and the falsifiability control for both cases above: a client +// that could make no request at all would satisfy them just as well. +func TestDefaultRematerializeBlobClient_ReachesTheHostWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + host := newCountingBlobHost(t) + + data, err := DefaultRematerializeBlobClient(true). + Fetch(context.Background(), host.server.URL, rematerializeGuardDID, rematerializeGuardCID) + + require.NoErrorf(t, err, + "the hatch is what a dev run of cmd/rematerialize-posts depends on: with it open the copy "+ + "must reach a loopback PDS; got: %v", err) + assert.NotEmpty(t, data, "the fixture serves bytes, so a successful copy must return them") + assert.Equalf(t, int64(1), host.requests.Load(), + "the listener was reached %d times rather than once", host.requests.Load()) +} + +// TestDefaultRematerializeBlobClient_GuardedIsTheDefaultForTheStateMachine pins +// the branch production actually runs, at the place it is actually taken. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the PERMISSIVE branch at +// every call site holding such a boolean. Here there is no boolean at all — +// blobClient() falls back to the default whenever Rematerializer.Blobs is nil, +// which is every construction in this package's own tests and every one in +// rematerialize_dryrun.go. So the guarded spelling has to be the one the +// fallback uses, and this is where that is checked. +func TestDefaultRematerializeBlobClient_GuardedIsTheDefaultForTheStateMachine(t *testing.T) { + t.Parallel() + + host := newCountingBlobHost(t) + + fallback := (&Rematerializer{}).blobClient() + require.NotNil(t, fallback, "a Rematerializer with no injected Blobs must still have a client") + + _, err := fallback.Fetch(context.Background(), host.server.URL, + rematerializeGuardDID, rematerializeGuardCID) + + assert.Zerof(t, host.requests.Load(), + "the state machine's DEFAULT blob client reached the listener %d times. Every caller that "+ + "does not inject one — including the dry run — copies blobs through this client", + host.requests.Load()) + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the fallback client is not the guarded one; got: %v", err) +} + +// TestDefaultRematerializeBlobClient_PreservesTheFetchTimeout guards a value the +// shared client would otherwise swallow, and this one is not a courtesy. +// +// NewSSRFSafeHTTPClient ships a 15s ceiling. A single blob here may be 100 MiB, +// which is why this site allows two MINUTES — so adopting the shared client +// without re-applying rematerializeBlobFetchTimeout would not "re-time" the +// copy, it would make large-blob copies fail outright, mid-migration, and be +// reported as a flaky PDS. +func TestDefaultRematerializeBlobClient_PreservesTheFetchTimeout(t *testing.T) { + t.Parallel() + + client, ok := DefaultRematerializeBlobClient(false).(*httpRematerializeBlobClient) + require.True(t, ok, "DefaultRematerializeBlobClient must return the concrete client these tests drive") + require.NotNil(t, client.client, "the blob client must hold an HTTP client") + + assert.Equalf(t, rematerializeBlobFetchTimeout, client.client.Timeout, + "the blob copy runs on a %v timeout instead of rematerializeBlobFetchTimeout (%v). A blob "+ + "here may be 100 MiB; inheriting the shared client's 15s ceiling would fail those "+ + "copies rather than merely hurrying them", + client.client.Timeout, rematerializeBlobFetchTimeout) +} + +// TestDefaultRematerializeBlobClient_PreservesTheCopyCap is the same trap in the +// other dimension, and it is the one that bites silently. +// +// maxRematerializeBlobBytes is 100 MiB. oauth.DefaultMaxResponseBytes is 32 MiB +// — SMALLER — so unlike every other conversion in this remediation, adopting the +// shared client here TIGHTENS the limit unless the cap is raised explicitly. +// oauth.DefaultMaxResponseBytes documents this obligation by name. +// +// The failure would arrive as an error on a blob between 32 and 100 MiB, in a +// tool that has already re-materialized the post and is about to delete the +// legacy record. That is the worst possible moment for a limit nobody chose. +func TestDefaultRematerializeBlobClient_PreservesTheCopyCap(t *testing.T) { + t.Parallel() + + client, ok := DefaultRematerializeBlobClient(false).(*httpRematerializeBlobClient) + require.True(t, ok, "DefaultRematerializeBlobClient must return the concrete client these tests drive") + + assert.Equalf(t, maxRematerializeBlobBytes, client.maxBytes, + "the production client's own copy cap is %d rather than maxRematerializeBlobBytes (%d)", + client.maxBytes, maxRematerializeBlobBytes) +} + +// TestDefaultRematerializeBlobClient_TheTransportDoesNotClampBelowTheCopyCap is +// the assertion the field check above cannot make. +// +// c.maxBytes is what Fetch enforces after the body arrives. The TRANSPORT has a +// second, independent ceiling, and it refuses an announced Content-Length over +// that ceiling before a byte of body is read — so a client whose maxBytes field +// says 100 MiB can still be unable to receive 40. +// +// It is asserted through an announced length rather than by moving 40 MiB +// through a test: the transport's check is `resp.ContentLength > maxResponseBytes` +// in RoundTrip, which is reached without a body. The declared length is short by +// design, so the read fails either way — what this test reads is WHICH failure, +// and ErrResponseTooLarge is the one that means the shared 32 MiB default is +// still in force. +// +// THE ANNOUNCED LENGTH IS ONE BYTE OVER THE COPY CAP, AND THAT IS THE WHOLE +// TEST. Announcing exactly maxRematerializeBlobBytes was the original fixture, +// and it passed against a transport clipping at exactly that number — which is +// the off-by-one that made Fetch's overrun probe unreachable in production. +// maxBytes+1 is the smallest length the two caps disagree about, so it is the +// only one that can tell them apart. See +// TestDefaultRematerializeBlobClient_AnOverrunKeepsItsCIDExplanation for what +// the extra byte is for. +func TestDefaultRematerializeBlobClient_TheTransportDoesNotClampBelowTheCopyCap(t *testing.T) { + t.Parallel() + + // The hatch is open because the fixture is on loopback; the cap under test is + // a different control from the address guard, and this is the only way to get + // a response header from a real transport in the unit tier. + const announced = maxRematerializeBlobBytes + 1 // the probing byte, which must reach Fetch + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", fmt.Sprint(announced)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("a short body under a large declared length")) + })) + t.Cleanup(server.Close) + + _, err := DefaultRematerializeBlobClient(true). + Fetch(context.Background(), server.URL, rematerializeGuardDID, rematerializeGuardCID) + + require.Error(t, err, "the declared length is not delivered, so this fetch fails either way") + assert.NotErrorIsf(t, err, covesoauth.ErrResponseTooLarge, + "a response declaring maxRematerializeBlobBytes+1 (%d) was refused by the TRANSPORT's own "+ + "ceiling, which defaults to %d — smaller than this site's cap. Two failures wear this "+ + "shape: the shared 32 MiB default still being in force, which fails every blob between "+ + "32 and 100 MiB mid-migration; and a cap set to exactly the copy cap, which clips the byte "+ + "Fetch reads past it to DETECT an overrun and turns the CID-hazard error into a byte "+ + "count; got: %v", + announced, covesoauth.DefaultMaxResponseBytes, err) +} + +// resolvingBlobClient builds the client the way production does and then +// replaces only its NAME RESOLUTION, so the client under test is the real one. +func resolvingBlobClient(t *testing.T, allowPrivateHosts bool, resolvesTo string) RematerializeBlobClient { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(resolvesTo) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", resolvesTo) + + return newGuardedRematerializeBlobClient(allowPrivateHosts, maxRematerializeBlobBytes, + covesoauth.WithHostResolver(func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + })) +} + +// TestDefaultRematerializeBlobClient_RefusesAWellFormedHostThatResolvesPrivate +// is the assertion a loopback-literal fixture cannot make: the guard's +// CLASSIFICATION pass, on a name that survives every earlier check. +// +// Nothing on this path validates the host's shape — HydrateBlobURL refuses only +// empty strings — so a literal fixture does reach classification here. This case +// still earns its place: it is the one that fails if a future edit satisfies the +// tests above with a literal-only check, which is exactly the mutation the +// jetstream and aggregator sites were caught by. +func TestDefaultRematerializeBlobClient_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + client := resolvingBlobClient(t, false, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + _, err := client.Fetch(context.Background(), rematerializeGuardHost, + rematerializeGuardDID, rematerializeGuardCID) + + require.Errorf(t, err, + "%s is a well-formed https host whose DNS answer was 127.0.0.1, and the blob copy fetched "+ + "it anyway. An author's serviceEndpoint is chosen by whoever minted their DID document, "+ + "and they own the zone", rematerializeGuardHost) + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a build where this client was never "+ + "converted looks identical; got: %v", err) +} + +// TestDefaultRematerializeBlobClient_ControlTheSameHostIsDialledWithTheHatchOpen +// is the falsifiability control for the case above. +// +// Identical constructor, identical seam, identical host — only the hatch +// differs. With it open the address is no longer refused, so the request +// proceeds to a dial, which fails because nothing is listening on loopback:443. +// That difference is what pins the refusal above to classification rather than to +// this test being unable to make requests at all. +func TestDefaultRematerializeBlobClient_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + client := resolvingBlobClient(t, true, "127.0.0.1") // coves:allow-host-literal: with the hatch open this is dialled and refused by the OS + + _, err := client.Fetch(context.Background(), rematerializeGuardHost, + rematerializeGuardDID, rematerializeGuardCID) + + require.Error(t, err, + "nothing listens on loopback:443, so this fetch must fail — if it succeeded, the seam is "+ + "not answering with the address this test gave it") + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either the gate is not "+ + "reaching the client, or the guarded case above proves nothing: a client that refuses "+ + "every address refuses that case too, for a reason unconnected to classification; got: %v", + err) +} diff --git a/internal/core/posts/rematerialize_blob_overrun_test.go b/internal/core/posts/rematerialize_blob_overrun_test.go new file mode 100644 index 0000000..8e2e579 --- /dev/null +++ b/internal/core/posts/rematerialize_blob_overrun_test.go @@ -0,0 +1,138 @@ +package posts + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + covesoauth "Coves/internal/atproto/oauth" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The overrun probe, driven through the client production actually builds. +// +// TestRematerializeBlobClient_Fetch_FailsRatherThanTruncating already covers the +// probe, but it hands newRematerializeBlobClient an httptest server.Client() — +// an UNGUARDED client with no byte cap of its own. So it proves Fetch's +// arithmetic and nothing about the transport underneath, which is exactly how +// the transport came to clip the probing byte off with every existing test +// green. + +// TestDefaultRematerializeBlobClient_AnOverrunKeepsItsCIDExplanation is the +// end-to-end version, at a size a test can move. +// +// # THE TWO CAPS ARE OFF BY ONE FROM EACH OTHER, ON PURPOSE +// +// Fetch reads c.maxBytes+1 through an io.LimitReader because io.ReadAll cannot +// tell "the body ended" from "the limit was reached" — the extra byte's +// existence IS the overrun signal, and its whole point is to produce an error +// that explains the hazard: a truncated blob is DIFFERENT bytes, so it lands +// under a different CID and the postv2 ends up pointing at media the repo does +// not serve. A transport cap set to exactly c.maxBytes clips that byte, the +// probe never completes, and the overrun surfaces as a generic +// ErrResponseTooLarge naming a limit no operator configured. +// +// # WHY THE BODY HERE IS CHUNKED +// +// The transport ALSO refuses an announced Content-Length above its cap, before +// a byte is read. That branch fires first for any response that declares its +// length, so it would mask the read path entirely. A chunked body — which is +// what a large blob stream and any transparently-decompressed response both are +// — reports ContentLength -1 and reaches the probing read, which is the path +// this off-by-one lives on. +// +// Not a security hole either way: an oversized blob is refused in both worlds. +// What the fix buys is the message, during a one-shot migration that has already +// written the postv2 and is about to delete the only intact copy. +func TestDefaultRematerializeBlobClient_AnOverrunKeepsItsCIDExplanation(t *testing.T) { + t.Parallel() + + const copyCap = 1024 + oversized := strings.Repeat("x", copyCap+64) + + // The hatch is open because the fixture is on loopback. The byte cap is a + // different control from the address guard. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + flusher, ok := w.(http.Flusher) + require.True(t, ok, "the fixture needs a Flusher to send a chunked body") + w.WriteHeader(http.StatusOK) + flusher.Flush() // no Content-Length: the response is chunked from here + _, _ = w.Write([]byte(oversized)) + })) + t.Cleanup(server.Close) + + client := newGuardedRematerializeBlobClient(true, copyCap) + _, err := client.Fetch(context.Background(), server.URL, + rematerializeGuardDID, rematerializeGuardCID) + + require.Error(t, err, "a body over the copy cap must never be accepted") + assert.NotErrorIsf(t, err, covesoauth.ErrResponseTooLarge, + "the overrun was refused by the TRANSPORT's cap instead of by Fetch's own probe. The transport is "+ + "clipping at exactly the copy cap, so the byte Fetch reads past it to DETECT an overrun never "+ + "arrives and `len(data) > c.maxBytes` is unreachable in production; got: %v", err) + assert.Containsf(t, err.Error(), "truncated", + "the overrun error no longer explains the hazard. An operator mid-migration needs to read that a "+ + "truncated copy is DIFFERENT bytes under a different CID, not a byte count against a limit "+ + "they never set; got: %v", err) +} + +// TestDefaultRematerializeBlobClient_TheBoundaryBlobIsRefusedByItsOwnCap is the +// declared-length half of the same boundary. +// +// A body of exactly copyCap+1 announces copyCap+1, so the transport's header +// check sees the ONE length that separates the two caps: refused when the +// transport clips at copyCap, delivered when it allows the probing byte. It is +// the smallest input that tells the two implementations apart with a +// Content-Length present. +func TestDefaultRematerializeBlobClient_TheBoundaryBlobIsRefusedByItsOwnCap(t *testing.T) { + t.Parallel() + + const copyCap = 1024 + body := strings.Repeat("x", copyCap+1) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", fmt.Sprint(len(body))) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + client := newGuardedRematerializeBlobClient(true, copyCap) + _, err := client.Fetch(context.Background(), server.URL, + rematerializeGuardDID, rematerializeGuardCID) + + require.Error(t, err, "a blob one byte over the copy cap must be refused") + assert.Containsf(t, err.Error(), "truncated", + "a blob of exactly copyCap+1 was refused by the transport's announced-length check rather than by "+ + "Fetch's overrun probe, so the CID hazard goes unexplained at the one size the two caps "+ + "disagree about; got: %v", err) +} + +// TestDefaultRematerializeBlobClient_ABlobAtTheCapStillArrives is the other side +// of the boundary, and it is what stops "+1" from drifting upward: a body of +// exactly the copy cap is legitimate and must come back byte-for-byte. +func TestDefaultRematerializeBlobClient_ABlobAtTheCapStillArrives(t *testing.T) { + t.Parallel() + + const copyCap = 1024 + body := strings.Repeat("y", copyCap) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + client := newGuardedRematerializeBlobClient(true, copyCap) + data, err := client.Fetch(context.Background(), server.URL, + rematerializeGuardDID, rematerializeGuardCID) + + require.NoError(t, err, "a blob of exactly the copy cap is inside the limit and must arrive") + assert.Equalf(t, body, string(data), + "a blob at exactly the copy cap came back changed, so the probing byte is being counted as "+ + "payload; got %d bytes", len(data)) +} diff --git a/internal/core/posts/rematerialize_blob_seam_test.go b/internal/core/posts/rematerialize_blob_seam_test.go new file mode 100644 index 0000000..388d2a7 --- /dev/null +++ b/internal/core/posts/rematerialize_blob_seam_test.go @@ -0,0 +1,39 @@ +package posts + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestDefaultRematerializeBlobClient_TakesNoOptionsFromOutsideThisPackage is the +// same fence as users' TestNewProfileBackfillClient_TakesNoOptionsFromOutside +// ThisPackage, over the constructor that makes the same claim. +// +// The doc comment said "Production passes nothing; it cannot open the guard, +// which only allowPrivateHosts does." The parameter was `opts ...covesoauth. +// Option` and covesoauth.WithPrivateAddressesAllowed() is exported, so the +// second clause was false — any caller in the tree could open the guard on the +// batch tool that copies blobs from whatever host a federated community's PDSURL +// or an author's DID document names. +// +// The seam it existed for did not have to be exported to work: +// newGuardedRematerializeBlobClient is in this package, and this package is +// where the tests that need it live. +func TestDefaultRematerializeBlobClient_TakesNoOptionsFromOutsideThisPackage(t *testing.T) { + t.Parallel() + + fn := reflect.TypeOf(DefaultRematerializeBlobClient) + + assert.Falsef(t, fn.IsVariadic(), + "DefaultRematerializeBlobClient is variadic (%s), so a caller outside this package can pass "+ + "covesoauth options — including the exported WithPrivateAddressesAllowed(). Its doc comment "+ + "says the guard 'only allowPrivateHosts' opens; while this parameter exists that is a "+ + "convention, not a type fact, and there is no coves:allow-ssrf-hatch marker on such a call "+ + "for the audit to find. newGuardedRematerializeBlobClient is where the test seam belongs", fn) + + assert.Equalf(t, 1, fn.NumIn(), + "DefaultRematerializeBlobClient takes %d parameters. The dev gate is the only thing a caller "+ + "outside this package has to say about the client it gets", fn.NumIn()) +} diff --git a/internal/core/posts/rematerialize_blob_test.go b/internal/core/posts/rematerialize_blob_test.go index a0cfed4..51c2569 100644 --- a/internal/core/posts/rematerialize_blob_test.go +++ b/internal/core/posts/rematerialize_blob_test.go @@ -57,6 +57,12 @@ func TestRematerializeBlobClient_Fetch_ReturnsTheWholeBodyUnderTheCap(t *testing // error and a 404 into a single false — so a network blip refused a healthy // record, and any future change of polarity would have licensed deleting the // last copy of a blob. +// +// THE HATCH IS OPEN IN THESE SUBTESTS because they serve their fixtures from +// httptest, which listens on loopback — exactly what the address guard refuses. +// They are about what a STATUS CODE means, not about which addresses may be +// dialled; rematerialize_blob_guard_test.go owns that, and the last subtest +// keeps the guarded spelling because it never gets as far as a dial. func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *testing.T) { t.Run("200 is present", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -64,7 +70,7 @@ func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *tes })) defer server.Close() - present, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + present, err := DefaultRematerializeBlobClient(true).Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") require.NoError(t, err) assert.True(t, present) }) @@ -75,7 +81,7 @@ func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *tes })) defer server.Close() - present, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + present, err := DefaultRematerializeBlobClient(true).Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") require.NoErrorf(t, err, "a definite 404 is an ANSWER — the blob is not there — and must not be reported as a failure to ask") assert.False(t, present) }) @@ -86,7 +92,7 @@ func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *tes })) defer server.Close() - _, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + _, err := DefaultRematerializeBlobClient(true).Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") require.Errorf(t, err, "a 503 was collapsed into 'absent'. The caller uses this answer to decide whether it is safe to delete the record that keeps the community's "+ "only copy of the bytes alive, and a server that could not answer has told it nothing") @@ -97,12 +103,12 @@ func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *tes url := server.URL server.Close() // nothing is listening now - _, err := DefaultRematerializeBlobClient().Present(context.Background(), url, "did:plc:x2222222222222222222222", "bafkrei1") + _, err := DefaultRematerializeBlobClient(true).Present(context.Background(), url, "did:plc:x2222222222222222222222", "bafkrei1") require.Errorf(t, err, "a transport failure must surface; reporting it as 'absent' turns a blip into a refusal and a polarity slip into data loss") }) t.Run("an unbuildable URL is an error, never a false", func(t *testing.T) { - _, err := DefaultRematerializeBlobClient().Present(context.Background(), "", "did:plc:x2222222222222222222222", "bafkrei1") + _, err := DefaultRematerializeBlobClient(false).Present(context.Background(), "", "did:plc:x2222222222222222222222", "bafkrei1") require.Errorf(t, err, "a URL that could not be built means the question was never asked") }) } diff --git a/internal/core/posts/rematerialize_outer_test.go b/internal/core/posts/rematerialize_outer_test.go index c71abba..d9c4873 100644 --- a/internal/core/posts/rematerialize_outer_test.go +++ b/internal/core/posts/rematerialize_outer_test.go @@ -155,7 +155,7 @@ func TestRematerialize_OuterContract_RealPDS_MovesPostAndIsIdempotent(t *testing // Real author-repo credentials for the author, over the real PDS. authorFactory := func(_ context.Context, authorDID string, _ *oauth.ClientSessionData) (posts.AuthorRepo, error) { require.Equalf(t, authorAcct.DID, authorDID, "the tool asked for a repo other than the post's author") - generic, err := pds.NewFromAccessToken(pdsServer.URL(), authorAcct.DID, authorAcct.AccessToken) + generic, err := pds.NewFromAccessToken(pdsServer.URL(), authorAcct.DID, authorAcct.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) repo, ok := generic.(posts.AuthorRepo) require.True(t, ok, "the PDS client must implement the author-repo write surface") @@ -163,7 +163,7 @@ func TestRematerialize_OuterContract_RealPDS_MovesPostAndIsIdempotent(t *testing } // Real community-repo credentials, and the DIRECT acceptance writer over them. - communityGeneric, err := pds.NewFromAccessToken(pdsServer.URL(), communityAcct.DID, communityAcct.AccessToken) + communityGeneric, err := pds.NewFromAccessToken(pdsServer.URL(), communityAcct.DID, communityAcct.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) communityRepo, ok := communityGeneric.(posts.CommunityRepo) require.True(t, ok, "the PDS client must implement the community-repo write surface") @@ -290,13 +290,13 @@ func TestRematerialize_OuterContract_CopiesEmbedBlobToAuthorRepo(t *testing.T) { } authorFactory := func(_ context.Context, _ string, _ *oauth.ClientSessionData) (posts.AuthorRepo, error) { - generic, err := pds.NewFromAccessToken(pdsServer.URL(), authorAcct.DID, authorAcct.AccessToken) + generic, err := pds.NewFromAccessToken(pdsServer.URL(), authorAcct.DID, authorAcct.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) repo, ok := generic.(posts.AuthorRepo) require.True(t, ok) return repo, nil } - communityGeneric, err := pds.NewFromAccessToken(pdsServer.URL(), communityAcct.DID, communityAcct.AccessToken) + communityGeneric, err := pds.NewFromAccessToken(pdsServer.URL(), communityAcct.DID, communityAcct.AccessToken, pds.PrivateHostOptions(true)...) require.NoError(t, err) communityRepo, ok := communityGeneric.(posts.CommunityRepo) require.True(t, ok) @@ -306,7 +306,26 @@ func TestRematerialize_OuterContract_CopiesEmbedBlobToAuthorRepo(t *testing.T) { source := &realLegacySource{community: communityGeneric, staged: []posts.LegacyPost{legacy}} ledger := postgres.NewRematerializeLedger(testkit.DB(t)) communityRepos := func(_ context.Context, _ string) (posts.CommunityRepo, error) { return communityRepo, nil } - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer, CommunityRepos: communityRepos} + + // THE HATCH IS OPEN, AND THIS IS THE ONLY TEST IN THE TREE THAT NEEDS IT. + // + // Rematerializer.blobClient() falls back to the GUARDED default, which refuses + // private, loopback and link-local addresses — and the CI stack's PDS is on + // loopback, which is exactly what it refuses. Every other Rematerializer test + // stages records carrying no blobs, so the fallback client is constructed and + // never dialled; this one copies real bytes through it. + // + // It is passed as an explicit Blobs client rather than reached by loosening + // the fallback, for the reason blobs and imageproxy pass PrivateHostOptions at + // their construction: the decision to dial a private address is made ONCE, in + // the open, by whoever built the thing — and the guarded spelling stays the + // one a caller gets by omission. Nothing about the state machine's own + // behaviour changes; this replaces the client, not the path. + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer, + CommunityRepos: communityRepos, + Blobs: posts.DefaultRematerializeBlobClient(true), + } _, err = tool.RematerializeOne(ctx, legacy) require.NoError(t, err) diff --git a/internal/core/posts/service.go b/internal/core/posts/service.go index fa0feda..98afba0 100644 --- a/internal/core/posts/service.go +++ b/internal/core/posts/service.go @@ -46,6 +46,11 @@ type postService struct { // that retracts, from a hosted community's repo, the acceptance the fast // path put there. See compensateAuthorDelete. acceptanceWithdrawal AcceptanceWithdrawer + + // pdsClientOptions is the SSRF dev gate for every PDS client this service + // builds from a community's own credentials. Empty — the zero value — means + // GUARDED, so a wiring that forgets it is strict rather than open. + pdsClientOptions []pds.ClientOption } // PostServiceOption configures optional postService dependencies. Options keep the @@ -60,6 +65,19 @@ func WithBlockChecker(checker BlockChecker) PostServiceOption { return func(s *postService) { s.blockChecker = checker } } +// WithPDSClientOptions supplies the SSRF dev gate for the PDS clients this +// service builds against a COMMUNITY's repo, on that community's credentials. +// +// Two paths need it and both dial `community.PDSURL`, a database column rather +// than configuration: deleteCommunityPost's direct client, and the acceptance +// withdrawer's repo factory built in NewPostService. Passing nothing leaves both +// guarded, which is why this is an option and not a required parameter — the +// omission fails closed. cmd/server passes pds.PrivateHostOptions(cfg.IsDevEnv)...; +// tests against the CI stack's loopback PDS pass pds.PrivateHostOptions(true)... +func WithPDSClientOptions(opts ...pds.ClientOption) PostServiceOption { + return func(s *postService) { s.pdsClientOptions = opts } +} + // NewPostService creates a new post service // aggregatorService, blobService, unfurlService, and blueskyService can be nil if not needed (e.g., in tests or minimal setups) func NewPostService( @@ -97,7 +115,8 @@ func NewPostService( // community this instance does not host it answers ErrCommunityNotHosted and // the delete path skips, which is the common case. if s.acceptanceWithdrawal == nil && communityService != nil { - s.acceptanceWithdrawal = NewCommunityRecordWriter(NewCommunityRepoFactory(communityService), time.Now) + s.acceptanceWithdrawal = NewCommunityRecordWriter( + NewCommunityRepoFactory(communityService, s.pdsClientOptions...), time.Now) } // The admission policy is mandatory, and a missing or partial one panics @@ -2003,8 +2022,10 @@ func (s *postService) deleteCommunityPost(ctx context.Context, userDID, communit return fmt.Errorf("failed to refresh community credentials: %w", err) } - // 6. Create PDS client for community repository - pdsClient, err := pds.NewFromAccessToken(community.PDSURL, community.DID, community.PDSAccessToken) + // 6. Create PDS client for community repository. The options carry the SSRF + // dev gate; empty means guarded, and community.PDSURL is a database column. + pdsClient, err := pds.NewFromAccessToken(community.PDSURL, community.DID, community.PDSAccessToken, + s.pdsClientOptions...) if err != nil { return fmt.Errorf("failed to create PDS client: %w", err) } diff --git a/internal/core/posts/service_author_posts_query_test.go b/internal/core/posts/service_author_posts_query_test.go index 7309f5c..0cc3e6a 100644 --- a/internal/core/posts/service_author_posts_query_test.go +++ b/internal/core/posts/service_author_posts_query_test.go @@ -93,7 +93,8 @@ func newAuthorPostsFixture(t *testing.T) *authorPostsFixture { // the post service consults this one only to resolve an identifier to a DID // and to check the community exists. communityService := communities.NewCommunityServiceWithPDSFactory( - communityRepo, pdsURL, fixtures.InstanceDID(), "", nil, nil, nil) + communityRepo, pdsURL, fixtures.InstanceDID(), "", nil, nil, nil, + communities.PrivateHostOptions(true)...) // The optional post collaborators (aggregators, blobs, unfurl, bluesky) are // all write-path concerns and stay nil. postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, diff --git a/internal/core/posts/service_create_validation_test.go b/internal/core/posts/service_create_validation_test.go index 40cb9a5..baceee8 100644 --- a/internal/core/posts/service_create_validation_test.go +++ b/internal/core/posts/service_create_validation_test.go @@ -64,7 +64,8 @@ func TestService_CreateResolvesTheCommunityAndValidatesTheRequest(t *testing.T) // instance hosts, so the service's notion of "here" has to match the domain // in the community's handle. communityService := communities.NewCommunityServiceWithPDSFactory( - communityRepo, pdsURL, instanceDID, instanceDomain, nil, nil, nil) + communityRepo, pdsURL, instanceDID, instanceDomain, nil, nil, nil, + communities.PrivateHostOptions(true)...) postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests())) diff --git a/internal/core/posts/service_writeforward_test.go b/internal/core/posts/service_writeforward_test.go index e608721..0b88562 100644 --- a/internal/core/posts/service_writeforward_test.go +++ b/internal/core/posts/service_writeforward_test.go @@ -240,7 +240,7 @@ func (r *authorRepoRegistry) factory() posts.AuthorRepoFactory { if host == "" { host = r.pds.URL() } - client, err := pds.NewFromAccessToken(host, account.DID, account.AccessToken) + client, err := pds.NewFromAccessToken(host, account.DID, account.AccessToken, pds.PrivateHostOptions(true)...) if err != nil { return nil, fmt.Errorf("opening the repo of %s: %w", authorDID, err) } @@ -319,9 +319,10 @@ func newPostFixture(t *testing.T) *postFixture { pdsServer.URL(), instanceDID, instanceDomain, - communities.NewPDSAccountProvisioner(instanceDomain, pdsServer.URL()), - testkit.PasswordAuthFactory(pds.NewFromAccessToken), + communities.NewPDSAccountProvisioner(instanceDomain, pdsServer.URL(), communities.PrivateHostOptions(true)...), + testkit.PasswordAuthFactory(pds.NewFromAccessToken, pds.PrivateHostOptions(true)...), nil, + communities.PrivateHostOptions(true)..., ) authorRepos := newAuthorRepoRegistry(pdsServer) @@ -353,7 +354,12 @@ func newPostFixture(t *testing.T) *postFixture { // fail. It is scripted to refuse, so a fast path that consulted it at // all fails these tests loudly. &scriptedDecider{code: posts.DecisionRuleViolation}, - posts.NewCommunityRecordWriter(posts.NewCommunityRepoFactory(communityService), time.Now), + posts.NewCommunityRecordWriter( + // The hatch, because every community repo this factory opens is on the + // CI stack's PDS at loopback — exactly the address class the guard + // refuses. Production wiring passes pds.PrivateHostOptions(cfg.IsDevEnv). + posts.NewCommunityRepoFactory(communityService, pds.PrivateHostOptions(true)...), + time.Now), posts.NewCommunityCredentialRefresher(communityService), ) acceptor := &acceptorSpy{delegate: engine} @@ -389,6 +395,14 @@ func (f *postFixture) writePathOptions() []posts.PostServiceOption { return []posts.PostServiceOption{ posts.WithAuthorRepoFactory(f.authorRepos.factory()), posts.WithSyncAcceptance(f.admissions, f.acceptor), + + // The SSRF dev gate for the clients the service builds against a + // COMMUNITY's repo — deleteCommunityPost's direct client and the + // acceptance withdrawer's factory. It belongs here rather than at one + // fixture because it is a property of WHERE these tests point (the CI + // stack's loopback PDS), not of what any one of them is proving, and the + // neighbouring fixtures dial the same PDS through the same two paths. + posts.WithPDSClientOptions(pds.PrivateHostOptions(true)...), } } diff --git a/internal/core/unfurl/body_cap_test.go b/internal/core/unfurl/body_cap_test.go new file mode 100644 index 0000000..3553fb2 --- /dev/null +++ b/internal/core/unfurl/body_cap_test.go @@ -0,0 +1,245 @@ +package unfurl + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The byte cap at the ONLY site in this tree that returns a response body to its +// caller. +// +// # THE DEFECT THESE TESTS WERE WRITTEN AGAINST +// +// NewService told the transport maxUnfurlBodyBytes, and both HTML read paths +// then wrapped the body in io.LimitReader(resp.Body, maxUnfurlBodyBytes) — the +// SAME number. A LimitReader stops at exactly its allowance, so cappedBody was +// never asked for the byte past the cap that IS its overrun signal. An over-cap +// page therefore arrived TRUNCATED, with no error, and went straight into +// parseOpenGraph and html.Parse — both of which are error-tolerant by design and +// will happily produce a document from half a file. What came back was an +// og:title and og:description that read as a complete page, cached for 24 hours +// and served into a post. +// +// imageproxy/fetcher.go and posts' newGuardedRematerializeBlobClient both set +// their transport cap to their own limit PLUS ONE for exactly this reason, and +// both say so in a comment. Unfurl was the one converted site that did not — and +// it is the one where the consequence is content rather than a failed fetch. +// +// # WHY THE FIXTURES DECLARE NO CONTENT-LENGTH +// +// The transport has a second, earlier control: it refuses a response whose +// ANNOUNCED length exceeds the cap, before a byte of body is read. A fixture +// that sets Content-Length is stopped there and proves nothing about the read +// path — the defect above would look fixed. A host that wants its body read is +// simply chunked, which costs an attacker nothing and is what these fixtures do. +// +// # WHY THE PAGES ARE REALLY 10 MB +// +// The cap is a package constant that no seam parameterises, so shrinking it for +// a test would mean testing a number production does not run. Ten megabytes over +// loopback is cheap enough to pay for testing the real one. + +// overCapPage builds an HTML page of exactly size bytes whose og: tags sit in +// the first few hundred, so a truncated read still yields a complete-looking +// result. That ordering is the point: it is what makes the silent truncation +// LOOK like a successful unfurl rather than like a parse failure. +func overCapPage(t *testing.T, size int) string { + t.Helper() + + // The kagiproxy.com is what fetchKagiKite needs to return a result at + // all; without it that path fails with "no image found" and would report + // green for a reason that has nothing to do with the cap. + head := `` + + `` + + `` + + `` + + `topology

` + tail := `

` + + padding := size - len(head) - len(tail) + require.Positivef(t, padding, "the fixture must be larger than its own markup (%d bytes)", len(head)+len(tail)) + + page := head + strings.Repeat("a", padding) + tail + require.Lenf(t, page, size, "the fixture must be EXACTLY %d bytes, not near it", size) + return page +} + +// chunkedTarget serves body without declaring its length, so the transport's +// announced-length refusal is out of the picture and what is exercised is the +// streamed read. +func chunkedTarget(t *testing.T, body string) *httptest.Server { + t.Helper() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + // Deliberately no Content-Length. Flushing after the header commits the + // response to chunked encoding, so net/http cannot infer one from a + // buffered body. + w.WriteHeader(http.StatusOK) + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + return server +} + +// htmlFetch is one of the two paths that reads a body into a document. The +// oEmbed path is excluded because it decodes JSON: json.Decode over a truncated +// document FAILS, so that path already refuses rather than truncating. +type htmlFetch struct { + name string + fetch func(ctx context.Context, url string, client *http.Client) (*UnfurlResult, error) +} + +func htmlFetchPaths() []htmlFetch { + return []htmlFetch{ + { + name: "opengraph", + fetch: func(ctx context.Context, url string, client *http.Client) (*UnfurlResult, error) { + return fetchOpenGraph(ctx, url, client, "CovesBot/1.0") + }, + }, + { + name: "kagi", + fetch: func(ctx context.Context, url string, client *http.Client) (*UnfurlResult, error) { + return fetchKagiKite(ctx, url, client, "CovesBot/1.0") + }, + }, + } +} + +// TestUnfurl_AnOverCapPageIsRefusedRatherThanParsedAsComplete is the binding +// assertion. +// +// One byte past the cap is the tightest fixture that can distinguish "refused" +// from "truncated and parsed": under the defect it is silently clipped back to +// the cap, the og: tags at the top survive, and a caller gets a result it has no +// way to know is short. +func TestUnfurl_AnOverCapPageIsRefusedRatherThanParsedAsComplete(t *testing.T) { + t.Parallel() + + page := overCapPage(t, maxUnfurlBodyBytes+1) + server := chunkedTarget(t, page) + + for _, path := range htmlFetchPaths() { + t.Run(path.name, func(t *testing.T) { + t.Parallel() + + result, err := path.fetch(context.Background(), server.URL, hatchOpenClient(t, 30*time.Second)) + + require.Errorf(t, err, + "the %s path accepted a %d-byte page through a %d-byte cap and returned a result. The "+ + "transport cap equals the io.LimitReader's allowance, so the reader stops at exactly "+ + "the cap and cappedBody is never asked for the byte whose EXISTENCE is the overrun "+ + "signal — the page is clipped, handed to an error-tolerant parser, and comes back "+ + "looking whole. This is the one fetch site in the tree that returns the response body "+ + "to its caller, so what a truncated read produces is a truncated og:title cached for "+ + "24 hours and served into a post", + path.name, len(page), maxUnfurlBodyBytes) + + assert.Nilf(t, result, + "the %s path returned a result ALONGSIDE the error. An enrichment path that logs the "+ + "error and carries on — which is what unfurl is — then caches the truncated document "+ + "anyway", path.name) + + // ErrPageTooLarge and not ErrResponseTooLarge, because WHICH of the + // two caps fired is the fence around the transport's +1. A transport + // told exactly maxUnfurlBodyBytes also refuses this page — from the + // wrong layer, clipping readCappedBody's probing byte and making its + // own overrun branch unreachable in production. The page is refused + // either way, so nothing but the sentinel can tell the two apart. + assert.ErrorIsf(t, err, ErrPageTooLarge, + "the %s path refused the over-cap page, but not through this site's own probe. "+ + "readCappedBody reads maxUnfurlBodyBytes+1 so the extra byte's existence is the "+ + "signal; if the transport cap does not leave room for that byte, the probe is clipped "+ + "and `len(data) > maxUnfurlBodyBytes` becomes dead code — which is the state the "+ + "truncation defect was found in; got: %v", path.name, err) + }) + } +} + +// TestUnfurl_APageAtTheCapStillArrivesWhole is the fence around the assertion +// above, and it is why the transport is told maxUnfurlBodyBytes+1 rather than +// having the read paths tightened to refuse earlier. +// +// A cap that refused at exactly its own limit would satisfy the over-cap test +// and quietly reject every page in the last byte of the allowance. The boundary +// belongs one byte higher than the largest page that must succeed. +func TestUnfurl_APageAtTheCapStillArrivesWhole(t *testing.T) { + t.Parallel() + + page := overCapPage(t, maxUnfurlBodyBytes) + server := chunkedTarget(t, page) + + for _, path := range htmlFetchPaths() { + t.Run(path.name, func(t *testing.T) { + t.Parallel() + + result, err := path.fetch(context.Background(), server.URL, hatchOpenClient(t, 30*time.Second)) + + require.NoErrorf(t, err, + "the %s path refused a page of EXACTLY the %d-byte cap. The limit is the largest body "+ + "that must be accepted, not the first one refused; got: %v", + path.name, maxUnfurlBodyBytes, err) + require.NotNilf(t, result, "the %s path returned no result for a page inside the cap", path.name) + assert.Equalf(t, leakedTitle, result.Title, + "the %s path parsed a page at the cap but lost its og:title, so the body did not survive "+ + "the read intact", path.name) + }) + } +} + +// TestUnfurl_AnAnnouncedOverCapLengthIsStillRefusedBeforeTheBody keeps the +// EARLIER half of the control from being lost while the read paths are fixed. +// +// The transport refuses a response whose declared Content-Length exceeds its cap +// before allocating anything, and raising that cap to make room for the probing +// byte is exactly the kind of edit that could slacken it to the shared 32 MiB +// default instead. A host that declares its length is the ordinary case; the +// chunked fixtures above deliberately dodge this check, so nothing else covers +// it. +func TestUnfurl_AnAnnouncedOverCapLengthIsStillRefusedBeforeTheBody(t *testing.T) { + t.Parallel() + + // Comfortably above this site's 10 MB and comfortably below the shared + // 32 MiB default, so it separates the two rather than sitting on an edge. + const announced = maxUnfurlBodyBytes * 2 + require.Less(t, int64(announced), int64(covesoauth.DefaultMaxResponseBytes), + "the fixture must declare a length the SHARED default would allow, or it proves nothing") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.Header().Set("Content-Length", fmt.Sprint(announced)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(secretInternalHTML)) + })) + t.Cleanup(server.Close) + + for _, path := range htmlFetchPaths() { + t.Run(path.name, func(t *testing.T) { + t.Parallel() + + _, err := path.fetch(context.Background(), server.URL, hatchOpenClient(t, 30*time.Second)) + + require.Errorf(t, err, "the %s path accepted a response declaring %d bytes", path.name, announced) + assert.ErrorIsf(t, err, covesoauth.ErrResponseTooLarge, + "the %s path did not refuse a declared length of %d on the ANNOUNCED-length check, so this "+ + "client is running on oauth.DefaultMaxResponseBytes (%d) rather than on a cap derived "+ + "from maxUnfurlBodyBytes (%d). The read paths bound what is ALLOCATED; this check is "+ + "the half that refuses before a byte of body arrives; got: %v", + path.name, announced, covesoauth.DefaultMaxResponseBytes, maxUnfurlBodyBytes, err) + }) + } +} diff --git a/internal/core/unfurl/circuit_breaker.go b/internal/core/unfurl/circuit_breaker.go index 54815f9..f8ba92d 100644 --- a/internal/core/unfurl/circuit_breaker.go +++ b/internal/core/unfurl/circuit_breaker.go @@ -1,10 +1,13 @@ package unfurl import ( + "errors" "fmt" "log" "sync" "time" + + covesoauth "Coves/internal/atproto/oauth" ) // circuitState represents the state of a circuit breaker @@ -97,8 +100,43 @@ func (cb *circuitBreaker) recordSuccess(provider string) { } } -// recordFailure records a failed unfurl attempt +// recordFailure records a failed unfurl attempt. +// +// # A GUARD REFUSAL IS NOT A FAILURE, AND COUNTING IT IS A DENIAL OF SERVICE +// +// The SSRF guard refuses a private, loopback or link-local destination before a +// packet leaves the process, so a refusal carries no information about whether +// the provider is up — nobody spoke to it. Counting one is wrong twice over. +// +// The first cost is availability, and it is an attack. The breaker's bucket for +// the OpenGraph path is the constant string "opengraph": ONE bucket for the +// whole instance, not one per host. Three pasted `http://127.0.0.1/` links +// therefore disabled link previews for every user of this instance for +// openDuration, at the cost of three posts, repeatable on expiry. The guard did +// its job on each request and the aggregate was the outage — a denial of service +// delivered THROUGH the security control. +// +// The second is diagnostic: the log line below names a healthy provider as the +// failing party and quotes an address error as the evidence, which is exactly +// the wrong place to send an operator looking. +// +// THE CHECK LIVES HERE, not at the three call sites in UnfurlURL, because those +// three sit a few lines apart and each looks like the one above it. That is the +// shape the three-unguarded-clients defect in providers.go took; one check at +// the single point where a failure is COUNTED cannot be forgotten by the fourth +// path someone adds. errors.Is and not a comparison, because what arrives here +// has been through http.Client.Do's *url.Error and one fmt.Errorf of UnfurlURL's +// own. func (cb *circuitBreaker) recordFailure(provider string, err error) { + if errors.Is(err, covesoauth.ErrBlockedAddress) { + log.Printf( + "[UNFURL-CIRCUIT] Not counting a refused address against provider '%s': %v", + provider, + err, + ) + return + } + cb.mu.Lock() defer cb.mu.Unlock() diff --git a/internal/core/unfurl/circuit_breaker_guard_test.go b/internal/core/unfurl/circuit_breaker_guard_test.go new file mode 100644 index 0000000..d101330 --- /dev/null +++ b/internal/core/unfurl/circuit_breaker_guard_test.go @@ -0,0 +1,162 @@ +package unfurl + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The circuit breaker, and the one class of error it must never count. +// +// # THE DEFECT THESE TESTS WERE WRITTEN AGAINST +// +// UnfurlURL fed every fetch error into recordFailure, including the SSRF guard's +// refusal. The breaker opens after 3 consecutive failures and stays open for 5 +// minutes, and its key for the OpenGraph path is the constant string +// "opengraph" — one bucket for the whole instance, not one per host. So three +// pasted `http://127.0.0.1/` links disabled link previews SITE-WIDE for five +// minutes, for every user, repeatably, at the cost of three posts. +// +// That is a denial of service delivered THROUGH the security control: the guard +// does its job perfectly on each request and the aggregate is an outage. It is +// also a mis-attribution — the breaker exists to stop hammering a provider that +// is failing, and a refused address says nothing about whether the provider is +// up. The log line it writes names a healthy provider as the failing party. +// +// # WHY THE CHECK LIVES IN recordFailure AND NOT AT THE CALL SITES +// +// UnfurlURL calls recordFailure from THREE places — kagi, oEmbed and OpenGraph — +// a few lines apart, each looking like the one above it. That is the exact shape +// the three-client defect in providers.go took, and converting two of three is +// how it survives review. One check at the single point where a failure is +// counted cannot be forgotten by the fourth path someone adds. + +// alwaysMissRepo is a cache that never hits and never stores, so every UnfurlURL +// call below reaches the fetch. Nothing here is about caching. +type alwaysMissRepo struct{} + +func (alwaysMissRepo) Get(context.Context, string) (*UnfurlResult, error) { return nil, nil } + +func (alwaysMissRepo) Set(context.Context, string, *UnfurlResult, time.Duration) error { return nil } + +// TestUnfurlService_AGuardRefusalDoesNotOpenTheCircuit is the binding assertion, +// stated the way a user experiences it. +// +// The count is failureThreshold+1 refusals and then one more: the breaker opens +// AT the threshold, so the assertion has to be made on a request that comes +// after it would have opened. What that request must come back with is the +// guard's own refusal — proof it was ATTEMPTED — rather than a breaker error +// naming a provider nobody has spoken to. +func TestUnfurlService_AGuardRefusalDoesNotOpenTheCircuit(t *testing.T) { + t.Parallel() + + // The service as production builds it: PrivateHostOptions(false), so the + // guard is on and loopback is refused before a packet leaves the process. + svc, ok := NewService(alwaysMissRepo{}, PrivateHostOptions(false)...).(*service) + require.True(t, ok, "NewService must return the concrete *service this package's tests drive") + + threshold := svc.circuitBreaker.failureThreshold + require.Positive(t, threshold, "the premise: the breaker must have a threshold to trip") + + // Distinct paths on the same host, because the breaker's bucket is the + // PROVIDER ("opengraph"), not the URL — which is precisely why three links + // take out previews for everyone. + var lastErr error + for i := 0; i <= threshold; i++ { + //nolint:noctx // the guard refuses before any dial; the context is irrelevant here + _, lastErr = svc.UnfurlURL(context.Background(), + fmt.Sprintf("http://127.0.0.1/%d", i)) // coves:allow-host-literal: the address the guard must refuse, and the payload of the DoS + + require.Errorf(t, lastErr, "attempt %d against a loopback address must be refused", i+1) + require.ErrorIsf(t, lastErr, covesoauth.ErrBlockedAddress, + "attempt %d failed for a reason other than the guard, so this test is measuring the wrong "+ + "thing; got: %v", i+1, lastErr) + } + + // The subject. This request is one past the threshold, so under the defect + // the breaker is open and the error names a provider instead of an address. + assert.NotContains(t, lastErr.Error(), "circuit breaker open", + "after %d guard refusals the OpenGraph circuit is open, so link previews are disabled "+ + "INSTANCE-WIDE for %s — for every user, at the cost of %d pasted loopback links, repeatable "+ + "on expiry. A refusal is the guard working; counting it as provider failure turns the "+ + "security control into the outage", + threshold+1, svc.circuitBreaker.openDuration, threshold) + + canAttempt, breakerErr := svc.circuitBreaker.canAttempt("opengraph") + assert.Truef(t, canAttempt, + "the OpenGraph circuit refuses further attempts after %d refused ADDRESSES. The breaker exists "+ + "to stop hammering a provider that is failing, and nothing here has learned anything about "+ + "whether opengraph is up: not one packet left the process; got: %v", threshold+1, breakerErr) +} + +// TestUnfurlService_AGenuineProviderFailureStillOpensTheCircuit is the fence. +// +// The cheapest wrong fix for the test above is to stop recording failures, or to +// widen the exclusion until it swallows the timeouts and 5xx the breaker was +// built for. This is the behaviour that must survive: a provider that really is +// failing still gets cut off. +func TestUnfurlService_AGenuineProviderFailureStillOpensTheCircuit(t *testing.T) { + t.Parallel() + + breaker := newCircuitBreaker() + for i := 0; i < breaker.failureThreshold; i++ { + breaker.recordFailure("opengraph", fmt.Errorf("HTTP request returned status 503")) + } + + canAttempt, err := breaker.canAttempt("opengraph") + assert.Falsef(t, canAttempt, + "%d consecutive 503s left the circuit closed. Excluding guard refusals must not disarm the "+ + "breaker for the failures it was built for — a provider answering 503 in a loop is exactly "+ + "what it stops hammering", breaker.failureThreshold) + require.Error(t, err, "an open circuit must explain itself to the caller") + assert.Contains(t, err.Error(), "circuit breaker open", + "the open circuit must keep the message UnfurlURL logs and the tests above assert the ABSENCE "+ + "of; got: %v", err) +} + +// TestCircuitBreaker_AWrappedGuardRefusalIsRecognised pins the matching +// mechanism at the unit level. +// +// The refusal that arrives at recordFailure has been through http.Client.Do, +// which wraps it in a *url.Error, and then through UnfurlURL's own +// fmt.Errorf("failed to fetch OpenGraph data: %w"). A check comparing errors +// with == or reading err.Error() would pass against a bare sentinel and fail +// against every real one, so the test hands it the shape production produces. +func TestCircuitBreaker_AWrappedGuardRefusalIsRecognised(t *testing.T) { + t.Parallel() + + breaker := newCircuitBreaker() + wrapped := fmt.Errorf("failed to fetch OpenGraph data: %w", + fmt.Errorf(`Get "http://127.0.0.1/": %w`, // coves:allow-host-literal: the *url.Error wrapping production produces + &covesoauth.BlockedAddressError{Host: "127.0.0.1"})) // coves:allow-host-literal: the refused host, carried on the typed error + + require.ErrorIs(t, wrapped, covesoauth.ErrBlockedAddress, + "the premise: the fixture must be a wrapped guard refusal") + + for i := 0; i < breaker.failureThreshold*2; i++ { + breaker.recordFailure("opengraph", wrapped) + } + + canAttempt, err := breaker.canAttempt("opengraph") + assert.Truef(t, canAttempt, + "%d wrapped guard refusals opened the circuit. The refusal reaches recordFailure through a "+ + "*url.Error and one fmt.Errorf, so the exclusion has to be errors.Is and not an identity or "+ + "substring comparison — either of those passes on a bare sentinel and fails on every "+ + "refusal production actually produces; got: %v", breaker.failureThreshold*2, err) + + // Not merely "the circuit stayed closed" — the counter must never have moved. + // A breaker that counted refusals but refused to open on them would satisfy + // the assertion above and still cut off a provider on its first REAL failure + // after a run of refusals, because the count would already be at the + // threshold's door. + assert.Zerof(t, breaker.failures["opengraph"], + "the breaker recorded %d failures against 'opengraph' for refusals that never left the process. "+ + "A leftover count means the next genuine failure trips a breaker it did not earn", + breaker.failures["opengraph"]) +} diff --git a/internal/core/unfurl/errors.go b/internal/core/unfurl/errors.go index 1f6cea9..8a07881 100644 --- a/internal/core/unfurl/errors.go +++ b/internal/core/unfurl/errors.go @@ -11,4 +11,16 @@ var ( // ErrInvalidTTL is returned when the provided TTL is invalid (e.g., negative or zero) ErrInvalidTTL = errors.New("invalid TTL: must be positive") + + // ErrPageTooLarge is returned when a remote page delivers more than + // maxUnfurlBodyBytes. + // + // IT IS A REFUSAL, NOT A TRUNCATION, and that distinction is this site's + // alone among the fetch sites in this tree. Every other one discards the + // body or treats it as opaque bytes; this one parses the body and returns + // its CONTENT — og:title, og:description, og:image — into an UnfurlResult + // that is cached for a day and served into a post. parseOpenGraph and + // html.Parse are both error-tolerant, so half a document yields a result + // that reads as whole. A caller must be told the difference. + ErrPageTooLarge = errors.New("unfurl target exceeds the response size limit") ) diff --git a/internal/core/unfurl/kagi_test.go b/internal/core/unfurl/kagi_test.go index 7ba17f0..01477e0 100644 --- a/internal/core/unfurl/kagi_test.go +++ b/internal/core/unfurl/kagi_test.go @@ -35,7 +35,7 @@ func TestFetchKagiKite_Success(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") require.NoError(t, err) assert.Equal(t, "article", result.Type) @@ -63,7 +63,7 @@ func TestFetchKagiKite_NoImage(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") assert.Error(t, err) assert.Nil(t, result) @@ -89,7 +89,7 @@ func TestFetchKagiKite_FallbackToTitle(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") require.NoError(t, err) assert.Equal(t, "Fallback Title", result.Title) @@ -115,7 +115,7 @@ func TestFetchKagiKite_ImageWithAltText(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") require.NoError(t, err) assert.Equal(t, "News Story", result.Title) @@ -132,7 +132,7 @@ func TestFetchKagiKite_HTTPError(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") assert.Error(t, err) assert.Nil(t, result) @@ -160,7 +160,7 @@ func TestFetchKagiKite_Timeout(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 100*time.Millisecond, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 100*time.Millisecond), "TestBot/1.0") assert.Error(t, err) assert.Nil(t, result) @@ -186,7 +186,7 @@ func TestFetchKagiKite_MultipleImages_PicksSecond(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") require.NoError(t, err) // We skip the first image (often a header/logo) and use the second @@ -213,7 +213,7 @@ func TestFetchKagiKite_OnlyNonKagiImages_NoMatch(t *testing.T) { ctx := context.Background() - result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + result, err := fetchKagiKite(ctx, server.URL, hatchOpenClient(t, 5*time.Second), "TestBot/1.0") assert.Error(t, err) assert.Nil(t, result) diff --git a/internal/core/unfurl/opengraph_test.go b/internal/core/unfurl/opengraph_test.go index 8a03e9e..4cc1cbd 100644 --- a/internal/core/unfurl/opengraph_test.go +++ b/internal/core/unfurl/opengraph_test.go @@ -164,7 +164,7 @@ func TestFetchOpenGraph_Success(t *testing.T) { defer server.Close() ctx := context.Background() - result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + result, err := fetchOpenGraph(ctx, server.URL, hatchOpenClient(t, 10*time.Second), "CovesBot/1.0") require.NoError(t, err) require.NotNil(t, result) @@ -183,7 +183,7 @@ func TestFetchOpenGraph_HTTPError(t *testing.T) { defer server.Close() ctx := context.Background() - result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + result, err := fetchOpenGraph(ctx, server.URL, hatchOpenClient(t, 10*time.Second), "CovesBot/1.0") require.Error(t, err) assert.Nil(t, result) assert.Contains(t, err.Error(), "404") @@ -209,7 +209,7 @@ func TestFetchOpenGraph_Timeout(t *testing.T) { defer close(release) ctx := context.Background() - result, err := fetchOpenGraph(ctx, server.URL, 100*time.Millisecond, "CovesBot/1.0") + result, err := fetchOpenGraph(ctx, server.URL, hatchOpenClient(t, 100*time.Millisecond), "CovesBot/1.0") require.Error(t, err) assert.Nil(t, result) } @@ -225,7 +225,7 @@ func TestFetchOpenGraph_NoMetadata(t *testing.T) { defer server.Close() ctx := context.Background() - result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + result, err := fetchOpenGraph(ctx, server.URL, hatchOpenClient(t, 10*time.Second), "CovesBot/1.0") require.NoError(t, err) require.NotNil(t, result) diff --git a/internal/core/unfurl/post_unfurl_integration_test.go b/internal/core/unfurl/post_unfurl_integration_test.go index c3a8272..cbce4a8 100644 --- a/internal/core/unfurl/post_unfurl_integration_test.go +++ b/internal/core/unfurl/post_unfurl_integration_test.go @@ -50,6 +50,7 @@ func TestPostUnfurl_UnsupportedURL(t *testing.T) { nil, nil, nil, + communities.PrivateHostOptions(true)..., ) // Create post service WITHOUT unfurl service @@ -150,6 +151,7 @@ func TestPostUnfurl_MissingEmbedType(t *testing.T) { nil, nil, nil, + communities.PrivateHostOptions(true)..., ) postService := posts.NewPostService( @@ -303,8 +305,21 @@ func TestPostUnfurl_E2E_WithJetstream(t *testing.T) { identityResolver := identity.NewResolver(db, identityConfig) userService := users.NewUserService(userRepo, identityResolver, testkit.Endpoints().PDS.BaseURL, nil, "") + // THE HATCH IS OPEN BECAUSE THE FIXTURE IS ON LOOPBACK, and for no other + // reason. unfurlTarget below is an httptest server, so its address is + // exactly the class the SSRF guard refuses — the same reason + // blobs/fetch_guard_test.go builds its service with WithPrivateHostsAllowed. + // + // This is the honest repair rather than the convenient one. The alternative — + // handing this test a client that skips the guarded transport — would keep it + // green while removing the thing it exercises: the request below would no + // longer travel the path production travels, and a regression in that path + // would leave this test passing. The hatch changes ONE decision, the address + // classification, and leaves the vetted dial, the byte cap and the timeout + // exactly where production has them. unfurlService := unfurl.NewService(unfurlRepo, unfurl.WithTimeout(30*time.Second), + unfurl.WithPrivateHostsAllowed(), ) // The URL this test unfurls is served by an httptest server rather than by a diff --git a/internal/core/unfurl/providers.go b/internal/core/unfurl/providers.go index d78fa70..1e1e501 100644 --- a/internal/core/unfurl/providers.go +++ b/internal/core/unfurl/providers.go @@ -1,6 +1,7 @@ package unfurl import ( + "bytes" "context" "encoding/json" "fmt" @@ -8,11 +9,54 @@ import ( "net/http" "net/url" "strings" - "time" "golang.org/x/net/html" ) +// maxUnfurlBodyBytes is how much of a remote response this site is willing to +// hold in memory: 10 MB, the limit both HTML reads below have always enforced +// through an io.LimitReader. +// +// IT IS ALSO WHAT THE TRANSPORT IS TOLD, PLUS ONE, in NewService. The shared +// SSRF-safe client defaults to oauth.DefaultMaxResponseBytes (32 MiB), which is +// larger, so adopting it without passing this value would triple what a pasted +// link can make this process allocate — silently, since every existing fixture +// still passes under the looser bound. Both layers read the same constant so the +// two cannot drift, and the +1 is the room readCappedBody needs to PROBE past +// this limit rather than truncate at it. See readCappedBody for why that byte +// decides between a refusal and a half-page served into a post. +const maxUnfurlBodyBytes = 10 * 1024 * 1024 + +// readCappedBody reads a remote page and REFUSES one that runs past this site's +// limit, instead of handing back the part that fitted. +// +// # WHY THE LIMIT READER PROBES ONE BYTE PAST THE CAP +// +// io.ReadAll over an io.LimitReader cannot tell "the body ended" from "the limit +// was reached" — both arrive as EOF and a nil error. Reading maxUnfurlBodyBytes +// exactly is therefore a silent truncation: a page one byte too long comes back +// clipped, and because parseOpenGraph and html.Parse are both error-tolerant by +// design, the clipped bytes still yield an og:title and og:description. The +// caller gets an UnfurlResult that reads as a complete page, caches it for 24 +// hours and serves it into a post. The extra byte's EXISTENCE is the signal, so +// it has to be asked for. +// +// It follows that the transport must be told maxUnfurlBodyBytes+1 (NewService +// does): a transport cap set to exactly the limit clips the probing byte and +// restores the truncation this function exists to prevent. imageproxy/fetcher.go +// and posts' newGuardedRematerializeBlobClient are the same pairing, and the +// only reason they were not the same defect. +func readCappedBody(body io.Reader) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(body, maxUnfurlBodyBytes+1)) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + if len(data) > maxUnfurlBodyBytes { + return nil, fmt.Errorf("%w: read more than %d bytes", ErrPageTooLarge, maxUnfurlBodyBytes) + } + return data, nil +} + // Provider configuration var oEmbedEndpoints = map[string]string{ "streamable.com": "https://api.streamable.com/oembed", @@ -81,7 +125,7 @@ func isOEmbedProvider(urlStr string) bool { } // fetchOEmbed fetches oEmbed data from the provider -func fetchOEmbed(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*oEmbedResponse, error) { +func fetchOEmbed(ctx context.Context, urlStr string, client *http.Client, userAgent string) (*oEmbedResponse, error) { domain := extractDomain(urlStr) endpoint, exists := oEmbedEndpoints[domain] if !exists { @@ -99,8 +143,6 @@ func fetchOEmbed(ctx context.Context, urlStr string, timeout time.Duration, user req.Header.Set("User-Agent", userAgent) - // Create HTTP client with timeout - client := &http.Client{Timeout: timeout} resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch oEmbed data: %w", err) @@ -172,7 +214,7 @@ type openGraphData struct { } // fetchOpenGraph fetches OpenGraph metadata from a URL -func fetchOpenGraph(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { +func fetchOpenGraph(ctx context.Context, urlStr string, client *http.Client, userAgent string) (*UnfurlResult, error) { // Create HTTP request req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { @@ -181,8 +223,6 @@ func fetchOpenGraph(ctx context.Context, urlStr string, timeout time.Duration, u req.Header.Set("User-Agent", userAgent) - // Create HTTP client with timeout - client := &http.Client{Timeout: timeout} resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch URL: %w", err) @@ -193,11 +233,9 @@ func fetchOpenGraph(ctx context.Context, urlStr string, timeout time.Duration, u return nil, fmt.Errorf("HTTP request returned status %d", resp.StatusCode) } - // Read response body (limit to 10MB to prevent abuse) - limitedReader := io.LimitReader(resp.Body, 10*1024*1024) - body, err := io.ReadAll(limitedReader) + body, err := readCappedBody(resp.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) + return nil, err } // Parse OpenGraph metadata @@ -312,7 +350,7 @@ func getAttr(n *html.Node, key string) string { // fetchKagiKite handles special unfurling for Kagi Kite news pages // Kagi Kite pages use client-side rendering, so og:image tags aren't available at SSR time // Instead, we parse the HTML to extract the story image from the page content -func fetchKagiKite(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { +func fetchKagiKite(ctx context.Context, urlStr string, client *http.Client, userAgent string) (*UnfurlResult, error) { // Create HTTP request req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { @@ -321,8 +359,6 @@ func fetchKagiKite(ctx context.Context, urlStr string, timeout time.Duration, us req.Header.Set("User-Agent", userAgent) - // Create HTTP client with timeout - client := &http.Client{Timeout: timeout} resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch URL: %w", err) @@ -333,11 +369,19 @@ func fetchKagiKite(ctx context.Context, urlStr string, timeout time.Duration, us return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) } - // Limit response size to 10MB - limitedReader := io.LimitReader(resp.Body, 10*1024*1024) + // Read before parsing, rather than handing html.Parse the capped reader + // directly, because the overrun has to be decided on the WHOLE body: a + // streaming parse would consume the page and only then meet the extra byte, + // by which point a document already exists to return. Reading first costs + // nothing — io.ReadAll's buffer is the same 10 MB the parse was going to + // allocate anyway. + body, err := readCappedBody(resp.Body) + if err != nil { + return nil, err + } // Parse HTML - doc, err := html.Parse(limitedReader) + doc, err := html.Parse(bytes.NewReader(body)) if err != nil { return nil, fmt.Errorf("failed to parse HTML: %w", err) } diff --git a/internal/core/unfurl/service.go b/internal/core/unfurl/service.go index 748d92c..2edd9ae 100644 --- a/internal/core/unfurl/service.go +++ b/internal/core/unfurl/service.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "log" + "net/http" "time" + + covesoauth "Coves/internal/atproto/oauth" ) // Service handles URL unfurling with caching @@ -16,9 +19,16 @@ type Service interface { type service struct { repo Repository circuitBreaker *circuitBreaker + httpClient *http.Client userAgent string timeout time.Duration cacheTTL time.Duration + + // allowPrivateHosts disables the SSRF guard on every unfurl fetch. NEVER set + // in production: the URL unfurled here is a link a signed-up account pasted + // into a post, and this is the only fetch site in the tree that hands the + // RESPONSE BODY back to the caller. + allowPrivateHosts bool } // NewService creates a new unfurl service @@ -35,12 +45,86 @@ func NewService(repo Repository, opts ...ServiceOption) Service { opt(s) } + // ONE CLIENT FOR ALL THREE FETCH PATHS. providers.go used to build an + // unguarded &http.Client{} inside each of fetchOEmbed, fetchOpenGraph and + // fetchKagiKite; they now take this one, so a path cannot be left behind by a + // conversion that only reached the two that looked alike. + // + // The SSRF-safe transport of internal/atproto/oauth resolves the host, + // refuses private, loopback and link-local addresses, and then dials only the + // address it vetted — closing the check-then-dial window a naive guard leaves + // open. It matters more here than at any other fetch site in the tree: every + // other one discards the response body or treats it as opaque bytes, while + // this one PARSES it and RETURNS ITS CONTENT — og:title, og:description and + // og:image land in an UnfurlResult, in the unfurl cache, and then in a post + // read by strangers. The URL is a link a signed-up account pasted, so the + // primitive being closed is not "can the AppView reach this address" but + // "tell me what it said". + // + // THE BYTE CAP IS THIS SITE'S OWN, not the transport's 32 MiB default: + // adopting the default would triple what a remote host can make this process + // allocate, and nothing in the suite would notice because a looser bound + // fails no existing fixture. See maxUnfurlBodyBytes. + // + // ONE BYTE ABOVE THAT LIMIT, deliberately, the way imageproxy/fetcher.go and + // posts' newGuardedRematerializeBlobClient are. Both HTML read paths probe + // maxUnfurlBodyBytes+1 through an io.LimitReader so that an oversized page is + // DETECTED rather than truncated; a transport cap set to exactly the limit + // clips the probing byte, the LimitReader returns a full-but-short read with + // no error, and an error-tolerant parser turns half a page into a + // complete-looking UnfurlResult. See maxUnfurlBodyBytes and ErrPageTooLarge. + // + // THE TIMEOUT IS THE CONFIGURED ONE, restored over the shared client's own + // 15s ceiling the way blobs.NewBlobService and imageproxy.NewPDSFetcher + // restore theirs. This service defaults to 10s and cmd/server passes + // WithTimeout(unfurlTimeout), its own 10s constant, so both paths land on the + // same number; loosening every unfurl by five seconds — on the path a post + // write blocks behind — would be a second change wearing an SSRF fix's + // clothes. + clientOpts := append( + covesoauth.PrivateAddressOptions(s.allowPrivateHosts), + covesoauth.WithMaxResponseBytes(maxUnfurlBodyBytes+1), + ) + s.httpClient = covesoauth.NewSSRFSafeHTTPClient(clientOpts...) + s.httpClient.Timeout = s.timeout + return s } // ServiceOption configures the service type ServiceOption func(*service) +// WithPrivateHostsAllowed disables the SSRF address guard on every unfurl fetch. +// +// THE NAME IS THE CONTRACT: production must not call this. cmd/server derives +// the value from config once (the IS_DEV_ENV gate); tests that serve their +// fixtures from httptest pass it because loopback is exactly what the guard +// refuses. +func WithPrivateHostsAllowed() ServiceOption { // coves:allow-ssrf-hatch: this IS the hatch itself; the name is the contract + return func(s *service) { s.allowPrivateHosts = true } +} + +// PrivateHostOptions returns the options a caller holding an allow-private +// boolean should pass to NewService: the hatch when it is set, and NOTHING when +// it is not. +// +// It mirrors oauth.PrivateAddressOptions and imageproxy.PrivateHostOptions, and +// it is a function rather than an `if` in cmd/server/wiring.go for the reason +// documented there: `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` takes the +// PERMISSIVE branch at every call site holding such a boolean. A unit test +// against this function is the only place in the repository where the branch +// production actually runs is ever evaluated. Do not inline it back. +// +// FALSE RETURNS ZERO OPTIONS, AND THAT IS THE CONTRACT — not "options that are +// safe", but none, so that what production gets is exactly the constructor's +// own defaults. +func PrivateHostOptions(allowPrivate bool) []ServiceOption { + if !allowPrivate { + return nil + } + return []ServiceOption{WithPrivateHostsAllowed()} // coves:allow-ssrf-hatch: the gate helper allow-branch; its false branch returns nothing +} + // WithTimeout sets the HTTP timeout for oEmbed requests func WithTimeout(timeout time.Duration) ServiceOption { return func(s *service) { @@ -96,7 +180,7 @@ func (s *service) UnfurlURL(ctx context.Context, urlStr string) (*UnfurlResult, } log.Printf("[UNFURL] Cache miss for %s, fetching via Kagi parser...", urlStr) - result, err = fetchKagiKite(ctx, urlStr, s.timeout, s.userAgent) + result, err = fetchKagiKite(ctx, urlStr, s.httpClient, s.userAgent) if err != nil { s.circuitBreaker.recordFailure(provider, err) return nil, err @@ -125,7 +209,7 @@ func (s *service) UnfurlURL(ctx context.Context, urlStr string) (*UnfurlResult, log.Printf("[UNFURL] Cache miss for %s, fetching from oEmbed...", urlStr) // Fetch from oEmbed provider - oembed, err := fetchOEmbed(ctx, urlStr, s.timeout, s.userAgent) + oembed, err := fetchOEmbed(ctx, urlStr, s.httpClient, s.userAgent) if err != nil { s.circuitBreaker.recordFailure(provider, err) return nil, fmt.Errorf("failed to fetch oEmbed data: %w", err) @@ -148,7 +232,7 @@ func (s *service) UnfurlURL(ctx context.Context, urlStr string) (*UnfurlResult, log.Printf("[UNFURL] Cache miss for %s, fetching via OpenGraph...", urlStr) // Fetch via OpenGraph - result, err = fetchOpenGraph(ctx, urlStr, s.timeout, s.userAgent) + result, err = fetchOpenGraph(ctx, urlStr, s.httpClient, s.userAgent) if err != nil { s.circuitBreaker.recordFailure(provider, err) return nil, fmt.Errorf("failed to fetch OpenGraph data: %w", err) diff --git a/internal/core/unfurl/ssrf_guard_test.go b/internal/core/unfurl/ssrf_guard_test.go new file mode 100644 index 0000000..84a4178 --- /dev/null +++ b/internal/core/unfurl/ssrf_guard_test.go @@ -0,0 +1,560 @@ +package unfurl + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The unfurl fetch, against a destination a signed-up account chose. +// +// # WHY THIS IS THE MOST CAPABLE OF THE NINE SITES +// +// Every other site on the remediation list either discards the response body or +// treats it as opaque bytes. This one PARSES the body and RETURNS ITS CONTENT to +// the caller: og:title, og:description and og:image come back in an UnfurlResult, +// get written to the unfurl cache, and are then served to whoever reads the post. +// So the primitive is not "can the AppView reach this address" — that is the +// image proxy's, and it is a port scanner. This one is "tell me what it said". +// +// The URL is a link pasted into a post. The cost of the credential is a signup. +// The AppView shares a network with its Postgres, its PDS, its Jetstream and, in +// production, a metadata endpoint at 169.254.169.254 that hands credentials to +// anything that can reach it — and an internal HTTP endpoint that answers with a +// JSON banner, an error page naming a version, or an admin console title is an +// endpoint whose content lands in a public post. +// +// # WHY THREE PATHS AND NOT ONE +// +// providers.go built THREE separate `&http.Client{Timeout: timeout}` values, one +// per fetch function. Converting two of three is the shape this failure takes: +// each is a few lines apart, each looks like the one above it, and the survivor +// is reachable by choosing a different link. So every case below runs against all +// three individually rather than against whichever one the service happens to +// route to. +// +// # WHY REACHABILITY IS ASSERTED AND NOT ONLY THE ERROR +// +// Mutation testing produced an implementation that classified +// every address correctly, emitted a byte-identical error message, and refused +// the request AFTER delivering it. Every error-message assertion in the suite +// passed against it. For a destination a stranger named, the packet leaving IS +// the SSRF — so each case stands up a real listener and asserts its handler was +// never invoked. +// +// # WHY THESE TESTS ARE NOT PARALLEL +// +// The oEmbed path routes through the package-level oEmbedEndpoints map: the +// endpoint it dials is looked up from the link's domain, not taken from the link, +// so the only way to point that path at a listener this test owns is to register +// a domain in the map. Mutating it races TestIsOEmbedProvider and TestIsSupported, +// which read it under t.Parallel(). A test that does not call t.Parallel() runs in +// the sequential phase, and the testing package resumes parallel tests only once +// that phase is over — so serial is what makes the registration safe, and it is +// cheap here because nothing below waits on anything. + +// secretInternalHTML is what an internal endpoint answers with, written so that a +// leak is visible in the assertion rather than inferred from a nil check. +// +// It carries a kagiproxy.com as well as the og: tags because fetchKagiKite +// fails with "no image found" without one, and a path that errors for an +// unrelated reason proves nothing about the guard. +const secretInternalHTML = ` + + + Internal Admin Console + + + + + + internal network topology + +` + +// secretInternalOEmbed is the same leak in the shape the oEmbed path parses. +const secretInternalOEmbed = `{ + "version": "1.0", + "type": "video", + "title": "Internal Admin Console", + "description": "cluster credentials rotate at 04:00 UTC", + "thumbnail_url": "https://internal.invalid/img/internal-topology.png", + "provider_name": "Internal" +}` + +// leakedTitle is the one string that must never come back from a refused unfurl. +const leakedTitle = "Internal Admin Console" + +// countingTarget is a listener that answers like an internal service and records +// whether anything ever reached it. It listens on loopback, which is exactly the +// address class the guard exists to refuse, so the counter doubles as the +// assertion. +type countingTarget struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingTarget(t *testing.T, contentType, body string) *countingTarget { + t.Helper() + + target := &countingTarget{} + target.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + target.requests.Add(1) + w.Header().Set("Content-Type", contentType) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(target.server.Close) + return target +} + +// unfurledContent is what a caller of any of the three paths ends up holding, +// normalised across their three different return types. +// +// returned is separate from the three strings on purpose: "an error came back +// alongside a populated result" and "nothing came back" are different outcomes, +// and only the second one is a closed site. A caller that ignores the error — or +// logs it and carries on, which is what a best-effort enrichment path does — gets +// the leak either way. +type unfurledContent struct { + title string + description string + image string + returned bool +} + +// providerPath describes one of the three guarded unfurl constructions. +type providerPath struct { + name string + why string + contentType string + body string + + // targetURL maps a listener's base URL onto the URL this path must be asked + // to unfurl, registering whatever package state the path needs to route + // there. It takes *testing.T so a registration can be undone by t.Cleanup. + targetURL func(t *testing.T, base string) string + + fetch func(ctx context.Context, target string, client *http.Client) (unfurledContent, error) +} + +// providerPaths enumerates the three fetch sites, so the refusal test and the +// hatch test cannot drift apart in which paths they cover. +func providerPaths() []providerPath { + return []providerPath{ + { + name: "OpenGraph", + contentType: "text/html; charset=utf-8", + body: secretInternalHTML, + why: "the default path: every link that is not a known oEmbed provider lands here, so this " + + "is the one an attacker reaches by pasting any URL at all", + targetURL: func(_ *testing.T, base string) string { return base + "/article" }, + fetch: func(ctx context.Context, target string, client *http.Client) (unfurledContent, error) { + result, err := fetchOpenGraph(ctx, target, client, "CovesBot/1.0") + if result == nil { + return unfurledContent{}, err + } + return unfurledContent{ + title: result.Title, + description: result.Description, + image: result.ThumbnailURL, + returned: true, + }, err + }, + }, + { + name: "Kagi Kite", + contentType: "text/html; charset=utf-8", + body: secretInternalHTML, + why: "a third client construction, converted or not independently of the other two. It is " + + "unreachable through UnfurlURL today (see the dead-branch note below), which is exactly " + + "why it is the one a conversion forgets", + targetURL: func(_ *testing.T, base string) string { return base + "/abc/science/9" }, + fetch: func(ctx context.Context, target string, client *http.Client) (unfurledContent, error) { + result, err := fetchKagiKite(ctx, target, client, "TestBot/1.0") + if result == nil { + return unfurledContent{}, err + } + return unfurledContent{ + title: result.Title, + description: result.Description, + image: result.ThumbnailURL, + returned: true, + }, err + }, + }, + { + name: "oEmbed", + contentType: "application/json", + body: secretInternalOEmbed, + why: "the endpoint here is looked up from a package map rather than taken from the link, so " + + "it is the path that looks safest and is the easiest to leave unconverted. The map is " + + "not a control: it is a routing table, and a single entry pointed anywhere private — by " + + "an edit, or by a provider domain whose DNS answer changes — dials it with no guard at all", + targetURL: func(t *testing.T, base string) string { + t.Helper() + // A reserved TLD, so a lookup can never escape even if this + // registration were ever left behind by a failed cleanup. + const domain = "oembed-ssrf-probe.invalid" + oEmbedEndpoints[domain] = base + t.Cleanup(func() { delete(oEmbedEndpoints, domain) }) + return "https://" + domain + "/watch" + }, + fetch: func(ctx context.Context, target string, client *http.Client) (unfurledContent, error) { + oembed, err := fetchOEmbed(ctx, target, client, "CovesBot/1.0") + if oembed == nil { + return unfurledContent{}, err + } + return unfurledContent{ + title: oembed.Title, + description: oembed.Description, + image: oembed.ThumbnailURL, + returned: true, + }, err + }, + }, + } +} + +// serviceHTTPClient builds a service the way cmd/server does and hands back the +// client it constructed. +// +// GOING THROUGH NewService IS THE POINT. A test that built its own SSRF-safe +// client would assert that internal/atproto/oauth works, which its own tests +// covers exhaustively. What is unproven here is that THIS package's constructor +// wires one — with this site's own byte cap and this site's own timeout — and +// hands it to all three fetch functions. Reaching for the field is how that stays +// the thing under test. +func serviceHTTPClient(t *testing.T, opts ...ServiceOption) *http.Client { + t.Helper() + + svc, ok := NewService(nil, opts...).(*service) + require.True(t, ok, "NewService must return the concrete *service this package's tests drive") + require.NotNil(t, svc.httpClient, + "NewService built no HTTP client, so every fetch below would nil-panic rather than assert anything") + return svc.httpClient +} + +// guardedClient is what production gets: the dev gate answering false, and +// nothing else applied. +func guardedClient(t *testing.T, timeout time.Duration) *http.Client { + t.Helper() + return serviceHTTPClient(t, append([]ServiceOption{WithTimeout(timeout)}, PrivateHostOptions(false)...)...) +} + +// hatchOpenClient is what a developer and every httptest fixture in this package +// gets: the dev gate answering true. +func hatchOpenClient(t *testing.T, timeout time.Duration) *http.Client { + t.Helper() + return serviceHTTPClient(t, append([]ServiceOption{WithTimeout(timeout)}, PrivateHostOptions(true)...)...) +} + +// TestUnfurlProviders_RefuseAPrivateAddressWithoutReachingIt is the binding +// contract for unfurl egress. +// +// Each of the three fetch sites is pointed at a real listener on loopback and +// must refuse it before a packet leaves the process — and must hand back nothing. +// +// Not parallel: see the oEmbed registration note in the file header. +func TestUnfurlProviders_RefuseAPrivateAddressWithoutReachingIt(t *testing.T) { + for _, path := range providerPaths() { + t.Run(path.name, func(t *testing.T) { + target := newCountingTarget(t, path.contentType, path.body) + + content, err := path.fetch( + context.Background(), + path.targetURL(t, target.server.URL), + guardedClient(t, 5*time.Second), + ) + + require.Errorf(t, err, + "the %s path fetched a loopback address successfully. The URL is a link an account "+ + "pasted into a post, and this path returns what the address ANSWERED — %s", + path.name, path.why) + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the %s path failed, but not because the guard refused it: the error does not match "+ + "covesoauth.ErrBlockedAddress. An unguarded client pointed at an address that "+ + "happens to be unreachable fails too, and looks identical from here — so without "+ + "this assertion the case above passes on a site that was never converted; got: %v", + path.name, err) + + assert.Zerof(t, target.requests.Load(), + "the %s path reached the listener %d times. The refusal happened, but it happened "+ + "AFTER the request was delivered — which prevents none of the SSRF and is precisely "+ + "the implementation that passed every error-message assertion during "+ + "mutation testing", path.name, target.requests.Load()) + + assert.Falsef(t, content.returned, + "the %s path returned a result alongside its error. This site is the one that hands "+ + "RESPONSE CONTENT back, and a caller that logs the error and carries on — which is "+ + "what a best-effort enrichment path does — publishes it anyway. A refused unfurl "+ + "must be nothing at all, not an error with a payload attached", path.name) + + assert.NotContainsf(t, content.title, leakedTitle, + "the %s path returned the internal page's title (%q) from an address it was supposed "+ + "to refuse. This is the harm: the og: tags of an internal endpoint, parsed and "+ + "handed to the caller that named it", path.name, content.title) + assert.Emptyf(t, content.description, + "the %s path returned a description (%q) from a refused address", path.name, content.description) + assert.Emptyf(t, content.image, + "the %s path returned an image URL (%q) from a refused address", path.name, content.image) + }) + } +} + +// TestUnfurlProviders_ReachTheListenerWhenTheHatchIsOpen is the other direction, +// and it is not a nicety. +// +// Every unit test in this package serves its fixture from httptest, which listens +// on loopback — kagi_test.go and opengraph_test.go both do, through this same +// helper — and a local dev stack runs everything on the developer's own machine. +// Without a working hatch the guard takes all of that with it. +// +// It is also the behavioural half of the PrivateHostOptions contract: the client +// here is built from PrivateHostOptions(true), so a helper that returns the wrong +// option — or none — fails here rather than silently leaving developers with a +// client that cannot reach anything local. +// +// Not parallel: see the oEmbed registration note in the file header. +func TestUnfurlProviders_ReachTheListenerWhenTheHatchIsOpen(t *testing.T) { + for _, path := range providerPaths() { + t.Run(path.name, func(t *testing.T) { + target := newCountingTarget(t, path.contentType, path.body) + + content, err := path.fetch( + context.Background(), + path.targetURL(t, target.server.URL), + hatchOpenClient(t, 5*time.Second), + ) + + require.NoErrorf(t, err, + "the %s path was refused with the hatch open. Every fixture in this package and every "+ + "dev stack depends on this: PrivateHostOptions(true) must produce a client that "+ + "reaches loopback; got: %v", path.name, err) + require.Truef(t, content.returned, "the %s path returned no result with the hatch open", path.name) + + assert.Equalf(t, leakedTitle, content.title, + "the %s path came back with title %q. The body must survive the guarded transport "+ + "unchanged — a conversion that reaches the listener but mangles what it parses is "+ + "a different regression wearing the same green", path.name, content.title) + assert.NotEmptyf(t, content.description, "the %s path lost the description", path.name) + assert.NotEmptyf(t, content.image, "the %s path lost the image URL", path.name) + assert.Equalf(t, int64(1), target.requests.Load(), + "the %s path reached the listener %d times rather than once", + path.name, target.requests.Load()) + }) + } +} + +// recordingRepo is a cold unfurl cache that records what was written to it. +type recordingRepo struct { + mu sync.Mutex + stored []*UnfurlResult +} + +func (r *recordingRepo) Get(context.Context, string) (*UnfurlResult, error) { return nil, nil } + +func (r *recordingRepo) Set(_ context.Context, _ string, result *UnfurlResult, _ time.Duration) error { + r.mu.Lock() + defer r.mu.Unlock() + r.stored = append(r.stored, result) + return nil +} + +func (r *recordingRepo) writes() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.stored) +} + +// TestUnfurlService_UnfurlURL_RefusesAPrivateAddressAndCachesNothing proves the +// assembled path is closed, not just the fetch function in isolation. +// +// # WHY THE CACHE ASSERTION IS HERE +// +// UnfurlURL writes every successful result to the unfurl cache with a 24h TTL, +// and the read path serves that cache before it checks anything else. So a leak +// that got through once is served from Postgres for a day afterwards, to every +// reader of the post, with no further requests to the internal address and +// nothing left in the logs to connect the two. "Nothing was cached" is therefore +// a distinct claim from "an error was returned", and it is the one that bounds +// the blast radius. +// +// The repository starts cold and says so: if Get ever returned a hit, UnfurlURL +// would answer from it without calling the fetch at all, and this case would pass +// having exercised nothing. +func TestUnfurlService_UnfurlURL_RefusesAPrivateAddressAndCachesNothing(t *testing.T) { + t.Parallel() + + target := newCountingTarget(t, "text/html; charset=utf-8", secretInternalHTML) + repo := &recordingRepo{} + + svc := NewService(repo, append([]ServiceOption{WithTimeout(5 * time.Second)}, PrivateHostOptions(false)...)...) + + result, err := svc.UnfurlURL(context.Background(), target.server.URL+"/article") + + require.Error(t, err, + "the assembled unfurl service fetched a loopback address. This is the production shape: a link "+ + "pasted into a post by anyone with an account") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the service must pass the guard's refusal through with its identity intact, so a log line and "+ + "an alert can tell a security refusal from a site that was merely down; got: %v", err) + assert.Nil(t, result, + "the service returned a result for a refused address. UnfurlURL's result is embedded in the post "+ + "and served to every reader of it") + assert.Zerof(t, target.requests.Load(), + "the service reached the listener %d times", target.requests.Load()) + assert.Zerof(t, repo.writes(), + "the service wrote %d entr(y/ies) to the unfurl cache for a refused address. A cached leak is "+ + "served for the full 24h TTL without any further request to the internal address, so a "+ + "single success outlives the request that caused it", repo.writes()) +} + +// TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed is the +// single most important assertion for this call site. +// +// `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +// T0+T1+T2 — runs the PERMISSIVE branch here and at every other site holding such +// a boolean. A green merge gate therefore proves nothing whatsoever about whether +// unfurl is guarded in production. This function is the one place in the +// repository where the production branch is ever evaluated, which is why the gate +// must be a pure function and not an `if cfg.IsDevEnv` in cmd/server/wiring.go. +// +// The claim is not "the options returned are safe". It is that there are NONE: +// length zero, nothing applied, the constructor's own defaults left untouched. An +// edit that appends a diagnostic option, or returns a one-element slice holding a +// no-op "explicitly deny" closure, keeps every behavioural test green while moving +// the untested branch from "provably applies nothing" to "applies something +// believed harmless". If this assertion is ever in the way, the answer is not to +// relax it. +func TestPrivateHostOptions_ReturnsZeroOptionsWhenPrivateHostsAreDisallowed(t *testing.T) { + t.Parallel() + + opts := PrivateHostOptions(false) + + assert.Lenf(t, opts, 0, + "PrivateHostOptions(false) returned %d option(s). The production branch — the one IS_DEV_ENV=true "+ + "keeps `make ci` from ever evaluating — must contribute nothing at all, so that what "+ + "production gets is exactly the constructor's own defaults", len(opts)) +} + +// TestPrivateHostOptions_DisallowedServiceIsGuarded is the behavioural half of +// the assertion above: zero options has to also MEAN a guarded client. +// +// The length check alone would still pass if NewService's own default ever +// regressed to permissive — the helper would correctly be returning nothing, onto +// a base that no longer refuses anything. +func TestPrivateHostOptions_DisallowedServiceIsGuarded(t *testing.T) { + t.Parallel() + + target := newCountingTarget(t, "text/html; charset=utf-8", secretInternalHTML) + + result, err := fetchOpenGraph( + context.Background(), + target.server.URL+"/article", + guardedClient(t, 5*time.Second), + "CovesBot/1.0", + ) + + require.Error(t, err, + "a client built from PrivateHostOptions(false) reached a loopback address. This is the branch "+ + "production runs and CI never does") + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must be the guard's, matchable by identity — a request that failed for some other "+ + "reason is not the same control and would not hold in production; got: %v", err) + assert.Nil(t, result, "a refused fetch must return no metadata") + assert.Zerof(t, target.requests.Load(), + "the listener was reached %d times, so the packet left the process", target.requests.Load()) +} + +// TestUnfurlFetch_BodyCapStaysAtThisSitesOwnLimit is the clamp trap in the +// direction this site can fall. +// +// providers.go caps both HTML reads at 10 MB through an io.LimitReader. The +// shared transport carries a cap of its own, defaulting to 32 MiB, and a +// conversion that simply adopts NewSSRFSafeHTTPClient() inherits that default — +// which does not fail anything, does not warn, and quietly triples what a remote +// host can make this process allocate before anything objects. Nothing in the +// suite would notice: 32 MiB is larger, so every existing fixture still passes. +// +// The window between the two limits is where they can be told apart. 12 MiB is +// above this site's 10 MB and below the transport's 32 MiB default, so it is +// refused by a correctly converted client and accepted by one carrying the +// package default. +// +// # WHY A DECLARED LENGTH AND NOT A REAL BODY +// +// A streamed 12 MiB body would NOT discriminate, and the reason is worth knowing +// before setting the cap: the site reads through io.LimitReader(resp.Body, 10MB), +// which stops requesting bytes at exactly 10 MB, so a transport cap set to 10 MB +// is never asked for the byte that would trip it. The oversized body arrives +// silently truncated — the pre-existing behaviour — and no error is produced by +// either layer. The announced-length branch is the one that actually fires, and +// the one this asserts. +func TestUnfurlFetch_BodyCapStaysAtThisSitesOwnLimit(t *testing.T) { + t.Parallel() + + // Above providers.go's 10 MB, below covesoauth.DefaultMaxResponseBytes. + const declared = 12 << 20 + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Length", strconv.Itoa(declared)) + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + + // The hatch is open so the ADDRESS is out of the picture entirely: the only + // thing left that can refuse this response is a byte cap. + result, err := fetchOpenGraph( + context.Background(), + server.URL+"/article", + hatchOpenClient(t, 5*time.Second), + "CovesBot/1.0", + ) + + require.Errorf(t, err, + "a response declaring %d bytes was accepted by a site whose own limit is 10 MB. The cap that "+ + "should have refused it is the shared transport's, and it is only this permissive if the "+ + "conversion inherited covesoauth.DefaultMaxResponseBytes (%d) instead of passing this "+ + "site's own limit", declared, covesoauth.DefaultMaxResponseBytes) + assert.ErrorIsf(t, err, covesoauth.ErrResponseTooLarge, + "the response was refused, but not by the byte cap. Any other failure here means the cap is "+ + "still whatever the transport defaults to and this assertion is passing by accident; got: %v", err) + assert.Nil(t, result, "an over-sized response must yield no metadata") +} + +// TestUnfurlService_PreservesTheConfiguredTimeout guards the setting the shared +// client would otherwise swallow. +// +// NewSSRFSafeHTTPClient returns a client with its own 15s ceiling. This service's +// own default is 10s, and cmd/server passes the same value explicitly through +// unfurl.WithTimeout. Adopting the shared client without restoring the caller's +// value LOOSENS every unfurl by five seconds — a change nobody asked for, +// arriving as part of an SSRF fix, on the path a post write blocks behind. +// blobs.NewBlobService and imageproxy.NewPDSFetcher both restore their own for +// the same reason. +func TestUnfurlService_PreservesTheConfiguredTimeout(t *testing.T) { + t.Parallel() + + const configured = 27 * time.Second + + client := serviceHTTPClient(t, WithTimeout(configured)) + + assert.Equalf(t, configured, client.Timeout, + "the service's client runs on a %v timeout instead of the configured %v. The shared SSRF client "+ + "ships a 15s ceiling of its own, so a call site that adopts it without re-applying its own "+ + "value hands operators a setting that no longer does anything", + client.Timeout, configured) +} diff --git a/internal/core/userblocks/service_writeforward_test.go b/internal/core/userblocks/service_writeforward_test.go index feb6a0f..d0a84ec 100644 --- a/internal/core/userblocks/service_writeforward_test.go +++ b/internal/core/userblocks/service_writeforward_test.go @@ -72,7 +72,7 @@ func newWriteForwardFixture(t *testing.T) *writeForwardFixture { // DID, and resolveIdentifier short-circuits before touching it. Handle // resolution is service_test.go's subject. service := userblocks.NewServiceWithPDSFactory(repo, nil, - testkit.PasswordAuthFactory(pds.NewFromAccessToken)) + testkit.PasswordAuthFactory(pds.NewFromAccessToken, pds.PrivateHostOptions(true)...)) return &writeForwardFixture{ service: service, diff --git a/internal/core/users/profile_backfill_cap_test.go b/internal/core/users/profile_backfill_cap_test.go new file mode 100644 index 0000000..9b94ab7 --- /dev/null +++ b/internal/core/users/profile_backfill_cap_test.go @@ -0,0 +1,107 @@ +package users + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + covesoauth "Coves/internal/atproto/oauth" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The backfill fetch's byte ceiling, which the conversion onto the shared client +// left behind. +// +// FetchProfileRecord has capped itself at maxProfileResponseBytes (1 MiB) since +// before this remediation, through an io.LimitReader. What the conversion did +// not carry across is the TRANSPORT's own cap, which defaults to +// oauth.DefaultMaxResponseBytes — 32 MiB, THIRTY-TWO TIMES this site's own +// limit. oauth.DefaultMaxResponseBytes documents the obligation by name: a +// caller with its own limit has to state it. +// +// The gap is not academic. The LimitReader bounds what this process ALLOCATES +// and nothing else; the transport's cap is the half that refuses an announced +// length before a byte of body is read, and refuses it on a fetch whose host is +// whatever the indexed user's PDS says it is, from a goroutine detached with +// context.WithoutCancel that nothing waits on. + +// TestNewProfileBackfillClient_AppliesThisSitesOwnByteCap pins the announced +// length at the boundary the two caps disagree about. +// +// A declared length between 1 MiB and 32 MiB is delivered under the shared +// default and refused under this site's own. Asserted through the header rather +// than by moving a megabyte, because the transport's check is +// `resp.ContentLength > maxResponseBytes` in RoundTrip and is reached without a +// body: the fetch fails either way, and what the test reads is WHICH failure. +func TestNewProfileBackfillClient_AppliesThisSitesOwnByteCap(t *testing.T) { + t.Parallel() + + // Comfortably above this site's 1 MiB and comfortably below the shared + // 32 MiB, so it separates the two and cannot be confused for either edge. + const announced = maxProfileResponseBytes * 4 + require.Less(t, int64(announced), int64(covesoauth.DefaultMaxResponseBytes), + "the fixture must declare a length the SHARED default would allow, or it proves nothing") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Length", fmt.Sprint(announced)) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"value":{}}`)) + })) + t.Cleanup(server.Close) + + // The hatch is open because the fixture is on loopback. The byte cap is a + // different control from the address guard. + _, err := FetchProfileRecord(context.Background(), NewProfileBackfillClient(true), + server.URL, "did:plc:profilebackfillcap22222") + + require.Error(t, err, "the declared length is not delivered, so this fetch fails either way") + assert.ErrorIsf(t, err, covesoauth.ErrResponseTooLarge, + "a PDS declaring %d bytes was not refused by the transport, so this client is running on "+ + "oauth.DefaultMaxResponseBytes (%d) rather than on maxProfileResponseBytes (%d) — its own "+ + "limit, thirty-two times smaller. The site's io.LimitReader bounds only what io.ReadAll "+ + "allocates; the announced-length refusal is the half that was dropped; got: %v", + announced, covesoauth.DefaultMaxResponseBytes, maxProfileResponseBytes, err) +} + +// TestNewProfileBackfillClient_AProfileAtTheCapStillArrives is what stops the +// cap from being tightened by accident, and it is why the number applied is +// maxProfileResponseBytes EXACTLY rather than the +1 the rematerialize and image +// proxy sites use. +// +// Those two read one byte PAST their own limit through an io.LimitReader so that +// an overrun is detected rather than silently truncated, and a transport cap set +// to exactly their limit would clip that probing byte. FetchProfileRecord reads +// exactly maxProfileResponseBytes and no more, so there is no probe to make room +// for — and a cap one byte above its own limit would be a number describing +// nothing. +func TestNewProfileBackfillClient_AProfileAtTheCapStillArrives(t *testing.T) { + t.Parallel() + + // A well-formed record padded out to exactly the cap, so the boundary is + // exercised by a response the caller must accept AND parse. + prefix := `{"uri":"at://x","value":{"$type":"` + ProfileCollection + `","displayName":"` + suffix := `"}}` + padding := maxProfileResponseBytes - len(prefix) - len(suffix) + require.Positive(t, padding, "the fixture must fit inside the cap it is testing") + body := prefix + strings.Repeat("a", padding) + suffix + require.Len(t, body, maxProfileResponseBytes, "the fixture must be EXACTLY the cap, not near it") + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + + _, err := FetchProfileRecord(context.Background(), NewProfileBackfillClient(true), + server.URL, "did:plc:profilebackfillcap22222") + + assert.NoErrorf(t, err, + "a response of exactly maxProfileResponseBytes (%d) was refused. The cap is this site's own "+ + "limit and a body at it is legitimate; a transport cap BELOW it would refuse profiles the "+ + "LimitReader was written to accept", maxProfileResponseBytes) +} diff --git a/internal/core/users/profile_backfill_fallback_guard_test.go b/internal/core/users/profile_backfill_fallback_guard_test.go new file mode 100644 index 0000000..cbcbd64 --- /dev/null +++ b/internal/core/users/profile_backfill_fallback_guard_test.go @@ -0,0 +1,191 @@ +package users + +import ( + "context" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The hole in the work that just shipped. +// +// # WHAT WAS ACTUALLY CLOSED +// +// NewProfileBackfillClient is the guard boundary used by +// cmd/server/wiring.go at it. What it guarded was the client wiring INJECTS. +// WithProfileBackfill still ends with +// +// if client == nil { +// client = &http.Client{Timeout: 10 * time.Second} +// } +// +// so the service has TWO ways to acquire a backfill client and only one of them +// goes past the gate. Anything that enables backfill without handing over a +// client — a second wiring path, an integration harness, a future caller +// copying the option's own documented "Pass nil to use a default client" — +// silently gets an unguarded one, at a call site whose whole job is to fetch an +// address a stranger chose. +// +// # WHY A NIL DEFAULT IS THE WORST PLACE FOR THIS +// +// The guarded path is the one a reader checks. `WithProfileBackfill(client)` +// with a real argument reads as guarded, because the thing being passed in was +// built by the gate — and the reader stops there, having answered the question. +// The nil branch is not a call site anyone reviews; it is what a call site +// DEGRADES TO. So the failure mode is not "someone wired this wrongly", it is +// "someone wired this in the way the doc comment suggests". +// +// The severity is unchanged from the injected path — see +// profile_backfill_guard_test.go for what `pdsURL` is and why nothing observes +// this goroutine — so these tests do not restate it. They pin one claim: THE +// FALLBACK IS THE GATE'S OWN CLIENT, not a second client that happens to exist. + +// TestWithProfileBackfill_NilYieldsAGuardedClient is the binding contract. +// +// It drives FetchProfileRecord with whatever the option left on the service, +// rather than inspecting the client, because "guarded" is a behaviour: a field +// assertion would pass against any client built by any constructor that happened +// to look right. +func TestWithProfileBackfill_NilYieldsAGuardedClient(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + service := &userService{} + WithProfileBackfill(nil)(service) + require.NotNil(t, service.profileBackfillClient, + "WithProfileBackfill(nil) must still enable backfill — a nil client here disables the "+ + "feature silently instead of defaulting it") + + _, err := FetchProfileRecord(context.Background(), + service.profileBackfillClient, pds.server.URL, backfillGuardDID) + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times through the NIL-argument path. The explicit guard "+ + "covers only the client wiring injects; this branch hands back a bare "+ + "http.Client and the PDS URL it dials comes from an indexed user's record", + pds.requests.Load()) + + require.Error(t, err, + "a service configured with WithProfileBackfill(nil) fetched a loopback address "+ + "successfully. Nothing waits on this goroutine in production, so this would leave no "+ + "trace at all") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the fetch failed, but not because the guard refused the address. Without the guard's own "+ + "identity this assertion would pass against the current build, where the fallback is "+ + "`&http.Client{Timeout: 10 * time.Second}` and the failure is merely a PDS that could "+ + "not be reached; got: %v", err) +} + +// TestWithProfileBackfill_NilFallbackIsTheGatesOwnClient ties the fallback to +// the constructor whose classification pass is already pinned. +// +// # WHY A BEHAVIOURAL TEST IS NOT ENOUGH ON ITS OWN +// +// The case above proves the fallback refuses a loopback LITERAL. A client that +// refuses every address refuses that one too, and so does one whose transport +// was hand-rolled to check `IsLoopback` and nothing else — neither of which +// would refuse a name that RESOLVES to 169.254.169.254, which is the address +// this whole remediation exists for. +// +// The seam that proves classification (covesoauth.WithHostResolver) cannot be +// pushed through WithProfileBackfill: the option takes a whole client and there +// is nothing to pass options to. So this test makes the bridge instead — the +// fallback must be built by NewProfileBackfillClient, whose classification is +// pinned by TestFetchProfileRecord_RefusesAWellFormedHostThatResolvesPrivate +// and its hatch-open control. The transport TYPE is the observable that shows +// which constructor built it; a bare http.Client carries a nil Transport, and +// stdlib's default carries *http.Transport. +func TestWithProfileBackfill_NilFallbackIsTheGatesOwnClient(t *testing.T) { + t.Parallel() + + service := &userService{} + WithProfileBackfill(nil)(service) + require.NotNil(t, service.profileBackfillClient, "WithProfileBackfill(nil) must leave a client") + + fallback := reflect.TypeOf(service.profileBackfillClient.Transport) + gated := reflect.TypeOf(NewProfileBackfillClient(false).Transport) + + require.NotNilf(t, service.profileBackfillClient.Transport, + "the fallback client has a nil Transport, which is stdlib's DefaultTransport — an "+ + "unguarded client wearing no marking at all") + assert.Equalf(t, gated, fallback, + "the nil fallback is a %v, not the %v NewProfileBackfillClient(false) builds. Only the "+ + "gate's transport carries the classification pass; a second client that merely refuses "+ + "loopback would satisfy the behavioural test above and still dial 169.254.169.254", + fallback, gated) +} + +// TestWithProfileBackfill_NilFallbackKeepsTheBackfillDeadline pins the value the +// conversion is most likely to lose. +// +// The shared SSRF client ships a 15s ceiling. backfillProfile detaches from its +// caller with context.WithoutCancel, so profileBackfillTimeout is the ONLY +// deadline this goroutine has — adopting the gate without re-applying it (which +// NewProfileBackfillClient already does) would silently extend every backfill +// started through the nil path, with no request lifetime underneath to bound it. +func TestWithProfileBackfill_NilFallbackKeepsTheBackfillDeadline(t *testing.T) { + t.Parallel() + + service := &userService{} + WithProfileBackfill(nil)(service) + require.NotNil(t, service.profileBackfillClient, "WithProfileBackfill(nil) must leave a client") + + assert.Equalf(t, profileBackfillTimeout, service.profileBackfillClient.Timeout, + "the nil-fallback client runs on a %v timeout instead of profileBackfillTimeout (%v). This "+ + "goroutine is detached from its caller's context, so this value is its only deadline", + service.profileBackfillClient.Timeout, profileBackfillTimeout) +} + +// TestWithProfileBackfill_KeepsTheInjectedClient is the falsifiability control, +// and it is not a formality. +// +// The cheapest way to make every assertion above pass is to ignore the argument +// and always build the gate's guarded client — which would delete the dev hatch +// wiring depends on, and every httptest-backed fixture with it, since loopback +// is exactly what the guard refuses. The claim is pointer identity: an injected +// client must arrive at the field UNTOUCHED, so the fallback is reachable only +// when there is genuinely nothing to fall back from. +func TestWithProfileBackfill_KeepsTheInjectedClient(t *testing.T) { + t.Parallel() + + injected := NewProfileBackfillClient(true) + + service := &userService{} + WithProfileBackfill(injected)(service) + + assert.Samef(t, injected, service.profileBackfillClient, + "WithProfileBackfill replaced the client it was handed. cmd/server passes the gate's "+ + "output — including the DEV build, where the hatch is open — so overriding the "+ + "argument would guard the fallback by making every caller's own choice unreachable") +} + +// TestWithProfileBackfill_TheInjectedHatchStillReachesThePDS is the other half +// of that control, stated as behaviour rather than as a pointer. +// +// Pointer identity would still hold if a later change copied the client and +// rebuilt its transport. This is the property that actually matters to a +// developer's local stack: an injected hatch-open client reaches loopback. +func TestWithProfileBackfill_TheInjectedHatchStillReachesThePDS(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + service := &userService{} + WithProfileBackfill(NewProfileBackfillClient(true))(service) + + input, err := FetchProfileRecord(context.Background(), + service.profileBackfillClient, pds.server.URL, backfillGuardDID) + + require.NoErrorf(t, err, + "an injected hatch-open client must still reach a loopback PDS — this is what every dev "+ + "stack and every httptest fixture in this tree depends on; got: %v", err) + require.NotNil(t, input, "the fixture serves a displayName, so a profile must come back") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} diff --git a/internal/core/users/profile_backfill_guard_test.go b/internal/core/users/profile_backfill_guard_test.go new file mode 100644 index 0000000..58e0cf3 --- /dev/null +++ b/internal/core/users/profile_backfill_guard_test.go @@ -0,0 +1,304 @@ +package users + +import ( + "bytes" + "context" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + covesoauth "Coves/internal/atproto/oauth" +) + +// The profile backfill's PDS fetch, from a goroutine nobody is waiting on. +// +// # WHY THIS SITE IS DIFFERENT FROM THE OTHER EIGHT +// +// Every other fetch in this remediation returns its error to a caller who does +// something with it — a handler maps it to a status, a service wraps it, a test +// asserts on it. This one runs in `go s.backfillProfile(context.WithoutCancel(ctx), …)`. +// Nothing waits on it, nothing retries it, and its only output is a log line. So +// the SSRF here is invisible in both directions: an attacker gets no response +// body back, and an operator gets no signal that anything was attempted. +// +// That asymmetry is why this file asserts TWO separate properties. The guard +// must refuse the address, and the refusal must be OBSERVABLE — because a +// security control in a detached goroutine that fails silently is +// indistinguishable, from outside, from one that was never wired. +// +// # WHAT THE FETCH IS POINTED AT +// +// `pdsURL` comes from the indexed user's record. Users are indexed from the +// firehose as well as from login, so the value arrives from another instance's +// account data — a stranger's choice, resolved and dialled from inside the +// AppView's own network. + +const ( + backfillGuardDID = "did:plc:backfillguard" + + // backfillGuardHost passes every shape check this path applies and is a name + // rather than an address, so classification is the only thing that can refuse + // it. `.example` is reserved by RFC 2606, so nothing resolves it for real if + // the seam is ever bypassed. + backfillGuardHost = "https://user-pds.example" +) + +// countingPDS records whether the backfill ever reached a listener. +type countingPDS struct { + server *httptest.Server + requests atomic.Int64 +} + +func newCountingPDS(t *testing.T) *countingPDS { + t.Helper() + + pds := &countingPDS{} + pds.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + pds.requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"value":{"displayName":"leaked from an internal endpoint"}}`)) + })) + t.Cleanup(pds.server.Close) + return pds +} + +// TestFetchProfileRecord_RefusesAPrivatePDSWithoutReachingIt is the binding +// contract for the fetch itself. +// +// It drives FetchProfileRecord rather than the goroutine because the goroutine +// swallows the error by design; the observability test below is what covers +// that half. +func TestFetchProfileRecord_RefusesAPrivatePDSWithoutReachingIt(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + _, err := FetchProfileRecord(context.Background(), + NewProfileBackfillClient(false), pds.server.URL, backfillGuardDID) + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times. The PDS URL comes from an indexed user's record, so a "+ + "stranger chose this address, and the request leaving the process is the SSRF whatever "+ + "comes back", pds.requests.Load()) + + require.Error(t, err, + "the backfill fetched a loopback address successfully. Nothing waits on this goroutine, so "+ + "in production this would have happened with no trace at all") + + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the fetch failed, but not because the guard refused the address. A client that simply could "+ + "not reach the host fails identically, so without the guard's own identity this "+ + "assertion would pass against a build where NewProfileBackfillClient was never "+ + "converted; got: %v", err) +} + +// TestNewProfileBackfillClient_ReachesThePDSWhenTheHatchIsOpen is the other +// direction, and the falsifiability control for the case above. +func TestNewProfileBackfillClient_ReachesThePDSWhenTheHatchIsOpen(t *testing.T) { + t.Parallel() + + pds := newCountingPDS(t) + + input, err := FetchProfileRecord(context.Background(), + NewProfileBackfillClient(true), pds.server.URL, backfillGuardDID) + + require.NoErrorf(t, err, + "the hatch is what a dev stack depends on: NewProfileBackfillClient(true) must reach a "+ + "loopback PDS; got: %v", err) + require.NotNil(t, input, "the fixture serves a displayName, so a profile must come back") + assert.Equalf(t, int64(1), pds.requests.Load(), + "the listener was reached %d times rather than once", pds.requests.Load()) +} + +// TestNewProfileBackfillClient_PreservesTheConfiguredTimeout guards the setting +// the shared client would otherwise swallow. +// +// The 10s here is not an ordinary timeout: backfillProfile detaches from the +// caller's context with context.WithoutCancel, so profileBackfillTimeout is the +// ONLY deadline this goroutine has. Inheriting the shared client's 15s would +// silently extend every backfill, and there is no request lifetime left to bound +// it. +func TestNewProfileBackfillClient_PreservesTheConfiguredTimeout(t *testing.T) { + t.Parallel() + + client := NewProfileBackfillClient(false) + + require.NotNil(t, client, "the gate must return a client") + assert.Equalf(t, profileBackfillTimeout, client.Timeout, + "the backfill client runs on a %v timeout instead of profileBackfillTimeout (%v). This "+ + "goroutine is detached from its caller's context, so this value is its only deadline", + client.Timeout, profileBackfillTimeout) +} + +// resolvingBackfillClient builds the client the way production does and then +// replaces only its NAME RESOLUTION, so the client under test is the real one. +func resolvingBackfillClient(t *testing.T, allowPrivateHosts bool, resolvesTo string) *http.Client { + t.Helper() + + // Checked, not assumed: isPrivateIP(nil) is false, so a typo'd fixture would + // classify as PUBLIC and certify the guard against nothing. + ip := net.ParseIP(resolvesTo) + require.NotNilf(t, ip, "the test's own answer %q must parse as an IP address", resolvesTo) + + return newProfileBackfillClient(allowPrivateHosts, + covesoauth.WithHostResolver(func(context.Context, string) ([]net.IP, error) { + return []net.IP{ip}, nil + })) +} + +// TestFetchProfileRecord_RefusesAWellFormedHostThatResolvesPrivate is the +// assertion a loopback-literal fixture cannot make: the guard's CLASSIFICATION +// pass, on a name that survives every earlier check. +func TestFetchProfileRecord_RefusesAWellFormedHostThatResolvesPrivate(t *testing.T) { + t.Parallel() + + client := resolvingBackfillClient(t, false, "127.0.0.1") // coves:allow-host-literal: the address the seam answers with; the guard refuses it before any dial + + _, err := FetchProfileRecord(context.Background(), client, backfillGuardHost, backfillGuardDID) + + require.Errorf(t, err, + "%s is a well-formed https host whose DNS answer was 127.0.0.1, and the backfill fetched it "+ + "anyway. A user's PDS URL is chosen by whoever runs their PDS, and they own the zone", + backfillGuardHost) + assert.ErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the refusal must carry the guard's identity, or a build where this client was never "+ + "converted looks identical; got: %v", err) +} + +// TestFetchProfileRecord_ControlTheSameHostIsDialledWithTheHatchOpen is the +// falsifiability control for the case above. +// +// Identical client, identical seam, identical host — only the hatch differs. +// With it open the address is no longer refused, so the request proceeds to a +// dial, which fails because nothing is listening on loopback:443. That +// difference is what pins the refusal above to classification rather than to +// this test being unable to make requests at all. +func TestFetchProfileRecord_ControlTheSameHostIsDialledWithTheHatchOpen(t *testing.T) { + t.Parallel() + + client := resolvingBackfillClient(t, true, "127.0.0.1") // coves:allow-host-literal: with the hatch open this is dialled and refused by the OS + + _, err := FetchProfileRecord(context.Background(), client, backfillGuardHost, backfillGuardDID) + + require.Error(t, err, + "nothing listens on loopback:443, so this fetch must fail — if it succeeded, the seam is "+ + "not answering with the address this test gave it") + assert.NotErrorIsf(t, err, covesoauth.ErrBlockedAddress, + "the hatch was open and the address was still refused by the guard. Either the gate is not "+ + "reaching the client, or the guarded case above proves nothing: a client that refuses "+ + "every address refuses that case too, for a reason unconnected to classification; got: %v", + err) +} + +// populatedRepo answers the one call backfillProfile makes after a SUCCESSFUL +// fetch, and panics on anything else. +// +// It exists because of what the RED run showed: with the fetch unguarded, the +// blocked-path test sailed past the refusal, reached the re-check at +// service.go:597 and panicked on a nil repository — a segfault instead of a +// readable failure. The embedded nil interface keeps that property for every +// OTHER method, which is the repo's convention (see aggregator's fakeUserService): +// a call nobody predicted panics immediately rather than returning a zero value +// that quietly changes what the test proves. +// +// GetByDID reports a user who already has a display name, so the +// firehose-won-the-race branch returns before any write. That keeps these tests +// about the fetch and its log line, and nothing else. +type populatedRepo struct { + UserRepository +} + +func (populatedRepo) GetByDID(context.Context, string) (*User, error) { + return &User{DID: backfillGuardDID, DisplayName: "already indexed by the firehose"}, nil +} + +// TestBackfillProfile_ABlockedFetchIsObservable is the second half of this +// site's contract, and the one the other eight call sites do not need. +// +// # WHY A SILENT REFUSAL IS ITS OWN DEFECT +// +// backfillProfile runs detached: no caller receives its error, no retry +// consumes it, no status code reflects it. If a blocked fetch produced nothing, +// then from outside the process a guarded build and an unguarded one would be +// indistinguishable — and so would a working backfill and one that has been +// refusing every user since a config change. An operator's only evidence that +// this control exists is the log line, which makes the log line part of the +// control rather than decoration around it. +// +// # WHY THIS TEST IS NOT PARALLEL +// +// It swaps the default slog handler, which is process-global. slog.SetDefault is +// the only seam here because backfillProfile logs through the package-level +// slog.Warn rather than an injected logger — which is itself worth knowing: it +// is why this assertion has to reach for global state, and why a future change +// that gives userService its own *slog.Logger would make this test better. +func TestBackfillProfile_ABlockedFetchIsObservable(t *testing.T) { + pds := newCountingPDS(t) + + var logged bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + service := &userService{ + userRepo: populatedRepo{}, + profileBackfillClient: NewProfileBackfillClient(false), + } + + // Called synchronously. IndexUser spawns this in a goroutine, but the + // behaviour under test is the function's own, and driving it directly is + // what keeps this test free of the sleep-for-a-goroutine pattern the audit + // forbids. + service.backfillProfile(context.Background(), backfillGuardDID, pds.server.URL) + + assert.Zerof(t, pds.requests.Load(), + "the listener was reached %d times", pds.requests.Load()) + + output := logged.String() + require.NotEmptyf(t, output, + "a blocked profile backfill produced NO log output at all. This goroutine is detached, so "+ + "the log line is the only evidence the refusal ever happened — without it an operator "+ + "cannot tell a guarded build from an unguarded one, or a working backfill from one that "+ + "has been refusing every user since a config change") + + assert.Containsf(t, output, backfillGuardDID, + "the log line does not name the DID whose backfill was refused, so an operator cannot tell "+ + "which user needs reconciling with cmd/backfill-profiles. Got: %s", output) + assert.Containsf(t, output, "SSRF blocked", + "the log line does not say the address was refused by the guard, so a refusal reads as an "+ + "ordinary network failure and gets triaged as a flaky remote PDS. Got: %s", output) +} + +// TestBackfillProfile_ASuccessfulFetchIsAlsoObservable is the control: a +// message-matching assertion is only worth something if the message can differ. +// +// Without this, a backfillProfile that logged the same warning unconditionally — +// or one that could not fetch anything at all — would satisfy the test above. +func TestBackfillProfile_ASuccessfulFetchIsAlsoObservable(t *testing.T) { + pds := newCountingPDS(t) + + var logged bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logged, &slog.HandlerOptions{Level: slog.LevelWarn}))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + service := &userService{ + userRepo: populatedRepo{}, + profileBackfillClient: NewProfileBackfillClient(true), + } + + service.backfillProfile(context.Background(), backfillGuardDID, pds.server.URL) + + assert.Equalf(t, int64(1), pds.requests.Load(), + "with the hatch open the backfill must reach the PDS; it was reached %d times", + pds.requests.Load()) + assert.NotContainsf(t, logged.String(), "SSRF blocked", + "a backfill that reached its PDS still logged an SSRF refusal, so the assertion in the test "+ + "above matches whatever this function logs and proves nothing. Got: %s", logged.String()) +} diff --git a/internal/core/users/profile_backfill_seam_test.go b/internal/core/users/profile_backfill_seam_test.go new file mode 100644 index 0000000..6ab54e4 --- /dev/null +++ b/internal/core/users/profile_backfill_seam_test.go @@ -0,0 +1,45 @@ +package users + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestNewProfileBackfillClient_TakesNoOptionsFromOutsideThisPackage makes the +// constructor's doc comment a fact about the type rather than a description of +// current habits. +// +// That comment says "Production passes nothing; it cannot open the guard." The +// first half was observation and the second half was false: the parameter was +// `opts ...covesoauth.Option`, an EXPORTED type, and covesoauth. +// WithPrivateAddressesAllowed() is an exported option that does precisely what +// the sentence promises cannot happen — from cmd/server, from a handler, from +// anywhere, with no coves:allow-ssrf-hatch marker for the audit to find. +// +// A confidently wrong comment in a security file is worse than no comment: it is +// the thing a reviewer checks INSTEAD of the code. So the seam moved to an +// unexported constructor, this package's tests call that, and the exported one +// now takes the gate and nothing else — which is what the sentence claims. +// +// Reflection, because the property is about a SIGNATURE and a signature is not +// otherwise observable from a test; a call that tried to pass an option simply +// would not compile, and a test that does not compile is not a test. +func TestNewProfileBackfillClient_TakesNoOptionsFromOutsideThisPackage(t *testing.T) { + t.Parallel() + + fn := reflect.TypeOf(NewProfileBackfillClient) + + assert.Falsef(t, fn.IsVariadic(), + "NewProfileBackfillClient is variadic (%s), so a caller outside this package can pass "+ + "covesoauth options — including WithPrivateAddressesAllowed(), which is exported. Its doc "+ + "comment says production 'cannot open the guard'; while this parameter exists that is a "+ + "convention rather than a type fact, and a security comment that overstates its enforcement "+ + "is what a reviewer trusts instead of reading the code. newProfileBackfillClient is where the "+ + "test seam belongs", fn) + + assert.Equalf(t, 1, fn.NumIn(), + "NewProfileBackfillClient takes %d parameters. The dev gate is the only thing a caller outside "+ + "this package has to say about the client it gets", fn.NumIn()) +} diff --git a/internal/core/users/service.go b/internal/core/users/service.go index 99e2b2f..ad5e36a 100644 --- a/internal/core/users/service.go +++ b/internal/core/users/service.go @@ -2,6 +2,7 @@ package users import ( "Coves/internal/atproto/identity" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/core/blobs" "bytes" "context" @@ -84,18 +85,117 @@ type userService struct { profileBackfillClient *http.Client } +// NewProfileBackfillClient builds the client the detached profile backfill +// fetches through, and is the dev gate for this call site. +// +// # WHAT IT IS GUARDING +// +// backfillProfile fetches `pdsURL` — taken from the indexed user's record, so +// chosen by whoever that user's PDS says it is — from a goroutine detached from +// the request that started it. Nothing waits on it and nothing retries it, so +// today a refusal, a timeout and a hang are all equally invisible. That is the +// second half of what makes this site worth closing: an SSRF nobody can see is +// also a control nobody can tell is working. +// +// # THE BOOLEAN IS THE GATE +// +// It takes an allow-private boolean rather than returning options, because +// WithProfileBackfill takes a whole client and there is nothing to append to. +// The contract is the same as PrivateHostOptions everywhere else: FALSE MUST +// PRODUCE A GUARDED CLIENT, and `.env.ci:140` sets IS_DEV_ENV=true so `make ci` +// never evaluates that branch anywhere else. This function is where it is +// tested. +// +// # THE DEV GATE IS THE ONLY THING A CALLER MAY SAY +// +// It takes the gate and nothing else, so "production cannot open the guard" is a +// fact about the signature rather than a description of current call sites. It +// used to take `opts ...covesoauth.Option` as a test seam — an EXPORTED type, +// and covesoauth.WithPrivateAddressesAllowed() is an exported option, so any +// package in the tree could open the guard here with nothing for the audit to +// grep. The seam is real and still needed (a guard test that builds its own +// client proves only that internal/atproto/oauth works), so it moved down to +// newProfileBackfillClient, which is unexported and reachable only from this +// package's own tests. That mirrors jetstream's newWellKnownClient, blobs' +// newBlobUploadClient and the aggregator's registerHTTPClient. +// +// # THE TIMEOUT IS THIS SITE'S OWN, AND IT IS THE SECOND OF TWO +// +// profileBackfillTimeout is re-applied over the shared client's 15s ceiling. +// backfillProfile detaches from its caller with context.WithoutCancel and then +// re-bounds with context.WithTimeout on the SAME constant, so the two agree by +// construction and the fetch is bounded even if this line is lost. Keep them +// equal: a client ceiling above the context deadline makes this field dead, and +// one below it silently shortens every backfill. +// +// # SO IS THE BYTE CEILING, AND IT IS THIRTY-TWO TIMES SMALLER THAN THE DEFAULT +// +// maxProfileResponseBytes is 1 MiB against oauth.DefaultMaxResponseBytes's +// 32 MiB, and oauth.DefaultMaxResponseBytes documents by name that a caller +// holding its own limit has to state it. FetchProfileRecord's io.LimitReader is +// only half of that limit: it bounds what io.ReadAll ALLOCATES and has no say +// over a response that ANNOUNCES its length, which the transport refuses before +// a byte of body is read. Inheriting the shared default left this site — a fetch +// against whatever host the indexed user's PDS names, from a goroutine nothing +// waits on — running a ceiling thirty-two times looser than the one written +// beside the read. +// +// EXACTLY THE LIMIT, NOT ONE ABOVE IT, and the difference from +// imageproxy/fetcher.go and posts' newGuardedRematerializeBlobClient is in what +// those two sites read. Both probe one byte PAST their own cap so that an +// oversized body is DETECTED rather than truncated, and a transport cap set to +// exactly their cap clips the probing byte. FetchProfileRecord reads +// maxProfileResponseBytes and stops, so there is no probe to make room for and a +// cap one byte higher would be a number describing nothing. +func NewProfileBackfillClient(allowPrivateHosts bool) *http.Client { + return newProfileBackfillClient(allowPrivateHosts) +} + +// newProfileBackfillClient is NewProfileBackfillClient with the test seam +// attached, unexported so that the seam cannot be reached from outside this +// package. See NewProfileBackfillClient for everything the numbers below mean. +func newProfileBackfillClient(allowPrivateHosts bool, opts ...covesoauth.Option) *http.Client { + // The site's own cap goes in BEFORE opts, so the test seam keeps its ability + // to override — and after PrivateAddressOptions, which is one-way and cannot + // be affected by either. + client := covesoauth.NewSSRFSafeHTTPClient(append( + covesoauth.PrivateAddressOptions(allowPrivateHosts), + append([]covesoauth.Option{ + covesoauth.WithMaxResponseBytes(maxProfileResponseBytes), + }, opts...)..., + )...) + client.Timeout = profileBackfillTimeout + return client +} + // UserServiceOption configures optional behavior on the user service. type UserServiceOption func(*userService) // WithProfileBackfill enables best-effort profile backfill during IndexUser (see -// profileBackfillClient). Pass nil to use a default client with a 10s timeout. -// The fetch+store runs in a detached goroutine so it never blocks IndexUser -// callers (OAuth login, Jetstream consumers); failures are logged only — run -// cmd/backfill-profiles to reconcile users whose backfill fetch failed. +// profileBackfillClient). Pass nil to use NewProfileBackfillClient's GUARDED +// client. The fetch+store runs in a detached goroutine so it never blocks +// IndexUser callers (OAuth login, Jetstream consumers); failures are logged only +// — run cmd/backfill-profiles to reconcile users whose backfill fetch failed. +// +// # THE NIL FALLBACK GOES THROUGH THE GATE, NOT AROUND IT +// +// It used to be `&http.Client{Timeout: 10 * time.Second}` — a second way for +// this service to acquire a backfill client, and the only one that never met +// NewProfileBackfillClient. That mattered because the nil branch is not a call +// site anyone reviews; it is what a call site DEGRADES TO, so the failure mode +// was "someone wired this the way the doc comment suggested" rather than +// "someone wired this wrongly". A guard whose default is unguarded is not a +// guard. +// +// The hatch is deliberately unreachable from here: false is hardcoded because +// the argument this option takes is a whole client, so a caller that wants the +// dev hatch has NewProfileBackfillClient(true) to pass — which cmd/server +// already does, from cfg.IsDevEnv. Falling back is for callers that expressed no +// opinion, and what those get has to be the safe value. func WithProfileBackfill(client *http.Client) UserServiceOption { return func(s *userService) { if client == nil { - client = &http.Client{Timeout: 10 * time.Second} + client = NewProfileBackfillClient(false) } s.profileBackfillClient = client } @@ -160,7 +260,7 @@ func NewUserService( defaultPDS: defaultPDS, turnstile: turnstile, pdsAdminPassword: pdsAdminPassword, - pdsAdminClient: &http.Client{Timeout: pdsAdminCallTimeout}, + pdsAdminClient: &http.Client{Timeout: pdsAdminCallTimeout}, // coves:allow-bare-client: pdsAdminClient talks to the AppView's own PDS from operator config } for _, opt := range opts { opt(s) @@ -322,7 +422,7 @@ func (s *userService) RegisterAccount(ctx context.Context, req RegisterAccountRe httpReq.Header.Set("Content-Type", "application/json") // Set timeout to prevent hanging on slow/unavailable PDS - client := &http.Client{ + client := &http.Client{ // coves:allow-bare-client: endpoint is built from s.defaultPDS, operator config, never a caller-supplied host Timeout: 10 * time.Second, } resp, err := client.Do(httpReq) diff --git a/internal/core/users/turnstile.go b/internal/core/users/turnstile.go index 9b6d223..bf0fb69 100644 --- a/internal/core/users/turnstile.go +++ b/internal/core/users/turnstile.go @@ -67,7 +67,7 @@ func NewCloudflareTurnstile(secret string, opts ...TurnstileOption) TurnstileVer c := &cloudflareTurnstile{ secret: secret, siteverifyURL: defaultTurnstileSiteverifyURL, - httpClient: &http.Client{Timeout: turnstileHTTPTimeout}, + httpClient: &http.Client{Timeout: turnstileHTTPTimeout}, // coves:allow-bare-client: challenges.cloudflare.com, a fixed host; no caller-supplied URL reaches it } for _, opt := range opts { opt(c) diff --git a/internal/db/postgres/community_feed_test.go b/internal/db/postgres/community_feed_test.go index 3dbdefc..3b24f1d 100644 --- a/internal/db/postgres/community_feed_test.go +++ b/internal/db/postgres/community_feed_test.go @@ -69,6 +69,7 @@ func newCommunityFeedHandler(db *sql.DB) *communityFeed.GetCommunityHandler { nil, nil, nil, + communities.PrivateHostOptions(true)..., ) feedService := communityFeeds.NewCommunityFeedService( postgres.NewCommunityFeedRepository(db, feedCursorSecret), diff --git a/internal/notify/telegram/client.go b/internal/notify/telegram/client.go index 38cb29b..7af2159 100644 --- a/internal/notify/telegram/client.go +++ b/internal/notify/telegram/client.go @@ -65,7 +65,7 @@ func NewClient(cfg Config) (*Client, error) { botToken: cfg.BotToken, chatID: cfg.ChatID, baseURL: strings.TrimSuffix(baseURL, "/"), - httpClient: &http.Client{Timeout: cfg.Timeout}, + httpClient: &http.Client{Timeout: cfg.Timeout}, // coves:allow-bare-client: api.telegram.org, a fixed host in this package's own const; no caller-supplied URL reaches it }, nil } diff --git a/internal/validation/domain.go b/internal/validation/domain.go new file mode 100644 index 0000000..d8ab77d --- /dev/null +++ b/internal/validation/domain.go @@ -0,0 +1,228 @@ +// Package validation holds predicates shared by every call site that builds a +// URL, a query or a command out of something a caller supplied. +// +// # WHY THE HOSTNAME VALIDATOR LIVES HERE AND NOT NEXT TO A CALL SITE +// +// It started as `normalizeDomain`, unexported, in +// internal/api/handlers/aggregator. That is where the first injection was found +// and closed, and being unexported is the reason the second call site — the +// community consumer's `.well-known/did.json` fetch, whose URL construction is +// byte-identical in shape — stayed open through that fix. Someone who noticed +// could not have called it. +// +// A security predicate that two packages need is a package, not a copy. The +// corpus it is tested against is shared for the same reason and lives at +// tests/domaincorpus. +package validation + +import ( + "errors" + "strings" +) + +const ( + // maxDomainLength is RFC 1035's 255-octet wire form written out as text: the + // octet count includes a length byte for the first label and a zero byte for + // the root, neither of which appears in the dotted string. A longer name + // cannot be carried by DNS, so accepting one only means handing a resolver + // something that is guaranteed to fail. + maxDomainLength = 253 + + // maxLabelLength is the single-octet length prefix DNS puts in front of each + // label, which leaves 63 characters for the label itself. + maxLabelLength = 63 + + // punycodePrefix marks a label as the ASCII encoding of a non-ASCII name. + // It is the one reason a top-level domain may carry digits — `example.рф` + // reaches a resolver as `example.xn--p1ai` — and so the one exemption from + // the all-alphabetic rule below. + punycodePrefix = "xn--" +) + +// ErrDomainInvalid is returned when a caller-supplied domain is not a hostname. +// +// It is a distinct sentinel rather than a formatted error because TWO separate +// mechanisms refuse most of the same inputs — this validator, and the +// SSRF-guarded transport wired in underneath the call sites that use it — so a +// test that only asserts "an error came back" cannot tell which one fired. A +// build where the validator had been quietly deleted and the guard caught the +// residue would look identical, and the residue is NOT the same set: the guard +// sees an address, and `internal-admin/v1/secrets?x=y#` is a *path* injection +// against whatever `internal-admin` resolves to. +var ErrDomainInvalid = errors.New("domain is not a valid hostname") + +// NormalizeDomain checks that domain is a hostname and returns its canonical +// form. +// +// This is a POSITIVE allowlist on DNS shape, not a blocklist on addresses, and +// that is the whole design. A blocklist has to start by asking "is this an IP", +// and net.ParseIP answers no for 2130706433, 127.1 and 0x7f.0.0.1 — every one +// of which a resolver turns into loopback. Asking instead "is this a hostname" +// refuses all three for the same reason it refuses `localhost`: they are not +// two-label names ending in a TLD. +// +// The accepted shape: +// +// - two or more labels separated by single dots, no leading or trailing dot +// - each label 1-63 characters of ASCII letters, digits and hyphens, and +// neither first nor last character a hyphen +// - the final label is entirely alphabetic, or punycode (so `127.1`, +// `example.123` and `127.0.0.0x1` are refused while `example.xn--p1ai` is +// not) +// - 253 characters total at most +// +// Everything a URL can carry besides the host is therefore refused by +// construction: no scheme remnant, no userinfo, no port, no path, no query, no +// fragment, no whitespace, no control characters, no non-ASCII. +// +// The returned string is the domain lowercased. DNS is case-insensitive, so +// refusing EXAMPLE.COM would turn a legitimate registration away over nothing; +// leaving the case alone would give one domain several spellings in the logs +// and in anything that later compares them. Canonicalising is the third option +// and the only one that costs nothing. +// +// IT HAS NO OPINION ABOUT WHAT A CLIENT WRAPPED THE HOSTNAME IN. `https:// +// example.com` is refused here — it is a URL, not a hostname — while the +// aggregator's registration handler goes on accepting it, because that handler +// strips a small, enumerated set of client sloppiness BEFORE calling this and +// validates the result. Unwrapping belongs at the call site that has a reason to +// tolerate it; the predicate stays a predicate. +// +// On refusal the returned string is empty: a caller that drops the error must +// not come away holding something it can put in a URL. +func NormalizeDomain(domain string) (string, error) { + // Length first, on the raw string. Lowercasing ASCII cannot change a + // string's length, so measuring before is the same as measuring after, and + // doing it here keeps an absurdly long input from being walked label by + // label. + if len(domain) > maxDomainLength { + return "", ErrDomainInvalid + } + + // Split rather than a single pass over the bytes, because every rule below + // is a rule about a label, and the dots are where the structure lives. Split + // yields an empty string for each position where a dot is missing a label, + // so a leading dot, a trailing dot, a doubled dot and the empty input all + // arrive at the same zero-length label the loop refuses. That is why none of + // them needs a case of its own. + labels := strings.Split(domain, ".") + if len(labels) < 2 { + // A single label is `localhost`, `internal-admin`, `2130706433` — names + // with no public existence that resolve to whatever the AppView's own + // network or resolver decides they mean. A hostname a stranger can prove + // ownership of has a public suffix under it, so requiring the dot costs + // a legitimate registrant nothing. + return "", ErrDomainInvalid + } + + for _, label := range labels { + if !isHostnameLabel(label) { + return "", ErrDomainInvalid + } + } + + // The TLD requirement falls on the last label only, because a leading-digit + // label like `1.example.com` is legal per RFC 1123 and in use. It is what + // separates `127.1` and `example.123` from a hostname without this function + // ever having to decide whether something is an IP address: every spelling + // of an address ends in a number, and no TLD is one. + if !isTLDLabel(labels[len(labels)-1]) { + return "", ErrDomainInvalid + } + + return strings.ToLower(domain), nil +} + +// isHostnameLabel reports whether label is one DNS label of the preferred +// syntax: 1-63 characters of ASCII letters, digits and hyphens, with a hyphen +// in neither the first nor the last position. +// +// The character set is an allowlist, and that is the load-bearing part. Byte +// comparison rather than the unicode package: unicode.IsLetter says yes to `ä` +// and unicode.IsDigit to the Arabic-Indic digits, which would let a name +// through that a resolver cannot use as written and that reads as a different +// name to a human than it does to punycode. Anything outside this set — a +// colon, a slash, an at sign, a percent, a space, CR, LF, NUL, or any byte of a +// multi-byte rune — is a character that belongs to some other part of a URL or +// to no URL at all, and it is refused here rather than left for a URL parser to +// interpret generously. +func isHostnameLabel(label string) bool { + if len(label) == 0 || len(label) > maxLabelLength { + return false + } + if label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for i := 0; i < len(label); i++ { + c := label[i] + switch { + case c >= 'a' && c <= 'z': + case c >= 'A' && c <= 'Z': + case c >= '0' && c <= '9': + case c == '-': + default: + return false + } + } + return true +} + +// isTLDLabel reports whether label is a plausible top-level domain: entirely +// alphabetic, or punycode. +// +// # WHY NOT "CONTAINS A LETTER", WHICH IS WHAT THIS USED TO BE +// +// Because a hex octet is a number that contains a letter. The old rule was +// meant as a proxy for "this is a TLD and not an address", and it read as one +// until the hex spellings were tried against it: +// +// 127.0.0.0x1 accepted, and getaddrinfo resolves it to 127.0.0.1 +// 127.0.0x1 accepted, likewise +// 0x7f.0.0.0x1 accepted, likewise +// +// The rejection table's `0x7f.0.0.1` row hid this for as long as it existed: +// that name is refused on its final `1`, and nothing in the validator ever +// looked at the `0x7f`. Move the hex to the LAST label and the proxy waves the +// name through on the `x`. A CGO_ENABLED=0 production build happens to fail +// closed — the pure-Go resolver NXDOMAINs all three — but that is an accident +// of the build, not this function doing its job, and a developer's build +// reaches loopback through an endpoint that needs no credential. +// +// So the rule is the positive one, and it must not be relaxed back to anything +// that admits a digit outside punycode: +// +// - ALPHABETIC, so no spelling of an integer can satisfy it. +// - OR PUNYCODE, because an internationalised TLD reaches a resolver as ASCII +// with digits in it, and `example.xn--p1ai` (Russia's `.рф`) is a real +// registration. The prefix is anchored: a label that merely CONTAINS +// `xn--` is not punycode and inherits nothing. +// +// The IANA root holds no TLD that is neither all-alphabetic nor `xn--`- +// prefixed, so nothing legitimate is turned away. +// +// The punycode arm leans on isHostnameLabel having already run: every caller +// checks the character set first, so what follows the prefix here is letters, +// digits and hyphens by construction rather than by a second scan. +func isTLDLabel(label string) bool { + if isAlphabetic(label) { + return true + } + return len(label) > len(punycodePrefix) && + strings.EqualFold(label[:len(punycodePrefix)], punycodePrefix) +} + +// isAlphabetic reports whether label is non-empty and made only of ASCII +// letters. Byte comparison rather than the unicode package, for the reason +// isHostnameLabel gives at length: unicode.IsLetter says yes to `ä`. +func isAlphabetic(label string) bool { + if len(label) == 0 { + return false + } + for i := 0; i < len(label); i++ { + c := label[i] + if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) { + return false + } + } + return true +} diff --git a/internal/validation/domain_test.go b/internal/validation/domain_test.go new file mode 100644 index 0000000..5f08746 --- /dev/null +++ b/internal/validation/domain_test.go @@ -0,0 +1,229 @@ +package validation + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "Coves/tests/domaincorpus" +) + +// NormalizeDomain is the shared half of a two-mechanism defence, and this file +// is its whole contract. +// +// Two call sites build a URL by concatenating a caller-supplied domain into it: +// the aggregator's registration handler (`https://` + domain + +// `/.well-known/atproto-did`, reachable with no credential) and the community +// consumer's DID-document fetch (`https://` + domain + `/.well-known/did.json`, +// reachable by anyone federated). String concatenation into a URL is the whole +// vulnerability: every part of a URL that comes after the host can be smuggled +// in through the host, because the parser does not know the concatenation was +// supposed to stop. +// +// # WHY A GUARDED DIALLER IS NOT ENOUGH, WHICH IS WHY THIS FUNCTION EXISTS +// +// The SSRF-safe transport refuses private ADDRESSES. It has no opinion about +// paths, and `internal-admin/v1/secrets?x=y#` does not name a private address — +// it names whatever `internal-admin` resolves to, which on a corporate resolver +// or a split-horizon DNS is a public-looking answer, and then requests +// `/v1/secrets?x=y` from it. Wiring the guard and skipping this validator closes +// the address half and leaves the URL-structure half wide open. +// +// # WHY THE CASES LIVE IN tests/domaincorpus +// +// So the two call sites cannot drift. See that package's doc comment. + +func TestNormalizeDomain_RefusesAnythingThatIsNotAHostname(t *testing.T) { + t.Parallel() + + for _, tt := range domaincorpus.Invalid() { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + normalized, err := NormalizeDomain(tt.Domain) + + require.ErrorIsf(t, err, ErrDomainInvalid, + "NormalizeDomain(%q) must refuse this with ErrDomainInvalid; accepting it lets a "+ + "caller aim an AppView .well-known fetch at %q", + tt.Domain, tt.Domain) + require.Emptyf(t, normalized, + "NormalizeDomain(%q) refused but still returned %q; a caller that drops the error "+ + "must not come away holding something it can concatenate into a URL", + tt.Domain, normalized) + }) + } +} + +// The other half of the allowlist: refusing everything is not a fix. +func TestNormalizeDomain_AcceptsHostnames(t *testing.T) { + t.Parallel() + + for _, tt := range domaincorpus.Valid() { + t.Run(tt.Name, func(t *testing.T) { + t.Parallel() + + normalized, err := NormalizeDomain(tt.Domain) + + require.NoErrorf(t, err, + "NormalizeDomain(%q) refused a hostname; the allowlist has to admit the domains "+ + "real aggregators and real federated instances use, or both call sites are closed", + tt.Domain) + require.Equalf(t, tt.Want, normalized, "NormalizeDomain(%q) canonical form", tt.Domain) + }) + } +} + +// The final-label rule, stated in one place and in both directions. +// +// It lives in its own test rather than as more rows in the corpus because a +// table cannot show a RULE — it shows a set of inputs, and a reader checking +// whether the rule is sound has to reconstruct it from forty scattered cases. +// That reconstruction is exactly what went wrong once: the rule "the final label +// contains at least one letter" reads as "this is a TLD and not a number", and +// it is not — `0x1` is a number that contains a letter, and the corpus's +// `0x7f.0.0.1` row hid it, because that name is refused on its `1` and never on +// its `0x7f`. +// +// The rule that holds is the positive one, and both halves are load-bearing: +// +// - ALPHABETIC, so no spelling of an integer can satisfy it. Not "contains a +// letter", which every hex octet does. +// - OR PUNYCODE (`xn--`), because an internationalised TLD reaches a resolver +// as ASCII with digits in it — `example.xn--p1ai` is Russia's `.рф` and is a +// real registration both call sites must accept. +// +// Nothing else needs admitting: the IANA root has no TLD that is neither +// all-alphabetic nor `xn--`-prefixed. +// +// THIS TEST MUST SURVIVE THE MOVE INTACT. A +// promotion that carried the corpus across but left this behind would restore +// the hex bypass with the suite green. +func TestNormalizeDomain_TheFinalLabelIsAlphabeticOrPunycode(t *testing.T) { + t.Parallel() + + refused := []struct { + name string + domain string + }{ + // The three hex bypasses, restated here so this test fails on its own + // if the rule is loosened later. + {name: "hex final octet", domain: "127.0.0.0x1"}, + {name: "hex final octet with omitted octets", domain: "127.0.0x1"}, + {name: "hex first and final octets", domain: "0x7f.0.0.0x1"}, + + // The general case the three above are instances of. No demonstrated + // resolver treats `example.a1` as an address — this row is here because + // the RULE is "alphabetic or punycode", and a fix that special-cased the + // string `0x` would satisfy every row above while leaving the proxy in + // place for whatever spelling comes next. + {name: "alphanumeric final label", domain: "example.a1"}, + {name: "digit-leading alphanumeric final label", domain: "example.1a"}, + + // Already covered by the corpus; kept so this test states the whole rule. + {name: "all-numeric final label", domain: "example.123"}, + + // `xn--` has to be a PREFIX, not a substring. A label that merely + // contains it is not punycode and must not inherit the exemption. + {name: "xn-- in the middle of the final label", domain: "example.axn--p1ai"}, + } + + for _, tt := range refused { + t.Run("refused/"+tt.name, func(t *testing.T) { + t.Parallel() + + normalized, err := NormalizeDomain(tt.domain) + + require.ErrorIsf(t, err, ErrDomainInvalid, + "NormalizeDomain(%q) accepted a final label that is neither alphabetic nor punycode. "+ + "Letter-bearing was only ever a proxy for 'not a number', and a hex octet defeats it: "+ + "%q reaches a resolver as an address, which is the whole class this validator exists "+ + "to refuse", tt.domain, tt.domain) + require.Emptyf(t, normalized, + "NormalizeDomain(%q) refused but still returned %q", tt.domain, normalized) + }) + } + + // The other half. A rule tightened until it refuses everything is not a fix, + // and each of these is a shape a real instance runs under. + accepted := []struct { + name string + domain string + want string + }{ + {name: "an ordinary TLD", domain: "example.com", want: "example.com"}, + {name: "a multi-part public suffix", domain: "sub.example.co.uk", want: "sub.example.co.uk"}, + {name: "a long alphabetic TLD", domain: "aggregator.museum", want: "aggregator.museum"}, + // The punycode exemption, and the only reason it exists: this is `.рф`. + {name: "a punycode TLD carrying digits", domain: "example.xn--p1ai", want: "example.xn--p1ai"}, + // Punycode in a non-final label needs no exemption — the final label is + // `com` — but it must go on working, and it is the more common shape. + {name: "a punycode label under an alphabetic TLD", domain: "xn--n3h.example.com", want: "xn--n3h.example.com"}, + // Digits are legal anywhere but the last label, per RFC 1123. + {name: "a digit-leading label under an alphabetic TLD", domain: "1.example.com", want: "1.example.com"}, + {name: "uppercase, canonicalised rather than refused", domain: "EXAMPLE.COM", want: "example.com"}, + {name: "uppercase punycode TLD", domain: "example.XN--P1AI", want: "example.xn--p1ai"}, + } + + for _, tt := range accepted { + t.Run("accepted/"+tt.name, func(t *testing.T) { + t.Parallel() + + normalized, err := NormalizeDomain(tt.domain) + + require.NoErrorf(t, err, + "NormalizeDomain(%q) refused a hostname a real instance runs under. Tightening the "+ + "final-label rule to close the hex spellings must not close the TLDs that carry digits "+ + "legitimately — punycode is how every internationalised domain reaches a resolver", tt.domain) + require.Equalf(t, tt.want, normalized, "NormalizeDomain(%q) canonical form", tt.domain) + }) + } +} + +// The length limits, as boundaries rather than as "something long". A cap that +// is only tested with an obviously oversized input pins no number at all. +func TestNormalizeDomain_BoundsTheNameLength(t *testing.T) { + t.Parallel() + + const maxName = 253 // RFC 1035's 255-octet wire form, written out as text + const maxLabel = 63 + + atLimit := nameOfLength(t, maxName) + overLimit := nameOfLength(t, maxName+1) + + normalized, err := NormalizeDomain(atLimit) + require.NoErrorf(t, err, + "a %d-character name is the longest DNS can carry and must be accepted", maxName) + require.Equal(t, atLimit, normalized, "a name at the length limit is returned unchanged") + + _, err = NormalizeDomain(overLimit) + require.ErrorIsf(t, err, ErrDomainInvalid, + "a %d-character name exceeds the %d-character limit and cannot resolve, so it must be "+ + "refused here rather than handed to a resolver", maxName+1, maxName) + + longestLabel := strings.Repeat("a", maxLabel) + ".com" + normalized, err = NormalizeDomain(longestLabel) + require.NoErrorf(t, err, "a %d-character label is the longest DNS allows and must be accepted", maxLabel) + require.Equal(t, longestLabel, normalized, "a label at the length limit is returned unchanged") + + _, err = NormalizeDomain(strings.Repeat("a", maxLabel+1) + ".com") + require.ErrorIsf(t, err, ErrDomainInvalid, + "a %d-character label exceeds the %d-character limit", maxLabel+1, maxLabel) +} + +// nameOfLength builds an ordinary hostname of exactly n characters: 63-character +// labels until the remainder fits in one more. The length boundary above is only +// a boundary if the accepted and refused cases differ in nothing but the count, +// which is what this guarantees. +func nameOfLength(t *testing.T, n int) string { + t.Helper() + require.Greaterf(t, n, 64, "nameOfLength cannot build a %d-character name with two labels", n) + + var labels []string + for n > 64 { + labels = append(labels, strings.Repeat("a", 63)) + n -= 64 // the label plus the dot that follows it + } + labels = append(labels, strings.Repeat("a", n)) + return strings.Join(labels, ".") +} diff --git a/scripts/ci-runner.sh b/scripts/ci-runner.sh index 4ce665e..f4be8c8 100755 --- a/scripts/ci-runner.sh +++ b/scripts/ci-runner.sh @@ -81,6 +81,20 @@ echo # to learn about it twenty minutes later. bash /src/scripts/test-audit.sh +# The SSRF fence, a SIBLING of the audit above rather than a category in it: +# test-audit.sh scans test code and says so in its scope note, this one scans +# production sources exclusively. +# +# It runs in the tier and not merely on demand because it is the only mechanism +# here that can notice a new unguarded client. .env.ci:140 sets IS_DEV_ENV=true, +# so every guarded call site in this very run takes the PERMISSIVE branch — the +# suite that follows is structurally incapable of telling a guarded deployment +# from an unguarded one, and a fence nobody runs is a fence that is not standing. +# +# tests/audit exercises the script's own rules against planted violations, so +# this line is the fence applied and that package is the fence proven. +bash /src/scripts/ssrf-audit.sh + # --------------------------------------------------------------------------- # 1e. Test template database # --------------------------------------------------------------------------- diff --git a/scripts/ssrf-audit.sh b/scripts/ssrf-audit.sh new file mode 100755 index 0000000..1ac28a6 --- /dev/null +++ b/scripts/ssrf-audit.sh @@ -0,0 +1,992 @@ +#!/usr/bin/env bash +# Counts SSRF-guard regressions in PRODUCTION code. HARD GATE — a nonzero count +# fails. +# +# WHAT THIS IS FOR +# +# The shared guard protects attacker-influenced egress across the application; +# docs/SSRF_SECURITY.md describes its maintained security contract. This audit +# is the mechanism that stops a new fetch site appearing unguarded, and it exists +# because the ordinary test suite cannot identify every client construction. +# +# `.env.ci:140` sets IS_DEV_ENV=true, so `make ci` — the hermetic merge gate, +# running T0+T1+T2 — takes the PERMISSIVE branch at every call site that holds an +# allow-private boolean. A green merge gate is therefore compatible with every +# guarded site being unguarded in production. The suite cannot see this; only a +# grep over the source can. +# +# It is also a grep because human enumeration was tried and measured: the +# remediation document's own list of call sites undercounted four separate times. +# +# THIS IS A SIBLING OF scripts/test-audit.sh, NOT A CATEGORY IN IT. That script +# audits TEST code and says so in its scope note ("Production sources are out of +# scope for every category except testing.Short()"). This one audits production +# sources exclusively. The exemption mechanism, output format and failure +# semantics are deliberately identical, so there is one convention to learn. +# +# THE FLOOR IS ZERO — with exemptions that are declared, not assumed. +# +# Every check here is a grep over text, and production code has legitimate +# reasons to build an HTTP client: a fixed vendor API, the AppView's own +# configured PDS, the guarded client's own construction. Those are exempted +# individually, in the source, with a reason — so each one is a decision someone +# made and can be found by grepping for the marker: +# +# client := &http.Client{...} // coves:allow-bare-client: +# opts := PrivateHostOptions(true) // coves:allow-ssrf-hatch: +# +# and, where per-line markers would be noise because the whole file's subject +# matter IS the thing being counted (the guard's own transport), a file-scope +# form declared once near the top of the file: +# +# // coves:allow-bare-client-file: +# +# File-scope exemptions are printed on EVERY run with their reason and hit count, +# so a broad exemption cannot go quiet. +# +# WHAT THE MARKER ACTUALLY REQUIRES, stated because each of these was once looser +# than the paragraph above implied: +# +# * IT MUST BE IN A COMMENT, quote-aware. A marker inside a string literal used +# to silence the rule, which is what happens when a log message or an error +# string quotes the convention while explaining it. +# * IT MUST CARRY A NON-EMPTY REASON. The bare token used to be enough. A +# reason nobody has to write is a reason nobody writes, and the reason is the +# only part of an entry a reviewer can disagree with. +# * IT APPLIES ON ITS OWN LINE OR THE LINE BELOW, and only to its own category: +# a justification about a bare client cannot license a resolver override. +# * IT MUST STILL HAVE SOMETHING TO EXEMPT. An exemption outliving its +# violation is a standing grant that no rule reports, and the next unguarded +# client written under it inherits a reason composed about different code. +# Rule 5 counts those. +# +# ONE CAVEAT THE NAMES HIDE: `coves:allow-bare-client` is a marker FAMILY, not a +# per-rule marker. Rules 1, 1b, 1c, 1d, 1e, 1f and 1g all name it, so one marker +# — and especially one `-file:` marker — exempts a file from every bare-client +# rule at once, not from the one that fired. That is a real breadth, it is why +# the file-scope table prints on every run, and it is why splitting a file is +# usually the better fix than broadening its exemption. +# +# IT IS ALSO WHY 1c, 1e AND 1f REFUTE THE DEFAULT BY NAME. Those three fire on a +# literal's OPENING line while the value they object to sits on a later line, so +# a marker written about the bare client on that later line exempts rule 1 or 1b +# and leaves the literal reading as remediated — one reason, written about one of +# the two rules, silencing both. The refuted list is what keeps the second +# question separate from the first. +# +# AN ALLOWLIST ENTRY IS A DOCUMENTED DECISION TO BE UNGUARDED. That is the whole +# weight of the marker, and it is why an unlisted occurrence fails the build +# rather than being snapshotted into a baseline: a wrong entry is worse than a +# noisy audit, because it converts an oversight into a recorded choice nobody +# revisits. +# +# THESE ARE TRIPWIRES, NOT PROOFS. A grep is bypassable by construction — a +# client built by a helper in another package, a transport assigned after +# construction. They catch drift cheaply. The actual guarantee is that each +# converted site's own guard test refuses a private address. +# +# WHAT IS KNOWINGLY NOT COVERED, because a false claim of coverage is worse than +# a documented hole — the hole gets read, the claim gets trusted: +# +# * PRODUCTION GO OUTSIDE cmd/ AND internal/. The corpus is those two roots and +# nothing else. Today that leaves scripts/setup_dev_aggregator.go unscanned — +# `go list` reports it as a real package, and it reaches the network twice +# with http.Post — so this is a live hole and not a theoretical one. +# +# Widening the corpus is the obvious fix and cannot be done from this file: +# the widened set immediately contains violations, and the markers that would +# settle them have to be written in the files themselves. What IS enforced is +# that the roots exist and hold production Go (rule 0), because the failure +# that hides everything is not a file outside the corpus, it is a corpus of +# no files reporting green over the whole tree. +# +# * A METHOD CALL ON AN ALREADY-BUILT CLIENT. `c.Do(req)` and `c.Get(url)` are +# how every CONVERTED site uses its guarded client; today the tree holds 26 +# `.Do(` and 141 `.Get(` calls in production code, almost all of them +# correct. There is no text distinguishing a call on a guarded client from a +# call on an unguarded one, so a rule here would be a permanent floor of ~167 +# over exactly the code the remediation fixed — the count reviewers learn to +# read past. +# +# THE SHAPES ARE CAUGHT AT CONSTRUCTION INSTEAD — but only the constructions +# named here, and the claim is worth no more than its list. Rule 1 covers the +# struct literal, new(), DefaultClient, the package-level helpers, and the +# zero-value client however it is declared (`var c http.Client`, a struct +# field, a parameter). Rule 1b covers the transport underneath and the +# single-host proxy constructor, rule 1c the xrpc.Client that names no client +# at all, rule 1e the reverse proxy assembled by hand, rule 1f the identity +# directory whose client field is a VALUE (so omitting it compiles and yields +# a zero-value client rather than a nil-pointer panic), rule 1g the four +# library constructors that install http.DefaultClient for you, and rule 1d +# the aliased import that renames past all of them. +# +# NOT COVERED AT CONSTRUCTION: any OTHER library type with an optional +# `*http.Client`, `http.Client` or `Transport` field. The list in rules +# 1c/1e/1f/1g is an ENUMERATION OF THIS TREE, not a rule that generalises — +# it grew from two entries to seven because four more were already here when +# someone looked, which is the honest measure of how well the enumeration +# tracks the code. A type this list does not name is invisible. +# +# WHAT MAKES THAT SURVIVABLE IS THAT THE HATCH IS NOT ENUMERATED. Rule 3 asks +# about the field assignment every hatch performs, so a new subsystem's guard +# can be spelled however it likes and its OFF switch is still counted. The +# enumerated rules are the tripwires; that one is the mechanism. +# +# * A CLIENT BUILT BY A HELPER IN ANOTHER PACKAGE and returned. Nothing at the +# call site says what it is. Rule 1 catches the helper's own construction, +# which is inside this tree; a client from a VENDORED dependency is not +# visible to any rule here. +# +# * A TRANSPORT ASSIGNED AFTER CONSTRUCTION — `c.Transport = tr` where tr came +# from elsewhere. +# +# The aliased-import case IS covered, by rule 1d, and not at the call site: the +# call cannot be greppable (`nh.Get(url)` has no distinguishing text) but the +# import that creates the alias must exist, and there is no reason in this tree +# to rename net/http. +# +# Usage: +# scripts/ssrf-audit.sh summary table; exits nonzero on any violation +# scripts/ssrf-audit.sh -v table plus every offending file:line +# +# Environment: +# SSRF_AUDIT_ROOT directory to scan instead of the repository root. +# +# This is a test seam, and it is the reason this script has tests at all. +# An audit rule that cannot fail is the most useless artifact in the +# repository, and asserting that a rule bites is not the same as watching it +# bite; tests/audit plants a violating fixture under a temporary root and +# requires a nonzero exit. It cannot weaken any rule — whatever root it +# names is scanned by the identical pass the repository root goes through. +set -uo pipefail + +if [[ -n ${SSRF_AUDIT_ROOT:-} ]]; then + REPO_ROOT=$SSRF_AUDIT_ROOT +else + REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +fi +cd "$REPO_ROOT" || exit 2 + +VERBOSE=0 +if [[ ${1:-} == "-v" || ${1:-} == "--verbose" ]]; then + VERBOSE=1 +fi + +CYAN='\033[36m' +YELLOW='\033[33m' +GREEN='\033[32m' +RED='\033[31m' +RESET='\033[0m' + +TOTAL=0 +ALL_PATTERNS="" +ALL_MARKERS="" +declare -a ROWS=() +declare -a DETAILS=() +declare -a EXEMPT_FILES=() + +echo +printf "${CYAN}═══════════════════════════════════════════════════════════════════════${RESET}\n" +printf "${CYAN} SSRF GUARD REGRESSION AUDIT${RESET} (hard gate — any violation fails)\n" +printf "${CYAN}═══════════════════════════════════════════════════════════════════════${RESET}\n" +echo + +# --------------------------------------------------------------------------- +# File scope +# --------------------------------------------------------------------------- + +# Production code: every Go file under cmd/ and internal/ that is not a test. +# +# TEST FILES ARE OUT OF SCOPE, and not as a convenience. A guard test has to +# stand up an httptest server and dial it, and httptest listens on LOOPBACK — +# exactly the address class the guard refuses. Auditing test files here would +# make the guard's own tests unwritable, and tests/ is where the hatch and the +# resolver seam are SUPPOSED to appear. +# +# tests/ is excluded for the same reason even though tests/testkit holds +# non-_test.go files: it is test support, and scripts/test-audit.sh already owns +# that tree. +PROD_ROOTS=(cmd internal) + +prod_code_files() { + find "${PROD_ROOTS[@]}" -type f -name '*.go' 2>/dev/null | + grep -v '_test\.go$' | sort -u +} + +# --------------------------------------------------------------------------- +# Reading a line the way Go reads it +# --------------------------------------------------------------------------- + +# split_go_line code|comment +# +# Filters stdin, printing for each line either the code before its `//` comment +# or the comment itself. Line count is preserved, so a caller can still address +# lines by number after filtering. +# +# THE QUOTE TRACKING IS THE POINT, and it is why this is not `sed 's|//.*||'`. +# Three separate checks in this script ask "does this line's TEXT contain X", +# and text is not the same question as code: +# +# * `Client:` written in a COMMENT inside an xrpc.Client literal satisfied the +# remediation check for rule 1c. An audit satisfied by a note about the fix +# is measuring its own documentation — the same defect rule 4 shipped with, +# where the Dockerfile's explanation of CGO_ENABLED=0 satisfied the check for +# CGO_ENABLED=0. +# * An exemption marker written inside a STRING silenced a rule, which is what +# happens when a log line or an error message quotes the convention. +# * `"https://example.com"` is in nearly every literal this script reads, so a +# naive comment strip would cut a line in half at the URL's slashes and take +# any brace after it with it — miscounting the literal. +split_go_line() { + awk -v mode="$1" ' + function comment_at(s, i, n, c, q, inq, inraw) { + n = length(s) + for (i = 1; i <= n; i++) { + c = substr(s, i, 1) + if (inraw) { if (c == "`") { inraw = 0 }; continue } + if (inq) { + if (c == "\\") { i++; continue } + if (c == q) { inq = 0 } + continue + } + if (c == "`") { inraw = 1; continue } + if (c == "\"" || c == "\047") { inq = 1; q = c; continue } + if (c == "/" && substr(s, i + 1, 1) == "/") { return i } + } + return 0 + } + { + at = comment_at($0) + if (mode == "code") { print (at ? substr($0, 1, at - 1) : $0) } + else { print (at ? substr($0, at) : "") } + } + ' +} + +# marker_on_line +# +# True when carries IN A COMMENT and with a non-empty reason +# after the colon. +# +# Both halves are load-bearing, and neither was checked. The old test was a +# fixed-string grep for the marker without its trailing colon, anywhere on the +# line — so `// coves:allow-bare-client` with nothing after it silenced the rule +# as well as a justification would, and so did the same text inside a string. +# +# The reason is not decoration: the header stakes the whole allowlist on an entry +# being a DOCUMENTED DECISION, and the reason is the only part of one a reviewer +# can disagree with. A reason nobody has to write is a reason nobody writes. +marker_on_line() { + local file=$1 line=$2 marker=$3 + + [[ $line -ge 1 ]] || return 1 + sed -n "${line}p" "$file" 2>/dev/null | split_go_line comment | + grep -qE "${marker}:[[:space:]]*[^[:space:]]" +} + +# --------------------------------------------------------------------------- +# Scanning +# --------------------------------------------------------------------------- + +# composite_literal_body +# +# Prints the composite literal that OPENS on , from that line through the +# line where its braces balance again. Used by the fifth argument to scan, so a +# rule can ask about the literal's contents rather than about one line's text. +# +# Brace counting rather than a fixed window, and rather than stopping at the +# first closing brace, because these literals nest: every xrpc.Client in this +# tree holds an &xrpc.AuthInfo{...} whose `},` closes several lines before the +# outer one does. A window that stopped there would miss any field written after +# it and report the remediated form as a violation — which is the failure that +# teaches people the audit is wrong. +# +# COMMENTS ARE STRIPPED BEFORE THE LITERAL IS READ, which is the difference +# between asking what a literal DOES and asking what it SAYS. `Client:` written +# in a comment inside an xrpc.Client literal used to satisfy rule 1c's +# remediation check, so the literal that merely mentions the fix passed as though +# it had applied it. Stripping also keeps a brace inside a comment out of the +# depth count. +composite_literal_body() { + local file=$1 start=$2 + + split_go_line code < "$file" 2>/dev/null | awk -v start="$start" ' + NR < start { next } + { + body = $0 + # Everything before the opening brace belongs to the enclosing + # expression; counting it would start the depth in the wrong place. + if (NR == start) { sub(/^[^{]*/, "", body) } + print $0 + depth += gsub(/\{/, "{", body) - gsub(/\}/, "}", body) + if (depth <= 0) { exit } + } + ' +} + +# scan