diff --git a/.gitignore b/.gitignore index a3d6177..4e49dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,6 @@ # Go tooling coverage.out + +# Local Claude Code session/tooling state +.claude/ diff --git a/LOOP_STATE.md b/LOOP_STATE.md index 70d6919..f57bdd9 100644 --- a/LOOP_STATE.md +++ b/LOOP_STATE.md @@ -7,7 +7,7 @@ update this file → schedule next. Stop the loop when every task is `done`. | # | Task | Status | Commit | Notes | |---|------|--------|--------|-------| | 1 | 01-scaffold-storage | done | (see git log) | reviewed by 7 reviewers, 18 fixes applied | -| 2 | 02-ap-protocol | pending | | | +| 2 | 02-ap-protocol | done | (see git log) | 5 reviewers incl. security; 14 fixes (critical: actor-id binding; high: SSRF, webfinger host confusion) | | 3 | 03-identity-repos | pending | | | | 4 | 04-sync-firehose | pending | | | | 5 | 05-materializer | pending | | | @@ -55,3 +55,31 @@ and deferred TODOs here) explicit names, mapped via pq.Error.Constraint in uniqueViolation(). - pr-review-toolkit plugin agents unavailable in this session — the loop emulates them with general-purpose agents (works fine; keep doing it). + +### From task 02 (internal/ap protocol layer — tasks 05/06 consume this) +- FetchObject/FetchActor error branches mirror ResolveStrongRef: IsNotFound + (404/401/403) → task 05 fetches ancestor chain; IsTombstoned (410 or + Tombstone body) → drop subtree (consent). SignatureError → IsValidation. +- ap.Object is one universal struct. ap.Time has .OK()/.Valid — a present + but malformed `published` is non-nil but OK()==false; task 05 MUST call + OK() before deriving rkeys/TIDs (zero Time would collide/mis-sort). +- FetchCollection signals ErrCollectionTruncated when it hits the page cap + or a next-loop — task 06 backfill must treat that as "resume needed", + NOT complete. Bare-IRI collection items come back with only ID set + (Type==""); re-fetch them. +- SSRF egress guard is ON by default; config.AllowPrivateAddresses / + ClientOptions.AllowPrivateAddresses (env ALLOW_PRIVATE_FETCH, dev-only) + disables it. ANY test/consumer hitting 127.0.0.1 httptest servers must + set AllowPrivateAddresses=true or fetches are blocked at dial time. +- Task 06 inbox wiring: Verifier.Verify(ctx, req, body) returns the signing + actor id, enforces same-authority binding (actor.ID host == keyId host) + and requires host+date+(request-target)+digest signed. It does ONE + fresh-key refetch on verify failure (key rotation) — task 06 should gate + that retry (only when key came from cache) to bound forgery amplification. + ServiceActor.DocumentJSON() ready to serve at /actor; inbox convention + https://{host}/inbox. service_keys table (migration 005) holds the + bridge's RSA key UNENCRYPTED (documented tradeoff; not user key material). +- Lemmy HTTP-sig facts (activitypub-federation-rust): Digest required on + EVERY request incl. GET; keyId is {actorID}#main-key; hs2019 treated as + rsa-sha256; 1h date-skew window. +- .claude/ is gitignored (session/tooling state, incl. scheduled_tasks.lock). diff --git a/go.mod b/go.mod index 8efa3bc..95235ce 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/lib/pq v1.12.3 github.com/pressly/goose/v3 v3.27.2 github.com/stretchr/testify v1.11.1 + golang.org/x/sync v0.21.0 + golang.org/x/time v0.15.0 ) require ( @@ -16,6 +18,5 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/sync v0.21.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 34165a6..eace9f9 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/ap/client.go b/internal/ap/client.go new file mode 100644 index 0000000..63f20d9 --- /dev/null +++ b/internal/ap/client.go @@ -0,0 +1,819 @@ +package ap + +import ( + "bytes" + "context" + "crypto/rsa" + "encoding/json" + stderrors "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" + "golang.org/x/time/rate" + + "tidepool/internal/errors" +) + +// Client defaults. All are overridable through ClientOptions. +const ( + // DefaultMaxResponseBytes caps AP response bodies. Lemmy objects are a + // few KB; a 50-item outbox page is a few hundred KB. Anything past a few + // MB is either abuse or a collection we should be paging through. + DefaultMaxResponseBytes = 5 << 20 // 5 MiB + // DefaultPerHostRPS rate-limits outbound requests per remote host. + DefaultPerHostRPS = 5 + // DefaultPerHostBurst allows short bursts (ancestor-chain fetches). + DefaultPerHostBurst = 10 + // DefaultMaxAttempts bounds retries for transient failures (5xx, 429, + // network errors): initial request plus two retries. + DefaultMaxAttempts = 3 + // DefaultRetryBaseDelay is the first backoff step (doubles per attempt, + // with jitter). + DefaultRetryBaseDelay = 500 * time.Millisecond + // DefaultMaxCollectionPages caps how many pages FetchCollection walks. + DefaultMaxCollectionPages = 10 + // DefaultRequestTimeout bounds a single HTTP attempt. + DefaultRequestTimeout = 30 * time.Second + // DefaultKeyCacheTTL is how long resolved actor public keys are cached. + // bridgy-fed caches keys aggressively for the same reason: key PEMs are + // stable and re-fetching actors per-delivery is the main cost of + // signature verification. + DefaultKeyCacheTTL = time.Hour + // maxRedirects bounds manually-followed redirects. Redirects are + // re-signed per hop ((request-target) changes), like bridgy-fed does. + maxRedirects = 5 + // defaultMaxKeyCacheEntries bounds the resolved-key cache so unique, + // attacker-controlled keyIds cannot grow it without limit. + defaultMaxKeyCacheEntries = 4096 + // defaultMaxLimiters bounds the per-host rate-limiter map for the same + // reason (unique hostnames). + defaultMaxLimiters = 4096 +) + +// ErrStop can be returned by a FetchCollection visit callback to stop paging +// early without reporting an error. +var ErrStop = stderrors.New("ap: stop iteration") + +// ErrCollectionTruncated marks a FetchCollection walk that stopped before +// reaching the end of the collection (page cap or a next-pointer loop). It is +// distinct from a clean completion so backfill (task 05) can resume or log +// rather than silently miss older items. Use errors.As to recover the +// *CollectionTruncatedError for the resume point. +var ErrCollectionTruncated = stderrors.New("ap: collection truncated") + +// CollectionTruncatedError carries where a truncated walk stopped. +type CollectionTruncatedError struct { + // Pages is how many pages were walked before stopping. + Pages int + // Next is the unfetched next-page IRI (empty if truncation was a loop). + Next string +} + +func (e *CollectionTruncatedError) Error() string { + if e.Next != "" { + return fmt.Sprintf("ap: collection truncated after %d pages, next=%s", e.Pages, e.Next) + } + return fmt.Sprintf("ap: collection truncated after %d pages", e.Pages) +} + +func (e *CollectionTruncatedError) Unwrap() error { return ErrCollectionTruncated } + +// HTTPError reports a non-success AP response that is not otherwise mapped +// to a sentinel (404 → ErrNotFound, 410 → ErrTombstoned are mapped instead). +type HTTPError struct { + URL string + StatusCode int +} + +func (e HTTPError) Error() string { + return fmt.Sprintf("ap: GET %s: unexpected status %d", e.URL, e.StatusCode) +} + +// ClientOptions configures a Client. The zero value of every field means +// "use the default above". +type ClientOptions struct { + // UserAgent is sent on every request (config.UserAgent). + UserAgent string + // Signer signs outbound requests. Optional: nil sends unsigned requests + // (many instances require authorized fetch, so production always sets + // it; tests may not). + Signer *Signer + // HTTPClient overrides the underlying client (tests). Redirects are + // handled by Client itself; any CheckRedirect on this client is + // replaced. + HTTPClient *http.Client + // MaxResponseBytes caps response bodies. + MaxResponseBytes int64 + // PerHostRPS / PerHostBurst configure per-host rate limiting. + PerHostRPS float64 + PerHostBurst int + // MaxAttempts bounds attempts per request (1 = no retries). + MaxAttempts int + // RetryBaseDelay is the first backoff step. + RetryBaseDelay time.Duration + // MaxCollectionPages caps FetchCollection paging. + MaxCollectionPages int + // KeyCacheTTL is how long resolved public keys are cached. + KeyCacheTTL time.Duration + // AllowPrivateAddresses disables the SSRF egress guard so the client may + // fetch loopback/private addresses. Default false (guard on); wire it from + // config.AllowPrivateAddresses. Tests that hit httptest servers on + // 127.0.0.1 set it true. + AllowPrivateAddresses bool +} + +// Client fetches ActivityPub objects with signed GETs, per-host rate +// limiting, retries, response-size caps, and in-flight dedupe. It also sends +// signed POSTs (task 06 uses this for Follow) and implements KeyResolver for +// signature verification. +type Client struct { + httpClient *http.Client + userAgent string + signer *Signer + maxResponseBytes int64 + perHostRPS rate.Limit + perHostBurst int + maxAttempts int + retryBaseDelay time.Duration + maxCollectionPages int + keyCacheTTL time.Duration + guard *egressGuard + // requestBudget bounds a deduped fetch's detached lifetime (see + // getDeduped): the shared request outlives the initiating caller's + // context, so it needs its own upper bound. + requestBudget time.Duration + + inflight singleflight.Group + + mu sync.Mutex + limiters map[string]*limiterEntry + keyCache map[string]cachedKey + // maxKeyCacheEntries / maxLimiters bound the per-keyID and per-host maps + // so an attacker minting unique keyIds/hosts cannot grow them without + // limit. Overridable in tests. + maxKeyCacheEntries int + maxLimiters int + + // sleep is stubbed in tests to avoid real backoff waits. + sleep func(ctx context.Context, d time.Duration) error + // now is stubbed in tests for key-cache expiry. + now func() time.Time +} + +type cachedKey struct { + key *rsa.PublicKey + ownerID string + expiresAt time.Time +} + +type limiterEntry struct { + limiter *rate.Limiter + lastUsed time.Time +} + +// NewClient creates a Client with the given options. +func NewClient(opts ClientOptions) *Client { + guard := newEgressGuard(opts.AllowPrivateAddresses) + + // Shallow-copy the caller's client so we never mutate their value (setting + // CheckRedirect/Transport on a shared *http.Client is a surprising side + // effect), and default the request timeout when unset. + var httpClient http.Client + if opts.HTTPClient != nil { + httpClient = *opts.HTTPClient + } + if httpClient.Timeout == 0 { + httpClient.Timeout = DefaultRequestTimeout + } + // Redirects are followed manually so each hop gets a fresh signature. + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + // Install the SSRF dial guard on the transport so the actually-resolved IP + // is validated (defeating DNS rebinding), not just the hostname string. + httpClient.Transport = guardedTransport(httpClient.Transport, guard) + + c := &Client{ + httpClient: &httpClient, + userAgent: opts.UserAgent, + signer: opts.Signer, + maxResponseBytes: opts.MaxResponseBytes, + perHostRPS: rate.Limit(opts.PerHostRPS), + perHostBurst: opts.PerHostBurst, + maxAttempts: opts.MaxAttempts, + retryBaseDelay: opts.RetryBaseDelay, + maxCollectionPages: opts.MaxCollectionPages, + keyCacheTTL: opts.KeyCacheTTL, + guard: guard, + limiters: make(map[string]*limiterEntry), + keyCache: make(map[string]cachedKey), + maxKeyCacheEntries: defaultMaxKeyCacheEntries, + maxLimiters: defaultMaxLimiters, + now: time.Now, + sleep: func(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } + }, + } + if c.userAgent == "" { + c.userAgent = "tidepool/0.1" + } + if c.maxResponseBytes <= 0 { + c.maxResponseBytes = DefaultMaxResponseBytes + } + if c.perHostRPS <= 0 { + c.perHostRPS = DefaultPerHostRPS + } + if c.perHostBurst <= 0 { + c.perHostBurst = DefaultPerHostBurst + } + if c.maxAttempts <= 0 { + c.maxAttempts = DefaultMaxAttempts + } + if c.retryBaseDelay <= 0 { + c.retryBaseDelay = DefaultRetryBaseDelay + } + if c.maxCollectionPages <= 0 { + c.maxCollectionPages = DefaultMaxCollectionPages + } + if c.keyCacheTTL <= 0 { + c.keyCacheTTL = DefaultKeyCacheTTL + } + // A deduped fetch runs detached from the initiating caller's context, so + // bound it: worst case is every attempt timing out plus its backoff. + c.requestBudget = time.Duration(c.maxAttempts)*DefaultRequestTimeout + + time.Duration(c.maxAttempts)*c.retryBaseDelay + return c +} + +// guardedTransport returns a RoundTripper that validates the resolved IP at +// dial time. When base is a *http.Transport (the common case, including the +// default and httptest clients) it clones it and wraps DialContext; any other +// RoundTripper is returned unchanged (URL-level checks in waitForHost still +// apply, but such transports are only used in bespoke tests). +func guardedTransport(base http.RoundTripper, guard *egressGuard) http.RoundTripper { + if base == nil { + base = http.DefaultTransport + } + t, ok := base.(*http.Transport) + if !ok { + return base + } + cloned := t.Clone() + baseDial := cloned.DialContext + if baseDial == nil { + baseDial = (&net.Dialer{Timeout: DefaultRequestTimeout, KeepAlive: 30 * time.Second}).DialContext + } + cloned.DialContext = guard.dialContext(baseDial) + return cloned +} + +// FetchObject fetches an AP object by IRI with a signed GET. +// +// Error mapping (typed so the materializer can branch): +// - 404 (and 401/403, which secure-mode instances return for objects we +// cannot see) → errors.IsNotFound; +// - 410 Gone → errors.IsTombstoned — the object was deleted upstream, the +// caller must treat it as a tombstone, never re-fetch; +// - a 200 whose body is a Tombstone object → errors.IsTombstoned (Lemmy +// serves Tombstones with 410, Mastodon with 404, others with 200); +// - other non-2xx → HTTPError. +func (c *Client) FetchObject(ctx context.Context, iri string) (*Object, error) { + body, err := c.getDeduped(ctx, iri) + if err != nil { + return nil, err + } + obj, err := ParseObject(body) + if err != nil { + return nil, fmt.Errorf("ap: fetch %s: %w", iri, err) + } + if obj.IsTombstone() { + return nil, errors.NewTombstonedError("ap_object", iri) + } + return obj, nil +} + +// FetchActor fetches an AP actor document and validates that it is one +// (Group/Person/Application/Service with an id). +func (c *Client) FetchActor(ctx context.Context, iri string) (*Object, error) { + obj, err := c.FetchObject(ctx, iri) + if err != nil { + return nil, err + } + if !obj.IsActor() { + return nil, errors.NewValidationError("actor", + fmt.Sprintf("object %s has type %q, want an actor type", iri, obj.Type)) + } + if obj.ID == "" { + return nil, errors.NewValidationError("actor", "actor document has no id") + } + return obj, nil +} + +// FetchCollection pages through a Collection/OrderedCollection, calling +// visit for every item. It handles both inline items (Lemmy outboxes are a +// single OrderedCollection with orderedItems) and paged collections +// (first/next chains, Mastodon style). Paging stops after MaxCollectionPages +// pages, when next is absent, or when visit returns an error (ErrStop stops +// silently; anything else propagates). +func (c *Client) FetchCollection(ctx context.Context, iri string, visit func(*Object) error) error { + coll, err := c.FetchObject(ctx, iri) + if err != nil { + return err + } + if !coll.IsCollection() { + return errors.NewValidationError("collection", + fmt.Sprintf("object %s has type %q, want a collection type", iri, coll.Type)) + } + + seen := map[string]bool{} + if coll.ID != "" { + seen[coll.ID] = true + } + page := coll + for pageCount := 1; ; pageCount++ { + if err := visitItems(page, visit); err != nil { + if stderrors.Is(err, ErrStop) { + return nil + } + return err + } + + nextIRI := "" + switch { + case page.Next != nil && page.Next.ID != "": + nextIRI = page.Next.ID + case len(page.Items)+len(page.OrderedItems) == 0 && page.First != nil: + // A collection header without inline items: descend into first, + // which may be an inline page object or a bare IRI. + first := page.First + if len(first.Items)+len(first.OrderedItems) > 0 || first.Type != "" { + page = first + if page.ID != "" { + if seen[page.ID] { + return &CollectionTruncatedError{Pages: pageCount} + } + seen[page.ID] = true + } + continue + } + nextIRI = first.ID + } + if nextIRI == "" { + // No further pages: the walk reached the end of the collection. + return nil + } + if pageCount >= c.maxCollectionPages { + // Page cap hit with more to fetch: report truncation so the caller + // can resume rather than mistaking it for a complete walk. + return &CollectionTruncatedError{Pages: pageCount, Next: nextIRI} + } + if seen[nextIRI] { + // A next-pointer loop: stop, but signal that we did not reach the + // natural end of the collection. + return &CollectionTruncatedError{Pages: pageCount, Next: nextIRI} + } + seen[nextIRI] = true + + page, err = c.FetchObject(ctx, nextIRI) + if err != nil { + return fmt.Errorf("ap: fetch collection page %s: %w", nextIRI, err) + } + // A next pointer must resolve to a collection page. If it resolves to + // something else (a Note, an actor, an error page served with 200), + // treat it as a broken chain rather than a clean end-of-collection. + if !page.IsCollection() { + return errors.NewValidationError("collection", + fmt.Sprintf("collection page %s has type %q, want a collection page type", nextIRI, page.Type)) + } + } +} + +func visitItems(page *Object, visit func(*Object) error) error { + for i := range page.OrderedItems { + if err := visit(&page.OrderedItems[i]); err != nil { + return err + } + } + for i := range page.Items { + if err := visit(&page.Items[i]); err != nil { + return err + } + } + return nil +} + +// SendActivity signed-POSTs an activity to a remote inbox (task 06 uses this +// for Follow/Undo). The activity is JSON-encoded as-is; a Signer must be +// configured. +func (c *Client) SendActivity(ctx context.Context, inboxURL string, activity any) error { + if c.signer == nil { + return errors.NewValidationError("signer", "SendActivity requires a configured Signer") + } + payload, err := json.Marshal(activity) + if err != nil { + return fmt.Errorf("ap: encode activity: %w", err) + } + + var lastErr error + for attempt := 0; attempt < c.maxAttempts; attempt++ { + if err := c.backoff(ctx, attempt); err != nil { + return err + } + if err := c.waitForHost(ctx, inboxURL); err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, inboxURL, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("ap: build POST %s: %w", inboxURL, err) + } + req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("Content-Type", ContentTypeActivityJSON) + if err := c.signer.SignRequest(req, payload); err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + lastErr = fmt.Errorf("ap: POST %s: %w", inboxURL, err) + continue + } + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return nil + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500: + lastErr = HTTPError{URL: inboxURL, StatusCode: resp.StatusCode} + continue + default: + return HTTPError{URL: inboxURL, StatusCode: resp.StatusCode} + } + } + return lastErr +} + +// ResolveKey implements KeyResolver: it resolves a signature keyId to the +// owning actor's RSA public key, caching results for KeyCacheTTL. The actor +// document is fetched from the keyId minus fragment (Lemmy convention: +// "{actor}#main-key"). +func (c *Client) ResolveKey(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) { + c.mu.Lock() + cached, ok := c.keyCache[keyID] + c.mu.Unlock() + if ok && c.now().Before(cached.expiresAt) { + return cached.key, cached.ownerID, nil + } + return c.resolveKeyUncached(ctx, keyID) +} + +// resolveKeyUncached fetches and validates the actor for keyID, bypassing the +// positive cache. It writes a positive cache entry only on success — a +// spoofed or mismatched resolution is never cached. +func (c *Client) resolveKeyUncached(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) { + // The URL we will fetch is derived solely from the keyId. The signer's + // claimed identity (actor.ID) is NOT trusted until it is shown to belong + // to the same authority as this fetch URL — otherwise a malicious host at + // https://evil.example/spoof could serve an actor document claiming + // id=https://lemmy.world/u/alice and we would attribute the signature to + // alice. + fetchURL := ActorIDFromKeyID(keyID) + actor, err := c.FetchActor(ctx, fetchURL) + if err != nil { + return nil, "", err + } + if actor.PublicKey == nil || actor.PublicKey.PublicKeyPem == "" { + return nil, "", errors.NewValidationError("publicKey", + fmt.Sprintf("actor %s publishes no publicKeyPem", fetchURL)) + } + // The self-asserted actor id must live on the same authority (scheme+host + // +port) as the keyId-derived fetch URL. This is the core actor-identity + // binding: it stops a cross-host actor document from claiming another + // host's identity. + if !sameAuthority(actor.ID, fetchURL) { + return nil, "", SignatureError{Reason: fmt.Sprintf( + "actor id %q is not on the same authority as key id %q", actor.ID, keyID)} + } + // If the actor names its key, it must be the one we were asked for — + // otherwise any actor could satisfy any keyId. + if actor.PublicKey.ID != "" && actor.PublicKey.ID != keyID { + return nil, "", SignatureError{Reason: fmt.Sprintf( + "actor %s publishes key %q, not %q", actor.ID, actor.PublicKey.ID, keyID)} + } + // When the key names its owner, it must be the actor that published it. + if actor.PublicKey.Owner != "" && actor.PublicKey.Owner != actor.ID { + return nil, "", SignatureError{Reason: fmt.Sprintf( + "key %q claims owner %q but is published by %q", keyID, actor.PublicKey.Owner, actor.ID)} + } + key, err := ParsePublicKeyPEM([]byte(actor.PublicKey.PublicKeyPem)) + if err != nil { + return nil, "", err + } + + c.mu.Lock() + c.evictKeyCacheLocked() + c.keyCache[keyID] = cachedKey{key: key, ownerID: actor.ID, expiresAt: c.now().Add(c.keyCacheTTL)} + c.mu.Unlock() + return key, actor.ID, nil +} + +// InvalidateKey drops any cached key for keyID so the next ResolveKey +// re-fetches. Task 06 calls this (via ResolveKeyFresh) when a cached key +// stops verifying, so a remote key rotation doesn't blackhole deliveries for +// the full cache TTL. +func (c *Client) InvalidateKey(keyID string) { + c.mu.Lock() + delete(c.keyCache, keyID) + c.mu.Unlock() +} + +// ResolveKeyFresh invalidates any cached key for keyID and resolves it again +// from the network. It implements the freshKeyResolver hook the Verifier uses +// for a single retry after a cached key fails to verify (key rotation). +func (c *Client) ResolveKeyFresh(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) { + c.InvalidateKey(keyID) + return c.resolveKeyUncached(ctx, keyID) +} + +// evictKeyCacheLocked keeps keyCache within maxKeyCacheEntries. It first +// sweeps expired entries, then, if still at the cap, evicts the entry closest +// to expiry. Callers must hold c.mu. +func (c *Client) evictKeyCacheLocked() { + if len(c.keyCache) < c.maxKeyCacheEntries { + return + } + now := c.now() + for k, v := range c.keyCache { + if !now.Before(v.expiresAt) { + delete(c.keyCache, k) + } + } + for len(c.keyCache) >= c.maxKeyCacheEntries { + var oldestKey string + var oldest time.Time + first := true + for k, v := range c.keyCache { + if first || v.expiresAt.Before(oldest) { + oldestKey, oldest, first = k, v.expiresAt, false + } + } + delete(c.keyCache, oldestKey) + } +} + +// sameAuthority reports whether two absolute URLs share scheme, host, and +// port (the authority). Unparseable or non-absolute inputs never match. +func sameAuthority(a, b string) bool { + ua, err := url.Parse(a) + if err != nil || ua.Host == "" { + return false + } + ub, err := url.Parse(b) + if err != nil || ub.Host == "" { + return false + } + return strings.EqualFold(ua.Scheme, ub.Scheme) && + strings.EqualFold(ua.Host, ub.Host) +} + +// getDeduped collapses concurrent GETs for the same IRI into one request. +// +// The shared fetch runs on a context DETACHED from the initiating caller's +// context (with its own budget), so that if the initiator cancels, the fetch +// — which every waiter depends on — is not cancelled out from under the +// others. Each caller still selects on its own ctx.Done, so an individual +// caller that cancels stops waiting immediately; it just doesn't poison the +// shared request. Values from the caller's ctx are preserved for the fetch. +func (c *Client) getDeduped(ctx context.Context, iri string) ([]byte, error) { + return c.getDedupedMode(ctx, iri, fetchModeObject) +} + +func (c *Client) getDedupedMode(ctx context.Context, iri string, mode fetchMode) ([]byte, error) { + ch := c.inflight.DoChan(iri, func() (any, error) { + fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), c.requestBudget) + defer cancel() + return c.get(fetchCtx, iri, mode) + }) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-ch: + if result.Err != nil { + return nil, result.Err + } + body, ok := result.Val.([]byte) + if !ok { + return nil, fmt.Errorf("ap: unexpected dedupe result type %T", result.Val) + } + return body, nil + } +} + +// fetchMode selects how non-success statuses map to errors. AP object fetches +// treat 401/403 as "unavailable" (NotFound); WebFinger must NOT, so a +// Cloudflare 403 / defederation is distinguishable from a genuinely missing +// account. +type fetchMode int + +const ( + fetchModeObject fetchMode = iota + fetchModeWebFinger +) + +// get performs a signed GET with retries, redirects, rate limiting, and the +// response-size cap, returning the response body. +func (c *Client) get(ctx context.Context, iri string, mode fetchMode) ([]byte, error) { + var lastErr error + for attempt := 0; attempt < c.maxAttempts; attempt++ { + if err := c.backoff(ctx, attempt); err != nil { + return nil, err + } + body, retryable, err := c.getOnce(ctx, iri, mode) + if err == nil { + return body, nil + } + if !retryable { + return nil, err + } + lastErr = err + } + return nil, lastErr +} + +// getOnce performs one GET (following up to maxRedirects manually, re-signing +// each hop). retryable reports whether the failure is transient. +func (c *Client) getOnce(ctx context.Context, iri string, mode fetchMode) (body []byte, retryable bool, err error) { + target := iri + for redirects := 0; ; redirects++ { + if err := c.waitForHost(ctx, target); err != nil { + return nil, false, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) + if err != nil { + return nil, false, fmt.Errorf("ap: build GET %s: %w", target, err) + } + req.Header.Set("User-Agent", c.userAgent) + req.Header.Set("Accept", acceptActivityJSON) + if c.signer != nil { + if err := c.signer.SignRequest(req, nil); err != nil { + return nil, false, err + } + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, true, fmt.Errorf("ap: GET %s: %w", target, err) + } + + if isRedirect(resp.StatusCode) { + location := resp.Header.Get("Location") + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + if location == "" { + return nil, false, HTTPError{URL: target, StatusCode: resp.StatusCode} + } + if redirects >= maxRedirects { + return nil, false, fmt.Errorf("ap: GET %s: too many redirects", iri) + } + next, err := url.Parse(location) + if err != nil { + return nil, false, fmt.Errorf("ap: GET %s: bad redirect location %q: %w", target, location, err) + } + target = req.URL.ResolveReference(next).String() + continue + } + + body, err := c.readBody(resp) + if err != nil { + return nil, false, fmt.Errorf("ap: GET %s: %w", target, err) + } + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return body, false, nil + case resp.StatusCode == http.StatusGone: + // 410 Gone is how Lemmy serves deleted objects (with a Tombstone + // body). Surface tombstone semantics so the materializer maps it + // to "drop, never re-fetch". + return nil, false, errors.NewTombstonedError("ap_object", iri) + case resp.StatusCode == http.StatusNotFound: + return nil, false, errors.NewNotFoundError("ap_object", iri) + case resp.StatusCode == http.StatusUnauthorized, + resp.StatusCode == http.StatusForbidden: + // For AP objects, 401/403 mean secure-mode instances declined an + // unauthorized fetch of an object that may well exist; from the + // bridge's perspective it is unavailable, treated as NotFound. + // WebFinger is different: a 401/403 there (Cloudflare, a + // defederating instance) must NOT be flattened to "account does + // not exist", so surface it as a distinguishable HTTPError that + // preserves the status. + if mode == fetchModeWebFinger { + return nil, false, HTTPError{URL: iri, StatusCode: resp.StatusCode} + } + return nil, false, errors.NewNotFoundError("ap_object", iri) + case resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500: + return nil, true, HTTPError{URL: iri, StatusCode: resp.StatusCode} + default: + return nil, false, HTTPError{URL: iri, StatusCode: resp.StatusCode} + } + } +} + +// readBody drains the response body through the size cap and closes it. +func (c *Client) readBody(resp *http.Response) ([]byte, error) { + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, c.maxResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + if int64(len(body)) > c.maxResponseBytes { + return nil, fmt.Errorf("response exceeds %d byte cap", c.maxResponseBytes) + } + return body, nil +} + +// backoff sleeps before retry attempts (none before the first), honoring ctx. +func (c *Client) backoff(ctx context.Context, attempt int) error { + if attempt == 0 { + if err := ctx.Err(); err != nil { + return err + } + return nil + } + delay := c.retryBaseDelay << (attempt - 1) + // Full jitter: uniform in [delay/2, delay]. + delay = delay/2 + time.Duration(rand.Int64N(int64(delay/2)+1)) + return c.sleep(ctx, delay) +} + +// waitForHost validates the URL against the egress guard (scheme, userinfo, +// IP literals) and applies the per-host rate limit. The resolved-IP guard +// runs later at dial time (defeating DNS rebinding). +func (c *Client) waitForHost(ctx context.Context, rawURL string) error { + parsed, err := url.Parse(rawURL) + if err != nil { + return errors.NewValidationError("url", fmt.Sprintf("unparseable url %q: %v", rawURL, err)) + } + if err := c.guard.checkURL(parsed); err != nil { + return err + } + host := strings.ToLower(parsed.Hostname()) + + c.mu.Lock() + entry, ok := c.limiters[host] + if !ok { + c.evictLimitersLocked() + entry = &limiterEntry{limiter: rate.NewLimiter(c.perHostRPS, c.perHostBurst)} + c.limiters[host] = entry + } + entry.lastUsed = c.now() + limiter := entry.limiter + c.mu.Unlock() + + if err := limiter.Wait(ctx); err != nil { + return fmt.Errorf("ap: rate limit wait for %s: %w", host, err) + } + return nil +} + +// evictLimitersLocked keeps the per-host limiter map within maxLimiters by +// evicting the least-recently-used entry. Callers must hold c.mu. +func (c *Client) evictLimitersLocked() { + for len(c.limiters) >= c.maxLimiters { + var oldestHost string + var oldest time.Time + first := true + for h, e := range c.limiters { + if first || e.lastUsed.Before(oldest) { + oldestHost, oldest, first = h, e.lastUsed, false + } + } + delete(c.limiters, oldestHost) + } +} + +func isRedirect(status int) bool { + switch status { + case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, + http.StatusTemporaryRedirect, http.StatusPermanentRedirect: + return true + } + return false +} diff --git a/internal/ap/client_test.go b/internal/ap/client_test.go new file mode 100644 index 0000000..589941d --- /dev/null +++ b/internal/ap/client_test.go @@ -0,0 +1,806 @@ +package ap + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" +) + +// newTestClient builds a client with instant retries pointed at a test +// server, signing with the shared test key. +func newTestClient(t *testing.T, opts ClientOptions) *Client { + t.Helper() + if opts.Signer == nil { + opts.Signer = NewSigner(testKeyID, testRSAKey(t)) + } + if opts.UserAgent == "" { + opts.UserAgent = "tidepool-test/0" + } + // Tests hit httptest servers on 127.0.0.1; relax the SSRF egress guard the + // way local dev does (config.AllowPrivateAddresses). Egress-guard behavior + // itself is covered by the dedicated tests in egress_test.go. + opts.AllowPrivateAddresses = true + c := NewClient(opts) + c.sleep = func(ctx context.Context, _ time.Duration) error { return ctx.Err() } + return c +} + +func TestFetchObject_SignedGET(t *testing.T) { + key := testRSAKey(t) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + var sawRequest *http.Request + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sawRequest = r.Clone(context.Background()) + w.Header().Set("Content-Type", ContentTypeActivityJSON) + _, _ = w.Write(loadFixture(t, "page_lemmy_world.json")) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + obj, err := client.FetchObject(context.Background(), server.URL+"/post/49131386") + require.NoError(t, err) + assert.Equal(t, TypePage, obj.Type) + assert.Equal(t, "https://lemmy.world/post/49131386", obj.ID) + + require.NotNil(t, sawRequest) + assert.Contains(t, sawRequest.Header.Get("Accept"), "application/activity+json") + assert.Equal(t, "tidepool-test/0", sawRequest.Header.Get("User-Agent")) + + // The GET must carry a signature Lemmy would accept: verify it + // server-side, empty body. + _, err = verifier.Verify(context.Background(), sawRequest, nil) + require.NoError(t, err, "signed GET must verify against the signer's public key") + fields := parseSignatureHeader(sawRequest.Header.Get("Signature")) + assert.Equal(t, "(request-target) host date digest", fields["headers"]) +} + +func TestFetchObject_StatusMapping(t *testing.T) { + cases := []struct { + status int + check func(t *testing.T, err error) + }{ + {http.StatusNotFound, func(t *testing.T, err error) { + assert.True(t, errors.IsNotFound(err), "404 → IsNotFound, got %v", err) + assert.False(t, errors.IsTombstoned(err)) + }}, + {http.StatusGone, func(t *testing.T, err error) { + assert.True(t, errors.IsTombstoned(err), "410 → IsTombstoned, got %v", err) + assert.False(t, errors.IsNotFound(err), "tombstoned must NOT satisfy IsNotFound") + }}, + {http.StatusForbidden, func(t *testing.T, err error) { + assert.True(t, errors.IsNotFound(err), "403 (authorized fetch) → IsNotFound, got %v", err) + }}, + {http.StatusUnauthorized, func(t *testing.T, err error) { + assert.True(t, errors.IsNotFound(err), "401 → IsNotFound, got %v", err) + }}, + {http.StatusTeapot, func(t *testing.T, err error) { + var httpErr HTTPError + require.ErrorAs(t, err, &httpErr) + assert.Equal(t, http.StatusTeapot, httpErr.StatusCode) + }}, + } + for _, tc := range cases { + t.Run(fmt.Sprint(tc.status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + _, err := client.FetchObject(context.Background(), server.URL+"/x") + require.Error(t, err) + tc.check(t, err) + }) + } +} + +func TestFetchObject_TombstoneBodyIsTombstoned(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Tombstone","id":"https://x.example/post/1","formerType":"Page"}`)) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + _, err := client.FetchObject(context.Background(), server.URL+"/post/1") + require.Error(t, err) + assert.True(t, errors.IsTombstoned(err), + "a 200 Tombstone body must surface tombstone semantics, got %v", err) +} + +func TestFetchObject_RetriesTransientFailures(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if hits.Add(1) < 3 { + w.WriteHeader(http.StatusBadGateway) + return + } + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{MaxAttempts: 3}) + obj, err := client.FetchObject(context.Background(), server.URL+"/1") + require.NoError(t, err) + assert.Equal(t, TypeNote, obj.Type) + assert.Equal(t, int32(3), hits.Load()) +} + +func TestFetchObject_DoesNotRetryHardFailures(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{MaxAttempts: 3}) + _, err := client.FetchObject(context.Background(), server.URL+"/1") + require.Error(t, err) + assert.Equal(t, int32(1), hits.Load(), "404 must not be retried") +} + +func TestFetchObject_ResponseSizeCap(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Note","content":"` + strings.Repeat("x", 4096) + `"}`)) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{MaxResponseBytes: 1024}) + _, err := client.FetchObject(context.Background(), server.URL+"/big") + require.Error(t, err) + assert.Contains(t, err.Error(), "cap") +} + +func TestFetchObject_RejectsBadURLs(t *testing.T) { + client := newTestClient(t, ClientOptions{}) + for _, iri := range []string{"ftp://example.com/x", "not-a-url", "https:///nohost"} { + _, err := client.FetchObject(context.Background(), iri) + assert.Error(t, err, "iri %q", iri) + } +} + +func TestFetchObject_ContextCancelled(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := newTestClient(t, ClientOptions{}) + _, err := client.FetchObject(ctx, server.URL+"/x") + require.ErrorIs(t, err, context.Canceled) +} + +func TestFetchObject_FollowsRedirectsWithFreshSignatures(t *testing.T) { + key := testRSAKey(t) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + mux.HandleFunc("/old", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "/new", http.StatusMovedPermanently) + }) + mux.HandleFunc("/new", func(w http.ResponseWriter, r *http.Request) { + // The redirected request must carry a signature valid for /new — + // a naively replayed signature would still say (request-target) /old. + if _, err := verifier.Verify(context.Background(), r, nil); err != nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/new"}`)) + }) + + client := newTestClient(t, ClientOptions{}) + obj, err := client.FetchObject(context.Background(), server.URL+"/old") + require.NoError(t, err) + assert.Equal(t, "https://x.example/new", obj.ID) +} + +func TestFetchActor_RejectsNonActors(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(loadFixture(t, "page_lemmy_world.json")) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + _, err := client.FetchActor(context.Background(), server.URL+"/post/1") + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestFetchActor_Group(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(loadFixture(t, "group_lemmy_world.json")) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + actor, err := client.FetchActor(context.Background(), server.URL+"/c/technology") + require.NoError(t, err) + assert.Equal(t, TypeGroup, actor.Type) +} + +func TestFetchCollection_InlineItems(t *testing.T) { + // Lemmy outboxes: one OrderedCollection, all items inline, no paging. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(loadFixture(t, "outbox_lemmy_world.json")) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + var types []string + err := client.FetchCollection(context.Background(), server.URL+"/outbox", func(item *Object) error { + types = append(types, item.Type) + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{TypeAnnounce, TypeAnnounce}, types) +} + +func TestFetchCollection_PagedWithCap(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + const totalPages = 5 + mux.HandleFunc("/collection", func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + if page == "" { + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "OrderedCollection", "id": server.URL + "/collection", + "totalItems": totalPages, "first": server.URL + "/collection?page=1", + }) + return + } + n := int(page[0] - '0') + doc := map[string]any{ + "type": "OrderedCollectionPage", + "id": server.URL + "/collection?page=" + page, + "orderedItems": []map[string]any{ + {"type": "Note", "id": fmt.Sprintf("https://x.example/%d", n)}, + }, + } + if n < totalPages { + doc["next"] = fmt.Sprintf("%s/collection?page=%d", server.URL, n+1) + } + _ = json.NewEncoder(w).Encode(doc) + }) + + // Uncapped walk sees every page. + client := newTestClient(t, ClientOptions{}) + var ids []string + err := client.FetchCollection(context.Background(), server.URL+"/collection", func(item *Object) error { + ids = append(ids, item.ID) + return nil + }) + require.NoError(t, err) + assert.Len(t, ids, totalPages) + + // Capped walk stops at MaxCollectionPages pages (header + first page + + // one next = 3 fetches, 2 item-bearing pages) and reports truncation so + // the caller can resume rather than mistaking it for a complete walk. + client = newTestClient(t, ClientOptions{MaxCollectionPages: 3}) + ids = nil + err = client.FetchCollection(context.Background(), server.URL+"/collection", func(item *Object) error { + ids = append(ids, item.ID) + return nil + }) + require.ErrorIs(t, err, ErrCollectionTruncated, "page cap must signal truncation, not clean completion") + var truncated *CollectionTruncatedError + require.ErrorAs(t, err, &truncated) + assert.Equal(t, 3, truncated.Pages) + assert.NotEmpty(t, truncated.Next, "truncation must carry the resume pointer") + assert.Len(t, ids, 2, "page cap must stop the walk early") + + // ErrStop halts silently. + client = newTestClient(t, ClientOptions{}) + ids = nil + err = client.FetchCollection(context.Background(), server.URL+"/collection", func(item *Object) error { + ids = append(ids, item.ID) + return ErrStop + }) + require.NoError(t, err) + assert.Len(t, ids, 1) + + // A visit error propagates. + client = newTestClient(t, ClientOptions{}) + wantErr := fmt.Errorf("translation exploded") + err = client.FetchCollection(context.Background(), server.URL+"/collection", func(item *Object) error { + return wantErr + }) + require.ErrorIs(t, err, wantErr) +} + +func TestFetchCollection_NextLoopTerminates(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + mux.HandleFunc("/collection", func(w http.ResponseWriter, _ *http.Request) { + // Malicious/broken server: next points back at itself. + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "OrderedCollectionPage", "id": server.URL + "/collection", + "orderedItems": []map[string]any{{"type": "Note", "id": "https://x.example/1"}}, + "next": server.URL + "/collection", + }) + }) + + client := newTestClient(t, ClientOptions{}) + count := 0 + err := client.FetchCollection(context.Background(), server.URL+"/collection", func(*Object) error { + count++ + return nil + }) + require.ErrorIs(t, err, ErrCollectionTruncated, + "a next-pointer loop is truncation, not a clean end-of-collection") + assert.Equal(t, 1, count, "a next-pointer loop must terminate after one visit") +} + +// TestFetchCollection_NonPageNextRejected: a next pointer that resolves to +// something that is not a collection page (a Note, an actor, an HTML error +// page served with 200) must be a hard error, not a silent clean finish that +// would make backfill think it saw the whole collection. +func TestFetchCollection_NonPageNextRejected(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + + mux.HandleFunc("/collection", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "OrderedCollectionPage", "id": server.URL + "/collection", + "orderedItems": []map[string]any{{"type": "Note", "id": "https://x.example/1"}}, + "next": server.URL + "/notapage", + }) + }) + mux.HandleFunc("/notapage", func(w http.ResponseWriter, _ *http.Request) { + // A 200 that is a Note, not a collection page. + _ = json.NewEncoder(w).Encode(map[string]any{"type": "Note", "id": "https://x.example/2"}) + }) + + client := newTestClient(t, ClientOptions{}) + err := client.FetchCollection(context.Background(), server.URL+"/collection", func(*Object) error { return nil }) + require.Error(t, err) + assert.True(t, errors.IsValidation(err), + "a next pointer resolving to a non-collection must be a validation error, not a clean finish") +} + +func TestFetchCollection_RejectsNonCollections(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(loadFixture(t, "person_lemmy_world.json")) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + err := client.FetchCollection(context.Background(), server.URL+"/u/x", func(*Object) error { return nil }) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestGetDeduped_CollapsesConcurrentFetches(t *testing.T) { + var hits atomic.Int32 + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + <-release + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + const workers = 8 + var wg sync.WaitGroup + objErrs := make([]error, workers) + for i := 0; i < workers; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + _, objErrs[i] = client.FetchObject(context.Background(), server.URL+"/1") + }(i) + } + // Give every worker time to join the in-flight group, then release. + time.Sleep(100 * time.Millisecond) + close(release) + wg.Wait() + + for i, err := range objErrs { + require.NoError(t, err, "worker %d", i) + } + assert.Equal(t, int32(1), hits.Load(), "concurrent fetches of one IRI must hit the server once") +} + +// TestGetDeduped_InitiatorCancelDoesNotPoisonWaiters proves the shared fetch +// runs on a context detached from the initiating caller: if caller A (whose +// ctx started the request) cancels, caller B — deduped onto the same in-flight +// request — must still succeed, not inherit A's cancellation (finding 4). +func TestGetDeduped_InitiatorCancelDoesNotPoisonWaiters(t *testing.T) { + entered := make(chan struct{}, 1) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + select { + case entered <- struct{}{}: + default: + } + <-release + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + iri := server.URL + "/1" + + ctxA, cancelA := context.WithCancel(context.Background()) + var aErr, bErr error + var aObj, bObj *Object + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + aObj, aErr = client.FetchObject(ctxA, iri) // initiator + }() + // Wait until A's request is actually in flight (server handler entered) so + // B joins the same singleflight call. + <-entered + wg.Add(1) + go func() { + defer wg.Done() + bObj, bErr = client.FetchObject(context.Background(), iri) // waiter + }() + // Give B a moment to join the in-flight group, then cancel the initiator. + time.Sleep(50 * time.Millisecond) + cancelA() + // Let A observe cancellation before the shared request completes. + time.Sleep(50 * time.Millisecond) + close(release) + wg.Wait() + + require.ErrorIs(t, aErr, context.Canceled, "the cancelling initiator stops waiting") + require.NoError(t, bErr, "the waiter must not be poisoned by the initiator's cancel") + require.NotNil(t, bObj) + assert.Equal(t, "https://x.example/1", bObj.ID) + _ = aObj +} + +func TestPerHostRateLimiting(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + // Burst 1 at 20 rps: the second request must wait ~50ms. + client := newTestClient(t, ClientOptions{PerHostRPS: 20, PerHostBurst: 1}) + ctx := context.Background() + + start := time.Now() + _, err := client.FetchObject(ctx, server.URL+"/a") + require.NoError(t, err) + _, err = client.FetchObject(ctx, server.URL+"/b") + require.NoError(t, err) + elapsed := time.Since(start) + assert.GreaterOrEqual(t, elapsed, 40*time.Millisecond, + "second request to the same host must be rate limited") +} + +func TestSendActivity_SignedPOST(t *testing.T) { + key := testRSAKey(t) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + var receivedBody []byte + var verifyErr error + var sawContentType string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + receivedBody = body + sawContentType = r.Header.Get("Content-Type") + _, verifyErr = verifier.Verify(context.Background(), r, body) + w.WriteHeader(http.StatusAccepted) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{}) + follow := map[string]any{ + "@context": "https://www.w3.org/ns/activitystreams", + "type": TypeFollow, + "id": "https://bridge.example/activities/follow/1", + "actor": testActorID, + "object": "https://lemmy.world/c/technology", + } + require.NoError(t, client.SendActivity(context.Background(), server.URL+"/inbox", follow)) + + require.NoError(t, verifyErr, "inbox POST must carry a valid signature over the body") + assert.Equal(t, ContentTypeActivityJSON, sawContentType) + assert.Contains(t, string(receivedBody), `"Follow"`) +} + +func TestSendActivity_RetriesAndFails(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + client := newTestClient(t, ClientOptions{MaxAttempts: 2}) + err := client.SendActivity(context.Background(), server.URL+"/inbox", map[string]any{"type": "Follow"}) + require.Error(t, err) + assert.Equal(t, int32(2), hits.Load()) + + var httpErr HTTPError + require.ErrorAs(t, err, &httpErr) + assert.Equal(t, http.StatusServiceUnavailable, httpErr.StatusCode) +} + +func TestSendActivity_RequiresSigner(t *testing.T) { + client := NewClient(ClientOptions{}) + err := client.SendActivity(context.Background(), "https://lemmy.world/inbox", map[string]any{}) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestResolveKey_FetchesAndCaches(t *testing.T) { + // Serve an actor document that publishes our test public key. + key := testRSAKey(t) + publicPEM, err := EncodePublicKeyPEM(&key.PublicKey) + require.NoError(t, err) + + var hits atomic.Int32 + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + actorID := server.URL + "/u/alice" + keyID := actorID + "#main-key" + mux.HandleFunc("/u/alice", func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "Person", "id": actorID, "preferredUsername": "alice", + "publicKey": map[string]string{ + "id": keyID, "owner": actorID, "publicKeyPem": string(publicPEM), + }, + }) + }) + + client := newTestClient(t, ClientOptions{}) + resolved, ownerID, err := client.ResolveKey(context.Background(), keyID) + require.NoError(t, err) + assert.True(t, key.PublicKey.Equal(resolved)) + assert.Equal(t, actorID, ownerID) + + // Second resolve is served from cache. + _, _, err = client.ResolveKey(context.Background(), keyID) + require.NoError(t, err) + assert.Equal(t, int32(1), hits.Load(), "resolved keys must be cached") + + // Expired cache re-fetches. + client.now = func() time.Time { return time.Now().Add(2 * DefaultKeyCacheTTL) } + _, _, err = client.ResolveKey(context.Background(), keyID) + require.NoError(t, err) + assert.Equal(t, int32(2), hits.Load(), "expired cache entries must be re-fetched") +} + +// TestVerify_RefetchesRotatedKey proves the key-rotation recovery path +// (finding 5): a key is cached, the remote rotates it, and a signature made +// with the NEW key initially fails against the stale cached key — but the +// Verifier's one-shot ResolveKeyFresh retry re-fetches and verifies. Without +// it the actor's deliveries would blackhole for the full cache TTL. +func TestVerify_RefetchesRotatedKey(t *testing.T) { + oldKey := testRSAKey(t) + newKey, err := GenerateRSAKey() + require.NoError(t, err) + + oldPEM, err := EncodePublicKeyPEM(&oldKey.PublicKey) + require.NoError(t, err) + newPEM, err := EncodePublicKeyPEM(&newKey.PublicKey) + require.NoError(t, err) + + var ( + mu sync.Mutex + servedPEM = string(oldPEM) + fetchCount int + ) + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + actorID := server.URL + "/u/rotator" + keyID := actorID + "#main-key" + mux.HandleFunc("/u/rotator", func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + pem := servedPEM + fetchCount++ + mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "Person", "id": actorID, "preferredUsername": "rotator", + "publicKey": map[string]string{"id": keyID, "owner": actorID, "publicKeyPem": pem}, + }) + }) + + client := newTestClient(t, ClientOptions{}) + // Warm the cache with the OLD key. + _, _, err = client.ResolveKey(context.Background(), keyID) + require.NoError(t, err) + + // Remote rotates to the NEW key. + mu.Lock() + servedPEM = string(newPEM) + mu.Unlock() + + // A request signed with the new key arrives. + signer := NewSigner(keyID, newKey) + req := httptest.NewRequest(http.MethodGet, actorID, nil) + require.NoError(t, signer.SignRequest(req, nil)) + + verifier := NewVerifier(client) + ownerID, err := verifier.Verify(context.Background(), req, nil) + require.NoError(t, err, "verification must recover by re-fetching the rotated key") + assert.Equal(t, actorID, ownerID) + + mu.Lock() + assert.GreaterOrEqual(t, fetchCount, 2, "the stale cached key must trigger exactly one refetch") + mu.Unlock() +} + +// TestKeyCacheEviction proves the resolved-key cache is bounded (finding 8): +// an attacker minting unique keyIds cannot grow it without limit. Expired +// entries are swept first; if still at the cap the nearest-to-expiry entry is +// evicted. +func TestKeyCacheEviction(t *testing.T) { + c := newTestClient(t, ClientOptions{}) + base := time.Now() + c.now = func() time.Time { return base } + c.maxKeyCacheEntries = 4 + + // Insert past the cap through the same locked path resolveKeyUncached uses. + for i := 0; i < 20; i++ { + c.mu.Lock() + c.evictKeyCacheLocked() + c.keyCache[fmt.Sprintf("https://h%d.example/u/x#main-key", i)] = + cachedKey{expiresAt: base.Add(time.Hour)} + c.mu.Unlock() + } + c.mu.Lock() + size := len(c.keyCache) + c.mu.Unlock() + assert.LessOrEqual(t, size, c.maxKeyCacheEntries, "key cache must stay within its cap") + + // Expired entries are swept when the cache is under pressure. + c.mu.Lock() + c.keyCache = map[string]cachedKey{ + "expired1": {expiresAt: base.Add(-time.Hour)}, + "expired2": {expiresAt: base.Add(-time.Minute)}, + "live1": {expiresAt: base.Add(time.Hour)}, + "live2": {expiresAt: base.Add(time.Hour)}, + } + c.evictKeyCacheLocked() // len==cap(4) triggers the expired sweep + _, e1 := c.keyCache["expired1"] + _, e2 := c.keyCache["expired2"] + _, l1 := c.keyCache["live1"] + c.mu.Unlock() + assert.False(t, e1, "expired entries must be swept") + assert.False(t, e2) + assert.True(t, l1, "live entries survive the sweep") +} + +// TestLimiterEviction proves the per-host limiter map is bounded (finding 8). +func TestLimiterEviction(t *testing.T) { + c := newTestClient(t, ClientOptions{}) + c.maxLimiters = 4 + ctx := context.Background() + for i := 0; i < 20; i++ { + require.NoError(t, c.waitForHost(ctx, fmt.Sprintf("https://h%d.example/x", i))) + } + c.mu.Lock() + size := len(c.limiters) + c.mu.Unlock() + assert.LessOrEqual(t, size, c.maxLimiters, "limiter map must stay within its cap") +} + +func TestResolveKey_RejectsKeyIDMismatch(t *testing.T) { + key := testRSAKey(t) + publicPEM, err := EncodePublicKeyPEM(&key.PublicKey) + require.NoError(t, err) + + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + actorID := server.URL + "/u/mallory" + mux.HandleFunc("/u/mallory", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "Person", "id": actorID, + "publicKey": map[string]string{ + "id": actorID + "#other-key", + "owner": actorID, + "publicKeyPem": string(publicPEM), + }, + }) + }) + + client := newTestClient(t, ClientOptions{}) + _, _, err = client.ResolveKey(context.Background(), actorID+"#main-key") + require.Error(t, err) + assert.True(t, errors.IsValidation(err), + "an actor publishing a different keyId must not satisfy the requested one") +} + +// TestResolveKey_RejectsCrossHostIDSpoof is the core actor-identity-binding +// test: a malicious host serves an actor document whose self-asserted id +// belongs to a DIFFERENT authority (lemmy.world) while signing with a keyId on +// its own host. ResolveKey must refuse to attribute the key to lemmy.world's +// actor, and must not cache the spoof as a positive result. +func TestResolveKey_RejectsCrossHostIDSpoof(t *testing.T) { + key := testRSAKey(t) + publicPEM, err := EncodePublicKeyPEM(&key.PublicKey) + require.NoError(t, err) + + var hits atomic.Int32 + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + keyID := server.URL + "/spoof#main-key" + spoofedID := "https://lemmy.world/u/alice" + mux.HandleFunc("/spoof", func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + _ = json.NewEncoder(w).Encode(map[string]any{ + // Claims to be lemmy.world's alice, but is served from the test host. + "type": "Person", "id": spoofedID, "preferredUsername": "alice", + "publicKey": map[string]string{ + "id": keyID, "owner": spoofedID, "publicKeyPem": string(publicPEM), + }, + }) + }) + + client := newTestClient(t, ClientOptions{}) + _, _, err = client.ResolveKey(context.Background(), keyID) + require.Error(t, err) + require.ErrorIs(t, err, errors.ErrInvalidInput, + "a cross-authority id must be rejected (signature error)") + + // Must not have been cached as a positive key: a second resolve re-fetches. + _, _, err = client.ResolveKey(context.Background(), keyID) + require.Error(t, err) + assert.Equal(t, int32(2), hits.Load(), "a rejected/spoofed resolution must not be cached") +} + +// TestResolveKey_RejectsOwnerMismatch: the actor lives on the right authority +// and publishes the right keyId, but the key names a different owner than the +// actor that published it. Reject — otherwise a key could be laundered through +// an unrelated actor document. +func TestResolveKey_RejectsOwnerMismatch(t *testing.T) { + key := testRSAKey(t) + publicPEM, err := EncodePublicKeyPEM(&key.PublicKey) + require.NoError(t, err) + + mux := http.NewServeMux() + server := httptest.NewServer(mux) + defer server.Close() + actorID := server.URL + "/u/bob" + keyID := actorID + "#main-key" + mux.HandleFunc("/u/bob", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "type": "Person", "id": actorID, "preferredUsername": "bob", + "publicKey": map[string]string{ + "id": keyID, "owner": server.URL + "/u/eve", "publicKeyPem": string(publicPEM), + }, + }) + }) + + client := newTestClient(t, ClientOptions{}) + _, _, err = client.ResolveKey(context.Background(), keyID) + require.Error(t, err) + assert.True(t, errors.IsValidation(err), + "a key whose owner is not the publishing actor must be rejected") +} diff --git a/internal/ap/egress.go b/internal/ap/egress.go new file mode 100644 index 0000000..28470e8 --- /dev/null +++ b/internal/ap/egress.go @@ -0,0 +1,138 @@ +package ap + +import ( + "context" + "fmt" + "net" + "net/url" + + "tidepool/internal/errors" +) + +// egressGuard blocks outbound requests to addresses an SSRF attacker would +// pivot through: loopback, RFC1918 private, link-local, unique-local, +// multicast, unspecified, and the cloud-metadata endpoint. It rejects +// non-http(s) schemes and URL userinfo, validates any IP literal in a URL, +// and — crucially — validates the actually-resolved IP at dial time so a +// hostname that resolves to a blocked address (or rebinds between the DNS +// answer and the connect) is refused. This mirrors bridgy-fed's +// are_urls_safe egress check. +// +// The guard is config-gated (config.AllowPrivateAddresses / +// ClientOptions.AllowPrivateAddresses): production keeps it on; local dev and +// the httptest-based tests that hit 127.0.0.1 turn it off. +type egressGuard struct { + allowPrivate bool + // lookupIPAddr resolves a hostname to candidate IPs. It is a field (not a + // direct net.Resolver call) so tests can inject a DNS-rebinding scenario — + // a hostname that resolves to a blocked address — without a real DNS server. + lookupIPAddr func(ctx context.Context, host string) ([]net.IPAddr, error) +} + +func newEgressGuard(allowPrivate bool) *egressGuard { + return &egressGuard{ + allowPrivate: allowPrivate, + lookupIPAddr: net.DefaultResolver.LookupIPAddr, + } +} + +// checkURL validates scheme, userinfo, and any IP-literal host before a +// request is dialed. Hostname-based blocking happens at dial time (checkIP +// against the resolved address). +func (g *egressGuard) checkURL(u *url.URL) error { + if u.Scheme != "http" && u.Scheme != "https" { + return errors.NewValidationError("url", fmt.Sprintf("unsupported scheme %q", u.Scheme)) + } + if u.User != nil { + return errors.NewValidationError("url", "url must not contain userinfo") + } + host := u.Hostname() + if host == "" { + return errors.NewValidationError("url", fmt.Sprintf("url %q has no host", u.String())) + } + if ip := net.ParseIP(host); ip != nil { + return g.checkIP(ip) + } + return nil +} + +// checkIP rejects addresses in the blocked ranges unless the guard is +// disabled for dev/test. +func (g *egressGuard) checkIP(ip net.IP) error { + if g.allowPrivate { + return nil + } + if isBlockedIP(ip) { + return errors.NewValidationError("url", + fmt.Sprintf("address %s is not an allowed egress target", ip)) + } + return nil +} + +// metadataV4 is the cloud instance-metadata address (AWS/GCP/Azure/etc.). It +// is link-local so isBlockedIP already catches it; naming it makes intent +// explicit and covers the mapped forms. +var metadataV4 = net.IPv4(169, 254, 169, 254) + +func isBlockedIP(ip net.IP) bool { + if ip == nil { + return true + } + if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsInterfaceLocalMulticast() || + ip.IsMulticast() || ip.IsUnspecified() { + return true + } + if ip.Equal(metadataV4) { + return true + } + return false +} + +type dialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + +// dialContext wraps a base dialer, resolving the host and validating every +// candidate IP before connecting, then dialing a validated IP directly so a +// DNS answer cannot rebind to a blocked address between check and connect. +func (g *egressGuard) dialContext(base dialFunc) dialFunc { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + if g.allowPrivate { + return base(ctx, network, addr) + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + // IP literal: validate and dial as-is. + if ip := net.ParseIP(host); ip != nil { + if err := g.checkIP(ip); err != nil { + return nil, err + } + return base(ctx, network, addr) + } + ips, err := g.lookupIPAddr(ctx, host) + if err != nil { + return nil, err + } + var lastErr error + for _, ipa := range ips { + if err := g.checkIP(ipa.IP); err != nil { + lastErr = err + continue + } + // Dial the exact IP we validated (not the hostname) to defeat DNS + // rebinding. + conn, err := base(ctx, network, net.JoinHostPort(ipa.IP.String(), port)) + if err != nil { + lastErr = err + continue + } + return conn, nil + } + if lastErr == nil { + lastErr = errors.NewValidationError("url", + fmt.Sprintf("host %q did not resolve to any address", host)) + } + return nil, lastErr + } +} diff --git a/internal/ap/egress_test.go b/internal/ap/egress_test.go new file mode 100644 index 0000000..49a3144 --- /dev/null +++ b/internal/ap/egress_test.go @@ -0,0 +1,163 @@ +package ap + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" +) + +func TestIsBlockedIP(t *testing.T) { + blocked := []string{ + "127.0.0.1", "::1", // loopback + "10.0.0.5", "192.168.1.1", "172.16.0.1", // RFC1918 private + "169.254.169.254", // cloud metadata (link-local) + "169.254.0.1", // link-local + "fe80::1", // link-local v6 + "fc00::1", "fd12:3456::1", // unique-local v6 + "224.0.0.1", "ff02::1", // multicast + "0.0.0.0", "::", // unspecified + } + for _, s := range blocked { + ip := net.ParseIP(s) + require.NotNil(t, ip, s) + assert.True(t, isBlockedIP(ip), "%s must be blocked", s) + } + + allowed := []string{"8.8.8.8", "1.1.1.1", "93.184.216.34", "2606:2800:220:1::1"} + for _, s := range allowed { + ip := net.ParseIP(s) + require.NotNil(t, ip, s) + assert.False(t, isBlockedIP(ip), "%s must be allowed", s) + } +} + +func TestCheckURL_SchemeAndUserinfo(t *testing.T) { + g := newEgressGuard(false) + cases := []string{ + "ftp://example.com/x", // non-http scheme + "file:///etc/passwd", // non-http scheme + "https://user:pass@example.com", // userinfo + "https://127.0.0.1/x", // blocked IP literal + "https://169.254.169.254/x", // metadata literal + } + for _, raw := range cases { + u, err := url.Parse(raw) + require.NoError(t, err, raw) + assert.Error(t, g.checkURL(u), "%s must be refused", raw) + } + + // A public hostname passes the URL-level check (its resolved IP is checked + // later at dial time). + u, _ := url.Parse("https://lemmy.world/c/technology") + assert.NoError(t, g.checkURL(u)) + + // With the guard relaxed (dev/test), private literals are allowed. + off := newEgressGuard(true) + u, _ = url.Parse("https://127.0.0.1:8443/x") + assert.NoError(t, off.checkURL(u)) +} + +// TestFetch_BlocksLoopbackWhenGuardOn: with the guard on (production default), +// a fetch to a 127.0.0.1 httptest server is refused before any request. This +// is the initial-request egress check. +func TestFetch_BlocksLoopbackWhenGuardOn(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + // Guard ON: do NOT set AllowPrivateAddresses. + client := NewClient(ClientOptions{UserAgent: "tidepool-test/0", Signer: NewSigner(testKeyID, testRSAKey(t))}) + _, err := client.FetchObject(context.Background(), server.URL+"/1") + require.Error(t, err) + assert.True(t, errors.IsValidation(err), "loopback fetch must be an egress validation error, got %v", err) + assert.Contains(t, err.Error(), "egress") +} + +// TestFetch_AllowsLoopbackWhenGuardOff: the test/dev override lets the same +// fetch through. +func TestFetch_AllowsLoopbackWhenGuardOff(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"type":"Note","id":"https://x.example/1"}`)) + })) + defer server.Close() + + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + Signer: NewSigner(testKeyID, testRSAKey(t)), + AllowPrivateAddresses: true, + }) + obj, err := client.FetchObject(context.Background(), server.URL+"/1") + require.NoError(t, err) + assert.Equal(t, "https://x.example/1", obj.ID) +} + +// TestDialGuard_BlocksResolvedMetadata proves the DNS-rebinding / redirect +// defense: a hostname (which passes the URL-level check because it is not an +// IP literal) that RESOLVES to the cloud-metadata address is refused at dial +// time, when the actually-connected IP is validated. Every request — the +// initial one AND each redirect hop — dials through exactly this path. +func TestDialGuard_BlocksResolvedMetadata(t *testing.T) { + g := newEgressGuard(false) + g.lookupIPAddr = func(_ context.Context, _ string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.IPv4(169, 254, 169, 254)}}, nil + } + dialed := false + base := func(_ context.Context, _, _ string) (net.Conn, error) { + dialed = true + return stubConn{}, nil + } + _, err := g.dialContext(base)(context.Background(), "tcp", "metadata.attacker.test:80") + require.Error(t, err, "a hostname resolving to metadata must be refused at dial time") + assert.False(t, dialed, "the guard must not dial a blocked resolved address") +} + +// TestDialGuard_BlocksResolvedPrivate covers a rebind to an RFC1918 address. +func TestDialGuard_BlocksResolvedPrivate(t *testing.T) { + g := newEgressGuard(false) + g.lookupIPAddr = func(_ context.Context, _ string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.IPv4(10, 0, 0, 5)}}, nil + } + base := func(_ context.Context, _, _ string) (net.Conn, error) { return stubConn{}, nil } + _, err := g.dialContext(base)(context.Background(), "tcp", "internal.attacker.test:80") + require.Error(t, err) +} + +// TestDialGuard_DialsValidatedIP: a hostname resolving to a public IP is +// dialed by its resolved IP literal (not the hostname), defeating a rebind +// between the DNS answer and the connect. +func TestDialGuard_DialsValidatedIP(t *testing.T) { + g := newEgressGuard(false) + g.lookupIPAddr = func(_ context.Context, _ string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.IPv4(93, 184, 216, 34)}}, nil + } + var dialedAddr string + base := func(_ context.Context, _, addr string) (net.Conn, error) { + dialedAddr = addr + return stubConn{}, nil + } + _, err := g.dialContext(base)(context.Background(), "tcp", "example.test:443") + require.NoError(t, err) + assert.Equal(t, "93.184.216.34:443", dialedAddr, + "the guard must dial the validated IP literal, not the hostname") +} + +// TestDialGuard_IPLiteralAddr validates a blocked IP-literal address passed +// straight to the dialer (no resolution step). +func TestDialGuard_IPLiteralAddr(t *testing.T) { + g := newEgressGuard(false) + base := func(_ context.Context, _, _ string) (net.Conn, error) { return stubConn{}, nil } + _, err := g.dialContext(base)(context.Background(), "tcp", "169.254.169.254:80") + require.Error(t, err, "a blocked IP literal must be refused at dial time") +} + +// stubConn is a no-op net.Conn so the dial guard tests never touch the network. +type stubConn struct{ net.Conn } diff --git a/internal/ap/httpsig.go b/internal/ap/httpsig.go new file mode 100644 index 0000000..339d789 --- /dev/null +++ b/internal/ap/httpsig.go @@ -0,0 +1,451 @@ +// This file implements draft-cavage HTTP signatures (the flavor the +// fediverse actually speaks), exactly as Lemmy validates them. +// +// What Lemmy requires, from LemmyNet/activitypub-federation-rust +// src/http_signatures.rs (the library Lemmy federates with): +// +// - Verification uses http_signature_normalization Config::new() +// .set_expiration(EXPIRES_AFTER).require_digest(), where EXPIRES_AFTER is +// one hour. require_digest() marks the Digest header REQUIRED among the +// signed headers for EVERY verified request — including signed GETs (the +// crate comment says digest "doesn't make sense for GET", but Lemmy +// applies one shared config to inbox POSTs and authorized fetches alike). +// We therefore always send a Digest header, computed over the empty body +// for GETs, and always include it in the signature. bridgy-fed does the +// same (HTTP_SIG_HEADERS = ('Date', 'Host', 'Digest', '(request-target)') +// for both GET and POST) and interoperates with Lemmy in production. +// - Signatures are RSA PKCS#1 v1.5 over SHA-256 (Pkcs1v15Sign::new::). +// Lemmy requires RSA actor keys; this is the bridge's AP-side key, +// distinct from the atproto secp256k1 repo keys. +// - keyId must be "{actorID}#main-key" — Lemmy's signing_actor() extracts +// the actor id with the regex keyId="([^"]+)#([^"]+)". +// - Lemmy's own outbound signatures cover +// "(request-target) content-type date digest host" (see the crate's +// test_sign fixture); our Verify accepts any signed-header set that +// includes (request-target) and date, and requires digest on requests +// with a body. +// +// We sign "(request-target) host date digest" plus, on requests with a body, +// "content-type". The signature algorithm parameter is emitted as +// "rsa-sha256"; on verification "hs2019" is treated as rsa-sha256, matching +// bridgy-fed's compat note (Mastodon emits hs2019 but still means RSA-SHA256). + +package ap + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "fmt" + "net/http" + "strings" + "time" + + "tidepool/internal/errors" +) + +// ServiceKeyBits is the RSA modulus size for generated AP keys, matching +// Lemmy's generate_actor_keypair (2048). +const ServiceKeyBits = 2048 + +// maxDateSkew is how far a request's Date header may deviate from now before +// verification fails. Lemmy tolerates one hour (EXPIRES_AFTER); we match it. +const maxDateSkew = time.Hour + +// GenerateRSAKey generates a new 2048-bit RSA keypair for AP signing. +func GenerateRSAKey() (*rsa.PrivateKey, error) { + key, err := rsa.GenerateKey(rand.Reader, ServiceKeyBits) + if err != nil { + return nil, fmt.Errorf("ap: generate RSA key: %w", err) + } + return key, nil +} + +// EncodePrivateKeyPEM encodes an RSA private key as PKCS#8 PEM, the format +// Lemmy uses for its own keys. +func EncodePrivateKeyPEM(key *rsa.PrivateKey) ([]byte, error) { + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return nil, fmt.Errorf("ap: marshal private key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), nil +} + +// ParsePrivateKeyPEM decodes a PKCS#8 or PKCS#1 PEM RSA private key. +func ParsePrivateKeyPEM(pemBytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil, fmt.Errorf("ap: private key PEM: no PEM block found") + } + if parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes); err == nil { + rsaKey, ok := parsed.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("ap: private key PEM: not an RSA key (%T)", parsed) + } + return rsaKey, nil + } + rsaKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("ap: parse private key PEM: %w", err) + } + return rsaKey, nil +} + +// EncodePublicKeyPEM encodes an RSA public key as SPKI PEM ("PUBLIC KEY"), +// the format every fediverse implementation publishes in publicKeyPem. +func EncodePublicKeyPEM(key *rsa.PublicKey) ([]byte, error) { + der, err := x509.MarshalPKIXPublicKey(key) + if err != nil { + return nil, fmt.Errorf("ap: marshal public key: %w", err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der}), nil +} + +// ParsePublicKeyPEM decodes an SPKI ("PUBLIC KEY") or PKCS#1 ("RSA PUBLIC +// KEY") PEM RSA public key, as found in actors' publicKeyPem fields. +func ParsePublicKeyPEM(pemBytes []byte) (*rsa.PublicKey, error) { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil, fmt.Errorf("ap: public key PEM: no PEM block found") + } + if parsed, err := x509.ParsePKIXPublicKey(block.Bytes); err == nil { + rsaKey, ok := parsed.(*rsa.PublicKey) + if !ok { + return nil, fmt.Errorf("ap: public key PEM: not an RSA key (%T)", parsed) + } + return rsaKey, nil + } + rsaKey, err := x509.ParsePKCS1PublicKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("ap: parse public key PEM: %w", err) + } + return rsaKey, nil +} + +// Signer signs outbound HTTP requests with the bridge's AP-side RSA key. +type Signer struct { + keyID string + privateKey *rsa.PrivateKey +} + +// NewSigner creates a Signer. keyID is the full key id published in the +// actor document, e.g. "https://bridge.example/actor#main-key". +func NewSigner(keyID string, privateKey *rsa.PrivateKey) *Signer { + return &Signer{keyID: keyID, privateKey: privateKey} +} + +// KeyID returns the signer's key id. +func (s *Signer) KeyID() string { return s.keyID } + +// SignRequest adds Date, Host, Digest, and Signature headers to req, signing +// over "(request-target) host date digest" (plus content-type when the +// request carries a body). body must be the exact request payload (nil for +// GET). The caller sets req.Body itself; SignRequest only reads body to +// compute the digest. +func (s *Signer) SignRequest(req *http.Request, body []byte) error { + now := time.Now().UTC().Format(http.TimeFormat) + req.Header.Set("Date", now) + // Go sends req.Host (or req.URL.Host) as the Host header; mirror it in + // the header map so the signing string and the wire agree. + host := req.Host + if host == "" { + host = req.URL.Host + } + req.Host = host + + digest := "SHA-256=" + base64.StdEncoding.EncodeToString(sha256Sum(body)) + req.Header.Set("Digest", digest) + + signedHeaders := []string{"(request-target)", "host", "date", "digest"} + if len(body) > 0 { + // Lemmy signs content-type on its own POSTs; include it for parity. + if req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", ContentTypeActivityJSON) + } + signedHeaders = append(signedHeaders, "content-type") + } + + signingString := buildSigningString(req, host, signedHeaders) + hashed := sha256.Sum256([]byte(signingString)) + signature, err := rsa.SignPKCS1v15(rand.Reader, s.privateKey, crypto.SHA256, hashed[:]) + if err != nil { + return fmt.Errorf("ap: sign request: %w", err) + } + + req.Header.Set("Signature", fmt.Sprintf( + `keyId="%s",algorithm="rsa-sha256",headers="%s",signature="%s"`, + s.keyID, + strings.Join(signedHeaders, " "), + base64.StdEncoding.EncodeToString(signature), + )) + return nil +} + +func sha256Sum(data []byte) []byte { + sum := sha256.Sum256(data) + return sum[:] +} + +// buildSigningString assembles the draft-cavage signing string for the given +// header list, using req's method, path, and headers. +func buildSigningString(req *http.Request, host string, headers []string) string { + lines := make([]string, 0, len(headers)) + for _, header := range headers { + switch header { + case "(request-target)": + target := req.URL.RequestURI() + lines = append(lines, fmt.Sprintf("(request-target): %s %s", + strings.ToLower(req.Method), target)) + case "host": + lines = append(lines, "host: "+host) + default: + lines = append(lines, header+": "+req.Header.Get(header)) + } + } + return strings.Join(lines, "\n") +} + +// KeyResolver resolves a signature keyId to the RSA public key the signing +// actor publishes. The AP client implements it (fetch actor, read +// publicKey.publicKeyPem) with caching; tests inject stubs. +type KeyResolver interface { + // ResolveKey returns the public key for keyID and the id of the actor + // that owns it. + ResolveKey(ctx context.Context, keyID string) (key *rsa.PublicKey, ownerID string, err error) +} + +// freshKeyResolver is an optional extension of KeyResolver: it re-resolves a +// key bypassing any positive cache. The Verifier uses it for a single +// retry-with-fresh-key when a cached key fails to verify a signature — the +// remote may have rotated its key while we still held the old one in cache, +// which is otherwise indistinguishable from a forgery and would blackhole all +// of that actor's deliveries for the full cache TTL. The Client implements it +// via ResolveKeyFresh (task 06 relies on this path). +type freshKeyResolver interface { + ResolveKeyFresh(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) +} + +// KeyResolverFunc adapts a function to the KeyResolver interface. +type KeyResolverFunc func(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) + +// ResolveKey calls f. +func (f KeyResolverFunc) ResolveKey(ctx context.Context, keyID string) (*rsa.PublicKey, string, error) { + return f(ctx, keyID) +} + +// SignatureError reports a failed HTTP signature verification. It unwraps to +// ErrInvalidInput so task 06 can reject unauthenticated deliveries uniformly. +type SignatureError struct { + Reason string +} + +func (e SignatureError) Error() string { + return "ap: http signature verification failed: " + e.Reason +} + +// Unwrap makes errors.Is(err, errors.ErrInvalidInput) true. +func (e SignatureError) Unwrap() error { return errors.ErrInvalidInput } + +// Verifier checks inbound draft-cavage HTTP signatures (task 06 wires it to +// the inbox). Keys are resolved through a KeyResolver. +type Verifier struct { + resolver KeyResolver + // now is stubbed in tests. + now func() time.Time +} + +// NewVerifier creates a Verifier that resolves signing keys with resolver. +func NewVerifier(resolver KeyResolver) *Verifier { + return &Verifier{resolver: resolver, now: time.Now} +} + +// Verify checks the request's HTTP signature and returns the AP actor id +// that owns the signing key. body must be the request payload already read +// by the caller (the inbox buffers it anyway to parse the activity). +// +// Validation rules (matching Lemmy's verifier plus the digest rules +// bridgy-fed applies): +// - a Signature header must be present, with keyId, headers, signature; +// - "(request-target)" and "date" must be among the signed headers; +// - the Date header must be within maxDateSkew of now (Lemmy: one hour); +// - requests with a body must have a Digest header, it must be signed, +// and it must match SHA-256(body); +// - algorithm, when present, must be rsa-sha256 or hs2019 (treated as +// rsa-sha256, the bridgy-fed compat rule). +// +// All verification failures unwrap to errors.ErrInvalidInput via +// SignatureError; key-resolution failures propagate as-is. +func (v *Verifier) Verify(ctx context.Context, req *http.Request, body []byte) (actorID string, err error) { + header := req.Header.Get("Signature") + if header == "" { + return "", SignatureError{Reason: "missing Signature header"} + } + fields := parseSignatureHeader(header) + keyID := fields["keyid"] + signatureB64 := fields["signature"] + if keyID == "" || signatureB64 == "" { + return "", SignatureError{Reason: "Signature header missing keyId or signature"} + } + switch algorithm := fields["algorithm"]; algorithm { + case "", "rsa-sha256", "hs2019": + // hs2019 in the wild means "figure it out"; every fediverse + // implementation that sends it uses RSA-SHA256 (bridgy-fed applies + // the same mapping). + default: + return "", SignatureError{Reason: fmt.Sprintf("unsupported algorithm %q", algorithm)} + } + + signedHeaders := strings.Fields(strings.ToLower(fields["headers"])) + if len(signedHeaders) == 0 { + // Per draft-cavage the default is "date" alone. + signedHeaders = []string{"date"} + } + if !containsString(signedHeaders, "(request-target)") { + return "", SignatureError{Reason: "(request-target) not signed"} + } + if !containsString(signedHeaders, "date") { + return "", SignatureError{Reason: "date not signed"} + } + // Host must be signed: it binds the signature to the request's authority. + // Every fediverse signer (Lemmy, Mastodon, bridgy-fed) signs host, and + // without it a signature captured for one host could be replayed against + // another. Signer always includes it. + if !containsString(signedHeaders, "host") { + return "", SignatureError{Reason: "host not signed"} + } + + // Date skew. + date, err := http.ParseTime(req.Header.Get("Date")) + if err != nil { + return "", SignatureError{Reason: "missing or malformed Date header"} + } + if skew := v.now().Sub(date); skew > maxDateSkew || skew < -maxDateSkew { + return "", SignatureError{Reason: fmt.Sprintf("date skew %s exceeds %s", skew.Round(time.Second), maxDateSkew)} + } + + // Digest: required and checked whenever the request carries a body + // (POST inbox deliveries). GETs without a body may omit it. + if len(body) > 0 { + if !containsString(signedHeaders, "digest") { + return "", SignatureError{Reason: "digest not signed on request with body"} + } + digest := req.Header.Get("Digest") + expected := base64.StdEncoding.EncodeToString(sha256Sum(body)) + if !digestMatches(digest, expected) { + return "", SignatureError{Reason: "digest mismatch"} + } + } + + key, ownerID, err := v.resolver.ResolveKey(ctx, keyID) + if err != nil { + return "", fmt.Errorf("ap: resolve signing key %q: %w", keyID, err) + } + + host := req.Host + if host == "" { + host = req.URL.Host + } + signingString := buildSigningString(req, host, signedHeaders) + signature, err := base64.StdEncoding.DecodeString(signatureB64) + if err != nil { + return "", SignatureError{Reason: "signature is not valid base64"} + } + hashed := sha256.Sum256([]byte(signingString)) + if rsa.VerifyPKCS1v15(key, crypto.SHA256, hashed[:], signature) == nil { + return ownerID, nil + } + // The (possibly cached) key did not verify. If the resolver can bypass its + // cache, try exactly once with a freshly-fetched key: a remote key rotation + // leaves a stale key in cache that fails every signature until it expires. + // We only re-verify when the fresh key actually differs, so a forged + // signature does not gain extra verification attempts. + if fresh, ok := v.resolver.(freshKeyResolver); ok { + freshKey, freshOwner, ferr := fresh.ResolveKeyFresh(ctx, keyID) + if ferr == nil && !freshKey.Equal(key) && + rsa.VerifyPKCS1v15(freshKey, crypto.SHA256, hashed[:], signature) == nil { + return freshOwner, nil + } + } + return "", SignatureError{Reason: "signature does not verify"} +} + +// digestMatches compares a Digest header (possibly multi-valued, e.g. +// "SHA-256=xxx,sha-512=yyy") against the expected SHA-256 base64 value. +func digestMatches(header, expectedB64 string) bool { + for _, part := range strings.Split(header, ",") { + part = strings.TrimSpace(part) + algorithm, value, found := strings.Cut(part, "=") + if !found { + continue + } + if strings.EqualFold(algorithm, "SHA-256") && value == expectedB64 { + return true + } + } + return false +} + +// parseSignatureHeader parses the comma-separated key="value" pairs of a +// draft-cavage Signature header. Keys are lower-cased. Values may contain +// commas (base64 never does, but headers lists don't either — split on +// `",` boundaries would be fragile, so scan properly). +func parseSignatureHeader(header string) map[string]string { + fields := make(map[string]string) + rest := header + for rest != "" { + rest = strings.TrimLeft(rest, " \t,") + equals := strings.IndexByte(rest, '=') + if equals < 0 { + break + } + key := strings.ToLower(strings.TrimSpace(rest[:equals])) + rest = rest[equals+1:] + var value string + if strings.HasPrefix(rest, `"`) { + closing := strings.IndexByte(rest[1:], '"') + if closing < 0 { + value = rest[1:] + rest = "" + } else { + value = rest[1 : 1+closing] + rest = rest[closing+2:] + } + } else { + comma := strings.IndexByte(rest, ',') + if comma < 0 { + value = rest + rest = "" + } else { + value = rest[:comma] + rest = rest[comma+1:] + } + value = strings.TrimSpace(value) + } + fields[key] = value + } + return fields +} + +func containsString(list []string, want string) bool { + for _, v := range list { + if v == want { + return true + } + } + return false +} + +// ActorIDFromKeyID strips the fragment from a keyId, yielding the actor id +// ("https://host/u/alice#main-key" → "https://host/u/alice"). Lemmy resolves +// keys the same way. +func ActorIDFromKeyID(keyID string) string { + if hash := strings.IndexByte(keyID, '#'); hash >= 0 { + return keyID[:hash] + } + return keyID +} diff --git a/internal/ap/httpsig_test.go b/internal/ap/httpsig_test.go new file mode 100644 index 0000000..93b8959 --- /dev/null +++ b/internal/ap/httpsig_test.go @@ -0,0 +1,308 @@ +package ap + +import ( + "context" + "crypto/rsa" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" +) + +// testKey generates a throwaway RSA key once per test binary — key +// generation is the slow part of these tests. +var testKeyOnce = struct { + key *rsa.PrivateKey + err error + ok bool +}{} + +func testRSAKey(t *testing.T) *rsa.PrivateKey { + t.Helper() + if !testKeyOnce.ok { + testKeyOnce.key, testKeyOnce.err = GenerateRSAKey() + testKeyOnce.ok = true + } + require.NoError(t, testKeyOnce.err) + return testKeyOnce.key +} + +const testActorID = "https://bridge.example/actor" +const testKeyID = testActorID + "#main-key" + +func staticResolver(key *rsa.PublicKey) KeyResolver { + return KeyResolverFunc(func(_ context.Context, keyID string) (*rsa.PublicKey, string, error) { + return key, ActorIDFromKeyID(keyID), nil + }) +} + +func TestSignRequest_GETHeaderSet(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + + req := httptest.NewRequest(http.MethodGet, "https://lemmy.world/c/technology?page=2", nil) + require.NoError(t, signer.SignRequest(req, nil)) + + // The exact header set Lemmy's verifier accepts for GETs. Digest MUST be + // present and signed even on GET: Lemmy's verify_signature_inner applies + // require_digest() to every request (activitypub-federation-rust + // src/http_signatures.rs), and bridgy-fed ships the same set. + assert.NotEmpty(t, req.Header.Get("Date")) + _, err := http.ParseTime(req.Header.Get("Date")) + assert.NoError(t, err, "Date must be a valid HTTP date") + // SHA-256 of the empty body. + assert.Equal(t, "SHA-256=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", req.Header.Get("Digest")) + + fields := parseSignatureHeader(req.Header.Get("Signature")) + assert.Equal(t, testKeyID, fields["keyid"]) + assert.Equal(t, "rsa-sha256", fields["algorithm"]) + assert.Equal(t, "(request-target) host date digest", fields["headers"]) + assert.NotEmpty(t, fields["signature"]) + + // The signing string must include the query in (request-target). + assert.Equal(t, "https://lemmy.world/c/technology?page=2", req.URL.String()) +} + +func TestSignRequest_POSTHeaderSet(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + body := []byte(`{"type":"Follow"}`) + + req := httptest.NewRequest(http.MethodPost, "https://lemmy.world/c/technology/inbox", nil) + req.Header.Set("Content-Type", ContentTypeActivityJSON) + require.NoError(t, signer.SignRequest(req, body)) + + fields := parseSignatureHeader(req.Header.Get("Signature")) + assert.Equal(t, "(request-target) host date digest content-type", fields["headers"], + "POSTs sign content-type too, matching Lemmy's own outbound set") + assert.NotEmpty(t, req.Header.Get("Digest")) + assert.NotEqual(t, "SHA-256=47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", req.Header.Get("Digest"), + "POST digest covers the body, not the empty string") +} + +func TestSignVerifyRoundTrip_GET(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://lemmy.world/post/49131386", nil) + require.NoError(t, signer.SignRequest(req, nil)) + + actorID, err := verifier.Verify(context.Background(), req, nil) + require.NoError(t, err) + assert.Equal(t, testActorID, actorID) +} + +func TestSignVerifyRoundTrip_POST(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + body := []byte(`{"type":"Announce","id":"https://lemmy.world/activities/announce/x"}`) + + req := httptest.NewRequest(http.MethodPost, "https://bridge.example/inbox", nil) + require.NoError(t, signer.SignRequest(req, body)) + + actorID, err := verifier.Verify(context.Background(), req, body) + require.NoError(t, err) + assert.Equal(t, testActorID, actorID) +} + +func TestVerify_RejectsTamperedBody(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + body := []byte(`{"type":"Announce"}`) + + req := httptest.NewRequest(http.MethodPost, "https://bridge.example/inbox", nil) + require.NoError(t, signer.SignRequest(req, body)) + + _, err := verifier.Verify(context.Background(), req, []byte(`{"type":"Delete"}`)) + require.Error(t, err) + assert.True(t, errors.IsValidation(err), "signature failures unwrap to ErrInvalidInput, got %v", err) + assert.Contains(t, err.Error(), "digest") +} + +func TestVerify_RejectsTamperedTarget(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + // Replay the signed headers against a different path. + replayed := httptest.NewRequest(http.MethodGet, "https://bridge.example/inbox", nil) + replayed.Header = req.Header.Clone() + replayed.Host = req.Host + + _, err := verifier.Verify(context.Background(), replayed, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not verify") +} + +func TestVerify_RejectsWrongKey(t *testing.T) { + key := testRSAKey(t) + otherKey, err := GenerateRSAKey() + require.NoError(t, err) + + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&otherKey.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + + _, err = verifier.Verify(context.Background(), req, nil) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestVerify_RejectsDateSkew(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + // Pretend the request arrives 90 minutes later — past Lemmy's one-hour + // window. + verifier.now = func() time.Time { return time.Now().Add(90 * time.Minute) } + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + + _, err := verifier.Verify(context.Background(), req, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "skew") +} + +func TestVerify_RequiresSignature(t *testing.T) { + key := testRSAKey(t) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodPost, "https://bridge.example/inbox", nil) + _, err := verifier.Verify(context.Background(), req, []byte(`{}`)) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestVerify_RequiresDigestOnBody(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + body := []byte(`{"type":"Like"}`) + + // Sign as GET (no digest in signed set)… + req := httptest.NewRequest(http.MethodPost, "https://bridge.example/inbox", nil) + require.NoError(t, signer.SignRequest(req, nil)) + // …then claim a body: must be rejected because digest isn't signed over + // the actual payload. + _, err := verifier.Verify(context.Background(), req, body) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestVerify_RequiresHostSigned(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + // Strip host from the signed header set (still present in the header map). + sig := req.Header.Get("Signature") + sig = strings.Replace(sig, "(request-target) host date digest", "(request-target) date digest", 1) + req.Header.Set("Signature", sig) + + _, err := verifier.Verify(context.Background(), req, nil) + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) + assert.Contains(t, err.Error(), "host") +} + +func TestVerify_AcceptsHS2019Algorithm(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + // Mastodon labels RSA-SHA256 signatures "hs2019"; the bytes are the same. + req.Header.Set("Signature", strings.Replace(req.Header.Get("Signature"), + `algorithm="rsa-sha256"`, `algorithm="hs2019"`, 1)) + + _, err := verifier.Verify(context.Background(), req, nil) + require.NoError(t, err) +} + +func TestVerify_RejectsUnknownAlgorithm(t *testing.T) { + key := testRSAKey(t) + signer := NewSigner(testKeyID, key) + verifier := NewVerifier(staticResolver(&key.PublicKey)) + + req := httptest.NewRequest(http.MethodGet, "https://bridge.example/actor", nil) + require.NoError(t, signer.SignRequest(req, nil)) + req.Header.Set("Signature", strings.Replace(req.Header.Get("Signature"), + `algorithm="rsa-sha256"`, `algorithm="ed25519"`, 1)) + + _, err := verifier.Verify(context.Background(), req, nil) + require.Error(t, err) +} + +func TestParseSignatureHeader(t *testing.T) { + // Real-world shape (Lemmy emits exactly this ordering). + fields := parseSignatureHeader( + `keyId="https://lemmy.world/c/technology#main-key",algorithm="rsa-sha256",` + + `headers="(request-target) content-type date digest host",signature="c2ln"`) + assert.Equal(t, "https://lemmy.world/c/technology#main-key", fields["keyid"]) + assert.Equal(t, "rsa-sha256", fields["algorithm"]) + assert.Equal(t, "(request-target) content-type date digest host", fields["headers"]) + assert.Equal(t, "c2ln", fields["signature"]) + + // Spacing and unquoted values tolerated. + fields = parseSignatureHeader(`keyId="k" , algorithm=hs2019, signature="s"`) + assert.Equal(t, "k", fields["keyid"]) + assert.Equal(t, "hs2019", fields["algorithm"]) + assert.Equal(t, "s", fields["signature"]) +} + +func TestDigestMatches(t *testing.T) { + assert.True(t, digestMatches("SHA-256=abc", "abc")) + assert.True(t, digestMatches("sha-256=abc", "abc"), "algorithm name is case-insensitive") + assert.True(t, digestMatches("SHA-512=zzz, SHA-256=abc", "abc"), "multi-valued digest headers") + assert.False(t, digestMatches("SHA-256=xyz", "abc")) + assert.False(t, digestMatches("", "abc")) +} + +func TestKeyPEMRoundTrip(t *testing.T) { + key := testRSAKey(t) + + privatePEM, err := EncodePrivateKeyPEM(key) + require.NoError(t, err) + assert.Contains(t, string(privatePEM), "BEGIN PRIVATE KEY") + parsedPrivate, err := ParsePrivateKeyPEM(privatePEM) + require.NoError(t, err) + assert.True(t, key.Equal(parsedPrivate)) + + publicPEM, err := EncodePublicKeyPEM(&key.PublicKey) + require.NoError(t, err) + assert.Contains(t, string(publicPEM), "BEGIN PUBLIC KEY") + parsedPublic, err := ParsePublicKeyPEM(publicPEM) + require.NoError(t, err) + assert.True(t, key.PublicKey.Equal(parsedPublic)) +} + +func TestParsePublicKeyPEM_LemmyFixture(t *testing.T) { + // The publicKeyPem published by a real Lemmy actor must parse. + person := parseFixture(t, "person_lemmy_world.json") + require.NotNil(t, person.PublicKey) + key, err := ParsePublicKeyPEM([]byte(person.PublicKey.PublicKeyPem)) + require.NoError(t, err) + assert.Equal(t, ServiceKeyBits, key.Size()*8, "Lemmy actor keys are RSA-2048") +} + +func TestActorIDFromKeyID(t *testing.T) { + assert.Equal(t, "https://lemmy.world/u/alice", ActorIDFromKeyID("https://lemmy.world/u/alice#main-key")) + assert.Equal(t, "https://lemmy.world/u/alice", ActorIDFromKeyID("https://lemmy.world/u/alice")) +} diff --git a/internal/ap/live_smoke_test.go b/internal/ap/live_smoke_test.go new file mode 100644 index 0000000..c62cff7 --- /dev/null +++ b/internal/ap/live_smoke_test.go @@ -0,0 +1,64 @@ +package ap + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestLiveSmoke is the manual smoke test from the task-02 definition of +// done: signed GET, WebFinger, and collection paging against a live Lemmy +// instance. It talks to the real lemmy.world, so it is opt-in only — +// CI and normal runs use the recorded fixtures. Run it with: +// +// TIDEPOOL_LIVE_SMOKE=1 go test ./internal/ap/ -run TestLiveSmoke -v +func TestLiveSmoke(t *testing.T) { + if testing.Short() || !liveSmokeEnabled() { + t.Skip("TIDEPOOL_LIVE_SMOKE not set; skipping network smoke test") + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // The signing key is throwaway: lemmy.world does not run authorized + // fetch, so the signature is not verified server-side — the smoke proves + // the signed request shape is accepted end to end. + key := testRSAKey(t) + client := NewClient(ClientOptions{ + UserAgent: "tidepool/0.1 (+https://tidepool.invalid; task-02 smoke)", + Signer: NewSigner("https://tidepool.invalid/actor#main-key", key), + }) + + // WebFinger: community handle → Group actor URL. + actorURL, err := client.ResolveHandle(ctx, "!technology@lemmy.world") + require.NoError(t, err) + assert.Equal(t, "https://lemmy.world/c/technology", actorURL) + + // Signed GET: Group actor. + group, err := client.FetchActor(ctx, actorURL) + require.NoError(t, err) + assert.Equal(t, TypeGroup, group.Type) + require.NotNil(t, group.PublicKey) + + handle, err := ActorHandle(group) + require.NoError(t, err) + assert.Equal(t, "!technology@lemmy.world", handle) + + // Collection paging over the live outbox (single inline page on Lemmy). + items := 0 + err = client.FetchCollection(ctx, group.Outbox, func(item *Object) error { + items++ + assert.Equal(t, TypeAnnounce, item.Type) + return nil + }) + require.NoError(t, err) + assert.Greater(t, items, 0, "live outbox should contain announces") + t.Logf("live smoke OK: actor=%s outbox items=%d", group.ID, items) +} + +func liveSmokeEnabled() bool { + return os.Getenv("TIDEPOOL_LIVE_SMOKE") != "" +} diff --git a/internal/ap/service_actor.go b/internal/ap/service_actor.go new file mode 100644 index 0000000..e3037ff --- /dev/null +++ b/internal/ap/service_actor.go @@ -0,0 +1,136 @@ +package ap + +import ( + "context" + "crypto/rsa" + "encoding/json" + "fmt" + + "tidepool/internal/errors" + "tidepool/internal/store" +) + +// ServiceKeyName is the service_keys row holding the bridge's AP-side RSA +// private key. +const ServiceKeyName = "service-actor" + +// ServiceActorPath is the path the bridge's actor document will be served +// from (the HTTP route itself lands with the inbox in task 06). +const ServiceActorPath = "/actor" + +// serviceActorContext is the JSON-LD context for the service actor document: +// core AS2 plus the security vocabulary that defines publicKey. +const serviceActorContext = `["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1"]` + +// ServiceActor is the bridge's own AP identity: an Application actor whose +// RSA key signs every outbound request (fetches and Follows). This is the +// AP-side interop key — entirely distinct from the atproto secp256k1 repo +// keys task 03 mints. +type ServiceActor struct { + // ID is the actor's canonical id, https://{hostname}/actor. + ID string + // Hostname is the bridge's public hostname (config.BridgeHostname). + Hostname string + // Key is the actor's RSA private key. + Key *rsa.PrivateKey +} + +// LoadOrCreateServiceActor returns the bridge's service actor, loading its +// RSA key from the service_keys store or generating and persisting one on +// first run. Losing a concurrent-bootstrap insert race falls back to the +// winner's key, so every process converges on the same keypair. +func LoadOrCreateServiceActor(ctx context.Context, keys store.ServiceKeys, hostname string) (*ServiceActor, error) { + if hostname == "" { + return nil, errors.NewValidationError("hostname", "must not be empty") + } + + stored, err := keys.Get(ctx, ServiceKeyName) + switch { + case err == nil: + // Existing key. + case errors.IsNotFound(err): + key, err := GenerateRSAKey() + if err != nil { + return nil, err + } + pemBytes, err := EncodePrivateKeyPEM(key) + if err != nil { + return nil, err + } + stored, err = keys.Create(ctx, ServiceKeyName, pemBytes) + if errors.IsAlreadyExists(err) { + // Another instance won the bootstrap race; use its key. + stored, err = keys.Get(ctx, ServiceKeyName) + if err != nil { + return nil, fmt.Errorf("ap: reload service key after lost create race: %w", err) + } + } else if err != nil { + return nil, fmt.Errorf("ap: persist service key: %w", err) + } + default: + return nil, fmt.Errorf("ap: load service key: %w", err) + } + + key, err := ParsePrivateKeyPEM(stored.PrivateKeyPEM) + if err != nil { + return nil, fmt.Errorf("ap: stored service key is corrupt: %w", err) + } + + return &ServiceActor{ + ID: "https://" + hostname + ServiceActorPath, + Hostname: hostname, + Key: key, + }, nil +} + +// KeyID is the id the actor document publishes for its public key and the +// keyId sent on signatures. Lemmy's key resolution splits keyId on '#', so +// the fragment must be present; #main-key matches Lemmy's own convention. +func (a *ServiceActor) KeyID() string { return a.ID + "#main-key" } + +// InboxURL is the actor's inbox (served by task 06). +func (a *ServiceActor) InboxURL() string { return "https://" + a.Hostname + "/inbox" } + +// OutboxURL is the actor's outbox. +func (a *ServiceActor) OutboxURL() string { return "https://" + a.Hostname + "/outbox" } + +// Signer returns a request signer using the actor's key. +func (a *ServiceActor) Signer() *Signer { return NewSigner(a.KeyID(), a.Key) } + +// Document builds the Application actor document served at +// https://{hostname}/actor. Lemmy requires the publicKey block (RSA, SPKI +// PEM) to accept our signed requests. +func (a *ServiceActor) Document() (*Object, error) { + publicPEM, err := EncodePublicKeyPEM(&a.Key.PublicKey) + if err != nil { + return nil, err + } + return &Object{ + Context: json.RawMessage(serviceActorContext), + ID: a.ID, + Type: TypeApplication, + PreferredUsername: a.Hostname, + Name: "Tidepool bridge", + Summary: "Bridges threadiverse communities into atproto. https://" + a.Hostname, + Inbox: a.InboxURL(), + Outbox: a.OutboxURL(), + PublicKey: &PublicKey{ + ID: a.KeyID(), + Owner: a.ID, + PublicKeyPem: string(publicPEM), + }, + }, nil +} + +// DocumentJSON renders the actor document as AS2 JSON. +func (a *ServiceActor) DocumentJSON() ([]byte, error) { + doc, err := a.Document() + if err != nil { + return nil, err + } + data, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("ap: encode service actor document: %w", err) + } + return data, nil +} diff --git a/internal/ap/service_actor_test.go b/internal/ap/service_actor_test.go new file mode 100644 index 0000000..2a6f2cf --- /dev/null +++ b/internal/ap/service_actor_test.go @@ -0,0 +1,166 @@ +package ap + +import ( + "context" + "net/http/httptest" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" + "tidepool/internal/store" +) + +// fakeServiceKeys is an in-memory store.ServiceKeys. The postgres +// implementation is covered by internal/store's real-DB tests; these tests +// exercise the bootstrap logic. +type fakeServiceKeys struct { + mu sync.Mutex + rows map[string][]byte + // createHook runs inside Create before the insert (to simulate races). + createHook func() +} + +func newFakeServiceKeys() *fakeServiceKeys { + return &fakeServiceKeys{rows: map[string][]byte{}} +} + +func (f *fakeServiceKeys) Create(_ context.Context, name string, pem []byte) (*store.ServiceKey, error) { + if f.createHook != nil { + f.createHook() + } + f.mu.Lock() + defer f.mu.Unlock() + if _, exists := f.rows[name]; exists { + return nil, errors.NewConflictError("service_key", "name", name) + } + f.rows[name] = pem + return &store.ServiceKey{ID: 1, Name: name, PrivateKeyPEM: pem}, nil +} + +func (f *fakeServiceKeys) Get(_ context.Context, name string) (*store.ServiceKey, error) { + f.mu.Lock() + defer f.mu.Unlock() + pem, ok := f.rows[name] + if !ok { + return nil, errors.NewNotFoundError("service_key", name) + } + return &store.ServiceKey{ID: 1, Name: name, PrivateKeyPEM: pem}, nil +} + +func TestLoadOrCreateServiceActor_GeneratesThenLoads(t *testing.T) { + keys := newFakeServiceKeys() + ctx := context.Background() + + first, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + require.NoError(t, err) + assert.Equal(t, "https://bridge.example/actor", first.ID) + assert.Equal(t, "https://bridge.example/actor#main-key", first.KeyID()) + assert.Equal(t, "https://bridge.example/inbox", first.InboxURL()) + require.NotNil(t, first.Key) + + // The key must have been persisted as parseable PKCS#8 PEM. + stored, err := keys.Get(ctx, ServiceKeyName) + require.NoError(t, err) + storedKey, err := ParsePrivateKeyPEM(stored.PrivateKeyPEM) + require.NoError(t, err) + assert.True(t, first.Key.Equal(storedKey)) + + // A second bootstrap loads the same key instead of generating a new one. + second, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + require.NoError(t, err) + assert.True(t, first.Key.Equal(second.Key), "restarts must reuse the persisted key") +} + +func TestLoadOrCreateServiceActor_LosesBootstrapRace(t *testing.T) { + keys := newFakeServiceKeys() + ctx := context.Background() + + // Simulate a concurrent instance winning the insert between our Get + // (not found) and Create. + winnerKey, err := GenerateRSAKey() + require.NoError(t, err) + winnerPEM, err := EncodePrivateKeyPEM(winnerKey) + require.NoError(t, err) + keys.createHook = func() { + keys.mu.Lock() + if _, exists := keys.rows[ServiceKeyName]; !exists { + keys.rows[ServiceKeyName] = winnerPEM + } + keys.mu.Unlock() + } + + actor, err := LoadOrCreateServiceActor(ctx, keys, "bridge.example") + require.NoError(t, err) + assert.True(t, winnerKey.Equal(actor.Key), + "losing the create race must converge on the winner's key") +} + +func TestLoadOrCreateServiceActor_RequiresHostname(t *testing.T) { + _, err := LoadOrCreateServiceActor(context.Background(), newFakeServiceKeys(), "") + require.Error(t, err) + assert.True(t, errors.IsValidation(err)) +} + +func TestLoadOrCreateServiceActor_CorruptKey(t *testing.T) { + keys := newFakeServiceKeys() + _, err := keys.Create(context.Background(), ServiceKeyName, []byte("not a pem")) + require.NoError(t, err) + + _, err = LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + require.Error(t, err) + assert.Contains(t, err.Error(), "corrupt") +} + +func TestServiceActorDocument(t *testing.T) { + keys := newFakeServiceKeys() + actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + require.NoError(t, err) + + docJSON, err := actor.DocumentJSON() + require.NoError(t, err) + + // The document must parse back through our own tolerant vocab and look + // like an actor Lemmy can resolve keys from. + doc, err := ParseObject(docJSON) + require.NoError(t, err) + assert.True(t, doc.IsActor()) + assert.Equal(t, TypeApplication, doc.Type) + assert.Equal(t, "https://bridge.example/actor", doc.ID) + assert.Equal(t, "https://bridge.example/inbox", doc.Inbox) + assert.NotEmpty(t, doc.PreferredUsername, "webfinger reverse resolution needs preferredUsername") + + require.NotNil(t, doc.PublicKey) + assert.Equal(t, "https://bridge.example/actor#main-key", doc.PublicKey.ID, + "keyId must carry the #main-key fragment Lemmy's regex expects") + assert.Equal(t, doc.ID, doc.PublicKey.Owner) + published, err := ParsePublicKeyPEM([]byte(doc.PublicKey.PublicKeyPem)) + require.NoError(t, err) + assert.True(t, actor.Key.PublicKey.Equal(published)) + + // The JSON-LD context must include the security vocabulary that defines + // publicKey. + assert.Contains(t, string(doc.Context), "https://w3id.org/security/v1") +} + +func TestServiceActorSigner_RoundTrip(t *testing.T) { + keys := newFakeServiceKeys() + actor, err := LoadOrCreateServiceActor(context.Background(), keys, "bridge.example") + require.NoError(t, err) + + // Resolve the verification key exactly the way a remote instance would: + // from the actor document's publicKeyPem. + doc, err := actor.Document() + require.NoError(t, err) + remoteKey, err := ParsePublicKeyPEM([]byte(doc.PublicKey.PublicKeyPem)) + require.NoError(t, err) + verifier := NewVerifier(staticResolver(remoteKey)) + + req := httptest.NewRequest("GET", "https://lemmy.world/c/technology", nil) + require.NoError(t, actor.Signer().SignRequest(req, nil)) + signerActorID, err := verifier.Verify(context.Background(), req, nil) + require.NoError(t, err) + assert.Equal(t, actor.ID, signerActorID) +} diff --git a/internal/ap/testdata/announce_create_note.json b/internal/ap/testdata/announce_create_note.json new file mode 100644 index 0000000..af16305 --- /dev/null +++ b/internal/ap/testdata/announce_create_note.json @@ -0,0 +1,57 @@ +{ + "actor": "https://lemmy.world/c/technology", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "id": "https://lemmy.zip/activities/create/8d4e2f1a-9b3c-4a7d-8e6f-1c2b3a4d5e6f", + "actor": "https://lemmy.zip/u/tixooo", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "type": "Note", + "id": "https://lemmy.zip/comment/27485395", + "attributedTo": "https://lemmy.zip/u/tixooo", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://lemmy.world/c/technology", + "https://sh.itjust.works/u/DemandtheOxfordComma" + ], + "content": "

I think it was the theme for all of human history, just the scale is different now.

\n", + "inReplyTo": "https://sh.itjust.works/comment/26248018", + "mediaType": "text/html", + "source": { + "content": "I think it was the theme for all of human history, just the scale is different now. ", + "mediaType": "text/markdown" + }, + "published": "2026-07-07T05:28:23.817032Z", + "tag": [ + { + "href": "https://sh.itjust.works/u/DemandtheOxfordComma", + "name": "@DemandtheOxfordComma@sh.itjust.works", + "type": "Mention" + } + ], + "distinguished": false, + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "attachment": [] + }, + "cc": [ + "https://lemmy.world/c/technology" + ], + "type": "Create", + "audience": "https://lemmy.world/c/technology" + }, + "cc": [ + "https://lemmy.world/c/technology/followers" + ], + "type": "Announce", + "id": "https://lemmy.world/activities/announce/create/7b8c9d0e-1f2a-3b4c-5d6e-7f8a9b0c1d2e" +} \ No newline at end of file diff --git a/internal/ap/testdata/announce_create_page_lemmy_world.json b/internal/ap/testdata/announce_create_page_lemmy_world.json new file mode 100644 index 0000000..b81a355 --- /dev/null +++ b/internal/ap/testdata/announce_create_page_lemmy_world.json @@ -0,0 +1,60 @@ +{ + "actor": "https://lemmy.world/c/technology", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "id": "https://lemmy.world/activities/create/599f915a-c61e-49fd-b590-28dc86593dc6", + "actor": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "type": "Page", + "id": "https://lemmy.world/post/49131386", + "attributedTo": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://lemmy.world/c/technology", + "https://www.w3.org/ns/activitystreams#Public" + ], + "name": "Inside the history of DRAM price-fixing lawsuits — how HBM allocations could make a difference after two decades of failed cases", + "cc": [], + "mediaType": "text/html", + "attachment": [ + { + "href": "https://www.tomshardware.com/pc-components/dram/samsung-sk-hynix-and-micron-face-a-third-dram-price-fixing-lawsuit", + "mediaType": "text/html; charset=utf-8", + "type": "Link" + } + ], + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/98840222-90ff-4a7f-83ee-6090488648d4.jpeg" + }, + "sensitive": false, + "published": "2026-07-07T03:27:37.028201Z", + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "tag": [ + { + "href": "https://lemmy.world/post/49131386", + "name": "#technology", + "type": "Hashtag" + } + ] + }, + "cc": [ + "https://lemmy.world/c/technology" + ], + "type": "Create", + "audience": "https://lemmy.world/c/technology" + }, + "cc": [ + "https://lemmy.world/c/technology/followers" + ], + "type": "Announce", + "id": "https://lemmy.world/activities/announce/create/6a91b0d9-c1e5-45d6-be8b-ce248306867e" +} \ No newline at end of file diff --git a/internal/ap/testdata/announce_like.json b/internal/ap/testdata/announce_like.json new file mode 100644 index 0000000..e7b57ba --- /dev/null +++ b/internal/ap/testdata/announce_like.json @@ -0,0 +1,17 @@ +{ + "actor": "https://lemmy.world/c/technology", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "actor": "https://lemmy.zip/u/tixooo", + "object": "https://lemmy.world/post/49131386", + "type": "Like", + "id": "https://lemmy.zip/activities/like/3f4e5d6c-7b8a-490e-b1c2-d3e4f5a6b7c8" + }, + "cc": [ + "https://lemmy.world/c/technology/followers" + ], + "type": "Announce", + "id": "https://lemmy.world/activities/announce/like/2a3b4c5d-6e7f-4809-9a0b-c1d2e3f4a5b6" +} \ No newline at end of file diff --git a/internal/ap/testdata/delete_page.json b/internal/ap/testdata/delete_page.json new file mode 100644 index 0000000..7615698 --- /dev/null +++ b/internal/ap/testdata/delete_page.json @@ -0,0 +1,13 @@ +{ + "actor": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": "https://lemmy.world/post/49131386", + "cc": [ + "https://lemmy.world/c/technology" + ], + "type": "Delete", + "id": "https://lemmy.world/activities/delete/9c8b7a6d-5e4f-4321-a0b9-c8d7e6f5a4b3", + "audience": "https://lemmy.world/c/technology" +} \ No newline at end of file diff --git a/internal/ap/testdata/group_lemmy_world.json b/internal/ap/testdata/group_lemmy_world.json new file mode 100644 index 0000000..03af96c --- /dev/null +++ b/internal/ap/testdata/group_lemmy_world.json @@ -0,0 +1,46 @@ +{ + "@context": [ + "https://join-lemmy.org/context.json", + "https://www.w3.org/ns/activitystreams" + ], + "type": "Group", + "id": "https://lemmy.world/c/technology", + "preferredUsername": "technology", + "inbox": "https://lemmy.world/c/technology/inbox", + "followers": "https://lemmy.world/c/technology/followers", + "publicKey": { + "id": "https://lemmy.world/c/technology#main-key", + "owner": "https://lemmy.world/c/technology", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwr6LKYHJz8UyVZAuSbc+\nURRbobWjq8bteM7McE9JmyJYRcNxqvvlx45hcJniIKm2gv4OKxJjKMhJ1SOYpI7e\noDVE75ua9ATu6/mvTKYOvE9T6G5VwFB9HkGjOQNNNh4Ely0Ytg60DkTr6J2unzYr\nFqedprG7xi3PRUVWE+VTv8OkCyi7rXiq5KLbU0eV4qi4C13LvqpQXh5z2GhejXkZ\ne8stX1gh5FTtHiVE/DwahFTGNWk4sq415vD/p7I1OC9Wp5m/ZRR1YcJnpUAe/29/\n/e4L+7tKwjt040E2Boi/M9qUV/JkXDmhm3IGjx5bRev588EbZNpTda0kuQzOrB6Y\n2QIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "name": "Technology", + "summary": "

This is a most excellent place for technology news and articles.

\n
\n

Our Rules

\n
\n
    \n
  1. Follow the lemmy.world rules.
    2. \n
  2. Only tech related news or articles.
    3. \n
  3. Be excellent to each other!
    4. \n
  4. Mod approved content bots can post up to 10 articles per day.
    5. \n
  5. Threads asking for personal tech support may be deleted.
    6. \n
  6. Politics threads may be removed.
    7. \n
  7. No memes allowed as posts, OK to post as comments.
    8. \n
  8. Only approved bots from the list below, this includes using AI responses and summaries. To ask if your bot can be added please contact a mod.
    9. \n
  9. Check for duplicates before posting, duplicates may be removed
    10. \n
  10. Accounts 7 days and younger will have their posts automatically removed.
    11. \n
\n
\n

Approved Bots

\n
\n\n", + "source": { + "content": "This is a [most excellent](https://media.giphy.com/media/v1.Y2lkPTc5MGI3NjExN3N0NmhuODNib3d3Nzg0OHU2bTFqMXAzNW42Y2JsOTVmenNsNG8ycSZlcD12MV9pbnRlcm5hbF9naWZfYnlfaWQmY3Q9Zw/l46CDHTqbmnGZyxKo/giphy.gif) place for technology news and articles.\n \n---\n \n## Our Rules\n \n---\n \n1. Follow the [lemmy.world rules.](https://mastodon.world/about) \n2. Only tech related news or articles.\n3. Be [excellent to each other!](https://www.youtube.com/watch?v=rph_1DODXDU)\n4. Mod approved content bots can post up to 10 articles per day.\n5. Threads asking for personal tech support may be deleted.\n6. Politics threads may be removed.\n7. No memes allowed as posts, OK to post as comments.\n8. Only approved bots from the list below, this includes using AI responses and summaries. To ask if your bot can be added please contact a mod.\n9. Check for duplicates before posting, duplicates may be removed\n10. Accounts 7 days and younger will have their posts automatically removed. \n \n---\n \n## Approved Bots\n \n---\n \n- [@L4s@lemmy.world](https://lemmy.world/u/L4s) \n- [@autotldr@lemmings.world](https://lemmings.world/u/autotldr) \n- [@PipedLinkBot@feddit.rocks](https://feddit.rocks/u/PipedLinkBot) \n- [@wikibot@lemmy.world](https://lemmy.world/u/wikibot) ", + "mediaType": "text/markdown" + }, + "icon": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/2a85bf08-5c07-4326-8dad-f549bfb2189e.png" + }, + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/be818d42-f96f-4a33-b963-2a6e35938d54.png" + }, + "sensitive": false, + "attributedTo": "https://lemmy.world/c/technology/moderators", + "postingRestrictedToMods": false, + "outbox": "https://lemmy.world/c/technology/outbox", + "endpoints": { + "sharedInbox": "https://lemmy.world/inbox" + }, + "featured": "https://lemmy.world/c/technology/featured", + "language": [ + { + "identifier": "en", + "name": "English" + } + ], + "published": "2023-06-11T02:16:17.173483Z", + "updated": "2025-12-29T20:50:24.593029Z" +} \ No newline at end of file diff --git a/internal/ap/testdata/note_lemmy_zip.json b/internal/ap/testdata/note_lemmy_zip.json new file mode 100644 index 0000000..6b8f6ea --- /dev/null +++ b/internal/ap/testdata/note_lemmy_zip.json @@ -0,0 +1,38 @@ +{ + "@context": [ + "https://join-lemmy.org/context.json", + "https://www.w3.org/ns/activitystreams" + ], + "type": "Note", + "id": "https://lemmy.zip/comment/27485395", + "attributedTo": "https://lemmy.zip/u/tixooo", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "cc": [ + "https://lemmy.world/c/technology", + "https://sh.itjust.works/u/DemandtheOxfordComma" + ], + "content": "

I think it was the theme for all of human history, just the scale is different now.

\n", + "inReplyTo": "https://sh.itjust.works/comment/26248018", + "mediaType": "text/html", + "source": { + "content": "I think it was the theme for all of human history, just the scale is different now. ", + "mediaType": "text/markdown" + }, + "published": "2026-07-07T05:28:23.817032Z", + "tag": [ + { + "href": "https://sh.itjust.works/u/DemandtheOxfordComma", + "name": "@DemandtheOxfordComma@sh.itjust.works", + "type": "Mention" + } + ], + "distinguished": false, + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "attachment": [] +} \ No newline at end of file diff --git a/internal/ap/testdata/outbox_lemmy_world.json b/internal/ap/testdata/outbox_lemmy_world.json new file mode 100644 index 0000000..aaf981b --- /dev/null +++ b/internal/ap/testdata/outbox_lemmy_world.json @@ -0,0 +1,131 @@ +{ + "@context": [ + "https://join-lemmy.org/context.json", + "https://www.w3.org/ns/activitystreams" + ], + "type": "OrderedCollection", + "id": "https://lemmy.world/c/technology/outbox", + "totalItems": 50, + "orderedItems": [ + { + "actor": "https://lemmy.world/c/technology", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "id": "https://lemmy.world/activities/create/599f915a-c61e-49fd-b590-28dc86593dc6", + "actor": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "type": "Page", + "id": "https://lemmy.world/post/49131386", + "attributedTo": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://lemmy.world/c/technology", + "https://www.w3.org/ns/activitystreams#Public" + ], + "name": "Inside the history of DRAM price-fixing lawsuits — how HBM allocations could make a difference after two decades of failed cases", + "cc": [], + "mediaType": "text/html", + "attachment": [ + { + "href": "https://www.tomshardware.com/pc-components/dram/samsung-sk-hynix-and-micron-face-a-third-dram-price-fixing-lawsuit", + "mediaType": "text/html; charset=utf-8", + "type": "Link" + } + ], + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/98840222-90ff-4a7f-83ee-6090488648d4.jpeg" + }, + "sensitive": false, + "published": "2026-07-07T03:27:37.028201Z", + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "tag": [ + { + "href": "https://lemmy.world/post/49131386", + "name": "#technology", + "type": "Hashtag" + } + ] + }, + "cc": [ + "https://lemmy.world/c/technology" + ], + "type": "Create", + "audience": "https://lemmy.world/c/technology" + }, + "cc": [ + "https://lemmy.world/c/technology/followers" + ], + "type": "Announce", + "id": "https://lemmy.world/activities/announce/create/6a91b0d9-c1e5-45d6-be8b-ce248306867e" + }, + { + "actor": "https://lemmy.world/c/technology", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "id": "https://lemmy.world/activities/create/6f26283b-505a-42de-afd5-bd3981df830f", + "actor": "https://lemmy.world/u/Aweigh", + "to": [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "object": { + "type": "Page", + "id": "https://lemmy.world/post/49122698", + "attributedTo": "https://lemmy.world/u/Aweigh", + "to": [ + "https://lemmy.world/c/technology", + "https://www.w3.org/ns/activitystreams#Public" + ], + "name": "Microsoft cuts 4,800 jobs and shrinks Xbox in 'significant restructure'", + "cc": [], + "mediaType": "text/html", + "attachment": [ + { + "href": "https://www.bbc.com/news/articles/c36yy27rnpeo", + "mediaType": "text/html; charset=utf-8", + "type": "Link" + } + ], + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/06b96f69-3ae5-4c0b-91aa-631a5ed90080.jpeg" + }, + "sensitive": false, + "published": "2026-07-06T22:01:15.771031Z", + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "tag": [ + { + "href": "https://lemmy.world/post/49122698", + "name": "#technology", + "type": "Hashtag" + } + ] + }, + "cc": [ + "https://lemmy.world/c/technology" + ], + "type": "Create", + "audience": "https://lemmy.world/c/technology" + }, + "cc": [ + "https://lemmy.world/c/technology/followers" + ], + "type": "Announce", + "id": "https://lemmy.world/activities/announce/create/43e8f9ba-b755-492e-900e-035c99d36747" + } + ] +} \ No newline at end of file diff --git a/internal/ap/testdata/page_lemmy_world.json b/internal/ap/testdata/page_lemmy_world.json new file mode 100644 index 0000000..d998d15 --- /dev/null +++ b/internal/ap/testdata/page_lemmy_world.json @@ -0,0 +1,41 @@ +{ + "@context": [ + "https://join-lemmy.org/context.json", + "https://www.w3.org/ns/activitystreams" + ], + "type": "Page", + "id": "https://lemmy.world/post/49131386", + "attributedTo": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "to": [ + "https://lemmy.world/c/technology", + "https://www.w3.org/ns/activitystreams#Public" + ], + "name": "Inside the history of DRAM price-fixing lawsuits — how HBM allocations could make a difference after two decades of failed cases", + "cc": [], + "mediaType": "text/html", + "attachment": [ + { + "href": "https://www.tomshardware.com/pc-components/dram/samsung-sk-hynix-and-micron-face-a-third-dram-price-fixing-lawsuit", + "mediaType": "text/html; charset=utf-8", + "type": "Link" + } + ], + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/98840222-90ff-4a7f-83ee-6090488648d4.jpeg" + }, + "sensitive": false, + "published": "2026-07-07T03:27:37.028201Z", + "language": { + "identifier": "en", + "name": "English" + }, + "audience": "https://lemmy.world/c/technology", + "tag": [ + { + "href": "https://lemmy.world/post/49131386", + "name": "#technology", + "type": "Hashtag" + } + ] +} \ No newline at end of file diff --git a/internal/ap/testdata/person_lemmy_world.json b/internal/ap/testdata/person_lemmy_world.json new file mode 100644 index 0000000..27e49f1 --- /dev/null +++ b/internal/ap/testdata/person_lemmy_world.json @@ -0,0 +1,29 @@ +{ + "@context": [ + "https://join-lemmy.org/context.json", + "https://www.w3.org/ns/activitystreams" + ], + "type": "Person", + "id": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "preferredUsername": "LeftLeaningFreedomFighters", + "inbox": "https://lemmy.world/u/LeftLeaningFreedomFighters/inbox", + "outbox": "https://lemmy.world/u/LeftLeaningFreedomFighters/outbox", + "publicKey": { + "id": "https://lemmy.world/u/LeftLeaningFreedomFighters#main-key", + "owner": "https://lemmy.world/u/LeftLeaningFreedomFighters", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAutWsHnK/k+ODe0pktuTD\nIDFCxJny5M1Vno748QTW3yKTvA3L2EGcN8PTvubQ29r8JPt+qVF4XeQZllwq55xD\nsbnU+QHevy/snvXtwsT8luxy9HDfCKXL/JzBMebLSyJMraja7efHxWVnh7uIjacL\n3WNDHLN/pCj9nzArwENMeesolMUVbpmafiEKGLor7lNdLl7jpXBgc5jjYks9HRbC\nJqmeDzrqkBFj/8QFoGjQRjSA/jTVJ0phOXUNzvbT1JNYmJzmtwxy0bQa9+Rf/uKj\n7yXwNSrrBfn1/Ir+0e/2MScfEBc9QybfblAemVyCotIFlXb1ab3RHfwnyHwqPIsA\nPwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "name": "Surprised Neelix", + "icon": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/761c9aaf-70a2-4a50-a92c-9ada2025ff51.png" + }, + "image": { + "type": "Image", + "url": "https://lemmy.world/pictrs/image/8065bb13-9415-4f30-a427-d34e33ffea1b.png" + }, + "endpoints": { + "sharedInbox": "https://lemmy.world/inbox" + }, + "published": "2026-06-27T20:59:46.499970Z" +} \ No newline at end of file diff --git a/internal/ap/testdata/webfinger_group.json b/internal/ap/testdata/webfinger_group.json new file mode 100644 index 0000000..16187fb --- /dev/null +++ b/internal/ap/testdata/webfinger_group.json @@ -0,0 +1 @@ +{"subject":"acct:technology@lemmy.world","links":[{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://lemmy.world/c/technology","template":null},{"rel":"self","type":"application/activity+json","href":"https://lemmy.world/c/technology","template":null,"properties":{"https://www.w3.org/ns/activitystreams#type":"Group"}}]} \ No newline at end of file diff --git a/internal/ap/vocab.go b/internal/ap/vocab.go new file mode 100644 index 0000000..f0bf207 --- /dev/null +++ b/internal/ap/vocab.go @@ -0,0 +1,563 @@ +// Package ap implements the client side of the Lemmy-flavored ActivityPub +// protocol: tolerant AS2 vocabulary types, WebFinger resolution, draft-cavage +// HTTP signatures, and a signed-fetch client with collection paging. +// +// The vocabulary deliberately avoids a full JSON-LD processor. Like granary's +// as2.py it treats AP as "json-ld-lite": one open Object struct covering every +// type Lemmy/FEP-1b12 emits, with custom unmarshallers for the fields that may +// legally appear as a bare IRI string, a single object, or an array of either. +// Unknown fields are ignored, never fatal. +package ap + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "reflect" + "time" +) + +// AS2 media types. Lemmy serves and accepts application/activity+json; +// Mastodon prefers the ld+json profile form. We send both in Accept. +const ( + ContentTypeActivityJSON = "application/activity+json" + ContentTypeLDJSON = `application/ld+json; profile="https://www.w3.org/ns/activitystreams"` + acceptActivityJSON = ContentTypeActivityJSON + `, application/ld+json; profile="https://www.w3.org/ns/activitystreams"; q=0.9` +) + +// PublicAudience is the special AS2 collection meaning "public". +const PublicAudience = "https://www.w3.org/ns/activitystreams#Public" + +// AP object/activity types Lemmy and FEP-1b12 emit. +const ( + TypeGroup = "Group" + TypePerson = "Person" + TypeApplication = "Application" + TypeService = "Service" + + TypePage = "Page" + TypeNote = "Note" + TypeArticle = "Article" + + TypeCreate = "Create" + TypeUpdate = "Update" + TypeDelete = "Delete" + TypeAnnounce = "Announce" + TypeFollow = "Follow" + TypeAccept = "Accept" + TypeReject = "Reject" + TypeUndo = "Undo" + TypeLike = "Like" + TypeDislike = "Dislike" + + TypeTombstone = "Tombstone" + TypeImage = "Image" + TypeLink = "Link" + TypeHashtag = "Hashtag" + TypeMention = "Mention" + + TypeCollection = "Collection" + TypeOrderedCollection = "OrderedCollection" + TypeCollectionPage = "CollectionPage" + TypeOrderedCollectionPage = "OrderedCollectionPage" +) + +// Object is the universal AS2 object: every field any Lemmy/FEP-1b12 type +// uses, in one open struct. The Type field discriminates. Fields that may be +// a bare IRI, an inline object, or an array use the tolerant wrapper types +// below (Ref, Refs, Audience, Links, Tags, Languages). +type Object struct { + Context json.RawMessage `json:"@context,omitempty"` + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + + // Addressing. Lemmy emits arrays of IRI strings; other implementations + // emit single strings or inline objects. Audience flattens all of them + // to IRIs. + To Audience `json:"to,omitempty"` + Cc Audience `json:"cc,omitempty"` + Bto Audience `json:"bto,omitempty"` + Bcc Audience `json:"bcc,omitempty"` + // Audience on Lemmy objects/activities is the community IRI (FEP-1b12). + Audience Audience `json:"audience,omitempty"` + + // Actor/attribution: string or object (or array of either). + Actor *Object `json:"actor,omitempty"` + AttributedTo Refs `json:"attributedTo,omitempty"` + + // The activity payload: string or inline object, possibly nested + // (Announce{Create{Note}}). + Object *Object `json:"object,omitempty"` + Target *Object `json:"target,omitempty"` + + // Content. + Name string `json:"name,omitempty"` + Content string `json:"content,omitempty"` + Summary string `json:"summary,omitempty"` + MediaType string `json:"mediaType,omitempty"` + Source *Source `json:"source,omitempty"` + URL Links `json:"url,omitempty"` + InReplyTo *Object `json:"inReplyTo,omitempty"` + Tag Tags `json:"tag,omitempty"` + Attach Tags `json:"attachment,omitempty"` + Icon *Object `json:"icon,omitempty"` + Image *Object `json:"image,omitempty"` + Published *Time `json:"published,omitempty"` + Updated *Time `json:"updated,omitempty"` + + // Lemmy emits a single language object on posts/comments but an ARRAY + // of language objects on Group actors; Languages accepts both. + Language Languages `json:"language,omitempty"` + + // Lemmy extensions. + Sensitive *bool `json:"sensitive,omitempty"` + CommentsEnabled *bool `json:"commentsEnabled,omitempty"` + PostingRestrictedToMods *bool `json:"postingRestrictedToMods,omitempty"` + Stickied *bool `json:"stickied,omitempty"` + Distinguished *bool `json:"distinguished,omitempty"` + + // Actor plumbing. + PreferredUsername string `json:"preferredUsername,omitempty"` + Inbox string `json:"inbox,omitempty"` + Outbox string `json:"outbox,omitempty"` + Followers string `json:"followers,omitempty"` + Following string `json:"following,omitempty"` + Featured string `json:"featured,omitempty"` + Moderators string `json:"moderators,omitempty"` + Endpoints *Endpoints `json:"endpoints,omitempty"` + PublicKey *PublicKey `json:"publicKey,omitempty"` + + // Tombstone. + FormerType string `json:"formerType,omitempty"` + Deleted *Time `json:"deleted,omitempty"` + + // Collections. First/Next may be a bare IRI or an inline page object. + TotalItems int `json:"totalItems,omitempty"` + First *Object `json:"first,omitempty"` + Last *Object `json:"last,omitempty"` + Next *Object `json:"next,omitempty"` + Prev *Object `json:"prev,omitempty"` + PartOf string `json:"partOf,omitempty"` + Items Refs `json:"items,omitempty"` + OrderedItems Refs `json:"orderedItems,omitempty"` +} + +// objectAlias strips Object's methods so (un)marshalling the plain struct +// does not recurse through the tolerant wrappers. +type objectAlias Object + +// UnmarshalJSON accepts either a bare IRI string (decoding to an Object with +// only ID set — how references appear on the wire) or a JSON object. This is +// the core of tolerant string-or-object parsing: every *Object field gets it +// for free. +func (o *Object) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || string(data) == "null" { + return nil + } + if data[0] == '"' { + var iri string + if err := json.Unmarshal(data, &iri); err != nil { + return err + } + *o = Object{ID: iri} + return nil + } + if data[0] == '[' { + // Some servers emit a single-valued field (actor/object/target) as a + // one-element array. The tolerant-parse contract says degrade, don't + // fail: take the first non-null element rather than sinking the whole + // object. Extra elements are dropped (these fields are logically + // single-valued in the shapes we consume). + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + for _, item := range raw { + trimmed := bytes.TrimSpace(item) + if len(trimmed) == 0 || string(trimmed) == "null" { + continue + } + return o.UnmarshalJSON(trimmed) + } + *o = Object{} + return nil + } + var alias objectAlias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + *o = Object(alias) + return nil +} + +// MarshalJSON emits a bare IRI string when only the ID is set (the compact +// wire form of a reference), otherwise the full object. +func (o Object) MarshalJSON() ([]byte, error) { + if o.ID != "" && o.isIDOnly() { + return json.Marshal(o.ID) + } + return json.Marshal(objectAlias(o)) +} + +// isIDOnly reports whether every field except ID is zero. A DeepEqual against +// the zero Object short-circuits on the first non-zero field, unlike the old +// marshal-and-byte-compare which paid an O(n) JSON encode on every Object +// marshal (including large collection pages). +func (o Object) isIDOnly() bool { + clone := o + clone.ID = "" + return reflect.DeepEqual(clone, Object{}) +} + +// IsActor reports whether the object's type is an AP actor type. +func (o *Object) IsActor() bool { + switch o.Type { + case TypeGroup, TypePerson, TypeApplication, TypeService: + return true + } + return false +} + +// IsCollection reports whether the object is a collection or collection page. +func (o *Object) IsCollection() bool { + switch o.Type { + case TypeCollection, TypeOrderedCollection, TypeCollectionPage, TypeOrderedCollectionPage: + return true + } + return false +} + +// IsTombstone reports whether the object is an AS2 Tombstone. +func (o *Object) IsTombstone() bool { return o.Type == TypeTombstone } + +// IsPublic reports whether the object is addressed to the AS2 public +// collection in to, cc, or audience. +func (o *Object) IsPublic() bool { + for _, list := range []Audience{o.To, o.Cc, o.Audience} { + for _, iri := range list { + // Both spellings appear in the wild ("Public", "as:Public", + // full IRI); Lemmy always uses the full IRI. + if iri == PublicAudience || iri == "as:Public" || iri == "Public" { + return true + } + } + } + return false +} + +// Host returns the hostname of the object's canonical id, or "" if the id +// is absent or unparseable. +func (o *Object) Host() string { + if o.ID == "" { + return "" + } + u, err := url.Parse(o.ID) + if err != nil { + return "" + } + return u.Hostname() +} + +// URLString returns the object's primary url as a string (first Link href), +// or "" if none. +func (o *Object) URLString() string { + for _, link := range o.URL { + if link.Href != "" { + return link.Href + } + } + return "" +} + +// Time is a tolerant AP timestamp. Lemmy emits RFC3339 with sub-second +// precision; other fediverse software occasionally emits variants. A value +// that fails every known layout parses to the zero Time instead of failing +// the whole object (tolerant parsing: a bad date must never be fatal). +// +// Valid distinguishes "present and parsed" from "present but unparseable / +// wrong JSON type": a bad `published` yields a non-nil *Time whose Valid is +// false, so downstream code (task 05 rkey/TID derivation) can tell it apart +// from a real timestamp with OK() rather than being fooled by a nil check or +// a fabricated year-0001 value. +type Time struct { + time.Time + // Valid is true only when the wire value parsed to a real timestamp. + Valid bool +} + +// OK reports whether the timestamp is present, parsed, and non-zero. Task 05 +// calls it before deriving a TID from a published date. +func (t *Time) OK() bool { + return t != nil && t.Valid && !t.IsZero() +} + +// apTimeLayouts are tried in order when parsing AP timestamps. +var apTimeLayouts = []string{ + time.RFC3339Nano, // Lemmy, Mastodon: 2026-07-07T03:27:37.028201Z + time.RFC3339, // + "2006-01-02T15:04:05.999999", // missing zone, seen from misbehaving servers + "2006-01-02T15:04:05", // + time.RFC1123, // legacy OStatus-era software + time.RFC1123Z, // +} + +// UnmarshalJSON parses the timestamp tolerantly; unparseable values yield +// the zero Time, never an error. +func (t *Time) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + // Not a JSON string (e.g. a number or object) — tolerate as invalid. + t.Time = time.Time{} + t.Valid = false + return nil //nolint:nilerr // tolerance is deliberate here + } + for _, layout := range apTimeLayouts { + if parsed, err := time.Parse(layout, s); err == nil { + t.Time = parsed.UTC() + t.Valid = true + return nil + } + } + // Present but unparseable: mark invalid so callers don't mistake the zero + // time for a real (year-0001) timestamp. + t.Time = time.Time{} + t.Valid = false + return nil +} + +// MarshalJSON emits RFC3339 UTC, Lemmy style. An invalid or zero timestamp +// marshals to null rather than a fabricated year-0001 date, so a malformed +// inbound `published` never round-trips as a plausible-looking timestamp. +func (t Time) MarshalJSON() ([]byte, error) { + if !t.Valid || t.IsZero() { + return []byte("null"), nil + } + return json.Marshal(t.UTC().Format("2006-01-02T15:04:05.000000Z07:00")) +} + +// NewTime wraps a time.Time as an AP timestamp. +func NewTime(t time.Time) *Time { return &Time{Time: t, Valid: true} } + +// Refs is a list of objects/references that may appear on the wire as a +// single string, a single object, or an array of either. +type Refs []Object + +// UnmarshalJSON accepts string | object | array of (string | object). +func (rs *Refs) UnmarshalJSON(data []byte) error { + return unmarshalOneOrMany(data, (*[]Object)(rs)) +} + +// First returns the first ref, or nil if the list is empty. +func (rs Refs) First() *Object { + if len(rs) == 0 { + return nil + } + return &rs[0] +} + +// FirstID returns the id of the first ref, or "". +func (rs Refs) FirstID() string { + if first := rs.First(); first != nil { + return first.ID + } + return "" +} + +// Audience is a list of IRIs that may appear on the wire as a single string, +// a single object (id extracted), or an array of either. +type Audience []string + +// UnmarshalJSON accepts string | object | array of (string | object) and +// flattens everything to IRI strings. +func (a *Audience) UnmarshalJSON(data []byte) error { + var refs Refs + if err := refs.UnmarshalJSON(data); err != nil { + return err + } + out := make(Audience, 0, len(refs)) + for i := range refs { + if refs[i].ID != "" { + out = append(out, refs[i].ID) + } + } + *a = out + return nil +} + +// Contains reports whether the audience includes the given IRI. +func (a Audience) Contains(iri string) bool { + for _, v := range a { + if v == iri { + return true + } + } + return false +} + +// Link is an AS2 Link (or Hashtag/Mention tag, or attachment). Lemmy post +// attachments are Links with href+mediaType; tags are Hashtags/Mentions with +// href+name. +type Link struct { + Type string `json:"type,omitempty"` + Href string `json:"href,omitempty"` + Name string `json:"name,omitempty"` + MediaType string `json:"mediaType,omitempty"` + // Image attachments (PieFed) carry url instead of href. + URL string `json:"url,omitempty"` +} + +// UnmarshalJSON accepts a bare IRI string (→ Href) or a Link object. +func (l *Link) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || string(data) == "null" { + return nil + } + if data[0] == '"' { + var href string + if err := json.Unmarshal(data, &href); err != nil { + return err + } + *l = Link{Href: href} + return nil + } + type plain Link + var p plain + if err := json.Unmarshal(data, &p); err != nil { + return err + } + *l = Link(p) + return nil +} + +// MarshalJSON emits a bare string when only Href is set. +func (l Link) MarshalJSON() ([]byte, error) { + if l.Type == "" && l.Name == "" && l.MediaType == "" && l.URL == "" && l.Href != "" { + return json.Marshal(l.Href) + } + type plain Link + return json.Marshal(plain(l)) +} + +// Links is a list of Link that may appear as string | object | array. +type Links []Link + +// UnmarshalJSON accepts string | object | array of (string | object). +func (ls *Links) UnmarshalJSON(data []byte) error { + return unmarshalOneOrMany(data, ls) +} + +// Tags is a list of Link used for `tag` and `attachment`, tolerating +// single-value and array forms. +type Tags []Link + +// UnmarshalJSON accepts string | object | array of (string | object). +func (ts *Tags) UnmarshalJSON(data []byte) error { + return unmarshalOneOrMany(data, (*[]Link)(ts)) +} + +// unmarshalOneOrMany decodes JSON that may be a single value or an array of +// values into a slice whose element type has a tolerant UnmarshalJSON. +func unmarshalOneOrMany[S ~[]E, E any](data []byte, out *S) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || string(data) == "null" { + return nil + } + if data[0] == '[' { + var raw []json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + result := make(S, len(raw)) + for i, item := range raw { + if err := json.Unmarshal(item, &result[i]); err != nil { + return err + } + } + *out = result + return nil + } + var single E + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *out = S{single} + return nil +} + +// Source carries the original markdown of an object (Lemmy always includes +// it alongside the rendered HTML content). +type Source struct { + Content string `json:"content,omitempty"` + MediaType string `json:"mediaType,omitempty"` +} + +// Language is Lemmy's language tag ({identifier, name}). +type Language struct { + Identifier string `json:"identifier,omitempty"` + Name string `json:"name,omitempty"` +} + +// Languages tolerates Lemmy's two wire forms: a single language object on +// posts/comments, an array of them on Group actors. +type Languages []Language + +// UnmarshalJSON accepts object | array of objects. +func (ls *Languages) UnmarshalJSON(data []byte) error { + data = bytes.TrimSpace(data) + if len(data) == 0 || string(data) == "null" { + return nil + } + if data[0] == '[' { + type plain []Language + var p plain + if err := json.Unmarshal(data, &p); err != nil { + return err + } + *ls = Languages(p) + return nil + } + var single Language + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *ls = Languages{single} + return nil +} + +// Endpoints is the actor endpoints object; Lemmy only uses sharedInbox. +type Endpoints struct { + SharedInbox string `json:"sharedInbox,omitempty"` +} + +// PublicKey is the actor's RSA public key as published for HTTP signatures. +// Lemmy requires RSA keys and keyId "{actorID}#main-key". +type PublicKey struct { + ID string `json:"id,omitempty"` + Owner string `json:"owner,omitempty"` + PublicKeyPem string `json:"publicKeyPem,omitempty"` +} + +// SharedInboxOrInbox returns the actor's shared inbox when present, +// otherwise its own inbox (Lemmy prefers sharedInbox for delivery). +func (o *Object) SharedInboxOrInbox() string { + if o.Endpoints != nil && o.Endpoints.SharedInbox != "" { + return o.Endpoints.SharedInbox + } + return o.Inbox +} + +// ParseObject decodes an AP object from JSON, requiring that the payload is +// a JSON object (not a bare string/array). +func ParseObject(data []byte) (*Object, error) { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || trimmed[0] != '{' { + return nil, fmt.Errorf("ap: payload is not a JSON object") + } + var obj Object + if err := json.Unmarshal(trimmed, &obj); err != nil { + return nil, fmt.Errorf("ap: parse object: %w", err) + } + return &obj, nil +} diff --git a/internal/ap/vocab_test.go b/internal/ap/vocab_test.go new file mode 100644 index 0000000..560e529 --- /dev/null +++ b/internal/ap/vocab_test.go @@ -0,0 +1,423 @@ +package ap + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixture provenance: group_lemmy_world.json, person_lemmy_world.json, +// page_lemmy_world.json, note_lemmy_zip.json, outbox_lemmy_world.json +// (truncated to 2 items), announce_create_page_lemmy_world.json, and +// webfinger_group.json were captured live from lemmy.world / lemmy.zip on +// 2026-07-06 with `curl -H 'Accept: application/activity+json'`. +// announce_create_note.json and announce_like.json are constructed: the +// Announce/Create wrapper shapes are copied verbatim from the live outbox +// items, the inner Note is the live comment, and the inner Like follows +// Lemmy's crates/apub/src/protocol/activities/voting/vote.rs shape. +// delete_page.json follows Lemmy's deletion protocol shape. + +func loadFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", name)) + require.NoError(t, err, "fixture %s must exist", name) + return data +} + +func parseFixture(t *testing.T, name string) *Object { + t.Helper() + obj, err := ParseObject(loadFixture(t, name)) + require.NoError(t, err, "fixture %s must parse", name) + return obj +} + +func TestParseGroupFixture(t *testing.T) { + group := parseFixture(t, "group_lemmy_world.json") + + assert.Equal(t, TypeGroup, group.Type) + assert.True(t, group.IsActor()) + assert.Equal(t, "https://lemmy.world/c/technology", group.ID) + assert.Equal(t, "technology", group.PreferredUsername) + assert.Equal(t, "Technology", group.Name) + assert.Equal(t, "lemmy.world", group.Host()) + assert.Equal(t, "https://lemmy.world/c/technology/inbox", group.Inbox) + assert.Equal(t, "https://lemmy.world/c/technology/outbox", group.Outbox) + assert.Equal(t, "https://lemmy.world/c/technology/followers", group.Followers) + assert.Equal(t, "https://lemmy.world/c/technology/featured", group.Featured) + assert.Equal(t, "https://lemmy.world/inbox", group.SharedInboxOrInbox()) + + // Lemmy Group attributedTo is the moderators collection IRI. + assert.Equal(t, "https://lemmy.world/c/technology/moderators", group.AttributedTo.FirstID()) + + require.NotNil(t, group.PublicKey) + assert.Equal(t, "https://lemmy.world/c/technology#main-key", group.PublicKey.ID) + assert.Equal(t, group.ID, group.PublicKey.Owner) + _, err := ParsePublicKeyPEM([]byte(group.PublicKey.PublicKeyPem)) + require.NoError(t, err, "publicKeyPem must be a parseable RSA key") + + // Lemmy quirk: Group.language is an ARRAY of language objects. + require.Len(t, group.Language, 1) + assert.Equal(t, "en", group.Language[0].Identifier) + + // Lemmy extensions. + require.NotNil(t, group.PostingRestrictedToMods) + assert.False(t, *group.PostingRestrictedToMods) + require.NotNil(t, group.Sensitive) + assert.False(t, *group.Sensitive) + + // Markdown source alongside rendered HTML. + require.NotNil(t, group.Source) + assert.Equal(t, "text/markdown", group.Source.MediaType) + assert.NotEmpty(t, group.Source.Content) + assert.NotEmpty(t, group.Summary) + + require.NotNil(t, group.Icon) + assert.Equal(t, TypeImage, group.Icon.Type) + assert.NotEmpty(t, group.Icon.URLString()) + + require.NotNil(t, group.Published) + assert.Equal(t, 2023, group.Published.Year()) + require.NotNil(t, group.Updated) +} + +func TestParsePersonFixture(t *testing.T) { + person := parseFixture(t, "person_lemmy_world.json") + + assert.Equal(t, TypePerson, person.Type) + assert.True(t, person.IsActor()) + assert.Equal(t, "https://lemmy.world/u/LeftLeaningFreedomFighters", person.ID) + assert.Equal(t, "LeftLeaningFreedomFighters", person.PreferredUsername) + assert.Equal(t, "Surprised Neelix", person.Name) + require.NotNil(t, person.PublicKey) + assert.Equal(t, person.ID+"#main-key", person.PublicKey.ID) + assert.Equal(t, "https://lemmy.world/inbox", person.SharedInboxOrInbox()) + require.NotNil(t, person.Icon) + assert.NotEmpty(t, person.Icon.URLString()) +} + +func TestParsePageFixture(t *testing.T) { + page := parseFixture(t, "page_lemmy_world.json") + + assert.Equal(t, TypePage, page.Type) + assert.Equal(t, "https://lemmy.world/post/49131386", page.ID) + assert.Equal(t, "https://lemmy.world/u/LeftLeaningFreedomFighters", page.AttributedTo.FirstID()) + assert.NotEmpty(t, page.Name) + assert.True(t, page.IsPublic(), "post addressed to as:Public must be public") + assert.True(t, page.To.Contains("https://lemmy.world/c/technology")) + assert.True(t, page.Audience.Contains("https://lemmy.world/c/technology"), + "audience (bare string on the wire) must normalize to a list") + + // Link post: the external URL rides in attachment. + require.Len(t, page.Attach, 1) + assert.Equal(t, TypeLink, page.Attach[0].Type) + assert.Contains(t, page.Attach[0].Href, "tomshardware.com") + + // Image embed. + require.NotNil(t, page.Image) + assert.Equal(t, TypeImage, page.Image.Type) + assert.Contains(t, page.Image.URLString(), "pictrs") + + // Single language object (vs the Group's array form). + require.Len(t, page.Language, 1) + assert.Equal(t, "en", page.Language[0].Identifier) + + require.NotNil(t, page.Sensitive) + assert.False(t, *page.Sensitive) + + require.Len(t, page.Tag, 1) + assert.Equal(t, TypeHashtag, page.Tag[0].Type) + assert.Equal(t, "#technology", page.Tag[0].Name) + + require.NotNil(t, page.Published) + assert.Equal(t, + time.Date(2026, 7, 7, 3, 27, 37, 28201000, time.UTC), + page.Published.Time) +} + +func TestParseNoteFixture(t *testing.T) { + note := parseFixture(t, "note_lemmy_zip.json") + + assert.Equal(t, TypeNote, note.Type) + assert.Equal(t, "https://lemmy.zip/comment/27485395", note.ID) + assert.Equal(t, "https://lemmy.zip/u/tixooo", note.AttributedTo.FirstID()) + + // The parent pointer — the whole reason comments need strongRef + // resolution. + require.NotNil(t, note.InReplyTo) + assert.Equal(t, "https://sh.itjust.works/comment/26248018", note.InReplyTo.ID) + + assert.Contains(t, note.Content, "human history") + require.NotNil(t, note.Source) + assert.Equal(t, "text/markdown", note.Source.MediaType) + assert.Contains(t, note.Source.Content, "human history") + + require.Len(t, note.Tag, 1) + assert.Equal(t, TypeMention, note.Tag[0].Type) + + // distinguished is a Lemmy extension on comments. + require.NotNil(t, note.Distinguished) + assert.False(t, *note.Distinguished) + + // Empty attachment array must parse to empty, not fail. + assert.Empty(t, note.Attach) +} + +func TestParseAnnounceCreatePageFixture(t *testing.T) { + announce := parseFixture(t, "announce_create_page_lemmy_world.json") + + assert.Equal(t, TypeAnnounce, announce.Type) + assert.Equal(t, "https://lemmy.world/c/technology", announce.Actor.ID) + assert.True(t, announce.IsPublic()) + assert.True(t, announce.Cc.Contains("https://lemmy.world/c/technology/followers")) + + create := announce.Object + require.NotNil(t, create) + assert.Equal(t, TypeCreate, create.Type) + assert.Equal(t, "https://lemmy.world/u/LeftLeaningFreedomFighters", create.Actor.ID) + + page := create.Object + require.NotNil(t, page) + assert.Equal(t, TypePage, page.Type) + assert.Equal(t, "https://lemmy.world/post/49131386", page.ID) + assert.NotEmpty(t, page.Name) +} + +func TestParseAnnounceCreateNoteFixture(t *testing.T) { + announce := parseFixture(t, "announce_create_note.json") + + require.NotNil(t, announce.Object) + assert.Equal(t, TypeCreate, announce.Object.Type) + note := announce.Object.Object + require.NotNil(t, note) + assert.Equal(t, TypeNote, note.Type) + require.NotNil(t, note.InReplyTo) + assert.Equal(t, "https://sh.itjust.works/comment/26248018", note.InReplyTo.ID) +} + +func TestParseAnnounceLikeFixture(t *testing.T) { + announce := parseFixture(t, "announce_like.json") + + assert.Equal(t, TypeAnnounce, announce.Type) + like := announce.Object + require.NotNil(t, like) + assert.Equal(t, TypeLike, like.Type) + assert.Equal(t, "https://lemmy.zip/u/tixooo", like.Actor.ID) + // The liked object is a bare IRI string on the wire. + require.NotNil(t, like.Object) + assert.Equal(t, "https://lemmy.world/post/49131386", like.Object.ID) + assert.Empty(t, like.Object.Type) +} + +func TestParseDeleteFixture(t *testing.T) { + deleteActivity := parseFixture(t, "delete_page.json") + + assert.Equal(t, TypeDelete, deleteActivity.Type) + require.NotNil(t, deleteActivity.Object) + assert.Equal(t, "https://lemmy.world/post/49131386", deleteActivity.Object.ID) +} + +func TestParseOutboxFixture(t *testing.T) { + outbox := parseFixture(t, "outbox_lemmy_world.json") + + assert.Equal(t, TypeOrderedCollection, outbox.Type) + assert.True(t, outbox.IsCollection()) + assert.Equal(t, 50, outbox.TotalItems) + // Truncated at capture time; every item is a full Announce activity. + require.Len(t, outbox.OrderedItems, 2) + for i := range outbox.OrderedItems { + item := &outbox.OrderedItems[i] + assert.Equal(t, TypeAnnounce, item.Type) + require.NotNil(t, item.Object) + assert.Equal(t, TypeCreate, item.Object.Type) + } +} + +func TestParseTombstone(t *testing.T) { + obj, err := ParseObject([]byte(`{ + "type": "Tombstone", + "id": "https://lemmy.world/post/1", + "formerType": "Page", + "deleted": "2026-01-02T03:04:05.000000Z" + }`)) + require.NoError(t, err) + assert.True(t, obj.IsTombstone()) + assert.Equal(t, TypePage, obj.FormerType) + require.NotNil(t, obj.Deleted) + assert.Equal(t, 2026, obj.Deleted.Year()) +} + +// TestTolerantFields exercises the string-or-object-or-array wire variants +// that differ across fediverse implementations (granary's as2 quirk list). +func TestTolerantFields(t *testing.T) { + obj, err := ParseObject([]byte(`{ + "id": "https://example.com/note/1", + "type": "Note", + "to": "https://www.w3.org/ns/activitystreams#Public", + "cc": [{"id": "https://example.com/u/bob"}, "https://example.com/u/carol"], + "attributedTo": [{"type": "Person", "id": "https://example.com/u/alice"}], + "icon": "https://example.com/icon.png", + "url": {"type": "Link", "href": "https://example.com/note/1.html"}, + "tag": {"type": "Hashtag", "href": "https://example.com/tag/x", "name": "#x"}, + "language": {"identifier": "de", "name": "Deutsch"}, + "object": "https://example.com/note/0" + }`)) + require.NoError(t, err) + + assert.True(t, obj.IsPublic(), "bare-string to must be understood") + assert.Equal(t, Audience{"https://example.com/u/bob", "https://example.com/u/carol"}, obj.Cc) + assert.Equal(t, "https://example.com/u/alice", obj.AttributedTo.FirstID()) + assert.Equal(t, TypePerson, obj.AttributedTo.First().Type) + require.NotNil(t, obj.Icon) + assert.Equal(t, "https://example.com/icon.png", obj.Icon.ID, "bare-string icon decodes as IRI-only ref") + assert.Equal(t, "https://example.com/note/1.html", obj.URLString()) + require.Len(t, obj.Tag, 1) + assert.Equal(t, "#x", obj.Tag[0].Name) + require.Len(t, obj.Language, 1) + assert.Equal(t, "de", obj.Language[0].Identifier) + require.NotNil(t, obj.Object) + assert.Equal(t, "https://example.com/note/0", obj.Object.ID) +} + +func TestUnknownFieldsIgnored(t *testing.T) { + obj, err := ParseObject([]byte(`{ + "id": "https://example.com/note/1", + "type": "Note", + "content": "hi", + "somePieFedExtension": {"nested": ["values", 1, true]}, + "litepub:capabilities": {"acceptsChatMessages": false} + }`)) + require.NoError(t, err, "unknown fields must never be fatal") + assert.Equal(t, "hi", obj.Content) +} + +func TestTolerantTimestamps(t *testing.T) { + cases := map[string]bool{ // value → expect parsed (non-zero) + `"2026-07-07T03:27:37.028201Z"`: true, + `"2026-07-07T03:27:37Z"`: true, + `"2026-07-07T03:27:37+02:00"`: true, + `"2026-07-07T03:27:37.028201"`: true, // zone-less, seen in the wild + `"Mon, 06 Jul 2026 10:00:00 GMT"`: true, // legacy RFC1123 + `"not a date"`: false, + `12345`: false, // wrong JSON type + `{"unexpected": "object"}`: false, + } + for raw, wantParsed := range cases { + var parsed Time + err := json.Unmarshal([]byte(raw), &parsed) + require.NoError(t, err, "timestamp %s must not error", raw) + assert.Equal(t, wantParsed, !parsed.IsZero(), "timestamp %s parsed-ness", raw) + } + + // And a bad published date must not sink the whole object. + obj, err := ParseObject([]byte(`{"id": "x", "type": "Note", "published": "yesterday-ish"}`)) + require.NoError(t, err) + assert.True(t, obj.Published.IsZero()) + // A present-but-unparseable timestamp is a non-nil *Time, but OK() reports + // it as unusable so task-05 rkey/TID derivation isn't fooled (finding 6). + require.NotNil(t, obj.Published, "the field was present, so *Time is non-nil") + assert.False(t, obj.Published.OK(), "a malformed published must be detectable as invalid") + assert.False(t, obj.Published.Valid) + + // And it must NOT round-trip as a fabricated year-0001 date: it marshals to + // null, not "0001-01-01T...". + out, err := json.Marshal(obj) + require.NoError(t, err) + assert.Contains(t, string(out), `"published":null`, + "a malformed published re-serializes as null, never a plausible timestamp") + assert.NotContains(t, string(out), "0001-01-01") + + // A real published parses, OK()s true, and round-trips. + good, err := ParseObject([]byte(`{"id": "x", "type": "Note", "published": "2026-07-07T03:27:37.028201Z"}`)) + require.NoError(t, err) + assert.True(t, good.Published.OK()) + goodOut, err := json.Marshal(good) + require.NoError(t, err) + assert.Contains(t, string(goodOut), "2026-07-07T03:27:37") +} + +// TestTolerantSingleValueArrays covers finding 10: a server that wraps a +// logically single-valued field (actor/object/target) in a one-element array +// must degrade to the first element, not sink the whole object. +func TestTolerantSingleValueArrays(t *testing.T) { + obj, err := ParseObject([]byte(`{ + "id": "https://example.com/activities/1", + "type": "Create", + "actor": ["https://example.com/u/alice"], + "object": [{"id": "https://example.com/note/1", "type": "Note", "content": "hi"}], + "target": ["https://example.com/c/tech", "https://example.com/c/ignored"] + }`)) + require.NoError(t, err, "an array-wrapped single-value field must not be fatal") + + require.NotNil(t, obj.Actor) + assert.Equal(t, "https://example.com/u/alice", obj.Actor.ID) + require.NotNil(t, obj.Object) + assert.Equal(t, "https://example.com/note/1", obj.Object.ID) + assert.Equal(t, "hi", obj.Object.Content) + require.NotNil(t, obj.Target) + assert.Equal(t, "https://example.com/c/tech", obj.Target.ID, + "extra array elements are dropped; the first wins") + + // An empty array degrades to a zero (nil-id) object rather than failing. + empty, err := ParseObject([]byte(`{"id": "x", "type": "Create", "object": []}`)) + require.NoError(t, err) + require.NotNil(t, empty.Object) + assert.Empty(t, empty.Object.ID) +} + +func TestRefMarshalRoundTrip(t *testing.T) { + // A bare-IRI ref re-marshals as a bare string; an inline object stays an + // object. + obj, err := ParseObject([]byte(`{"id": "a", "type": "Like", "object": "https://example.com/post/1"}`)) + require.NoError(t, err) + out, err := json.Marshal(obj) + require.NoError(t, err) + assert.Contains(t, string(out), `"object":"https://example.com/post/1"`) + + obj, err = ParseObject([]byte(`{"id": "a", "type": "Create", "object": {"id": "b", "type": "Note", "content": "hi"}}`)) + require.NoError(t, err) + out, err = json.Marshal(obj) + require.NoError(t, err) + assert.Contains(t, string(out), `"content":"hi"`) +} + +// TestFixturesRoundTrip proves every committed fixture survives +// parse → marshal → parse with the fields we read intact. +func TestFixturesRoundTrip(t *testing.T) { + fixtures := []string{ + "group_lemmy_world.json", + "person_lemmy_world.json", + "page_lemmy_world.json", + "note_lemmy_zip.json", + "announce_create_page_lemmy_world.json", + "announce_create_note.json", + "announce_like.json", + "delete_page.json", + "outbox_lemmy_world.json", + } + for _, name := range fixtures { + first := parseFixture(t, name) + encoded, err := json.Marshal(first) + require.NoError(t, err, "%s must re-marshal", name) + second, err := ParseObject(encoded) + require.NoError(t, err, "%s must re-parse", name) + assert.Equal(t, first.ID, second.ID, name) + assert.Equal(t, first.Type, second.Type, name) + assert.Equal(t, len(first.OrderedItems), len(second.OrderedItems), name) + if first.Object != nil { + require.NotNil(t, second.Object, name) + assert.Equal(t, first.Object.ID, second.Object.ID, name) + } + } +} + +func TestParseObjectRejectsNonObjects(t *testing.T) { + for _, raw := range []string{``, `"just-a-string"`, `[1,2,3]`, `null`} { + _, err := ParseObject([]byte(raw)) + assert.Error(t, err, "payload %q must be rejected", raw) + } +} diff --git a/internal/ap/webfinger.go b/internal/ap/webfinger.go new file mode 100644 index 0000000..42dec5f --- /dev/null +++ b/internal/ap/webfinger.go @@ -0,0 +1,159 @@ +package ap + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + "tidepool/internal/errors" +) + +// WebFingerLink is one entry of a JRD document's links array. +type WebFingerLink struct { + Rel string `json:"rel,omitempty"` + Type string `json:"type,omitempty"` + Href string `json:"href,omitempty"` +} + +// WebFingerResponse is the JRD document served at /.well-known/webfinger. +type WebFingerResponse struct { + Subject string `json:"subject,omitempty"` + Aliases []string `json:"aliases,omitempty"` + Links []WebFingerLink `json:"links,omitempty"` +} + +// ParseHandle splits a fediverse handle into username and host. It accepts +// the forms Lemmy and Mastodon users write: +// +// user@instance, @user@instance, !community@instance, acct:user@instance +// +// The leading ! (Lemmy community sigil) and @ carry no meaning at the +// WebFinger layer — communities and users resolve identically. +func ParseHandle(handle string) (username, host string, err error) { + trimmed := strings.TrimSpace(handle) + trimmed = strings.TrimPrefix(trimmed, "acct:") + trimmed = strings.TrimPrefix(trimmed, "!") + trimmed = strings.TrimPrefix(trimmed, "@") + + username, host, found := strings.Cut(trimmed, "@") + // Reject anything that would let the host smuggle a path, query, fragment, + // or second authority into the WebFinger URL we build from it. + if !found || username == "" || host == "" || strings.ContainsAny(host, "@/ ?#") { + return "", "", errors.NewValidationError("handle", + fmt.Sprintf("%q is not of the form user@instance or !community@instance", handle)) + } + return username, host, nil +} + +// ResolveHandle resolves a fediverse handle (user@instance or +// !community@instance) to the actor's canonical AP id via WebFinger, +// preferring the rel="self" link with an ActivityPub media type. +func (c *Client) ResolveHandle(ctx context.Context, handle string) (actorURL string, err error) { + username, host, err := ParseHandle(handle) + if err != nil { + return "", err + } + + acct := fmt.Sprintf("acct:%s@%s", username, host) + query := url.Values{} + query.Set("resource", acct) + webfingerURL := fmt.Sprintf("https://%s/.well-known/webfinger?%s", host, query.Encode()) + + body, err := c.getDedupedMode(ctx, webfingerURL, fetchModeWebFinger) + if err != nil { + // Only a genuine 404 means "no such account". A 401/403 (Cloudflare, a + // defederating instance) is surfaced as-is so the caller can tell a + // blocked lookup apart from a missing account. + if errors.IsNotFound(err) { + return "", errors.NewNotFoundError("webfinger account", handle) + } + return "", fmt.Errorf("ap: webfinger %s: %w", handle, err) + } + + var jrd WebFingerResponse + if err := json.Unmarshal(body, &jrd); err != nil { + return "", fmt.Errorf("ap: webfinger %s: parse JRD: %w", handle, err) + } + + // Host-confusion guard: a self link must resolve to the same authority we + // queried (the interoperable WebFinger rule) — otherwise instance A could + // claim to speak for actors on instance B — OR the JRD subject must be the + // exact acct we asked for. Reject anything else. + subjectMatches := strings.EqualFold(jrd.Subject, acct) + + // Prefer rel=self with an AP media type; fall back to any rel=self. + var fallback string + for _, link := range jrd.Links { + if link.Rel != "self" || link.Href == "" { + continue + } + if !hrefAuthorityMatches(link.Href, host) && !subjectMatches { + continue + } + if isActivityJSONType(link.Type) { + return link.Href, nil + } + if fallback == "" { + fallback = link.Href + } + } + if fallback != "" { + return fallback, nil + } + return "", errors.NewNotFoundError("webfinger self link", handle) +} + +// hrefAuthorityMatches reports whether an href's authority (host:port) equals +// the queried WebFinger host. The comparison is case-insensitive. +func hrefAuthorityMatches(href, host string) bool { + u, err := url.Parse(href) + if err != nil || u.Host == "" { + return false + } + return strings.EqualFold(u.Host, host) +} + +// ActorHandle derives the canonical fediverse handle for an actor document +// (the reverse of ResolveHandle): preferredUsername@host-of-actor-id, with a +// leading ! for Groups (Lemmy community convention). +func ActorHandle(actor *Object) (string, error) { + if actor == nil { + return "", errors.NewValidationError("actor", "must not be nil") + } + if actor.PreferredUsername == "" { + return "", errors.NewValidationError("preferredUsername", + fmt.Sprintf("actor %s has no preferredUsername", actor.ID)) + } + host := actor.Host() + if host == "" { + return "", errors.NewValidationError("id", + fmt.Sprintf("actor id %q has no usable host", actor.ID)) + } + handle := actor.PreferredUsername + "@" + host + if actor.Type == TypeGroup { + return "!" + handle, nil + } + return handle, nil +} + +// ResolveActorHandle fetches an actor and returns both its handle and the +// fetched document (so callers don't fetch twice). +func (c *Client) ResolveActorHandle(ctx context.Context, actorURL string) (string, *Object, error) { + actor, err := c.FetchActor(ctx, actorURL) + if err != nil { + return "", nil, err + } + handle, err := ActorHandle(actor) + if err != nil { + return "", nil, err + } + return handle, actor, nil +} + +func isActivityJSONType(mediaType string) bool { + mediaType = strings.ToLower(mediaType) + return strings.HasPrefix(mediaType, "application/activity+json") || + strings.HasPrefix(mediaType, "application/ld+json") +} diff --git a/internal/ap/webfinger_test.go b/internal/ap/webfinger_test.go new file mode 100644 index 0000000..5af6a51 --- /dev/null +++ b/internal/ap/webfinger_test.go @@ -0,0 +1,211 @@ +package ap + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" +) + +func TestParseHandle(t *testing.T) { + cases := []struct { + in string + user string + host string + expectOK bool + }{ + {"technology@lemmy.world", "technology", "lemmy.world", true}, + {"!technology@lemmy.world", "technology", "lemmy.world", true}, + {"@alice@lemmy.world", "alice", "lemmy.world", true}, + {"acct:alice@lemmy.world", "alice", "lemmy.world", true}, + {" alice@lemmy.world ", "alice", "lemmy.world", true}, + {"alice@127.0.0.1:8443", "alice", "127.0.0.1:8443", true}, + {"alice", "", "", false}, + {"@lemmy.world", "", "", false}, + {"alice@", "", "", false}, + {"alice@lemmy.world/evil", "", "", false}, + {"alice@lemmy@world", "", "", false}, + {"", "", "", false}, + } + for _, tc := range cases { + user, host, err := ParseHandle(tc.in) + if !tc.expectOK { + assert.Error(t, err, "handle %q must be rejected", tc.in) + assert.True(t, errors.IsValidation(err), "handle %q error must be a validation error", tc.in) + continue + } + require.NoError(t, err, "handle %q", tc.in) + assert.Equal(t, tc.user, user, "handle %q", tc.in) + assert.Equal(t, tc.host, host, "handle %q", tc.in) + } +} + +// webfingerTestServer runs a TLS test server (ResolveHandle always speaks +// https) and returns it plus a client trusting its certificate. +func webfingerTestServer(t *testing.T, handler http.Handler) (*httptest.Server, *Client, string) { + t.Helper() + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + host := strings.TrimPrefix(server.URL, "https://") + + httpClient := server.Client() + client := NewClient(ClientOptions{ + UserAgent: "tidepool-test/0", + HTTPClient: httpClient, + AllowPrivateAddresses: true, // httptest server on 127.0.0.1 + }) + return server, client, host +} + +func TestResolveHandle_Community(t *testing.T) { + var sawResource string + var host string + _, client, h := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/.well-known/webfinger", r.URL.Path) + sawResource = r.URL.Query().Get("resource") + w.Header().Set("Content-Type", "application/jrd+json") + // Same shape as the live lemmy.world JRD, but self-hosted so the self + // link's authority matches the queried WebFinger host (the interoperable + // rule the host-confusion guard enforces). + _, _ = fmt.Fprintf(w, `{"subject":"acct:technology@%s","links":[ + {"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://%s/c/technology"}, + {"rel":"self","type":"application/activity+json","href":"https://%s/c/technology"} + ]}`, host, host, host) + })) + host = h + + actorURL, err := client.ResolveHandle(context.Background(), "!technology@"+host) + require.NoError(t, err) + assert.Equal(t, "https://"+host+"/c/technology", actorURL) + assert.Equal(t, "acct:technology@"+host, sawResource, + "the ! sigil must be stripped from the acct: resource") +} + +// TestResolveHandle_RejectsCrossHostSelfLink covers the WebFinger host-confusion +// guard: instance A must not be able to hand back a self link pointing at +// instance B (claiming to speak for B's actors). The subject also does not +// match, so there is no legitimate override. +func TestResolveHandle_RejectsCrossHostSelfLink(t *testing.T) { + _, client, host := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // subject names a foreign account and the self href is on evil.example. + _, _ = w.Write([]byte(`{"subject":"acct:alice@lemmy.world","links":[ + {"rel":"self","type":"application/activity+json","href":"https://evil.example/u/alice"} + ]}`)) + })) + + _, err := client.ResolveHandle(context.Background(), "alice@"+host) + require.Error(t, err) + assert.True(t, errors.IsNotFound(err), + "a self link on a different authority than the queried host must be rejected") +} + +// TestResolveHandle_AcceptsSubjectMatch: when the self href is cross-host but +// the JRD subject exactly matches the requested acct, the interoperable rule +// permits it (some redirect setups legitimately do this). +func TestResolveHandle_AcceptsSubjectMatch(t *testing.T) { + var host string + _, client, h := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = fmt.Fprintf(w, `{"subject":"acct:alice@%s","links":[ + {"rel":"self","type":"application/activity+json","href":"https://actors.example/u/alice"} + ]}`, host) + })) + host = h + + actorURL, err := client.ResolveHandle(context.Background(), "alice@"+host) + require.NoError(t, err) + assert.Equal(t, "https://actors.example/u/alice", actorURL, + "an exact subject match authorizes a cross-host self link") +} + +// TestResolveHandle_Forbidden: a 403 on WebFinger (Cloudflare, defederation) +// must be distinguishable from a genuinely missing account, not flattened to +// not-found (finding 9). +func TestResolveHandle_Forbidden(t *testing.T) { + _, client, host := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + + _, err := client.ResolveHandle(context.Background(), "alice@"+host) + require.Error(t, err) + assert.False(t, errors.IsNotFound(err), + "a 403 must NOT read as account-does-not-exist") + var httpErr HTTPError + require.ErrorAs(t, err, &httpErr, "the underlying status must be preserved") + assert.Equal(t, http.StatusForbidden, httpErr.StatusCode) +} + +func TestResolveHandle_NoSelfLink(t *testing.T) { + _, client, host := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"subject":"acct:x@y","links":[{"rel":"http://webfinger.net/rel/profile-page","type":"text/html","href":"https://x.example/@x"}]}`)) + })) + + _, err := client.ResolveHandle(context.Background(), "x@"+host) + require.Error(t, err) + assert.True(t, errors.IsNotFound(err)) +} + +func TestResolveHandle_UnknownAccount(t *testing.T) { + _, client, host := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + + _, err := client.ResolveHandle(context.Background(), "ghost@"+host) + require.Error(t, err) + assert.True(t, errors.IsNotFound(err)) +} + +func TestResolveHandle_PrefersActivityJSONLink(t *testing.T) { + var host string + _, client, h := webfingerTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Both self links are on the queried host; the AP-typed one wins. + _, _ = fmt.Fprintf(w, `{"subject":"acct:x@%s","links":[ + {"rel":"self","type":"text/html","href":"https://%s/html"}, + {"rel":"self","type":"application/activity+json","href":"https://%s/ap"} + ]}`, host, host, host) + })) + host = h + + actorURL, err := client.ResolveHandle(context.Background(), "x@"+host) + require.NoError(t, err) + assert.Equal(t, "https://"+host+"/ap", actorURL) +} + +func TestActorHandle(t *testing.T) { + group := parseFixture(t, "group_lemmy_world.json") + handle, err := ActorHandle(group) + require.NoError(t, err) + assert.Equal(t, "!technology@lemmy.world", handle, "groups get the Lemmy ! sigil") + + person := parseFixture(t, "person_lemmy_world.json") + handle, err = ActorHandle(person) + require.NoError(t, err) + assert.Equal(t, "LeftLeaningFreedomFighters@lemmy.world", handle) + + _, err = ActorHandle(&Object{ID: "https://x.example/u/anon", Type: TypePerson}) + assert.True(t, errors.IsValidation(err), "actor without preferredUsername must fail") + + _, err = ActorHandle(nil) + assert.True(t, errors.IsValidation(err)) +} + +func TestResolveActorHandle(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(loadFixture(t, "person_lemmy_world.json")) + })) + defer server.Close() + + client := NewClient(ClientOptions{UserAgent: "tidepool-test/0", AllowPrivateAddresses: true}) + handle, actor, err := client.ResolveActorHandle(context.Background(), server.URL+"/u/x") + require.NoError(t, err) + // The handle host comes from the actor's canonical id, not the URL we + // fetched from. + assert.Equal(t, "LeftLeaningFreedomFighters@lemmy.world", handle) + assert.Equal(t, TypePerson, actor.Type) +} diff --git a/internal/config/config.go b/internal/config/config.go index c06c122..0d6abaf 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "os" + "strings" ) const ( @@ -33,6 +34,12 @@ type Config struct { BridgeServiceDID string // UserAgent is sent on all outbound HTTP requests (signed fetches etc.). UserAgent string + // AllowPrivateAddresses disables the SSRF egress guard, letting the AP + // client fetch loopback/private/link-local/metadata addresses. It defaults + // to false (guard on) and must only be enabled for local development or + // tests that hit httptest servers on 127.0.0.1. Set ALLOW_PRIVATE_FETCH=1 + // to enable. + AllowPrivateAddresses bool } // Load reads configuration from the environment. logger must not be nil; @@ -75,6 +82,14 @@ func Load(logger *slog.Logger) (*Config, error) { // bridge's service DID, otherwise identity bootstrap mints one. cfg.BridgeServiceDID = os.Getenv("BRIDGE_SERVICE_DID") + // SSRF egress guard is on by default; only local dev/tests should relax + // it. Accept it in development but refuse to let production disable the + // guard silently. + cfg.AllowPrivateAddresses = boolVar("ALLOW_PRIVATE_FETCH") + if cfg.AllowPrivateAddresses && !isDevelopment { + return nil, fmt.Errorf("config: ALLOW_PRIVATE_FETCH must not be set in production") + } + defaultUserAgent := fmt.Sprintf("tidepool/0.1 (+https://%s)", cfg.BridgeHostname) cfg.UserAgent = os.Getenv("USER_AGENT") if cfg.UserAgent == "" { @@ -91,6 +106,16 @@ func (c *Config) IsDevelopment() bool { return c.Environment == EnvironmentDevelopment } +// boolVar reports whether an environment variable is set to a truthy value +// ("1", "true", "yes", case-insensitive). +func boolVar(name string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) { + case "1", "true", "yes", "on": + return true + } + return false +} + // stringVar returns the value of an environment variable. When unset it // falls back to the logged dev default in development and errors in // production. diff --git a/internal/db/migrations/005_create_service_keys.sql b/internal/db/migrations/005_create_service_keys.sql new file mode 100644 index 0000000..50a14e6 --- /dev/null +++ b/internal/db/migrations/005_create_service_keys.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- service_keys persists the bridge's own long-lived key material, keyed by +-- purpose name. Today it holds one row: the service actor's AP-side RSA +-- private key (PEM, PKCS#8) under the name 'service-actor'. This is the +-- bridge's OWN key, not escrowed user key material (that lives on +-- bridged_actors.signing_key), so it is stored unencrypted like any other +-- service credential. +-- Unique constraints carry explicit names so the store layer can map +-- SQLSTATE 23505 violations to precise conflict errors. +CREATE TABLE service_keys ( + id BIGSERIAL PRIMARY KEY, + name TEXT NOT NULL CHECK (name <> ''), -- key purpose, e.g. service-actor + private_key_pem BYTEA NOT NULL CHECK (length(private_key_pem) > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT service_keys_name_key UNIQUE (name) +); + +-- +goose Down +DROP TABLE IF EXISTS service_keys; diff --git a/internal/store/interfaces.go b/internal/store/interfaces.go index b1d2f61..99537f3 100644 --- a/internal/store/interfaces.go +++ b/internal/store/interfaces.go @@ -119,6 +119,21 @@ type Communities interface { ListByFollowState(ctx context.Context, state FollowState) ([]*Community, error) } +// ServiceKeys persists the bridge's own long-lived keys (today: the service +// actor's AP-side RSA private key). Keys are create-once: there is no update +// or delete, so a stored key can never be silently rotated out from under +// signatures already in flight. +type ServiceKeys interface { + // Create inserts a new named key and returns the stored row. An existing + // name returns an error satisfying errors.IsAlreadyExists — callers that + // lose a bootstrap race must Get the winner's key instead. + Create(ctx context.Context, name string, privateKeyPEM []byte) (*ServiceKey, error) + + // Get returns the key for a purpose name. A missing key is an error + // satisfying errors.IsNotFound. + Get(ctx context.Context, name string) (*ServiceKey, error) +} + // InboxEvents deduplicates inbound AP activities and records processing // outcomes. The queue-consumption side (ListPending and friends) is // deliberately deferred to task 06, which owns the processing loop. diff --git a/internal/store/migrations_test.go b/internal/store/migrations_test.go index d7c79ef..982cd14 100644 --- a/internal/store/migrations_test.go +++ b/internal/store/migrations_test.go @@ -24,7 +24,7 @@ func TestMigrations_UpDownUp(t *testing.T) { err := database.QueryRowContext(ctx, ` SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public' - AND table_name IN ('ap_objects', 'bridged_actors', 'communities', 'inbox_events') + AND table_name IN ('ap_objects', 'bridged_actors', 'communities', 'inbox_events', 'service_keys') `).Scan(&remaining) require.NoError(t, err) assert.Zero(t, remaining, "down migrations must drop every Tidepool table") @@ -54,6 +54,7 @@ func TestMigrations_UniqueConstraintNames(t *testing.T) { "communities_ap_group_id_key", "communities_did_key", "inbox_events_activity_id_key", + "service_keys_name_key", } for _, name := range expected { var exists bool diff --git a/internal/store/models.go b/internal/store/models.go index 4f3e4ba..8490c58 100644 --- a/internal/store/models.go +++ b/internal/store/models.go @@ -132,6 +132,16 @@ type Community struct { CreatedAt time.Time } +// ServiceKey is one of the bridge's own long-lived keys, keyed by purpose +// name (e.g. "service-actor" for the AP-side RSA signing key, stored as +// PKCS#8 PEM). +type ServiceKey struct { + ID int64 + Name string + PrivateKeyPEM []byte + CreatedAt time.Time +} + // InboxEvent is a received AP activity, recorded for dedupe and // processing bookkeeping. type InboxEvent struct { diff --git a/internal/store/service_keys.go b/internal/store/service_keys.go new file mode 100644 index 0000000..41e7cb1 --- /dev/null +++ b/internal/store/service_keys.go @@ -0,0 +1,66 @@ +package store + +import ( + "context" + "database/sql" + stderrors "errors" + "fmt" + + "tidepool/internal/errors" +) + +type postgresServiceKeys struct { + db *sql.DB +} + +// NewServiceKeys creates the postgres-backed service_keys repository. +func NewServiceKeys(db *sql.DB) ServiceKeys { + return &postgresServiceKeys{db: db} +} + +const serviceKeyColumns = ` id, name, private_key_pem, created_at` + +func (r *postgresServiceKeys) Create(ctx context.Context, name string, privateKeyPEM []byte) (*ServiceKey, error) { + if name == "" { + return nil, errors.NewValidationError("name", "must not be empty") + } + if len(privateKeyPEM) == 0 { + return nil, errors.NewValidationError("private_key_pem", "must not be empty") + } + + query := ` + INSERT INTO service_keys (name, private_key_pem) + VALUES ($1, $2) + RETURNING` + serviceKeyColumns + + key, err := scanServiceKey(r.db.QueryRowContext(ctx, query, name, privateKeyPEM)) + if err != nil { + // Keys are create-once by design: a concurrent bootstrap losing the + // insert race must re-Get the winner's key, never overwrite it. + if constraint, ok := uniqueViolation(err); ok && constraint == "service_keys_name_key" { + return nil, errors.NewConflictError("service_key", "name", name) + } + return nil, fmt.Errorf("create service_key %q: %w", name, err) + } + return key, nil +} + +func (r *postgresServiceKeys) Get(ctx context.Context, name string) (*ServiceKey, error) { + query := `SELECT` + serviceKeyColumns + ` FROM service_keys WHERE name = $1` + key, err := scanServiceKey(r.db.QueryRowContext(ctx, query, name)) + if err != nil { + if stderrors.Is(err, sql.ErrNoRows) { + return nil, errors.NewNotFoundError("service_key", name) + } + return nil, fmt.Errorf("get service_key %q: %w", name, err) + } + return key, nil +} + +func scanServiceKey(row rowScanner) (*ServiceKey, error) { + var key ServiceKey + if err := row.Scan(&key.ID, &key.Name, &key.PrivateKeyPEM, &key.CreatedAt); err != nil { + return nil, err + } + return &key, nil +} diff --git a/internal/store/service_keys_test.go b/internal/store/service_keys_test.go new file mode 100644 index 0000000..2055092 --- /dev/null +++ b/internal/store/service_keys_test.go @@ -0,0 +1,82 @@ +package store + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/errors" +) + +func TestServiceKeys_CreateAndGet(t *testing.T) { + repo := NewServiceKeys(testDB(t)) + ctx := context.Background() + + pemBytes := []byte("-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n") + created, err := repo.Create(ctx, "service-actor", pemBytes) + require.NoError(t, err) + assert.NotZero(t, created.ID) + assert.Equal(t, "service-actor", created.Name) + assert.Equal(t, pemBytes, created.PrivateKeyPEM) + assert.False(t, created.CreatedAt.IsZero()) + + got, err := repo.Get(ctx, "service-actor") + require.NoError(t, err) + assert.Equal(t, created.ID, got.ID) + assert.Equal(t, pemBytes, got.PrivateKeyPEM) +} + +func TestServiceKeys_CreateExistingNameConflicts(t *testing.T) { + repo := NewServiceKeys(testDB(t)) + ctx := context.Background() + + original := []byte("original-key-pem") + _, err := repo.Create(ctx, "service-actor", original) + require.NoError(t, err) + + // Create is create-once: the second insert must conflict and must NOT + // overwrite the stored key (a rotated key would invalidate the actor + // document other instances already served). + _, err = repo.Create(ctx, "service-actor", []byte("usurper-key-pem")) + require.Error(t, err) + assert.True(t, errors.IsAlreadyExists(err), "second create must satisfy IsAlreadyExists, got %v", err) + + got, err := repo.Get(ctx, "service-actor") + require.NoError(t, err) + assert.Equal(t, original, got.PrivateKeyPEM, "losing create must not clobber the stored key") +} + +func TestServiceKeys_GetMissingIsNotFound(t *testing.T) { + repo := NewServiceKeys(testDB(t)) + + _, err := repo.Get(context.Background(), "nope") + require.Error(t, err) + assert.True(t, errors.IsNotFound(err), "missing key must satisfy IsNotFound, got %v", err) +} + +func TestServiceKeys_CreateValidation(t *testing.T) { + repo := NewServiceKeys(testDB(t)) + ctx := context.Background() + + _, err := repo.Create(ctx, "", []byte("pem")) + assert.True(t, errors.IsValidation(err), "empty name must fail validation, got %v", err) + + _, err = repo.Create(ctx, "service-actor", nil) + assert.True(t, errors.IsValidation(err), "empty key must fail validation, got %v", err) +} + +func TestServiceKeys_DistinctNamesCoexist(t *testing.T) { + repo := NewServiceKeys(testDB(t)) + ctx := context.Background() + + _, err := repo.Create(ctx, "service-actor", []byte("key-a")) + require.NoError(t, err) + _, err = repo.Create(ctx, "some-future-key", []byte("key-b")) + require.NoError(t, err) + + got, err := repo.Get(ctx, "some-future-key") + require.NoError(t, err) + assert.Equal(t, []byte("key-b"), got.PrivateKeyPEM) +} diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5b3176d..a9e0c47 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -54,7 +54,7 @@ func testDB(t *testing.T) *sql.DB { require.NoError(t, testDatabaseErr, "connect and migrate test database") _, err := testDatabase.ExecContext(context.Background(), - `TRUNCATE ap_objects, bridged_actors, communities, inbox_events RESTART IDENTITY`) + `TRUNCATE ap_objects, bridged_actors, communities, inbox_events, service_keys RESTART IDENTITY`) require.NoError(t, err, "truncate test tables") return testDatabase