From d66b15cdb4e29cfc7e3300ab45edf07ab81f1b90 Mon Sep 17 00:00:00 2001 From: dawn Date: Wed, 2 Sep 2026 00:15:55 +0900 Subject: [PATCH] xrpc,knotmirror: add service-auth signer Signed-off-by: dawn --- cmd/deliberi/main.go | 2 - cmd/knotmirror/main.go | 18 +++- docker-compose.yml | 5 + knotmirror/config/config.go | 16 ++++ knotmirror/config/config_test.go | 53 +++++++++++ knotmirror/knotmirror.go | 19 +++- knotmirror/xrpc/xrpc.go | 6 ++ localinfra/deliberi.Dockerfile | 2 +- localinfra/knotmirror.Dockerfile | 2 +- localinfra/readme.md | 6 ++ nix/modules/knotmirror.nix | 32 +++++++ xrpc/serviceauth/signer.go | 65 +++++++++++++ xrpc/serviceauth/signer_test.go | 154 +++++++++++++++++++++++++++++++ 13 files changed, 373 insertions(+), 7 deletions(-) create mode 100644 knotmirror/config/config_test.go create mode 100644 xrpc/serviceauth/signer.go create mode 100644 xrpc/serviceauth/signer_test.go diff --git a/cmd/deliberi/main.go b/cmd/deliberi/main.go index 4760b66d1..64f74b799 100644 --- a/cmd/deliberi/main.go +++ b/cmd/deliberi/main.go @@ -45,11 +45,9 @@ func run(args []string) error { } func runDeliberi(ctx context.Context, cmd *cli.Command) error { - logger := log.FromContext(ctx) cfg, err := config.Load(ctx) if err != nil { return err } - logger.Debug("config loaded", "config", cfg) return deliberi.Run(ctx, cfg) } diff --git a/cmd/knotmirror/main.go b/cmd/knotmirror/main.go index 9867d68dc..79dc8768c 100644 --- a/cmd/knotmirror/main.go +++ b/cmd/knotmirror/main.go @@ -2,11 +2,13 @@ package main import ( "context" + "fmt" "log/slog" "os" "os/signal" "syscall" + "github.com/bluesky-social/indigo/atproto/atcrypto" "github.com/carlmjohnson/versioninfo" "github.com/urfave/cli/v3" "tangled.org/core/knotmirror" @@ -50,18 +52,30 @@ func run(args []string) error { Action: runMigrateDisk, Flags: []cli.Flag{}, }, + { + Name: "generate-service-auth-key", + Usage: "generate a private key for service authentication", + Action: generateServiceAuthKey, + }, } return app.Run(ctx, args) } +func generateServiceAuthKey(context.Context, *cli.Command) error { + key, err := atcrypto.GeneratePrivateKeyK256() + if err != nil { + return fmt.Errorf("generating service private key: %w", err) + } + fmt.Printf("MIRROR_SERVICE_PRIVATE_KEY=%s\n", key.Multibase()) + return nil +} + func runKnotMirror(ctx context.Context, cmd *cli.Command) error { - logger := log.FromContext(ctx) cfg, err := config.Load(ctx) if err != nil { return err } - logger.Debug("config loaded:", "config", cfg) return knotmirror.Run(ctx, cfg) } diff --git a/docker-compose.yml b/docker-compose.yml index b0e3166f2..e65019629 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -307,6 +307,10 @@ services: MIRROR_APPVIEW_URL: http://appview:3000 MIRROR_SEARCH_ZOEKT_URL: https://zoekt.tngl.boltless.dev/indexserver MIRROR_V2_HOST: http://gitmirror:9001 + MIRROR_DELIBERI_URL: http://deliberi:6565 + MIRROR_DELIBERI_DID: did:web:deliberi.tngl.boltless.dev + env_file: + - ./localinfra/certs/service-auth.env volumes: - knotmirror-data:/data - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro @@ -439,6 +443,7 @@ services: DELIBERI_PDS_HOST: https://pds.tngl.boltless.dev # PDS admin password for minting invite codes + creating accounts DELIBERI_PDS_ADMIN_SECRET: ${TANGLED_PDS_ADMIN_SECRET} + DELIBERI_TRUSTED_SERVICE_DIDS: did:web:mirror.tngl.boltless.dev volumes: - deliberi-data:/var/lib/deliberi - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro diff --git a/knotmirror/config/config.go b/knotmirror/config/config.go index 6344079f6..60987c0ec 100644 --- a/knotmirror/config/config.go +++ b/knotmirror/config/config.go @@ -2,8 +2,10 @@ package config import ( "context" + "fmt" "time" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/sethvargo/go-envconfig" ) @@ -26,6 +28,9 @@ type Config struct { MetricsListen string `env:"MIRROR_METRICS_LISTEN, default=127.0.0.1:7100"` AdminListen string `env:"MIRROR_ADMIN_LISTEN, default=127.0.0.1:7200"` V2Host string `env:"MIRROR_V2_HOST"` + DeliberiURL string `env:"MIRROR_DELIBERI_URL"` + DeliberiDID string `env:"MIRROR_DELIBERI_DID"` + ServicePrivateKey string `env:"MIRROR_SERVICE_PRIVATE_KEY"` } func (c *Config) BaseUrl() string { @@ -49,5 +54,16 @@ func Load(ctx context.Context) (*Config, error) { if err := envconfig.Process(ctx, &cfg); err != nil { return nil, err } + if (cfg.DeliberiURL == "") != (cfg.DeliberiDID == "") { + return nil, fmt.Errorf("MIRROR_DELIBERI_URL and MIRROR_DELIBERI_DID must be set together") + } + if cfg.DeliberiURL != "" && cfg.ServicePrivateKey == "" { + return nil, fmt.Errorf("MIRROR_SERVICE_PRIVATE_KEY is required when Deliberi is configured") + } + if cfg.DeliberiDID != "" { + if _, err := syntax.ParseDID(cfg.DeliberiDID); err != nil { + return nil, fmt.Errorf("MIRROR_DELIBERI_DID: %w", err) + } + } return &cfg, nil } diff --git a/knotmirror/config/config_test.go b/knotmirror/config/config_test.go new file mode 100644 index 000000000..bb8ca660b --- /dev/null +++ b/knotmirror/config/config_test.go @@ -0,0 +1,53 @@ +package config + +import ( + "context" + "testing" +) + +func TestLoadRequiresCompleteDeliberiServiceAuth(t *testing.T) { + for _, test := range []struct { + name string + url string + did string + privateKey string + wantError bool + }{ + {name: "disabled"}, + {name: "signer only", privateKey: "z-private-key-validated-at-startup"}, + { + name: "configured", + url: "http://deliberi.test", + did: "did:web:deliberi.test", + privateKey: "z-private-key-validated-at-startup", + }, + {name: "url only", url: "http://deliberi.test", wantError: true}, + {name: "did only", did: "did:web:deliberi.test", wantError: true}, + { + name: "invalid audience DID", + url: "http://deliberi.test", + did: "not-a-did", + privateKey: "z-private-key-validated-at-startup", + wantError: true, + }, + { + name: "missing key", + url: "http://deliberi.test", + did: "did:web:deliberi.test", + wantError: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("MIRROR_DB_URL", "postgresql://test") + t.Setenv("MIRROR_REDIS_ADDR", "127.0.0.1:6379") + t.Setenv("MIRROR_HOSTNAME", "mirror.test") + t.Setenv("MIRROR_DELIBERI_URL", test.url) + t.Setenv("MIRROR_DELIBERI_DID", test.did) + t.Setenv("MIRROR_SERVICE_PRIVATE_KEY", test.privateKey) + _, err := Load(context.Background()) + if (err != nil) != test.wantError { + t.Fatalf("Load() error = %v, wantError = %v", err, test.wantError) + } + }) + } +} diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go index b2f042db4..188138310 100644 --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -18,6 +18,7 @@ import ( "tangled.org/core/knotmirror/repoindexer" "tangled.org/core/knotmirror/xrpc" "tangled.org/core/log" + "tangled.org/core/xrpc/serviceauth" ) func Run(ctx context.Context, cfg *config.Config) error { @@ -27,6 +28,18 @@ func Run(ctx context.Context, cfg *config.Config) error { logger := log.FromContext(ctx) + var serviceSigner *serviceauth.Signer + if cfg.ServicePrivateKey != "" { + var err error + serviceSigner, err = serviceauth.NewSigner( + serviceauth.DidWeb(cfg.Hostname), + cfg.ServicePrivateKey, + ) + if err != nil { + return fmt.Errorf("initializing service auth signer: %w", err) + } + } + db, err := db.Make(ctx, cfg.DbUrl, 32) if err != nil { return fmt.Errorf("initializing db: %w", err) @@ -65,6 +78,7 @@ func Run(ctx context.Context, cfg *config.Config) error { crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, indexScheduler, cfg) xrpc := xrpc.New(logger, cfg, db, rdb, indexer, resolver, knotstream) + xrpc.SetServiceSigner(serviceSigner) adminpage := NewAdminServer(logger, db, resyncer, xrpc, resolver) // maintain repository list with tap @@ -77,8 +91,11 @@ func Run(ctx context.Context, cfg *config.Config) error { mux := chi.NewRouter() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Write([]byte("Welcome to a knotmirror server.\n")) + _, _ = w.Write([]byte("Welcome to a knotmirror server.\n")) }) + if serviceSigner != nil { + mux.Get("/.well-known/did.json", serviceSigner.ServeDIDDocument) + } mux.Mount("/xrpc", xrpc.Router()) if err := http.ListenAndServe(cfg.Listen, mux); err != nil { diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go index 82e06cadd..988e7d784 100644 --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -20,6 +20,7 @@ import ( "tangled.org/core/knotmirror/knotstream" "tangled.org/core/knotmirror/repoindexer" "tangled.org/core/log" + "tangled.org/core/xrpc/serviceauth" ) type Xrpc struct { @@ -35,6 +36,7 @@ type Xrpc struct { // write usually takes longer. for example; merge v2WriteClient *http.Client inflight *inflightTracker + serviceSigner *serviceauth.Signer } func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, indexer *repoindexer.Indexer, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { @@ -59,6 +61,10 @@ func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, } } +func (x *Xrpc) SetServiceSigner(signer *serviceauth.Signer) { + x.serviceSigner = signer +} + func (x *Xrpc) Router() http.Handler { r := chi.NewRouter() r.Use(metricsMiddleware) diff --git a/localinfra/deliberi.Dockerfile b/localinfra/deliberi.Dockerfile index 8485aca26..8f579e3b3 100644 --- a/localinfra/deliberi.Dockerfile +++ b/localinfra/deliberi.Dockerfile @@ -1,6 +1,6 @@ # Development only. Not for production use. -FROM golang:1.25-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache git build-base sqlite-dev diff --git a/localinfra/knotmirror.Dockerfile b/localinfra/knotmirror.Dockerfile index 807f56e23..b7efebef2 100644 --- a/localinfra/knotmirror.Dockerfile +++ b/localinfra/knotmirror.Dockerfile @@ -1,6 +1,6 @@ # Development only. Not for production use. -FROM golang:1.25-alpine AS build +FROM golang:1.26-alpine AS build RUN apk add --no-cache git diff --git a/localinfra/readme.md b/localinfra/readme.md index ebee64d40..03103e0f1 100644 --- a/localinfra/readme.md +++ b/localinfra/readme.md @@ -41,6 +41,12 @@ To make that work: -addext "basicConstraints=critical,CA:TRUE,pathlen:1" \ -addext "keyUsage=critical,keyCertSign,cRLSign" \ -addext "nameConstraints=critical,permitted;DNS:tngl.boltless.dev" + + test -f localinfra/certs/service-auth.env || ( + umask 077 + nix develop .#default -c go run ./cmd/knotmirror generate-service-auth-key \ + > localinfra/certs/service-auth.env + ) ``` 2. Trust generated `localinfra/certs/root.crt` in your system's trust store. - For example in MacOS, run diff --git a/nix/modules/knotmirror.nix b/nix/modules/knotmirror.nix index 07abe8050..4e732d06a 100644 --- a/nix/modules/knotmirror.nix +++ b/nix/modules/knotmirror.nix @@ -67,6 +67,24 @@ in description = "Database URL. postgresql expected (required)"; }; + deliberiUrl = mkOption { + type = types.str; + default = ""; + description = "base URL for Deliberi XRPC calls"; + }; + + deliberiDid = mkOption { + type = types.str; + default = ""; + description = "Deliberi service DID used as the service-auth audience"; + }; + + environmentFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "file with secret env vars such as MIRROR_SERVICE_PRIVATE_KEY"; + }; + atpPlcUrl = mkOption { type = types.str; default = "https://plc.directory"; @@ -118,6 +136,17 @@ in }; }; config = mkIf cfg.enable { + assertions = [ + { + assertion = (cfg.deliberiUrl == "") == (cfg.deliberiDid == ""); + message = "services.tangled.knotmirror deliberiUrl and deliberiDid must be set together"; + } + { + assertion = cfg.deliberiUrl == "" || cfg.environmentFile != null; + message = "services.tangled.knotmirror environmentFile is required when Deliberi is configured"; + } + ]; + environment.systemPackages = [ pkgs.git cfg.package @@ -163,6 +192,7 @@ in serviceConfig = { LogsDirectory = "knotmirror"; StateDirectory = "knotmirror"; + EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile; Environment = [ # TODO: add environment variables "MIRROR_LISTEN=${cfg.listenAddr}" @@ -179,6 +209,8 @@ in "MIRROR_SLURPER_CONCURRENCY=4" "MIRROR_SEARCH_ZOEKT_URL=${cfg.zoektUrl}" "MIRROR_V2_HOST=http://${cfg.v2.listenAddr}" + "MIRROR_DELIBERI_URL=${cfg.deliberiUrl}" + "MIRROR_DELIBERI_DID=${cfg.deliberiDid}" ]; ExecStart = "${getExe cfg.package} serve"; Restart = "always"; diff --git a/xrpc/serviceauth/signer.go b/xrpc/serviceauth/signer.go new file mode 100644 index 000000000..cead6165e --- /dev/null +++ b/xrpc/serviceauth/signer.go @@ -0,0 +1,65 @@ +package serviceauth + +import ( + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/auth" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type Signer struct { + issuer syntax.DID + privateKey atcrypto.PrivateKey + publicMultibase string +} + +func NewSigner(issuer syntax.DID, privateMultibase string) (*Signer, error) { + issuer, err := syntax.ParseDID(issuer.String()) + if err != nil { + return nil, fmt.Errorf("parsing service DID: %w", err) + } + privateKey, err := atcrypto.ParsePrivateMultibase(privateMultibase) + if err != nil { + return nil, fmt.Errorf("parsing service private key: %w", err) + } + publicKey, err := privateKey.PublicKey() + if err != nil { + return nil, fmt.Errorf("deriving service public key: %w", err) + } + return &Signer{ + issuer: issuer, + privateKey: privateKey, + publicMultibase: publicKey.Multibase(), + }, nil +} + +func (s *Signer) DID() syntax.DID { + return s.issuer +} + +func (s *Signer) Sign(audience syntax.DID, lxm syntax.NSID) (string, error) { + return auth.SignServiceAuth(s.issuer, audience.String(), time.Minute, &lxm, s.privateKey) +} + +func (s *Signer) DIDDocument() identity.DIDDocument { + issuer := s.issuer.String() + return identity.DIDDocument{ + DID: s.issuer, + VerificationMethod: []identity.DocVerificationMethod{{ + ID: issuer + "#atproto", + Type: "Multikey", + Controller: issuer, + PublicKeyMultibase: s.publicMultibase, + }}, + } +} + +func (s *Signer) ServeDIDDocument(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/did+json") + _ = json.NewEncoder(w).Encode(s.DIDDocument()) +} diff --git a/xrpc/serviceauth/signer_test.go b/xrpc/serviceauth/signer_test.go new file mode 100644 index 000000000..b2ca10b57 --- /dev/null +++ b/xrpc/serviceauth/signer_test.go @@ -0,0 +1,154 @@ +package serviceauth + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/bluesky-social/indigo/atproto/atcrypto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" +) + +func TestSignerProducesVerifiableLxmBoundToken(t *testing.T) { + privateKey, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + issuer := syntax.DID("did:web:mirror.example") + signer, err := NewSigner(issuer, privateKey.Multibase()) + if err != nil { + t.Fatal(err) + } + document := signer.DIDDocument() + parsed := identity.ParseIdentity(&document) + directory := identity.NewMockDirectory() + directory.Insert(parsed) + audience := syntax.DID("did:web:deliberi.example") + lxm := syntax.NSID("sh.tangled.identity.resolveCommitters") + token, err := signer.Sign(audience, lxm) + if err != nil { + t.Fatal(err) + } + + var authenticated syntax.DID + handler := NewServiceAuth( + slog.New(slog.NewTextHandler(io.Discard, nil)), + directory, + audience.String(), + ).VerifyServiceAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authenticated, _ = r.Context().Value(ActorDid).(syntax.DID) + w.WriteHeader(http.StatusNoContent) + })) + req := httptest.NewRequest(http.MethodPost, "/xrpc/"+lxm.String(), nil) + req.Header.Set("Authorization", "Bearer "+token) + response := httptest.NewRecorder() + handler.ServeHTTP(response, req) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } + if authenticated != issuer { + t.Fatalf("authenticated DID = %q, want %q", authenticated, issuer) + } +} + +func TestNewSignerRejectsInvalidPrivateKey(t *testing.T) { + if _, err := NewSigner(syntax.DID("did:web:mirror.example"), "not-a-key"); err == nil { + t.Fatal("expected invalid private key error") + } +} + +func TestNewSignerRejectsInvalidDID(t *testing.T) { + privateKey, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + if _, err := NewSigner(syntax.DID("not-a-did"), privateKey.Multibase()); err == nil { + t.Fatal("expected invalid DID error") + } +} + +func TestSignerServesItsDIDDocument(t *testing.T) { + privateKey, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + issuer := syntax.DID("did:web:mirror.example") + signer, err := NewSigner(issuer, privateKey.Multibase()) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + signer.ServeDIDDocument(response, httptest.NewRequest(http.MethodGet, "/.well-known/did.json", nil)) + if response.Header().Get("Content-Type") != "application/did+json" { + t.Fatalf("content type = %q", response.Header().Get("Content-Type")) + } + var document identity.DIDDocument + if err := json.Unmarshal(response.Body.Bytes(), &document); err != nil { + t.Fatal(err) + } + parsed := identity.ParseIdentity(&document) + publicKey, err := parsed.PublicKey() + if err != nil { + t.Fatal(err) + } + expected, err := privateKey.PublicKey() + if err != nil { + t.Fatal(err) + } + if document.DID != issuer || !publicKey.Equal(expected) { + t.Fatalf("served DID document does not match signer") + } +} + +func TestSignerRoundTripsThroughDIDWebResolution(t *testing.T) { + privateKey, err := atcrypto.GeneratePrivateKeyP256() + if err != nil { + t.Fatal(err) + } + var signer *Signer + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + signer.ServeDIDDocument(w, r) + })) + defer server.Close() + issuer := syntax.DID("did:web:mirror.example.com") + signer, err = NewSigner(issuer, privateKey.Multibase()) + if err != nil { + t.Fatal(err) + } + serverAddress := strings.TrimPrefix(server.URL, "https://") + transport := server.Client().Transport.(*http.Transport).Clone() + transport.DialContext = func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, serverAddress) + } + directory := &identity.BaseDirectory{ + HTTPClient: http.Client{Transport: transport}, + SkipHandleVerification: true, + } + audience := syntax.DID("did:web:deliberi.example") + lxm := syntax.NSID("sh.tangled.identity.resolveCommitters") + token, err := signer.Sign(audience, lxm) + if err != nil { + t.Fatal(err) + } + verified := NewServiceAuth( + slog.New(slog.NewTextHandler(io.Discard, nil)), + directory, + audience.String(), + ).VerifyServiceAuth(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + request := httptest.NewRequest(http.MethodPost, "/xrpc/"+lxm.String(), nil) + request.Header.Set("Authorization", "Bearer "+token) + response := httptest.NewRecorder() + verified.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } +} -- 2.51.2