From ed148a410e936abe83c6dbfbd0b0c9632ffe65f4 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Wed, 6 May 2026 22:06:41 +0900 Subject: [PATCH] appview,knotmirror: remove use of `idresolver` package from knotmirror modifying appview code becuase knotmirror depends on `appview/models` Signed-off-by: Seongmin Lee --- appview/models/label.go | 68 ------------------------ appview/state/labels.go | 76 +++++++++++++++++++++++++++ appview/state/state.go | 2 +- knotmirror/adminpage.go | 12 ++--- knotmirror/knotmirror.go | 47 ++++++++++++++--- knotmirror/xrpc/proxy.go | 2 +- knotmirror/xrpc/sync_request_crawl.go | 2 +- knotmirror/xrpc/xrpc.go | 8 +-- 8 files changed, 128 insertions(+), 89 deletions(-) create mode 100644 appview/state/labels.go diff --git a/appview/models/label.go b/appview/models/label.go index 13e1e988..cf1e6937 100644 --- a/appview/models/label.go +++ b/appview/models/label.go @@ -1,20 +1,15 @@ package models import ( - "context" "crypto/sha1" "encoding/hex" - "encoding/json" "errors" "fmt" "slices" "time" - "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/syntax" - "github.com/bluesky-social/indigo/xrpc" "tangled.org/core/api/tangled" - "tangled.org/core/idresolver" ) type ConcreteType string @@ -501,66 +496,3 @@ func ReduceLabelOps(ops []LabelOp) []LabelOp { return result } - -func FetchLabelDefs(r *idresolver.Resolver, aturis []string) ([]LabelDefinition, error) { - var labelDefs []LabelDefinition - ctx := context.Background() - - for _, dl := range aturis { - atUri, err := syntax.ParseATURI(dl) - if err != nil { - return nil, fmt.Errorf("failed to parse AT-URI %s: %v", dl, err) - } - if atUri.Collection() != tangled.LabelDefinitionNSID { - return nil, fmt.Errorf("expected AT-URI pointing %s collection: %s", tangled.LabelDefinitionNSID, atUri) - } - - owner, err := r.ResolveIdent(ctx, atUri.Authority().String()) - if err != nil { - return nil, fmt.Errorf("failed to resolve default label owner DID %s: %v", atUri.Authority(), err) - } - - xrpcc := xrpc.Client{ - Host: owner.PDSEndpoint(), - } - - record, err := atproto.RepoGetRecord( - ctx, - &xrpcc, - "", - atUri.Collection().String(), - atUri.Authority().String(), - atUri.RecordKey().String(), - ) - if err != nil { - return nil, fmt.Errorf("failed to get record for %s: %v", atUri, err) - } - - if record != nil { - bytes, err := record.Value.MarshalJSON() - if err != nil { - return nil, fmt.Errorf("failed to marshal record value for %s: %v", atUri, err) - } - - raw := json.RawMessage(bytes) - labelRecord := tangled.LabelDefinition{} - err = json.Unmarshal(raw, &labelRecord) - if err != nil { - return nil, fmt.Errorf("invalid record for %s: %w", atUri, err) - } - - labelDef, err := LabelDefinitionFromRecord( - atUri.Authority().String(), - atUri.RecordKey().String(), - labelRecord, - ) - if err != nil { - return nil, fmt.Errorf("failed to create label definition from record %s: %v", atUri, err) - } - - labelDefs = append(labelDefs, *labelDef) - } - } - - return labelDefs, nil -} diff --git a/appview/state/labels.go b/appview/state/labels.go new file mode 100644 index 00000000..f2767b87 --- /dev/null +++ b/appview/state/labels.go @@ -0,0 +1,76 @@ +package state + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/bluesky-social/indigo/xrpc" + "tangled.org/core/api/tangled" + "tangled.org/core/appview/models" +) + +func fetchLabelDefs(ctx context.Context, dir identity.Directory, aturis []string) ([]models.LabelDefinition, error) { + var labelDefs []models.LabelDefinition + + for _, dl := range aturis { + atUri, err := syntax.ParseATURI(dl) + if err != nil { + return nil, fmt.Errorf("failed to parse AT-URI %s: %v", dl, err) + } + if atUri.Collection() != tangled.LabelDefinitionNSID { + return nil, fmt.Errorf("expected AT-URI pointing %s collection: %s", tangled.LabelDefinitionNSID, atUri) + } + + owner, err := dir.Lookup(ctx, atUri.Authority()) + if err != nil { + return nil, fmt.Errorf("failed to resolve default label owner DID %s: %v", atUri.Authority(), err) + } + + xrpcc := xrpc.Client{ + Host: owner.PDSEndpoint(), + } + + record, err := atproto.RepoGetRecord( + ctx, + &xrpcc, + "", + atUri.Collection().String(), + atUri.Authority().String(), + atUri.RecordKey().String(), + ) + if err != nil { + return nil, fmt.Errorf("failed to get record for %s: %v", atUri, err) + } + + if record != nil { + bytes, err := record.Value.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal record value for %s: %v", atUri, err) + } + + raw := json.RawMessage(bytes) + labelRecord := tangled.LabelDefinition{} + err = json.Unmarshal(raw, &labelRecord) + if err != nil { + return nil, fmt.Errorf("invalid record for %s: %w", atUri, err) + } + + labelDef, err := models.LabelDefinitionFromRecord( + atUri.Authority().String(), + atUri.RecordKey().String(), + labelRecord, + ) + if err != nil { + return nil, fmt.Errorf("failed to create label definition from record %s: %v", atUri, err) + } + + labelDefs = append(labelDefs, *labelDef) + } + } + + return labelDefs, nil +} diff --git a/appview/state/state.go b/appview/state/state.go index b9da5ea2..f62cbb3d 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -745,7 +745,7 @@ func BackfillDefaultDefs(e db.Execer, r *idresolver.Resolver, defaults []string) return nil } - labelDefs, err := models.FetchLabelDefs(r, defaults) + labelDefs, err := fetchLabelDefs(context.Background(), r.Directory(), defaults) if err != nil { return err } diff --git a/knotmirror/adminpage.go b/knotmirror/adminpage.go index 6fa3d477..526b81c4 100644 --- a/knotmirror/adminpage.go +++ b/knotmirror/adminpage.go @@ -12,10 +12,10 @@ import ( "strconv" "time" + "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-chi/chi/v5" "tangled.org/core/appview/pagination" - "tangled.org/core/idresolver" "tangled.org/core/knotmirror/db" "tangled.org/core/knotmirror/models" "tangled.org/core/knotmirror/xrpc" @@ -31,16 +31,16 @@ type AdminServer struct { resyncer *Resyncer xrpc *xrpc.Xrpc logger *slog.Logger - resolver *idresolver.Resolver + dir identity.Directory } -func NewAdminServer(l *slog.Logger, database *sql.DB, resyncer *Resyncer, x *xrpc.Xrpc, resolver *idresolver.Resolver) *AdminServer { +func NewAdminServer(l *slog.Logger, database *sql.DB, resyncer *Resyncer, x *xrpc.Xrpc, dir identity.Directory) *AdminServer { return &AdminServer{ db: database, resyncer: resyncer, xrpc: x, logger: l, - resolver: resolver, + dir: dir, } } @@ -121,12 +121,12 @@ func (s *AdminServer) handleRepos() http.HandlerFunc { http.Error(w, fmt.Sprintf("invalid DID or handle: %s", didInput), http.StatusBadRequest) return } - resolved, rerr := s.resolver.ResolveHandle(r.Context(), handle.Normalize()) + resolved, rerr := s.dir.LookupHandle(r.Context(), handle.Normalize()) if rerr != nil { http.Error(w, fmt.Sprintf("could not resolve handle %q: %s", didInput, rerr), http.StatusBadRequest) return } - did = resolved.String() + did = resolved.DID.String() } } diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go index 692deebe..759b1a4f 100644 --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -3,14 +3,17 @@ package knotmirror import ( "context" "fmt" + "net" "net/http" _ "net/http/pprof" "time" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/identity/redisdir" + "github.com/carlmjohnson/versioninfo" "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/redis/go-redis/v9" - "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/db" "tangled.org/core/knotmirror/knotstream" @@ -34,11 +37,7 @@ func Run(ctx context.Context, cfg *config.Config) error { rdb := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr}) - resolver, err := idresolver.RedisResolver("redis://"+cfg.RedisAddr, cfg.PlcUrl) - if err != nil { - logger.Error("failed to create redis resolver for admin, falling back to default", "err", err) - resolver = idresolver.DefaultResolver(cfg.PlcUrl) - } + dir := makeDirectory(cfg.PlcUrl, cfg.RedisAddr) // NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive. gitm := NewCliGitMirrorManager(cfg.GitRepoBasePath, cfg.KnotUseSSL) @@ -64,8 +63,8 @@ func Run(ctx context.Context, cfg *config.Config) error { knotstream := knotstream.NewKnotStream(logger, db, cfg) crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, indexScheduler, cfg) - xrpc := xrpc.New(logger, cfg, db, rdb, indexer, resolver, knotstream) - adminpage := NewAdminServer(logger, db, resyncer, xrpc, resolver) + xrpc := xrpc.New(logger, cfg, db, rdb, indexer, dir, knotstream) + adminpage := NewAdminServer(logger, db, resyncer, xrpc, dir) // maintain repository list with tap // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. @@ -148,3 +147,35 @@ func Run(ctx context.Context, cfg *config.Config) error { logger.Info("shutdown complete") return nil } + +func makeDirectory(plcUrl, redisUrl string) identity.Directory { + base := identity.BaseDirectory{ + PLCURL: plcUrl, + HTTPClient: http.Client{ + Timeout: time.Second * 10, + Transport: &http.Transport{ + // would want this around 100ms for services doing lots of handle resolution. Impacts PLC connections as well, but not too bad. + IdleConnTimeout: time.Millisecond * 1000, + MaxIdleConns: 100, + }, + }, + Resolver: net.Resolver{ + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + d := net.Dialer{Timeout: time.Second * 3} + return d.DialContext(ctx, network, address) + }, + }, + TryAuthoritativeDNS: true, + // primary Bluesky and Tangled PDS instances only support HTTP resolution method + SkipDNSDomainSuffixes: []string{".bsky.social", ".tngl.sh"}, + UserAgent: "knotmirror/" + versioninfo.Short(), + } + hitTTL := time.Hour * 24 + errTTL := time.Second * 30 + invalidHandleTTL := time.Minute * 5 + dir, err := redisdir.NewRedisDirectory(&base, redisUrl, hitTTL, errTTL, invalidHandleTTL, 10_000) + if err != nil { + return identity.NewCacheDirectory(&base, 250_000, time.Hour*24, time.Minute*2, time.Minute*5) + } + return dir +} diff --git a/knotmirror/xrpc/proxy.go b/knotmirror/xrpc/proxy.go index c518bfde..6729aa8b 100644 --- a/knotmirror/xrpc/proxy.go +++ b/knotmirror/xrpc/proxy.go @@ -92,7 +92,7 @@ func (x *Xrpc) resolveKnot(ctx context.Context, repoDid syntax.DID) (*knotInfo, return &knotInfo{baseURL: knotURL, repoIdentifier: repo.RepoIdentifier()}, nil } - ident, err := x.resolver.ResolveIdent(ctx, repoDid.String()) + ident, err := x.dir.LookupDID(ctx, repoDid) if err != nil { return nil, fmt.Errorf("resolving repoDid %s: %w", repoDid, err) } diff --git a/knotmirror/xrpc/sync_request_crawl.go b/knotmirror/xrpc/sync_request_crawl.go index 900b8beb..0fe9cc79 100644 --- a/knotmirror/xrpc/sync_request_crawl.go +++ b/knotmirror/xrpc/sync_request_crawl.go @@ -47,7 +47,7 @@ func (x *Xrpc) RequestCrawl(w http.ResponseWriter, r *http.Request) { writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", *input.EnsureRepo)}) return } - owner, err := x.resolver.ResolveIdent(ctx, repoAt.Authority().String()) + owner, err := x.dir.Lookup(ctx, repoAt.Authority()) if err != nil || owner.Handle.IsInvalidHandle() { l.Error("failed to resolve ident", "err", err, "owner", repoAt.Authority().String()) writeErr(w, fmt.Errorf("failed to resolve repo owner")) diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go index 4456514e..55835bf3 100644 --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -9,11 +9,11 @@ import ( "time" "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/util/ssrf" "github.com/go-chi/chi/v5" "github.com/redis/go-redis/v9" "tangled.org/core/api/tangled" - "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/knotstream" "tangled.org/core/knotmirror/repoindexer" @@ -25,14 +25,14 @@ type Xrpc struct { db *sql.DB rdb *redis.Client indexer *repoindexer.Indexer - resolver *idresolver.Resolver + dir identity.Directory ks *knotstream.KnotStream logger *slog.Logger httpClient *http.Client inflight *inflightTracker } -func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, indexer *repoindexer.Indexer, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { +func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, indexer *repoindexer.Indexer, dir identity.Directory, ks *knotstream.KnotStream) *Xrpc { httpClient := &http.Client{ Timeout: 30 * time.Second, } @@ -44,7 +44,7 @@ func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, db: db, rdb: rdb, indexer: indexer, - resolver: resolver, + dir: dir, ks: ks, logger: log.SubLogger(logger, "xrpc"), httpClient: httpClient, -- 2.54.0