From 508a17ea9580200bdc7c4637a31a5f4a7ea9f8de Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Mon, 11 May 2026 20:22:18 +0900 Subject: [PATCH] knotmirror: add redis cache to listLanguages Signed-off-by: Seongmin Lee --- knotmirror/config/config.go | 1 + knotmirror/knotmirror.go | 5 ++++- knotmirror/xrpc/git_list_languages.go | 30 +++++++++++++++++++++++++++ knotmirror/xrpc/xrpc.go | 5 ++++- nix/modules/knotmirror.nix | 6 ++++++ 5 files changed, 45 insertions(+), 2 deletions(-) diff --git a/knotmirror/config/config.go b/knotmirror/config/config.go index 01767d68..51173b1d 100644 --- a/knotmirror/config/config.go +++ b/knotmirror/config/config.go @@ -11,6 +11,7 @@ type Config struct { PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"` TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"` DbUrl string `env:"MIRROR_DB_URL, required"` + RedisAddr string `env:"MIRROR_REDIS_ADDR, required"` KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified KnotSSRF bool `env:"MIRROR_KNOT_SSRF, default=false"` GitRepoBasePath string `env:"MIRROR_GIT_BASEPATH, default=repos"` diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go index f9236c2e..14571d64 100644 --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -8,6 +8,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/redis/go-redis/v9" "github.com/prometheus/client_golang/prometheus/promhttp" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" @@ -30,6 +31,8 @@ func Run(ctx context.Context, cfg *config.Config) error { return fmt.Errorf("initializing db: %w", err) } + rdb := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr}) + resolver := idresolver.DefaultResolver(cfg.PlcUrl) // NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive. @@ -53,7 +56,7 @@ func Run(ctx context.Context, cfg *config.Config) error { crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, cfg) adminpage := NewAdminServer(logger, db, resyncer) - xrpc := xrpc.New(logger, cfg, db, resolver, knotstream) + xrpc := xrpc.New(logger, cfg, db, rdb, resolver, knotstream) // maintain repository list with tap // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. diff --git a/knotmirror/xrpc/git_list_languages.go b/knotmirror/xrpc/git_list_languages.go index 7507a218..5bd158d2 100644 --- a/knotmirror/xrpc/git_list_languages.go +++ b/knotmirror/xrpc/git_list_languages.go @@ -2,6 +2,7 @@ package xrpc import ( "context" + "encoding/json" "fmt" "math" "net/http" @@ -13,6 +14,11 @@ import ( "tangled.org/core/knotserver/git" ) +const ( + RepoLanguagesByDid = "git_list_languages:repo:%s:%s" + RepoLanguagesTTL = 24 * time.Hour +) + func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { var ( repoQuery = r.URL.Query().Get("repo") @@ -27,6 +33,19 @@ func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { return } + if val, err := x.rdb.Get(r.Context(), fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil { + l.Debug("served from cache") + var langs []*tangled.GitTempListLanguages_Language + err = json.Unmarshal([]byte(val), &langs) + if err == nil { + writeJson(w, http.StatusOK, &tangled.GitTempListLanguages_Output{ + Ref: ref, + Languages: langs, + }) + return + } + } + out, err := x.listLanguages(r.Context(), repo, ref) if err != nil { l.Warn("local mirror failed, trying proxy", "err", err) @@ -59,6 +78,17 @@ func (x *Xrpc) listLanguages(ctx context.Context, repo syntax.ATURI, ref string) return nil, fmt.Errorf("analyzing languages: %w", err) } + langs := sizesToLanguages(sizes) + + go func() { + ctx := context.Background() + encoded, err := json.Marshal(langs) + if err != nil { + return + } + x.rdb.Set(ctx, fmt.Sprintf(RepoLanguagesByDid, repo, ref), encoded, RepoLanguagesTTL) + }() + return &tangled.GitTempListLanguages_Output{ Ref: ref, Languages: sizesToLanguages(sizes), diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go index 37827abf..3e770e10 100644 --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -10,6 +10,7 @@ import ( "github.com/bluesky-social/indigo/atproto/atclient" "github.com/go-chi/chi/v5" + "github.com/redis/go-redis/v9" "tangled.org/core/api/tangled" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" @@ -20,16 +21,18 @@ import ( type Xrpc struct { cfg *config.Config db *sql.DB + rdb *redis.Client resolver *idresolver.Resolver ks *knotstream.KnotStream logger *slog.Logger httpClient *http.Client } -func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { +func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { return &Xrpc{ cfg: cfg, db: db, + rdb: rdb, resolver: resolver, ks: ks, logger: log.SubLogger(logger, "xrpc"), diff --git a/nix/modules/knotmirror.nix b/nix/modules/knotmirror.nix index b9f2ed28..2e82d288 100644 --- a/nix/modules/knotmirror.nix +++ b/nix/modules/knotmirror.nix @@ -98,6 +98,11 @@ in cfg.package ]; + services.redis.servers.knotmirror = { + enable = true; + port = 6377; + }; + systemd.services.tap-knotmirror = { description = "knotmirror tap service"; after = ["network.target"]; @@ -139,6 +144,7 @@ in "MIRROR_HOSTNAME=${cfg.hostname}" "MIRROR_TAP_URL=http://localhost:${toString cfg.tap.port}" "MIRROR_DB_URL=${cfg.dbUrl}" + "MIRROR_REDIS_ADDR=localhost:6377" "MIRROR_GIT_BASEPATH=/var/lib/knotmirror/repos" "MIRROR_KNOT_USE_SSL=${boolToString cfg.knotUseSSL}" "MIRROR_KNOT_SSRF=${boolToString cfg.knotSSRF}" -- 2.51.2