From 9acac4bffbb7d24080e525153501a2fd74690152 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Mon, 11 May 2026 11:22:18 +0000 Subject: [PATCH] knotmirror: add redis cache to listLanguages Signed-off-by: Seongmin Lee --- knotmirror/config/config.go | 1 + knotmirror/knotmirror.go | 5 ++++- knotmirror/xrpc/git_list_languages.go | 30 ++++++++++++++++++++++++++++++ knotmirror/xrpc/xrpc.go | 5 ++++- nix/modules/knotmirror.nix | 6 ++++++ 5 file(s) changed, 45 insertion(s)(+), 2 deletion(s)(-) diff --git a/knotmirror/config/config.go b/knotmirror/config/config.go --- a/knotmirror/config/config.go +++ b/knotmirror/config/config.go @@ -11,6 +11,7 @@ type Config struct { PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"` TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"` DbUrl string `env:"MIRROR_DB_URL, required"` + RedisAddr string `env:"MIRROR_REDIS_ADDR, required"` KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified KnotSSRF bool `env:"MIRROR_KNOT_SSRF, default=false"` GitRepoBasePath string `env:"MIRROR_GIT_BASEPATH, default=repos"` diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -8,6 +8,7 @@ _ "net/http/pprof" "time" "github.com/go-chi/chi/v5" + "github.com/redis/go-redis/v9" "github.com/prometheus/client_golang/prometheus/promhttp" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" @@ -29,6 +30,8 @@ db, err := db.Make(ctx, cfg.DbUrl, 32) if err != nil { return fmt.Errorf("initializing db: %w", err) } + + rdb := redis.NewClient(&redis.Options{Addr: cfg.RedisAddr}) resolver := idresolver.DefaultResolver(cfg.PlcUrl) @@ -53,7 +56,7 @@ knotstream := knotstream.NewKnotStream(logger, db, cfg) crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, cfg) adminpage := NewAdminServer(logger, db, resyncer) - xrpc := xrpc.New(logger, cfg, db, resolver, knotstream) + xrpc := xrpc.New(logger, cfg, db, rdb, resolver, knotstream) // maintain repository list with tap // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. diff --git a/knotmirror/xrpc/git_list_languages.go b/knotmirror/xrpc/git_list_languages.go --- a/knotmirror/xrpc/git_list_languages.go +++ b/knotmirror/xrpc/git_list_languages.go @@ -2,6 +2,7 @@ package xrpc import ( "context" + "encoding/json" "fmt" "math" "net/http" @@ -13,6 +14,11 @@ "tangled.org/core/api/tangled" "tangled.org/core/knotserver/git" ) +const ( + RepoLanguagesByDid = "git_list_languages:repo:%s:%s" + RepoLanguagesTTL = 24 * time.Hour +) + func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { var ( repoQuery = r.URL.Query().Get("repo") @@ -27,6 +33,19 @@ writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) return } + if val, err := x.rdb.Get(r.Context(), fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil { + l.Debug("served from cache") + var langs []*tangled.GitTempListLanguages_Language + err = json.Unmarshal([]byte(val), &langs) + if err == nil { + writeJson(w, http.StatusOK, &tangled.GitTempListLanguages_Output{ + Ref: ref, + Languages: langs, + }) + return + } + } + out, err := x.listLanguages(r.Context(), repo, ref) if err != nil { l.Warn("local mirror failed, trying proxy", "err", err) @@ -58,6 +77,17 @@ sizes, err := gr.AnalyzeLanguages(ctx) if err != nil { return nil, fmt.Errorf("analyzing languages: %w", err) } + + langs := sizesToLanguages(sizes) + + go func() { + ctx := context.Background() + encoded, err := json.Marshal(langs) + if err != nil { + return + } + x.rdb.Set(ctx, fmt.Sprintf(RepoLanguagesByDid, repo, ref), encoded, RepoLanguagesTTL) + }() return &tangled.GitTempListLanguages_Output{ Ref: ref, diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -10,6 +10,7 @@ "time" "github.com/bluesky-social/indigo/atproto/atclient" "github.com/go-chi/chi/v5" + "github.com/redis/go-redis/v9" "tangled.org/core/api/tangled" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" @@ -20,16 +21,18 @@ type Xrpc struct { cfg *config.Config db *sql.DB + rdb *redis.Client resolver *idresolver.Resolver ks *knotstream.KnotStream logger *slog.Logger httpClient *http.Client } -func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { +func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { return &Xrpc{ cfg: cfg, db: db, + rdb: rdb, resolver: resolver, ks: ks, logger: log.SubLogger(logger, "xrpc"), diff --git a/nix/modules/knotmirror.nix b/nix/modules/knotmirror.nix --- a/nix/modules/knotmirror.nix +++ b/nix/modules/knotmirror.nix @@ -98,6 +98,11 @@ pkgs.git cfg.package ]; + services.redis.servers.knotmirror = { + enable = true; + port = 6377; + }; + systemd.services.tap-knotmirror = { description = "knotmirror tap service"; after = ["network.target"]; @@ -139,6 +144,7 @@ "MIRROR_LISTEN=${cfg.listenAddr}" "MIRROR_HOSTNAME=${cfg.hostname}" "MIRROR_TAP_URL=http://localhost:${toString cfg.tap.port}" "MIRROR_DB_URL=${cfg.dbUrl}" + "MIRROR_REDIS_ADDR=localhost:6377" "MIRROR_GIT_BASEPATH=/var/lib/knotmirror/repos" "MIRROR_KNOT_USE_SSL=${boolToString cfg.knotUseSSL}" "MIRROR_KNOT_SSRF=${boolToString cfg.knotSSRF}" -- tangled.sh