From 46ecd70b0e96ed854160ef5cec5080ccda2efa4a Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sat, 2 May 2026 15:50:27 +0900 Subject: [PATCH] knotmirror/xrpc: `git.{compareRevs,formatPatch,interdiffRevs}` Signed-off-by: Seongmin Lee --- knotmirror/config/config.go | 5 + knotmirror/knotmirror.go | 10 +- knotmirror/xrpc/git_compare_revs.go | 128 +++++++++++++++++++++ knotmirror/xrpc/git_format_patch.go | 150 +++++++++++++++++++++++++ knotmirror/xrpc/git_interdiff_revs.go | 153 ++++++++++++++++++++++++++ knotmirror/xrpc/xrpc.go | 8 +- nix/vm.nix | 4 + 7 files changed, 456 insertions(+), 2 deletions(-) create mode 100644 knotmirror/xrpc/git_compare_revs.go create mode 100644 knotmirror/xrpc/git_format_patch.go create mode 100644 knotmirror/xrpc/git_interdiff_revs.go diff --git a/knotmirror/config/config.go b/knotmirror/config/config.go index 01767d68..0733b1e3 100644 --- a/knotmirror/config/config.go +++ b/knotmirror/config/config.go @@ -11,6 +11,7 @@ type Config struct { PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"` TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"` DbUrl string `env:"MIRROR_DB_URL, required"` + Redis RedisConfig `env:",prefix=MIRROR_REDIS_"` KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified KnotSSRF bool `env:"MIRROR_KNOT_SSRF, default=false"` GitRepoBasePath string `env:"MIRROR_GIT_BASEPATH, default=repos"` @@ -31,6 +32,10 @@ func (c *Config) BaseUrl() string { return "http://" + c.Hostname } +type RedisConfig struct { + Addr string `env:"ADDR, default=localhost:6379"` +} + type SlurperConfig struct { PersistCursorPeriod time.Duration `env:"PERSIST_CURSOR_PERIOD, default=4s"` ConcurrencyPerHost int `env:"CONCURRENCY, default=4"` diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go index f9236c2e..781a177c 100644 --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -9,6 +9,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/redis/go-redis/v9" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/db" @@ -30,6 +31,13 @@ func Run(ctx context.Context, cfg *config.Config) error { return fmt.Errorf("initializing db: %w", err) } + var rdb *redis.Client + if cfg.Redis.Addr != "" { + rdb = redis.NewClient(&redis.Options{ + Addr: cfg.Redis.Addr, + }) + } + resolver := idresolver.DefaultResolver(cfg.PlcUrl) // NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive. @@ -53,7 +61,7 @@ func Run(ctx context.Context, cfg *config.Config) error { crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, cfg) adminpage := NewAdminServer(logger, db, resyncer) - xrpc := xrpc.New(logger, cfg, db, resolver, knotstream) + xrpc := xrpc.New(logger, cfg, db, rdb, resolver, knotstream) // maintain repository list with tap // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. diff --git a/knotmirror/xrpc/git_compare_revs.go b/knotmirror/xrpc/git_compare_revs.go new file mode 100644 index 00000000..4eec1f40 --- /dev/null +++ b/knotmirror/xrpc/git_compare_revs.go @@ -0,0 +1,128 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/git" +) + +const ( + RdbCompareRevs = "compare_revs:%s:%s" + RdbCompareRevsTTL = 24 * time.Hour +) + +func (x *Xrpc) CompareRevs(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + rev1 = r.URL.Query().Get("rev1") + rev2 = r.URL.Query().Get("rev2") + ) + + l := x.logger.With("method", "git.compareRevs") + ctx := r.Context() + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo1 parameter invalid: %s", repoQuery)}) + return + } + + if rev1 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev1 parameter"}) + return + } + if rev2 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev2 parameter"}) + return + } + + l = l.With("repo1", repo, "rev1", rev1, "rev2", rev2) + + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + l.Error("error building repo path", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + gr, err := git.PlainOpen(repoPath) + if err != nil { + l.Error("failed opening git repo", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + commit1, err := gr.ResolveRevision(rev1) + if err != nil { + l.Error("error resolving revision 1", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + + commit2, err := gr.ResolveRevision(rev2) + if err != nil { + l.Error("error resolving revision 2", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev2)}) + return + } + + rawPatch, err := x.compareRevs(ctx, gr, commit1, commit2) + if err != nil { + l.Error("error comparing revisions", "err", err.Error()) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + return + } + + rev1Hash := commit1.Hash.String() + rev2Hash := commit2.Hash.String() + writeJson(w, http.StatusOK, &tangled.GitTempCompareRevs_Output{ + Rev1: &rev1Hash, + Rev2: &rev2Hash, + Patch: rawPatch, + }) +} + +func (x *Xrpc) compareRevs(ctx context.Context, gr *git.GitRepo, commit1, commit2 *object.Commit) (string, error) { + l := x.logger + mergeBaseCommit, err := gr.MergeBase(commit1, commit2) + if err != nil { + return "", err + } + + if x.rdb != nil { + rawPatch, err := x.rdb.Get(ctx, fmt.Sprintf(RdbCompareRevs, mergeBaseCommit.Hash, commit2.Hash)).Result() + if err != nil { + // no-op + } else { + l.Debug("using cached patch") + return rawPatch, nil + } + } + + diffTree, err := gr.DiffTree(mergeBaseCommit, commit2) + if err != nil { + return "", err + } + + if x.rdb != nil { + go func() { + key := fmt.Sprintf(RdbCompareRevs, mergeBaseCommit.Hash, commit2.Hash) + l.Debug("caching compare patch", "key", key) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := x.rdb.Set(ctx, key, diffTree.Patch, RdbCompareRevsTTL).Result() + if err != nil { + l.Error("failed to cache compare result", "err", err) + } + }() + } + + return diffTree.Patch, nil +} diff --git a/knotmirror/xrpc/git_format_patch.go b/knotmirror/xrpc/git_format_patch.go new file mode 100644 index 00000000..825808ce --- /dev/null +++ b/knotmirror/xrpc/git_format_patch.go @@ -0,0 +1,150 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os/exec" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/git" +) + +const ( + RdbFormatPatch = "format_patch:%s:%s" + RdbFormatPatchTTL = 24 * time.Hour +) + +func (x *Xrpc) FormatPatch(w http.ResponseWriter, r *http.Request) { + var ( + repo1Query = r.URL.Query().Get("repo1") + rev1 = r.URL.Query().Get("rev1") + repo2Query = r.URL.Query().Get("repo2") // optional + rev2 = r.URL.Query().Get("rev2") + ) + + l := x.logger.With("method", "git.formatPatch") + ctx := r.Context() + + repo1, err := syntax.ParseATURI(repo1Query) + if err != nil || repo1.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo1 parameter invalid: %s", repo1Query)}) + return + } + + var repo2 syntax.ATURI + if repo2Query == "" { + repo2 = repo1 + } else { + repo2, err = syntax.ParseATURI(repo2Query) + if err != nil || repo2.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo2 parameter invalid: %s", repo2Query)}) + return + } + } + + if rev1 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev1 parameter"}) + return + } + if rev2 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev2 parameter"}) + return + } + + l = l.With("repo1", repo1, "repo2", repo2, "rev1", rev1, "rev2", rev2) + + repo2Path, err := x.makeRepoPath(ctx, repo2) + if err != nil { + l.Error("error building repo path", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + gr, err := git.PlainOpen(repo2Path) + if err != nil { + l.Error("failed opening git repo", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + repo1Path, err := x.makeRepoPath(ctx, repo1) + if err != nil { + l.Error("error building repo path", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + if repo2Path != repo1Path { + // fetch commit1 from repo1 to repo2 + repo1Remote := &url.URL{Scheme: "file", Path: repo1Path} // TODO: ensure repo1Path is absolute path + fetchCmd := exec.Command( + "git", + "-C", repo2Path, + "fetch", "--depth=1", + repo1Remote.String(), rev1, + ) + if err := fetchCmd.Run(); err != nil { + l.Error("error fetching rev1", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + } + + commit1, err := gr.ResolveRevision(rev1) + if err != nil { + l.Error("error resolving revision 1", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + + commit2, err := gr.ResolveRevision(rev2) + if err != nil { + l.Error("error resolving revision 2", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev2)}) + return + } + + rev1Hash := commit1.Hash.String() + rev2Hash := commit2.Hash.String() + + if x.rdb != nil { + rawPatch, err := x.rdb.Get(ctx, fmt.Sprintf(RdbFormatPatch, commit1.Hash, commit2.Hash)).Result() + if err != nil { + // no-op + } else { + writeJson(w, http.StatusOK, &tangled.GitTempFormatPatch_Output{ + Rev1: &rev1Hash, + Rev2: &rev2Hash, + Patch: rawPatch, + }) + return + } + } + + rawPatch, _, err := gr.FormatPatch(commit1, commit2) + if err != nil { + l.Error("error running format-patch", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + return + } + + writeJson(w, http.StatusOK, &tangled.GitTempFormatPatch_Output{ + Rev1: &rev1Hash, + Rev2: &rev2Hash, + Patch: rawPatch, + }) + + if x.rdb != nil { + go func() { + ctx := context.Background() + _, err := x.rdb.Set(ctx, fmt.Sprintf(RdbFormatPatch, commit1.Hash, commit2.Hash), rawPatch, RdbFormatPatchTTL).Result() + if err != nil { + l.Error("failed to cache compare result", "err", err) + } + }() + } +} diff --git a/knotmirror/xrpc/git_interdiff_revs.go b/knotmirror/xrpc/git_interdiff_revs.go new file mode 100644 index 00000000..f6751a6a --- /dev/null +++ b/knotmirror/xrpc/git_interdiff_revs.go @@ -0,0 +1,153 @@ +package xrpc + +import ( + "fmt" + "net/http" + "net/url" + "os/exec" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotserver/git" +) + +func (x *Xrpc) InterdiffRevs(w http.ResponseWriter, r *http.Request) { + var ( + targetRepoQuery = r.URL.Query().Get("targetRepo") + base = r.URL.Query().Get("base") // target branch + sourceRepoQuery = r.URL.Query().Get("sourceRepo") // optional + rev1 = r.URL.Query().Get("rev1") + rev2 = r.URL.Query().Get("rev2") + ) + + l := x.logger.With("method", "git.interdiffRevs") + ctx := r.Context() + + targetRepo, err := syntax.ParseATURI(targetRepoQuery) + if err != nil || targetRepo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo1 parameter invalid: %s", targetRepoQuery)}) + return + } + + var sourceRepo syntax.ATURI + if sourceRepoQuery == "" { + sourceRepo = targetRepo + } else { + sourceRepo, err = syntax.ParseATURI(sourceRepoQuery) + if err != nil || sourceRepo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo2 parameter invalid: %s", sourceRepoQuery)}) + return + } + } + + if base == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing base parameter"}) + return + } + if rev1 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev1 parameter"}) + return + } + if rev2 == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing rev2 parameter"}) + return + } + + l = l.With("target", targetRepo, "base", base, "source", sourceRepo, "rev1", rev1, "rev2", rev2) + + sourceRepoPath, err := x.makeRepoPath(ctx, sourceRepo) + if err != nil { + l.Error("error building repo path", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + gr, err := git.PlainOpen(sourceRepoPath) + if err != nil { + l.Error("failed opening git repo", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + + targetRepoPath, err := x.makeRepoPath(ctx, targetRepo) + if err != nil { + l.Error("error building repo path", "err", err) + writeJson(w, http.StatusNotFound, atclient.ErrorBody{Name: "RepoNotFound", Message: "repository not found"}) + return + } + if sourceRepoPath != targetRepoPath { + // fetch `base` from targetRepo to sourceRepo + targetRemote := &url.URL{Scheme: "file", Path: targetRepoPath} // TODO: ensure targetRepoPath is absolute path + fetchCmd := exec.Command( + "git", + "-C", sourceRepoPath, + "fetch", + targetRemote.String(), base, + ) + if err := fetchCmd.Run(); err != nil { + l.Error("error fetching base", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + } + + baseCommit, err := gr.ResolveRevision(base) + if err != nil { + l.Error("error resolving base rev", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + + commit1, err := gr.ResolveRevision(rev1) + if err != nil { + l.Error("error resolving revision 1", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev1)}) + return + } + + commit2, err := gr.ResolveRevision(rev2) + if err != nil { + l.Error("error resolving revision 2", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "RevisionNotFound", Message: fmt.Sprintf("error resolving revision %s", rev2)}) + return + } + + l = l.With("base", baseCommit.Hash, "rev1", commit1.Hash, "rev2", commit2.Hash) + l.Debug("interdiff") + + rev1Patch, err := x.compareRevs(ctx, gr, baseCommit, commit1) + if err != nil { + l.Error("error comparing base...rev1", "err", err.Error()) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + } + rev2Patch, err := x.compareRevs(ctx, gr, baseCommit, commit2) + if err != nil { + l.Error("error comparing base...rev2", "err", err.Error()) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + } + + // NOTE: we can't json-decode `patchutil.InterdiffResult`. + // So we just pass the unfinished values instead. + writeJson(w, http.StatusOK, struct{ + Patch1 string + Patch2 string + }{ + Patch1: rev1Patch, + Patch2: rev2Patch, + }) + + // TODO(boltless): run interdiff from knotmirror & pass diff to appview + + // rev1Diff, err := patchutil.AsDiff(rev1Patch) + // if err != nil { + // l.Error("error parsing base...rev1", "err", err.Error()) + // writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + // } + // rev2Diff, err := patchutil.AsDiff(rev2Patch) + // if err != nil { + // l.Error("error parsing base...rev2", "err", err.Error()) + // writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "CompareError", Message: "error comparing revisions"}) + // } + // + // writeJson(w, http.StatusOK, patchutil.Interdiff(rev1Diff, rev2Diff)) +} diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go index 37827abf..d2b060b3 100644 --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -10,6 +10,7 @@ import ( "github.com/bluesky-social/indigo/atproto/atclient" "github.com/go-chi/chi/v5" + "github.com/redis/go-redis/v9" "tangled.org/core/api/tangled" "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" @@ -20,16 +21,18 @@ import ( type Xrpc struct { cfg *config.Config db *sql.DB + rdb *redis.Client resolver *idresolver.Resolver ks *knotstream.KnotStream logger *slog.Logger httpClient *http.Client } -func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { +func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, rdb *redis.Client, resolver *idresolver.Resolver, ks *knotstream.KnotStream) *Xrpc { return &Xrpc{ cfg: cfg, db: db, + rdb: rdb, resolver: resolver, ks: ks, logger: log.SubLogger(logger, "xrpc"), @@ -55,6 +58,9 @@ func (x *Xrpc) Router() http.Handler { r.Get("/"+tangled.GitTempListCommitsNSID, x.ListCommits) r.Get("/"+tangled.GitTempListLanguagesNSID, x.ListLanguages) r.Get("/"+tangled.GitTempListTagsNSID, x.ListTags) + r.Get("/"+tangled.GitTempCompareRevsNSID, x.CompareRevs) + r.Get("/"+tangled.GitTempFormatPatchNSID, x.FormatPatch) + r.Get("/"+tangled.GitTempInterdiffRevsNSID, x.InterdiffRevs) r.Get("/"+tangled.RepoBlobNSID, x.RepoBlob) r.Post("/"+tangled.SyncRequestCrawlNSID, x.RequestCrawl) diff --git a/nix/vm.nix b/nix/vm.nix index ea1501f6..a2237657 100644 --- a/nix/vm.nix +++ b/nix/vm.nix @@ -148,6 +148,10 @@ in host all tnglr 127.0.0.1/32 trust ''; }; + services.redis.servers.km = { + enable = true; + port = 6379; + }; services.tangled.knotmirror = { enable = true; listenAddr = "0.0.0.0:7000"; -- 2.51.2