From 35ceae65d94dc95c4095aa9aa56a8282c65c6369 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Mon, 16 Feb 2026 15:35:55 +0000 Subject: [PATCH] appview,knotmirror: use knotmirror to read the repository Underlying types except the interface hasn't changed much. Removed `xrpcclient.HandleXrpcErr()` call as appview always expect knotmirror with compatible API. Signed-off-by: Seongmin Lee --- knotmirror/knotmirror.go | 21 +++++++++++++++++++++ appview/config/config.go | 37 +++++++++++++++++++++---------------- appview/pulls/pulls.go | 98 ++++++++++++++++---------------------------------------------------------------------------------- appview/repo/archive.go | 28 +++++++++------------------- appview/repo/artifact.go | 12 ++---------- appview/repo/branches.go | 17 +++++------------ appview/repo/compare.go | 14 +++----------- appview/repo/index.go | 79 +++++++++++++++++++++++++++---------------------------------------------------- appview/repo/log.go | 28 ++++++++++------------------ appview/repo/settings.go | 12 ++---------- appview/repo/tags.go | 31 ++++++++----------------------- appview/repo/tree.go | 13 +++---------- knotmirror/config/config.go | 11 +++++++++++ knotmirror/xrpc/git_getArchive.go | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_getBlob.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_getBranch.go | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_getTag.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_getTree.go | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_listBranches.go | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_listCommits.go | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_listLanguages.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/git_listTags.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/xrpc/xrpc.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git/git.go | 14 ++++++++++++++ 24 file(s) changed, 1082 insertion(s)(+), 263 deletion(s)(-) diff --git a/knotmirror/knotmirror.go b/knotmirror/knotmirror.go --- a/knotmirror/knotmirror.go +++ b/knotmirror/knotmirror.go @@ -7,11 +7,14 @@ _ "net/http/pprof" "time" + "github.com/go-chi/chi/v5" "github.com/prometheus/client_golang/prometheus/promhttp" + "tangled.org/core/idresolver" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/db" "tangled.org/core/knotmirror/knotstream" "tangled.org/core/knotmirror/models" + "tangled.org/core/knotmirror/xrpc" "tangled.org/core/log" ) @@ -26,6 +29,8 @@ if err != nil { return fmt.Errorf("initializing db: %w", err) } + + resolver := idresolver.DefaultResolver(cfg.PlcUrl) // NOTE: using plain git-cli for clone/fetch as go-git is too memory-intensive. gitm := NewCliGitMirrorManager(cfg.GitRepoBasePath, cfg.KnotUseSSL) @@ -44,6 +49,7 @@ } logger.Info(fmt.Sprintf("clearing resyning states: %d records updated", rows)) + xrpc := xrpc.New(logger, cfg, db, resolver) knotstream := knotstream.NewKnotStream(logger, db, cfg) crawler := NewCrawler(logger, db) resyncer := NewResyncer(logger, db, gitm, cfg) @@ -52,6 +58,21 @@ // maintain repository list with tap // NOTE: this can be removed once we introduce did-for-repo because then we can just listen to KnotStream for #identity events. tap := NewTapClient(logger, cfg, db, gitm, knotstream) + + // start http server + go func() { + logger.Info("starting http server", "addr", cfg.Listen) + + mux := chi.NewRouter() + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Welcome to a knotmirror server.\n")) + }) + mux.Mount("/xrpc", xrpc.Router()) + + if err := http.ListenAndServe(cfg.Listen, mux); err != nil { + logger.Error("xrpc server failed", "error", err) + } + }() // start metrics endpoint go func() { diff --git a/appview/config/config.go b/appview/config/config.go --- a/appview/config/config.go +++ b/appview/config/config.go @@ -46,6 +46,10 @@ PLCURL string `env:"URL, default=https://plc.directory"` } +type KnotMirrorConfig struct { + Url string `env:"URL, required"` +} + type JetstreamConfig struct { Endpoint string `env:"ENDPOINT, default=wss://jetstream1.us-east.bsky.network/subscribe"` } @@ -150,22 +154,23 @@ } type Config struct { - Core CoreConfig `env:",prefix=TANGLED_"` - Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"` - Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"` - Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"` - Resend ResendConfig `env:",prefix=TANGLED_RESEND_"` - Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"` - Camo CamoConfig `env:",prefix=TANGLED_CAMO_"` - Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"` - OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"` - Redis RedisConfig `env:",prefix=TANGLED_REDIS_"` - Plc PlcConfig `env:",prefix=TANGLED_PLC_"` - Pds PdsConfig `env:",prefix=TANGLED_PDS_"` - Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"` - Label LabelConfig `env:",prefix=TANGLED_LABEL_"` - Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"` - Sites SitesConfig `env:",prefix=TANGLED_SITES_"` + Core CoreConfig `env:",prefix=TANGLED_"` + Jetstream JetstreamConfig `env:",prefix=TANGLED_JETSTREAM_"` + Knotstream ConsumerConfig `env:",prefix=TANGLED_KNOTSTREAM_"` + Spindlestream ConsumerConfig `env:",prefix=TANGLED_SPINDLESTREAM_"` + Resend ResendConfig `env:",prefix=TANGLED_RESEND_"` + Posthog PosthogConfig `env:",prefix=TANGLED_POSTHOG_"` + Camo CamoConfig `env:",prefix=TANGLED_CAMO_"` + Avatar AvatarConfig `env:",prefix=TANGLED_AVATAR_"` + OAuth OAuthConfig `env:",prefix=TANGLED_OAUTH_"` + Redis RedisConfig `env:",prefix=TANGLED_REDIS_"` + Plc PlcConfig `env:",prefix=TANGLED_PLC_"` + Pds PdsConfig `env:",prefix=TANGLED_PDS_"` + Cloudflare Cloudflare `env:",prefix=TANGLED_CLOUDFLARE_"` + Label LabelConfig `env:",prefix=TANGLED_LABEL_"` + Bluesky BlueskyConfig `env:",prefix=TANGLED_BLUESKY_"` + Sites SitesConfig `env:",prefix=TANGLED_SITES_"` + KnotMirror KnotMirrorConfig `env:",prefix=TANGLED_KNOTMIRROR_"` } func LoadConfig(ctx context.Context) (*Config, error) { diff --git a/appview/pulls/pulls.go b/appview/pulls/pulls.go --- a/appview/pulls/pulls.go +++ b/appview/pulls/pulls.go @@ -411,16 +411,8 @@ return nil } - scheme := "http" - if !s.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, repo.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - - resp, err := tangled.RepoBranch(r.Context(), xrpcc, branch, fmt.Sprintf("%s/%s", repo.Did, repo.Name)) + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} + resp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, branch, repo.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { return nil } @@ -436,37 +428,17 @@ return pages.Unknown } - var knot, ownerDid, repoName string - + var sourceRepo syntax.ATURI if pull.PullSource.RepoAt != nil { // fork-based pulls - sourceRepo, err := db.GetRepoByAtUri(s.db, pull.PullSource.RepoAt.String()) - if err != nil { - log.Println("failed to get source repo", err) - return pages.Unknown - } - - knot = sourceRepo.Knot - ownerDid = sourceRepo.Did - repoName = sourceRepo.Name + sourceRepo = *pull.PullSource.RepoAt } else { // pulls within the same repo - knot = repo.Knot - ownerDid = repo.Did - repoName = repo.Name + sourceRepo = repo.RepoAt() } - scheme := "http" - if !s.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - - didSlashName := fmt.Sprintf("%s/%s", ownerDid, repoName) - branchResp, err := tangled.RepoBranch(r.Context(), xrpcc, pull.PullSource.Branch, didSlashName) + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} + branchResp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, pull.PullSource.Branch, sourceRepo.String()) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { log.Println("failed to call XRPC repo.branches", xrpcerr) @@ -904,17 +876,9 @@ switch r.Method { case http.MethodGet: - scheme := "http" - if !s.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { log.Println("failed to call XRPC repo.branches", xrpcerr) @@ -1535,24 +1499,12 @@ return } - scheme := "http" - if !s.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) if err != nil { - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - log.Println("failed to call XRPC repo.branches", xrpcerr) - s.pages.Error503(w) - return - } log.Println("failed to fetch branches", err) + s.pages.Error503(w) return } @@ -1607,6 +1559,8 @@ return } + xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} + forkVal := r.URL.Query().Get("fork") repoString := strings.SplitN(forkVal, "/", 2) forkOwnerDid := repoString[0] @@ -1622,17 +1576,7 @@ return } - sourceScheme := "http" - if !s.config.Core.Dev { - sourceScheme = "https" - } - sourceHost := fmt.Sprintf("%s://%s", sourceScheme, repo.Knot) - sourceXrpcc := &indigoxrpc.Client{ - Host: sourceHost, - } - - sourceRepo := fmt.Sprintf("%s/%s", forkOwnerDid, repo.Name) - sourceXrpcBytes, err := tangled.RepoBranches(r.Context(), sourceXrpcc, "", 0, sourceRepo) + sourceXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, repo.RepoAt().String()) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { log.Println("failed to call XRPC repo.branches for source", xrpcerr) @@ -1651,17 +1595,7 @@ return } - targetScheme := "http" - if !s.config.Core.Dev { - targetScheme = "https" - } - targetHost := fmt.Sprintf("%s://%s", targetScheme, f.Knot) - targetXrpcc := &indigoxrpc.Client{ - Host: targetHost, - } - - targetRepo := fmt.Sprintf("%s/%s", f.Did, f.Name) - targetXrpcBytes, err := tangled.RepoBranches(r.Context(), targetXrpcc, "", 0, targetRepo) + targetXrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { log.Println("failed to call XRPC repo.branches for target", xrpcerr) diff --git a/appview/repo/archive.go b/appview/repo/archive.go --- a/appview/repo/archive.go +++ b/appview/repo/archive.go @@ -8,6 +8,7 @@ "strings" "github.com/go-chi/chi/v5" + "tangled.org/core/api/tangled" ) func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { @@ -20,30 +21,19 @@ l.Error("failed to get repo and knot", "err", err) return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - didSlashRepo := f.DidSlashRepo() // build the xrpc url - u, err := url.Parse(host) - if err != nil { - l.Error("failed to parse host URL", "err", err) - rp.pages.Error503(w) - return - } - - u.Path = "/xrpc/sh.tangled.repo.archive" query := url.Values{} + query.Set("repo", f.RepoAt().String()) + query.Set("ref", ref) query.Set("format", "tar.gz") query.Set("prefix", r.URL.Query().Get("prefix")) - query.Set("ref", ref) - query.Set("repo", didSlashRepo) - u.RawQuery = query.Encode() - - xrpcURL := u.String() + xrpcURL := fmt.Sprintf( + "%s/xrpc/%s?%s", + rp.config.KnotMirror.Url, + tangled.GitTempGetArchiveNSID, + query.Encode(), + ) // make the get request resp, err := http.Get(xrpcURL) diff --git a/appview/repo/artifact.go b/appview/repo/artifact.go --- a/appview/repo/artifact.go +++ b/appview/repo/artifact.go @@ -313,17 +313,9 @@ return nil, err } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, repo) + xrpcBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, f.RepoAt().String()) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.tags", "err", xrpcerr) diff --git a/appview/repo/branches.go b/appview/repo/branches.go --- a/appview/repo/branches.go +++ b/appview/repo/branches.go @@ -21,18 +21,11 @@ l.Error("failed to get repo and knot", "err", err) return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.branches", "err", xrpcerr) + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) + if err != nil { + l.Error("failed to call XRPC repo.branches", "err", err) rp.pages.Error503(w) return } diff --git a/appview/repo/compare.go b/appview/repo/compare.go --- a/appview/repo/compare.go +++ b/appview/repo/compare.go @@ -27,17 +27,9 @@ return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) + branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.branches", "err", xrpcerr) rp.pages.Error503(w) @@ -74,7 +66,7 @@ head = queryHead } - tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) + tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.tags", "err", xrpcerr) rp.pages.Error503(w) diff --git a/appview/repo/index.go b/appview/repo/index.go --- a/appview/repo/index.go +++ b/appview/repo/index.go @@ -22,7 +22,6 @@ "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/pages" - "tangled.org/core/appview/xrpcclient" "tangled.org/core/orm" "tangled.org/core/types" @@ -42,37 +41,18 @@ return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - user := rp.oauth.GetMultiAccountUser(r) // Build index response from multiple XRPC calls - result, err := rp.buildIndexResponse(r.Context(), xrpcc, f, ref) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - if errors.Is(xrpcerr, xrpcclient.ErrXrpcUnsupported) { - l.Error("failed to call XRPC repo.index", "err", err) - rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ - LoggedInUser: user, - NeedsKnotUpgrade: true, - RepoInfo: rp.repoResolver.GetRepoInfo(r, user), - }) - return - } else { - l.Error("failed to build index response", "err", err) - rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ - LoggedInUser: user, - KnotUnreachable: true, - RepoInfo: rp.repoResolver.GetRepoInfo(r, user), - }) - return - } + result, err := rp.buildIndexResponse(r.Context(), f, ref) + if err != nil { + l.Error("failed to build index response", "err", err) + rp.pages.RepoIndexPage(w, pages.RepoIndexParams{ + LoggedInUser: user, + KnotUnreachable: true, + RepoInfo: rp.repoResolver.GetRepoInfo(r, user), + }) + return } tagMap := make(map[string][]string) @@ -133,7 +113,7 @@ } // TODO: a bit dirty - languageInfo, err := rp.getLanguageInfo(r.Context(), l, f, xrpcc, result.Ref, ref == "") + languageInfo, err := rp.getLanguageInfo(r.Context(), l, f, result.Ref, ref == "") if err != nil { l.Warn("failed to compute language percentages", "err", err) // non-fatal @@ -169,7 +149,6 @@ ctx context.Context, l *slog.Logger, repo *models.Repo, - xrpcc *indigoxrpc.Client, currentRef string, isDefaultRef bool, ) ([]types.RepoLanguageDetails, error) { @@ -182,14 +161,10 @@ if err != nil || langs == nil { // non-fatal, fetch langs from ks via XRPC - didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name) - ls, err := tangled.RepoLanguages(ctx, xrpcc, currentRef, didSlashRepo) + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + ls, err := tangled.GitTempListLanguages(ctx, xrpcc, currentRef, repo.RepoAt().String()) if err != nil { - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.languages", "err", xrpcerr) - return nil, xrpcerr - } - return nil, err + return nil, fmt.Errorf("calling knotmirror git.listLanguages: %w", err) } if ls == nil || ls.Languages == nil { @@ -258,13 +233,13 @@ } // buildIndexResponse creates a RepoIndexResponse by combining multiple xrpc calls in parallel -func (rp *Repo) buildIndexResponse(ctx context.Context, xrpcc *indigoxrpc.Client, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) { - didSlashRepo := fmt.Sprintf("%s/%s", repo.Did, repo.Name) +func (rp *Repo) buildIndexResponse(ctx context.Context, repo *models.Repo, ref string) (*types.RepoIndexResponse, error) { + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} // first get branches to determine the ref if not specified - branchesBytes, err := tangled.RepoBranches(ctx, xrpcc, "", 0, didSlashRepo) + branchesBytes, err := tangled.GitTempListBranches(ctx, xrpcc, "", 0, repo.RepoAt().String()) if err != nil { - return nil, fmt.Errorf("failed to call repoBranches: %w", err) + return nil, fmt.Errorf("calling knotmirror git.listBranches: %w", err) } var branchesResp types.RepoBranchesResponse @@ -296,7 +271,7 @@ var ( tagsResp types.RepoTagsResponse - treeResp *tangled.RepoTree_Output + treeResp *tangled.GitTempGetTree_Output logResp types.RepoLogResponse readmeContent string readmeFileName string @@ -304,22 +279,22 @@ // tags wg.Go(func() { - tagsBytes, err := tangled.RepoTags(ctx, xrpcc, "", 0, didSlashRepo) + tagsBytes, err := tangled.GitTempListTags(ctx, xrpcc, "", 0, repo.RepoAt().String()) if err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to call repoTags: %w", err)) + errs = errors.Join(errs, fmt.Errorf("failed to call git.ListTags: %w", err)) return } if err := json.Unmarshal(tagsBytes, &tagsResp); err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoTags: %w", err)) + errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListTags: %w", err)) } }) // tree/files wg.Go(func() { - resp, err := tangled.RepoTree(ctx, xrpcc, "", ref, didSlashRepo) + resp, err := tangled.GitTempGetTree(ctx, xrpcc, "", ref, repo.RepoAt().String()) if err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to call repoTree: %w", err)) + errs = errors.Join(errs, fmt.Errorf("failed to call git.GetTree: %w", err)) return } treeResp = resp @@ -327,14 +302,14 @@ // commits wg.Go(func() { - logBytes, err := tangled.RepoLog(ctx, xrpcc, "", 50, "", ref, didSlashRepo) + logBytes, err := tangled.GitTempListCommits(ctx, xrpcc, "", 50, ref, repo.RepoAt().String()) if err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to call repoLog: %w", err)) + errs = errors.Join(errs, fmt.Errorf("failed to call git.ListCommits: %w", err)) return } if err := json.Unmarshal(logBytes, &logResp); err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to unmarshal repoLog: %w", err)) + errs = errors.Join(errs, fmt.Errorf("failed to unmarshal git.ListCommits: %w", err)) } }) @@ -376,7 +351,7 @@ Readme: readmeContent, ReadmeFileName: readmeFileName, Commits: logResp.Commits, - Description: logResp.Description, + Description: "", Files: files, Branches: branchesResp.Branches, Tags: tagsResp.Tags, diff --git a/appview/repo/log.go b/appview/repo/log.go --- a/appview/repo/log.go +++ b/appview/repo/log.go @@ -40,14 +40,7 @@ ref := chi.URLParam(r, "ref") ref, _ = url.PathUnescape(ref) - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} limit := int64(60) cursor := "" @@ -57,10 +50,9 @@ cursor = strconv.Itoa(offset) } - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoLog(r.Context(), xrpcc, cursor, limit, "", ref, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.log", "err", xrpcerr) + xrpcBytes, err := tangled.GitTempListCommits(r.Context(), xrpcc, cursor, limit, ref, f.RepoAt().String()) + if err != nil { + l.Error("failed to call XRPC repo.log", "err", err) rp.pages.Error503(w) return } @@ -72,9 +64,9 @@ return } - tagBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.tags", "err", xrpcerr) + tagBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) + if err != nil { + l.Error("failed to call XRPC repo.tags", "err", err) rp.pages.Error503(w) return } @@ -93,9 +85,9 @@ } } - branchBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.branches", "err", xrpcerr) + branchBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) + if err != nil { + l.Error("failed to call XRPC repo.branches", "err", err) rp.pages.Error503(w) return } diff --git a/appview/repo/settings.go b/appview/repo/settings.go --- a/appview/repo/settings.go +++ b/appview/repo/settings.go @@ -386,17 +386,9 @@ f, err := rp.repoResolver.Resolve(r) user := rp.oauth.GetMultiAccountUser(r) - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) + xrpcBytes, err := tangled.GitTempListBranches(r.Context(), xrpcc, "", 0, f.RepoAt().String()) var result types.RepoBranchesResponse if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.branches", "err", xrpcerr) diff --git a/appview/repo/tags.go b/appview/repo/tags.go --- a/appview/repo/tags.go +++ b/appview/repo/tags.go @@ -27,18 +27,10 @@ l.Error("failed to get repo and knot", "err", err) return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 0, repo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.tags", "err", xrpcerr) + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + xrpcBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 0, f.RepoAt().String()) + if err != nil { + l.Error("failed to call XRPC repo.tags", "err", err) rp.pages.Error503(w) return } @@ -90,23 +82,16 @@ l.Error("failed to get repo and knot", "err", err) return } - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) tag := chi.URLParam(r, "tag") - xrpcBytes, err := tangled.RepoTag(r.Context(), xrpcc, repo, tag) + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + + xrpcBytes, err := tangled.GitTempGetTag(r.Context(), xrpcc, f.RepoAt().String(), tag) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { // if we don't match an existing tag, and the tag we're trying // to match is "latest", resolve to the most recent tag if tag == "latest" { - tagsBytes, err := tangled.RepoTags(r.Context(), xrpcc, "", 1, repo) + tagsBytes, err := tangled.GitTempListTags(r.Context(), xrpcc, "", 1, f.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.tags for latest", "err", xrpcerr) rp.pages.Error503(w) diff --git a/appview/repo/tree.go b/appview/repo/tree.go --- a/appview/repo/tree.go +++ b/appview/repo/tree.go @@ -33,16 +33,9 @@ treePath := chi.URLParam(r, "*") treePath, _ = url.PathUnescape(treePath) treePath = strings.TrimSuffix(treePath, "/") - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) - xrpcResp, err := tangled.RepoTree(r.Context(), xrpcc, treePath, ref, repo) + + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + xrpcResp, err := tangled.GitTempGetTree(r.Context(), xrpcc, treePath, ref, f.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.tree", "err", xrpcerr) rp.pages.Error503(w) diff --git a/knotmirror/config/config.go b/knotmirror/config/config.go --- a/knotmirror/config/config.go +++ b/knotmirror/config/config.go @@ -8,6 +8,7 @@ ) type Config struct { + PlcUrl string `env:"MIRROR_PLC_URL, default=https://plc.directory"` TapUrl string `env:"MIRROR_TAP_URL, default=http://localhost:2480"` DbUrl string `env:"MIRROR_DB_URL, required"` KnotUseSSL bool `env:"MIRROR_KNOT_USE_SSL, default=false"` // use SSL for Knot when not scheme is not specified @@ -16,8 +17,18 @@ GitRepoFetchTimeout time.Duration `env:"MIRROR_GIT_FETCH_TIMEOUT, default=600s"` ResyncParallelism int `env:"MIRROR_RESYNC_PARALLELISM, default=5"` Slurper SlurperConfig `env:",prefix=MIRROR_SLURPER_"` + UseSSL bool `env:"MIRROR_USE_SSL, default=false"` + Hostname string `env:"MIRROR_HOSTNAME, required"` + Listen string `env:"MIRROR_LISTEN, default=:7000"` MetricsListen string `env:"MIRROR_METRICS_LISTEN, default=127.0.0.1:7100"` AdminListen string `env:"MIRROR_ADMIN_LISTEN, default=127.0.0.1:7200"` +} + +func (c *Config) BaseUrl() string { + if c.UseSSL { + return "https://" + c.Hostname + } + return "http://" + c.Hostname } type SlurperConfig struct { diff --git a/knotmirror/xrpc/git_getArchive.go b/knotmirror/xrpc/git_getArchive.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_getArchive.go @@ -0,0 +1,106 @@ +package xrpc + +import ( + "compress/gzip" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/api/tangled" + "tangled.org/core/knotmirror/db" + "tangled.org/core/knotserver/git" +) + +func (x *Xrpc) GetArchive(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") + format = r.URL.Query().Get("format") + prefix = r.URL.Query().Get("prefix") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + if format != "tar.gz" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "only tar.gz format is supported"}) + return + } + if format == "" { + format = "tar.gz" + } + + l := x.logger.With("repo", repo, "ref", ref, "format", format, "prefix", prefix) + ctx := r.Context() + + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + l.Error("failed to resolve repo at-uri", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to resolve repo"}) + return + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + l.Error("failed to open git repo", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to open git repo"}) + return + } + + repoName, err := func() (string, error) { + r, err := db.GetRepoByAtUri(ctx, x.db, repo) + if err != nil { + return "", err + } + if r == nil { + return "", fmt.Errorf("repo not found: %s", repo) + } + return r.Name, nil + }() + if err != nil { + l.Error("failed to get repo name", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to retrieve repo name"}) + return + } + + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") + immutableLink := func() string { + params := url.Values{} + params.Set("repo", repo.String()) + params.Set("ref", gr.Hash().String()) + params.Set("format", format) + params.Set("prefix", prefix) + return fmt.Sprintf("%s/xrpc/%s?%s", x.cfg.BaseUrl(), tangled.GitTempGetArchiveNSID, params.Encode()) + }() + + filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) + + gw := gzip.NewWriter(w) + defer gw.Close() + + if err := gr.WriteTar(gw, prefix); err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + l.Error("writing tar file", "err", err.Error()) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if err := gw.Flush(); err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with logging the error + l.Error("flushing", "err", err.Error()) + w.WriteHeader(http.StatusInternalServerError) + return + } +} diff --git a/knotmirror/xrpc/git_getBlob.go b/knotmirror/xrpc/git_getBlob.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_getBlob.go @@ -0,0 +1,86 @@ +package xrpc + +import ( + "context" + "fmt" + "io" + "net/http" + "slices" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/knotserver/git" +) + +func (x *Xrpc) GetBlob(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) + path = r.URL.Query().Get("path") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + l := x.logger.With("repo", repo, "ref", ref, "path", path) + + if path == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing path parameter"}) + return + } + + file, err := x.getFile(r.Context(), repo, ref, path) + if err != nil { + // TODO: better error return + l.Error("failed to get blob", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get blob"}) + return + } + + reader, err := file.Reader() + if err != nil { + l.Error("failed to read blob", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"}) + return + } + defer reader.Close() + + w.Header().Set("Content-Type", "application/octet-stream") + if _, err := io.Copy(w, reader); err != nil { + l.Error("failed to serve the blob", "err", err) + } +} + +func (x *Xrpc) getFile(ctx context.Context, repo syntax.ATURI, ref, path string) (*object.File, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("resolving repo at-uri: %w", err) + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + return gr.File(path) +} + +var textualMimeTypes = []string{ + "application/json", + "application/xml", + "application/yaml", + "application/x-yaml", + "application/toml", + "application/javascript", + "application/ecmascript", +} + +// isTextualMimeType returns true if the MIME type represents textual content +// that should be served as text/plain for security reasons +func isTextualMimeType(mimeType string) bool { + return slices.Contains(textualMimeTypes, mimeType) +} diff --git a/knotmirror/xrpc/git_getBranch.go b/knotmirror/xrpc/git_getBranch.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_getBranch.go @@ -0,0 +1,85 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/git" +) + +// TODO: maybe rename to `sh.tangled.repo.temp.getCommit`? +// then, we should ensure the given `ref` is valid +func (x *Xrpc) GetBranch(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + nameQuery = r.URL.Query().Get("name") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + if nameQuery == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing name parameter"}) + return + } + branchName, _ := url.PathUnescape(nameQuery) + + l := x.logger.With("repo", repo, "branch", branchName) + + out, err := x.getBranch(r.Context(), repo, branchName) + if err != nil { + // TODO: better error return + l.Error("failed to get branch", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get branch"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) getBranch(ctx context.Context, repo syntax.ATURI, branchName string) (*tangled.GitTempGetBranch_Output, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) + } + + gr, err := git.PlainOpen(repoPath) + if err != nil { + return nil, fmt.Errorf("failed to open git repo: %w", err) + } + + ref, err := gr.Branch(branchName) + if err != nil { + return nil, fmt.Errorf("getting branch '%s': %w", branchName, err) + } + + commit, err := gr.Commit(ref.Hash()) + if err != nil { + return nil, fmt.Errorf("getting commit '%s': %w", ref.Hash(), err) + } + + out := tangled.GitTempGetBranch_Output{ + Name: ref.Name().Short(), + Hash: ref.Hash().String(), + When: commit.Author.When.Format(time.RFC3339), + Author: &tangled.GitTempDefs_Signature{ + Name: commit.Author.Name, + Email: commit.Author.Email, + When: commit.Author.When.Format(time.RFC3339), + }, + } + + if commit.Message != "" { + out.Message = &commit.Message + } + + return &out, nil +} diff --git a/knotmirror/xrpc/git_getTag.go b/knotmirror/xrpc/git_getTag.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_getTag.go @@ -0,0 +1,92 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/knotserver/git" + "tangled.org/core/types" +) + +func (x *Xrpc) GetTag(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + tagName = r.URL.Query().Get("tag") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + if tagName == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing 'tag' parameter"}) + return + } + + l := x.logger.With("repo", repo, "tag", tagName) + + out, err := x.getTag(r.Context(), repo, tagName) + if err != nil { + // TODO: better error return + l.Error("failed to get tag", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tag"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) getTag(ctx context.Context, repo syntax.ATURI, tagName string) (*types.RepoTagResponse, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) + } + + gr, err := git.PlainOpen(repoPath) + if err != nil { + return nil, fmt.Errorf("failed to open git repo: %w", err) + } + + // if this is not already formatted as refs/tags/v0.1.0, then format it + if !plumbing.ReferenceName(tagName).IsTag() { + tagName = plumbing.NewTagReferenceName(tagName).String() + } + + tag, err := func() (object.Tag, error) { + tags, err := gr.Tags(&git.TagsOptions{ + Pattern: tagName, + }) + if err != nil { + return object.Tag{}, err + } + if len(tags) != 1 { + return object.Tag{}, fmt.Errorf("expected 1 tag to be returned, got %d tags", len(tags)) + } + return tags[0], nil + }() + if err != nil { + return nil, fmt.Errorf("getting tag: %w", err) + } + + var target *object.Tag + if tag.Target != plumbing.ZeroHash { + target = &tag + } + + return &types.RepoTagResponse{ + Tag: &types.TagReference{ + Tag: target, + Reference: types.Reference{ + Name: tag.Name, + Hash: tag.Hash.String(), + }, + Message: tag.Message, + }, + }, nil +} diff --git a/knotmirror/xrpc/git_getTree.go b/knotmirror/xrpc/git_getTree.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_getTree.go @@ -0,0 +1,118 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "path/filepath" + "time" + "unicode/utf8" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/appview/pages/markup" + "tangled.org/core/knotserver/git" +) + +func (x *Xrpc) GetTree(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) + path = r.URL.Query().Get("path") // path can be empty (defaults to root) + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + l := x.logger.With("repo", repo, "ref", ref, "path", path) + + out, err := x.getTree(r.Context(), repo, ref, path) + if err != nil { + // TODO: better error return + l.Error("failed to get tree", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get tree"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) getTree(ctx context.Context, repo syntax.ATURI, ref, path string) (*tangled.GitTempGetTree_Output, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + files, err := gr.FileTree(ctx, path) + if err != nil { + return nil, fmt.Errorf("reading file tree: %w", err) + } + + // if any of these files are a readme candidate, pass along its blob contents too + var readmeFileName string + var readmeContents string + for _, file := range files { + if markup.IsReadmeFile(file.Name) { + contents, err := gr.RawContent(filepath.Join(path, file.Name)) + if err != nil { + x.logger.Error("failed to read contents of file", "path", path, "file", file.Name) + } + + if utf8.Valid(contents) { + readmeFileName = file.Name + readmeContents = string(contents) + break + } + } + } + + // convert NiceTree -> tangled.RepoTempGetTree_TreeEntry + treeEntries := make([]*tangled.GitTempGetTree_TreeEntry, len(files)) + for i, file := range files { + entry := &tangled.GitTempGetTree_TreeEntry{ + Name: file.Name, + Mode: file.Mode, + Size: file.Size, + } + if file.LastCommit != nil { + entry.Last_commit = &tangled.GitTempGetTree_LastCommit{ + Hash: file.LastCommit.Hash.String(), + Message: file.LastCommit.Message, + When: file.LastCommit.When.Format(time.RFC3339), + } + } + treeEntries[i] = entry + } + + var parentPtr *string + if path != "" { + parentPtr = &path + } + + var dotdotPtr *string + if path != "" { + dotdot := filepath.Dir(path) + if dotdot != "." { + dotdotPtr = &dotdot + } + } + + return &tangled.GitTempGetTree_Output{ + Ref: ref, + Parent: parentPtr, + Dotdot: dotdotPtr, + Files: treeEntries, + Readme: &tangled.GitTempGetTree_Readme{ + Filename: readmeFileName, + Contents: readmeContents, + }, + }, nil +} diff --git a/knotmirror/xrpc/git_listBranches.go b/knotmirror/xrpc/git_listBranches.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_listBranches.go @@ -0,0 +1,95 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "path/filepath" + "strconv" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotserver/git" + "tangled.org/core/types" +) + +func (x *Xrpc) ListBranches(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + limitQuery = r.URL.Query().Get("limit") + cursorQuery = r.URL.Query().Get("cursor") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + limit := 50 + if limitQuery != "" { + limit, err = strconv.Atoi(limitQuery) + if err != nil || limit < 1 || limit > 1000 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) + return + } + } + + var cursor int64 + if cursorQuery != "" { + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) + if err != nil || cursor < 0 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) + return + } + } + + l := x.logger.With("repo", repoQuery, "limit", limit, "cursor", cursor) + + out, err := x.listBranches(r.Context(), repo, limit, cursor) + if err != nil { + // TODO: better error return + l.Error("failed to list branches", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list branches"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) listBranches(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoBranchesResponse, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("resolving repo at-uri: %w", err) + } + + gr, err := git.PlainOpen(repoPath) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + branches, err := gr.Branches(&git.BranchesOptions{ + Limit: limit, + Offset: int(cursor), + }) + if err != nil { + return nil, fmt.Errorf("listing git branches: %w", err) + } + + return &types.RepoBranchesResponse{ + // TODO: include default branch and cursor + Branches: branches, + }, nil +} + +func (x *Xrpc) makeRepoPath(ctx context.Context, repo syntax.ATURI) (string, error) { + id, err := x.resolver.ResolveIdent(ctx, repo.Authority().String()) + if err != nil { + return "", err + } + + return filepath.Join( + x.cfg.GitRepoBasePath, + id.DID.String(), + repo.RecordKey().String(), + ), nil +} diff --git a/knotmirror/xrpc/git_listCommits.go b/knotmirror/xrpc/git_listCommits.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_listCommits.go @@ -0,0 +1,95 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotserver/git" + "tangled.org/core/types" +) + +func (x *Xrpc) ListCommits(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) + limitQuery = r.URL.Query().Get("limit") + cursorQuery = r.URL.Query().Get("cursor") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + limit := 50 + if limitQuery != "" { + limit, err = strconv.Atoi(limitQuery) + if err != nil || limit < 1 || limit > 1000 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) + return + } + } + + var cursor int64 + if cursorQuery != "" { + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) + if err != nil || cursor < 0 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) + return + } + } + + l := x.logger.With("repo", repo, "ref", ref) + + out, err := x.listCommits(r.Context(), repo, ref, limit, cursor) + if err != nil { + // TODO: better error return + l.Error("failed to list commits", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list commits"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) listCommits(ctx context.Context, repo syntax.ATURI, ref string, limit int, cursor int64) (*types.RepoLogResponse, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("resolving repo at-uri: %w", err) + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + offset := int(cursor) + + commits, err := gr.Commits(offset, limit) + if err != nil { + return nil, fmt.Errorf("listing git commits: %w", err) + } + + tcommits := make([]types.Commit, len(commits)) + for i, c := range commits { + tcommits[i].FromGoGitCommit(c) + } + + total, err := gr.TotalCommits() + if err != nil { + return nil, fmt.Errorf("counting total commits: %w", err) + } + + return &types.RepoLogResponse{ + Commits: tcommits, + Ref: ref, + Page: (offset / limit) + 1, + PerPage: limit, + Total: total, + Log: true, + }, nil +} diff --git a/knotmirror/xrpc/git_listLanguages.go b/knotmirror/xrpc/git_listLanguages.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_listLanguages.go @@ -0,0 +1,86 @@ +package xrpc + +import ( + "context" + "fmt" + "math" + "net/http" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/git" +) + +func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") + ) + l := x.logger.With("repo", repoQuery, "ref", ref) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + l.Error("invalid repo at-uri", "err", err) + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + out, err := x.listLanguages(r.Context(), repo, ref) + if err != nil { + l.Error("failed to list languages", "err", err) + writeErr(w, err) + return + } + + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) listLanguages(ctx context.Context, repo syntax.ATURI, ref string) (*tangled.GitTempListLanguages_Output, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("resolving repo at-uri: %w", err) + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + return nil, &atclient.APIError{StatusCode: http.StatusNotFound, Name: "RepoNotFound", Message: "failed to find git repo"} + } + + ctx, cancel := context.WithTimeout(ctx, 1*time.Second) + defer cancel() + + sizes, err := gr.AnalyzeLanguages(ctx) + if err != nil { + return nil, fmt.Errorf("analyzing languages: %w", err) + } + + return &tangled.GitTempListLanguages_Output{ + Ref: ref, + Languages: sizesToLanguages(sizes), + }, nil +} + +func sizesToLanguages(sizes git.LangBreakdown) []*tangled.GitTempListLanguages_Language { + var apiLanguages []*tangled.GitTempListLanguages_Language + var totalSize int64 + for _, size := range sizes { + totalSize += size + } + + for name, size := range sizes { + percentagef64 := float64(size) / float64(totalSize) * 100 + percentage := math.Round(percentagef64) + + lang := &tangled.GitTempListLanguages_Language{ + Name: name, + Size: size, + Percentage: int64(percentage), + } + + apiLanguages = append(apiLanguages, lang) + } + + return apiLanguages +} diff --git a/knotmirror/xrpc/git_listTags.go b/knotmirror/xrpc/git_listTags.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/git_listTags.go @@ -0,0 +1,98 @@ +package xrpc + +import ( + "context" + "fmt" + "net/http" + "strconv" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/knotserver/git" + "tangled.org/core/types" +) + +func (x *Xrpc) ListTags(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + limitQuery = r.URL.Query().Get("limit") + cursorQuery = r.URL.Query().Get("cursor") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + limit := 50 + if limitQuery != "" { + limit, err = strconv.Atoi(limitQuery) + if err != nil || limit < 1 || limit > 1000 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("limit parameter invalid: %s", limitQuery)}) + return + } + } + + var cursor int64 + if cursorQuery != "" { + cursor, err = strconv.ParseInt(cursorQuery, 10, 64) + if err != nil || cursor < 0 { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("cursor parameter invalid: %s", cursorQuery)}) + return + } + } + + l := x.logger.With("repo", repo, "limit", limit, "cursor", cursor) + + out, err := x.listTags(r.Context(), repo, limit, cursor) + if err != nil { + // TODO: better error return + l.Error("failed to list tags", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to list tags"}) + return + } + writeJson(w, http.StatusOK, out) +} + +func (x *Xrpc) listTags(ctx context.Context, repo syntax.ATURI, limit int, cursor int64) (*types.RepoTagsResponse, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("failed to resolve repo at-uri: %w", err) + } + + gr, err := git.PlainOpen(repoPath) + if err != nil { + return nil, fmt.Errorf("failed to open git repo: %w", err) + } + + tags, err := gr.Tags(&git.TagsOptions{ + Limit: limit, + Offset: int(cursor), + }) + if err != nil { + return nil, fmt.Errorf("failed to get git tags: %w", err) + } + + rtags := make([]*types.TagReference, len(tags)) + for i, tag := range tags { + var target *object.Tag + if tag.Target != plumbing.ZeroHash { + target = &tag + } + rtags[i] = &types.TagReference{ + Reference: types.Reference{ + Name: tag.Name, + Hash: tag.Hash.String(), + }, + Tag: target, + Message: tag.Message, + } + } + + return &types.RepoTagsResponse{ + Tags: rtags, + }, nil +} diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/xrpc.go @@ -0,0 +1,69 @@ +package xrpc + +import ( + "database/sql" + "encoding/json" + "errors" + "log/slog" + "net/http" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/go-chi/chi/v5" + "tangled.org/core/api/tangled" + "tangled.org/core/idresolver" + "tangled.org/core/knotmirror/config" + "tangled.org/core/log" +) + +type Xrpc struct { + cfg *config.Config + db *sql.DB + resolver *idresolver.Resolver + logger *slog.Logger +} + +func New(logger *slog.Logger, cfg *config.Config, db *sql.DB, resolver *idresolver.Resolver) *Xrpc { + return &Xrpc{ + cfg, + db, + resolver, + log.SubLogger(logger, "xrpc"), + } +} + +func (x *Xrpc) Router() http.Handler { + r := chi.NewRouter() + + r.Get("/"+tangled.GitTempGetArchiveNSID, x.GetArchive) + r.Get("/"+tangled.GitTempGetBlobNSID, x.GetBlob) + r.Get("/"+tangled.GitTempGetBranchNSID, x.GetBranch) + // r.Get("/"+tangled.GitTempGetCommitNSID, x.GetCommit) // todo + // r.Get("/"+tangled.GitTempGetDiffNSID, x.GetDiff) // todo + // r.Get("/"+tangled.GitTempGetEntityNSID, x.GetEntity) // todo + // r.Get("/"+tangled.GitTempGetHeadNSID, x.GetHead) // todo + r.Get("/"+tangled.GitTempGetTagNSID, x.GetTag) // using types.Response + r.Get("/"+tangled.GitTempGetTreeNSID, x.GetTree) + r.Get("/"+tangled.GitTempListBranchesNSID, x.ListBranches) // wip, unknown output + r.Get("/"+tangled.GitTempListCommitsNSID, x.ListCommits) + r.Get("/"+tangled.GitTempListLanguagesNSID, x.ListLanguages) + r.Get("/"+tangled.GitTempListTagsNSID, x.ListTags) + + return r +} + +func writeJson(w http.ResponseWriter, status int, response any) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(response); err != nil { + return err + } + return nil +} + +func writeErr(w http.ResponseWriter, err error) error { + var apiErr *atclient.APIError + if errors.As(err, &apiErr) { + return writeJson(w, apiErr.StatusCode, atclient.ErrorBody{Name: apiErr.Name, Message: apiErr.Message}) + } + return writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "internal server error"}) +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -199,6 +199,20 @@ return io.ReadAll(reader) } +func (g *GitRepo) File(path string) (*object.File, error) { + c, err := g.r.CommitObject(g.h) + if err != nil { + return nil, fmt.Errorf("commit object: %w", err) + } + + tree, err := c.Tree() + if err != nil { + return nil, fmt.Errorf("file tree: %w", err) + } + + return tree.File(path) +} + // read and parse .gitmodules func (g *GitRepo) Submodules() (*config.Modules, error) { c, err := g.r.CommitObject(g.h) -- tangled.sh