Something went wrong. Try again.
Monorepo for Tangled tangled.org
Something went wrong. Try again.
11 kB · 386 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387package knotmirror
import ( "context" "errors" "fmt" "log" "net/url" "os" "os/exec" "path/filepath" "regexp" "strings" "time"
"github.com/go-git/go-git/v5" gitconfig "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing/transport" "tangled.org/core/knotmirror/models")
type branch struct { Name string `json:"name"` Version string `json:"version"`}
type GitMirrorManager interface { Exist(repo *models.Repo) (bool, error) // Clone clones the repository as a mirror Clone(ctx context.Context, repo *models.Repo) error // Fetch fetches the repository Fetch(ctx context.Context, repo *models.Repo) error // Sync mirrors the repository. It will clone the repository if repository doesn't exist. Sync(ctx context.Context, repo *models.Repo) error DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) Delete(repo *models.Repo) error}
type CliGitMirrorManager struct { repoBasePath string knotUseSSL bool}
func NewCliGitMirrorManager(repoBasePath string, knotUseSSL bool) *CliGitMirrorManager { return &CliGitMirrorManager{ repoBasePath, knotUseSSL, }}
var _ GitMirrorManager = new(CliGitMirrorManager)
func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string { return filepath.Join(c.repoBasePath, repo.RepoDid.String())}
func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) { return isDir(c.makeRepoPath(repo))}
func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } return c.clone(ctx, path, url)}
func (c *CliGitMirrorManager) clone(ctx context.Context, path, url string) error { cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", url, path) if out, err := cmd.CombinedOutput(); err != nil { if ctx.Err() != nil { return ctx.Err() } msg := string(out) if classification := classifyCliError(msg); classification != nil { return classification } return fmt.Errorf("running 'git clone --mirror %s': %w\n%s", url, err, msg) } writeCommitGraph(ctx, path, 30*time.Second) return nil}
func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } return c.fetch(ctx, path, url)}
func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error { cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*") if out, err := cmd.CombinedOutput(); err != nil { if ctx.Err() != nil { return ctx.Err() } return fmt.Errorf("running 'git fetch': %w\n%s", err, string(out)) }
// TODO(boltless): make this dedicated event instead lsRemoteCmd := exec.CommandContext(ctx, "git", "ls-remote", "--symref", url, "HEAD") out, err := lsRemoteCmd.CombinedOutput() if err != nil { if ctx.Err() != nil { return ctx.Err() } return fmt.Errorf("running 'git ls-remote --symref': %w\n%s", err, string(out)) }
var headRef string for line := range strings.SplitSeq(string(out), "\n") { if !strings.HasPrefix(line, "ref: ") { continue } fields := strings.Fields(line) if len(fields) >= 2 { headRef = fields[1] break } } if headRef != "" { symrefCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "HEAD", headRef) if out, err := symrefCmd.CombinedOutput(); err != nil { if ctx.Err() != nil { return ctx.Err() } return fmt.Errorf("running 'git symbolic-ref HEAD %s': %w\n%s", headRef, err, string(out)) } }
writeCommitGraph(ctx, path, 3*time.Second) return nil}
func writeCommitGraph(ctx context.Context, path string, timeout time.Duration) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() if err := exec.CommandContext(ctx, "git", "-C", path, "commit-graph", "write", "--reachable", "--split").Run(); err != nil { log.Println("failed to run commit-graph", err) }}
func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) }
exist, err := isDir(path) if err != nil { return fmt.Errorf("checking repo path: %w", err) } if !exist { if err := c.clone(ctx, path, url); err != nil { return fmt.Errorf("cloning repo: %w", err) } } else { if err := c.fetch(ctx, path, url); err != nil { return fmt.Errorf("fetching repo: %w", err) } } return nil}
func (c *CliGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) { path := c.makeRepoPath(repo)
nameCmd := exec.CommandContext(ctx, "git", "-C", path, "symbolic-ref", "--short", "HEAD") nameOut, err := nameCmd.Output() if err != nil { return branch{}, err }
// --verify --quiet exits 1 with no output on an empty repo (unborn HEAD). revCmd := exec.CommandContext(ctx, "git", "-C", path, "rev-parse", "--verify", "--quiet", "HEAD") revOut, err := revCmd.Output() if err != nil { return branch{}, err }
version := strings.TrimSpace(string(revOut)) if version == "" { return branch{}, errors.New("git: no commits") }
return branch{ Name: strings.TrimSpace(string(nameOut)), Version: version, }, nil}
func (c *CliGitMirrorManager) Delete(repo *models.Repo) error { return os.RemoveAll(c.makeRepoPath(repo))}
var ( ErrDNSFailure = errors.New("git: knot: dns failure (could not resolve host)") ErrCertExpired = errors.New("git: knot: certificate has expired") ErrCertMismatch = errors.New("git: knot: certificate hostname mismatch") ErrTLSHandshake = errors.New("git: knot: tls handshake failure") ErrHTTPStatus = errors.New("git: knot: request url returned error") ErrUnreachable = errors.New("git: knot: could not connect to server") ErrRepoNotFound = errors.New("git: repo: repository not found"))
var ( reDNSFailure = regexp.MustCompile(`Could not resolve host:`) reCertExpired = regexp.MustCompile(`SSL certificate OpenSSL verify result: certificate has expired`) reCertMismatch = regexp.MustCompile(`SSL: no alternative certificate subject name matches target hostname`) reTLSHandshake = regexp.MustCompile(`TLS connect error: (.*)`) reHTTPStatus = regexp.MustCompile(`The requested URL returned error: (\d\d\d)`) reUnreachable = regexp.MustCompile(`Could not connect to server`) reRepoNotFound = regexp.MustCompile(`repository '.*?' not found`))
// classifyCliError classifies git cli error message. It will return nil for unknown error messagesfunc classifyCliError(stderr string) error { msg := strings.TrimSpace(stderr) if m := reTLSHandshake.FindStringSubmatch(msg); len(m) > 1 { return fmt.Errorf("%w: %s", ErrTLSHandshake, m[1]) } if m := reHTTPStatus.FindStringSubmatch(msg); len(m) > 1 { return fmt.Errorf("%w: %s", ErrHTTPStatus, m[1]) } switch { case reDNSFailure.MatchString(msg): return ErrDNSFailure case reCertExpired.MatchString(msg): return ErrCertExpired case reCertMismatch.MatchString(msg): return ErrCertMismatch case reUnreachable.MatchString(msg): return ErrUnreachable case reRepoNotFound.MatchString(msg): return ErrRepoNotFound } return nil}
type GoGitMirrorManager struct { repoBasePath string knotUseSSL bool}
func NewGoGitMirrorClient(repoBasePath string, knotUseSSL bool) *GoGitMirrorManager { return &GoGitMirrorManager{ repoBasePath, knotUseSSL, }}
var _ GitMirrorManager = new(GoGitMirrorManager)
func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string { return filepath.Join(c.repoBasePath, repo.RepoDid.String())}
func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) { return isDir(c.makeRepoPath(repo))}
func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } return c.clone(ctx, path, url)}
func (c *GoGitMirrorManager) clone(ctx context.Context, path, url string) error { _, err := git.PlainCloneContext(ctx, path, true, &git.CloneOptions{ URL: url, Mirror: true, }) if err != nil && !errors.Is(err, transport.ErrEmptyRemoteRepository) { return fmt.Errorf("cloning repo: %w", err) } return nil}
func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) }
return c.fetch(ctx, path, url)}
func (c *GoGitMirrorManager) fetch(ctx context.Context, path, url string) error { gr, err := git.PlainOpen(path) if err != nil { return fmt.Errorf("opening local repo: %w", err) } if err := gr.FetchContext(ctx, &git.FetchOptions{ RemoteURL: url, RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec("+refs/*:refs/*")}, Force: true, Prune: true, }); err != nil { return fmt.Errorf("fetching reppo: %w", err) } return nil}
func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) }
exist, err := isDir(path) if err != nil { return fmt.Errorf("checking repo path: %w", err) } if !exist { if err := c.clone(ctx, path, url); err != nil { return fmt.Errorf("cloning repo: %w", err) } } else { if err := c.fetch(ctx, path, url); err != nil { return fmt.Errorf("fetching repo: %w", err) } } return nil}
func (c *GoGitMirrorManager) DefaultBranch(ctx context.Context, repo *models.Repo) (branch, error) { gr, err := git.PlainOpen(c.makeRepoPath(repo)) if err != nil { return branch{}, fmt.Errorf("opening local repo: %w", err) } ref, err := gr.Head() if err != nil { return branch{}, fmt.Errorf("resolving HEAD: %w", err) } return branch{ Name: ref.Name().Short(), Version: ref.Hash().String(), }, nil}
func (c *GoGitMirrorManager) Delete(repo *models.Repo) error { return os.RemoveAll(c.makeRepoPath(repo))}
func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) { if !strings.Contains(knot, "://") { if knotUseSSL { knot = "https://" + knot } else { knot = "http://" + knot } }
u, err := url.Parse(knot) if err != nil { return "", err }
if u.Scheme != "http" && u.Scheme != "https" { return "", fmt.Errorf("unsupported scheme: %s", u.Scheme) }
u = u.JoinPath(repoIdentifier) return u.String(), nil}
func isDir(path string) (bool, error) { info, err := os.Stat(path) if err == nil && info.IsDir() { return true, nil } if os.IsNotExist(err) { return false, nil } return false, err}