From 0a9a35bd58b60e92dfa23ed43988e9c8a420ad50 Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Thu, 14 May 2026 00:02:45 +0900 Subject: [PATCH] nix,spindle: sync workflow files on `sh.tangled.git.refUpdate` Spindle will sync git repo when new repo is registered Spindle will listen to `sh.tangled.git.refUpdate` event from knot stream and sync its local git repo instead. Spindle's git repo will sparse-checkout only `/.tangled/workflows` directory. Spindle now requires git version >=2.49 for `--revision` flag in `git clone` command. References: - - Signed-off-by: Seongmin Lee --- go.mod | 1 + go.sum | 2 ++ spindle/git/git.go | 73 ++++++++++++++++++++++++++++++++++++++++++++ spindle/server.go | 33 +++++++++++++++++++- spindle/tapclient.go | 15 +++++++-- 5 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 spindle/git/git.go diff --git a/go.mod b/go.mod index 5a442303..84d2f6ad 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/gorilla/feeds v1.2.0 github.com/gorilla/sessions v1.4.0 github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 + github.com/hashicorp/go-version v1.9.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hiddeco/sshsig v0.2.0 github.com/hpcloud/tail v1.0.0 diff --git a/go.sum b/go.sum index fcad8535..4f30e60c 100644 --- a/go.sum +++ b/go.sum @@ -343,6 +343,8 @@ github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= diff --git a/spindle/git/git.go b/spindle/git/git.go new file mode 100644 index 00000000..f3e549c1 --- /dev/null +++ b/spindle/git/git.go @@ -0,0 +1,73 @@ +package git + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "strings" + + "github.com/hashicorp/go-version" +) + +func Version() (*version.Version, error) { + var buf bytes.Buffer + cmd := exec.Command("git", "version") + cmd.Stdout = &buf + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + return nil, err + } + fields := strings.Fields(buf.String()) + if len(fields) < 3 { + return nil, fmt.Errorf("invalid git version: %s", buf.String()) + } + + // version string is like: "git version 2.29.3" or "git version 2.29.3.windows.1" + versionString := fields[2] + if pos := strings.Index(versionString, "windows"); pos >= 1 { + versionString = versionString[:pos-1] + } + return version.NewVersion(versionString) +} + +const WorkflowDir = `/.tangled/workflows` + +func SparseSyncGitRepo(ctx context.Context, cloneUri, path, rev string) error { + exist, err := isDir(path) + if err != nil { + return err + } + if rev == "" { + rev = "HEAD" + } + if !exist { + if err := exec.Command("git", "clone", "--no-checkout", "--depth=1", "--filter=tree:0", "--revision="+rev, cloneUri, path).Run(); err != nil { + return fmt.Errorf("git clone: %w", err) + } + if err := exec.Command("git", "-C", path, "sparse-checkout", "set", "--no-cone", WorkflowDir).Run(); err != nil { + return fmt.Errorf("git sparse-checkout set: %w", err) + } + } else { + if err := exec.Command("git", "-C", path, "fetch", "--depth=1", "--filter=tree:0", "origin", rev).Run(); err != nil { + return fmt.Errorf("git pull: %w", err) + } + } + if err := exec.Command("git", "-C", path, "checkout", rev).Run(); err != nil { + return fmt.Errorf("git checkout: %w", err) + } + return nil +} + +func isDir(path string) (bool, error) { + info, err := os.Stat(path) + if err == nil && info.IsDir() { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} diff --git a/spindle/server.go b/spindle/server.go index 4df488d9..f8d516ae 100644 --- a/spindle/server.go +++ b/spindle/server.go @@ -8,6 +8,7 @@ import ( "log/slog" "maps" "net/http" + "path/filepath" "sync" "github.com/bluesky-social/indigo/atproto/syntax" @@ -24,6 +25,7 @@ import ( "tangled.org/core/spindle/db" "tangled.org/core/spindle/engine" "tangled.org/core/spindle/engines/nixery" + "tangled.org/core/spindle/git" "tangled.org/core/spindle/models" "tangled.org/core/spindle/queue" "tangled.org/core/spindle/secrets" @@ -353,7 +355,11 @@ func (s *Spindle) XrpcRouter() http.Handler { } func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, msg eventconsumer.Message) error { - if msg.Nsid == tangled.PipelineNSID { + l := log.FromContext(ctx).With("handler", "processKnotStream") + l = l.With("src", src.Key(), "msg.Nsid", msg.Nsid, "msg.Rkey", msg.Rkey) + switch msg.Nsid { + case tangled.PipelineNSID: + return nil tpl := tangled.Pipeline{} err := json.Unmarshal(msg.EventJson, &tpl) if err != nil { @@ -457,6 +463,31 @@ func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, } else { s.l.Error("failed to enqueue pipeline: queue is full") } + case tangled.GitRefUpdateNSID: + var event tangled.GitRefUpdate + if err := json.Unmarshal(msg.EventJson, &event); err != nil { + return err + } + repoDid, err := syntax.ParseDID(event.Repo) + if err != nil { + return fmt.Errorf("repo should be valid did: %w", err) + } + + l = l.With("repo", repoDid, "ref", event.Ref, "newSha", event.NewSha) + + repo, err := s.db.GetRepoByDid(repoDid) + if err != nil { + return fmt.Errorf("failed to query repo from db: %w", err) + } + + repoCloneUri := "" // {knot}/{did} + repoPath := filepath.Join("", string(repo.RepoDid)) + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, event.NewSha); err != nil { + return fmt.Errorf("syncing git repo: %w", err) + } + l.Info("synced git repo") + + // TODO: plan the pipeline } return nil diff --git a/spindle/tapclient.go b/spindle/tapclient.go index 23f6cfc0..f1da6587 100644 --- a/spindle/tapclient.go +++ b/spindle/tapclient.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "log/slog" + "path/filepath" "sync" "time" @@ -16,6 +17,7 @@ import ( "tangled.org/core/log" "tangled.org/core/rbac" "tangled.org/core/spindle/db" + "tangled.org/core/spindle/git" "tangled.org/core/tapc" ) @@ -122,17 +124,26 @@ func (t *Tap) processRepo(ctx context.Context, evt *tapc.RecordEventData) error src := eventconsumer.NewKnotSource(record.Knot) t.spindle.ks.AddSource(t.spindle.rootCtx, src) - if err := t.spindle.db.AddRepo(db.Repo{ + repo := db.Repo{ Knot: record.Knot, Owner: ownerDid, Rkey: rkey, RepoDid: repoDid, CreatedAt: record.CreatedAt, - }); err != nil { + } + + if err := t.spindle.db.AddRepo(repo); err != nil { l.Error("failed to add repo row", "err", err) return fmt.Errorf("add repo: %w", err) } + // setup sparse sync + repoCloneUri := "" // {knot}/{did} + repoPath := filepath.Join("", string(repo.RepoDid)) + if err := git.SparseSyncGitRepo(ctx, repoCloneUri, repoPath, ""); err != nil { + return fmt.Errorf("setting up sparse-clone git repo: %w", err) + } + legacyName := "" if record.Name != nil { legacyName = *record.Name -- 2.51.2