Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
3.9 kB · 156 lines
Go
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157// prefill-zoekt bulk-enqueues zoekt index tasks//// It reads a REPOS file with one repo DID per line://// did:plc:repository//// For each DID it resolves the knot from the DID document, then resolves the// remote HEAD branch + commit via `git ls-remote --symref`, and POSTs an// enqueue request to the indexserver's /admin/enqueueIndex endpoint.package main
import ( "bytes" "context" "encoding/json" "flag" "fmt" "log" "net/http" "os" "os/exec" "strings" "sync" "sync/atomic"
"github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/sourcegraph/zoekt")
func main() { reposPath := flag.String("repos", "REPOS", "path to repos list file (one DID per line)") server := flag.String("server", "http://localhost:6060", "indexserver base url") plc := flag.String("plc", "https://plc.directory", "atproto PLC directory url") concurrency := flag.Int("concurrency", 5, "number of repos to process in parallel") flag.Parse()
data, err := os.ReadFile(*reposPath) if err != nil { log.Fatalf("reading %s: %v", *reposPath, err) }
ctx := context.Background() dir := identity.BaseDirectory{PLCURL: *plc}
var ok, fail atomic.Int64 var wg sync.WaitGroup sem := make(chan struct{}, *concurrency)
for i, line := range strings.Split(string(data), "\n") { did := strings.TrimSpace(line) if did == "" { continue }
wg.Add(1) sem <- struct{}{} go func(i int, did string) { defer wg.Done() defer func() { <-sem }()
knot, err := resolveKnot(ctx, &dir, did) if err != nil { log.Printf("line %d: %s: resolving knot: %v", i+1, did, err) fail.Add(1) return }
branch, sha, err := resolveHead(knot, did) if err != nil { log.Printf("line %d: %s: resolving HEAD: %v", i+1, did, err) fail.Add(1) return }
if err := enqueue(*server, did, branch, sha); err != nil { log.Printf("line %d: %s: enqueue: %v", i+1, did, err) fail.Add(1) return } log.Printf("line %d: %s: enqueued %s@%s (knot=%s)", i+1, did, branch, sha, knot) ok.Add(1) }(i, did) }
wg.Wait() fmt.Printf("done: %d enqueued, %d failed\n", ok.Load(), fail.Load())}
func resolveKnot(ctx context.Context, dir identity.Directory, did string) (string, error) { d, err := syntax.ParseDID(did) if err != nil { return "", err } ident, err := dir.LookupDID(ctx, d) if err != nil { return "", err } knot := ident.PDSEndpoint() if knot == "" { return "", fmt.Errorf("no PDS endpoint in DID document") } return knot, nil}
func resolveHead(knot, did string) (branch, sha string, err error) { url := strings.TrimRight(knot, "/") + "/" + did out, err := exec.Command( "git", "-c", "http.sslVerify=false", "ls-remote", "--symref", url, "HEAD", ).Output() if err != nil { return "", "", fmt.Errorf("git ls-remote --symref %s HEAD: %w", url, err) } for line := range strings.SplitSeq(string(out), "\n") { fields := strings.Fields(line) if len(fields) < 2 { continue } switch { case fields[0] == "ref:": branch = strings.TrimPrefix(fields[1], "refs/heads/") case fields[1] == "HEAD": sha = fields[0] } } if branch == "" || sha == "" { return "", "", fmt.Errorf("could not resolve HEAD (branch=%q sha=%q)", branch, sha) } return branch, sha, nil}
func enqueue(server, did, branch, sha string) error { body, err := json.Marshal(map[string]any{ "repo": did, "branches": []zoekt.RepositoryBranch{{Name: branch, Version: sha}}, }) if err != nil { return err }
resp, err := http.Post(strings.TrimRight(server, "/")+"/admin/enqueueIndex", "application/json", bytes.NewReader(body)) if err != nil { return err } defer resp.Body.Close()
log.Println("status", resp.StatusCode)
if resp.StatusCode < 200 || resp.StatusCode >= 300 { return fmt.Errorf("status %d", resp.StatusCode) } return nil}