diff --git a/cmd/collectiondir/collectiondir.go b/cmd/collectiondir/collectiondir.go index 3ffafbb8..cbf533dc 100644 --- a/cmd/collectiondir/collectiondir.go +++ b/cmd/collectiondir/collectiondir.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "encoding/csv" "encoding/json" "errors" @@ -18,11 +19,11 @@ import ( "time" "github.com/earthboundkid/versioninfo/v2" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" ) func main() { - app := cli.App{ + app := cli.Command{ Name: "collectiondir", Usage: "collection directory service", Version: versioninfo.Short(), @@ -40,8 +41,7 @@ func main() { adminCrawlCmd, }, } - err := app.Run(os.Args) - if err != nil { + if err := app.Run(context.Background(), os.Args); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err.Error()) os.Exit(1) } @@ -57,14 +57,14 @@ var statsCmd = &cli.Command{ Required: true, }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { logLevel := slog.LevelInfo - if cctx.Bool("verbose") { + if cmd.Bool("verbose") { logLevel = slog.LevelDebug } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) slog.SetDefault(log) - pebblePath := cctx.String("pebble") + pebblePath := cmd.String("pebble") var db PebbleCollectionDirectory db.log = log err := db.Open(pebblePath) @@ -94,14 +94,14 @@ var exportCmd = &cli.Command{ Required: true, }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { logLevel := slog.LevelInfo - if cctx.Bool("verbose") { + if cmd.Bool("verbose") { logLevel = slog.LevelDebug } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) slog.SetDefault(log) - pebblePath := cctx.String("pebble") + pebblePath := cmd.String("pebble") var db PebbleCollectionDirectory db.log = log err := db.Open(pebblePath) @@ -112,7 +112,7 @@ var exportCmd = &cli.Command{ rows := make(chan CollectionDidTime, 100) go func() { - err := db.ReadAllPrimary(cctx.Context, rows) + err := db.ReadAllPrimary(ctx, rows) if err != nil { log.Error("db read", "path", pebblePath, "err", err) } @@ -153,14 +153,14 @@ var buildCmd = &cli.Command{ Required: true, }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { logLevel := slog.LevelInfo - if cctx.Bool("verbose") { + if cmd.Bool("verbose") { logLevel = slog.LevelDebug } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) slog.SetDefault(log) - pebblePath := cctx.String("pebble") + pebblePath := cmd.String("pebble") var db PebbleCollectionDirectory db.log = log err := db.Open(pebblePath) @@ -168,7 +168,7 @@ var buildCmd = &cli.Command{ return err } defer db.Close() - csvPath := cctx.String("csv") + csvPath := cmd.String("csv") var fin io.Reader if csvPath == "-" { fin = os.Stdin @@ -231,17 +231,17 @@ var adminCrawlCmd = &cli.Command{ Name: "url", Usage: "host:port of collectiondir server", Required: true, - EnvVars: []string{"COLLECTIONDIR_URL"}, + Sources: cli.EnvVars("COLLECTIONDIR_URL"), }, &cli.StringFlag{ Name: "auth", Usage: "Auth token for admin api", - EnvVars: []string{"ADMIN_AUTH"}, + Sources: cli.EnvVars("ADMIN_AUTH"), }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { logLevel := slog.LevelInfo - if cctx.Bool("verbose") { + if cmd.Bool("verbose") { logLevel = slog.LevelDebug } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) @@ -249,21 +249,21 @@ var adminCrawlCmd = &cli.Command{ var hostList []string - serverUrl, err := url.Parse(cctx.String("url")) + serverUrl, err := url.Parse(cmd.String("url")) if err != nil { var e2 error // try to fixup a bare host:port which can confuse url.Parse - serverUrl, e2 = url.Parse("http://" + cctx.String("url")) + serverUrl, e2 = url.Parse("http://" + cmd.String("url")) if e2 != nil { return fmt.Errorf("could not parse url, %w", err) } } requestCrawlUrl := serverUrl.JoinPath("/admin/pds/requestCrawl") - if cctx.IsSet("list") { - fin, err := os.Open(cctx.String("list")) + if cmd.IsSet("list") { + fin, err := os.Open(cmd.String("list")) if err != nil { - return fmt.Errorf("%s: could not open, %w", cctx.String("list"), err) + return fmt.Errorf("%s: could not open, %w", cmd.String("list"), err) } defer fin.Close() bufin := bufio.NewScanner(fin) @@ -272,20 +272,20 @@ var adminCrawlCmd = &cli.Command{ } err = bufin.Err() if err != nil { - return fmt.Errorf("%s: error reading, %w", cctx.String("list"), err) + return fmt.Errorf("%s: error reading, %w", cmd.String("list"), err) } - } else if cctx.IsSet("csv") { - fin, err := os.Open(cctx.String("csv")) + } else if cmd.IsSet("csv") { + fin, err := os.Open(cmd.String("csv")) if err != nil { - return fmt.Errorf("%s: could not open, %w", cctx.String("csv"), err) + return fmt.Errorf("%s: could not open, %w", cmd.String("csv"), err) } defer fin.Close() data, err := csv.NewReader(fin).ReadAll() if err != nil { - return fmt.Errorf("%s: could not read, %w", cctx.String("csv"), err) + return fmt.Errorf("%s: could not read, %w", cmd.String("csv"), err) } if len(data) < 2 { - return fmt.Errorf("%s: empty CSV file", cctx.String("csv")) + return fmt.Errorf("%s: empty CSV file", cmd.String("csv")) } headerRow := data[0] hostCol := -1 @@ -297,7 +297,7 @@ var adminCrawlCmd = &cli.Command{ } } if hostCol < 0 { - return fmt.Errorf("%s: header missing 'host' or 'hostname'", cctx.String("csv")) + return fmt.Errorf("%s: header missing 'host' or 'hostname'", cmd.String("csv")) } for _, row := range data[1:] { hostList = append(hostList, row[hostCol]) @@ -310,8 +310,8 @@ var adminCrawlCmd = &cli.Command{ client := http.Client{Timeout: 1 * time.Second} var headers http.Header = make(http.Header) - if cctx.IsSet("auth") { - headers.Add("Authorization", "Bearer "+cctx.String("auth")) + if cmd.IsSet("auth") { + headers.Add("Authorization", "Bearer "+cmd.String("auth")) } headers.Add("Content-Type", "application/json") var response *http.Response diff --git a/cmd/collectiondir/crawl.go b/cmd/collectiondir/crawl.go index fe8ce258..f6679e64 100644 --- a/cmd/collectiondir/crawl.go +++ b/cmd/collectiondir/crawl.go @@ -15,7 +15,7 @@ import ( "github.com/bluesky-social/indigo/util" "github.com/bluesky-social/indigo/xrpc" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" "golang.org/x/time/rate" ) @@ -55,14 +55,14 @@ var offlineCrawlCmd = &cli.Command{ &cli.StringFlag{ Name: "ratelimit-header", Usage: "secret for friend PDSes", - EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"}, + Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"), }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelInfo})) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - hostname := cctx.String("host") + hostname := cmd.String("host") hosturl, err := url.Parse(hostname) if err != nil { hosturl = new(url.URL) @@ -73,13 +73,13 @@ var offlineCrawlCmd = &cli.Command{ Host: hosturl.String(), Client: util.RobustHTTPClient(), } - if cctx.IsSet("ratelimit-header") { + if cmd.IsSet("ratelimit-header") { rpcClient.Headers = map[string]string{ - "x-ratelimit-bypass": cctx.String("ratelimit-header"), + "x-ratelimit-bypass": cmd.String("ratelimit-header"), } } log.Info("will crawl", "url", rpcClient.Host) - csvOutPath := cctx.String("csv-out") + csvOutPath := cmd.String("csv-out") var fout io.Writer = os.Stdout if csvOutPath != "" { if csvOutPath == "-" { @@ -91,7 +91,7 @@ var offlineCrawlCmd = &cli.Command{ } } } - qps := cctx.Float64("qps") + qps := cmd.Float64("qps") results := make(chan DidCollection, 100) defer close(results) go DidCollectionsToCsv(fout, results) diff --git a/cmd/collectiondir/serve.go b/cmd/collectiondir/serve.go index 0abf074f..36c6d0ee 100644 --- a/cmd/collectiondir/serve.go +++ b/cmd/collectiondir/serve.go @@ -33,7 +33,7 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/urfave/cli/v2" + "github.com/urfave/cli/v3" ) var serveCmd = &cli.Command{ @@ -42,12 +42,12 @@ var serveCmd = &cli.Command{ &cli.StringFlag{ Name: "api-listen", Value: ":2510", - EnvVars: []string{"COLLECTIONS_API_LISTEN"}, + Sources: cli.EnvVars("COLLECTIONS_API_LISTEN"), }, &cli.StringFlag{ Name: "metrics-listen", Value: ":2511", - EnvVars: []string{"COLLECTIONS_METRICS_LISTEN"}, + Sources: cli.EnvVars("COLLECTIONS_METRICS_LISTEN"), }, &cli.StringFlag{ Name: "pebble", @@ -62,12 +62,12 @@ var serveCmd = &cli.Command{ &cli.StringFlag{ Name: "upstream", Usage: "URL, e.g. wss://bsky.network", - EnvVars: []string{"COLLECTIONS_UPSTREAM"}, + Sources: cli.EnvVars("COLLECTIONS_UPSTREAM"), }, &cli.StringFlag{ Name: "admin-token", Usage: "admin authentication", - EnvVars: []string{"COLLECTIONS_ADMIN_TOKEN"}, + Sources: cli.EnvVars("COLLECTIONS_ADMIN_TOKEN"), }, &cli.Float64Flag{ Name: "crawl-qps", @@ -77,32 +77,32 @@ var serveCmd = &cli.Command{ &cli.StringFlag{ Name: "ratelimit-header", Usage: "secret for friend PDSes", - EnvVars: []string{"BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"}, + Sources: cli.EnvVars("BSKY_SOCIAL_RATE_LIMIT_SKIP", "RATE_LIMIT_HEADER"), }, &cli.Uint64Flag{ Name: "clist-min-dids", Usage: "filter collection list to >= N dids", Value: 5, - EnvVars: []string{"COLLECTIONS_CLIST_MIN_DIDS"}, + Sources: cli.EnvVars("COLLECTIONS_CLIST_MIN_DIDS"), }, &cli.IntFlag{ Name: "max-did-collections", Usage: "stop recording new collections per did after it has >= this many collections", Value: 1000, - EnvVars: []string{"COLLECTIONS_MAX_DID_COLLECTIONS"}, + Sources: cli.EnvVars("COLLECTIONS_MAX_DID_COLLECTIONS"), }, &cli.StringFlag{ Name: "sets-json-path", Usage: "file path of JSON file containing static word sets", - EnvVars: []string{"HEPA_SETS_JSON_PATH", "COLLECTIONS_SETS_JSON_PATH"}, + Sources: cli.EnvVars("HEPA_SETS_JSON_PATH", "COLLECTIONS_SETS_JSON_PATH"), }, &cli.BoolFlag{ Name: "verbose", }, }, - Action: func(cctx *cli.Context) error { + Action: func(ctx context.Context, cmd *cli.Command) error { var server collectionServer - return server.run(cctx) + return server.run(ctx, cmd) }, } @@ -165,29 +165,29 @@ type activeCrawl struct { stats *CrawlStats } -func (cs *collectionServer) run(cctx *cli.Context) error { +func (cs *collectionServer) run(ctx context.Context, cmd *cli.Command) error { signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) cs.shutdown = make(chan struct{}) level := slog.LevelInfo - if cctx.Bool("verbose") { + if cmd.Bool("verbose") { level = slog.LevelDebug } log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: level})) slog.SetDefault(log) - if cctx.IsSet("ratelimit-header") { - cs.ratelimitHeader = cctx.String("ratelimit-header") + if cmd.IsSet("ratelimit-header") { + cs.ratelimitHeader = cmd.String("ratelimit-header") } - if cctx.IsSet("sets-json-path") { - badwords, err := loadBadwords(cctx.String("sets-json-path")) + if cmd.IsSet("sets-json-path") { + badwords, err := loadBadwords(cmd.String("sets-json-path")) if err != nil { return err } cs.badwords = badwords } - cs.MinDidsForCollectionList = cctx.Uint64("clist-min-dids") - cs.MaxDidCollections = cctx.Int("max-did-collections") + cs.MinDidsForCollectionList = cmd.Uint64("clist-min-dids") + cs.MaxDidCollections = cmd.Int("max-did-collections") cs.ingestFirehose = make(chan DidCollection, 1000) cs.ingestCrawl = make(chan DidCollection, 1000) var err error @@ -196,12 +196,12 @@ func (cs *collectionServer) run(cctx *cli.Context) error { return fmt.Errorf("lru init, %w", err) } cs.log = log - cs.ctx = cctx.Context - cs.AdminToken = cctx.String("admin-token") + cs.ctx = ctx + cs.AdminToken = cmd.String("admin-token") cs.ExepctedAuthHeader = "Bearer " + cs.AdminToken cs.wg.Add(1) go cs.ingestReceiver() - pebblePath := cctx.String("pebble") + pebblePath := cmd.String("pebble") cs.pcd = &PebbleCollectionDirectory{ log: cs.log, } @@ -209,7 +209,7 @@ func (cs *collectionServer) run(cctx *cli.Context) error { if err != nil { return fmt.Errorf("%s: failed to open pebble db: %w", pebblePath, err) } - cs.dauDirectoryDir = cctx.String("dau-directory") + cs.dauDirectoryDir = cmd.String("dau-directory") if cs.dauDirectoryDir != "" { err := cs.openDau() if err != nil { @@ -218,18 +218,18 @@ func (cs *collectionServer) run(cctx *cli.Context) error { } cs.statsCacheFresh.L = &cs.statsCacheLock - apiServerEcho, err := cs.createApiServer(cctx.Context, cctx.String("api-listen")) + apiServerEcho, err := cs.createApiServer(ctx, cmd.String("api-listen")) if err != nil { return err } cs.wg.Add(1) - go func() { cs.StartApiServer(cctx.Context, apiServerEcho) }() + go func() { cs.StartApiServer(ctx, apiServerEcho) }() - cs.createMetricsServer(cctx.String("metrics-listen")) + cs.createMetricsServer(cmd.String("metrics-listen")) cs.wg.Add(1) - go func() { cs.StartMetricsServer(cctx.Context) }() + go func() { cs.StartMetricsServer(ctx) }() - upstream := cctx.String("upstream") + upstream := cmd.String("upstream") if upstream != "" { fh := Firehose{ Log: log,