From 37343725ebdb907270964e53cc91b023de1c9182 Mon Sep 17 00:00:00 2001 From: Aly Raffauf Date: Fri, 3 Jul 2026 16:54:42 -0400 Subject: [PATCH] add fully configurable CLI with kong --- README.md | 38 ++++++++++++++++++++++++++----- cmd/asterism/main.go | 53 ++++++++++++++++++++++++++++++++------------ go.mod | 1 + go.sum | 8 +++++++ 4 files changed, 81 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 7ee9bd7..b9a95a4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Asterism, meanwhile, consumes cryptographically verifiable events directly from ## What it does -Asterism connects directly to the relay Firehose (`com.atproto.sync.subscribeRepos`), decodes each repo commit's CAR-framed CBOR blocks itself, and recursively walks each record for link references (strong refs, AT-URIs, DIDs, URLs). Links are stored keyed by target, source collection, and field path. On startup it backfills existing repos for your configured collections so the index is useful immediately. +Asterism connects directly to the relay Firehose (`com.atproto.sync.subscribeRepos`), decodes each repo commit's CAR-framed CBOR blocks itself, and recursively walks each record for link references (strong refs, AT-URIs, DIDs, URLs). Links are stored keyed by target, source collection, and field path. It can optionally backfill existing repos for your configured collections on startup so the index is useful immediately. This matters for two reasons: @@ -30,14 +30,42 @@ Relay ──► Asterism (raw commits, filter locally) The typical deployment indexes only the collections your app queries: ```bash -go run ./cmd/asterism/ -collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment +go run ./cmd/asterism/ --collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment ``` -This connects to the relay Firehose, backfills existing repos for those collections in the background, stores links in an sqlite database at `asterism.db`, and serves the query API on `:8081`. +This connects to the relay Firehose, stores links in an sqlite database at `asterism.db`, and serves the query API on `:8081`. Firehose events that are too large to include inline are fetched through the repo API automatically. -To index all collections (Constellation-equivalent scope, not recommended for sovereign deployments): +To also backfill existing repos for your configured collections on startup: ```bash +go run ./cmd/asterism/ --backfill --collections sh.tangled.graph.follow,sh.tangled.repo.issue,sh.tangled.feed.comment +``` + +To live-index all collections (Constellation-equivalent scope, not recommended for sovereign deployments): + +```bash +go run ./cmd/asterism/ +``` + +### Configuration + +Every flag can also be set with an environment variable. + +| Flag | Environment variable | Default | Description | +|---|---|---|---| +| `--collections` | `ASTERISM_COLLECTIONS` | empty | Comma-separated collection NSIDs to index. Empty means all collections. | +| `--backfill` | `ASTERISM_BACKFILL` | false | Backfill existing repos for configured collections on startup. | +| `--database` | `ASTERISM_DATABASE` | `asterism.db` | SQLite database path. | +| `--listen` | `ASTERISM_LISTEN` | `:8081` | HTTP API listen address. | +| `--relay` | `ASTERISM_RELAY` | `relay1.us-east.bsky.network` | Relay host. Asterism derives the Firehose websocket and relay HTTP API URLs from this host. | + +For example: + +```bash +ASTERISM_DATABASE=/var/lib/asterism/asterism.db \ +ASTERISM_LISTEN=:8081 \ +ASTERISM_COLLECTIONS=sh.tangled.graph.follow,sh.tangled.repo.issue \ +ASTERISM_BACKFILL=true \ go run ./cmd/asterism/ ``` @@ -126,7 +154,7 @@ Note the DID filter parameter is `did` here, not `linkDid` like `getManyToMany` **Near term** - [x] Full Constellation API parity (`getBacklinksCount`, `getBacklinkDids`, `getBacklinks`, `getManyToMany`, `getManyToManyCounts`) -- [ ] Configurable listen address, database path, and relay URL +- [x] Configurable listen address, database path, relay host, and startup backfill - [ ] Account deletion and deactivation handling - [x] Graceful shutdown and Firehose reconnect diff --git a/cmd/asterism/main.go b/cmd/asterism/main.go index 674ba8f..13beb5a 100644 --- a/cmd/asterism/main.go +++ b/cmd/asterism/main.go @@ -2,11 +2,11 @@ package main import ( "context" - "flag" "fmt" "log/slog" "strings" + "github.com/alecthomas/kong" "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/xrpc" @@ -16,7 +16,17 @@ import ( "github.com/alyraffauf/asterism/internal/store" ) -const relayURL = "wss://relay1.us-east.bsky.network/xrpc/com.atproto.sync.subscribeRepos" +const ( + subscribeReposPath = "/xrpc/com.atproto.sync.subscribeRepos" +) + +type CLI struct { + Collections string `help:"Comma-separated list of collection NSIDs to filter on. Empty means all." env:"ASTERISM_COLLECTIONS"` + Backfill bool `help:"Backfill existing repos on startup." env:"ASTERISM_BACKFILL"` + Database string `help:"SQLite database path." env:"ASTERISM_DATABASE" default:"asterism.db"` + Listen string `help:"HTTP listen address." env:"ASTERISM_LISTEN" default:":8081"` + Relay string `help:"Relay host." env:"ASTERISM_RELAY" default:"relay1.us-east.bsky.network"` +} func parseCollections(raw string) map[string]struct{} { if raw == "" { @@ -35,44 +45,59 @@ func parseCollections(raw string) map[string]struct{} { return wanted } +func relayHTTPHost(relayHost string) string { + return "https://" + relayHost +} + +func subscribeReposURL(relayHost string) string { + return "wss://" + relayHost + subscribeReposPath +} + func main() { - collectionsFlag := flag.String("collections", "", "comma-separated list of collection NSIDs to filter on (empty means all)") - flag.Parse() + var cli CLI + kong.Parse(&cli, + kong.Name("asterism"), + kong.Description("AT Protocol link index."), + ) ctx := context.Background() logger := slog.Default() - linkStore, err := store.Open("asterism.db") + linkStore, err := store.Open(cli.Database) if err != nil { panic(err) } server := &api.Server{Store: linkStore} go func() { - if err := server.Run(":8081"); err != nil { + if err := server.Run(cli.Listen); err != nil { panic(err) } }() - wantedCollections := parseCollections(*collectionsFlag) + wantedCollections := parseCollections(cli.Collections) var collections []string for collection := range wantedCollections { collections = append(collections, collection) } + relayURL := subscribeReposURL(cli.Relay) + bf := &backfill.Backfill{ - Client: &xrpc.Client{Host: "https://relay1.us-east.bsky.network"}, + Client: &xrpc.Client{Host: relayHTTPHost(cli.Relay)}, Directory: identity.DefaultDirectory(), Store: linkStore, } - if len(collections) > 0 { - go func() { - if err := bf.Run(ctx, collections); err != nil { - fmt.Println("backfill error:", err) - } - }() + if cli.Backfill { + if len(collections) > 0 { + go func() { + if err := bf.Run(ctx, collections); err != nil { + fmt.Println("backfill error:", err) + } + }() + } } consumer := &firehose.Consumer{ diff --git a/go.mod b/go.mod index 7d9ab42..9fcc667 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/alyraffauf/asterism go 1.26.4 require ( + github.com/alecthomas/kong v1.15.0 github.com/bluesky-social/indigo v0.0.0-20260629160527-dfe5578fd537 github.com/gorilla/websocket v1.5.3 github.com/ipfs/go-cid v0.4.1 diff --git a/go.sum b/go.sum index 46eeb6e..6d449a6 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,12 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b h1:5/++qT1/z812ZqBvqQt6ToRswSuPZ/B33m6xVHRzADU= github.com/RussellLuo/slidingwindow v0.0.0-20200528002341-535bb99d338b/go.mod h1:4+EPqMRApwwE/6yo6CxiHoSnBzjRr3jsqer7frxP8y4= +github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= +github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/kong v1.15.0 h1:BVJstKbpO73zKpmIu+m/aLRrNmWwxXPIGTNin9VmLVI= +github.com/alecthomas/kong v1.15.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= +github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= +github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -59,6 +65,8 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= -- 2.51.2