From c18cbcdd1edc0265bcfcc4bb894676426e63b52f Mon Sep 17 00:00:00 2001 From: dhawos Date: Tue, 23 Jun 2026 07:31:13 +0200 Subject: [PATCH] indexing into elasticsearch --- .env.dev | 5 +++ .envrc | 1 + cmd/indexerv2/main.go | 73 ++++++++++++++++++++++++++++++------------- cmd/root.go | 31 ++++++++++++++++++ config.yaml | 0 config/viper.go | 5 +++ docker-compose.yaml | 28 ----------------- go.mod | 12 +++++++ go.sum | 25 +++++++++++++++ main.go | 7 +++++ 10 files changed, 138 insertions(+), 49 deletions(-) create mode 100644 .env.dev create mode 100644 cmd/root.go create mode 100644 config.yaml create mode 100644 config/viper.go create mode 100644 main.go diff --git a/.env.dev b/.env.dev new file mode 100644 index 0000000..618247f --- /dev/null +++ b/.env.dev @@ -0,0 +1,5 @@ +SKYFEEDGO_FIREHOSE="relay1.eurosky.network" +SKYFEEDGO_ELASTICSEARCH_HOST="localhost" +SKYFEEDGO_ELASTICSEARCH_PORT="9200" +SKYFEEDGO_ELASTICSEARCH_USERNAME="elastic" +SKYFEEDGO_ELASTICSEARCH_PASSWORD="password" diff --git a/.envrc b/.envrc index 3550a30..2c18343 100644 --- a/.envrc +++ b/.envrc @@ -1 +1,2 @@ +dotenv .env.dev use flake diff --git a/cmd/indexerv2/main.go b/cmd/indexerv2/main.go index 1a4c6d4..9b37fbb 100644 --- a/cmd/indexerv2/main.go +++ b/cmd/indexerv2/main.go @@ -1,28 +1,59 @@ -package main +package indexerv2 import ( - "context" + "fmt" + "bluesky-custom-feeds/config" "bluesky-custom-feeds/indexer" + + "github.com/spf13/cobra" ) -func main() { - ctx := context.Background() - process := indexer.NewFirehoseListener("relay1.eurosky.network") - channels := indexer.NewBlueskyChannels() - esClient, err := indexer.NewElasticTypedClient("http://elastic:password@localhost:9200") - if err != nil { - panic(err) - } - err = indexer.InitIndex(ctx, esClient) - if err != nil { - panic(err) - } - go indexer.HandlePosts(ctx, channels.Posts, esClient) - go indexer.HandleLikes(channels.Likes) - go indexer.HandleReposts(channels.Reposts) - err = process.SubscribeFirehose(ctx, channels) - if err != nil { - panic(err) - } +type FirehoseConfig struct { + Host string +} + +type ElasticsearchConfig struct { + Host string + Username string + Password string + Port uint +} + +type IndexerConfig struct { + Firehose FirehoseConfig + Elasticsearch ElasticsearchConfig +} + +var Cmd = &cobra.Command{ + Use: "indexer", + Short: "Commands for ingesting from the firehose", + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + var indexerConfig IndexerConfig + + if err := config.Loader.Unmarshal(&indexerConfig); err != nil { + return fmt.Errorf("config: unable to decode into struct: %w", err) + } + fmt.Println(indexerConfig) + process := indexer.NewFirehoseListener(indexerConfig.Firehose.Host) + channels := indexer.NewBlueskyChannels() + esURL := fmt.Sprintf("http://%s:%s@%s:%d", indexerConfig.Elasticsearch.Username, indexerConfig.Elasticsearch.Password, indexerConfig.Elasticsearch.Host, indexerConfig.Elasticsearch.Port) + esClient, err := indexer.NewElasticTypedClient(esURL) + if err != nil { + panic(err) + } + err = indexer.InitIndex(ctx, esClient) + if err != nil { + panic(err) + } + go indexer.HandlePosts(ctx, channels.Posts, esClient) + go indexer.HandleLikes(channels.Likes) + go indexer.HandleReposts(channels.Reposts) + err = process.SubscribeFirehose(ctx, channels) + if err != nil { + panic(err) + } + return nil + }, } diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..97d6964 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "strings" + + "bluesky-custom-feeds/cmd/indexerv2" + "bluesky-custom-feeds/config" + "github.com/spf13/cobra" +) + +func init() { + config.Loader.SetConfigName("config") + config.Loader.AddConfigPath("$HOME/.appname") + config.Loader.AddConfigPath(".") + config.Loader.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + config.Loader.AutomaticEnv() + config.Loader.SetEnvPrefix("SKYFEEDGO") + + err := config.Loader.ReadInConfig() + if err != nil { + panic(err) + } + + RootCmd.AddCommand(indexerv2.Cmd) +} + +var RootCmd = &cobra.Command{ + Use: "skyfeedgo", + Short: "A CLI to help using custom feeds on bluesky", + SilenceUsage: true, +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/config/viper.go b/config/viper.go new file mode 100644 index 0000000..c80d80a --- /dev/null +++ b/config/viper.go @@ -0,0 +1,5 @@ +package config + +import "github.com/spf13/viper" + +var Loader = viper.NewWithOptions(viper.ExperimentalBindStruct()) diff --git a/docker-compose.yaml b/docker-compose.yaml index 124d9a5..f5614c0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,37 +11,9 @@ services: ELASTIC_PASSWORD: "password" volumes: - elasticsearch:/usr/share/elasticsearch/data - tap: - image: ghcr.io/bluesky-social/indigo/tap@sha256:5e20bfe416d29fcd215ed8bf99f10b2ab825a6de4e5599846dd33967ade2abeb # 0.1.10 - networks: - - tap - ports: - - 2480:2480 - depends_on: - - postgresql - environment: - TAP_DATABASE_URL: postgresql://tap:tap@postgresql:5432/tap - TAP_NO_REPLAY: "true" - TAP_RELAY_URL: https://relay1.eurosky.network - TAP_SIGNAL_COLLECTION: app.bsky.actor.profile - TAP_COLLECTION_FILTERS: "app.bsky.feed.post,app.bsky.feed.like,app.bsky.feed.repost" - postgresql: - image: postgres@sha256:1b1689b20d16a014a3d195653381cf2caa75a41a92d93b255a9d6ea29fd353aa # 0.1.10 - networks: - - tap - ports: - - 5432:5432 - volumes: - - postgresql:/var/lib/postgresql - environment: - POSTGRES_DB: tap - POSTGRES_USER: tap - POSTGRES_PASSWORD: tap networks: elasticsearch: - tap: volumes: elasticsearch: - postgresql: diff --git a/go.mod b/go.mod index 498ea5f..fdcb911 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/earthboundkid/versioninfo/v2 v2.24.1 // indirect github.com/elastic/elastic-transport-go/v8 v8.9.0 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gammazero/chanqueue v1.1.2 // indirect github.com/gammazero/deque v1.2.1 // indirect @@ -32,6 +33,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.3 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/goccy/go-yaml v1.19.2 // indirect github.com/gocql/gocql v1.7.0 // indirect @@ -43,6 +45,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/bbloom v0.1.0 // indirect github.com/ipfs/boxo v0.41.0 // indirect github.com/ipfs/go-block-format v0.2.3 // indirect @@ -96,7 +99,15 @@ require ( github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.60.0 // indirect github.com/rogpeppe/go-internal v1.15.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/spf13/viper v1.21.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect @@ -111,6 +122,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.28.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.28.0 // indirect golang.org/x/crypto v0.53.0 // indirect golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect diff --git a/go.sum b/go.sum index aa62cb7..fa65af4 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,7 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI= github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -47,6 +48,8 @@ github.com/filecoin-project/go-clock v0.1.0 h1:SFbYIM75M8NnFm1yMHhN9Ahy3W5bEZV9g github.com/filecoin-project/go-clock v0.1.0/go.mod h1:4uB/O4PvOjlx1VCMdZ9MyDZXRm//gkj1ELEbxfI1AZs= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/chanqueue v1.1.2 h1:dZEsxlyANZMyeTRemABqZF8QM9BnE4NBI43Oh3y5fIU= @@ -72,6 +75,8 @@ github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJ github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -107,6 +112,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/bbloom v0.1.0 h1:nIWwfIE3AaG7RCDQIsrUonGCOTp7qSXzxH7ab/ss964= github.com/ipfs/bbloom v0.1.0/go.mod h1:lDy3A3i6ndgEW2z1CaRFvDi5/ZTzgM1IxA/pkL7Wgts= github.com/ipfs/boxo v0.41.0 h1:diKlFosOG2e1mgSO1CXqcMSnHvtn6ubUvaCf9iF8AIY= @@ -287,12 +294,28 @@ github.com/quic-go/quic-go v0.60.0/go.mod h1:wpKpjmPpftl30sL6pFh7REVpjbcCVy4zt2v github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -306,6 +329,8 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..62f44d6 --- /dev/null +++ b/main.go @@ -0,0 +1,7 @@ +package main + +import "bluesky-custom-feeds/cmd" + +func main() { + cmd.RootCmd.Execute() +} -- 2.51.2