diff --git a/knotserver/events.go b/knotserver/events.go --- a/knotserver/events.go +++ b/knotserver/events.go @@ -50,12 +50,12 @@ } // complete backfill first before going to live data + l.Info("going through backfill", "cursor", cursor) l.Debug("going through backfill", "cursor", cursor) if err := h.streamOps(conn, &cursor); err != nil { l.Error("failed to backfill", "err", err) return } - for { // wait for new data or timeout select { diff --git a/spindle/exec.go b/spindle/exec.go deleted file mode 100644 --- a/spindle/exec.go +++ /dev/null @@ -1,54 +0,0 @@ -package spindle - -import ( - "context" - "encoding/json" - "fmt" - - "tangled.sh/tangled.sh/core/api/tangled" -) - -func (s *Spindle) exec(ctx context.Context, src string, msg []byte) error { - pipeline := tangled.Pipeline{} - data := map[string]any{} - err := json.Unmarshal(msg, &data) - if err != nil { - fmt.Println("error unmarshalling", err) - return err - } - - if data["nsid"] == tangled.PipelineNSID { - event, ok := data["event"] - if !ok { - s.l.Error("no event in message") - return nil - } - - rawEvent, err := json.Marshal(event) - if err != nil { - return err - } - - err = json.Unmarshal(rawEvent, &pipeline) - if err != nil { - return err - } - - rkey, ok := data["rkey"].(string) - if !ok { - s.l.Error("no rkey in message") - return nil - } - - err = s.eng.SetupPipeline(ctx, &pipeline, rkey) - if err != nil { - return err - } - err = s.eng.StartWorkflows(ctx, &pipeline, rkey) - if err != nil { - return err - } - } - - return nil -} diff --git a/spindle/server.go b/spindle/server.go --- a/spindle/server.go +++ b/spindle/server.go @@ -1,6 +1,7 @@ package spindle import ( + "encoding/json" "fmt" "log/slog" "net/http" @@ -68,11 +69,14 @@ go func() { logger.Info("starting event consumer") - ec := knotclient.NewEventConsumer(knotclient.ConsumerConfig{ - Sources: []string{"ws://localhost:5555/events"}, + knotEventSource := knotclient.NewEventSource("localhost:5555") + ccfg := knotclient.ConsumerConfig{ Logger: logger, ProcessFunc: spindle.exec, - }) + } + ccfg.AddEventSource(knotEventSource) + + ec := knotclient.NewEventConsumer(ccfg) ec.Start(ctx) }() @@ -88,4 +92,26 @@ mux.HandleFunc("/events", s.Events) return mux +} + +func (s *Spindle) exec(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error { + pipeline := tangled.Pipeline{} + err := json.Unmarshal(msg.EventJson, &pipeline) + if err != nil { + fmt.Println("error unmarshalling", err) + return err + } + + if msg.Nsid == tangled.PipelineNSID { + err = s.eng.SetupPipeline(ctx, &pipeline, msg.Rkey) + if err != nil { + return err + } + err = s.eng.StartWorkflows(ctx, &pipeline, msg.Rkey) + if err != nil { + return err + } + } + + return nil } diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -1,6 +1,7 @@ package state import ( + "context" "encoding/json" "fmt" "slices" @@ -17,7 +18,7 @@ "github.com/posthog/posthog-go" ) -func KnotstreamConsumer(c *config.Config, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client) (*kc.EventConsumer, error) { +func KnotstreamConsumer(ctx context.Context, c *config.Config, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client) (*kc.EventConsumer, error) { knots, err := db.GetCompletedRegistrations(d) if err != nil { return nil, err @@ -35,7 +36,7 @@ cfg := kc.ConsumerConfig{ Sources: srcs, - ProcessFunc: knotstreamIngester(d, enforcer, posthog, c.Core.Dev), + ProcessFunc: knotstreamIngester(ctx, d, enforcer, posthog, c.Core.Dev), RetryInterval: c.Knotstream.RetryInterval, MaxRetryInterval: c.Knotstream.MaxRetryInterval, ConnectionTimeout: c.Knotstream.ConnectionTimeout, @@ -49,8 +50,8 @@ return kc.NewEventConsumer(cfg), nil } -func knotstreamIngester(d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) kc.ProcessFunc { - return func(source kc.EventSource, msg kc.Message) error { +func knotstreamIngester(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, dev bool) kc.ProcessFunc { + return func(ctx context.Context, source kc.EventSource, msg kc.Message) error { switch msg.Nsid { case tangled.GitRefUpdateNSID: return ingestRefUpdate(d, enforcer, posthog, dev, source, msg) diff --git a/appview/state/state.go b/appview/state/state.go --- a/appview/state/state.go +++ b/appview/state/state.go @@ -48,7 +48,7 @@ knotstream *knotclient.EventConsumer } -func Make(config *config.Config) (*State, error) { +func Make(ctx context.Context, config *config.Config) (*State, error) { d, err := db.Make(config.Core.DbPath) if err != nil { return nil, err @@ -104,16 +104,16 @@ if err != nil { return nil, fmt.Errorf("failed to create jetstream client: %w", err) } - err = jc.StartJetstream(context.Background(), appview.Ingest(wrapper, enforcer)) + err = jc.StartJetstream(ctx, appview.Ingest(wrapper, enforcer)) if err != nil { return nil, fmt.Errorf("failed to start jetstream watcher: %w", err) } - knotstream, err := KnotstreamConsumer(config, d, enforcer, posthog) + knotstream, err := KnotstreamConsumer(ctx, config, d, enforcer, posthog) if err != nil { return nil, fmt.Errorf("failed to start knotstream consumer: %w", err) } - knotstream.Start(context.Background()) + knotstream.Start(ctx) state := &State{ d, diff --git a/cmd/appview/main.go b/cmd/appview/main.go --- a/cmd/appview/main.go +++ b/cmd/appview/main.go @@ -14,13 +14,15 @@ func main() { slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, nil))) - c, err := config.LoadConfig(context.Background()) + ctx := context.Background() + + c, err := config.LoadConfig(ctx) if err != nil { log.Println("failed to load config", "error", err) return } - state, err := state.Make(c) + state, err := state.Make(ctx, c) if err != nil { log.Fatal(err) diff --git a/cmd/eventconsumer/main.go b/cmd/eventconsumer/main.go --- a/cmd/eventconsumer/main.go +++ b/cmd/eventconsumer/main.go @@ -24,19 +24,18 @@ return } - var srcs []knotclient.EventSource - for k := range strings.SplitSeq(*knots, ",") { - srcs = append(srcs, knotclient.EventSource{k}) - } - - consumer := knotclient.NewEventConsumer(knotclient.ConsumerConfig{ - Sources: srcs, + ccfg := knotclient.ConsumerConfig{ ProcessFunc: processEvent, RetryInterval: *retryFlag, MaxRetryInterval: *maxRetryFlag, WorkerCount: *workerCount, Dev: true, - }) + } + for k := range strings.SplitSeq(*knots, ",") { + ccfg.AddEventSource(knotclient.NewEventSource(k)) + } + + consumer := knotclient.NewEventConsumer(ccfg) ctx, cancel := context.WithCancel(context.Background()) consumer.Start(ctx) @@ -45,7 +44,7 @@ consumer.Stop() } -func processEvent(source knotclient.EventSource, msg knotclient.Message) error { +func processEvent(_ context.Context, source knotclient.EventSource, msg knotclient.Message) error { fmt.Printf("From %s (%s, %s): %s\n", source.Knot, msg.Rkey, msg.Nsid, string(msg.EventJson)) return nil }