From d438e62db13d6c3ebb457894c8a2f99d76b50e06 Mon Sep 17 00:00:00 2001 From: Ashlynne Mitchell Date: Thu, 8 Jan 2026 19:01:20 +1100 Subject: [PATCH] refactor: namespace existing functionality to Drinkup.Firehose --- AGENTS.md | 300 ++++---------------------- CHANGELOG.md | 7 + examples/basic_consumer.ex | 6 +- examples/multiple_consumers.ex | 10 +- examples/record_consumer.ex | 5 +- lib/{drinkup.ex => firehose.ex} | 6 +- lib/{ => firehose}/consumer.ex | 4 +- lib/{ => firehose}/event.ex | 4 +- lib/{ => firehose}/event/account.ex | 2 +- lib/{ => firehose}/event/commit.ex | 2 +- lib/{ => firehose}/event/identity.ex | 2 +- lib/{ => firehose}/event/info.ex | 2 +- lib/{ => firehose}/event/sync.ex | 2 +- lib/{ => firehose}/options.ex | 2 +- lib/{ => firehose}/record_consumer.ex | 12 +- lib/{ => firehose}/socket.ex | 4 +- 16 files changed, 79 insertions(+), 291 deletions(-) rename lib/{drinkup.ex => firehose.ex} (89%) rename lib/{ => firehose}/consumer.ex (69%) rename lib/{ => firehose}/event.ex (94%) rename lib/{ => firehose}/event/account.ex (96%) rename lib/{ => firehose}/event/commit.ex (98%) rename lib/{ => firehose}/event/identity.ex (92%) rename lib/{ => firehose}/event/info.ex (90%) rename lib/{ => firehose}/event/sync.ex (93%) rename lib/{ => firehose}/options.ex (93%) rename lib/{ => firehose}/record_consumer.ex (85%) rename lib/{ => firehose}/socket.ex (98%) diff --git a/AGENTS.md b/AGENTS.md index 5909d69..e6d04ca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,260 +1,40 @@ -# AGENTS.md - -This file provides guidance for agentic coding assistants working with the -Drinkup codebase. - -## Project Overview - -Drinkup is an Elixir library for consuming events from an ATProtocol relay -(firehose/`com.atproto.sync.subscribeRepos`). It uses OTP principles with -GenStatem for managing WebSocket connections and Task.Supervisor for concurrent -event processing. - -## Build, Lint, and Test Commands - -### Running Tests - -```bash -# Run all tests -mix test - -# Run a single test file -mix test test/drinkup_test.exs - -# Run a specific test by line number -mix test test/drinkup_test.exs:5 - -# Run tests with coverage -mix test --cover - -# Run tests matching a pattern -mix test --only [tag_name] -``` - -### Formatting and Linting - -```bash -# Format code (uses .formatter.exs config) -mix format - -# Check if code is formatted -mix format --check-formatted - -# Run Credo for static code analysis -mix credo - -# Run Credo strictly -mix credo --strict -``` - -### Compilation and Documentation - -```bash -# Compile the project -mix compile - -# Clean build artifacts -mix clean - -# Generate documentation -mix docs - -# Run Dialyzer for type checking (if configured) -mix dialyzer -``` - -## Code Style Guidelines - -### Module Structure - -- Use `defmodule` with clear, descriptive names following `Drinkup.` - namespace -- Place module documentation (`@moduledoc`) immediately after `defmodule` -- Group related functionality within nested modules (e.g., - `Drinkup.Event.Commit.RepoOp`) -- Order module contents: module attributes, types, public functions, private - functions - -### Imports and Aliases - -- Use `require` for macros (e.g., `require Logger`) -- Use `alias` to shorten module names, prefer explicit aliases over `import` -- Group in order: `require`, `alias`, `import` -- Example: - ```elixir - require Logger - alias Drinkup.{Event, Options} - ``` - -### Type Specifications - -- Use TypedStruct for structs with typed fields (dependency: - `{:typedstruct, "~> 0.5"}`) -- Define `@type` specs for complex types, unions, and public APIs -- Use `@spec` for all public functions -- Use `enforce: true` for required TypedStruct fields -- Example: - - ```elixir - use TypedStruct - - typedstruct enforce: true do - field :consumer, module() - field :name, atom(), default: Drinkup - field :cursor, pos_integer() | nil, enforce: false - end - ``` - -### Naming Conventions - -- Modules: PascalCase (`Drinkup.Event.Commit`) -- Functions: snake_case (`handle_event/1`, `from/1`) -- Variables: snake_case (`repo_op`, `last_seq`) -- Private functions: prefix with `defp`, mark with `@spec` if complex -- Atoms: lowercase with underscores (`:ok`, `:connect_timeout`) -- Behaviours: use `@behaviour` (not `@behavior`) - -### Function Definitions - -- Pattern match in function heads when possible -- Use guard clauses for simple type/value checks -- Prefer multiple function heads over large case statements -- Example: - ```elixir - def valid_seq?(nil, seq) when is_integer(seq), do: true - def valid_seq?(last_seq, nil) when is_integer(last_seq), do: true - def valid_seq?(last_seq, seq) when is_integer(last_seq) and is_integer(seq), - do: seq > last_seq - def valid_seq?(_last_seq, _seq), do: false - ``` - -### Error Handling - -- Use `try/rescue` for expected errors, catch and log appropriately -- Use Logger for errors: - `Logger.error("Message: #{Exception.format(:error, e, __STACKTRACE__)}")` -- Return tagged tuples: `{:ok, result}` or `{:error, reason}` -- Use `with` for chaining operations that may fail -- Example from Socket module: - ```elixir - with {:ok, header, next} <- CAR.DagCbor.decode(frame), - {:ok, payload, _} <- CAR.DagCbor.decode(next), - {%{"op" => @op_regular}, _} <- {header, payload} do - # happy path - else - {:error, reason} -> Logger.warning("Failed to decode: #{inspect(reason)}") - end - ``` - -### OTP and Concurrency Patterns - -- Use `child_spec/1` for custom supervisor specifications -- Prefer `GenServer` for stateful processes, `:gen_statem` for state machines -- Use `Task.Supervisor` for concurrent, fire-and-forget work (see - `Event.dispatch/2`) -- Register processes via Registry for named lookups -- Define proper restart strategies (`:permanent`, `:transient`, `:temporary`) - -### Comments - -- Avoid obvious comments; prefer self-documenting code -- Use `# TODO:` for future improvements (see existing TODOs in codebase) -- Use `# DEPRECATED` for deprecated fields (see Commit struct) -- Document complex algorithms or non-obvious business logic -- Use module-level `@moduledoc` and function-level `@doc` for public APIs - -### Formatting - -- Use `mix format` (configured in `.formatter.exs`) -- Import deps for formatting: `import_deps: [:typedstruct]` -- Line length: default Elixir formatter settings -- Use 2-space indentation (enforced by formatter) - -### Testing - -- Use ExUnit for tests (files in `test/` with `_test.exs` suffix) -- Use `use ExUnit.Case` in test modules -- Use `doctest Module` for testing documentation examples -- Tag tests for selective running: `@tag :integration` -- Use descriptive test names: `test "validates sequence numbers correctly"` - -## Project-Specific Patterns - -### Consumer Behaviour Pattern - -- Implement `@behaviour Drinkup.Consumer` with `handle_event/1` callback -- Use pattern matching to handle different event types -- Return any value; errors are caught by Task.Supervisor wrapper - -### RecordConsumer Macro Pattern - -- Use `use Drinkup.RecordConsumer` with `collections:` opt for filtering -- Override `handle_create/1`, `handle_update/1`, `handle_delete/1` as needed -- Collections can be exact strings or Regex patterns: `~r/app\.bsky\.graph\..+/` - -### WebSocket State Machine - -- Socket module uses `:gen_statem` with states: `:disconnected`, - `:connecting_http`, `:connecting_ws`, `:connected` -- State functions match on events: `state_name(:enter, from, data)` or - `state_name(:info, msg, data)` -- Use `{:next_event, :internal, event}` for internal state transitions - -## Dependencies - -- `{:gun, "~> 2.2"}` - HTTP/WebSocket client -- `{:car, "~> 0.1.0"}` - CAR (Content Addressable aRchive) format -- `{:cbor, "~> 1.0.0"}` - CBOR encoding/decoding -- `{:typedstruct, "~> 0.5"}` - Typed structs -- `{:credo, "~> 1.7"}` - Static analysis (dev/test only) - -## Common Tasks - -### Adding a New Event Type - -1. Create `lib/event/your_event.ex` with TypedStruct definition -2. Add `from/1` function to parse payload -3. Add pattern match in `Drinkup.Event.from/2` -4. Add to `@type t()` union in `Drinkup.Event` -5. Update `CHANGELOG.md` under `[Unreleased]` section with the new feature - -### Debugging Connection Issues - -- Check `:gun` connection logs in Socket module -- Verify sequence tracking with `Event.valid_seq?/2` -- Monitor state transitions: `:disconnected` → `:connecting_http` → - `:connecting_ws` → `:connected` - -## Changelog Management - -**IMPORTANT**: After completing any feature or fixing a bug from a previous -release, you MUST update `CHANGELOG.md`. - -### Changelog Format - -- Follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format -- Uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) -- Group changes under appropriate sections: `Added`, `Changed`, `Deprecated`, - `Removed`, `Fixed`, `Security` - -### When to Update - -- **New features**: Add under `## [Unreleased]` → `### Added` -- **Bug fixes**: Add under `## [Unreleased]` → `### Fixed` -- **Breaking changes**: Add under `## [Unreleased]` → `### Breaking Changes` -- **Deprecations**: Add under `## [Unreleased]` → `### Deprecated` -- **Security fixes**: Add under `## [Unreleased]` → `### Security` - -### Example Entry - -```markdown -## [Unreleased] - -### Added - -- Support for `#handle` event type in firehose consumer - -### Fixed - -- Sequence validation now correctly handles nil cursor on initial connection -``` +# Agent Guidelines for Drinkup + +## Commands + +- **Test**: `mix test` (all), `mix test test/path/to/file_test.exs` (single file), `mix test test/path/to/file_test.exs:42` (single test at line) +- **Format**: `mix format` (auto-formats all code) +- **Lint**: `mix credo` (static analysis), `mix credo --strict` (strict mode) +- **Compile**: `mix compile` +- **Docs**: `mix docs` +- **Type Check**: `mix dialyzer` (if configured) + +## Code Style + +- **Imports**: Use `alias` for modules (e.g., `alias Drinkup.Firehose.{Event, Options}`), `require` for macros (e.g., `require Logger`) +- **Formatting**: Elixir 1.18+, auto-formatted via `.formatter.exs` with `import_deps: [:typedstruct]` +- **Naming**: snake_case for functions/variables, PascalCase for modules, `:lowercase_atoms` for atoms, `@behaviour` (not `@behavior`) +- **Types**: Use `@type` and `@spec` for all functions; use TypedStruct for structs with `enforce: true` for required fields +- **Moduledocs**: Public modules need `@moduledoc`, public functions need `@doc` with examples +- **Error Handling**: Return `{:ok, result}` or `{:error, reason}` tuples; use `with` for chaining operations; log errors with `Logger.error("#{Exception.format(:error, e, __STACKTRACE__)}")` +- **Pattern Matching**: Prefer pattern matching in function heads over conditionals; use guard clauses when appropriate +- **OTP**: Use `child_spec/1` for custom supervisor specs; `:gen_statem` for state machines; `Task.Supervisor` for concurrent tasks; Registry for named lookups +- **Tests**: Use ExUnit with `use ExUnit.Case`; use `doctest Module` for documentation examples +- **Dependencies**: Core deps include gun (WebSocket), car (CAR format), cbor (encoding), TypedStruct (typed structs), Credo (linting) + +## Project Structure + +- **Namespace**: All firehose functionality under `Drinkup.Firehose.*` + - `Drinkup.Firehose` - Main supervisor + - `Drinkup.Firehose.Consumer` - Behaviour for handling all events + - `Drinkup.Firehose.RecordConsumer` - Macro for handling commit record events with filtering + - `Drinkup.Firehose.Event` - Event types (`Commit`, `Sync`, `Identity`, `Account`, `Info`) + - `Drinkup.Firehose.Socket` - `:gen_statem` WebSocket connection manager +- **Consumer Pattern**: Implement `@behaviour Drinkup.Firehose.Consumer` with `handle_event/1` +- **RecordConsumer Pattern**: `use Drinkup.Firehose.RecordConsumer, collections: [~r/app\.bsky\.graph\..+/, "app.bsky.feed.post"]` with `handle_create/1`, `handle_update/1`, `handle_delete/1` overrides + +## Important Notes + +- **Update CHANGELOG.md** when adding features, changes, or fixes under `## [Unreleased]` with appropriate sections (`Added`, `Changed`, `Fixed`, `Deprecated`, `Removed`, `Security`) +- **WebSocket States**: Socket uses `:disconnected` → `:connecting_http` → `:connecting_ws` → `:connected` flow +- **Sequence Tracking**: Use `Event.valid_seq?/2` to validate sequence numbers from firehose diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fcd551..fa1674c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Breaking Change + +- Existing behaviour moved to `Drinkup.Firehose` namespace, to make way for + alternate sync systems. + ## [0.1.0] - 2025-05-26 Initial release. diff --git a/examples/basic_consumer.ex b/examples/basic_consumer.ex index ef040cd..0f114bf 100644 --- a/examples/basic_consumer.ex +++ b/examples/basic_consumer.ex @@ -1,7 +1,7 @@ defmodule BasicConsumer do - @behaviour Drinkup.Consumer + @behaviour Drinkup.Firehose.Consumer - def handle_event(%Drinkup.Event.Commit{} = event) do + def handle_event(%Drinkup.Firehose.Event.Commit{} = event) do IO.inspect(event, label: "Got commit event") end @@ -18,7 +18,7 @@ defmodule ExampleSupervisor do @impl true def init(_) do children = [ - {Drinkup, %{consumer: BasicConsumer}} + {Drinkup.Firehose, %{consumer: BasicConsumer}} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/examples/multiple_consumers.ex b/examples/multiple_consumers.ex index d825d96..3fa41a3 100644 --- a/examples/multiple_consumers.ex +++ b/examples/multiple_consumers.ex @@ -1,5 +1,5 @@ defmodule PostDeleteConsumer do - use Drinkup.RecordConsumer, collections: ["app.bsky.feed.post"] + use Drinkup.Firehose.RecordConsumer, collections: ["app.bsky.feed.post"] def handle_delete(record) do IO.inspect(record, label: "update") @@ -7,9 +7,9 @@ defmodule PostDeleteConsumer do end defmodule IdentityConsumer do - @behaviour Drinkup.Consumer + @behaviour Drinkup.Firehose.Consumer - def handle_event(%Drinkup.Event.Identity{} = event) do + def handle_event(%Drinkup.Firehose.Event.Identity{} = event) do IO.inspect(event, label: "identity event") end @@ -26,8 +26,8 @@ defmodule ExampleSupervisor do @impl true def init(_) do children = [ - {Drinkup, %{consumer: PostDeleteConsumer}}, - {Drinkup, %{consumer: IdentityConsumer, name: :identities}} + {Drinkup.Firehose, %{consumer: PostDeleteConsumer}}, + {Drinkup.Firehose, %{consumer: IdentityConsumer, name: :identities}} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/examples/record_consumer.ex b/examples/record_consumer.ex index b5ff0b5..945b7d3 100644 --- a/examples/record_consumer.ex +++ b/examples/record_consumer.ex @@ -1,5 +1,6 @@ defmodule ExampleRecordConsumer do - use Drinkup.RecordConsumer, collections: [~r/app\.bsky\.graph\..+/, "app.bsky.feed.post"] + use Drinkup.Firehose.RecordConsumer, + collections: [~r/app\.bsky\.graph\..+/, "app.bsky.feed.post"] def handle_create(record) do IO.inspect(record, label: "create") @@ -24,7 +25,7 @@ defmodule ExampleSupervisor do @impl true def init(_) do children = [ - {Drinkup, %{consumer: ExampleRecordConsumer}} + {Drinkup.Firehose, %{consumer: ExampleRecordConsumer}} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/lib/drinkup.ex b/lib/firehose.ex similarity index 89% rename from lib/drinkup.ex rename to lib/firehose.ex index 866ea25..93b7dab 100644 --- a/lib/drinkup.ex +++ b/lib/firehose.ex @@ -1,13 +1,13 @@ -defmodule Drinkup do +defmodule Drinkup.Firehose do use Supervisor - alias Drinkup.Options + alias Drinkup.Firehose.Options @dialyzer nowarn_function: {:init, 1} @impl true def init({%Options{name: name} = drinkup_options, supervisor_options}) do children = [ {Task.Supervisor, name: {:via, Registry, {Drinkup.Registry, {name, Tasks}}}}, - {Drinkup.Socket, drinkup_options} + {Drinkup.Firehose.Socket, drinkup_options} ] Supervisor.start_link( diff --git a/lib/consumer.ex b/lib/firehose/consumer.ex similarity index 69% rename from lib/consumer.ex rename to lib/firehose/consumer.ex index 1e8d254..68203bc 100644 --- a/lib/consumer.ex +++ b/lib/firehose/consumer.ex @@ -1,9 +1,9 @@ -defmodule Drinkup.Consumer do +defmodule Drinkup.Firehose.Consumer do @moduledoc """ An unopinionated consumer of the Firehose. Will receive all events, not just commits. """ - alias Drinkup.Event + alias Drinkup.Firehose.Event @callback handle_event(Event.t()) :: any() end diff --git a/lib/event.ex b/lib/firehose/event.ex similarity index 94% rename from lib/event.ex rename to lib/firehose/event.ex index d0ed051..eabc300 100644 --- a/lib/event.ex +++ b/lib/firehose/event.ex @@ -1,6 +1,6 @@ -defmodule Drinkup.Event do +defmodule Drinkup.Firehose.Event do require Logger - alias Drinkup.{Event, Options} + alias Drinkup.Firehose.{Event, Options} @type t() :: Event.Commit.t() diff --git a/lib/event/account.ex b/lib/firehose/event/account.ex similarity index 96% rename from lib/event/account.ex rename to lib/firehose/event/account.ex index 9e6d39f..306c792 100644 --- a/lib/event/account.ex +++ b/lib/firehose/event/account.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Event.Account do +defmodule Drinkup.Firehose.Event.Account do @moduledoc """ Struct for account events from the ATProto Firehose. """ diff --git a/lib/event/commit.ex b/lib/firehose/event/commit.ex similarity index 98% rename from lib/event/commit.ex rename to lib/firehose/event/commit.ex index d678b5d..81550aa 100644 --- a/lib/event/commit.ex +++ b/lib/firehose/event/commit.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Event.Commit do +defmodule Drinkup.Firehose.Event.Commit do @moduledoc """ Struct for commit events from the ATProto Firehose. """ diff --git a/lib/event/identity.ex b/lib/firehose/event/identity.ex similarity index 92% rename from lib/event/identity.ex rename to lib/firehose/event/identity.ex index 99e62cd..1592353 100644 --- a/lib/event/identity.ex +++ b/lib/firehose/event/identity.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Event.Identity do +defmodule Drinkup.Firehose.Event.Identity do @moduledoc """ Struct for identity events from the ATProto Firehose. """ diff --git a/lib/event/info.ex b/lib/firehose/event/info.ex similarity index 90% rename from lib/event/info.ex rename to lib/firehose/event/info.ex index 813462b..92fec2d 100644 --- a/lib/event/info.ex +++ b/lib/firehose/event/info.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Event.Info do +defmodule Drinkup.Firehose.Event.Info do @moduledoc """ Struct for info events from the ATProto Firehose. """ diff --git a/lib/event/sync.ex b/lib/firehose/event/sync.ex similarity index 93% rename from lib/event/sync.ex rename to lib/firehose/event/sync.ex index 0825324..18a81f6 100644 --- a/lib/event/sync.ex +++ b/lib/firehose/event/sync.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Event.Sync do +defmodule Drinkup.Firehose.Event.Sync do @moduledoc """ Struct for sync events from the ATProto Firehose. """ diff --git a/lib/options.ex b/lib/firehose/options.ex similarity index 93% rename from lib/options.ex rename to lib/firehose/options.ex index 5a4441a..53e4bae 100644 --- a/lib/options.ex +++ b/lib/firehose/options.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.Options do +defmodule Drinkup.Firehose.Options do use TypedStruct @default_host "https://bsky.network" diff --git a/lib/record_consumer.ex b/lib/firehose/record_consumer.ex similarity index 85% rename from lib/record_consumer.ex rename to lib/firehose/record_consumer.ex index dff42b4..aaf917c 100644 --- a/lib/record_consumer.ex +++ b/lib/firehose/record_consumer.ex @@ -1,4 +1,4 @@ -defmodule Drinkup.RecordConsumer do +defmodule Drinkup.Firehose.RecordConsumer do @moduledoc """ An opinionated consumer of the Firehose that eats consumers """ @@ -11,15 +11,15 @@ defmodule Drinkup.RecordConsumer do {collections, _opts} = Keyword.pop(opts, :collections, []) quote location: :keep do - @behaviour Drinkup.Consumer - @behaviour Drinkup.RecordConsumer + @behaviour Drinkup.Firehose.Consumer + @behaviour Drinkup.Firehose.RecordConsumer - def handle_event(%Drinkup.Event.Commit{} = event) do + def handle_event(%Drinkup.Firehose.Event.Commit{} = event) do event.ops |> Enum.filter(fn %{path: path} -> path |> String.split("/") |> Enum.at(0) |> matches_collections?() end) - |> Enum.map(&Drinkup.RecordConsumer.Record.from(&1, event.repo)) + |> Enum.map(&Drinkup.Firehose.RecordConsumer.Record.from(&1, event.repo)) |> Enum.each(&apply(__MODULE__, :"handle_#{&1.action}", [&1])) end @@ -56,7 +56,7 @@ defmodule Drinkup.RecordConsumer do end defmodule Record do - alias Drinkup.Event.Commit.RepoOp + alias Drinkup.Firehose.Event.Commit.RepoOp use TypedStruct typedstruct do diff --git a/lib/socket.ex b/lib/firehose/socket.ex similarity index 98% rename from lib/socket.ex rename to lib/firehose/socket.ex index 74600a3..61fed13 100644 --- a/lib/socket.ex +++ b/lib/firehose/socket.ex @@ -1,10 +1,10 @@ -defmodule Drinkup.Socket do +defmodule Drinkup.Firehose.Socket do @moduledoc """ gen_statem process for managing the websocket connection to an ATProto relay. """ require Logger - alias Drinkup.{Event, Options} + alias Drinkup.Firehose.{Event, Options} @behaviour :gen_statem @timeout :timer.seconds(5) -- 2.51.2