diff --git a/server/src/at_record_server/catalog_index.gleam b/server/src/at_record_server/catalog_index.gleam index 9e0a3cc..e6796c6 100644 --- a/server/src/at_record_server/catalog_index.gleam +++ b/server/src/at_record_server/catalog_index.gleam @@ -1,19 +1,69 @@ -//// Prototype in-memory catalog index: an actor over a Dict of `BrowseRow` -//// keyed by uri, seeded by a boot backfill and kept live by `jetstream_consumer`. -//// Lost on restart by design (rebuildable, and small enough to hold in memory). +//// The catalog index port: durable public stats over releases, adoptions +//// (shelf.entry events that carry a release ref), and edits, plus the +//// Jetstream replay cursor. Backends live in this module (in-memory, dev +//// fallback, actor over a Dict) and in catalog_index_postgres (durable); +//// wiring picks one at the composition root based on DATABASE_URL. Kept +//// additive: `upsert`/`delete`/`list` signatures never change so existing +//// callers compile untouched. import at_record_server/browse.{type BrowseRow} import at_record_server/parallel import gleam/dict.{type Dict} import gleam/erlang/process.{type Subject} +import gleam/list +import gleam/option.{type Option} import gleam/otp/actor import gleam/result +/// One shelf.entry event that carries a release ref: an edge from a did's +/// crate entry to the release it adopted. Keyed by the entry's own at-uri, so +/// a later status change on the same entry (own -> want -> gone) overwrites +/// rather than accumulates. +pub type Adoption { + Adoption( + entry_uri: String, + did: String, + release_uri: String, + status: String, + created_at: String, + ) +} + +/// One catalog.edit event: a proposed change to some catalog entity, keyed by +/// the edit record's own at-uri. +pub type Edit { + Edit( + edit_uri: String, + did: String, + subject_uri: String, + entity: String, + created_at: String, + ) +} + pub type Store { Store( upsert: fn(BrowseRow) -> Nil, delete: fn(String) -> Nil, list: fn() -> List(BrowseRow), + upsert_adoption: fn(Adoption) -> Nil, + delete_adoption: fn(String) -> Nil, + upsert_edit: fn(Edit) -> Nil, + delete_edit: fn(String) -> Nil, + adoption_count: fn(String) -> Int, + adoption_counts: fn() -> Dict(String, Int), + edits_for: fn(String) -> List(Edit), + save_cursor: fn(Int) -> Nil, + load_cursor: fn() -> Option(Int), + ) +} + +type State { + State( + releases: Dict(String, BrowseRow), + adoptions: Dict(String, Adoption), + edits: Dict(String, Edit), + cursor: Option(Int), ) } @@ -21,11 +71,29 @@ type Msg { Upsert(BrowseRow) Delete(String) List(Subject(List(BrowseRow))) + UpsertAdoption(Adoption) + DeleteAdoption(String) + UpsertEdit(Edit) + DeleteEdit(String) + AdoptionCount(String, Subject(Int)) + AdoptionCounts(Subject(Dict(String, Int))) + EditsFor(String, Subject(List(Edit))) + SaveCursor(Int) + LoadCursor(Subject(Option(Int))) +} + +fn initial_state() -> State { + State( + releases: dict.new(), + adoptions: dict.new(), + edits: dict.new(), + cursor: option.None, + ) } pub fn start() -> Result(Store, actor.StartError) { use started <- result.map( - actor.new(dict.new()) |> actor.on_message(handle) |> actor.start, + actor.new(initial_state()) |> actor.on_message(handle) |> actor.start, ) let subject = started.data Store( @@ -43,19 +111,121 @@ pub fn start() -> Result(Store, actor.StartError) { Error(Nil) -> [] } }, + upsert_adoption: fn(adoption) { + process.send(subject, UpsertAdoption(adoption)) + Nil + }, + delete_adoption: fn(entry_uri) { + process.send(subject, DeleteAdoption(entry_uri)) + Nil + }, + upsert_edit: fn(edit) { + process.send(subject, UpsertEdit(edit)) + Nil + }, + delete_edit: fn(edit_uri) { + process.send(subject, DeleteEdit(edit_uri)) + Nil + }, + adoption_count: fn(release_uri) { + case parallel.try_call(subject, 1000, AdoptionCount(release_uri, _)) { + Ok(count) -> count + Error(Nil) -> 0 + } + }, + adoption_counts: fn() { + case parallel.try_call(subject, 1000, AdoptionCounts) { + Ok(counts) -> counts + Error(Nil) -> dict.new() + } + }, + edits_for: fn(subject_uri) { + case parallel.try_call(subject, 1000, EditsFor(subject_uri, _)) { + Ok(edits) -> edits + Error(Nil) -> [] + } + }, + save_cursor: fn(time_us) { + process.send(subject, SaveCursor(time_us)) + Nil + }, + load_cursor: fn() { + case parallel.try_call(subject, 1000, LoadCursor) { + Ok(cursor) -> cursor + Error(Nil) -> option.None + } + }, ) } -fn handle( - state: Dict(String, BrowseRow), - msg: Msg, -) -> actor.Next(Dict(String, BrowseRow), Msg) { +fn handle(state: State, msg: Msg) -> actor.Next(State, Msg) { case msg { - Upsert(row) -> actor.continue(dict.insert(state, row.uri, row)) - Delete(uri) -> actor.continue(dict.delete(state, uri)) + Upsert(row) -> + actor.continue( + State(..state, releases: dict.insert(state.releases, row.uri, row)), + ) + Delete(uri) -> + actor.continue(State(..state, releases: dict.delete(state.releases, uri))) List(reply) -> { - process.send(reply, dict.values(state)) + process.send(reply, dict.values(state.releases)) + actor.continue(state) + } + UpsertAdoption(adoption) -> + actor.continue( + State( + ..state, + adoptions: dict.insert(state.adoptions, adoption.entry_uri, adoption), + ), + ) + DeleteAdoption(entry_uri) -> + actor.continue( + State(..state, adoptions: dict.delete(state.adoptions, entry_uri)), + ) + UpsertEdit(edit) -> + actor.continue( + State(..state, edits: dict.insert(state.edits, edit.edit_uri, edit)), + ) + DeleteEdit(edit_uri) -> + actor.continue(State(..state, edits: dict.delete(state.edits, edit_uri))) + AdoptionCount(release_uri, reply) -> { + process.send(reply, count_for(state, release_uri)) + actor.continue(state) + } + AdoptionCounts(reply) -> { + process.send(reply, all_counts(state)) + actor.continue(state) + } + EditsFor(subject_uri, reply) -> { + process.send( + reply, + dict.values(state.edits) + |> list.filter(fn(e) { e.subject_uri == subject_uri }), + ) + actor.continue(state) + } + SaveCursor(time_us) -> + actor.continue(State(..state, cursor: option.Some(time_us))) + LoadCursor(reply) -> { + process.send(reply, state.cursor) actor.continue(state) } } } + +fn count_for(state: State, release_uri: String) -> Int { + dict.values(state.adoptions) + |> list.filter(fn(a) { a.release_uri == release_uri }) + |> list.length +} + +fn all_counts(state: State) -> Dict(String, Int) { + dict.values(state.adoptions) + |> list.fold(dict.new(), fn(acc, a) { + dict.upsert(acc, a.release_uri, fn(existing) { + case existing { + option.Some(n) -> n + 1 + option.None -> 1 + } + }) + }) +} diff --git a/server/test/catalog_index_test.gleam b/server/test/catalog_index_test.gleam new file mode 100644 index 0000000..1cc86cf --- /dev/null +++ b/server/test/catalog_index_test.gleam @@ -0,0 +1,129 @@ +import at_record_server/browse.{BrowseRow} +import at_record_server/catalog_index.{Adoption, Edit} +import gleam/dict +import gleam/list +import gleam/option.{None, Some} + +fn row(uri: String) -> browse.BrowseRow { + BrowseRow( + uri:, + cid: "bafyrow", + title: "Title", + artist_display: None, + genres: [], + styles: [], + released: None, + country: None, + cover: None, + thumb_url: None, + discogs_id: None, + created_at: "2024-01-01T00:00:00Z", + publisher_did: "did:plc:pub", + publisher_handle: "pub.test", + publisher_pds: "https://pds.test", + supersedes: None, + based_on: None, + ) +} + +pub fn upsert_and_list_test() { + let assert Ok(store) = catalog_index.start() + store.upsert(row("at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1")) + store.upsert(row("at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r2")) + assert list.length(store.list()) == 2 +} + +pub fn delete_removes_row_test() { + let assert Ok(store) = catalog_index.start() + let uri = "at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1" + store.upsert(row(uri)) + store.delete(uri) + assert store.list() == [] +} + +pub fn upsert_overwrites_by_uri_test() { + let assert Ok(store) = catalog_index.start() + let uri = "at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1" + store.upsert(row(uri)) + store.upsert(BrowseRow(..row(uri), title: "Retitled")) + let assert [only] = store.list() + assert only.title == "Retitled" +} + +pub fn adoption_upsert_delete_and_count_test() { + let assert Ok(store) = catalog_index.start() + let release = "at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1" + store.upsert_adoption(Adoption( + entry_uri: "at://did:plc:a/dev.mokkenstorm.crate.shelf.entry/e1", + did: "did:plc:a", + release_uri: release, + status: "acquired", + created_at: "2024-01-01T00:00:00Z", + )) + store.upsert_adoption(Adoption( + entry_uri: "at://did:plc:b/dev.mokkenstorm.crate.shelf.entry/e2", + did: "did:plc:b", + release_uri: release, + status: "wanted", + created_at: "2024-01-02T00:00:00Z", + )) + assert store.adoption_count(release) == 2 + assert store.adoption_counts() == dict.from_list([#(release, 2)]) + store.delete_adoption("at://did:plc:a/dev.mokkenstorm.crate.shelf.entry/e1") + assert store.adoption_count(release) == 1 +} + +pub fn adoption_upsert_by_entry_uri_overwrites_test() { + let assert Ok(store) = catalog_index.start() + let entry_uri = "at://did:plc:a/dev.mokkenstorm.crate.shelf.entry/e1" + let release = "at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1" + store.upsert_adoption(Adoption( + entry_uri:, + did: "did:plc:a", + release_uri: release, + status: "acquired", + created_at: "2024-01-01T00:00:00Z", + )) + store.upsert_adoption(Adoption( + entry_uri:, + did: "did:plc:a", + release_uri: release, + status: "sold", + created_at: "2024-02-01T00:00:00Z", + )) + assert store.adoption_count(release) == 1 +} + +pub fn edit_upsert_delete_and_edits_for_test() { + let assert Ok(store) = catalog_index.start() + let subject = "at://did:plc:pub/dev.mokkenstorm.crate.catalog.release/r1" + let edit_uri = "at://did:plc:a/dev.mokkenstorm.crate.catalog.edit/x1" + store.upsert_edit(Edit( + edit_uri:, + did: "did:plc:a", + subject_uri: subject, + entity: "release", + created_at: "2024-01-01T00:00:00Z", + )) + assert store.edits_for(subject) + == [ + Edit( + edit_uri:, + did: "did:plc:a", + subject_uri: subject, + entity: "release", + created_at: "2024-01-01T00:00:00Z", + ), + ] + store.delete_edit(edit_uri) + assert store.edits_for(subject) == [] +} + +pub fn cursor_save_and_load_test() { + let assert Ok(store) = catalog_index.start() + assert store.load_cursor() == None + store.save_cursor(1_700_000_000_000_000) + assert store.load_cursor() == Some(1_700_000_000_000_000) + store.save_cursor(1_700_000_001_000_000) + assert store.load_cursor() == Some(1_700_000_001_000_000) +} diff --git a/server/test/support.gleam b/server/test/support.gleam index c4fe77d..d5b544b 100644 --- a/server/test/support.gleam +++ b/server/test/support.gleam @@ -17,6 +17,7 @@ import at_record_server/oauth/sessions import at_record_server/oauth/sessions_memory import at_record_server/oauth/store import atproto/xrpc +import gleam/dict import gleam/dynamic/decode import gleam/json import gleam/option.{None} @@ -42,9 +43,20 @@ fn known_users_of(users: List(known_users.KnownUser)) -> known_users.Store { } fn empty_catalog_index() -> catalog_index.Store { - catalog_index.Store(upsert: fn(_) { Nil }, delete: fn(_) { Nil }, list: fn() { - [] - }) + catalog_index.Store( + upsert: fn(_) { Nil }, + delete: fn(_) { Nil }, + list: fn() { [] }, + upsert_adoption: fn(_) { Nil }, + delete_adoption: fn(_) { Nil }, + upsert_edit: fn(_) { Nil }, + delete_edit: fn(_) { Nil }, + adoption_count: fn(_) { 0 }, + adoption_counts: fn() { dict.new() }, + edits_for: fn(_) { [] }, + save_cursor: fn(_) { Nil }, + load_cursor: fn() { None }, + ) } /// A `Config` wired to in-memory pending-flow/session stores, so tests never