From 41acff41da18d2d752ac0b3e79a8ce4c8777b31c Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Fri, 10 Jul 2026 00:45:57 +0200 Subject: [PATCH] feat(catalog): mint artist+genre entities from Discogs details, enrich releases (artistDisplay, country, genres, identifiers, creditedArtists), resolve release on record detail; drop retired shelf.item lexicon --- .../dev/mokkenstorm/crate/catalog/artist.json | 59 +++ .../dev/mokkenstorm/crate/catalog/genre.json | 38 ++ .../mokkenstorm/crate/catalog/release.json | 8 +- .../dev/mokkenstorm/crate/shelf/item.json | 56 --- .../at_record_server/catalog_entities.gleam | 340 ++++++++++++++++++ .../src/at_record_server/catalog_import.gleam | 169 +++++++-- server/src/at_record_server/covers.gleam | 13 +- server/src/at_record_server/crate.gleam | 15 + .../src/at_record_server/discogs_client.gleam | 119 +++++- .../src/at_record_server/handlers/amend.gleam | 12 +- .../src/at_record_server/handlers/shelf.gleam | 80 ++++- .../oauth/client_metadata.gleam | 2 + server/src/at_record_server/promotion.gleam | 258 +++++++++---- server/test/catalog_entities_test.gleam | 25 ++ server/test/crate_test.gleam | 60 ++++ server/test/promotion_test.gleam | 173 ++++++++- shared/test/at_record_shared_test.gleam | 116 ++++-- 17 files changed, 1335 insertions(+), 208 deletions(-) create mode 100644 lexicons/dev/mokkenstorm/crate/catalog/artist.json create mode 100644 lexicons/dev/mokkenstorm/crate/catalog/genre.json delete mode 100644 lexicons/dev/mokkenstorm/crate/shelf/item.json create mode 100644 server/src/at_record_server/catalog_entities.gleam create mode 100644 server/test/catalog_entities_test.gleam diff --git a/lexicons/dev/mokkenstorm/crate/catalog/artist.json b/lexicons/dev/mokkenstorm/crate/catalog/artist.json new file mode 100644 index 0000000..5d1da03 --- /dev/null +++ b/lexicons/dev/mokkenstorm/crate/catalog/artist.json @@ -0,0 +1,59 @@ +{ + "lexicon": 1, + "id": "dev.mokkenstorm.crate.catalog.artist", + "defs": { + "main": { + "type": "record", + "description": "Canonical artist record (proof of concept). Interop with other atproto music apps happens through externalIds (MusicBrainz ids as the neutral key), not through shared lexicons yet.", + "key": "tid", + "record": { + "type": "object", + "required": ["name", "createdAt"], + "properties": { + "name": { + "type": "string", + "maxGraphemes": 256, + "maxLength": 2560 + }, + "sortName": { + "type": "string", + "maxGraphemes": 256, + "maxLength": 2560, + "description": "Name for collation (e.g. 'Aphex Twin' sorts as-is; 'The Beatles' as 'Beatles, The')." + }, + "disambiguation": { + "type": "string", + "maxGraphemes": 256, + "maxLength": 2560, + "description": "Short clarifier when names collide, MusicBrainz-style." + }, + "externalIds": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.mokkenstorm.crate.defs#externalId" + } + }, + "thumbUrl": { + "type": "string", + "format": "uri" + }, + "supersedes": { + "type": "ref", + "ref": "dev.mokkenstorm.crate.defs#catalogRef", + "description": "Same-author replacement of an earlier revision; consumers should re-point." + }, + "basedOn": { + "type": "ref", + "ref": "dev.mokkenstorm.crate.defs#catalogRef", + "description": "The record this one was derived from or amends (an addendum by any author); backlinks surface alternates on the original." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/dev/mokkenstorm/crate/catalog/genre.json b/lexicons/dev/mokkenstorm/crate/catalog/genre.json new file mode 100644 index 0000000..320bcad --- /dev/null +++ b/lexicons/dev/mokkenstorm/crate/catalog/genre.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "dev.mokkenstorm.crate.catalog.genre", + "defs": { + "main": { + "type": "record", + "description": "Genre as a record (proof of concept). Genres on releases stay plain strings; this exists to explore what a shared vocabulary entity would look like. MusicBrainz's genre list is the natural externalIds target.", + "key": "tid", + "record": { + "type": "object", + "required": ["name", "createdAt"], + "properties": { + "name": { + "type": "string", + "maxGraphemes": 128, + "maxLength": 1280 + }, + "description": { + "type": "string", + "maxGraphemes": 512, + "maxLength": 5120 + }, + "externalIds": { + "type": "array", + "items": { + "type": "ref", + "ref": "dev.mokkenstorm.crate.defs#externalId" + } + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/lexicons/dev/mokkenstorm/crate/catalog/release.json b/lexicons/dev/mokkenstorm/crate/catalog/release.json index 27fb492..8edd0aa 100644 --- a/lexicons/dev/mokkenstorm/crate/catalog/release.json +++ b/lexicons/dev/mokkenstorm/crate/catalog/release.json @@ -15,6 +15,12 @@ "maxGraphemes": 256, "maxLength": 2560 }, + "artistDisplay": { + "type": "string", + "maxGraphemes": 256, + "maxLength": 2560, + "description": "Display-ready artist credit line, so a release renders without artist entities resolved." + }, "externalIds": { "type": "array", "items": { @@ -117,7 +123,7 @@ }, "credit": { "type": "object", - "required": ["artist", "snapshot"], + "required": ["artist"], "properties": { "artist": { "type": "ref", diff --git a/lexicons/dev/mokkenstorm/crate/shelf/item.json b/lexicons/dev/mokkenstorm/crate/shelf/item.json deleted file mode 100644 index 9fca1b5..0000000 --- a/lexicons/dev/mokkenstorm/crate/shelf/item.json +++ /dev/null @@ -1,56 +0,0 @@ -{ - "lexicon": 1, - "id": "dev.mokkenstorm.crate.shelf.item", - "defs": { - "main": { - "type": "record", - "description": "A user's relationship to a release: the pivot between users and records. Status distinguishes owned from wanted. One entry per release per user. v1: `release` is optional until the catalog authority and lazy promotion exist; an unpromoted entry carries only its `snapshot`.", - "key": "tid", - "record": { - "type": "object", - "required": ["snapshot", "status", "createdAt"], - "properties": { - "release": { - "type": "ref", - "ref": "dev.mokkenstorm.crate.defs#catalogRef" - }, - "snapshot": { - "type": "ref", - "ref": "dev.mokkenstorm.crate.defs#snapshot" - }, - "status": { "type": "string", "knownValues": ["owned", "wanted"] }, - "mediaGrade": { - "type": "string", - "knownValues": ["M", "NM", "VG+", "VG", "G+", "G", "F", "P"] - }, - "sleeveGrade": { - "type": "string", - "knownValues": ["M", "NM", "VG+", "VG", "G+", "G", "F", "P"] - }, - "folder": { - "type": "string", - "maxGraphemes": 128, - "maxLength": 1280 - }, - "rating": { "type": "integer", "minimum": 1, "maximum": 5 }, - "notes": { - "type": "string", - "maxGraphemes": 1000, - "maxLength": 10000 - }, - "acquiredAt": { "type": "string", "format": "datetime" }, - "source": { - "type": "ref", - "ref": "dev.mokkenstorm.crate.defs#source" - }, - "createdAt": { "type": "string", "format": "datetime" }, - "updatedAt": { "type": "string", "format": "datetime" }, - "deleted": { - "type": "boolean", - "description": "Soft-delete tombstone: true withdraws the entry. Hard removal uses deleteRecord." - } - } - } - } - } -} diff --git a/server/src/at_record_server/catalog_entities.gleam b/server/src/at_record_server/catalog_entities.gleam new file mode 100644 index 0000000..12a1c75 --- /dev/null +++ b/server/src/at_record_server/catalog_entities.gleam @@ -0,0 +1,340 @@ +//// Best-effort minting of the PoC `catalog.artist` and `catalog.genre` +//// records that a release references or seeds vocabulary from. Own-repo dedup +//// only: artists key on their discogs id, genres on their lowercased name. +//// Adopting other users' entities via Constellation is out of scope. A +//// non-429 write failure is logged and counted so the import surfaces it; a +//// 429 stops the run so the item retries, and a failed mint never enters the +//// dedup dict, so the next reference to the same entity retries the mint. + +import at_record/gen/catalog/artist as catalog_artist +import at_record/gen/catalog/genre as catalog_genre +import at_record/gen/defs +import at_record_server/discogs_client +import at_record_server/oauth/sessions.{type OauthSession} +import at_record_server/provenance +import atproto/repo +import atproto/xrpc.{type Client} +import gleam/dict.{type Dict} +import gleam/dynamic/decode +import gleam/int +import gleam/json.{type Json} +import gleam/list +import gleam/option.{None, Some} +import gleam/set.{type Set} +import gleam/string +import wisp + +/// Canonical entity refs already in our own repo: artists keyed by discogs id, +/// genres by lowercased name. Loaded once per run so repeats reuse. +pub type Entities { + Entities( + artists: Dict(String, defs.CatalogRef), + genres: Dict(String, defs.CatalogRef), + ) +} + +/// The write outcome of an ensure pass: how many non-429 writes failed (to be +/// surfaced in the import note) and whether a 429 asks the run to stop. +pub type MintReport { + MintReport(failed: Int, rate_limited: Bool) +} + +pub const empty_report = MintReport(failed: 0, rate_limited: False) + +pub fn combine(a: MintReport, b: MintReport) -> MintReport { + MintReport( + failed: a.failed + b.failed, + rate_limited: a.rate_limited || b.rate_limited, + ) +} + +type MintError { + RateLimited + Failed +} + +pub fn load(client: Client, session: OauthSession) -> Entities { + Entities( + artists: own_artists(client, session), + genres: own_genres(client, session), + ) +} + +/// Own artist records keyed by discogs external id. +pub fn own_artists( + client: Client, + session: OauthSession, +) -> Dict(String, defs.CatalogRef) { + case + repo.list_records( + client, + session.pds, + session.access_token, + session.did, + catalog_artist.collection, + artist_row_decoder(), + ) + { + Error(_) -> dict.new() + Ok(rows) -> + rows + |> list.flat_map(fn(row) { + let #(uri, cid, ids) = row + ids + |> list.filter(fn(x) { x.provider == "discogs" }) + |> list.map(fn(x) { + #(x.id, defs.CatalogRef(uri:, cid:, external_ids: None)) + }) + }) + |> dict.from_list + } +} + +fn artist_row_decoder() -> decode.Decoder( + #(String, String, List(defs.ExternalId)), +) { + use uri <- decode.field("uri", decode.string) + use cid <- decode.field("cid", decode.string) + use ids <- decode.field("value", { + use ids <- decode.optional_field( + "externalIds", + [], + decode.list(defs.external_id_decoder()), + ) + decode.success(ids) + }) + decode.success(#(uri, cid, ids)) +} + +/// Own genre records keyed by lowercased name. +pub fn own_genres( + client: Client, + session: OauthSession, +) -> Dict(String, defs.CatalogRef) { + case + repo.list_records( + client, + session.pds, + session.access_token, + session.did, + catalog_genre.collection, + genre_row_decoder(), + ) + { + Error(_) -> dict.new() + Ok(rows) -> + rows + |> list.map(fn(row) { + let #(uri, cid, name) = row + #( + string.lowercase(name), + defs.CatalogRef(uri:, cid:, external_ids: None), + ) + }) + |> dict.from_list + } +} + +fn genre_row_decoder() -> decode.Decoder(#(String, String, String)) { + use uri <- decode.field("uri", decode.string) + use cid <- decode.field("cid", decode.string) + use name <- decode.field("value", decode.at(["name"], decode.string)) + decode.success(#(uri, cid, name)) +} + +/// The distinct keys not already `known`, first occurrence preserved. Pure, so +/// the dedup rule is testable without minting (mirrors discogs_client.plan_import). +pub fn distinct_missing( + known: Set(String), + keys: List(String), +) -> List(String) { + keys + |> list.fold(#(known, []), fn(acc, key) { + let #(seen, out) = acc + case set.contains(seen, key) { + True -> acc + False -> #(set.insert(seen, key), [key, ..out]) + } + }) + |> fn(acc) { acc.1 |> list.reverse } +} + +fn index_first(items: List(a), key: fn(a) -> String) -> Dict(String, a) { + list.fold(items, dict.new(), fn(d, item) { + let k = key(item) + case dict.has_key(d, k) { + True -> d + False -> dict.insert(d, k, item) + } + }) +} + +/// Reuse or mint a `catalog.artist` per discogs artist, growing the dict. +/// Returns the credited refs in original order (skipping any that failed). +pub fn ensure_artists( + client: Client, + session: OauthSession, + own: Dict(String, defs.CatalogRef), + artists: List(discogs_client.DetailArtist), +) -> #(Dict(String, defs.CatalogRef), List(defs.CatalogRef), MintReport) { + case artists { + [] -> #(own, [], empty_report) + _ -> { + let keys = artists |> list.map(fn(a) { int.to_string(a.id) }) + let by_key = index_first(artists, fn(a) { int.to_string(a.id) }) + let distinct = distinct_missing(set.new(), keys) + let new_keys = distinct_missing(dict.keys(own) |> set.from_list, keys) + let #(own, report) = + mint_missing(own, new_keys, fn(key) { + case dict.get(by_key, key) { + Ok(artist) -> mint_artist(client, session, artist) + Error(Nil) -> Error(Failed) + } + }) + let refs = distinct |> list.filter_map(fn(k) { dict.get(own, k) }) + #(own, refs, report) + } + } +} + +/// Reuse or mint a `catalog.genre` per name, deduped case-insensitively. The +/// refs are returned for symmetry; releases keep genres as plain strings. +pub fn ensure_genres( + client: Client, + session: OauthSession, + own: Dict(String, defs.CatalogRef), + names: List(String), +) -> #(Dict(String, defs.CatalogRef), List(defs.CatalogRef), MintReport) { + let named = names |> list.filter(fn(n) { string.trim(n) != "" }) + case named { + [] -> #(own, [], empty_report) + _ -> { + let key = fn(n) { string.lowercase(string.trim(n)) } + let keys = named |> list.map(key) + let by_key = index_first(named, key) + let distinct = distinct_missing(set.new(), keys) + let new_keys = distinct_missing(dict.keys(own) |> set.from_list, keys) + let #(own, report) = + mint_missing(own, new_keys, fn(k) { + case dict.get(by_key, k) { + Ok(name) -> mint_genre(client, session, name) + Error(Nil) -> Error(Failed) + } + }) + let refs = distinct |> list.filter_map(fn(k) { dict.get(own, k) }) + #(own, refs, report) + } + } +} + +// Mint each missing key in turn, stopping at a 429; a failed mint is counted +// and its key left absent so a later reference retries it. +fn mint_missing( + own: Dict(String, defs.CatalogRef), + new_keys: List(String), + mint: fn(String) -> Result(defs.CatalogRef, MintError), +) -> #(Dict(String, defs.CatalogRef), MintReport) { + list.fold(new_keys, #(own, empty_report), fn(acc, key) { + let #(own, report) = acc + case report.rate_limited { + True -> acc + False -> + case mint(key) { + Ok(ref) -> #(dict.insert(own, key, ref), report) + Error(RateLimited) -> #(own, MintReport(..report, rate_limited: True)) + Error(Failed) -> #( + own, + MintReport(..report, failed: report.failed + 1), + ) + } + } + }) +} + +fn mint_artist( + client: Client, + session: OauthSession, + artist: discogs_client.DetailArtist, +) -> Result(defs.CatalogRef, MintError) { + let id = int.to_string(artist.id) + let record = + catalog_artist.CatalogArtist( + name: artist.name, + created_at: provenance.now_rfc3339(), + external_ids: Some([ + defs.ExternalId( + provider: "discogs", + id:, + url: Some("https://www.discogs.com/artist/" <> id), + ), + ]), + based_on: None, + disambiguation: None, + sort_name: None, + supersedes: None, + thumb_url: None, + ) + create( + client, + session, + catalog_artist.collection, + catalog_artist.encode_catalog_artist(record), + "artist mint failed for discogs:" <> id, + ) +} + +fn mint_genre( + client: Client, + session: OauthSession, + name: String, +) -> Result(defs.CatalogRef, MintError) { + let record = + catalog_genre.CatalogGenre( + name:, + created_at: provenance.now_rfc3339(), + description: None, + external_ids: None, + ) + create( + client, + session, + catalog_genre.collection, + catalog_genre.encode_catalog_genre(record), + "genre mint failed for " <> name, + ) +} + +fn create( + client: Client, + session: OauthSession, + collection: String, + record: Json, + context: String, +) -> Result(defs.CatalogRef, MintError) { + case + repo.create_record( + client, + session.pds, + session.access_token, + session.did, + collection, + record, + ) + { + Ok(created) -> + Ok(defs.CatalogRef(uri: created.uri, cid: created.cid, external_ids: None)) + Error(xrpc.BadStatus(status: 429, ..)) -> Error(RateLimited) + Error(e) -> { + wisp.log_warning(context <> ": " <> xrpc_error(e)) + Error(Failed) + } + } +} + +fn xrpc_error(e: xrpc.XrpcError) -> String { + case e { + xrpc.RequestFailed(m) -> m + xrpc.BadStatus(status:, ..) -> "status " <> int.to_string(status) + xrpc.DecodeFailed(m) -> m + } +} diff --git a/server/src/at_record_server/catalog_import.gleam b/server/src/at_record_server/catalog_import.gleam index e3de5d5..6c212a1 100644 --- a/server/src/at_record_server/catalog_import.gleam +++ b/server/src/at_record_server/catalog_import.gleam @@ -7,6 +7,7 @@ import at_record/gen/defs.{Snapshot} import at_record/gen/repo/strong_ref.{RepoStrongRef} import at_record/gen/shelf/entry.{type ShelfEntry, ShelfEntry} import at_record/storage +import at_record_server/catalog_entities import at_record_server/context.{type Context} import at_record_server/covers import at_record_server/crate @@ -45,8 +46,10 @@ type RunState { seen: Set(String), covered: Set(String), own: dict.Dict(String, promotion.OwnRelease), + entities: catalog_entities.Entities, genesis_by_id: dict.Dict(String, storage.StoredItem(ShelfEntry)), budget: Int, + entity_failures: Int, ) } @@ -75,7 +78,12 @@ pub fn run( session, auth, creds, - initial_state(stored, promotion.own_releases(client, session), cap), + initial_state( + stored, + promotion.own_releases(client, session), + catalog_entities.load(client, session), + cap, + ), 1, ImportRun(imported: 0, updated: 0, skipped: 0, done: True, note: None), ) @@ -97,6 +105,7 @@ pub fn seen_discogs_ids( fn initial_state( stored: List(storage.StoredItem(ShelfEntry)), own: dict.Dict(String, promotion.OwnRelease), + entities: catalog_entities.Entities, cap: Int, ) -> RunState { let seen = seen_discogs_ids(stored) @@ -128,7 +137,15 @@ fn initial_state( }) |> dict.keys |> set.from_list - RunState(seen:, covered:, own:, genesis_by_id:, budget: cap) + RunState( + seen:, + covered:, + own:, + entities:, + genesis_by_id:, + budget: cap, + entity_failures: 0, + ) } fn import_pages( @@ -171,8 +188,15 @@ fn import_pages( ) let planned = list.length(plan.new) + list.length(plan.backfill) let unseen_left = list.length(found.releases) > planned + plan.skipped - let #(written, own, stop_note) = - write_releases(ctx, client, session, state.own, plan.new) + let #(written, own, entities, entity_failed, stop_note) = + write_releases( + ctx, + client, + session, + state.own, + state.entities, + plan.new, + ) let #(updated, cover_missing, backfill_note) = case stop_note { Some(_) -> #(0, 0, None) None -> @@ -194,11 +218,15 @@ fn import_pages( let budget = state.budget - written - updated let last_page = found.page >= found.pages let stop = option.or(stop_note, backfill_note) + let failures = state.entity_failures + entity_failed case stop, last_page, unseen_left || budget <= 0 { Some(note), _, _ -> ImportRun(..acc, done: False, note: Some(note)) - None, True, False -> ImportRun(..acc, done: True) - None, True, True -> ImportRun(..acc, done: False) - None, False, True -> ImportRun(..acc, done: False) + None, True, False -> + ImportRun(..acc, done: True) |> note_entity_failures(failures) + None, True, True -> + ImportRun(..acc, done: False) |> note_entity_failures(failures) + None, False, True -> + ImportRun(..acc, done: False) |> note_entity_failures(failures) None, False, False -> import_pages( ctx, @@ -211,7 +239,9 @@ fn import_pages( seen: plan.seen, covered: plan.covered, own:, + entities:, budget:, + entity_failures: failures, ), page + 1, acc, @@ -238,7 +268,7 @@ fn write_backfills( |> result.replace_error("no genesis"), ) use cover <- result.try( - best_cover(ctx, client, session, release) + best_cover(ctx, client, session, release, fetch_details(ctx, release)) |> option.to_result("no cover"), ) ShelfEntry( @@ -297,29 +327,62 @@ fn write_backfills( } } +// Surfaces counted entity-write failures via the note when nothing more urgent +// (a stop/rate-limit note) already claims it. +fn note_entity_failures(run: ImportRun, failures: Int) -> ImportRun { + case run.note, failures > 0 { + None, True -> + ImportRun( + ..run, + note: Some( + int.to_string(failures) + <> " artist/genre records failed; run again to fill them", + ), + ) + _, _ -> run + } +} + // Stop at the first failure so a rate limit becomes a partial, resumable run. fn write_releases( ctx: Context, client: Client, session: OauthSession, own: dict.Dict(String, promotion.OwnRelease), + entities: catalog_entities.Entities, releases: List(discogs_client.Release), -) -> #(Int, dict.Dict(String, promotion.OwnRelease), Option(String)) { +) -> #( + Int, + dict.Dict(String, promotion.OwnRelease), + catalog_entities.Entities, + Int, + Option(String), +) { case releases { - [] -> #(0, own, None) + [] -> #(0, own, entities, 0, None) [release, ..rest] -> { - let #(own, written) = write_genesis(ctx, client, session, own, release) + let #(own, entities, entity_failed, written) = + write_genesis(ctx, client, session, own, entities, release) case written { Ok(_) -> { - let #(n, own, note) = write_releases(ctx, client, session, own, rest) - #(n + 1, own, note) + let #(n, own, entities, ef, note) = + write_releases(ctx, client, session, own, entities, rest) + #(n + 1, own, entities, entity_failed + ef, note) } Error(xrpc.BadStatus(status: 429, ..)) -> #( 0, own, + entities, + entity_failed, Some("PDS rate limited; run again later"), ) - Error(_) -> #(0, own, Some("PDS write failed; run again to resume")) + Error(_) -> #( + 0, + own, + entities, + entity_failed, + Some("PDS write failed; run again to resume"), + ) } } } @@ -330,14 +393,60 @@ fn write_genesis( client: Client, session: OauthSession, own: dict.Dict(String, promotion.OwnRelease), + entities: catalog_entities.Entities, release: discogs_client.Release, ) -> #( dict.Dict(String, promotion.OwnRelease), + catalog_entities.Entities, + Int, Result(repo.CreatedRecord, xrpc.XrpcError), ) { - let cover = best_cover(ctx, client, session, release) - let #(own, promoted) = - promotion.adopt_or_mint(ctx.catalog, client, session, own, release, cover) + let details = fetch_details(ctx, release) + let cover = best_cover(ctx, client, session, release, details) + let outcome = + promotion.adopt_or_mint( + ctx.catalog, + client, + session, + own, + entities, + release, + cover, + details, + ) + case outcome.report.rate_limited { + // Entity write hit a 429: abort before the shelf event so the item retries. + True -> #( + outcome.own, + outcome.entities, + outcome.report.failed, + Error(xrpc.BadStatus( + status: 429, + error: None, + message: None, + body: "entity write rate limited", + )), + ) + False -> + write_genesis_event(client, session, outcome, release, cover, details) + } +} + +fn write_genesis_event( + client: Client, + session: OauthSession, + outcome: promotion.Outcome, + release: discogs_client.Release, + cover: Option(blob.Blob), + _details: Option(discogs_client.ReleaseDetails), +) -> #( + dict.Dict(String, promotion.OwnRelease), + catalog_entities.Entities, + Int, + Result(repo.CreatedRecord, xrpc.XrpcError), +) { + let own = outcome.own + let promoted = outcome.promoted let base_source = provenance.source("discogs-import") |> provenance.with_external(discogs_client.external_id(release)) @@ -374,23 +483,41 @@ fn write_genesis( source: Some(source), created_at: provenance.now_rfc3339(), ) - #(own, event_log.append(client, session, event)) + #( + own, + outcome.entities, + outcome.report.failed, + event_log.append(client, session, event), + ) +} + +// The widened per-release fetch, shared by the cover and the mint enrichment. +fn fetch_details( + ctx: Context, + release: discogs_client.Release, +) -> Option(discogs_client.ReleaseDetails) { + discogs_client.release_details( + ctx.discogs.send, + ctx.discogs.auth, + release.discogs_id, + ) + |> option.from_result } -// Full-res primary image first (one extra Discogs call), then cover, thumb. +// Full-res primary image first (from the shared details), then cover, thumb. fn best_cover( ctx: Context, client: Client, session: OauthSession, release: discogs_client.Release, + details: Option(discogs_client.ReleaseDetails), ) -> Option(blob.Blob) { covers.best_discogs_cover( - ctx.discogs, ctx.atproto.client, client, session.pds, session.access_token, - release.discogs_id, + details, [release.cover_url, release.thumb_url], ) } diff --git a/server/src/at_record_server/covers.gleam b/server/src/at_record_server/covers.gleam index fde3743..c58a8a8 100644 --- a/server/src/at_record_server/covers.gleam +++ b/server/src/at_record_server/covers.gleam @@ -1,7 +1,6 @@ //// Fetch cover-art bytes and store them as a blob in the user's repo, so a //// record is self-contained instead of hotlinking a CDN that may rot. -import at_record_server/context import at_record_server/discogs_client import atproto/blob.{type Blob} import atproto/repo @@ -17,20 +16,18 @@ import wisp // Must match the lexicon's cover maxSize. const max_bytes = 2_000_000 -/// The full-resolution chain for a Discogs release: primary image (one extra -/// rate-limited call), then the caller's fallbacks (cover, thumb). +/// The full-resolution chain for a Discogs release: the already-fetched +/// primary image from `release_details`, then the caller's fallbacks (cover, +/// thumb). The details are threaded in so the mint path reuses one API call. pub fn best_discogs_cover( - discogs: context.Discogs, fetch_client: Client, pds_client: Client, pds: String, token: String, - discogs_id: Int, + details: Option(discogs_client.ReleaseDetails), fallbacks: List(Option(String)), ) -> Option(Blob) { - let full = - discogs_client.primary_image_url(discogs.send, discogs.auth, discogs_id) - |> result.unwrap(option.None) + let full = details |> option.then(fn(d) { d.primary_image_url }) fetch_and_upload_any(fetch_client, pds_client, pds, token, [full, ..fallbacks]) } diff --git a/server/src/at_record_server/crate.gleam b/server/src/at_record_server/crate.gleam index 6fbfc3d..15d5003 100644 --- a/server/src/at_record_server/crate.gleam +++ b/server/src/at_record_server/crate.gleam @@ -4,6 +4,7 @@ //// by rkey (TID) and reduced: status from the latest status-changing action, //// every other field last-write-wins. +import at_record/gen/catalog/release.{type CatalogRelease} import at_record/gen/defs.{type CatalogRef, type Price, type Snapshot} import at_record/gen/shelf/entry.{type ShelfEntry} import at_record/storage.{type StoredItem} @@ -165,3 +166,17 @@ fn opt( None -> [] } } + +/// Display-relevant fields off a resolved catalog release, for read paths +/// that only need to show them, not round-trip the whole record. +pub fn encode_release_info(release: CatalogRelease) -> Json { + json.object( + list.flatten([ + opt("artistDisplay", release.artist_display, json.string), + opt("genres", release.genres, fn(items) { json.array(items, json.string) }), + opt("styles", release.styles, fn(items) { json.array(items, json.string) }), + opt("country", release.country, json.string), + opt("released", release.released, json.string), + ]), + ) +} diff --git a/server/src/at_record_server/discogs_client.gleam b/server/src/at_record_server/discogs_client.gleam index f504ab4..12b077d 100644 --- a/server/src/at_record_server/discogs_client.gleam +++ b/server/src/at_record_server/discogs_client.gleam @@ -13,6 +13,7 @@ import gleam/option.{type Option, None, Some} import gleam/result import gleam/set.{type Set} import gleam/string +import wisp const host = "api.discogs.com" @@ -236,13 +237,34 @@ fn option_then(o: Option(a), f: fn(a) -> Option(b)) -> Option(b) { } } -/// The full-resolution primary image of a release, via the per-release -/// endpoint (one extra rate-limited call; callers fall back to cover_image). -pub fn primary_image_url( +/// One artist credited on a release, from the per-release endpoint's `artists` +/// array. anv/join/role are ignored for the PoC entity records. +pub type DetailArtist { + DetailArtist(id: Int, name: String) +} + +/// The widened per-release fetch: everything one `/releases/{id}` call yields +/// that the mint path needs, so the cover fetch and enrichment share one call. +pub type ReleaseDetails { + ReleaseDetails( + primary_image_url: Option(String), + artists: List(DetailArtist), + genres: List(String), + styles: List(String), + country: Option(String), + released: Option(String), + barcodes: List(String), + ) +} + +/// The per-release endpoint, fetched once. Preserves the primary-image +/// selection (type "primary" first, else the first image with a uri) and +/// leniently decodes the enrichment fields; a missing optional never fails it. +pub fn release_details( send: Sender, auth: Option(Auth), discogs_id: Int, -) -> Result(Option(String), String) { +) -> Result(ReleaseDetails, String) { let base = request.new() |> request.set_method(http.Get) @@ -262,25 +284,108 @@ pub fn primary_image_url( case send(req) { Error(e) -> Error(e) Ok(#(200, body)) -> - json.parse(body, images_decoder()) + json.parse(body, release_details_decoder()) |> map_error(fn(_) { "could not parse discogs release" }) + |> result.map(warn_if_no_artists(discogs_id, _)) Ok(#(status, _)) -> Error("discogs returned status " <> int.to_string(status)) } } -fn images_decoder() -> decode.Decoder(Option(String)) { +// A release with no artists is unusual; log it as a possible upstream drift. +fn warn_if_no_artists( + discogs_id: Int, + details: ReleaseDetails, +) -> ReleaseDetails { + case details.artists { + [] -> { + wisp.log_warning( + "discogs release " <> int.to_string(discogs_id) <> " had no artists", + ) + details + } + _ -> details + } +} + +fn release_details_decoder() -> decode.Decoder(ReleaseDetails) { let image = { use type_ <- decode.optional_field("type", "", decode.string) use uri <- decode.optional_field("uri", "", decode.string) decode.success(#(type_, uri)) } + let artist = { + use id <- decode.optional_field("id", 0, decode.int) + use name <- decode.optional_field("name", "", decode.string) + decode.success(DetailArtist(id:, name:)) + } + let identifier = { + use type_ <- decode.optional_field("type", "", decode.string) + use value <- decode.optional_field("value", "", decode.string) + decode.success(#(type_, value)) + } use images <- decode.optional_field("images", [], decode.list(image)) + use artists <- decode.optional_field("artists", [], decode.list(artist)) + use genres <- decode.optional_field("genres", [], decode.list(decode.string)) + use styles <- decode.optional_field("styles", [], decode.list(decode.string)) + use country <- decode.optional_field( + "country", + None, + decode.optional(decode.string), + ) + use released <- decode.optional_field( + "released", + None, + decode.optional(decode.string), + ) + use year <- decode.optional_field("year", None, decode.optional(decode.int)) + use identifiers <- decode.optional_field( + "identifiers", + [], + decode.list(identifier), + ) let primary = images |> list.find(fn(i) { i.0 == "primary" && i.1 != "" }) |> result.lazy_or(fn() { list.find(images, fn(i) { i.1 != "" }) }) - decode.success(primary |> result.map(fn(i) { i.1 }) |> option.from_result) + |> result.map(fn(i) { i.1 }) + |> option.from_result + let barcodes = + identifiers + |> list.filter(fn(i) { i.0 == "Barcode" }) + |> list.map(fn(i) { string.trim(i.1) }) + |> list.filter(fn(v) { v != "" }) + decode.success(ReleaseDetails( + primary_image_url: primary, + artists: artists |> list.filter(fn(a) { a.name != "" }), + genres:, + styles:, + country: option_non_empty(country), + released: released_string(released, year), + barcodes:, + )) +} + +fn option_non_empty(o: Option(String)) -> Option(String) { + case o { + Some(s) -> non_empty(string.trim(s)) + None -> None + } +} + +// Prefer the `released` date string; fall back to the `year` int (0 = unknown). +fn released_string( + released: Option(String), + year: Option(Int), +) -> Option(String) { + case option_non_empty(released) { + Some(r) -> Some(r) + None -> + case year { + Some(0) | None -> None + Some(y) -> Some(int.to_string(y)) + } + } } /// One page of the connected user's collection (folder 0 = All). The caller diff --git a/server/src/at_record_server/handlers/amend.gleam b/server/src/at_record_server/handlers/amend.gleam index 6a5633f..16560eb 100644 --- a/server/src/at_record_server/handlers/amend.gleam +++ b/server/src/at_record_server/handlers/amend.gleam @@ -14,6 +14,7 @@ import at_record_server/context.{ } import at_record_server/covers import at_record_server/crate +import at_record_server/discogs_client import at_record_server/event_log import at_record_server/oauth/sessions.{type OauthSession} import at_record_server/promotion @@ -204,6 +205,7 @@ fn load_base( let snap = entry.snapshot catalog_release.CatalogRelease( title: snap |> option.map(fn(s) { s.title }) |> option.unwrap(""), + artist_display: snap |> option.map(fn(s) { s.artist_display }), created_at: provenance.now_rfc3339(), external_ids: None, released: snap @@ -251,13 +253,19 @@ fn refetch_cover( |> option.from_result |> option.then(fn(x) { int.parse(x.id) |> option.from_result }), ) + let details = + discogs_client.release_details( + ctx.discogs.send, + ctx.discogs.auth, + discogs_id, + ) + |> option.from_result covers.best_discogs_cover( - ctx.discogs, ctx.atproto.client, client, session.pds, session.access_token, - discogs_id, + details, [old.thumb_url], ) } diff --git a/server/src/at_record_server/handlers/shelf.gleam b/server/src/at_record_server/handlers/shelf.gleam index 8021ee0..6363eae 100644 --- a/server/src/at_record_server/handlers/shelf.gleam +++ b/server/src/at_record_server/handlers/shelf.gleam @@ -2,11 +2,14 @@ //// folded crate and per-entry timeline, and writing immutable events (genesis //// add + later actions). Every mutation is a new event, never an edit. +import at_record/gen/catalog/release as catalog_release import at_record/gen/defs.{ExternalId, Snapshot} import at_record/gen/repo/strong_ref.{RepoStrongRef} import at_record/gen/shelf/entry.{ type ShelfEntry, ShelfEntry, encode_shelf_entry, } +import at_record/storage.{type StoredItem} +import at_record_server/catalog_entities import at_record_server/context.{ type Context, error_json, require_session, with_pds_client, } @@ -144,23 +147,46 @@ pub fn shelf_timeline( Ok(stored) -> case event_log.entry_events(stored, entry_id) { Error(Nil) -> error_json(404, "unknown entry") - Ok(events) -> - json.object([ - #("entryId", json.string(entry_id)), - #( - "events", - json.array( - list.map(events, fn(s) { s.value }), - encode_shelf_entry, - ), - ), - ]) + Ok(events) -> { + let release = resolve_release(ctx, events) + json.object( + list.flatten([ + [ + #("entryId", json.string(entry_id)), + #( + "events", + json.array( + list.map(events, fn(s) { s.value }), + encode_shelf_entry, + ), + ), + ], + case release { + Some(r) -> [#("release", crate.encode_release_info(r))] + None -> [] + }, + ]), + ) |> json.to_string |> wisp.json_response(200) + } } } } +// Best-effort read: no ref, or a fetch miss, just means the response omits it. +fn resolve_release( + ctx: Context, + events: List(StoredItem(ShelfEntry)), +) -> Option(catalog_release.CatalogRelease) { + crate.fold(events) + |> list.first + |> result.map(fn(e) { e.release }) + |> result.unwrap(None) + |> option.then(fn(ref) { ctx.catalog.fetch_release(ref.uri) }) + |> option.map(fn(pair) { pair.1 }) +} + pub fn purge_entry(req: Request, ctx: Context, entry_id: String) -> Response { use id, session <- require_session(req, ctx) use client, session <- with_pds_client(ctx, id, session) @@ -289,15 +315,21 @@ fn do_add( ]) None -> None } - let cover = case form.discogs_id |> option.then(parse_id) { + // One /releases/{id} fetch, shared by the cover and the mint enrichment. + let details = case form.discogs_id |> option.then(parse_id) { Some(id) -> + discogs_client.release_details(ctx.discogs.send, ctx.discogs.auth, id) + |> option.from_result + None -> None + } + let cover = case form.discogs_id |> option.then(parse_id) { + Some(_) -> covers.best_discogs_cover( - ctx.discogs, ctx.atproto.client, client, session.pds, session.access_token, - id, + details, [form.cover_url, form.thumb_url], ) None -> @@ -309,7 +341,8 @@ fn do_add( [form.cover_url, form.thumb_url], ) } - let #(release, source) = promote_manual(ctx, client, session, form, cover) + let #(release, source) = + promote_manual(ctx, client, session, form, cover, details) ShelfEntry( subject: None, action:, @@ -343,6 +376,7 @@ fn promote_manual( session: OauthSession, form: AddForm, cover: option.Option(blob.Blob), + details: option.Option(discogs_client.ReleaseDetails), ) -> #(option.Option(defs.CatalogRef), defs.Source) { let parsed = form.discogs_id |> option.then(parse_id) case parsed { @@ -360,9 +394,19 @@ fn promote_manual( instance_id: None, ) let own = promotion.own_releases(client, session) - let #(_own, promoted) = - promotion.adopt_or_mint(ctx.catalog, client, session, own, seed, cover) - case promoted { + let entities = catalog_entities.load(client, session) + let outcome = + promotion.adopt_or_mint( + ctx.catalog, + client, + session, + own, + entities, + seed, + cover, + details, + ) + case outcome.promoted { Some(p) -> #( Some(p.release), provenance.source("manual") diff --git a/server/src/at_record_server/oauth/client_metadata.gleam b/server/src/at_record_server/oauth/client_metadata.gleam index cc1a5c6..21c1eef 100644 --- a/server/src/at_record_server/oauth/client_metadata.gleam +++ b/server/src/at_record_server/oauth/client_metadata.gleam @@ -18,6 +18,8 @@ pub const scopes = [ "atproto", "repo:dev.mokkenstorm.crate.shelf.entry", "repo:dev.mokkenstorm.crate.catalog.release", + "repo:dev.mokkenstorm.crate.catalog.artist", + "repo:dev.mokkenstorm.crate.catalog.genre", "repo:dev.mokkenstorm.crate.catalog.edit", "blob:image/*", ] diff --git a/server/src/at_record_server/promotion.gleam b/server/src/at_record_server/promotion.gleam index 766f000..6f1bd54 100644 --- a/server/src/at_record_server/promotion.gleam +++ b/server/src/at_record_server/promotion.gleam @@ -7,6 +7,7 @@ import at_record/gen/catalog/release as catalog_release import at_record/gen/defs import at_record_server/catalog_deps.{type Deps} +import at_record_server/catalog_entities import at_record_server/discogs_client import at_record_server/oauth/sessions.{type OauthSession} import at_record_server/provenance @@ -28,6 +29,17 @@ pub type Promoted { Promoted(release: defs.CatalogRef, origin: String) } +/// The result of an adopt-or-mint: the (possibly grown) own-release map and +/// entity dicts, the outcome, and the entity-write report the import surfaces. +pub type Outcome { + Outcome( + own: Dict(String, OwnRelease), + entities: catalog_entities.Entities, + promoted: Option(Promoted), + report: catalog_entities.MintReport, + ) +} + /// A release we already published, with enough detail to decide amendments. pub type OwnRelease { OwnRelease(ref: defs.CatalogRef, cover_size: Option(Int)) @@ -93,41 +105,58 @@ fn own_row_decoder() -> decode.Decoder( } /// Resolve the canonical ref for a release: own repo first, then the -/// Constellation backlink index, else mint. Returns the (possibly grown) own -/// map alongside the outcome. +/// Constellation backlink index, else mint (with artist/genre entities and the +/// Discogs enrichment when `details` are supplied). Returns the grown maps. pub fn adopt_or_mint( deps: Deps, client: Client, session: OauthSession, own: Dict(String, OwnRelease), + entities: catalog_entities.Entities, release: discogs_client.Release, cover: Option(Blob), -) -> #(Dict(String, OwnRelease), Option(Promoted)) { + details: Option(discogs_client.ReleaseDetails), +) -> Outcome { let id = int.to_string(release.discogs_id) case dict.get(own, id) { // Reuse our record; corrections happen through the manual amend flow. - Ok(existing) -> #( - own, - Some(Promoted(release: existing.ref, origin: "promotion")), - ) + Ok(existing) -> + Outcome( + own:, + entities:, + promoted: Some(Promoted(release: existing.ref, origin: "promotion")), + report: catalog_entities.empty_report, + ) Error(Nil) -> case discover(deps, session, id) { - Some(ref) -> #(own, Some(Promoted(release: ref, origin: "adoption"))) - None -> - case mint(client, session, release, cover, None) { - Some(ref) -> #( - dict.insert( - own, - id, - OwnRelease( - ref:, - cover_size: cover |> option.map(fn(b) { b.size }), + Some(ref) -> + Outcome( + own:, + entities:, + promoted: Some(Promoted(release: ref, origin: "adoption")), + report: catalog_entities.empty_report, + ) + None -> { + let #(entities, minted, report) = + mint(client, session, entities, release, cover, details, None) + case minted { + Some(ref) -> + Outcome( + own: dict.insert( + own, + id, + OwnRelease( + ref:, + cover_size: cover |> option.map(fn(b) { b.size }), + ), ), - ), - Some(Promoted(release: ref, origin: "promotion")), - ) - None -> #(own, None) + entities:, + promoted: Some(Promoted(release: ref, origin: "promotion")), + report:, + ) + None -> Outcome(own:, entities:, promoted: None, report:) } + } } } } @@ -180,59 +209,164 @@ pub fn parse_at_uri(uri: String) -> Option(#(String, String, String)) { } } +// Artists and genres are minted before the release so a rate-limited entity +// write aborts the whole item (nothing seen), letting the next run retry both. fn mint( client: Client, session: OauthSession, + entities: catalog_entities.Entities, release: discogs_client.Release, cover: Option(Blob), + details: Option(discogs_client.ReleaseDetails), supersedes: Option(defs.CatalogRef), -) -> Option(defs.CatalogRef) { +) -> #( + catalog_entities.Entities, + Option(defs.CatalogRef), + catalog_entities.MintReport, +) { let id = int.to_string(release.discogs_id) - let record = - catalog_release.CatalogRelease( - title: release.title, - created_at: provenance.now_rfc3339(), - external_ids: Some([discogs_client.external_id(release)]), - released: release.year |> option.map(int.to_string), - thumb_url: release.thumb_url, - cover:, - based_on: None, - country: None, - credited_artists: None, - formats: None, - genres: None, - identifiers: None, - labels: None, - master: None, - styles: None, - supersedes:, - tracklist: None, - ) - let created = - repo.create_record( - client, - session.pds, - session.access_token, - session.did, - catalog_release.collection, - catalog_release.encode_catalog_release(record), + let artists = details |> option.map(fn(d) { d.artists }) |> option.unwrap([]) + let #(artist_dict, credits, artist_report) = + catalog_entities.ensure_artists(client, session, entities.artists, artists) + case artist_report.rate_limited { + True -> #( + catalog_entities.Entities(..entities, artists: artist_dict), + None, + artist_report, ) - case created { - Ok(created) -> - Some(defs.CatalogRef( - uri: created.uri, - cid: created.cid, - external_ids: None, - )) - Error(e) -> { - wisp.log_warning( - "release mint failed for discogs:" <> id <> ": " <> xrpc_error(e), - ) - None + False -> { + let names = + details + |> option.map(fn(d) { list.append(d.genres, d.styles) }) + |> option.unwrap([]) + let #(genre_dict, _refs, genre_report) = + catalog_entities.ensure_genres(client, session, entities.genres, names) + let entities = + catalog_entities.Entities(artists: artist_dict, genres: genre_dict) + let report = catalog_entities.combine(artist_report, genre_report) + case genre_report.rate_limited { + True -> #(entities, None, report) + False -> { + let record = + catalog_release.CatalogRelease( + title: release.title, + created_at: provenance.now_rfc3339(), + external_ids: Some([discogs_client.external_id(release)]), + artist_display: artist_display(release), + released: released(release, details), + country: details |> option.then(fn(d) { d.country }), + genres: details |> option.then(fn(d) { non_empty_list(d.genres) }), + styles: details |> option.then(fn(d) { non_empty_list(d.styles) }), + identifiers: details + |> option.then(fn(d) { barcodes(d.barcodes) }), + credited_artists: credited_artists(credits), + thumb_url: release.thumb_url, + cover:, + based_on: None, + formats: None, + labels: None, + master: None, + supersedes:, + tracklist: None, + ) + let created = + repo.create_record( + client, + session.pds, + session.access_token, + session.did, + catalog_release.collection, + catalog_release.encode_catalog_release(record), + ) + case created { + Ok(created) -> #( + entities, + Some(defs.CatalogRef( + uri: created.uri, + cid: created.cid, + external_ids: None, + )), + report, + ) + Error(e) -> { + wisp.log_warning( + "release mint failed for discogs:" + <> id + <> ": " + <> xrpc_error(e), + ) + #(entities, None, report) + } + } + } + } } } } +fn credited_artists( + credits: List(defs.CatalogRef), +) -> Option(List(catalog_release.Credit)) { + case credits { + [] -> None + refs -> + Some( + list.map(refs, fn(r) { + catalog_release.Credit( + artist: r, + snapshot: None, + anv: None, + role: None, + join: None, + ) + }), + ) + } +} + +// The thin release's joined artist string, so a release renders without +// resolving artist entities. Always set (unless the string is empty). +fn artist_display(release: discogs_client.Release) -> Option(String) { + case release.artist { + "" -> None + a -> Some(a) + } +} + +fn released( + release: discogs_client.Release, + details: Option(discogs_client.ReleaseDetails), +) -> Option(String) { + let fallback = release.year |> option.map(int.to_string) + case details { + Some(d) -> option.or(d.released, fallback) + None -> fallback + } +} + +fn non_empty_list(items: List(a)) -> Option(List(a)) { + case items { + [] -> None + _ -> Some(items) + } +} + +fn barcodes(values: List(String)) -> Option(List(catalog_release.Identifier)) { + case values { + [] -> None + _ -> + Some( + list.map(values, fn(v) { + catalog_release.Identifier( + type_: "Barcode", + value: v, + description: None, + ) + }), + ) + } +} + fn xrpc_error(e: xrpc.XrpcError) -> String { case e { xrpc.RequestFailed(m) -> m diff --git a/server/test/catalog_entities_test.gleam b/server/test/catalog_entities_test.gleam new file mode 100644 index 0000000..4006fea --- /dev/null +++ b/server/test/catalog_entities_test.gleam @@ -0,0 +1,25 @@ +import at_record_server/catalog_entities +import gleam/set + +pub fn distinct_missing_dedups_within_batch_test() { + let out = + catalog_entities.distinct_missing(set.new(), ["a", "b", "a", "c", "b"]) + assert out == ["a", "b", "c"] +} + +pub fn distinct_missing_skips_already_known_test() { + let known = set.from_list(["a"]) + let out = catalog_entities.distinct_missing(known, ["a", "b", "a", "c"]) + assert out == ["b", "c"] +} + +// Genre keys are lowercased before this runs, so casing variants collapse here. +pub fn distinct_missing_collapses_lowercased_genre_keys_test() { + let out = + catalog_entities.distinct_missing(set.new(), ["rock", "rock", "jazz"]) + assert out == ["rock", "jazz"] +} + +pub fn distinct_missing_empty_is_empty_test() { + assert catalog_entities.distinct_missing(set.new(), []) == [] +} diff --git a/server/test/crate_test.gleam b/server/test/crate_test.gleam index 50af418..533b2a0 100644 --- a/server/test/crate_test.gleam +++ b/server/test/crate_test.gleam @@ -1,9 +1,13 @@ +import at_record/gen/catalog/release as catalog_release import at_record/gen/repo/strong_ref.{RepoStrongRef} import at_record/gen/shelf/entry.{type ShelfEntry, ShelfEntry} import at_record/storage.{type StoredItem, StoredItem} import at_record_server/crate +import gleam/dynamic/decode +import gleam/json import gleam/list import gleam/option.{None, Some} +import gleam/result fn event_uri(rkey: String) -> String { "at://did/dev.mokkenstorm.crate.shelf.entry/" <> rkey @@ -80,3 +84,59 @@ pub fn fold_drops_orphan_appends_test() { ]) assert list.map(folded, fn(e) { e.entry_id }) == ["3c"] } + +fn blank_release() -> catalog_release.CatalogRelease { + catalog_release.CatalogRelease( + title: "Spiderland", + artist_display: None, + created_at: "2026-01-01T00:00:00Z", + external_ids: None, + released: None, + thumb_url: None, + cover: None, + based_on: None, + country: None, + credited_artists: None, + formats: None, + genres: None, + identifiers: None, + labels: None, + master: None, + styles: None, + supersedes: None, + tracklist: None, + ) +} + +fn field_string(body: String, path: List(String)) -> Result(String, Nil) { + json.parse(body, decode.at(path, decode.string)) |> result.replace_error(Nil) +} + +fn field_present(body: String, path: List(String)) -> Bool { + json.parse(body, decode.at(path, decode.dynamic)) |> result.is_ok +} + +pub fn encode_release_info_includes_present_display_fields_test() { + let release = + catalog_release.CatalogRelease( + ..blank_release(), + artist_display: Some("Slint"), + genres: Some(["Rock"]), + styles: Some(["Math Rock"]), + country: Some("US"), + released: Some("1991-03-27"), + ) + let body = crate.encode_release_info(release) |> json.to_string + assert field_string(body, ["artistDisplay"]) == Ok("Slint") + assert field_string(body, ["country"]) == Ok("US") + assert field_string(body, ["released"]) == Ok("1991-03-27") +} + +pub fn encode_release_info_omits_absent_fields_test() { + let body = crate.encode_release_info(blank_release()) |> json.to_string + assert field_present(body, ["artistDisplay"]) == False + assert field_present(body, ["genres"]) == False + assert field_present(body, ["styles"]) == False + assert field_present(body, ["country"]) == False + assert field_present(body, ["released"]) == False +} diff --git a/server/test/promotion_test.gleam b/server/test/promotion_test.gleam index 389972a..4bb1e52 100644 --- a/server/test/promotion_test.gleam +++ b/server/test/promotion_test.gleam @@ -1,18 +1,41 @@ import at_record/gen/catalog/release as catalog_release import at_record/gen/defs.{CatalogRef} import at_record_server/catalog_deps.{type Deps, Deps} -import at_record_server/discogs_client.{Release} +import at_record_server/catalog_entities.{type Entities, Entities} +import at_record_server/discogs_client.{ + type ReleaseDetails, DetailArtist, Release, ReleaseDetails, +} import at_record_server/oauth/sessions.{type OauthSession, OauthSession} import at_record_server/promotion.{OwnRelease, Promoted} import atproto/constellation.{Backlink, BacklinksPage} import atproto/xrpc import gleam/bit_array import gleam/dict +import gleam/dynamic/decode import gleam/http/response +import gleam/json +import gleam/list import gleam/option.{None, Some} +import gleam/result import gose import kryptos/ec +fn no_entities() -> Entities { + Entities(artists: dict.new(), genres: dict.new()) +} + +fn details() -> ReleaseDetails { + ReleaseDetails( + primary_image_url: None, + artists: [DetailArtist(id: 1, name: "Slint")], + genres: ["Rock"], + styles: ["Post Rock"], + country: Some("US"), + released: Some("1991-03-27"), + barcodes: ["12345"], + ) +} + const own_did = "did:plc:me" const other_did = "did:plc:someone" @@ -62,6 +85,31 @@ fn offline_deps() -> Deps { ) } +/// Deps whose backlink index is empty, so adopt_or_mint falls through to mint. +fn no_backlinks_deps() -> Deps { + Deps( + backlinks: fn(_, _) { + Ok(BacklinksPage(total: 0, records: [], cursor: None)) + }, + fetch_release: fn(_) { panic as "fetch_release must not be called" }, + ) +} + +/// A stub PDS that answers every createRecord with a per-collection uri, and +/// runs `inspect` on each request body so a test can assert what was minted. +fn inspecting_client(inspect: fn(String) -> Nil) -> xrpc.Client { + xrpc.Client(send: fn(req) { + let body = bit_array.to_string(req.body) |> result.unwrap("") + inspect(body) + let collection = + json.parse(body, decode.at(["collection"], decode.string)) + |> result.unwrap("record") + let uri = "at://" <> own_did <> "/" <> collection <> "/1" + let out = "{\"uri\": \"" <> uri <> "\", \"cid\": \"bafy\"}" + Ok(response.new(200) |> response.set_body(bit_array.from_string(out))) + }) +} + fn dead_client() -> xrpc.Client { xrpc.Client(send: fn(_) { panic as "no PDS writes expected" }) } @@ -77,6 +125,7 @@ fn minting_client(uri: String) -> xrpc.Client { fn remote_release(discogs_id id: Int) -> catalog_release.CatalogRelease { catalog_release.CatalogRelease( title: "Spiderland", + artist_display: None, created_at: "2026-01-01T00:00:00Z", external_ids: Some([discogs_client.external_id(release(id))]), released: None, @@ -99,17 +148,20 @@ fn remote_release(discogs_id id: Int) -> catalog_release.CatalogRelease { pub fn own_map_hit_reuses_without_network_test() { let existing = own_release("at://" <> own_did <> "/release/3aaa") let own = dict.from_list([#("42", existing)]) - let #(own_after, promoted) = + let outcome = promotion.adopt_or_mint( offline_deps(), dead_client(), session(), own, + no_entities(), release(42), None, + None, ) - assert promoted == Some(Promoted(release: existing.ref, origin: "promotion")) - assert own_after == own + assert outcome.promoted + == Some(Promoted(release: existing.ref, origin: "promotion")) + assert outcome.own == own } pub fn verified_backlink_is_adopted_test() { @@ -135,16 +187,19 @@ pub fn verified_backlink_is_adopted_test() { Some(#("bafyforeign", remote_release(discogs_id: 42))) }, ) - let #(_, promoted) = + let outcome = promotion.adopt_or_mint( deps, dead_client(), session(), dict.new(), + no_entities(), release(42), None, + None, ) - let assert Some(Promoted(release: adopted, origin: "adoption")) = promoted + let assert Some(Promoted(release: adopted, origin: "adoption")) = + outcome.promoted assert adopted.uri == foreign_uri assert adopted.cid == "bafyforeign" } @@ -163,18 +218,21 @@ pub fn unverified_backlink_falls_through_to_mint_test() { fetch_release: fn(_) { Some(#("bafyx", remote_release(discogs_id: 999))) }, ) let minted_uri = "at://" <> own_did <> "/release/3new" - let #(own_after, promoted) = + let outcome = promotion.adopt_or_mint( deps, minting_client(minted_uri), session(), dict.new(), + no_entities(), release(42), None, + None, ) - let assert Some(Promoted(release: minted, origin: "promotion")) = promoted + let assert Some(Promoted(release: minted, origin: "promotion")) = + outcome.promoted assert minted.uri == minted_uri - assert dict.size(own_after) == 1 + assert dict.size(outcome.own) == 1 } pub fn constellation_outage_still_mints_test() { @@ -183,16 +241,109 @@ pub fn constellation_outage_still_mints_test() { backlinks: fn(_, _) { Error(xrpc.RequestFailed("down")) }, fetch_release: fn(_) { panic as "no candidates to verify" }, ) - let #(_, promoted) = + let outcome = promotion.adopt_or_mint( deps, minting_client("at://" <> own_did <> "/release/3ddd"), session(), dict.new(), + no_entities(), + release(42), + None, + None, + ) + let assert Some(Promoted(origin: "promotion", ..)) = outcome.promoted +} + +fn field_string(body: String, path: List(String)) -> Result(String, Nil) { + json.parse(body, decode.at(path, decode.string)) |> result.replace_error(Nil) +} + +fn field_strings( + body: String, + path: List(String), +) -> Result(List(String), Nil) { + json.parse(body, decode.at(path, decode.list(decode.string))) + |> result.replace_error(Nil) +} + +fn field_nested( + body: String, + path: List(String), + inner: List(String), +) -> Result(List(String), Nil) { + json.parse( + body, + decode.at(path, decode.list(decode.at(inner, decode.string))), + ) + |> result.replace_error(Nil) +} + +fn field_present(body: String, path: List(String)) -> Bool { + json.parse(body, decode.at(path, decode.dynamic)) |> result.is_ok +} + +pub fn mint_with_details_enriches_release_test() { + let client = + inspecting_client(fn(body) { + let collection = field_string(body, ["collection"]) |> result.unwrap("") + case collection == catalog_release.collection { + False -> Nil + True -> { + assert field_string(body, ["record", "artistDisplay"]) == Ok("Slint") + assert field_strings(body, ["record", "genres"]) == Ok(["Rock"]) + assert field_strings(body, ["record", "styles"]) == Ok(["Post Rock"]) + assert field_string(body, ["record", "country"]) == Ok("US") + assert field_string(body, ["record", "released"]) == Ok("1991-03-27") + assert field_nested(body, ["record", "identifiers"], ["type"]) + == Ok(["Barcode"]) + assert field_nested(body, ["record", "identifiers"], ["value"]) + == Ok(["12345"]) + assert field_nested(body, ["record", "creditedArtists"], [ + "artist", "uri", + ]) + |> result.map(list.length) + == Ok(1) + } + } + }) + let outcome = + promotion.adopt_or_mint( + no_backlinks_deps(), + client, + session(), + dict.new(), + no_entities(), release(42), None, + Some(details()), + ) + let assert Some(Promoted(origin: "promotion", ..)) = outcome.promoted + assert dict.size(outcome.entities.artists) == 1 + assert dict.size(outcome.entities.genres) == 2 +} + +pub fn mint_without_details_sets_only_artist_display_test() { + let client = + inspecting_client(fn(body) { + assert field_string(body, ["record", "artistDisplay"]) == Ok("Slint") + assert field_present(body, ["record", "genres"]) == False + assert field_present(body, ["record", "country"]) == False + assert field_present(body, ["record", "creditedArtists"]) == False + }) + let outcome = + promotion.adopt_or_mint( + no_backlinks_deps(), + client, + session(), + dict.new(), + no_entities(), + release(42), + None, + None, ) - let assert Some(Promoted(origin: "promotion", ..)) = promoted + let assert Some(Promoted(origin: "promotion", ..)) = outcome.promoted + assert dict.size(outcome.entities.artists) == 0 } pub fn parse_at_uri_extracts_did_collection_rkey_test() { diff --git a/shared/test/at_record_shared_test.gleam b/shared/test/at_record_shared_test.gleam index 800f53d..6b4e817 100644 --- a/shared/test/at_record_shared_test.gleam +++ b/shared/test/at_record_shared_test.gleam @@ -1,7 +1,9 @@ +import at_record/gen/catalog/artist import at_record/gen/catalog/edit -import at_record/gen/defs.{Snapshot, Source} +import at_record/gen/catalog/genre +import at_record/gen/defs.{ExternalId, Snapshot, Source} import at_record/gen/internal -import at_record/gen/shelf/item.{type ShelfItem, ShelfItem} +import at_record/gen/shelf/entry.{type ShelfEntry, ShelfEntry} import at_record/storage.{StoredItem} import gleam/dynamic/decode import gleam/json @@ -32,9 +34,19 @@ pub fn unknown_round_trips_test() { assert internal.dynamic_to_json(value) |> json.to_string == "[1,2,3]" } -fn sample() -> ShelfItem { - ShelfItem( +fn sample() -> ShelfEntry { + ShelfEntry( + action: "acquired", + counterparty: None, + created_at: "2026-06-26T00:00:00Z", + external_ids: None, + folder: None, + media_grade: Some("NM"), + notes: None, + price: None, + rating: Some(5), release: None, + sleeve_grade: None, snapshot: Snapshot( title: "Spiderland", artist_display: "Slint", @@ -42,14 +54,8 @@ fn sample() -> ShelfItem { format: Some("LP"), thumb_url: None, cover: None, - ), - status: "owned", - media_grade: Some("NM"), - sleeve_grade: None, - folder: None, - rating: Some(5), - notes: None, - acquired_at: None, + ) + |> Some, source: Source( origin: Some("manual"), client_agent: None, @@ -58,31 +64,29 @@ fn sample() -> ShelfItem { record: None, ) |> Some, - created_at: "2026-06-26T00:00:00Z", - updated_at: None, - deleted: None, + subject: None, ) } -pub fn shelf_item_round_trips_test() { +pub fn shelf_entry_round_trips_test() { let value = sample() let assert Ok(decoded) = value - |> item.encode_shelf_item + |> entry.encode_shelf_entry |> json.to_string - |> json.parse(item.shelf_item_decoder()) + |> json.parse(entry.shelf_entry_decoder()) assert decoded == value } pub fn collection_constant_test() { - assert item.collection == "dev.mokkenstorm.crate.shelf.item" + assert entry.collection == "dev.mokkenstorm.crate.shelf.entry" } pub fn optional_fields_are_omitted_not_nulled_test() { // `release` is None, so the key must be absent (not `"release":null`). let release_present = sample() - |> item.encode_shelf_item + |> entry.encode_shelf_entry |> json.to_string |> json.parse(decode.at(["release"], decode.dynamic)) |> result.is_ok @@ -94,9 +98,9 @@ pub fn stored_item_round_trips_test() { StoredItem(uri: "at://x/y/z", cid: "bafy", rkey: "z", value: sample()) let assert Ok(decoded) = stored - |> storage.encode_stored_item(item.encode_shelf_item) + |> storage.encode_stored_item(entry.encode_shelf_entry) |> json.to_string - |> json.parse(storage.stored_item_decoder(item.shelf_item_decoder())) + |> json.parse(storage.stored_item_decoder(entry.shelf_entry_decoder())) assert decoded == stored } @@ -142,3 +146,71 @@ pub fn catalog_edit_union_unknown_variant_round_trips_test() { let assert Ok("dev.mokkenstorm.crate.catalog.edit#artistFields") = json.parse(re, decode.at(["fields", "$type"], decode.string)) } + +fn sample_artist() -> artist.CatalogArtist { + artist.CatalogArtist( + based_on: None, + created_at: "2026-07-09T00:00:00Z", + disambiguation: Some("Welsh electronic musician"), + external_ids: Some([ + ExternalId( + id: "7e5cc599-2c00-4d80-9d24-d2c0143d2775", + provider: "musicbrainz", + url: None, + ), + ]), + name: "Aphex Twin", + sort_name: Some("Aphex Twin"), + supersedes: None, + thumb_url: None, + ) +} + +pub fn catalog_artist_round_trips_test() { + let value = sample_artist() + let assert Ok(decoded) = + value + |> artist.encode_catalog_artist + |> json.to_string + |> json.parse(artist.catalog_artist_decoder()) + assert decoded == value + assert artist.collection == "dev.mokkenstorm.crate.catalog.artist" +} + +pub fn catalog_genre_round_trips_test() { + let value = + genre.CatalogGenre( + created_at: "2026-07-09T00:00:00Z", + description: Some("Fourth-world ambient with a rhythmic backbone."), + external_ids: None, + name: "Ambient", + ) + let assert Ok(decoded) = + value + |> genre.encode_catalog_genre + |> json.to_string + |> json.parse(genre.catalog_genre_decoder()) + assert decoded == value + assert genre.collection == "dev.mokkenstorm.crate.catalog.genre" +} + +pub fn catalog_artist_optional_fields_are_omitted_not_nulled_test() { + let required_only = + artist.CatalogArtist( + based_on: None, + created_at: "2026-07-09T00:00:00Z", + disambiguation: None, + external_ids: None, + name: "Aphex Twin", + sort_name: None, + supersedes: None, + thumb_url: None, + ) + let sort_name_present = + required_only + |> artist.encode_catalog_artist + |> json.to_string + |> json.parse(decode.at(["sortName"], decode.dynamic)) + |> result.is_ok + assert sort_name_present == False +} -- 2.51.2