From ade527a5ed11193a229e243514c06aee01e601f8 Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Sun, 9 Aug 2026 23:29:10 +0200 Subject: [PATCH 1/4] fix: resume pagination by position when a live cursor's anchor row is gone --- server/src/crate_server/pagination.gleam | 132 ++++++++++++++++++----- server/test/pagination_test.gleam | 75 ++++++++++++- 2 files changed, 178 insertions(+), 29 deletions(-) diff --git a/server/src/crate_server/pagination.gleam b/server/src/crate_server/pagination.gleam index ae36f5f..e249ee4 100644 --- a/server/src/crate_server/pagination.gleam +++ b/server/src/crate_server/pagination.gleam @@ -1,14 +1,31 @@ //// Cursor pagination over an already-sorted, in-memory list. //// -//// Cursor convention: an opaque token, base64url(namespace <> ":" <> last -//// id) - scoped to a namespace (e.g. a shelf view) so switching filters can -//// never resume into the wrong slice, and never surfaced as an error: a -//// token that fails to decode, is scoped to another namespace, or names an -//// id no longer present in the list, all just restart the page from the -//// top. base64url (not standard base64) so the token is safe to drop -//// straight into a query string with no percent-encoding. +//// Cursor convention: an opaque token, base64url(namespace <> ":" <> +//// last_id [<> "|" <> tail_length]) - scoped to a namespace (e.g. a shelf +//// view) so switching filters can never resume into the wrong slice, and +//// never surfaced as an error: a token that fails to decode, or is scoped +//// to another namespace, just restarts the page from the top. base64url +//// (not standard base64) so the token is safe to drop straight into a +//// query string with no percent-encoding. +//// +//// `tail_length` is how many items followed the anchor when the cursor was +//// minted. It's what makes resuming safe on a *live* list, where a row can +//// vanish between two requests (an adoption deleted or superseded): if the +//// anchor id is gone, dropping the caller's current items down to its last +//// `tail_length` elements resumes right where the client left off instead +//// of replaying the page it already has. Older cursors minted before this +//// field existed carry no `tail_length` (no "|" segment); those still +//// decode fine, they just fall back to the pre-existing behavior of +//// restarting from the top when the anchor id can't be found - degrade +//// gracefully rather than break cursors already held by real clients. +//// +//// Insertions ahead of the anchor's original position (not just at the +//// head of the list) can still confuse the tail-length fallback; this is a +//// best-effort resume, not a linearizable one, matching the rest of this +//// module's philosophy of never erroring on a cursor. import gleam/bit_array +import gleam/int import gleam/list import gleam/option.{type Option, None, Some} import gleam/result @@ -18,17 +35,41 @@ import gleam/string /// and the ceiling any supplied `limit` is clamped to. pub const default_limit = 100 -/// Encode the opaque "resume after `last_id`" cursor for `namespace`. +/// A decoded cursor: the id to resume after, plus (when the cursor was +/// minted by this module rather than hand-built) how many items followed it +/// at mint time, used to resume by position if the id itself is gone. +pub type ResumePoint { + ResumePoint(last_id: String, tail_length: Option(Int)) +} + +/// Encode the opaque "resume after `last_id`" cursor for `namespace`, with +/// no recorded tail length. This is the pre-existing, position-less cursor +/// format: still produced on purpose (e.g. to build a cursor by hand in +/// tests) and still decodes and resumes correctly, just without the +/// stale-anchor fallback `page` gets from cursors it mints itself. pub fn encode_cursor(namespace: String, last_id: String) -> String { - bit_array.base64_url_encode( - bit_array.from_string(namespace <> ":" <> last_id), - False, - ) + encode_bytes(namespace <> ":" <> last_id) +} + +/// Encode the opaque "resume after `last_id`" cursor for `namespace`, +/// recording that `tail_length` items followed it in the list it was cut +/// from. Used internally by `page` to mint cursors that can resume by +/// position when the anchor id disappears. +fn encode_cursor_at( + namespace: String, + last_id: String, + tail_length: Int, +) -> String { + encode_bytes(namespace <> ":" <> last_id <> "|" <> int.to_string(tail_length)) +} + +fn encode_bytes(payload: String) -> String { + bit_array.base64_url_encode(bit_array.from_string(payload), False) } /// Decode a cursor scoped to `namespace`. Anything unparseable, malformed, /// or scoped to a different namespace decodes to `None`. -pub fn decode_cursor(namespace: String, cursor: String) -> Option(String) { +pub fn decode_cursor(namespace: String, cursor: String) -> Option(ResumePoint) { case bit_array.base64_url_decode(cursor) { Error(_) -> None Ok(bytes) -> @@ -36,22 +77,40 @@ pub fn decode_cursor(namespace: String, cursor: String) -> Option(String) { Error(_) -> None Ok(decoded) -> case string.split_once(decoded, ":") { - Ok(#(ns, last_id)) if ns == namespace -> Some(last_id) + Ok(#(ns, rest)) if ns == namespace -> Some(parse_resume_point(rest)) _ -> None } } } } +/// `rest` is `last_id` alone (older cursors) or `last_id <> "|" <> +/// tail_length` (cursors minted by `page`). An unparseable tail length is +/// treated the same as a missing one, degrading rather than rejecting the +/// whole cursor - the id half is still trustworthy. +fn parse_resume_point(rest: String) -> ResumePoint { + case string.split_once(rest, "|") { + Error(_) -> ResumePoint(rest, None) + Ok(#(last_id, tail)) -> + case int.parse(tail) { + Ok(n) -> ResumePoint(last_id, Some(n)) + Error(_) -> ResumePoint(last_id, None) + } + } +} + /// Slice a page out of `items` (already sorted by the caller in the order /// pagination should walk). /// /// No cursor and no limit means "return everything": the pre-pagination /// behavior, so a caller that never adopts paging sees no change. Otherwise -/// the cursor resumes just after the matching id (an unrecognized or stale -/// cursor restarts from the top), and the page is capped at `limit` -/// (defaulting to, and ceilinged at, `default_limit`). Returns the page plus -/// the next cursor, `None` once there's nothing left to fetch. +/// the cursor resumes just after the matching id when it's still present; +/// when it's gone (row deleted between requests on a live list) but the +/// cursor carries a recorded tail length, it resumes by position instead of +/// replaying the page the client already has. An unrecognized, foreign, or +/// position-less cursor still restarts from the top. The page is capped at +/// `limit` (defaulting to, and ceilinged at, `default_limit`). Returns the +/// page plus the next cursor, `None` once there's nothing left to fetch. pub fn page( items: List(a), id_of: fn(a) -> String, @@ -68,12 +127,15 @@ pub fn page( } let capped = clamp_limit(limit) let taken = list.take(remaining, capped) - let next_cursor = case list.length(remaining) > capped { + let tail_length = list.length(remaining) - capped + let next_cursor = case tail_length > 0 { False -> None True -> taken |> list.last - |> result.map(fn(last) { encode_cursor(namespace, id_of(last)) }) + |> result.map(fn(last) { + encode_cursor_at(namespace, id_of(last), tail_length) + }) |> option.from_result } #(taken, next_cursor) @@ -82,8 +144,10 @@ pub fn page( } /// Everything in `items` after the element whose id matches the decoded -/// cursor; the full list when the cursor doesn't decode or its id isn't -/// found (stale/garbage cursors restart rather than error). +/// cursor. When that id isn't found, falls back to the cursor's recorded +/// tail length if it has one (see the module doc); otherwise, and when the +/// cursor doesn't decode at all, the full list (stale/garbage cursors +/// restart rather than error). fn resume_after( items: List(a), id_of: fn(a) -> String, @@ -92,14 +156,34 @@ fn resume_after( ) -> List(a) { case decode_cursor(namespace, token) { None -> items - Some(last_id) -> + Some(ResumePoint(last_id, tail_length)) -> case list.split_while(items, fn(i) { id_of(i) != last_id }) { #(_, [_found, ..rest]) -> rest - #(_, []) -> items + #(_, []) -> resume_by_tail_length(items, tail_length) } } } +/// Resume by "the last `tail_length` items of the current list" when the +/// anchor id itself has vanished. This stays correct across deletions +/// anywhere in the list (the recorded count only ever over-covers what's +/// left, never under-covers it) as long as nothing gets inserted ahead of +/// where the anchor used to sit; new items appearing only at the head of a +/// live feed is the case this is built for. A cursor with no recorded +/// length (pre-existing cursors, see module doc) restarts from the top. +fn resume_by_tail_length(items: List(a), tail_length: Option(Int)) -> List(a) { + case tail_length { + None -> items + Some(n) -> { + let total = list.length(items) + case n >= total { + True -> items + False -> list.drop(items, total - n) + } + } + } +} + fn clamp_limit(limit: Option(Int)) -> Int { case limit { Some(n) if n >= 1 && n <= default_limit -> n diff --git a/server/test/pagination_test.gleam b/server/test/pagination_test.gleam index 564d67c..44e2f83 100644 --- a/server/test/pagination_test.gleam +++ b/server/test/pagination_test.gleam @@ -24,7 +24,8 @@ fn identity(s: String) -> String { pub fn decode_cursor_roundtrips_an_encoded_cursor_test() { let token = pagination.encode_cursor("owned", "item-005") - assert pagination.decode_cursor("owned", token) == Some("item-005") + assert pagination.decode_cursor("owned", token) + == Some(pagination.ResumePoint("item-005", None)) } pub fn decode_cursor_rejects_malformed_or_mismatched_tokens_test() { @@ -64,17 +65,24 @@ pub fn page_past_the_end_of_the_list_is_empty_with_no_next_cursor_test() { pub fn page_walks_across_pages_by_cursor_test() { let items = ids(5) [ - #(None, ["item-001", "item-002"], Some("item-002")), - #(Some("item-002"), ["item-003", "item-004"], Some("item-004")), + #(None, ["item-001", "item-002"], Some(#("item-002", 3))), + #(Some("item-002"), ["item-003", "item-004"], Some(#("item-004", 1))), #(Some("item-004"), ["item-005"], None), ] |> list.each(fn(row) { - let #(cursor_id, expected_page, next_id) = row + let #(cursor_id, expected_page, expected_next) = row let cursor = option.map(cursor_id, pagination.encode_cursor("owned", _)) let #(page, next) = pagination.page(items, identity, "owned", cursor, Some(2)) assert page == expected_page - assert next == option.map(next_id, pagination.encode_cursor("owned", _)) + // The minted cursor carries a resume position (how many items followed + // the anchor at mint time) alongside the id now, so compare on the + // decoded anchor rather than the raw token. + assert option.map(next, pagination.decode_cursor("owned", _)) + == option.map(expected_next, fn(anchor) { + let #(id, tail_length) = anchor + Some(pagination.ResumePoint(id, Some(tail_length))) + }) }) } @@ -99,3 +107,60 @@ pub fn page_with_only_a_cursor_and_no_limit_uses_the_default_limit_test() { assert list.length(page) == pagination.default_limit assert next != None } + +pub fn a_cursor_minted_by_page_resumes_by_position_when_its_anchor_vanishes_test() { + let items = ids(6) + let assert #(_first_page, Some(cursor)) = + pagination.page(items, identity, "owned", None, Some(2)) + as "cursor minted for a live list should carry a resume position" + + // The anchor (item-002) is gone by the next request, same as a deleted + // or superseded row on a live feed. Resuming must neither replay + // item-001/item-002 (already seen) nor skip item-003 (never seen). + let items_without_anchor = list.filter(items, fn(id) { id != "item-002" }) + let #(page, next) = + pagination.page( + items_without_anchor, + identity, + "owned", + Some(cursor), + Some(2), + ) + assert page == ["item-003", "item-004"] + assert next != None +} + +pub fn resuming_by_position_never_truncates_the_tail_even_if_more_than_the_anchor_vanished_test() { + let items = ids(6) + let assert #(_first_page, Some(cursor)) = + pagination.page(items, identity, "owned", None, Some(2)) + as "cursor minted for a live list should carry a resume position" + + // Everything up to and including the recorded tail length disappeared + // too. There's nothing safe left to skip, so the fallback must hand back + // what remains rather than guess past it. + let heavily_pruned = ["item-004", "item-005", "item-006"] + let #(page, _next) = + pagination.page(heavily_pruned, identity, "owned", Some(cursor), Some(2)) + assert page == ["item-004", "item-005"] +} + +pub fn an_old_format_cursor_with_a_missing_anchor_still_restarts_from_the_top_test() { + // Cursors minted before this module recorded a resume position carry no + // "|"-separated tail length; `encode_cursor` (with no position argument) + // still produces that format on purpose, so real cursors already held by + // clients keep decoding and keep degrading to a restart rather than + // erroring or crashing. + let items = ids(5) + let old_style = pagination.encode_cursor("owned", "item-002") + let items_without_anchor = list.filter(items, fn(id) { id != "item-002" }) + let #(page, _next) = + pagination.page( + items_without_anchor, + identity, + "owned", + Some(old_style), + Some(2), + ) + assert page == ["item-001", "item-003"] +} -- 2.51.2 From a990cc999cbb99ceb9d3ed85e29b9d010e7a2fa8 Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Mon, 10 Aug 2026 15:50:17 +0200 Subject: [PATCH 2/4] docs: trim the pagination cursor comments to the non-obvious parts --- server/src/crate_server/pagination.gleam | 86 ++++++------------------ server/test/pagination_test.gleam | 19 +----- 2 files changed, 22 insertions(+), 83 deletions(-) diff --git a/server/src/crate_server/pagination.gleam b/server/src/crate_server/pagination.gleam index e249ee4..1f28542 100644 --- a/server/src/crate_server/pagination.gleam +++ b/server/src/crate_server/pagination.gleam @@ -1,28 +1,11 @@ //// Cursor pagination over an already-sorted, in-memory list. //// -//// Cursor convention: an opaque token, base64url(namespace <> ":" <> -//// last_id [<> "|" <> tail_length]) - scoped to a namespace (e.g. a shelf -//// view) so switching filters can never resume into the wrong slice, and -//// never surfaced as an error: a token that fails to decode, or is scoped -//// to another namespace, just restarts the page from the top. base64url -//// (not standard base64) so the token is safe to drop straight into a -//// query string with no percent-encoding. -//// -//// `tail_length` is how many items followed the anchor when the cursor was -//// minted. It's what makes resuming safe on a *live* list, where a row can -//// vanish between two requests (an adoption deleted or superseded): if the -//// anchor id is gone, dropping the caller's current items down to its last -//// `tail_length` elements resumes right where the client left off instead -//// of replaying the page it already has. Older cursors minted before this -//// field existed carry no `tail_length` (no "|" segment); those still -//// decode fine, they just fall back to the pre-existing behavior of -//// restarting from the top when the anchor id can't be found - degrade -//// gracefully rather than break cursors already held by real clients. -//// -//// Insertions ahead of the anchor's original position (not just at the -//// head of the list) can still confuse the tail-length fallback; this is a -//// best-effort resume, not a linearizable one, matching the rest of this -//// module's philosophy of never erroring on a cursor. +//// A cursor is base64url(namespace <> ":" <> last_id [<> "|" <> tail_length]). +//// The namespace stops a filter switch resuming into the wrong slice, and no +//// cursor is ever an error: unparseable, foreign or stale ones restart from +//// the top. `tail_length` (items following the anchor when it was minted) +//// resumes by position once the anchor row itself is gone, which holds only +//// while nothing is inserted after the anchor. import gleam/bit_array import gleam/int @@ -31,30 +14,19 @@ import gleam/option.{type Option, None, Some} import gleam/result import gleam/string -/// The page size used when a cursor is given without an explicit `limit`, -/// and the ceiling any supplied `limit` is clamped to. +/// Default page size, and the ceiling on any supplied `limit`. pub const default_limit = 100 -/// A decoded cursor: the id to resume after, plus (when the cursor was -/// minted by this module rather than hand-built) how many items followed it -/// at mint time, used to resume by position if the id itself is gone. +/// The id to resume after, plus the tail length recorded at mint time. pub type ResumePoint { ResumePoint(last_id: String, tail_length: Option(Int)) } -/// Encode the opaque "resume after `last_id`" cursor for `namespace`, with -/// no recorded tail length. This is the pre-existing, position-less cursor -/// format: still produced on purpose (e.g. to build a cursor by hand in -/// tests) and still decodes and resumes correctly, just without the -/// stale-anchor fallback `page` gets from cursors it mints itself. +/// Encode a cursor with no tail length: resumes by id, restarts once it is gone. pub fn encode_cursor(namespace: String, last_id: String) -> String { encode_bytes(namespace <> ":" <> last_id) } -/// Encode the opaque "resume after `last_id`" cursor for `namespace`, -/// recording that `tail_length` items followed it in the list it was cut -/// from. Used internally by `page` to mint cursors that can resume by -/// position when the anchor id disappears. fn encode_cursor_at( namespace: String, last_id: String, @@ -67,8 +39,8 @@ fn encode_bytes(payload: String) -> String { bit_array.base64_url_encode(bit_array.from_string(payload), False) } -/// Decode a cursor scoped to `namespace`. Anything unparseable, malformed, -/// or scoped to a different namespace decodes to `None`. +/// Decode a cursor scoped to `namespace`. Unparseable, malformed or foreign +/// tokens decode to `None`. pub fn decode_cursor(namespace: String, cursor: String) -> Option(ResumePoint) { case bit_array.base64_url_decode(cursor) { Error(_) -> None @@ -84,10 +56,8 @@ pub fn decode_cursor(namespace: String, cursor: String) -> Option(ResumePoint) { } } -/// `rest` is `last_id` alone (older cursors) or `last_id <> "|" <> -/// tail_length` (cursors minted by `page`). An unparseable tail length is -/// treated the same as a missing one, degrading rather than rejecting the -/// whole cursor - the id half is still trustworthy. +// An unparseable tail length degrades rather than rejecting the cursor: the +// id half is still trustworthy. fn parse_resume_point(rest: String) -> ResumePoint { case string.split_once(rest, "|") { Error(_) -> ResumePoint(rest, None) @@ -99,18 +69,9 @@ fn parse_resume_point(rest: String) -> ResumePoint { } } -/// Slice a page out of `items` (already sorted by the caller in the order -/// pagination should walk). -/// -/// No cursor and no limit means "return everything": the pre-pagination -/// behavior, so a caller that never adopts paging sees no change. Otherwise -/// the cursor resumes just after the matching id when it's still present; -/// when it's gone (row deleted between requests on a live list) but the -/// cursor carries a recorded tail length, it resumes by position instead of -/// replaying the page the client already has. An unrecognized, foreign, or -/// position-less cursor still restarts from the top. The page is capped at -/// `limit` (defaulting to, and ceilinged at, `default_limit`). Returns the -/// page plus the next cursor, `None` once there's nothing left to fetch. +/// Slice a page out of `items`, already sorted by the caller, capped at +/// `limit`. No cursor and no limit returns everything. Returns the page plus +/// the cursor for the next one, `None` once nothing is left. pub fn page( items: List(a), id_of: fn(a) -> String, @@ -143,11 +104,6 @@ pub fn page( } } -/// Everything in `items` after the element whose id matches the decoded -/// cursor. When that id isn't found, falls back to the cursor's recorded -/// tail length if it has one (see the module doc); otherwise, and when the -/// cursor doesn't decode at all, the full list (stale/garbage cursors -/// restart rather than error). fn resume_after( items: List(a), id_of: fn(a) -> String, @@ -164,13 +120,9 @@ fn resume_after( } } -/// Resume by "the last `tail_length` items of the current list" when the -/// anchor id itself has vanished. This stays correct across deletions -/// anywhere in the list (the recorded count only ever over-covers what's -/// left, never under-covers it) as long as nothing gets inserted ahead of -/// where the anchor used to sit; new items appearing only at the head of a -/// live feed is the case this is built for. A cursor with no recorded -/// length (pre-existing cursors, see module doc) restarts from the top. +// Keeping the last `tail_length` items over-covers once rows are deleted (a +// row is reseen) and under-covers once rows are inserted after the anchor (a +// row is lost). fn resume_by_tail_length(items: List(a), tail_length: Option(Int)) -> List(a) { case tail_length { None -> items diff --git a/server/test/pagination_test.gleam b/server/test/pagination_test.gleam index 44e2f83..c17684a 100644 --- a/server/test/pagination_test.gleam +++ b/server/test/pagination_test.gleam @@ -12,8 +12,7 @@ fn ids(count: Int) -> List(String) { |> list.index_map(fn(_, i) { "item-" <> zero_padded(i + 1) }) } -// Zero-padded so string.compare sorts the same as numeric order, matching -// how entry ids (TIDs) sort lexically. +// Zero-padded so string.compare sorts numerically, as TID entry ids do. fn zero_padded(n: Int) -> String { string.pad_start(int.to_string(n), 3, "0") } @@ -75,9 +74,6 @@ pub fn page_walks_across_pages_by_cursor_test() { let #(page, next) = pagination.page(items, identity, "owned", cursor, Some(2)) assert page == expected_page - // The minted cursor carries a resume position (how many items followed - // the anchor at mint time) alongside the id now, so compare on the - // decoded anchor rather than the raw token. assert option.map(next, pagination.decode_cursor("owned", _)) == option.map(expected_next, fn(anchor) { let #(id, tail_length) = anchor @@ -114,9 +110,6 @@ pub fn a_cursor_minted_by_page_resumes_by_position_when_its_anchor_vanishes_test pagination.page(items, identity, "owned", None, Some(2)) as "cursor minted for a live list should carry a resume position" - // The anchor (item-002) is gone by the next request, same as a deleted - // or superseded row on a live feed. Resuming must neither replay - // item-001/item-002 (already seen) nor skip item-003 (never seen). let items_without_anchor = list.filter(items, fn(id) { id != "item-002" }) let #(page, next) = pagination.page( @@ -136,9 +129,7 @@ pub fn resuming_by_position_never_truncates_the_tail_even_if_more_than_the_ancho pagination.page(items, identity, "owned", None, Some(2)) as "cursor minted for a live list should carry a resume position" - // Everything up to and including the recorded tail length disappeared - // too. There's nothing safe left to skip, so the fallback must hand back - // what remains rather than guess past it. + // Fewer rows survive than the recorded tail length: nothing safe to skip. let heavily_pruned = ["item-004", "item-005", "item-006"] let #(page, _next) = pagination.page(heavily_pruned, identity, "owned", Some(cursor), Some(2)) @@ -146,11 +137,7 @@ pub fn resuming_by_position_never_truncates_the_tail_even_if_more_than_the_ancho } pub fn an_old_format_cursor_with_a_missing_anchor_still_restarts_from_the_top_test() { - // Cursors minted before this module recorded a resume position carry no - // "|"-separated tail length; `encode_cursor` (with no position argument) - // still produces that format on purpose, so real cursors already held by - // clients keep decoding and keep degrading to a restart rather than - // erroring or crashing. + // Cursors already held by clients carry no tail length. let items = ids(5) let old_style = pagination.encode_cursor("owned", "item-002") let items_without_anchor = list.filter(items, fn(id) { id != "item-002" }) -- 2.51.2 From 18db37810287218d306ff6bb5f646ed7cc2de8c6 Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Mon, 10 Aug 2026 17:33:31 +0200 Subject: [PATCH 3/4] fix: pick nc timeout flags that exist on the running platform --- scripts/dev.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/dev.sh b/scripts/dev.sh index 6fee513..66268f9 100755 --- a/scripts/dev.sh +++ b/scripts/dev.sh @@ -86,7 +86,15 @@ free_8080 # out with no hint as to why. The per-container `.orb.local` name resolves to # the default bridge instead, which is always routed, so it goes first, with # the inspected addresses as the fallback for other runtimes. -db_reachable() { nc -z -G 2 -w 2 "$1" 5432 >/dev/null 2>&1; } +# macOS nc needs -G to bound a connect to an unroutable host (-w alone hangs +# there); OpenBSD nc has no -G and bounds it with -w. Probe once rather than +# branching on uname, or every candidate reads as dead on the wrong platform. +if nc -G 1 -z 127.0.0.1 1 2>&1 | grep -q 'invalid option'; then + nc_timeout_flags="-w 2" +else + nc_timeout_flags="-G 2 -w 2" +fi +db_reachable() { nc -z $nc_timeout_flags "$1" 5432 >/dev/null 2>&1; } if [ -z "${DATABASE_URL:-}" ]; then db_cid="$(docker compose ps -q db 2>/dev/null | head -n1)" -- 2.51.2 From e83d09d42a3aa954d833325edf965cb6b33e1f24 Mon Sep 17 00:00:00 2001 From: Niels Mokkenstorm Date: Mon, 10 Aug 2026 16:57:24 +0200 Subject: [PATCH 4/4] docs: cut narration comments from the devnet seed scripts --- e2e/devnet/create-test-accounts.sh | 8 ++-- e2e/devnet/seed-social.sh | 74 +++++++++--------------------- e2e/devnet/setup.sh | 7 ++- 3 files changed, 29 insertions(+), 60 deletions(-) diff --git a/e2e/devnet/create-test-accounts.sh b/e2e/devnet/create-test-accounts.sh index 5654ecc..3f067c4 100755 --- a/e2e/devnet/create-test-accounts.sh +++ b/e2e/devnet/create-test-accounts.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash -# Creates the fixed pool of e2e test accounts. Handles are well-known (not -# generated per run) because Caddy's TLS termination and the PDS network -# alias are declared per-handle in docker-compose.yml/Caddyfile — adding an -# account means adding it there too, not just running this script. +# Handles are well-known rather than generated per run because Caddy's TLS +# termination and the PDS network alias are declared per-handle in +# docker-compose.yml and the Caddyfile: adding an account means editing those +# too. set -euo pipefail PASSWORD="${TEST_ACCOUNT_PASSWORD:-e2e-test-password}" diff --git a/e2e/devnet/seed-social.sh b/e2e/devnet/seed-social.sh index 397517e..2224daf 100755 --- a/e2e/devnet/seed-social.sh +++ b/e2e/devnet/seed-social.sh @@ -1,16 +1,10 @@ #!/usr/bin/env bash -# Seeds the devnet's fixed accounts with a follow graph, a small shared -# catalog of releases, and crate entries (grades, ratings, notes, prices, -# multi-event timelines, cover art, and deliberate feed choreography) so the -# following/followers, feed, and record-detail UI have data to render -# without touching the live network. The feed is built from ADOPTION rows: -# entries whose `release` field points at a catalog.release record, so most -# genesis entries below carry one. +# Not idempotent: expects a freshly wiped devnet. Everything below is created +# unconditionally, so a second run duplicates the whole graph and changes the +# feed shape the choreography comments describe. # -# Not idempotent: expects a freshly wiped devnet. Every account, follow, -# release, and entry below is unconditionally created, so a second run -# duplicates the whole graph and changes the feed shape the choreography -# comments describe. +# The feed is built from adoption rows (entries whose `release` points at a +# catalog.release record), which is why most genesis entries below carry one. set -euo pipefail if ((BASH_VERSINFO[0] < 4)); then @@ -23,8 +17,8 @@ PDS="${PDS_URL:-http://localhost:2584}" PASSWORD="${TEST_ACCOUNT_PASSWORD:-e2e-test-password}" # Every string value below is JSON-encoded through this instead of hand-quoted, -# so a title/artist/note containing a quote, backslash, or newline can never -# produce malformed JSON. python3 is already a dependency (the PNG encoder). +# so a title, artist, or note containing a quote, backslash, or newline can +# never produce malformed JSON. json_str() { python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$1" } @@ -70,26 +64,23 @@ follow() { > /dev/null } -# Appends `,"key":` when value is non-empty, else nothing; lets -# JSON builders below stay flat instead of branching per optional field. +# Lets the JSON builders below stay flat instead of branching per optional +# field. opt_str() { [[ -z "${2:-}" ]] && return 0 printf ',"%s":%s' "$1" "$(json_str "$2")" } -# Same as opt_str but for values that are already JSON (ints, objects). opt_raw() { [[ -z "${2:-}" ]] && return 0 printf ',"%s":%s' "$1" "$2" } subject_ref() { printf '{"uri":%s,"cid":%s}' "$(json_str "$1")" "$(json_str "$2")"; } -# catalogRef and strongRef share the {uri, cid} shape we need here; named -# separately so call sites read as what they point at. +# catalogRef and strongRef share the {uri, cid} shape; aliased rather than +# merged so call sites read as what they point at. catalog_ref() { subject_ref "$1" "$2"; } -# Writes a genesis shelf entry (no subject) and returns the raw createRecord -# response so chained callers can pull uri/cid for follow-up events. write_genesis() { local jwt="$1" repo="$2" action="$3" created="$4" artist="$5" title="$6" \ media="$7" sleeve="$8" rating="$9" folder="${10}" notes="${11}" \ @@ -103,8 +94,6 @@ write_genesis() { create_record "$jwt" "$repo" dev.mokkenstorm.crate.shelf.entry "$record" } -# Appends a follow-up event referencing a prior entry's strongRef. Discards -# the response: nothing here chains a third time. write_event() { local jwt="$1" repo="$2" action="$3" created="$4" subject_json="$5" \ rating="${6:-}" notes="${7:-}" media="${8:-}" sleeve="${9:-}" @@ -126,8 +115,6 @@ wanted_entry() { "" "" "" "$folder" "" "" "" "" "" "$release" > /dev/null } -# Writes an owned genesis entry then two follow-up events (rated, then -# annotated), each a bit later than the last so ordering is meaningful. owned_with_rating_and_notes() { local jwt="$1" repo="$2" artist="$3" title="$4" media="$5" sleeve="$6" \ folder="$7" counterparty="$8" price_amount="$9" price_currency="${10}" \ @@ -142,8 +129,6 @@ owned_with_rating_and_notes() { write_event "$jwt" "$repo" annotated "$T2" "$subject" "" "$notes" "" "" } -# Writes an owned genesis entry then a regraded follow-up with updated -# grades, e.g. after a closer inspection. owned_with_regrade() { local jwt="$1" repo="$2" artist="$3" title="$4" media="$5" sleeve="$6" \ folder="$7" price_amount="$8" price_currency="$9" cover="${10}" \ @@ -157,9 +142,6 @@ owned_with_regrade() { write_event "$jwt" "$repo" regraded "$T1" "$subject" "" "" "$new_media" "$new_sleeve" } -# Mints a shared-catalog release. The lexicon calls this "lazy promotion on -# first reference", so minting right before the entries that adopt it -# matches how the real flow orders things. mint_release() { local jwt="$1" repo="$2" artist="$3" title="$4" year="$5" genre="$6" local resp @@ -168,8 +150,7 @@ mint_release() { catalog_ref "$(uri_of "$resp")" "$(cid_of "$resp")" } -# Hand-rolled 300x300 solid PNG (stdlib struct+zlib only, no Pillow) so each -# account gets a distinct cover color without shipping binary fixtures. +# Hand-rolled with stdlib struct+zlib (no Pillow) to avoid a binary fixture. generate_cover() { local out="$1" r="$2" g="$3" b="$4" python3 - "$out" "$r" "$g" "$b" 300 <<'PY' @@ -198,17 +179,13 @@ with open(path, "wb") as f: PY } -# Best-effort: generates a cover, uploads it, and echoes the blob JSON -# fragment verbatim (per com.atproto.repo.uploadBlob's `{blob: {$type, ref: -# {$link}, mimeType, size}}` shape). Warns and returns empty on any failure -# instead of aborting the whole seed run. +# Best effort: warns and returns empty on failure rather than aborting the run. make_cover_blob() { local handle="$1" jwt="$2" r="$3" g="$4" b="$5" local png resp blob_json # `mktemp XXXXXX.png` is not a template on BSD/macOS mktemp (the X's - # must be trailing); it would return that literal path. `-t` picks a temp - # dir and suffixes a real random name; the upload sets content-type - # explicitly, so the .png extension was never load-bearing. + # must be trailing) and would return that literal path. `-t` gives a real + # random name; the upload sets content-type, so .png was never load-bearing. png="$(mktemp -t crate-cover)" if ! generate_cover "$png" "$r" "$g" "$b" 2>/dev/null; then echo "warning: cover generation failed for $handle, skipping cover" >&2 @@ -235,8 +212,7 @@ for handle in alice.test bob.test carol.test dave.test erin.test; do echo "session $handle -> ${DID[$handle]}" done -# Asymmetric graph for varied counts: alice is popular, dave follows all, -# erin only has followers. +# Asymmetric on purpose, so following and followers counts differ per account. follow "${JWT[alice.test]}" "${DID[alice.test]}" "${DID[bob.test]}" follow "${JWT[alice.test]}" "${DID[alice.test]}" "${DID[carol.test]}" follow "${JWT[bob.test]}" "${DID[bob.test]}" "${DID[alice.test]}" @@ -256,8 +232,8 @@ COVER[dave.test]="$(make_cover_blob dave.test "${JWT[dave.test]}" 196 154 44)" COVER[erin.test]="$(make_cover_blob erin.test "${JWT[erin.test]}" 140 68 196)" echo "cover art seeded" -# Small shared catalog: alice mints most of it, bob mints a couple, matching -# how a real lazy-promotion catalog fills in from whoever gets there first. +# Minters are lopsided on purpose: a lazy-promotion catalog fills in from +# whoever references a release first. RELEASE[R1]="$(mint_release "${JWT[alice.test]}" "${DID[alice.test]}" "Alice Coltrane" "Journey in Satchidananda" 1971 jazz)" RELEASE[R2]="$(mint_release "${JWT[alice.test]}" "${DID[alice.test]}" "Pharoah Sanders" "Karma" 1969 jazz)" RELEASE[R3]="$(mint_release "${JWT[alice.test]}" "${DID[alice.test]}" "Can" "Tago Mago" 1971 krautrock)" @@ -268,10 +244,8 @@ RELEASE[R7]="$(mint_release "${JWT[bob.test]}" "${DID[bob.test]}" "Faust" "Faust RELEASE[R8]="$(mint_release "${JWT[bob.test]}" "${DID[bob.test]}" "Broadcast" "Tender Buttons" 2005 electronic)" echo "catalog releases minted" -# alice: jazz. Two chained timelines (rated+annotated, then regraded), one -# plain owned entry, and a couple of wanted entries. Most owned/wanted rows -# adopt a catalog release; the plain owned entry and one wanted stay -# snapshot-only so that state is still represented. +# alice: one owned and one wanted row are deliberately left snapshot-only (no +# release), so the non-adopting state is represented too. owned_with_rating_and_notes "${JWT[alice.test]}" "${DID[alice.test]}" \ "Alice Coltrane" "Journey in Satchidananda" NM VG+ jazz "Bleecker Street Records" 4500 USD \ "${COVER[alice.test]}" 5 "Found at a stoop sale; the dead wax is pristine." "${RELEASE[R1]}" @@ -282,7 +256,6 @@ owned_entry "${JWT[alice.test]}" "${DID[alice.test]}" \ wanted_entry "${JWT[alice.test]}" "${DID[alice.test]}" "Dorothy Ashby" "Afro-Harping" jazz wanted_entry "${JWT[alice.test]}" "${DID[alice.test]}" "Sun Ra" "Space Is the Place" jazz "${RELEASE[R6]}" -# bob: krautrock. owned_with_rating_and_notes "${JWT[bob.test]}" "${DID[bob.test]}" \ "Can" "Tago Mago" NM NM krautrock "Record fair, Cologne" 5500 EUR \ "${COVER[bob.test]}" 5 "Side two still turns heads at parties." "${RELEASE[R3]}" @@ -293,7 +266,6 @@ owned_entry "${JWT[bob.test]}" "${DID[bob.test]}" \ wanted_entry "${JWT[bob.test]}" "${DID[bob.test]}" "Neu!" "Neu! 75" krautrock wanted_entry "${JWT[bob.test]}" "${DID[bob.test]}" "Harmonia" "Deluxe" krautrock "${RELEASE[R8]}" -# carol: electronic/pop, plus a subjectConverge entry (see below). owned_with_rating_and_notes "${JWT[carol.test]}" "${DID[carol.test]}" \ "Stereolab" "Dots and Loops" NM NM electronic "" 3800 EUR \ "${COVER[carol.test]}" 5 "Pulled this straight to the top of the pile." "${RELEASE[R5]}" @@ -309,7 +281,6 @@ owned_entry "${JWT[carol.test]}" "${DID[carol.test]}" \ "Can" "Tago Mago" NM VG+ 4 krautrock "Bought on a whim after bob wouldn't stop talking about it." \ "${COVER[carol.test]}" "${RELEASE[R3]}" -# dave: hip-hop, plus an actorBatch burst and an isolated single (see below). owned_with_rating_and_notes "${JWT[dave.test]}" "${DID[dave.test]}" \ "J Dilla" "Donuts" NM NM hiphop "Discogs seller" 6000 USD \ "${COVER[dave.test]}" 5 "Played it front to back the day it arrived." "${RELEASE[R4]}" @@ -321,8 +292,8 @@ wanted_entry "${JWT[dave.test]}" "${DID[dave.test]}" "MF DOOM" "Madvillainy" hip wanted_entry "${JWT[dave.test]}" "${DID[dave.test]}" "Flying Lotus" "Cosmogramma" hiphop "${RELEASE[R8]}" # actorBatch: feed_skeleton.collapse runs the subjectConverge pass first, so # any release a co-adopter also touches in the T0 window is swept away from -# dave before the actor pass ever sees it -- the Madvillainy (R2) and -# Cosmogramma (R8) rows above are exactly that: alice also adopts R2 and +# dave before the actor pass ever sees it. That is what happens to the +# Madvillainy (R2) and Cosmogramma (R8) rows above: alice also adopts R2 and # bob/carol/erin also adopt R8, so both converge instead of batching. R9 and # R10 are minted here and adopted only by dave, so together with Donuts (R4, # also dave-exclusive) they give dave exactly 3 same-did rows in the T0 @@ -342,7 +313,6 @@ write_genesis "${JWT[dave.test]}" "${DID[dave.test]}" acquired "$T_PAST" \ "Arthur Russell" "Calling Out of Context" VG VG "" ambient "" "" "" "" \ "${COVER[dave.test]}" "${RELEASE[R6]}" > /dev/null -# erin: ambient, plus a reasonImport burst (see below). owned_with_rating_and_notes "${JWT[erin.test]}" "${DID[erin.test]}" \ "Arthur Russell" "Calling Out of Context" NM NM ambient "" 4200 USD \ "${COVER[erin.test]}" 5 "Warmer and stranger every time it spins." "${RELEASE[R6]}" diff --git a/e2e/devnet/setup.sh b/e2e/devnet/setup.sh index 58906a9..54b7add 100755 --- a/e2e/devnet/setup.sh +++ b/e2e/devnet/setup.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -# Clones the two source-built devnet dependencies at commits verified to work -# with this compose file's CLI flags. Both repos are fast-moving (AI-assisted -# development, no published container image matching their own main), so -# pinning beats "clone main and hope" — re-pin deliberately, not by accident. +# SHAs pinned to commits verified against this compose file's CLI flags: both +# upstreams move fast and publish no image matching their own main. Re-pin +# deliberately, not by accident. set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")" -- 2.51.2