From 6284ccb3c5d197c417d417b7c767706da71f2ba6 Mon Sep 17 00:00:00 2001 From: Timothy Quilling Date: Sat, 15 Nov 2025 19:33:33 -0500 Subject: [PATCH] fix: tid rkeys --- .../down.sql | 1 + .../up.sql | 6 +++ parakeet/src/loaders/list.rs | 13 ++++--- parakeet/src/loaders/misc.rs | 37 +++++++++++-------- parakeet/src/loaders/profile.rs | 2 +- 5 files changed, 37 insertions(+), 22 deletions(-) diff --git a/migrations/2025-11-01-114847_queues_and_extensions/down.sql b/migrations/2025-11-01-114847_queues_and_extensions/down.sql index 2a86e69b..7f7dc43a 100644 --- a/migrations/2025-11-01-114847_queues_and_extensions/down.sql +++ b/migrations/2025-11-01-114847_queues_and_extensions/down.sql @@ -8,4 +8,5 @@ DROP TABLE IF EXISTS fetch_queue; -- Drop extensions DROP EXTENSION IF EXISTS pgrouting; DROP EXTENSION IF EXISTS postgis; +DROP EXTENSION IF EXISTS pg_trgm; DROP EXTENSION IF EXISTS pg_stat_statements; diff --git a/migrations/2025-11-01-114847_queues_and_extensions/up.sql b/migrations/2025-11-01-114847_queues_and_extensions/up.sql index 3372890a..d243cfa8 100644 --- a/migrations/2025-11-01-114847_queues_and_extensions/up.sql +++ b/migrations/2025-11-01-114847_queues_and_extensions/up.sql @@ -16,6 +16,12 @@ -- via shared_preload_libraries for optimal performance. CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +-- pg_trgm: Trigram matching for fuzzy text search +-- Enables similarity() function and % operator for fuzzy string matching. +-- Used by actor search, post search, and starter pack search to provide +-- typo-tolerant search results via trigram similarity scoring. +CREATE EXTENSION IF NOT EXISTS pg_trgm; + -- PostGIS: Geospatial extensions (required dependency for pgRouting) -- While Parakeet doesn't use geospatial features directly, this is a -- prerequisite for pgRouting's graph analysis capabilities. diff --git a/parakeet/src/loaders/list.rs b/parakeet/src/loaders/list.rs index 4fae6ccc..48627892 100644 --- a/parakeet/src/loaders/list.rs +++ b/parakeet/src/loaders/list.rs @@ -32,13 +32,12 @@ pub fn build_lists_batch_query() -> &'static str { l.description_facets, l.avatar_cid, l.status, - 'at://' || owner.did || '/app.bsky.graph.list/' || l.rkey::text as at_uri, owner.did as owner_did, COUNT(li.list_id)::bigint as item_count FROM lists l INNER JOIN actors owner ON l.actor_id = owner.id LEFT JOIN list_items li ON li.list_id = l.id - WHERE 'at://' || owner.did || '/app.bsky.graph.list/' || l.rkey::text = ANY($1) + WHERE 'at://' || owner.did || '/app.bsky.graph.list/' || i64_to_tid(l.rkey) = ANY($1) GROUP BY l.id, l.actor_id, l.rkey, l.cid, l.owner_actor_id, l.list_type, l.name, l.description, l.description_facets, l.avatar_cid, l.status, owner.did" } @@ -80,8 +79,6 @@ impl BatchFn for ListLoader { status: parakeet_db::types::RecordStatus, // Computed fields #[diesel(sql_type = diesel::sql_types::Text)] - at_uri: String, - #[diesel(sql_type = diesel::sql_types::Text)] owner_did: String, #[diesel(sql_type = diesel::sql_types::BigInt)] item_count: i64, @@ -99,6 +96,10 @@ impl BatchFn for ListLoader { }); HashMap::from_iter(lists_with_computed.into_iter().map(|l| { + // Encode TID using Rust utility function + let encoded_rkey = parakeet_db::tid_util::encode_tid(l.rkey); + let at_uri = format!("at://{}/app.bsky.graph.list/{}", l.owner_did, encoded_rkey); + let enriched = EnrichedList { list: models::List { id: l.id, @@ -113,12 +114,12 @@ impl BatchFn for ListLoader { avatar_cid: l.avatar_cid, status: l.status, }, - at_uri: l.at_uri.clone(), + at_uri: at_uri.clone(), owner: l.owner_did, cid: l.cid, created_at: l.created_at, // Computed from tid_timestamp(rkey) in SQL }; - (l.at_uri, (enriched, l.item_count)) + (at_uri, (enriched, l.item_count)) })) } } diff --git a/parakeet/src/loaders/misc.rs b/parakeet/src/loaders/misc.rs index fa769582..919e7df2 100644 --- a/parakeet/src/loaders/misc.rs +++ b/parakeet/src/loaders/misc.rs @@ -97,14 +97,14 @@ pub fn build_starterpacks_batch_query() -> &'static str { sp.description, sp.description_facets, sp.status, - 'at://' || owner.did || '/app.bsky.graph.starterpack/' || sp.rkey::text as at_uri, owner.did as owner_did, - 'at://' || list_owner.did || '/app.bsky.graph.list/' || l.rkey::text as list_uri + list_owner.did as list_owner_did, + l.rkey as list_rkey FROM starterpacks sp INNER JOIN actors owner ON sp.owner_actor_id = owner.id INNER JOIN lists l ON sp.list_id = l.id INNER JOIN actors list_owner ON l.actor_id = list_owner.id - WHERE 'at://' || owner.did || '/app.bsky.graph.starterpack/' || sp.rkey::text = ANY($1)" + WHERE 'at://' || owner.did || '/app.bsky.graph.starterpack/' || i64_to_tid(sp.rkey) = ANY($1)" } /// Build SQL query for resolving feed IDs to AT URIs @@ -154,11 +154,11 @@ impl BatchFn for StarterPackLoader { #[diesel(sql_type = parakeet_db::schema::sql_types::RecordStatus)] status: parakeet_db::types::RecordStatus, #[diesel(sql_type = diesel::sql_types::Text)] - at_uri: String, - #[diesel(sql_type = diesel::sql_types::Text)] owner_did: String, #[diesel(sql_type = diesel::sql_types::Text)] - list_uri: String, + list_owner_did: String, + #[diesel(sql_type = diesel::sql_types::BigInt)] + list_rkey: i64, } let starterpacks: Vec = diesel_async::RunQueryDsl::load( @@ -225,6 +225,12 @@ impl BatchFn for StarterPackLoader { }; HashMap::from_iter(starterpacks.into_iter().map(|row| { + // Encode TIDs using Rust utility functions + let encoded_rkey = parakeet_db::tid_util::encode_tid(row.rkey); + let encoded_list_rkey = parakeet_db::tid_util::encode_tid(row.list_rkey); + let at_uri = format!("at://{}/app.bsky.graph.starterpack/{}", row.owner_did, encoded_rkey); + let list_uri = format!("at://{}/app.bsky.graph.list/{}", row.list_owner_did, encoded_list_rkey); + let feeds = feeds_by_sp.get(&row.id).map(|feed_ids| { feed_ids.iter().filter_map(|id| feed_uris.get(id).cloned()).collect() }); @@ -234,7 +240,7 @@ impl BatchFn for StarterPackLoader { &row.name, &row.description, &row.description_facets, - &row.list_uri, + &list_uri, &feeds, &row.created_at, ); @@ -252,15 +258,15 @@ impl BatchFn for StarterPackLoader { description_facets: row.description_facets, status: row.status, }, - at_uri: row.at_uri.clone(), + at_uri: at_uri.clone(), owner: row.owner_did, - list: row.list_uri, + list: list_uri, feeds, cid: row.cid, created_at: row.created_at, // Computed from tid_timestamp(rkey) in SQL record, }; - (row.at_uri, enriched) + (at_uri, enriched) })) } } @@ -280,8 +286,7 @@ pub fn build_verifications_batch_query() -> &'static str { v.handle, v.display_name, subject_a.did as subject_did, - verifier_a.did as verifier_did, - 'at://' || subject_a.did || '/dev.bsky.verification.verification/' || v.rkey::text as at_uri + verifier_a.did as verifier_did FROM verification v INNER JOIN actors subject_a ON v.subject_actor_id = subject_a.id INNER JOIN actors verifier_a ON v.verifier_actor_id = verifier_a.id @@ -322,8 +327,6 @@ impl BatchFn> for VerificationLoader { subject_did: String, #[diesel(sql_type = diesel::sql_types::Text)] verifier_did: String, - #[diesel(sql_type = diesel::sql_types::Text)] - at_uri: String, } let verifications: Vec = diesel_async::RunQueryDsl::load( @@ -340,6 +343,10 @@ impl BatchFn> for VerificationLoader { // Group by subject DID let mut result: HashMap> = HashMap::new(); for row in verifications { + // Encode TID using Rust utility function + let encoded_rkey = parakeet_db::tid_util::encode_tid(row.rkey); + let at_uri = format!("at://{}/dev.bsky.verification.verification/{}", row.subject_did, encoded_rkey); + let enriched = EnrichedVerification { verification: models::Verification { id: row.id, @@ -352,7 +359,7 @@ impl BatchFn> for VerificationLoader { display_name: row.display_name, }, verifier: row.verifier_did, - at_uri: row.at_uri, + at_uri, created_at: row.created_at, // Computed from tid_timestamp(rkey) in SQL }; result.entry(row.subject_did).or_default().push(enriched); diff --git a/parakeet/src/loaders/profile.rs b/parakeet/src/loaders/profile.rs index 199d3316..6a1c609c 100644 --- a/parakeet/src/loaders/profile.rs +++ b/parakeet/src/loaders/profile.rs @@ -53,7 +53,7 @@ pub fn build_statuses_batch_query() -> &'static str { s.thumb_mime_type, s.thumb_cid, a.did, - (SELECT 'at://' || emb_a.did || '/app.bsky.feed.post/' || emb_p.rkey::text + (SELECT 'at://' || emb_a.did || '/app.bsky.feed.post/' || i64_to_tid(emb_p.rkey) FROM posts emb_p INNER JOIN actors emb_a ON emb_p.actor_id = emb_a.id WHERE emb_p.id = s.embed_post_id) as embed_uri -- 2.51.2