diff --git a/migrations/2025-12-06-200236_add_search_filter_indexes/down.sql b/migrations/2025-12-06-200236_add_search_filter_indexes/down.sql new file mode 100644 --- /dev/null +++ b/migrations/2025-12-06-200236_add_search_filter_indexes/down.sql @@ -0,0 +1,5 @@ +-- Rollback search filter indexes + +DROP INDEX IF EXISTS idx_posts_author_search; +DROP INDEX IF EXISTS idx_posts_lang_search; +DROP INDEX IF EXISTS idx_posts_mentions_search; diff --git a/migrations/2025-12-06-200236_add_search_filter_indexes/up.sql b/migrations/2025-12-06-200236_add_search_filter_indexes/up.sql new file mode 100644 --- /dev/null +++ b/migrations/2025-12-06-200236_add_search_filter_indexes/up.sql @@ -0,0 +1,28 @@ +-- Add composite indexes for common searchPosts filter combinations +-- These optimize queries using from:author and lang:language filters + +-- 1. Author search filter (from:) +-- Optimizes: searchPosts with from: parameter +-- Query: WHERE actor_id = ? AND tokens && ? AND status = 'complete' ORDER BY rkey DESC +CREATE INDEX IF NOT EXISTS idx_posts_author_search +ON posts (actor_id, rkey DESC) +WHERE status = 'complete'; + +-- 2. Language search filter (lang:) +-- Optimizes: searchPosts with lang: parameter +-- Query: WHERE langs @> ARRAY[?] AND tokens && ? AND status = 'complete' ORDER BY rkey DESC +-- Note: Uses GIN index on array containment (@>) for efficient language filtering +CREATE INDEX IF NOT EXISTS idx_posts_lang_search +ON posts USING gin (langs) +WHERE status = 'complete'; + +-- 3. Mentions search filter +-- Optimizes: searchPosts with mentions: parameter +-- Query: WHERE mentions @> ARRAY[?] AND tokens && ? ORDER BY rkey DESC +-- Note: Already has idx_posts_mentions_gin, but add partial index for complete posts +CREATE INDEX IF NOT EXISTS idx_posts_mentions_search +ON posts USING gin (mentions) +WHERE status = 'complete'; + +-- Note: The existing idx_posts_tokens_gin handles text search efficiently +-- These new indexes optimize common filter combinations used with text search diff --git a/parakeet/src/db.rs b/parakeet/src/db.rs --- a/parakeet/src/db.rs +++ b/parakeet/src/db.rs @@ -34,13 +34,12 @@ // Re-export commonly used functions pub use actors::{get_actor_ids_by_dids, get_actor_status, resolve_actor, ResolvedActor}; pub use bookmarks::get_user_bookmarks; -pub use feeds::{get_author_feed, get_list_feed, get_list_feed_by_ids, get_quotes, get_quotes_by_ids, get_reposted_by, get_reposted_by_ids, get_timeline_posts, get_timeline_posts_by_ids, get_timeline_reposts, get_timeline_reposts_by_ids, AuthorFeedFilter, AuthorFeedItem}; +pub use feeds::{get_author_feed, get_list_feed, get_list_feed_by_ids, get_quotes, get_quotes_by_ids, get_reposted_by, get_reposted_by_ids, get_timeline_posts, get_timeline_posts_by_ids, get_timeline_reposts, AuthorFeedFilter, AuthorFeedItem}; pub use feedgens::{get_actor_feedgens, get_all_feedgen_uris, get_feedgen_service_did}; pub use graph::{ - get_actor_followers, get_actor_followers_by_id, get_actor_follows, get_actor_follows_by_id, - get_actor_lists, get_followed_by_batch, get_following_batch, get_list_id_by_uri, get_list_items, - get_mutual_followers, get_mutual_followers_by_id, get_user_blocks, get_user_list_blocks, - get_user_list_mutes, get_user_mutes, + get_actor_followers, get_actor_follows, get_actor_lists, get_followed_by_batch, + get_following_batch, get_list_id_by_uri, get_list_items, get_mutual_followers, + get_user_blocks, get_user_list_blocks, get_user_list_mutes, get_user_mutes, }; pub use likes::{get_actor_likes, get_like_state, get_like_states, get_post_likes}; pub use notification_records::{get_follow_record, get_like_record, get_post_record, get_repost_record}; diff --git a/parakeet/src/id_cache_helpers.rs b/parakeet/src/id_cache_helpers.rs new file mode 100644 --- /dev/null +++ b/parakeet/src/id_cache_helpers.rs @@ -0,0 +1,284 @@ +//! IdCache helper functions with automatic database fallback +//! +//! These functions combine IdCache lookups with database queries, +//! automatically fetching and caching missing entries. + +use crate::xrpc::error::{Error, XrpcResult}; +use diesel::sql_types::{Array, Integer, Text}; +use diesel_async::pooled_connection::deadpool::Pool; +use diesel_async::{AsyncPgConnection, RunQueryDsl}; +use parakeet_db::id_cache::{CachedActor, CachedActorData, IdCache}; +use std::collections::HashMap; +use std::sync::Arc; + +/// Get actor_id for a DID, fetching from database if not cached +/// +/// This automatically: +/// 1. Checks IdCache for the DID +/// 2. If miss, queries database +/// 3. Updates cache with result +/// 4. Returns the actor_id +/// +/// # Errors +/// Returns error if DID not found in database +pub async fn get_actor_id_or_fetch( + pool: &Pool, + id_cache: &Arc, + did: &str, +) -> XrpcResult { + // Try cache first + if let Some(actor_id) = id_cache.get_actor_id_only(did).await { + return Ok(actor_id); + } + + // Cache miss - query database + let mut conn = pool.get().await.map_err(|e| { + Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to get database connection: {}", e)), + ) + })?; + + #[derive(diesel::QueryableByName)] + struct ActorRow { + #[diesel(sql_type = Integer)] + id: i32, + #[diesel(sql_type = diesel::sql_types::Nullable)] + handle: Option, + #[diesel(sql_type = diesel::sql_types::Text)] + sync_state: String, + } + + let actor: ActorRow = diesel::sql_query( + "SELECT id, handle, sync_state::text + FROM actors + WHERE did = $1" + ) + .bind::(did) + .get_result(&mut conn) + .await + .map_err(|e| match e { + diesel::result::Error::NotFound => { + Error::new( + axum::http::StatusCode::NOT_FOUND, + "ActorNotFound", + Some(format!("Actor not found: {}", did)), + ) + } + _ => Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to query actor: {}", e)), + ), + })?; + + // Cache both forward and reverse lookups + let is_allowlisted = matches!(actor.sync_state.as_str(), "synced" | "dirty" | "processing"); + + id_cache.set_actor_id( + did.to_string(), + CachedActor { + actor_id: actor.id, + is_allowlisted, + }, + ).await; + + id_cache.set_actor_data( + actor.id, + CachedActorData { + did: did.to_string(), + handle: actor.handle, + }, + ).await; + + Ok(actor.id) +} + +/// Get DIDs for multiple actor_ids, fetching from database for cache misses +/// +/// This automatically: +/// 1. Checks IdCache for each actor_id +/// 2. For cache misses, queries database in batch +/// 3. Updates cache with results +/// 4. Returns HashMap of actor_id → DID +/// +/// # Returns +/// HashMap with all requested actor_ids (omits IDs not found in database) +pub async fn get_actor_dids_or_fetch( + pool: &Pool, + id_cache: &Arc, + actor_ids: &[i32], +) -> XrpcResult> { + if actor_ids.is_empty() { + return Ok(HashMap::new()); + } + + // Try cache first + let cached = id_cache.get_actor_data_many(actor_ids).await; + let mut result: HashMap = cached + .into_iter() + .map(|(id, data)| (id, data.did)) + .collect(); + + // Find cache misses + let missing: Vec = actor_ids + .iter() + .filter(|id| !result.contains_key(id)) + .copied() + .collect(); + + if missing.is_empty() { + return Ok(result); + } + + // Query database for misses + let mut conn = pool.get().await.map_err(|e| { + Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to get database connection: {}", e)), + ) + })?; + + #[derive(diesel::QueryableByName)] + struct ActorRow { + #[diesel(sql_type = Integer)] + id: i32, + #[diesel(sql_type = Text)] + did: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + handle: Option, + } + + let db_results: Vec = diesel::sql_query( + "SELECT id, did, handle + FROM actors + WHERE id = ANY($1)" + ) + .bind::, _>(&missing) + .load(&mut conn) + .await + .map_err(|e| { + Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to resolve actor DIDs: {}", e)), + ) + })?; + + // Update cache and result + for row in db_results { + id_cache.set_actor_data( + row.id, + CachedActorData { + did: row.did.clone(), + handle: row.handle, + }, + ).await; + + result.insert(row.id, row.did); + } + + Ok(result) +} + +/// Get actor_ids for multiple DIDs, fetching from database for cache misses +/// +/// This automatically: +/// 1. Checks IdCache for each DID +/// 2. For cache misses, queries database in batch +/// 3. Updates cache with results +/// 4. Returns HashMap of DID → actor_id +/// +/// # Returns +/// HashMap with all requested DIDs (omits DIDs not found in database) +pub async fn get_actor_ids_or_fetch( + pool: &Pool, + id_cache: &Arc, + dids: &[String], +) -> XrpcResult> { + if dids.is_empty() { + return Ok(HashMap::new()); + } + + // Try cache first + let cached = id_cache.get_actor_ids(dids).await; + let mut result: HashMap = cached + .into_iter() + .map(|(did, cached)| (did, cached.actor_id)) + .collect(); + + // Find cache misses + let missing: Vec<&str> = dids + .iter() + .filter(|did| !result.contains_key(did.as_str())) + .map(|s| s.as_str()) + .collect(); + + if missing.is_empty() { + return Ok(result); + } + + // Query database for misses + let mut conn = pool.get().await.map_err(|e| { + Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to get database connection: {}", e)), + ) + })?; + + #[derive(diesel::QueryableByName)] + struct ActorRow { + #[diesel(sql_type = Integer)] + id: i32, + #[diesel(sql_type = Text)] + did: String, + #[diesel(sql_type = diesel::sql_types::Nullable)] + handle: Option, + #[diesel(sql_type = diesel::sql_types::Text)] + sync_state: String, + } + + let db_results: Vec = diesel::sql_query( + "SELECT id, did, handle, sync_state::text + FROM actors + WHERE did = ANY($1)" + ) + .bind::, _>(&missing) + .load(&mut conn) + .await + .map_err(|e| { + Error::new( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "DatabaseError", + Some(format!("Failed to resolve actor IDs: {}", e)), + ) + })?; + + // Update cache and result + for row in db_results { + let is_allowlisted = matches!(row.sync_state.as_str(), "synced" | "dirty" | "processing"); + + id_cache.set_actor_id( + row.did.clone(), + CachedActor { + actor_id: row.id, + is_allowlisted, + }, + ).await; + + id_cache.set_actor_data( + row.id, + CachedActorData { + did: row.did.clone(), + handle: row.handle, + }, + ).await; + + result.insert(row.did, row.id); + } + + Ok(result) +} diff --git a/parakeet/src/lib.rs b/parakeet/src/lib.rs --- a/parakeet/src/lib.rs +++ b/parakeet/src/lib.rs @@ -12,6 +12,7 @@ pub mod config; pub mod db; pub mod hydration; +pub mod id_cache_helpers; pub mod loaders; pub mod middleware; pub mod rate_limit; diff --git a/parakeet/src/db/feedgens.rs b/parakeet/src/db/feedgens.rs --- a/parakeet/src/db/feedgens.rs +++ b/parakeet/src/db/feedgens.rs @@ -6,41 +6,40 @@ /// Get feedgens owned by an actor, with cursor pagination /// -/// Returns list of (created_at, at_uri) tuples ordered by created_at DESC +/// Returns list of (created_at, actor_id, rkey) tuples ordered by created_at DESC pub async fn get_actor_feedgens( conn: &mut AsyncPgConnection, - owner_did: &str, + owner_actor_id: i32, cursor_timestamp: Option<&chrono::DateTime>, limit: u8, -) -> QueryResult, String)>> { - use diesel::sql_types::{BigInt, Nullable, Timestamptz}; +) -> QueryResult, i32, String)>> { + use diesel::sql_types::{BigInt, Integer, Nullable, Timestamptz}; #[derive(QueryableByName)] struct FeedgenRow { #[diesel(sql_type = Timestamptz)] created_at: chrono::DateTime, + #[diesel(sql_type = Integer)] + actor_id: i32, #[diesel(sql_type = Text)] - at_uri: String, + rkey: String, } let results: Vec = diesel::sql_query( - "SELECT f.created_at, - 'at://' || a.did || '/app.bsky.feed.generator/' || f.rkey::text as at_uri + "SELECT f.created_at, f.actor_id, f.rkey::text as rkey FROM feedgens f - INNER JOIN actors a ON f.actor_id = a.id - INNER JOIN actors owner ON f.owner_actor_id = owner.id - WHERE owner.did = $1 + WHERE f.owner_actor_id = $1 AND ($2::timestamptz IS NULL OR f.created_at < $2) ORDER BY f.created_at DESC LIMIT $3" ) - .bind::(owner_did) + .bind::(owner_actor_id) .bind::, _>(cursor_timestamp) .bind::(i64::from(limit)) .load(conn) .await?; - Ok(results.into_iter().map(|r| (r.created_at, r.at_uri)).collect()) + Ok(results.into_iter().map(|r| (r.created_at, r.actor_id, r.rkey)).collect()) } /// Get all feedgen URIs ordered by creation date diff --git a/parakeet/src/db/feeds.rs b/parakeet/src/db/feeds.rs --- a/parakeet/src/db/feeds.rs +++ b/parakeet/src/db/feeds.rs @@ -132,83 +132,6 @@ /// Get repost information for timeline posts /// -/// Returns list of (post_uri, reposter_did, indexed_at) tuples -pub async fn get_timeline_reposts( - conn: &mut AsyncPgConnection, - followed_dids: &[String], - post_uris: &[String], -) -> QueryResult)>> { - #[derive(QueryableByName)] - struct RepostRow { - #[diesel(sql_type = diesel::sql_types::Text)] - post_did: String, - #[diesel(sql_type = diesel::sql_types::BigInt)] - post_rkey: i64, - #[diesel(sql_type = diesel::sql_types::Text)] - reposter_did: String, - #[diesel(sql_type = diesel::sql_types::Timestamptz)] - indexed_at: chrono::DateTime, - } - - if followed_dids.is_empty() || post_uris.is_empty() { - return Ok(Vec::new()); - } - - // Parse post URIs to (did, rkey) tuples - let mut post_dids = Vec::new(); - let mut post_rkeys = Vec::new(); - - for uri in post_uris { - let parts: Vec<&str> = uri.trim_start_matches("at://").split('/').collect(); - if parts.len() >= 3 { - let did = parts[0]; - let rkey_base32 = parts[2]; - if let Ok(rkey_bigint) = parakeet_db::tid_util::decode_tid(rkey_base32) { - post_dids.push(did.to_string()); - post_rkeys.push(rkey_bigint); - } - } - } - - if post_dids.is_empty() { - return Ok(Vec::new()); - } - - // Use array parameter binding and UNNEST to prevent SQL injection - use diesel::sql_types::{Array, BigInt, Text}; - - diesel::sql_query( - "SELECT pa.did as post_did, p.rkey as post_rkey, - a.did as reposter_did, - tid_timestamp(r.rkey) as indexed_at - FROM reposts r - INNER JOIN actors a ON r.actor_id = a.id - INNER JOIN posts p ON r.post_actor_id = p.actor_id AND r.post_rkey = p.rkey - INNER JOIN actors pa ON p.actor_id = pa.id - INNER JOIN unnest($2::text[], $3::bigint[]) AS lookup(lookup_did, lookup_rkey) - ON pa.did = lookup.lookup_did AND p.rkey = lookup.lookup_rkey - WHERE a.did = ANY($1) - AND p.status = 'complete' - ORDER BY r.rkey DESC" - ) - .bind::, _>(followed_dids) - .bind::, _>(&post_dids) - .bind::, _>(&post_rkeys) - .load::(conn) - .await - .map(|rows| { - rows.into_iter() - .map(|r| { - let encoded_rkey = parakeet_db::tid_util::encode_tid(r.post_rkey); - let post_uri = format!("at://{}/app.bsky.feed.post/{}", r.post_did, encoded_rkey); - (post_uri, r.reposter_did, r.indexed_at) - }) - .collect() - }) -} - -/// Optimized get_timeline_reposts using actor IDs instead of DIDs (eliminates 2 actors JOINs) -/// /// Returns list of (post_actor_id, post_rkey, reposter_actor_id, indexed_at) tuples. /// The caller should resolve actor_ids → DIDs via IdCache and construct URIs. /// @@ -216,7 +139,7 @@ /// * `conn` - Database connection /// * `followed_actor_ids` - Actor IDs of followed users /// * `post_keys` - Post natural keys (actor_id, rkey) to filter by -pub async fn get_timeline_reposts_by_ids( +pub async fn get_timeline_reposts( conn: &mut AsyncPgConnection, followed_actor_ids: &[i32], post_keys: &[(i32, i64)], diff --git a/parakeet/src/db/graph.rs b/parakeet/src/db/graph.rs --- a/parakeet/src/db/graph.rs +++ b/parakeet/src/db/graph.rs @@ -1,7 +1,7 @@ //! Graph relationship queries (follows, mutes, blocks, lists) use diesel::prelude::*; -use diesel::sql_types::{BigInt, Integer, Nullable, Text, Timestamptz}; +use diesel::sql_types::{Array, BigInt, Integer, Nullable, Text, Timestamptz}; use diesel_async::{AsyncPgConnection, RunQueryDsl}; /// Get muted accounts for a user with cursor pagination @@ -72,91 +72,46 @@ /// Get blocked accounts for a user with cursor pagination /// -/// Returns list of (created_at, subject_did) tuples +/// Returns list of (created_at, subject_actor_id) tuples pub async fn get_user_blocks( conn: &mut AsyncPgConnection, - actor_did: &str, + actor_id: i32, cursor_timestamp: Option<&chrono::DateTime>, limit: u8, -) -> QueryResult, String)>> { +) -> QueryResult, i32)>> { #[derive(QueryableByName)] struct BlockRow { #[diesel(sql_type = Timestamptz)] created_at: chrono::DateTime, - #[diesel(sql_type = Text)] - subject_did: String, + #[diesel(sql_type = Integer)] + subject_actor_id: i32, } - // Use .bind() for cursor parameter to prevent SQL injection diesel::sql_query( - "SELECT DISTINCT ON (b.subject_actor_id) tid_timestamp(b.rkey) as created_at, subject.did as subject_did + "SELECT DISTINCT ON (b.subject_actor_id) tid_timestamp(b.rkey) as created_at, b.subject_actor_id FROM blocks b - INNER JOIN actors actor ON b.actor_id = actor.id - INNER JOIN actors subject ON b.subject_actor_id = subject.id - WHERE actor.did = $1 + WHERE b.actor_id = $1 AND ($2::timestamptz IS NULL OR tid_timestamp(b.rkey) < $2) ORDER BY b.subject_actor_id, b.rkey DESC LIMIT $3" ) - .bind::(actor_did) + .bind::(actor_id) .bind::, _>(cursor_timestamp) .bind::(i64::from(limit)) .load::(conn) .await .map(|rows| { rows.into_iter() - .map(|r| (r.created_at, r.subject_did)) + .map(|r| (r.created_at, r.subject_actor_id)) .collect() }) } /// Get followers of an actor with cursor pagination /// -/// Returns list of (rkey, follower_did) tuples ordered by rkey descending -pub async fn get_actor_followers( - conn: &mut AsyncPgConnection, - subject_did: &str, - cursor_rkey: Option, - limit: u8, -) -> QueryResult> { - #[derive(QueryableByName)] - struct FollowerRow { - #[diesel(sql_type = BigInt)] - rkey: i64, - #[diesel(sql_type = Text)] - follower_did: String, - } - - // Use .bind() for cursor parameter to prevent SQL injection - diesel::sql_query( - "SELECT DISTINCT ON (f.actor_id) f.rkey, follower.did as follower_did - FROM follows f - INNER JOIN actors subject ON f.subject_actor_id = subject.id - INNER JOIN actors follower ON f.actor_id = follower.id - WHERE subject.did = $1 - AND ($2::bigint IS NULL OR f.rkey < $2) - ORDER BY f.actor_id, f.rkey DESC - LIMIT $3" - ) - .bind::(subject_did) - .bind::, _>(cursor_rkey) - .bind::(i64::from(limit)) - .load::(conn) - .await - .map(|rows| { - rows.into_iter() - .map(|r| (r.rkey, r.follower_did)) - .collect() - }) -} - -/// Get followers of an actor with cursor pagination (OPTIMIZED VERSION) -/// /// Returns list of (rkey, follower_actor_id) tuples ordered by rkey descending. /// The caller should resolve follower_actor_ids → DIDs via IdCache. -/// -/// This version eliminates the 2x actors table JOINs that the non-optimized version has. -pub async fn get_actor_followers_by_id( +pub async fn get_actor_followers( conn: &mut AsyncPgConnection, subject_actor_id: i32, cursor_rkey: Option, @@ -193,51 +148,9 @@ /// Get accounts followed by an actor with cursor pagination /// -/// Returns list of (rkey, subject_did) tuples ordered by rkey descending -pub async fn get_actor_follows( - conn: &mut AsyncPgConnection, - actor_did: &str, - cursor_rkey: Option, - limit: u8, -) -> QueryResult> { - #[derive(QueryableByName)] - struct FollowRow { - #[diesel(sql_type = BigInt)] - rkey: i64, - #[diesel(sql_type = Text)] - subject_did: String, - } - - // Use .bind() for cursor parameter to prevent SQL injection - diesel::sql_query( - "SELECT DISTINCT ON (f.subject_actor_id) f.rkey, subject.did as subject_did - FROM follows f - INNER JOIN actors actor ON f.actor_id = actor.id - INNER JOIN actors subject ON f.subject_actor_id = subject.id - WHERE actor.did = $1 - AND ($2::bigint IS NULL OR f.rkey < $2) - ORDER BY f.subject_actor_id, f.rkey DESC - LIMIT $3" - ) - .bind::(actor_did) - .bind::, _>(cursor_rkey) - .bind::(i64::from(limit)) - .load::(conn) - .await - .map(|rows| { - rows.into_iter() - .map(|r| (r.rkey, r.subject_did)) - .collect() - }) -} - -/// Get accounts followed by an actor with cursor pagination (OPTIMIZED VERSION) -/// /// Returns list of (rkey, subject_actor_id) tuples ordered by rkey descending. /// The caller should resolve subject_actor_ids → DIDs via IdCache. -/// -/// This version eliminates the 2x actors table JOINs that the non-optimized version has. -pub async fn get_actor_follows_by_id( +pub async fn get_actor_follows( conn: &mut AsyncPgConnection, actor_id: i32, cursor_rkey: Option, @@ -274,125 +187,75 @@ /// Get follow relationships for batch queries /// -/// Returns (target_did, follower_did, rkey) tuples for actor following others +/// Returns (target_actor_id, follower_actor_id, rkey) tuples for actor following others pub async fn get_following_batch( conn: &mut AsyncPgConnection, - actor_did: &str, - other_dids: &[&str], -) -> QueryResult> { + actor_id: i32, + other_actor_ids: &[i32], +) -> QueryResult> { #[derive(QueryableByName)] struct FollowingRow { - #[diesel(sql_type = diesel::sql_types::Text)] - target_did: String, - #[diesel(sql_type = diesel::sql_types::Text)] - follower_did: String, + #[diesel(sql_type = Integer)] + target_actor_id: i32, + #[diesel(sql_type = Integer)] + follower_actor_id: i32, #[diesel(sql_type = diesel::sql_types::Text)] rkey: String, } diesel::sql_query( - "SELECT subject.did as target_did, actor.did as follower_did, f.rkey::text as rkey + "SELECT f.subject_actor_id as target_actor_id, f.actor_id as follower_actor_id, f.rkey::text as rkey FROM follows f - INNER JOIN actors actor ON f.actor_id = actor.id - INNER JOIN actors subject ON f.subject_actor_id = subject.id - WHERE actor.did = $1 AND subject.did = ANY($2)" + WHERE f.actor_id = $1 AND f.subject_actor_id = ANY($2)" ) - .bind::(actor_did) - .bind::, _>(other_dids) + .bind::(actor_id) + .bind::, _>(other_actor_ids) .load::(conn) .await .map(|rows| { rows.into_iter() - .map(|r| (r.target_did, r.follower_did, r.rkey)) + .map(|r| (r.target_actor_id, r.follower_actor_id, r.rkey)) .collect() }) } /// Get followed-by relationships for batch queries /// -/// Returns (follower_did, rkey) tuples for others following actor +/// Returns (follower_actor_id, rkey) tuples for others following actor pub async fn get_followed_by_batch( conn: &mut AsyncPgConnection, - actor_did: &str, - other_dids: &[&str], -) -> QueryResult> { + actor_id: i32, + other_actor_ids: &[i32], +) -> QueryResult> { #[derive(QueryableByName)] struct FollowedByRow { - #[diesel(sql_type = diesel::sql_types::Text)] - follower_did: String, + #[diesel(sql_type = Integer)] + follower_actor_id: i32, #[diesel(sql_type = diesel::sql_types::Text)] rkey: String, } diesel::sql_query( - "SELECT actor.did as follower_did, f.rkey::text as rkey + "SELECT f.actor_id as follower_actor_id, f.rkey::text as rkey FROM follows f - INNER JOIN actors subject ON f.subject_actor_id = subject.id - INNER JOIN actors actor ON f.actor_id = actor.id - WHERE subject.did = $1 AND actor.did = ANY($2)" + WHERE f.subject_actor_id = $1 AND f.actor_id = ANY($2)" ) - .bind::(actor_did) - .bind::, _>(other_dids) + .bind::(actor_id) + .bind::, _>(other_actor_ids) .load::(conn) .await .map(|rows| { rows.into_iter() - .map(|r| (r.follower_did, r.rkey)) + .map(|r| (r.follower_actor_id, r.rkey)) .collect() }) } /// Get known followers (mutual follows intersection) /// -/// Returns list of (created_at, follower_did) tuples for followers that viewer also follows -pub async fn get_mutual_followers( - conn: &mut AsyncPgConnection, - target_did: &str, - viewer_did: &str, - cursor_timestamp: Option<&chrono::DateTime>, - limit: u8, -) -> QueryResult, String)>> { - #[derive(QueryableByName)] - struct MutualFollowerRow { - #[diesel(sql_type = Timestamptz)] - created_at: chrono::DateTime, - #[diesel(sql_type = Text)] - follower_did: String, - } - - // Use .bind() for cursor parameter to prevent SQL injection - diesel::sql_query( - "SELECT DISTINCT ON (f1.actor_id) tid_timestamp(f1.rkey) as created_at, follower.did as follower_did - FROM follows f1 - INNER JOIN actors target ON f1.subject_actor_id = target.id - INNER JOIN actors follower ON f1.actor_id = follower.id - INNER JOIN follows f2 ON f1.actor_id = f2.subject_actor_id - INNER JOIN actors viewer ON f2.actor_id = viewer.id - WHERE target.did = $1 AND viewer.did = $2 - AND ($3::timestamptz IS NULL OR tid_timestamp(f1.rkey) < $3) - ORDER BY f1.actor_id, f1.rkey DESC - LIMIT $4" - ) - .bind::(target_did) - .bind::(viewer_did) - .bind::, _>(cursor_timestamp) - .bind::(i64::from(limit)) - .load::(conn) - .await - .map(|rows| { - rows.into_iter() - .map(|r| (r.created_at, r.follower_did)) - .collect() - }) -} - -/// Get known followers (mutual follows intersection) - OPTIMIZED VERSION -/// /// Returns list of (created_at, follower_actor_id) tuples for followers that viewer also follows. /// The caller should resolve follower_actor_ids → DIDs via IdCache. -/// -/// This version eliminates the 3x actors table JOINs that the non-optimized version has. -pub async fn get_mutual_followers_by_id( +pub async fn get_mutual_followers( conn: &mut AsyncPgConnection, target_actor_id: i32, viewer_actor_id: i32, @@ -433,10 +296,10 @@ /// Get lists owned by an actor with cursor pagination /// -/// Returns list of (created_at, at_uri) tuples +/// Returns list of (created_at, rkey) tuples pub async fn get_actor_lists( conn: &mut AsyncPgConnection, - actor_did: &str, + actor_id: i32, cursor_timestamp: Option<&chrono::DateTime>, limit: u8, ) -> QueryResult, String)>> { @@ -444,8 +307,6 @@ struct ListRow { #[diesel(sql_type = Text)] rkey: String, - #[diesel(sql_type = Text)] - did: String, } // Convert cursor timestamp to rkey for comparison (if provided) @@ -455,17 +316,15 @@ parakeet_db::tid_util::encode_tid(rkey) }); - // Use .bind() for cursor parameter to prevent SQL injection let rows = diesel::sql_query( - "SELECT l.rkey, a.did + "SELECT l.rkey FROM lists l - INNER JOIN actors a ON l.actor_id = a.id - WHERE a.did = $1 + WHERE l.actor_id = $1 AND ($2::text IS NULL OR l.rkey < $2) ORDER BY l.rkey DESC LIMIT $3" ) - .bind::(actor_did) + .bind::(actor_id) .bind::, _>(cursor_rkey_str.as_deref()) .bind::(i64::from(limit)) .load::(conn) @@ -477,8 +336,7 @@ .filter_map(|r| { let rkey_bigint = parakeet_db::tid_util::decode_tid(&r.rkey).ok()?; let created_at = parakeet_db::tid_util::tid_to_datetime(rkey_bigint); - let at_uri = format!("at://{}/app.bsky.graph.list/{}", r.did, r.rkey); - Some((created_at, at_uri)) + Some((created_at, r.rkey)) }) .collect()) } diff --git a/parakeet/src/db/likes.rs b/parakeet/src/db/likes.rs --- a/parakeet/src/db/likes.rs +++ b/parakeet/src/db/likes.rs @@ -136,21 +136,21 @@ /// Get likes by an actor with cursor pagination /// -/// Returns list of (created_at, subject_uri) tuples +/// Returns list of (created_at, post_actor_id, subject_rkey) tuples pub async fn get_actor_likes( conn: &mut AsyncPgConnection, - actor_did: &str, + actor_id: i32, cursor_timestamp: Option<&chrono::DateTime>, limit: u8, -) -> QueryResult, String)>> { - use diesel::sql_types::{BigInt, Nullable, Timestamptz}; +) -> QueryResult, i32, i64)>> { + use diesel::sql_types::{BigInt, Integer, Nullable, Timestamptz}; #[derive(QueryableByName)] struct ActorLikeRow { #[diesel(sql_type = Timestamptz)] created_at: chrono::DateTime, - #[diesel(sql_type = Text)] - subject_did: String, + #[diesel(sql_type = Integer)] + post_actor_id: i32, #[diesel(sql_type = BigInt)] subject_rkey: i64, #[diesel(sql_type = BigInt)] @@ -161,31 +161,25 @@ // Extract like rkey using array_position, use tid_timestamp for ordering/cursor diesel::sql_query( "SELECT - tid_timestamp(p.like_rkeys[array_position(p.like_actor_ids, a.id)]) as created_at, - pa.did as subject_did, + tid_timestamp(p.like_rkeys[array_position(p.like_actor_ids, $1)]) as created_at, + p.actor_id as post_actor_id, p.rkey as subject_rkey, - p.like_rkeys[array_position(p.like_actor_ids, a.id)] as like_rkey + p.like_rkeys[array_position(p.like_actor_ids, $1)] as like_rkey FROM posts p - INNER JOIN actors pa ON p.actor_id = pa.id - INNER JOIN actors a ON a.did = $1 WHERE p.status = 'complete' - AND p.like_actor_ids @> ARRAY[a.id] - AND ($2::timestamptz IS NULL OR tid_timestamp(p.like_rkeys[array_position(p.like_actor_ids, a.id)]) < $2) + AND p.like_actor_ids @> ARRAY[$1] + AND ($2::timestamptz IS NULL OR tid_timestamp(p.like_rkeys[array_position(p.like_actor_ids, $1)]) < $2) ORDER BY like_rkey DESC LIMIT $3" ) - .bind::(actor_did) + .bind::(actor_id) .bind::, _>(cursor_timestamp) .bind::(i64::from(limit)) .load::(conn) .await .map(|rows| { rows.into_iter() - .map(|r| { - let subject_rkey_str = parakeet_db::tid_util::encode_tid(r.subject_rkey); - let subject_uri = format!("at://{}/app.bsky.feed.post/{}", r.subject_did, subject_rkey_str); - (r.created_at, subject_uri) - }) + .map(|r| (r.created_at, r.post_actor_id, r.subject_rkey)) .collect() }) } diff --git a/parakeet/src/db/starterpacks.rs b/parakeet/src/db/starterpacks.rs --- a/parakeet/src/db/starterpacks.rs +++ b/parakeet/src/db/starterpacks.rs @@ -1,7 +1,6 @@ //! Starter pack queries use diesel::prelude::*; -use diesel::sql_types::Text; use diesel_async::{AsyncPgConnection, RunQueryDsl}; /// Get all starterpacks with their owners @@ -48,48 +47,41 @@ /// Get starterpacks owned by an actor with cursor pagination /// -/// Returns list of (created_at, at_uri) tuples +/// Returns list of (created_at, actor_id, rkey) tuples pub async fn get_owner_starterpacks( conn: &mut AsyncPgConnection, - owner_did: &str, + owner_actor_id: i32, cursor_timestamp: Option<&chrono::DateTime>, limit: u8, -) -> QueryResult, String)>> { - use diesel::sql_types::{BigInt, Nullable, Timestamptz}; +) -> QueryResult, i32, i64)>> { + use diesel::sql_types::{BigInt, Integer, Nullable, Timestamptz}; #[derive(QueryableByName)] - struct StarterPackWithUri { + struct StarterPackRow { #[diesel(sql_type = Timestamptz)] created_at: chrono::DateTime, - #[diesel(sql_type = Text)] - did: String, + #[diesel(sql_type = Integer)] + actor_id: i32, #[diesel(sql_type = BigInt)] rkey: i64, } - // Use .bind() for cursor parameter to prevent SQL injection diesel::sql_query( - "SELECT tid_timestamp(sp.rkey) as created_at, a.did, sp.rkey + "SELECT tid_timestamp(sp.rkey) as created_at, sp.actor_id, sp.rkey FROM starterpacks sp - INNER JOIN actors a ON sp.actor_id = a.id - INNER JOIN actors owner ON sp.owner_actor_id = owner.id - WHERE owner.did = $1 + WHERE sp.owner_actor_id = $1 AND ($2::timestamptz IS NULL OR tid_timestamp(sp.rkey) < $2) ORDER BY sp.rkey DESC LIMIT $3" ) - .bind::(owner_did) + .bind::(owner_actor_id) .bind::, _>(cursor_timestamp) .bind::(i64::from(limit)) - .load::(conn) + .load::(conn) .await .map(|rows| { rows.into_iter() - .map(|r| { - let encoded_rkey = parakeet_db::tid_util::encode_tid(r.rkey); - let at_uri = format!("at://{}/app.bsky.graph.starterpack/{}", r.did, encoded_rkey); - (r.created_at, at_uri) - }) + .map(|r| (r.created_at, r.actor_id, r.rkey)) .collect() }) } diff --git a/parakeet/src/db/suggestions.rs b/parakeet/src/db/suggestions.rs --- a/parakeet/src/db/suggestions.rs +++ b/parakeet/src/db/suggestions.rs @@ -29,34 +29,31 @@ .map(|rows| rows.into_iter().map(|(did, _count)| did).collect()) } -/// Get DIDs that a viewer follows +/// Get actor_ids that a viewer follows /// -/// Returns list of DIDs that the given viewer_did follows +/// Returns list of subject_actor_ids that the given viewer follows /// /// OPTIMIZED: Queries follows table directly by actor_id without joins. /// Since follows is now partitioned by actor_id, this hits only 1 chunk. -/// The actor_id resolution (DID → id) should use IdCache to avoid decompressing actors chunks. pub async fn get_followed_dids( conn: &mut AsyncPgConnection, - viewer_did: &str, -) -> QueryResult> { + viewer_actor_id: i32, +) -> QueryResult> { #[derive(QueryableByName)] - struct DidRow { - #[diesel(sql_type = diesel::sql_types::Text)] - did: String, + struct FollowRow { + #[diesel(sql_type = diesel::sql_types::Integer)] + subject_actor_id: i32, } diesel::sql_query( - "SELECT a2.did - FROM follows f - INNER JOIN actors a1 ON f.actor_id = a1.id - INNER JOIN actors a2 ON f.subject_actor_id = a2.id - WHERE a1.did = $1" + "SELECT subject_actor_id + FROM follows + WHERE actor_id = $1" ) - .bind::(viewer_did) - .load::(conn) + .bind::(viewer_actor_id) + .load::(conn) .await - .map(|rows| rows.into_iter().map(|r| r.did).collect()) + .map(|rows| rows.into_iter().map(|r| r.subject_actor_id).collect()) } /// Get DIDs that a viewer follows (cached version) @@ -176,37 +173,36 @@ /// Get suggested follows using collaborative filtering /// /// Finds accounts followed by the target actor's followers (accounts similar to target) +/// Returns list of suggested actor_ids pub async fn get_collaborative_filter_suggestions( conn: &mut AsyncPgConnection, - actor_did: &str, -) -> QueryResult> { + actor_id: i32, +) -> QueryResult> { #[derive(QueryableByName)] struct CandidateRow { - #[diesel(sql_type = diesel::sql_types::Text)] - candidate_did: String, + #[diesel(sql_type = diesel::sql_types::Integer)] + candidate_actor_id: i32, } diesel::sql_query( "WITH actor_followers AS ( SELECT DISTINCT f.actor_id FROM follows f - INNER JOIN actors a ON f.subject_actor_id = a.id - WHERE a.did = $1 + WHERE f.subject_actor_id = $1 LIMIT 1000 ), mutual_follows AS ( - SELECT DISTINCT a2.did as candidate_did + SELECT DISTINCT f.subject_actor_id as candidate_actor_id FROM follows f INNER JOIN actor_followers af ON f.actor_id = af.actor_id - INNER JOIN actors a2 ON f.subject_actor_id = a2.id - WHERE a2.did != $1 + WHERE f.subject_actor_id != $1 LIMIT 500 ) - SELECT candidate_did + SELECT candidate_actor_id FROM mutual_follows" ) - .bind::(actor_did) + .bind::(actor_id) .load::(conn) .await - .map(|rows| rows.into_iter().map(|r| r.candidate_did).collect()) + .map(|rows| rows.into_iter().map(|r| r.candidate_actor_id).collect()) } diff --git a/parakeet/src/xrpc/mod.rs b/parakeet/src/xrpc/mod.rs --- a/parakeet/src/xrpc/mod.rs +++ b/parakeet/src/xrpc/mod.rs @@ -3,7 +3,7 @@ mod com_atproto; mod community_lexicon; pub mod cursor; -mod error; +pub mod error; pub mod extract; pub mod helpers; pub mod jwt; diff --git a/parakeet/src/xrpc/app_bsky/feed/feedgen.rs b/parakeet/src/xrpc/app_bsky/feed/feedgen.rs --- a/parakeet/src/xrpc/app_bsky/feed/feedgen.rs +++ b/parakeet/src/xrpc/app_bsky/feed/feedgen.rs @@ -27,6 +27,13 @@ let did = get_actor_did(&state.dataloaders, query.actor).await?; + // Resolve DID → actor_id (auto-fetches from DB if not cached) + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &did, + ).await?; + check_actor_status(&mut conn, &did).await?; let limit = query.limit.unwrap_or(50).clamp(1, 100); @@ -34,7 +41,7 @@ let cursor_timestamp = datetime_cursor(query.cursor.as_ref()); let results = crate::db::get_actor_feedgens( &mut conn, - &did, + actor_id, cursor_timestamp.as_ref(), limit, ) @@ -44,13 +51,32 @@ .last() .map(|last| last.0.timestamp_millis().to_string()); - let at_uris = results.iter().map(|r| r.1.clone()).collect(); + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) + let actor_ids: Vec = results.iter().map(|r| r.1).collect(); + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; + + // Construct AT-URIs from DIDs + rkeys + let at_uris: Vec = results + .iter() + .filter_map(|r| { + let did = actor_id_to_did.get(&r.1)?; + Some(format!("at://{}/app.bsky.feed.generator/{}", did, r.2)) + }) + .collect(); let mut feeds = hyd.hydrate_feedgens(at_uris).await; let feeds = results .into_iter() - .filter_map(|r| feeds.remove(&r.1)) + .filter_map(|r| { + let did = actor_id_to_did.get(&r.1)?; + let at_uri = format!("at://{}/app.bsky.feed.generator/{}", did, r.2); + feeds.remove(&at_uri) + }) .collect(); Ok(Json(GetActorFeedRes { cursor, feeds })) diff --git a/parakeet/src/xrpc/app_bsky/feed/get_timeline.rs b/parakeet/src/xrpc/app_bsky/feed/get_timeline.rs --- a/parakeet/src/xrpc/app_bsky/feed/get_timeline.rs +++ b/parakeet/src/xrpc/app_bsky/feed/get_timeline.rs @@ -59,7 +59,7 @@ Some(auth.clone()), ); - let feed = hydrate_timeline_feed(&hyd, cached.post_uris, &state, &user_did).await; + let feed = hydrate_timeline_feed(&hyd, cached.post_uris, &state, cached_actor.actor_id).await; let hydrate_time = step_timer.elapsed().as_secs_f64() * 1000.0; tracing::info!(" ├─ Hydrate cached feed: {:.1} ms", hydrate_time); @@ -91,38 +91,24 @@ Some(auth.clone()), ); - // Get the accounts the user follows (uses IdCache to avoid decompressing actors chunks) + // Resolve user DID → actor_id step_timer = std::time::Instant::now(); - let follows = crate::db::get_followed_dids_cached(&mut conn, &state.id_cache, &user_did).await?; + let user_actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &user_did, + ).await?; + let user_resolve_time = step_timer.elapsed().as_secs_f64() * 1000.0; + if user_resolve_time >= 1.0 { + tracing::info!(" ├─ Resolve user actor_id: {:.1} ms", user_resolve_time); + } + + // Get the accounts the user follows (returns actor_ids directly) + step_timer = std::time::Instant::now(); + let followed_actor_ids = crate::db::get_followed_dids(&mut conn, user_actor_id).await?; let follows_time = step_timer.elapsed().as_secs_f64() * 1000.0; if follows_time >= 1.0 { - tracing::info!(" ├─ Get followed DIDs: {:.1} ms ({} follows)", follows_time, follows.len()); - } - - if follows.is_empty() { - // Return empty feed if user doesn't follow anyone - let total_time = start.elapsed().as_secs_f64() * 1000.0; - tracing::info!(" └─ getTimeline total (no follows): {:.1} ms", total_time); - return Ok(Json(GetTimelineRes { - cursor: None, - feed: Vec::new(), - })); - } - - // Convert followed DIDs to actor_ids using id_cache (avoids slow JOIN) - step_timer = std::time::Instant::now(); - let mut actor_id_to_did = std::collections::HashMap::new(); - let mut followed_actor_ids = Vec::with_capacity(follows.len()); - - for did in &follows { - if let Some(cached) = state.id_cache.get_actor_id(did).await { - actor_id_to_did.insert(cached.actor_id, did.clone()); - followed_actor_ids.push(cached.actor_id); - } - } - let conversion_time = step_timer.elapsed().as_secs_f64() * 1000.0; - if conversion_time >= 1.0 { - tracing::info!(" ├─ Convert DIDs to actor_ids: {:.1} ms ({} actors)", conversion_time, followed_actor_ids.len()); + tracing::info!(" ├─ Get followed actor_ids: {:.1} ms ({} follows)", follows_time, followed_actor_ids.len()); } if followed_actor_ids.is_empty() { @@ -150,19 +136,26 @@ let timeline_query_time = step_timer.elapsed().as_secs_f64() * 1000.0; tracing::info!(" ├─ Timeline query: {:.1} ms ({} posts)", timeline_query_time, results.len()); + // Batch resolve post actor_ids → DIDs + step_timer = std::time::Instant::now(); + let post_actor_ids: Vec = results.iter().map(|(_, actor_id, _)| *actor_id).collect(); + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &post_actor_ids, + ).await?; + let did_resolve_time = step_timer.elapsed().as_secs_f64() * 1000.0; + if did_resolve_time >= 1.0 { + tracing::info!(" ├─ Resolve post actor DIDs: {:.1} ms ({} actors)", did_resolve_time, actor_id_to_did.len()); + } + // Construct AT URIs from (actor_id, rkey) using our DID mapping // Track the last successfully converted post for cursor pagination let mut last_timestamp: Option> = None; - let mut skipped_posts = 0; let at_uris: Vec = results .iter() .filter_map(|(created_at, actor_id, rkey)| { - let did = actor_id_to_did.get(actor_id); - if did.is_none() { - skipped_posts += 1; - return None; - } - let did = did.unwrap(); + let did = actor_id_to_did.get(actor_id)?; let encoded_rkey = parakeet_db::tid_util::encode_tid(*rkey); last_timestamp = Some(*created_at); // Track last successful conversion Some(format!("at://{}/app.bsky.feed.post/{}", did, encoded_rkey)) @@ -170,6 +163,7 @@ .collect(); // Warn if we skipped posts due to missing DIDs (indicates id_cache inconsistency) + let skipped_posts = results.len() - at_uris.len(); if skipped_posts > 0 { tracing::warn!(" ⚠ Skipped {} posts due to missing DIDs in actor_id→DID mapping (got {} posts from DB, returning {} posts)", skipped_posts, results.len(), at_uris.len()); @@ -196,7 +190,7 @@ let (mut post_views, reposts_results) = tokio::join!( hyd.hydrate_posts(at_uris.clone()), async { - crate::db::get_timeline_reposts_by_ids(&mut conn, &followed_actor_ids, &post_keys) + crate::db::get_timeline_reposts(&mut conn, &followed_actor_ids, &post_keys) .await .unwrap_or_default() } @@ -204,59 +198,18 @@ let hydrate_time = step_timer.elapsed().as_secs_f64() * 1000.0; tracing::info!(" ├─ Hydrate posts + get reposts: {:.1} ms", hydrate_time); - // OPTIMIZATION: Batch resolve actor_ids → DIDs via IdCache (with DB fallback) + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) let all_actor_ids: std::collections::HashSet = reposts_results .iter() .flat_map(|(post_actor_id, _, reposter_actor_id, _)| vec![*post_actor_id, *reposter_actor_id]) .collect(); let actor_ids_vec: Vec = all_actor_ids.into_iter().collect(); - // Try cache first - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - #[diesel(sql_type = diesel::sql_types::Nullable)] - handle: Option, - } - - if let Ok(db_results) = diesel::sql_query("SELECT id, did, handle FROM actors WHERE id = ANY($1)") - .bind::, _>(&missing) - .load::(&mut conn) - .await - { - // Populate cache and map - for row in db_results { - actor_id_to_did.insert(row.id, row.did.clone()); - state.id_cache.set_actor_data( - row.id, - parakeet_db::id_cache::CachedActorData { - did: row.did, - handle: row.handle, - }, - ).await; - } - } - } + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids_vec, + ).await?; // Find repost information - construct post URIs and map reposter IDs to DIDs let mut repost_data = HashMap::new(); @@ -351,7 +304,7 @@ hyd: &StatefulHydrator<'_>, at_uris: Vec, state: &GlobalState, - user_did: &str, + user_actor_id: i32, ) -> Vec { if at_uris.is_empty() { return Vec::new(); @@ -387,11 +340,11 @@ } }; - // Parallelize: hydrate posts and get followed DIDs concurrently + // Parallelize: hydrate posts and get followed actor_ids concurrently let (mut post_views, follows) = tokio::join!( hyd.hydrate_posts(at_uris.clone()), async { - crate::db::get_followed_dids(&mut conn, user_did) + crate::db::get_followed_dids(&mut conn, user_actor_id) .await .unwrap_or_default() } @@ -424,14 +377,56 @@ // Find repost information let mut repost_data = HashMap::new(); - let reposts_results = crate::db::get_timeline_reposts(&mut conn, &follows, &at_uris) + // Parse AT-URIs to (actor_id, rkey) tuples for repost query + let mut post_keys = Vec::new(); + for uri in &at_uris { + let parts: Vec<&str> = uri.trim_start_matches("at://").split('/').collect(); + if parts.len() >= 3 { + let did = parts[0]; + let rkey_base32 = parts[2]; + if let Ok(rkey) = parakeet_db::tid_util::decode_tid(rkey_base32) { + // Resolve DID → actor_id + if let Ok(actor_id) = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + did, + ).await { + post_keys.push((actor_id, rkey)); + } + } + } + } + + let reposts_results = crate::db::get_timeline_reposts(&mut conn, &follows, &post_keys) .await .unwrap_or_default(); + // Batch resolve all actor_ids to DIDs for reposts + let mut all_actor_ids: Vec = reposts_results + .iter() + .flat_map(|(post_actor_id, _, reposter_actor_id, _)| vec![*post_actor_id, *reposter_actor_id]) + .collect(); + all_actor_ids.sort_unstable(); + all_actor_ids.dedup(); + + let actor_id_to_did_map = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &all_actor_ids, + ).await.unwrap_or_default(); + // Group by post_uri and take the most recent repost for each post - for (post_uri, reposter_did, indexed_at) in reposts_results { - if let std::collections::hash_map::Entry::Vacant(e) = repost_data.entry(post_uri) { - let _ = e.insert((reposter_did, indexed_at)); + for (post_actor_id, post_rkey, reposter_actor_id, indexed_at) in reposts_results { + if let (Some(post_did), Some(reposter_did)) = ( + actor_id_to_did_map.get(&post_actor_id), + actor_id_to_did_map.get(&reposter_actor_id) + ) { + let rkey_str = parakeet_db::tid_util::encode_tid(post_rkey); + let post_uri = format!("at://{}/app.bsky.feed.post/{}", post_did, rkey_str); + + if let std::collections::hash_map::Entry::Vacant(e) = repost_data.entry(post_uri) { + let _ = e.insert((reposter_did.clone(), indexed_at)); + } } } diff --git a/parakeet/src/xrpc/app_bsky/feed/likes.rs b/parakeet/src/xrpc/app_bsky/feed/likes.rs --- a/parakeet/src/xrpc/app_bsky/feed/likes.rs +++ b/parakeet/src/xrpc/app_bsky/feed/likes.rs @@ -32,20 +32,41 @@ let limit = query.limit.unwrap_or(50).clamp(1, 100); + // Resolve actor DID → actor_id + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &query.actor, + ).await?; + // Query actor likes let cursor_value = datetime_cursor(query.cursor.as_ref()); - let results = crate::db::get_actor_likes(&mut conn, &query.actor, cursor_value.as_ref(), limit).await?; + let results = crate::db::get_actor_likes(&mut conn, actor_id, cursor_value.as_ref(), limit).await?; // Generate cursor in ISO 8601 format (matches official API) let cursor = results .last() .map(|row| row.0.to_rfc3339()); + // Batch resolve post_actor_ids → DIDs + let actor_ids: Vec = results.iter().map(|r| r.1).collect(); + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; + + // Construct AT-URIs from resolved DIDs + rkeys let raw_feed = results .iter() - .map(|row| RawFeedItem::Post { - uri: row.1.clone(), - context: None, + .filter_map(|row| { + let did = actor_id_to_did.get(&row.1)?; + let rkey_str = parakeet_db::tid_util::encode_tid(row.2); + let uri = format!("at://{}/app.bsky.feed.post/{}", did, rkey_str); + Some(RawFeedItem::Post { + uri, + context: None, + }) }) .collect::>(); diff --git a/parakeet/src/xrpc/app_bsky/feed/search.rs b/parakeet/src/xrpc/app_bsky/feed/search.rs --- a/parakeet/src/xrpc/app_bsky/feed/search.rs +++ b/parakeet/src/xrpc/app_bsky/feed/search.rs @@ -7,7 +7,6 @@ use axum::extract::{Query, State}; use axum::response::{IntoResponse as _, Response}; use axum::Json; -use diesel_async::RunQueryDsl; use lexica::app_bsky::feed::PostView; use serde::{Deserialize, Serialize}; @@ -165,63 +164,13 @@ &results[..] }; - // Batch resolve actor_ids → DIDs via IdCache (with DB fallback) + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) let actor_ids: Vec = results_to_return.iter().map(|r| r.actor_id).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); - - // Try cache first - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - #[diesel(sql_type = diesel::sql_types::Nullable)] - handle: Option, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did, handle FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve actor DIDs: {}", e)), - ) - })?; - - // Populate cache and map - for row in db_results { - actor_id_to_did.insert(row.id, row.did.clone()); - state.id_cache.set_actor_data( - row.id, - parakeet_db::id_cache::CachedActorData { - did: row.did, - handle: row.handle, - }, - ).await; - } - } + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; // Construct URIs in Rust (maintaining search order) let uris: Vec = results_to_return diff --git a/parakeet/src/xrpc/app_bsky/graph/lists.rs b/parakeet/src/xrpc/app_bsky/graph/lists.rs --- a/parakeet/src/xrpc/app_bsky/graph/lists.rs +++ b/parakeet/src/xrpc/app_bsky/graph/lists.rs @@ -37,6 +37,13 @@ let did = get_actor_did(&state.dataloaders, query.actor).await?; + // Resolve DID → actor_id (auto-fetches from DB if not cached) + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &did, + ).await?; + let limit = query.limit.unwrap_or(50).clamp(1, 100); // Query actor lists @@ -46,7 +53,7 @@ let mut conn2 = state.pool.get().await?; let (status_result, results) = tokio::join!( check_actor_status(&mut conn, &did), - crate::db::get_actor_lists(&mut conn2, &did, cursor_value.as_ref(), limit) + crate::db::get_actor_lists(&mut conn2, actor_id, cursor_value.as_ref(), limit) ); status_result?; @@ -56,13 +63,20 @@ .last() .map(|last| last.0.timestamp_millis().to_string()); - let at_uris = results.iter().map(|r| r.1.clone()).collect(); + // Construct AT-URIs from DID + rkeys + let at_uris: Vec = results + .iter() + .map(|r| format!("at://{}/app.bsky.graph.list/{}", did, r.1)) + .collect(); let mut lists = hyd.hydrate_lists(at_uris).await; let lists = results .into_iter() - .filter_map(|r| lists.remove(&r.1)) + .filter_map(|r| { + let at_uri = format!("at://{}/app.bsky.graph.list/{}", did, r.1); + lists.remove(&at_uri) + }) .collect(); Ok(Json(GetListsRes { cursor, lists })) diff --git a/parakeet/src/xrpc/app_bsky/graph/relations.rs b/parakeet/src/xrpc/app_bsky/graph/relations.rs --- a/parakeet/src/xrpc/app_bsky/graph/relations.rs +++ b/parakeet/src/xrpc/app_bsky/graph/relations.rs @@ -32,15 +32,33 @@ // Parse cursor once let parsed_cursor = datetime_cursor(query.cursor.as_ref()); - // Query blocked accounts - let results = crate::db::get_user_blocks(&mut conn, &did, parsed_cursor.as_ref(), limit).await?; + // Resolve DID → actor_id (auto-fetches from DB if not cached) + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &did, + ).await?; + + // Query blocked accounts (returns actor_ids) + let results = crate::db::get_user_blocks(&mut conn, actor_id, parsed_cursor.as_ref(), limit).await?; // Generate cursor in ISO 8601 format (matches official API) let cursor = results .last() .map(|row| row.0.to_rfc3339()); - let dids = results.iter().map(|row| row.1.clone()).collect(); + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) + let actor_ids: Vec = results.iter().map(|row| row.1).collect(); + let dids_map = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; + + let dids: Vec = actor_ids + .into_iter() + .filter_map(|id| dids_map.get(&id).cloned()) + .collect(); let profiles = hyd.hydrate_profiles(dids).await; let blocks = profiles.into_values().collect::>(); @@ -72,95 +90,45 @@ // Parse TID cursor let parsed_cursor = crate::xrpc::tid_cursor(query.cursor.as_ref()); - // OPTIMIZATION: Resolve subject DID → actor_id via IdCache to use optimized query - let subject_actor_id_opt = state.id_cache.get_actor_id_only(&subj_did).await; + // Resolve subject DID → actor_id (auto-fetches from DB if not cached) + let subject_actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &subj_did, + ).await?; - // Hydrate subject profile first (needed either way) + // Hydrate subject profile let subject_opt = hyd.hydrate_profile(subj_did.clone()).await; let Some(subject) = subject_opt else { return Err(Error::not_found()); }; - // Query followers using optimized or non-optimized path - let (cursor, dids) = if let Some(subject_actor_id) = subject_actor_id_opt { - // OPTIMIZED PATH: Use _by_id version (0 JOINs!) - let results = crate::db::get_actor_followers_by_id(&mut conn, subject_actor_id, parsed_cursor, limit).await?; + // Query followers using optimized _by_id version (0 JOINs!) + let results = crate::db::get_actor_followers( + &mut conn, + subject_actor_id, + parsed_cursor, + limit, + ).await?; - // Generate cursor from TID (base32-encoded rkey) - let cursor = results - .last() - .map(|row| parakeet_db::tid_util::encode_tid(row.0)); + // Generate cursor from TID (base32-encoded rkey) + let cursor = results + .last() + .map(|row| parakeet_db::tid_util::encode_tid(row.0)); - // Batch resolve actor_ids → DIDs via IdCache - let actor_ids: Vec = results.iter().map(|row| row.1).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) + let actor_ids: Vec = results.iter().map(|row| row.1).collect(); + let dids_map = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; - // Batch resolve via IdCache - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve follower DIDs: {}", e)), - ) - })?; - - for row in db_results { - actor_id_to_did.insert(row.id, row.did); - } - } - - // Map actor_ids to DIDs, preserving order - let dids: Vec = actor_ids - .into_iter() - .filter_map(|id| actor_id_to_did.get(&id).cloned()) - .collect(); - - (cursor, dids) - } else { - // NON-OPTIMIZED PATH: Use original version with JOINs - let results = crate::db::get_actor_followers(&mut conn, &subj_did, parsed_cursor, limit).await?; - - // Generate cursor from TID (base32-encoded rkey) - let cursor = results - .last() - .map(|row| parakeet_db::tid_util::encode_tid(row.0)); - - let dids = results.iter().map(|row| row.1.clone()).collect(); - - (cursor, dids) - }; + // Map actor_ids to DIDs, preserving order + let dids: Vec = actor_ids + .into_iter() + .filter_map(|id| dids_map.get(&id).cloned()) + .collect(); let mut profiles = hyd.hydrate_profiles(dids.clone()).await; @@ -200,95 +168,45 @@ // Parse TID cursor let parsed_cursor = crate::xrpc::tid_cursor(query.cursor.as_ref()); - // OPTIMIZATION: Resolve subject DID → actor_id via IdCache to use optimized query - let subject_actor_id_opt = state.id_cache.get_actor_id_only(&subj_did).await; + // Resolve subject DID → actor_id (auto-fetches from DB if not cached) + let subject_actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &subj_did, + ).await?; - // Hydrate subject profile first (needed either way) + // Hydrate subject profile let subject_opt = hyd.hydrate_profile(subj_did.clone()).await; let Some(subject) = subject_opt else { return Err(Error::not_found()); }; - // Query follows using optimized or non-optimized path - let (cursor, dids) = if let Some(subject_actor_id) = subject_actor_id_opt { - // OPTIMIZED PATH: Use _by_id version (0 JOINs!) - let results = crate::db::get_actor_follows_by_id(&mut conn, subject_actor_id, parsed_cursor, limit).await?; + // Query follows using optimized _by_id version (0 JOINs!) + let results = crate::db::get_actor_follows( + &mut conn, + subject_actor_id, + parsed_cursor, + limit, + ).await?; - // Generate cursor from TID (base32-encoded rkey) - let cursor = results - .last() - .map(|row| parakeet_db::tid_util::encode_tid(row.0)); + // Generate cursor from TID (base32-encoded rkey) + let cursor = results + .last() + .map(|row| parakeet_db::tid_util::encode_tid(row.0)); - // Batch resolve actor_ids → DIDs via IdCache - let actor_ids: Vec = results.iter().map(|row| row.1).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) + let actor_ids: Vec = results.iter().map(|row| row.1).collect(); + let dids_map = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; - // Batch resolve via IdCache - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve followed DIDs: {}", e)), - ) - })?; - - for row in db_results { - actor_id_to_did.insert(row.id, row.did); - } - } - - // Map actor_ids to DIDs, preserving order - let dids: Vec = actor_ids - .into_iter() - .filter_map(|id| actor_id_to_did.get(&id).cloned()) - .collect(); - - (cursor, dids) - } else { - // NON-OPTIMIZED PATH: Use original version with JOINs - let results = crate::db::get_actor_follows(&mut conn, &subj_did, parsed_cursor, limit).await?; - - // Generate cursor from TID (base32-encoded rkey) - let cursor = results - .last() - .map(|row| parakeet_db::tid_util::encode_tid(row.0)); - - let dids = results.iter().map(|row| row.1.clone()).collect(); - - (cursor, dids) - }; + // Map actor_ids to DIDs, preserving order + let dids: Vec = actor_ids + .into_iter() + .filter_map(|id| dids_map.get(&id).cloned()) + .collect(); let mut profiles = hyd.hydrate_profiles(dids.clone()).await; @@ -332,7 +250,14 @@ query.others }; - // Step 3: Resolve all "others" to DIDs, track failures + // Step 3: Resolve actor DID → actor_id (auto-fetches from DB if not cached) + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &actor_did, + ).await?; + + // Step 4: Resolve all "others" to DIDs, track failures let mut resolved_others: Vec<(String, String)> = Vec::new(); // (original, did) let mut failed_others: Vec = Vec::new(); @@ -343,42 +268,73 @@ } } - // Step 4: Query relationships in batch - let mut conn = state.pool.get().await?; - let other_dids: Vec<&str> = resolved_others + // Step 5: Batch resolve other DIDs → actor_ids (auto-fetches from DB) + let other_dids: Vec = resolved_others .iter() - .map(|(_, did)| did.as_str()) + .map(|(_, did)| did.clone()) .collect(); + let other_dids_to_ids = crate::id_cache_helpers::get_actor_ids_or_fetch( + &state.pool, + &state.id_cache, + &other_dids, + ).await?; + let other_actor_ids: Vec = other_dids + .iter() + .filter_map(|did| other_dids_to_ids.get(did).copied()) + .collect(); + + // Step 6: Query relationships in batch + let mut conn = state.pool.get().await?; // Parallelize: query following and followed_by relationships concurrently let mut conn2 = state.pool.get().await?; let (following_result, followed_by_result) = tokio::join!( - crate::db::get_following_batch(&mut conn, &actor_did, &other_dids), - crate::db::get_followed_by_batch(&mut conn2, &actor_did, &other_dids) + crate::db::get_following_batch(&mut conn, actor_id, &other_actor_ids), + crate::db::get_followed_by_batch(&mut conn2, actor_id, &other_actor_ids) ); let following_rows = following_result?; let followed_by_rows = followed_by_result?; + // Step 7: Batch resolve all actor_ids from results → DIDs + let mut all_actor_ids: std::collections::HashSet = std::collections::HashSet::new(); + all_actor_ids.insert(actor_id); + for (target_id, follower_id, _) in &following_rows { + all_actor_ids.insert(*target_id); + all_actor_ids.insert(*follower_id); + } + for (follower_id, _) in &followed_by_rows { + all_actor_ids.insert(*follower_id); + } + let actor_ids_vec: Vec = all_actor_ids.into_iter().collect(); + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids_vec, + ).await?; + // Build maps for O(1) lookup, constructing AT-URIs let following_map: HashMap = following_rows .into_iter() - .map(|(target_did, follower_did, rkey)| { + .filter_map(|(target_actor_id, follower_actor_id, rkey)| { + let target_did = actor_id_to_did.get(&target_actor_id)?; + let follower_did = actor_id_to_did.get(&follower_actor_id)?; let uri = format!( "at://{}/app.bsky.graph.follow/{}", follower_did, rkey ); - (target_did, uri) + Some((target_did.clone(), uri)) }) .collect(); let followed_by_map: HashMap = followed_by_rows .into_iter() - .map(|(follower_did, rkey)| { + .filter_map(|(follower_actor_id, rkey)| { + let follower_did = actor_id_to_did.get(&follower_actor_id)?; let uri = format!( "at://{}/app.bsky.graph.follow/{}", follower_did, rkey ); - (follower_did, uri) + Some((follower_did.clone(), uri)) }) .collect(); @@ -432,92 +388,49 @@ let limit = query.limit.unwrap_or(50).clamp(1, 100); let parsed_cursor = datetime_cursor(query.cursor.as_ref()); - // OPTIMIZATION: Resolve target and viewer DIDs → actor_ids via IdCache to use optimized query - let target_actor_id_opt = state.id_cache.get_actor_id_only(&target_did).await; - let viewer_actor_id_opt = state.id_cache.get_actor_id_only(&viewer_did).await; + // Resolve target and viewer DIDs → actor_ids (auto-fetches from DB if not cached) + let target_actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &target_did, + ).await?; + let viewer_actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &viewer_did, + ).await?; - // Hydrate subject profile first (needed either way) + // Hydrate subject profile let subject_opt = hyd.hydrate_profile(target_did.clone()).await; let Some(subject) = subject_opt else { return Err(Error::not_found()); }; - // Query known followers using optimized or non-optimized path - let (cursor, dids) = if let (Some(target_actor_id), Some(viewer_actor_id)) = (target_actor_id_opt, viewer_actor_id_opt) { - // OPTIMIZED PATH: Use _by_id version (eliminates 3 actors JOINs!) - let results = crate::db::get_mutual_followers_by_id(&mut conn, target_actor_id, viewer_actor_id, parsed_cursor.as_ref(), limit).await?; + // Query known followers using optimized _by_id version (eliminates 3 actors JOINs!) + let results = crate::db::get_mutual_followers( + &mut conn, + target_actor_id, + viewer_actor_id, + parsed_cursor.as_ref(), + limit, + ).await?; - // Generate cursor - let cursor = results.last().map(|row| row.0.to_rfc3339()); + // Generate cursor + let cursor = results.last().map(|row| row.0.to_rfc3339()); - // Batch resolve actor_ids → DIDs via IdCache - let actor_ids: Vec = results.iter().map(|row| row.1).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) + let actor_ids: Vec = results.iter().map(|row| row.1).collect(); + let dids_map = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; - // Batch resolve via IdCache - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve known follower DIDs: {}", e)), - ) - })?; - - for row in db_results { - actor_id_to_did.insert(row.id, row.did); - } - } - - // Map actor_ids to DIDs, preserving order - let dids: Vec = actor_ids - .into_iter() - .filter_map(|id| actor_id_to_did.get(&id).cloned()) - .collect(); - - (cursor, dids) - } else { - // NON-OPTIMIZED PATH: Use original version with 3 actors JOINs - let results = crate::db::get_mutual_followers(&mut conn, &target_did, &viewer_did, parsed_cursor.as_ref(), limit).await?; - - // Generate cursor - let cursor = results.last().map(|row| row.0.to_rfc3339()); - - let dids = results.iter().map(|row| row.1.clone()).collect(); - - (cursor, dids) - }; + // Map actor_ids to DIDs, preserving order + let dids: Vec = actor_ids + .into_iter() + .filter_map(|id| dids_map.get(&id).cloned()) + .collect(); // Hydrate profiles for known followers let mut profiles = hyd.hydrate_profiles(dids.clone()).await; diff --git a/parakeet/src/xrpc/app_bsky/graph/starter_packs.rs b/parakeet/src/xrpc/app_bsky/graph/starter_packs.rs --- a/parakeet/src/xrpc/app_bsky/graph/starter_packs.rs +++ b/parakeet/src/xrpc/app_bsky/graph/starter_packs.rs @@ -28,25 +28,48 @@ let subj_did = get_actor_did(&state.dataloaders, query.actor).await?; + // Resolve DID → actor_id + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &subj_did, + ).await?; + check_actor_status(&mut conn, &subj_did).await?; let limit = query.limit.unwrap_or(50).clamp(1, 100); // Query starterpacks owned by the actor let cursor_value = datetime_cursor(query.cursor.as_ref()); - let results = crate::db::get_owner_starterpacks(&mut conn, &subj_did, cursor_value.as_ref(), limit).await?; + let results = crate::db::get_owner_starterpacks(&mut conn, actor_id, cursor_value.as_ref(), limit).await?; let cursor = results .last() .map(|last| last.0.timestamp_millis().to_string()); - let uris = results.iter().map(|r| r.1.clone()).collect(); + // Batch resolve actor_ids → DIDs + let actor_ids: Vec = results.iter().map(|r| r.1).collect(); + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; - let mut starter_packs = hyd.hydrate_starterpacks_basic(uris).await; + // Construct AT-URIs from resolved DIDs + rkeys + let uris: Vec = results + .iter() + .filter_map(|r| { + let did = actor_id_to_did.get(&r.1)?; + let rkey_str = parakeet_db::tid_util::encode_tid(r.2); + Some(format!("at://{}/app.bsky.graph.starterpack/{}", did, rkey_str)) + }) + .collect(); - let starter_packs = results + let mut starter_packs = hyd.hydrate_starterpacks_basic(uris.clone()).await; + + let starter_packs = uris .into_iter() - .filter_map(|r| starter_packs.remove(&r.1)) + .filter_map(|uri| starter_packs.remove(&uri)) .collect(); Ok(Json(StarterPacksRes { diff --git a/parakeet/src/xrpc/app_bsky/graph/suggestions.rs b/parakeet/src/xrpc/app_bsky/graph/suggestions.rs --- a/parakeet/src/xrpc/app_bsky/graph/suggestions.rs +++ b/parakeet/src/xrpc/app_bsky/graph/suggestions.rs @@ -35,6 +35,13 @@ // Resolve actor identifier to DID let actor_did = get_actor_did(&state.dataloaders, query.actor).await?; + // Resolve DID → actor_id + let actor_id = crate::id_cache_helpers::get_actor_id_or_fetch( + &state.pool, + &state.id_cache, + &actor_did, + ).await?; + // Compute suggestions // TODO: Consider adding moka cache if this becomes a bottleneck @@ -49,8 +56,27 @@ // Execute collaborative filtering query let mut conn = state.pool.get().await?; - let candidate_dids = - crate::db::get_collaborative_filter_suggestions(&mut conn, &actor_did).await?; + let candidate_actor_ids = + crate::db::get_collaborative_filter_suggestions(&mut conn, actor_id).await?; + + if candidate_actor_ids.is_empty() { + return Ok(Json(GetSuggestedFollowsByActorRes { + suggestions: Vec::new(), + })); + } + + // Batch resolve candidate actor_ids → DIDs + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &candidate_actor_ids, + ).await?; + + // Convert to DIDs for stats loading + let candidate_dids: Vec = candidate_actor_ids + .iter() + .filter_map(|id| actor_id_to_did.get(id).cloned()) + .collect(); if candidate_dids.is_empty() { return Ok(Json(GetSuggestedFollowsByActorRes { diff --git a/parakeet/src/xrpc/app_bsky/feed/posts/queries.rs b/parakeet/src/xrpc/app_bsky/feed/posts/queries.rs --- a/parakeet/src/xrpc/app_bsky/feed/posts/queries.rs +++ b/parakeet/src/xrpc/app_bsky/feed/posts/queries.rs @@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize}; use crate::hydration::StatefulHydrator; -use crate::xrpc::error::{Error, XrpcResult}; +use crate::xrpc::error::XrpcResult; use crate::xrpc::extract::{AtpAcceptLabelers, AtpAuth}; use crate::xrpc::{datetime_cursor, normalise_at_uri}; use crate::GlobalState; @@ -131,65 +131,13 @@ ) .await?; - // Batch resolve actor_ids → DIDs via IdCache (with DB fallback) + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) let actor_ids: Vec = results.iter().map(|(actor_id, _)| *actor_id).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); - - // Try cache first - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - #[diesel(sql_type = diesel::sql_types::Nullable)] - handle: Option, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did, handle FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve actor DIDs: {}", e)), - ) - })?; - - // Populate cache and map - for row in db_results { - actor_id_to_did.insert(row.id, row.did.clone()); - state.id_cache.set_actor_data( - row.id, - parakeet_db::id_cache::CachedActorData { - did: row.did, - handle: row.handle, - }, - ).await; - } - } + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; // Construct URIs in Rust (maintaining order) let uris: Vec = results @@ -298,65 +246,13 @@ // Call optimized function (0 actors JOINs!) let results = crate::db::get_reposted_by_ids(&mut conn, post_actor_id, post_rkey, cursor_rkey, limit).await?; - // Batch resolve actor_ids → DIDs via IdCache (with DB fallback) + // Batch resolve actor_ids → DIDs (auto-fetches from DB for cache misses) let actor_ids: Vec = results.iter().map(|(actor_id, _)| *actor_id).collect(); - let unique_actor_ids: std::collections::HashSet = actor_ids.iter().copied().collect(); - let actor_ids_vec: Vec = unique_actor_ids.into_iter().collect(); - - // Try cache first - let cached = state.id_cache.get_actor_data_many(&actor_ids_vec).await; - let mut actor_id_to_did: std::collections::HashMap = cached - .into_iter() - .map(|(id, data)| (id, data.did)) - .collect(); - - // Query DB for cache misses - let missing: Vec = actor_ids_vec - .iter() - .filter(|id| !actor_id_to_did.contains_key(id)) - .copied() - .collect(); - - if !missing.is_empty() { - use diesel::sql_types::{Array, Integer, Text}; - use diesel_async::RunQueryDsl; - - #[derive(diesel::QueryableByName)] - struct ActorRow { - #[diesel(sql_type = Integer)] - id: i32, - #[diesel(sql_type = Text)] - did: String, - #[diesel(sql_type = diesel::sql_types::Nullable)] - handle: Option, - } - - let db_results: Vec = diesel::sql_query( - "SELECT id, did, handle FROM actors WHERE id = ANY($1)" - ) - .bind::, _>(&missing) - .load(&mut conn) - .await - .map_err(|e| { - Error::new( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - "DatabaseError", - Some(format!("Failed to resolve actor DIDs: {}", e)), - ) - })?; - - // Populate cache and map - for row in db_results { - actor_id_to_did.insert(row.id, row.did.clone()); - state.id_cache.set_actor_data( - row.id, - parakeet_db::id_cache::CachedActorData { - did: row.did, - handle: row.handle, - }, - ).await; - } - } + let actor_id_to_did = crate::id_cache_helpers::get_actor_dids_or_fetch( + &state.pool, + &state.id_cache, + &actor_ids, + ).await?; // Build result tuples (timestamp, DID) maintaining order let results_with_dids: Vec<(chrono::DateTime, String)> = results