diff --git a/consumer/src/database_writer/mod.rs b/consumer/src/database_writer/mod.rs index be7d7209..227fc635 100644 --- a/consumer/src/database_writer/mod.rs +++ b/consumer/src/database_writer/mod.rs @@ -124,9 +124,6 @@ pub struct ProcessedEvent { /// Database operations to perform pub operations: Vec, - /// Cache keys to invalidate (legacy field) - pub cache_invalidations: Vec, - /// Cursor update for this event (batched saves every 10s) /// Contains Jetstream cursor timestamp in microseconds pub cursor: Option, diff --git a/consumer/src/database_writer/operations/executor.rs b/consumer/src/database_writer/operations/executor.rs index 141035b8..482bd26c 100644 --- a/consumer/src/database_writer/operations/executor.rs +++ b/consumer/src/database_writer/operations/executor.rs @@ -12,7 +12,7 @@ //! Counts are maintained automatically by database triggers. //! Operations return PostStatsDeltas for compatibility. -use super::{cache, DatabaseOperation}; +use super::DatabaseOperation; use ipld_core::cid::Cid; pub fn describe_operation(op: &DatabaseOperation) -> (String, Option) { @@ -117,12 +117,11 @@ pub fn describe_operation(op: &DatabaseOperation) -> (String, Option) { } } -/// Execute a single database operation and return cache invalidations +/// Execute a single database operation /// /// This function will be called by the database writer for each operation. -/// It returns cache invalidations that should be sent via NOTIFY for real-time cache updates. /// -/// Counts are maintained by database triggers. +/// Counts and cache invalidations are maintained by database triggers. /// Notifications, fetch queue, and PDS cache use PostgreSQL/moka. pub async fn execute_operation( pool: &deadpool_postgres::Pool, @@ -130,7 +129,7 @@ pub async fn execute_operation( op: DatabaseOperation, allowlist: &crate::db::Allowlist, pds_cache: &crate::external::pds_cache::PdsHostCache, -) -> eyre::Result> { +) -> eyre::Result<()> { use crate::db; match op { @@ -178,8 +177,6 @@ pub async fn execute_operation( ); } if rows > 0 { - let mut cache_invalidations = Vec::new(); - // If this is a reply, increment parent post's reply count if let Some(parent_uri) = parent_uri.as_ref() { // Resolve parent URI to natural key @@ -194,8 +191,7 @@ pub async fn execute_operation( AND NOT ($3 = ANY(COALESCE(reply_actor_ids, ARRAY[])))", &[&parent_actor_id, &parent_rkey, &actor_id, &rkey], ).await; - // Invalidate parent post cache since reply array changed - cache_invalidations.push(cache::invalidate_post(parent_uri)); + // Database trigger handles cache invalidation } } @@ -213,14 +209,13 @@ pub async fn execute_operation( AND NOT ($3 = ANY(COALESCE(quote_actor_ids, ARRAY[])))", &["ed_actor_id, "ed_rkey, &actor_id, &rkey], ).await; - // Invalidate quoted post cache since quote array changed - cache_invalidations.push(cache::invalidate_post(quoted_uri)); + // Database trigger handles cache invalidation } } - Ok(cache_invalidations) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertLike { @@ -250,13 +245,11 @@ pub async fn execute_operation( if rows > 0 { // Note: like arrays are updated directly in db::like_insert via LikeArrayOp // No deltas needed with array-only tracking + // Database trigger handles cache invalidation - // Invalidate post cache since like array changed - let cache_invalidations = vec![cache::invalidate_post(&subject_uri)]; - - Ok(cache_invalidations) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertRepost { @@ -288,12 +281,11 @@ pub async fn execute_operation( // Note: repost arrays will be updated in db::repost_insert (dual-write to reposts table + post arrays) // No deltas needed with array-only tracking - // Invalidate post cache since repost array changed - let cache_invalidations = vec![cache::invalidate_post(&subject_uri)]; + // Database trigger handles cache invalidation - Ok(cache_invalidations) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertFollow { @@ -306,9 +298,9 @@ pub async fn execute_operation( let rows = db::follow_insert(conn, rkey, actor_id, subject_actor_id, cid, record).await?; if rows > 0 { // Counts maintained by triggers - Ok(vec![]) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertBlock { @@ -319,7 +311,7 @@ pub async fn execute_operation( record, } => { db::block_insert(conn, rkey, actor_id, subject_actor_id, cid, record).await?; - Ok(vec![]) // Blocks don't affect aggregates (yet) + Ok(()) // Blocks don't affect aggregates (yet) } DatabaseOperation::InsertNotification { recipient_actor_id, @@ -335,7 +327,7 @@ pub async fn execute_operation( } => { // Skip self-notifications (author notifying themselves) if recipient_actor_id == author_actor_id { - return Ok(vec![]); + return Ok(()); } // Query recipient DID for allowlist check @@ -343,12 +335,12 @@ pub async fn execute_operation( let Some(recipient_did) = recipient_did else { tracing::debug!("Skipping notification - recipient actor_id not found: {}", recipient_actor_id); - return Ok(vec![]); + return Ok(()); }; // PostgreSQL-based notifications (only for allowlisted recipients) if !allowlist.cache.contains_did(&recipient_did) { - return Ok(vec![]); + return Ok(()); } // Check if thread is muted (for reply, quote, and mention notifications) @@ -363,7 +355,7 @@ pub async fn execute_operation( subj_actor_id, subj_rkey ); - return Ok(vec![]); + return Ok(()); } } @@ -400,7 +392,7 @@ pub async fn execute_operation( ); } - Ok(vec![]) // Notifications don't affect aggregates + Ok(()) // Notifications don't affect aggregates } DatabaseOperation::NotifyReplyChain { reply_uri, @@ -642,7 +634,7 @@ pub async fn execute_operation( ); } - Ok(vec![]) // Notifications don't affect aggregates + Ok(()) // Notifications don't affect aggregates } DatabaseOperation::UpsertActor { did, @@ -662,12 +654,12 @@ pub async fn execute_operation( timestamp, ) .await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::EnqueueFetch { uri, .. } => { // Enqueue to PostgreSQL fetch_queue table crate::db::fetch_queue::enqueue(conn, &uri).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertProfile { actor_id, cid, record } => { // Query DID from actor_id (needed by db functions) @@ -692,14 +684,14 @@ pub async fn execute_operation( crate::db::handle_resolution_queue::enqueue(pool, &did).await?; } - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertStatus { actor_id, cid, record } => { // Query DID from actor_id (needed by db functions) let did = db::actor_did_from_id(conn, actor_id).await?; db::status_upsert(conn, actor_id, &did, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertPostgate { rkey, @@ -708,7 +700,7 @@ pub async fn execute_operation( record, } => { db::postgate_upsert(conn, actor_id, rkey, cid, &record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertThreadgate { rkey, @@ -717,7 +709,7 @@ pub async fn execute_operation( record, } => { db::threadgate_upsert(conn, actor_id, rkey, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertList { rkey, @@ -729,9 +721,9 @@ pub async fn execute_operation( let inserted = db::list_upsert(conn, actor_id, &rkey, cid, record).await?; if inserted { // lists_count maintained by application - Ok(vec![]) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertListItem { @@ -742,7 +734,7 @@ pub async fn execute_operation( record, } => { db::list_item_insert(conn, actor_id, rkey, subject_actor_id, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::InsertListBlock { rkey, @@ -751,7 +743,7 @@ pub async fn execute_operation( record, } => { db::list_block_insert(conn, actor_id, rkey, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertFeedGenerator { rkey, @@ -764,9 +756,9 @@ pub async fn execute_operation( let inserted = db::feedgen_upsert(conn, actor_id, &rkey, cid, service_actor_id, record).await?; if inserted { // feeds_count maintained by application - Ok(vec![]) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::UpsertStarterPack { @@ -778,9 +770,9 @@ pub async fn execute_operation( let inserted = db::starter_pack_upsert(conn, actor_id, rkey, cid, record).await?; if inserted { // starterpacks_count maintained by application - Ok(vec![]) + Ok(()) } else { - Ok(vec![]) + Ok(()) } } DatabaseOperation::InsertVerification { @@ -791,26 +783,26 @@ pub async fn execute_operation( record, } => { db::verification_insert(conn, actor_id, rkey, subject_actor_id, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertLabeler { actor_id, cid, record } => { // Query DID from actor_id (needed by db functions) let did = db::actor_did_from_id(conn, actor_id).await?; db::labeler_upsert(conn, actor_id, &did, cid, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertNotificationDeclaration { actor_id, record } => { db::notif_decl_upsert(conn, actor_id, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertChatDeclaration { actor_id, record } => { db::chat_decl_upsert(conn, actor_id, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::UpsertBookmark { actor_id, rkey, record } => { db::bookmark_upsert(conn, actor_id, rkey, record).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::DeleteRecord { actor_id, @@ -820,8 +812,6 @@ pub async fn execute_operation( } => { use crate::relay::types::CollectionType; - let mut cache_invalidations = Vec::new(); - // actor_id is already resolved in the resolution phase // No database lookup needed here! @@ -840,7 +830,7 @@ pub async fn execute_operation( let post_uri = format!("at://{}/app.bsky.feed.post/{}", post_did, parakeet_db::models::i64_to_tid(post_rkey)); - cache_invalidations.push(cache::invalidate_post(&post_uri)); + // Database trigger handles cache invalidation // Remove PostgreSQL notification (recipient = author of liked post) if allowlist.cache.contains_did(&post_did) { @@ -863,20 +853,11 @@ pub async fn execute_operation( } } } - db::LikeSubject::Feedgen { actor_id: feedgen_actor_id, rkey: feedgen_rkey } => { - // Invalidate feedgen cache (no delta needed - feedgen_likes is separate table) - if let Ok(feedgen_did) = db::actor_did_from_id(conn, feedgen_actor_id).await { - let feedgen_uri = format!("at://{}/app.bsky.feed.generator/{}", - feedgen_did, feedgen_rkey); - cache_invalidations.push(cache::invalidate_feedgen(&feedgen_uri)); - } + db::LikeSubject::Feedgen { actor_id: _feedgen_actor_id, rkey: _feedgen_rkey } => { + // Database trigger handles cache invalidation for feedgen likes } - db::LikeSubject::Labeler { actor_id: labeler_actor_id } => { - // Invalidate labeler cache (no delta needed - labeler_likes is separate table) - if let Ok(labeler_did) = db::actor_did_from_id(conn, labeler_actor_id).await { - let labeler_uri = format!("at://{}/app.bsky.labeler.service/self", labeler_did); - cache_invalidations.push(cache::invalidate_labeler(&labeler_uri)); - } + db::LikeSubject::Labeler { actor_id: _labeler_actor_id } => { + // Database trigger handles cache invalidation for labeler likes } } } @@ -888,7 +869,7 @@ pub async fn execute_operation( // No deltas needed with array-only tracking // Invalidate post cache since repost array changed - cache_invalidations.push(cache::invalidate_post(&post_uri)); + // Database trigger handles cache invalidation // Remove PostgreSQL notification (recipient = author of reposted post) if allowlist.cache.contains_did(crate::utils::extract_did_from_uri(&post_uri).unwrap_or("")) { @@ -911,7 +892,7 @@ pub async fn execute_operation( } // Invalidate author feed cache when user deletes a repost - cache_invalidations.push(format!("authorfeed:{}:", actor_id)); + // Database trigger handles cache invalidation } } CollectionType::BskyFollow => { @@ -940,7 +921,7 @@ pub async fn execute_operation( } // Invalidate timeline cache when user unfollows someone - cache_invalidations.push(format!("timeline:{}:", actor_id)); + // Database trigger handles cache invalidation } } CollectionType::BskyFeedPost => { @@ -967,7 +948,7 @@ pub async fn execute_operation( let parent_uri = format!("at://{}/app.bsky.feed.post/{}", parent_did, parakeet_db::models::i64_to_tid(parent_rkey)); - cache_invalidations.push(cache::invalidate_post(&parent_uri)); + // Database trigger handles cache invalidation } } @@ -988,7 +969,7 @@ pub async fn execute_operation( let embed_uri = format!("at://{}/app.bsky.feed.post/{}", embed_did, parakeet_db::models::i64_to_tid(embed_rkey)); - cache_invalidations.push(cache::invalidate_post(&embed_uri)); + // Database trigger handles cache invalidation } } @@ -1013,24 +994,24 @@ pub async fn execute_operation( } // Invalidate author feed cache when user deletes a post - cache_invalidations.push(format!("authorfeed:{}:", actor_id)); + // Database trigger handles cache invalidation } } // Other delete operations that don't affect aggregates CollectionType::BskyProfile => { db::profile_delete(conn, actor_id).await?; // Invalidate profile cache when user deletes profile record - cache_invalidations.push(format!("profile#{}", actor_id)); + // Database trigger handles cache invalidation } CollectionType::BskyStatus => { db::status_delete(conn, actor_id).await?; // Invalidate profile cache when user deletes custom status - cache_invalidations.push(format!("profile#{}", actor_id)); + // Database trigger handles cache invalidation } CollectionType::BskyBlock => { db::block_delete(conn, rkey, actor_id).await?; // Invalidate timeline cache when user unblocks someone - cache_invalidations.push(format!("timeline:{}:", actor_id)); + // Database trigger handles cache invalidation } CollectionType::BskyFeedGen => { // Feedgens use arbitrary string rkeys - extract from at_uri @@ -1091,7 +1072,7 @@ pub async fn execute_operation( // Note: No generic records table to delete from anymore - Ok(cache_invalidations) + Ok(()) } DatabaseOperation::MaintainSelfLabels { actor_id, @@ -1103,7 +1084,7 @@ pub async fn execute_operation( let did = db::actor_did_from_id(conn, actor_id).await?; db::maintain_self_labels(conn, &did, cid, &at_uri, labels).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::MaintainPostgateDetaches { post_uri, @@ -1115,7 +1096,7 @@ pub async fn execute_operation( // - Uses direct database lookups for IDs // - Does single efficient UPDATE db::postgate_maintain_detaches_cached(conn, &post_uri, &detached_uris, disable_effective).await?; - Ok(vec![]) + Ok(()) } DatabaseOperation::CachePdsMapping { did, @@ -1124,7 +1105,7 @@ pub async fn execute_operation( } => { // Cache PDS mapping in moka-based cache (in-memory, TTL: 24h) pds_cache.set(did, host).await; - Ok(vec![]) + Ok(()) } DatabaseOperation::EnsureMinimumPostStats { post_uri: _, @@ -1141,7 +1122,7 @@ pub async fn execute_operation( // // For now, this operation is a no-op to avoid breaking callers. // TODO: Remove this operation entirely once callers are updated. - Ok(vec![]) + Ok(()) } } } diff --git a/consumer/src/database_writer/operations/handlers/block.rs b/consumer/src/database_writer/operations/handlers/block.rs index 8be643b6..1a1b2342 100644 --- a/consumer/src/database_writer/operations/handlers/block.rs +++ b/consumer/src/database_writer/operations/handlers/block.rs @@ -8,7 +8,6 @@ pub fn handle_block( record: AppBskyGraphBlock, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { @@ -34,7 +33,6 @@ pub fn handle_block( }); // Invalidate timeline cache when user blocks someone - cache_invalidations.push(format!("timeline:{}:", ctx.actor_id)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/declarations.rs b/consumer/src/database_writer/operations/handlers/declarations.rs index 73edce72..ba17f6ff 100644 --- a/consumer/src/database_writer/operations/handlers/declarations.rs +++ b/consumer/src/database_writer/operations/handlers/declarations.rs @@ -8,17 +8,15 @@ pub fn handle_notification_declaration( record: AppBskyNotificationDeclaration, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { operations.push(DatabaseOperation::UpsertNotificationDeclaration { actor_id: ctx.actor_id, record, }); - cache_invalidations.push(format!("profile#{}", ctx.actor_id)); } - (operations, cache_invalidations) + (operations, vec![]) } pub fn handle_chat_declaration( @@ -26,15 +24,13 @@ pub fn handle_chat_declaration( record: ChatBskyActorDeclaration, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { operations.push(DatabaseOperation::UpsertChatDeclaration { actor_id: ctx.actor_id, record, }); - cache_invalidations.push(format!("profile#{}", ctx.actor_id)); } - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/feedgen.rs b/consumer/src/database_writer/operations/handlers/feedgen.rs index bb43a158..442a7c21 100644 --- a/consumer/src/database_writer/operations/handlers/feedgen.rs +++ b/consumer/src/database_writer/operations/handlers/feedgen.rs @@ -8,7 +8,6 @@ pub fn handle_feedgen( record: AppBskyFeedGenerator, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); let labels = record.labels.clone(); @@ -22,7 +21,7 @@ pub fn handle_feedgen( "Feedgen record missing service_actor_id - reference extraction may have filtered out the DID. \ This can happen if the DID field is empty, whitespace-only, or actor creation failed." ); - return (operations, cache_invalidations); + return (operations, vec![]); }; operations.push(DatabaseOperation::UpsertFeedGenerator { @@ -44,7 +43,6 @@ pub fn handle_feedgen( // Note: Can't determine did_insert without database, so we queue cache invalidation // The batch writer will decide based on INSERT result - cache_invalidations.push(format!("feedgen#{}", ctx.at_uri)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/follow.rs b/consumer/src/database_writer/operations/handlers/follow.rs index 80825fcb..bfae6c76 100644 --- a/consumer/src/database_writer/operations/handlers/follow.rs +++ b/consumer/src/database_writer/operations/handlers/follow.rs @@ -8,7 +8,6 @@ pub fn handle_follow( record: AppBskyGraphFollow, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { @@ -52,7 +51,6 @@ pub fn handle_follow( )); // Invalidate timeline cache when user follows someone - cache_invalidations.push(format!("timeline:{}:", ctx.actor_id)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/labeler.rs b/consumer/src/database_writer/operations/handlers/labeler.rs index bd490c74..dd5279ce 100644 --- a/consumer/src/database_writer/operations/handlers/labeler.rs +++ b/consumer/src/database_writer/operations/handlers/labeler.rs @@ -8,7 +8,6 @@ pub fn handle_labeler( record: AppBskyLabelerService, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { let labels = record.labels.clone(); @@ -28,8 +27,7 @@ pub fn handle_labeler( }); } - cache_invalidations.push(format!("labeler#{}", ctx.actor_id)); } - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/list.rs b/consumer/src/database_writer/operations/handlers/list.rs index 05007fe2..d72b3663 100644 --- a/consumer/src/database_writer/operations/handlers/list.rs +++ b/consumer/src/database_writer/operations/handlers/list.rs @@ -9,7 +9,6 @@ pub fn handle_list( record: AppBskyGraphList, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); // Lists can use both TID and arbitrary string rkeys (like "nfb", "bblock") // If it's a TID, validate the timestamp is within acceptable range @@ -39,9 +38,8 @@ pub fn handle_list( } // Queue cache invalidation (batch writer decides based on insert result) - cache_invalidations.push(format!("list#{}", ctx.at_uri)); - (operations, cache_invalidations) + (operations, vec![]) } pub fn handle_list_block( @@ -87,8 +85,6 @@ pub fn handle_list_item( let rkey_i64 = parakeet_db::models::tid_to_i64(&ctx.rkey) .expect("TID validation passed, conversion should succeed"); - let mut cache_invalidations = Vec::new(); - cache_invalidations.push(format!("list#{}", record.list)); let operations = vec![DatabaseOperation::InsertListItem { rkey: rkey_i64, @@ -99,5 +95,5 @@ pub fn handle_list_item( record, }]; - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/misc.rs b/consumer/src/database_writer/operations/handlers/misc.rs index b615e252..02914221 100644 --- a/consumer/src/database_writer/operations/handlers/misc.rs +++ b/consumer/src/database_writer/operations/handlers/misc.rs @@ -7,12 +7,10 @@ pub fn handle_alpa_status( ctx: &super::RecordContext, _record: FmTealAlpaActorStatus, ) -> (Vec, Vec) { - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { // TODO: implement fm.teal.alpa.actor.status table - cache_invalidations.push(format!("profile#{}", ctx.actor_id)); } - (vec![], cache_invalidations) + (vec![], vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/post.rs b/consumer/src/database_writer/operations/handlers/post.rs index 6118db44..3982d2f4 100644 --- a/consumer/src/database_writer/operations/handlers/post.rs +++ b/consumer/src/database_writer/operations/handlers/post.rs @@ -18,7 +18,6 @@ pub fn handle_post( let rkey = &ctx.rkey; let source = ctx.source; let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); let maybe_reply = record.reply.as_ref().map(|v| v.parent.uri.clone()); let maybe_root = record @@ -212,9 +211,8 @@ pub fn handle_post( } // Invalidate author feed cache when user creates a post - cache_invalidations.push(format!("authorfeed:{}:", actor_id)); - (operations, cache_invalidations) + (operations, vec![]) } pub fn handle_postgate( @@ -273,7 +271,6 @@ pub fn handle_threadgate( } let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { @@ -292,7 +289,6 @@ pub fn handle_threadgate( record, }); - cache_invalidations.push(format!("post#{}", ctx.at_uri)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/profile.rs b/consumer/src/database_writer/operations/handlers/profile.rs index c962dbbb..b9537107 100644 --- a/consumer/src/database_writer/operations/handlers/profile.rs +++ b/consumer/src/database_writer/operations/handlers/profile.rs @@ -9,7 +9,6 @@ pub fn handle_profile( mut record: AppBskyActorProfile, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { let labels = record.labels.clone(); @@ -36,10 +35,9 @@ pub fn handle_profile( }); } - cache_invalidations.push(format!("profile#{}", ctx.actor_id)); } - (operations, cache_invalidations) + (operations, vec![]) } pub fn handle_status( @@ -47,7 +45,6 @@ pub fn handle_status( record: AppBskyActorStatus, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); if ctx.rkey == "self" { operations.push(DatabaseOperation::UpsertStatus { @@ -55,8 +52,7 @@ pub fn handle_status( cid: ctx.cid, record, }); - cache_invalidations.push(format!("profile#{}", ctx.actor_id)); } - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/repost.rs b/consumer/src/database_writer/operations/handlers/repost.rs index 1d4a3504..dae87897 100644 --- a/consumer/src/database_writer/operations/handlers/repost.rs +++ b/consumer/src/database_writer/operations/handlers/repost.rs @@ -8,7 +8,6 @@ pub fn handle_repost( record: AppBskyFeedRepost, ) -> (Vec, Vec) { let mut operations = Vec::new(); - let mut cache_invalidations = Vec::new(); let subject_uri = record.subject.uri.clone(); @@ -59,7 +58,6 @@ pub fn handle_repost( } // Invalidate author feed cache when user creates a repost - cache_invalidations.push(format!("authorfeed:{}:", ctx.actor_id)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/handlers/starterpack.rs b/consumer/src/database_writer/operations/handlers/starterpack.rs index 77e0043f..5876a710 100644 --- a/consumer/src/database_writer/operations/handlers/starterpack.rs +++ b/consumer/src/database_writer/operations/handlers/starterpack.rs @@ -7,7 +7,6 @@ pub fn handle_starterpack( ctx: &super::RecordContext, record: AppBskyGraphStarterPack, ) -> (Vec, Vec) { - let mut cache_invalidations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { @@ -26,7 +25,6 @@ pub fn handle_starterpack( record, }]; - cache_invalidations.push(format!("starterpacks#{}", ctx.at_uri)); - (operations, cache_invalidations) + (operations, vec![]) } diff --git a/consumer/src/database_writer/operations/processor.rs b/consumer/src/database_writer/operations/processor.rs index 84174057..2b69e8ab 100644 --- a/consumer/src/database_writer/operations/processor.rs +++ b/consumer/src/database_writer/operations/processor.rs @@ -84,7 +84,6 @@ pub fn process_record_to_operations( ProcessedEvent { operations, - cache_invalidations, cursor: None, source, } diff --git a/consumer/src/database_writer/routing.rs b/consumer/src/database_writer/routing.rs index 4baf1b3f..92e7692f 100644 --- a/consumer/src/database_writer/routing.rs +++ b/consumer/src/database_writer/routing.rs @@ -58,8 +58,6 @@ pub fn classify_operation(op: &DatabaseOperation) -> WorkerType { pub struct PartitionedEvent { /// Operations for this worker pub operations: Vec, - /// Cache invalidations (shared across all workers) - pub cache_invalidations: Vec, /// Cursor update (only relevant for Jetstream events) /// Contains Jetstream cursor timestamp in microseconds pub cursor: Option, diff --git a/consumer/src/database_writer/workers.rs b/consumer/src/database_writer/workers.rs index 7142f629..7790860c 100644 --- a/consumer/src/database_writer/workers.rs +++ b/consumer/src/database_writer/workers.rs @@ -303,8 +303,8 @@ async fn execute_bulk_operations( allowlist, pds_cache, ).await { - Ok(_cache_invalidations) => { - // Actor counts maintained by triggers + Ok(()) => { + // Actor counts and cache maintained by triggers } Err(e) => { tracing::error!(error = ?e, "Failed to execute individual operation from bulk batch"); @@ -482,7 +482,6 @@ async fn resolve_event( ProcessedEvent { operations: vec![operation], - cache_invalidations, cursor: None, // Will be set below source: unresolved.source, } @@ -757,20 +756,20 @@ fn spawn_source_dispatcher( let tx = worker_pools.get_worker(source, worker_type); drop(tx.send(routing::PartitionedEvent { operations: vec![op], - cache_invalidations: Vec::new(), cursor: None, source, })); } - // Send cache/cursor to metadata worker - let metadata_tx = worker_pools.get_metadata_worker(source); - drop(metadata_tx.send(routing::PartitionedEvent { - operations: Vec::new(), - cache_invalidations: event.cache_invalidations, - cursor: event.cursor, - source, - })); + // Send cursor to metadata worker (if present) + if event.cursor.is_some() { + let metadata_tx = worker_pools.get_metadata_worker(source); + drop(metadata_tx.send(routing::PartitionedEvent { + operations: Vec::new(), + cursor: event.cursor, + source, + })); + } // Update counters batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -796,20 +795,20 @@ fn spawn_source_dispatcher( let tx = worker_pools.get_worker(source, worker_type); drop(tx.send(routing::PartitionedEvent { operations: vec![op], - cache_invalidations: Vec::new(), cursor: None, source, })); } - // Send cache/cursor to metadata worker - let metadata_tx = worker_pools.get_metadata_worker(source); - drop(metadata_tx.send(routing::PartitionedEvent { - operations: Vec::new(), - cache_invalidations: event.cache_invalidations, - cursor: event.cursor, - source, - })); + // Send cursor to metadata worker (if present) + if event.cursor.is_some() { + let metadata_tx = worker_pools.get_metadata_worker(source); + drop(metadata_tx.send(routing::PartitionedEvent { + operations: Vec::new(), + cursor: event.cursor, + source, + })); + } // Update counters batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -1138,7 +1137,6 @@ fn spawn_worker(config: WorkerConfig) -> tokio::task::JoinHandle tokio::task::JoinHandle { + Ok(()) => { affected += 1; - // Actor counts maintained by triggers - event_cache_invalidations.extend(cache_inv); + // Actor counts and cache maintained by triggers } Err(e) => { // Use {:#} to show full error chain with causes @@ -1227,27 +1224,8 @@ fn spawn_worker(config: WorkerConfig) -> tokio::task::JoinHandle { - // Sent successfully - cache worker will batch and NOTIFY - } - Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { - // Channel full - drop invalidations (cache will expire naturally) - counter!("cache_invalidation_dropped").increment(1); - tracing::trace!(worker = name, "Cache invalidation channel full, dropping batch"); - } - Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { - // Channel closed - cache worker has stopped - tracing::warn!(worker = name, "Cache invalidation worker has stopped"); - } - } - } + // Cache invalidations are now handled by database triggers via pg_notify + // No need to send them from the application layer // Update cursor AFTER successful database operations if let Some(cursor_timestamp) = event.cursor { diff --git a/consumer/src/workers/backfill/collection_fetch.rs b/consumer/src/workers/backfill/collection_fetch.rs index 5587aedd..6757f148 100644 --- a/consumer/src/workers/backfill/collection_fetch.rs +++ b/consumer/src/workers/backfill/collection_fetch.rs @@ -174,7 +174,6 @@ async fn fetch_and_process_collection( event_tx .send(WriterEvent::Resolved(Box::new(ProcessedEvent { operations: vec![operation], - cache_invalidations: vec![], cursor: None, source: EventSource::Backfill, }))) diff --git a/consumer/src/workers/backfill/repo.rs b/consumer/src/workers/backfill/repo.rs index 25b4a32b..3f96feb9 100644 --- a/consumer/src/workers/backfill/repo.rs +++ b/consumer/src/workers/backfill/repo.rs @@ -239,7 +239,6 @@ pub async fn insert_repo( event_tx.send(crate::database_writer::WriterEvent::Resolved(Box::new( crate::database_writer::ProcessedEvent { operations: vec![operation], - cache_invalidations: vec![], cursor: None, source: EventSource::Backfill, } diff --git a/consumer/src/workers/fetch/worker.rs b/consumer/src/workers/fetch/worker.rs index 2da492e5..34a32df7 100644 --- a/consumer/src/workers/fetch/worker.rs +++ b/consumer/src/workers/fetch/worker.rs @@ -379,7 +379,7 @@ async fn process_fetch_job( // Create ProcessedEvent for batch writer let event = ProcessedEvent { operations, - cache_invalidations: processed.cache_invalidations, + cursor: None, source: crate::database_writer::EventSource::FetchQueue, }; diff --git a/consumer/src/workers/jetstream/processing.rs b/consumer/src/workers/jetstream/processing.rs index c7005f55..7b43a377 100644 --- a/consumer/src/workers/jetstream/processing.rs +++ b/consumer/src/workers/jetstream/processing.rs @@ -131,7 +131,7 @@ pub fn process_identity_event( Some(ProcessedEvent { operations, - cache_invalidations: vec![format!("profile#{}", did)], + cursor: None, // Cursor is set by caller source: EventSource::Jetstream, }) @@ -179,7 +179,7 @@ pub fn process_account_event( Some(ProcessedEvent { operations, - cache_invalidations: vec![format!("profile#{}", did)], + cursor: None, // Cursor is set by caller source: EventSource::Jetstream, }) diff --git a/consumer/src/workers/jetstream/worker.rs b/consumer/src/workers/jetstream/worker.rs index 3eeb1f0f..1c872771 100644 --- a/consumer/src/workers/jetstream/worker.rs +++ b/consumer/src/workers/jetstream/worker.rs @@ -70,7 +70,6 @@ async fn run_worker(idx: u8, ctx: &mut WorkerContext, rx: &mut Receiver