diff --git a/Cargo.lock b/Cargo.lock index 8029c205..4c4ff2c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -644,6 +644,7 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" name = "consumer" version = "0.1.0" dependencies = [ + "async-trait", "chrono", "chrono-tz", "ciborium", @@ -657,6 +658,7 @@ dependencies = [ "foldhash", "futures", "futures-channel", + "futures-util", "ipld-core", "iroh-car", "lexica", diff --git a/consumer/Cargo.toml b/consumer/Cargo.toml index 672f3dce..cc7b9437 100644 --- a/consumer/Cargo.toml +++ b/consumer/Cargo.toml @@ -15,6 +15,7 @@ figment = { version = "0.10.19", features = ["env", "toml"] } flume = { version = "0.11", features = ["async"] } foldhash = "0.1.4" futures = "0.3.31" +futures-util = "0.3.31" ipld-core = "0.4.1" iroh-car = "0.5.1" lexica = { path = "../lexica" } @@ -46,6 +47,7 @@ futures-channel = "0.3.31" urlencoding = "2.1" color-eyre = "0.6.5" unicode-segmentation = "1.10" +async-trait = "0.1" [lints.rust] diff --git a/consumer/src/cmd.rs b/consumer/src/cmd.rs index 65fa602c..a6ba1a04 100644 --- a/consumer/src/cmd.rs +++ b/consumer/src/cmd.rs @@ -2,13 +2,10 @@ use clap::Parser; #[derive(Debug, Parser)] pub struct Cli { - /// Run backfill threads - #[arg(long, default_value_t = false)] - pub backfill: bool, - /// Run the firehose consumer and indexer + /// Run the Tap consumer and indexer #[arg(long, default_value_t = false)] pub indexer: bool, - /// Connect to label services and ingest labels + /// Connect to label services and ingest labels (currently disabled) #[arg(long, default_value_t = false)] pub labels: bool, } diff --git a/consumer/src/config.rs b/consumer/src/config.rs index 3ec09623..62844b04 100644 --- a/consumer/src/config.rs +++ b/consumer/src/config.rs @@ -37,104 +37,30 @@ pub struct Config { /// Configuration items specific to indexer pub indexer: Option, - /// Configuration items specific to backfill - pub backfill: Option, - /// Configuration items specific to record fetching - pub record_fetch: Option, } #[derive(Debug, Deserialize, Clone)] pub struct IndexerConfig { - /// Jetstream server URL - pub jetstream_source: Option, - /// DIDs to filter on when using Jetstream (empty = all DIDs) - #[serde(default)] - pub jetstream_wanted_dids: Vec, - /// Maximum message size in bytes for Jetstream (0 = no limit) - #[serde(default)] - pub jetstream_max_message_size: usize, - /// Whether to use compression for Jetstream - #[serde(default)] - pub jetstream_use_compression: bool, + /// Tap WebSocket URL (e.g., "ws://localhost:2480/channel") + pub tap_websocket_url: String, + /// Tap admin API URL (e.g., "http://localhost:2480") + pub tap_admin_url: String, + /// Tap admin password (if authentication is required) + pub tap_admin_password: Option, /// Number of worker threads for processing events #[serde(default = "default_indexer_workers")] pub workers: u8, - /// Starting cursor (timestamp in microseconds) for Jetstream - pub start_timestamp: Option, - /// Whether to resolve handles as part of `#identity` events. - /// You can use this to move handle resolution out of event handling and into another place. - #[serde(default)] - pub skip_handle_validation: bool, + /// Maximum number of unacknowledged events to buffer + #[serde(default = "default_max_pending_acks")] + pub max_pending_acks: usize, } -#[derive(Clone, Debug, Deserialize)] -pub struct BackfillConfig { - #[serde(default = "default_backfill_workers")] - pub workers: u8, - #[serde(default)] - pub skip_aggregation: bool, - #[serde(default = "default_download_workers")] - pub download_workers: usize, - #[serde(default = "default_download_buffer")] - pub download_buffer: usize, - pub download_tmp_dir: String, -} - -const fn default_backfill_workers() -> u8 { - 4 -} const fn default_indexer_workers() -> u8 { 4 } -const fn default_download_workers() -> usize { - 25 -} - -const fn default_download_buffer() -> usize { - 25_000 -} - -#[derive(Clone, Debug, Deserialize)] -pub struct RecordFetchConfig { - /// URL of the Slingshot instance to use for record fetching - #[serde(default = "default_slingshot_url")] - pub slingshot_url: String, - /// URL of the Bluesky public API for fallback fetching - #[serde(default = "default_bluesky_api_url")] - pub bluesky_api_url: String, - /// Timeout in seconds for record fetch requests - #[serde(default = "default_fetch_timeout")] - pub timeout_secs: u64, - /// Number of concurrent workers for fetching records - #[serde(default = "default_fetch_workers")] - pub workers: u8, +const fn default_max_pending_acks() -> usize { + 1000 } -impl Default for RecordFetchConfig { - fn default() -> Self { - Self { - slingshot_url: default_slingshot_url(), - bluesky_api_url: default_bluesky_api_url(), - timeout_secs: default_fetch_timeout(), - workers: default_fetch_workers(), - } - } -} - -fn default_slingshot_url() -> String { - "https://slingshot.microcosm.blue".to_owned() -} - -fn default_bluesky_api_url() -> String { - "https://public.api.bsky.app".to_owned() -} - -const fn default_fetch_timeout() -> u64 { - 5 -} - -const fn default_fetch_workers() -> u8 { - 12 -} diff --git a/consumer/src/database_writer/bulk_processor.rs b/consumer/src/database_writer/bulk_processor.rs index 6d867706..aca3f0b7 100644 --- a/consumer/src/database_writer/bulk_processor.rs +++ b/consumer/src/database_writer/bulk_processor.rs @@ -391,7 +391,7 @@ pub async fn process_bulk_records( None }; - let resolved_actor_ids = crate::database_writer::workers::ResolvedActorIds { + let resolved_actor_ids = super::operations::ResolvedActorIds { subject_actor_id, service_actor_id, parent_author_actor_id, diff --git a/consumer/src/database_writer/cache_notify.rs b/consumer/src/database_writer/cache_notify.rs deleted file mode 100644 index 21e00e4e..00000000 --- a/consumer/src/database_writer/cache_notify.rs +++ /dev/null @@ -1,97 +0,0 @@ -//! PostgreSQL NOTIFY helpers for cache invalidation -//! -//! This module sends NOTIFY messages to PostgreSQL when records change. -//! Parakeet listens for these notifications and invalidates its in-memory cache. -//! -//! ## Architecture -//! -//! - Consumer (this code): Sends NOTIFY messages via PostgreSQL -//! - Parakeet: Listens via LISTEN and invalidates moka cache -//! - Decoupled: Consumer can run without parakeet and vice versa -//! -//! ## Channel -//! -//! All cache invalidations use a single channel: `cache_invalidate` -//! -//! ## Message Format -//! -//! Messages are cache key prefixes that parakeet should invalidate: -//! - `timeline:{actor_id}:` - Invalidate all timeline entries for this actor -//! - `authorfeed:{actor_id}:` - Invalidate all author feed entries for this actor -//! - `profile#{actor_id}` - Invalidate profile for this actor -//! - `post#{uri}` - Invalidate specific post -//! - `labeler#{actor_id}` - Invalidate labeler info - -use deadpool_postgres::Object; - -/// Send a cache invalidation notification via PostgreSQL NOTIFY -/// -/// This sends a message on the `cache_invalidate` channel that parakeet will receive. -/// Parakeet invalidates all cache entries matching the given prefix. -/// -/// ## Performance -/// -/// NOTIFY is lightweight and non-blocking. It's fire-and-forget - if parakeet -/// isn't running, the message is simply lost (which is fine, cache will fill on demand). -/// -/// ## Usage -/// -/// ```rust,ignore -/// // Invalidate all timeline entries for actor 123 -/// send_cache_invalidation(&conn, "timeline:123:").await?; -/// -/// // Invalidate specific post -/// send_cache_invalidation(&conn, "post#at://did:plc:xyz/app.bsky.feed.post/abc").await?; -/// ``` -pub async fn send_cache_invalidation( - conn: &Object, - cache_key: &str, -) -> Result<(), eyre::Error> { - // Execute NOTIFY command - // Note: We use simple_query instead of execute because NOTIFY doesn't support $1 parameters - let query = format!("NOTIFY cache_invalidate, '{}'", escape_notify_payload(cache_key)); - conn.simple_query(&query).await?; - Ok(()) -} - -/// Send multiple cache invalidation notifications in a single transaction -/// -/// More efficient than calling `send_cache_invalidation` multiple times. -/// All notifications are sent atomically. -pub async fn send_cache_invalidations( - conn: &Object, - cache_keys: &[String], -) -> Result<(), eyre::Error> { - if cache_keys.is_empty() { - return Ok(()); - } - - for key in cache_keys { - send_cache_invalidation(conn, key).await?; - } - Ok(()) -} - -/// Escape single quotes in NOTIFY payload -/// -/// PostgreSQL NOTIFY payloads are string literals, so we need to escape single quotes. -/// Single quotes are escaped by doubling them: ' becomes '' -fn escape_notify_payload(s: &str) -> String { - s.replace('\'', "''") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_escape_notify_payload() { - assert_eq!(escape_notify_payload("simple"), "simple"); - assert_eq!(escape_notify_payload("with'quote"), "with''quote"); - assert_eq!( - escape_notify_payload("multiple'quotes'here"), - "multiple''quotes''here" - ); - assert_eq!(escape_notify_payload("timeline:123:"), "timeline:123:"); - } -} diff --git a/consumer/src/database_writer/cache_worker.rs b/consumer/src/database_writer/cache_worker.rs deleted file mode 100644 index 1385ae26..00000000 --- a/consumer/src/database_writer/cache_worker.rs +++ /dev/null @@ -1,161 +0,0 @@ -//! Cache invalidation worker -//! -//! This module implements a dedicated worker that handles PostgreSQL NOTIFY for cache invalidations. -//! -//! ## Why a separate worker? -//! -//! PostgreSQL NOTIFY acquires a global database lock during COMMIT (see async.c:956 in postgres source). -//! This lock serializes ALL commits across the entire database instance, which would severely -//! impact write throughput under high concurrent load. -//! -//! By isolating NOTIFY calls in a dedicated worker: -//! - Main data writers never touch NOTIFY, never get serialized -//! - Only this single worker is affected by the global lock -//! - Main ingestion throughput remains unaffected -//! -//! ## Architecture -//! -//! ```text -//! Main Writers (45) Cache Worker (1) -//! │ │ -//! ├─ Execute operations │ -//! ├─ Emit CacheInvalidate ──┼→ Bounded channel (1000) -//! ├─ COMMIT (no NOTIFY!) │ │ -//! └─ Continue │ ↓ -//! │ Batch every 10ms -//! │ NOTIFY (global lock here) -//! └─ Never blocks main writers -//! ``` -//! -//! ## Backpressure -//! -//! The channel is bounded (1000 capacity). If it fills: -//! - New cache invalidations are dropped (graceful degradation) -//! - Cache entries will expire naturally via TTL -//! - Metrics track dropped invalidations -//! -//! ## Batching -//! -//! Cache keys are batched every 10ms or 100 keys (whichever comes first). -//! This reduces the number of NOTIFY calls while maintaining responsiveness. - -use deadpool_postgres::Pool; -use metrics::counter; -use std::collections::HashSet; -use std::time::Duration; -use tokio::sync::mpsc; -use tokio::time::Instant; - -/// Batch size for cache invalidations (keys) -const BATCH_SIZE: usize = 100; - -/// Batch timeout for cache invalidations (milliseconds) -const BATCH_TIMEOUT_MS: u64 = 10; - -/// Cache invalidation worker -/// -/// Receives cache keys via bounded channel, batches them, and sends NOTIFY commands. -/// Runs in isolation to prevent global NOTIFY lock from blocking main writers. -pub async fn cache_invalidation_worker( - pool: Pool, - mut rx: mpsc::Receiver>, - name: String, -) { - tracing::info!(worker = %name, "Cache invalidation worker started"); - - let mut batch: HashSet = HashSet::new(); - let mut batch_start = Instant::now(); - - loop { - // Wait for either: - // 1. New cache keys - // 2. Batch timeout (10ms) - let timeout = tokio::time::sleep(Duration::from_millis(BATCH_TIMEOUT_MS)); - tokio::pin!(timeout); - - tokio::select! { - // Received cache keys - maybe_keys = rx.recv() => { - match maybe_keys { - Some(keys) => { - // Add keys to batch (HashSet deduplicates) - batch.extend(keys); - counter!("cache_invalidation_keys_received").increment(batch.len() as u64); - - // Flush if batch is full - if batch.len() >= BATCH_SIZE { - flush_batch(&pool, &mut batch, &mut batch_start, &name).await; - } - } - None => { - // Channel closed, flush remaining batch and exit - if !batch.is_empty() { - flush_batch(&pool, &mut batch, &mut batch_start, &name).await; - } - tracing::info!(worker = %name, "Cache invalidation worker shutting down"); - return; - } - } - } - - // Batch timeout reached - _ = &mut timeout => { - if !batch.is_empty() { - flush_batch(&pool, &mut batch, &mut batch_start, &name).await; - } - } - } - } -} - -/// Flush the current batch of cache invalidations via NOTIFY -async fn flush_batch( - pool: &Pool, - batch: &mut HashSet, - batch_start: &mut Instant, - worker_name: &str, -) { - let batch_size = batch.len(); - let batch_duration = batch_start.elapsed(); - - // Convert to Vec for NOTIFY - let keys: Vec = batch.drain().collect(); - - // Get database connection - let conn = match pool.get().await { - Ok(conn) => conn, - Err(e) => { - tracing::error!( - worker = %worker_name, - error = ?e, - "Failed to get database connection for cache invalidation" - ); - counter!("cache_invalidation_connection_error").increment(1); - return; - } - }; - - // Send NOTIFY commands (this acquires global lock, but only affects this worker) - if let Err(e) = crate::database_writer::cache_notify::send_cache_invalidations(&conn, &keys).await { - tracing::warn!( - worker = %worker_name, - keys = batch_size, - error = ?e, - "Failed to send cache invalidation NOTIFY" - ); - counter!("cache_invalidation_notify_error").increment(1); - } else { - counter!("cache_invalidation_batches_sent").increment(1); - counter!("cache_invalidation_keys_sent").increment(batch_size as u64); - - tracing::trace!( - worker = %worker_name, - keys = batch_size, - duration_ms = batch_duration.as_millis(), - "Sent cache invalidation batch" - ); - } - - // Reset batch timer - *batch_start = Instant::now(); -} diff --git a/consumer/src/database_writer/mod.rs b/consumer/src/database_writer/mod.rs index 227fc635..5b4545a6 100644 --- a/consumer/src/database_writer/mod.rs +++ b/consumer/src/database_writer/mod.rs @@ -5,34 +5,32 @@ //! all the data needed for database operations. A background database writer //! task drains the queue and performs database writes asynchronously. //! -//! Architecture: 45-worker pool (3 sources × 5 types × 3 workers) -//! - Sources: Jetstream (high priority), FetchQueue (medium), Backfill (low) +//! Architecture: 30-worker pool (2 sources × 5 types × 3 workers) +//! - Sources: Tap live (high priority), Tap backfill (medium) //! - Types: Actor, Like, Social, Post, Metadata //! - Per-DID ordering maintained through advisory locks pub mod bulk_processor; pub mod bulk_types; -pub mod cache_notify; -pub mod cache_worker; pub mod locking; pub mod operations; pub mod reference_extraction; pub mod routing; pub mod timestamp; -pub mod workers; +pub mod workers_tap; // acquire_did_locks is used internally by workers, not exposed pub use bulk_types::{BulkOperations, UnresolvedRecord}; -pub use operations::{process_record_to_operations, DatabaseOperation}; +pub use operations::{process_record_to_operations, DatabaseOperation, ResolvedActorIds}; pub use reference_extraction::{extract_references, RecordReferences}; pub use timestamp::{validate_record_timestamp_with_tid, validate_tid_timestamp}; -pub use workers::spawn_database_writer; +pub use workers_tap::spawn_database_writer_tap; /// Wrapper for events sent to the database writer /// /// The database writer accepts four types of events: -/// - UnresolvedEvent: Needs actor_id/post_id resolution (from Jetstream workers) -/// - ProcessedEvent: Already resolved, ready for dispatch (from Backfill/Fetch workers) +/// - UnresolvedEvent: Needs actor_id/post_id resolution (from Tap workers) +/// - ProcessedEvent: Already resolved, ready for dispatch (from Fetch workers) /// - UnresolvedBulk: Bulk unresolved events from backfill (50+ records) /// - ResolvedBulk: Bulk resolved events ready for COPY operations #[derive(Debug)] @@ -60,12 +58,10 @@ pub enum WriterEvent { /// Event source for priority queue routing #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EventSource { - /// High priority: Real-time Jetstream events (processed first) - Jetstream, - /// Medium priority: Fetch queue completions (processed when no Jetstream) - FetchQueue, - /// Low priority: Bulk backfill operations (processed when idle) - Backfill, + /// High priority: Real-time Tap events (processed first) + Tap, + /// Medium priority: Tap historical backfill events + TapBackfill, } /// Type of unresolved event (discriminates between creates/updates and deletes) @@ -84,7 +80,7 @@ pub enum UnresolvedEventType { /// An unresolved event that needs actor_id (and potentially post_id) resolution /// -/// Jetstream workers stay database-free by producing these events. +/// Tap workers stay database-free by producing these events. /// Resolution workers in the database writer will: /// 1. Resolve/create actor stub (get actor_id) /// 2. For posts with reply/quote: resolve/create parent post stubs (get parent_post_id) @@ -108,7 +104,7 @@ pub struct UnresolvedEvent { /// Event source for priority routing pub source: EventSource, - /// Cursor update for this event (Jetstream cursor timestamp in microseconds) + /// Cursor update for this event (cursor timestamp in microseconds) pub cursor: Option, } @@ -125,7 +121,7 @@ pub struct ProcessedEvent { pub operations: Vec, /// Cursor update for this event (batched saves every 10s) - /// Contains Jetstream cursor timestamp in microseconds + /// Contains cursor timestamp in microseconds pub cursor: Option, /// Event source for priority queue routing diff --git a/consumer/src/database_writer/operations/cache.rs b/consumer/src/database_writer/operations/cache.rs deleted file mode 100644 index 8556b0bd..00000000 --- a/consumer/src/database_writer/operations/cache.rs +++ /dev/null @@ -1,192 +0,0 @@ -//! Cache invalidation helpers -//! -//! This module provides centralized functions for generating cache -//! invalidation keys. Cache keys follow specific patterns that are used -//! by the parakeet server for caching various resources. - -/// Invalidate profile cache for a given DID -/// -/// Used when: profile, status, labeler, or declaration is updated -/// -/// # Arguments -/// -/// * `did` - The DID of the actor whose profile cache should be invalidated -/// -/// # Returns -/// -/// Cache key in format: `profile#{did}` -#[inline] -pub fn invalidate_profile(did: &str) -> String { - format!("profile#{}", did) -} - -/// Invalidate post cache for a given URI -/// -/// Used when: post, threadgate, or postgate is updated/deleted -/// -/// # Arguments -/// -/// * `uri` - The AT-URI of the post -/// -/// # Returns -/// -/// Cache key in format: `post#{uri}` -#[inline] -pub fn invalidate_post(uri: &str) -> String { - format!("post#{}", uri) -} - -/// Invalidate timeline cache for a given DID (wildcard) -/// -/// Used when: user follows/unfollows/blocks/unblocks someone -/// -/// # Arguments -/// -/// * `did` - The DID of the actor whose timeline cache should be invalidated -/// -/// # Returns -/// -/// Cache key pattern in format: `timeline:{did}:*` -#[inline] -pub fn invalidate_timeline(did: &str) -> String { - format!("timeline:{}:*", did) -} - -/// Invalidate author feed cache for a given DID (wildcard) -/// -/// Used when: user creates/deletes a post or repost -/// -/// # Arguments -/// -/// * `did` - The DID of the actor whose author feed cache should be invalidated -/// -/// # Returns -/// -/// Cache key pattern in format: `authorfeed:{did}:*` -#[inline] -pub fn invalidate_author_feed(did: &str) -> String { - format!("authorfeed:{}:*", did) -} - -/// Invalidate feed generator cache for a given URI -/// -/// Used when: feed generator is created/updated -/// -/// # Arguments -/// -/// * `uri` - The AT-URI of the feed generator -/// -/// # Returns -/// -/// Cache key in format: `feedgen#{uri}` -#[inline] -pub fn invalidate_feedgen(uri: &str) -> String { - format!("feedgen#{}", uri) -} - -/// Invalidate list cache for a given URI -/// -/// Used when: list, list item, or list block is updated -/// -/// # Arguments -/// -/// * `uri` - The AT-URI of the list -/// -/// # Returns -/// -/// Cache key in format: `list#{uri}` -#[inline] -pub fn invalidate_list(uri: &str) -> String { - format!("list#{}", uri) -} - -/// Invalidate starter pack cache for a given URI -/// -/// Used when: starter pack is created/updated/deleted -/// -/// # Arguments -/// -/// * `uri` - The AT-URI of the starter pack -/// -/// # Returns -/// -/// Cache key in format: `starterpacks#{uri}` -#[inline] -pub fn invalidate_starterpack(uri: &str) -> String { - format!("starterpacks#{}", uri) -} - -/// Invalidate labeler cache for a given DID -/// -/// Used when: labeler service is updated/deleted -/// -/// # Arguments -/// -/// * `did` - The DID of the labeler -/// -/// # Returns -/// -/// Cache key in format: `labeler#{did}` -#[inline] -pub fn invalidate_labeler(did: &str) -> String { - format!("labeler#{}", did) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_invalidate_profile() { - let key = invalidate_profile("did:plc:test123"); - assert_eq!(key, "profile#did:plc:test123"); - } - - #[test] - fn test_invalidate_post() { - let key = invalidate_post("at://did:plc:test123/app.bsky.feed.post/abc123"); - assert_eq!(key, "post#at://did:plc:test123/app.bsky.feed.post/abc123"); - } - - #[test] - fn test_invalidate_timeline() { - let key = invalidate_timeline("did:plc:test123"); - assert_eq!(key, "timeline:did:plc:test123:*"); - } - - #[test] - fn test_invalidate_author_feed() { - let key = invalidate_author_feed("did:plc:test123"); - assert_eq!(key, "authorfeed:did:plc:test123:*"); - } - - #[test] - fn test_invalidate_feedgen() { - let key = invalidate_feedgen("at://did:plc:test123/app.bsky.feed.generator/abc123"); - assert_eq!( - key, - "feedgen#at://did:plc:test123/app.bsky.feed.generator/abc123" - ); - } - - #[test] - fn test_invalidate_list() { - let key = invalidate_list("at://did:plc:test123/app.bsky.graph.list/abc123"); - assert_eq!(key, "list#at://did:plc:test123/app.bsky.graph.list/abc123"); - } - - #[test] - fn test_invalidate_starterpack() { - let key = invalidate_starterpack("at://did:plc:test123/app.bsky.graph.starterpack/abc123"); - assert_eq!( - key, - "starterpacks#at://did:plc:test123/app.bsky.graph.starterpack/abc123" - ); - } - - #[test] - fn test_invalidate_labeler() { - let key = invalidate_labeler("did:plc:test123"); - assert_eq!(key, "labeler#did:plc:test123"); - } -} diff --git a/consumer/src/database_writer/operations/executor.rs b/consumer/src/database_writer/operations/executor.rs index 482bd26c..cc9ff15e 100644 --- a/consumer/src/database_writer/operations/executor.rs +++ b/consumer/src/database_writer/operations/executor.rs @@ -98,9 +98,6 @@ pub fn describe_operation(op: &DatabaseOperation) -> (String, Option) { format!("DeleteRecord({:?})", collection), Some(at_uri.clone()), ), - DatabaseOperation::EnqueueFetch { uri, .. } => { - ("EnqueueFetch".to_string(), Some(uri.clone())) - } DatabaseOperation::MaintainSelfLabels { at_uri, .. } => { ("MaintainSelfLabels".to_string(), Some(at_uri.clone())) } @@ -108,12 +105,6 @@ pub fn describe_operation(op: &DatabaseOperation) -> (String, Option) { "MaintainPostgateDetaches".to_string(), Some(post_uri.clone()), ), - DatabaseOperation::CachePdsMapping { did, .. } => { - ("CachePdsMapping".to_string(), Some(did.clone())) - } - DatabaseOperation::EnsureMinimumPostStats { post_uri, .. } => { - ("EnsureMinimumPostStats".to_string(), Some(post_uri.clone())) - } } } @@ -122,13 +113,11 @@ pub fn describe_operation(op: &DatabaseOperation) -> (String, Option) { /// This function will be called by the database writer for each operation. /// /// Counts and cache invalidations are maintained by database triggers. -/// Notifications, fetch queue, and PDS cache use PostgreSQL/moka. +/// Execute database operations pub async fn execute_operation( pool: &deadpool_postgres::Pool, conn: &mut deadpool_postgres::Object, op: DatabaseOperation, - allowlist: &crate::db::Allowlist, - pds_cache: &crate::external::pds_cache::PdsHostCache, ) -> eyre::Result<()> { use crate::db; @@ -330,18 +319,6 @@ pub async fn execute_operation( return Ok(()); } - // Query recipient DID for allowlist check - let recipient_did = db::actor_did_from_id_opt(conn, recipient_actor_id).await?; - - let Some(recipient_did) = recipient_did else { - tracing::debug!("Skipping notification - recipient actor_id not found: {}", recipient_actor_id); - return Ok(()); - }; - - // PostgreSQL-based notifications (only for allowlisted recipients) - if !allowlist.cache.contains_did(&recipient_did) { - return Ok(()); - } // Check if thread is muted (for reply, quote, and mention notifications) // Only posts can be thread roots, so only check for post subjects @@ -495,32 +472,8 @@ pub async fn execute_operation( } }; - // Query ancestor DID for allowlist check - let ancestor_did = db::actor_did_from_id_opt(conn, ancestor_actor_id).await?; - - let Some(ancestor_did) = ancestor_did else { - tracing::debug!( - "Skipping reply-chain notification at level {}: ancestor actor_id={} has no DID", - level, - ancestor_actor_id - ); - level += 1; - if let Some(parent) = ancestor_parent_uri { - current_uri = parent; - continue; - } else { - break; - } - }; - - if !allowlist.cache.contains_did(&ancestor_did) { - // Only notify allowlisted users - tracing::debug!( - "Skipping reply-chain notification at level {}: actor_id={} not allowlisted", - level, - ancestor_actor_id - ); - } else if db::is_thread_muted(conn, ancestor_actor_id, subject_actor_id, subject_rkey_i64).await? { + // Check if thread is muted for this ancestor + if db::is_thread_muted(conn, ancestor_actor_id, subject_actor_id, subject_rkey_i64).await? { // Check if ancestor has muted this thread using actor_ids and i64 rkey tracing::debug!( "Skipping reply-chain notification at level {}: actor_id={} muted thread (root_post_actor_id={}, rkey={})", @@ -656,34 +609,12 @@ pub async fn execute_operation( .await?; Ok(()) } - DatabaseOperation::EnqueueFetch { uri, .. } => { - // Enqueue to PostgreSQL fetch_queue table - crate::db::fetch_queue::enqueue(conn, &uri).await?; - Ok(()) - } DatabaseOperation::UpsertProfile { actor_id, cid, record } => { // Query DID from actor_id (needed by db functions) let did = db::actor_did_from_id(conn, actor_id).await?; db::profile_upsert(conn, actor_id, &did, cid, record).await?; - - // Only enqueue handle resolution if we don't have a handle yet - // Check if the actor has a NULL or empty handle - let row = conn.query_opt( - "SELECT handle FROM actors WHERE id = $1", - &[&actor_id], - ).await?; - - let needs_resolution = row.is_some_and(|r| { - let handle: Option = r.get(0); - handle.is_none_or(|h| h.is_empty()) - }); - - if needs_resolution { - tracing::debug!("Enqueuing handle resolution for actor_id={} (no handle found)", actor_id); - crate::db::handle_resolution_queue::enqueue(pool, &did).await?; - } - + // Note: Handle resolution is now managed by Tap, not enqueued separately Ok(()) } DatabaseOperation::UpsertStatus { actor_id, cid, record } => { @@ -827,29 +758,27 @@ pub async fn execute_operation( // Invalidate post cache if let Ok(post_did) = db::actor_did_from_id(conn, post_actor_id).await { - let post_uri = format!("at://{}/app.bsky.feed.post/{}", + let _post_uri = format!("at://{}/app.bsky.feed.post/{}", post_did, parakeet_db::models::i64_to_tid(post_rkey)); // Database trigger handles cache invalidation // Remove PostgreSQL notification (recipient = author of liked post) - if allowlist.cache.contains_did(&post_did) { - if let Err(e) = - crate::external::pg_notifications::remove_notification( - pool, - actor_id, // author_actor_id (liker) - "like", // record_type - rkey, // record_rkey (i64) - ) - .await - { - tracing::warn!( - "Failed to remove like notification from PostgreSQL: actor_id={}, rkey={}, error={}", - actor_id, - rkey, - e - ); - } + if let Err(e) = + crate::external::pg_notifications::remove_notification( + pool, + actor_id, // author_actor_id (liker) + "like", // record_type + rkey, // record_rkey (i64) + ) + .await + { + tracing::warn!( + "Failed to remove like notification from PostgreSQL: actor_id={}, rkey={}, error={}", + actor_id, + rkey, + e + ); } } } @@ -864,7 +793,7 @@ pub async fn execute_operation( } CollectionType::BskyFeedRepost => { // Returns post URI if deleted - if let Some(post_uri) = db::repost_delete(conn, rkey, actor_id).await? { + if let Some(_post_uri) = db::repost_delete(conn, rkey, actor_id).await? { // Note: repost arrays will be updated in db::repost_delete (remove from post arrays) // No deltas needed with array-only tracking @@ -872,23 +801,21 @@ pub async fn execute_operation( // Database trigger handles cache invalidation // Remove PostgreSQL notification (recipient = author of reposted post) - if allowlist.cache.contains_did(crate::utils::extract_did_from_uri(&post_uri).unwrap_or("")) { - if let Err(e) = - crate::external::pg_notifications::remove_notification( - pool, - actor_id, // author_actor_id (reposter) - "repost", // record_type - rkey, // record_rkey (i64) - ) - .await - { - tracing::warn!( - "Failed to remove repost notification from PostgreSQL: actor_id={}, rkey={}, error={}", - actor_id, - rkey, - e - ); - } + if let Err(e) = + crate::external::pg_notifications::remove_notification( + pool, + actor_id, // author_actor_id (reposter) + "repost", // record_type + rkey, // record_rkey (i64) + ) + .await + { + tracing::warn!( + "Failed to remove repost notification from PostgreSQL: actor_id={}, rkey={}, error={}", + actor_id, + rkey, + e + ); } // Invalidate author feed cache when user deletes a repost @@ -897,27 +824,25 @@ pub async fn execute_operation( } CollectionType::BskyFollow => { // Returns subject (target DID) if deleted - if let Some(target_did) = db::follow_delete(conn, rkey, actor_id).await? { + if let Some(_target_did) = db::follow_delete(conn, rkey, actor_id).await? { // Counts maintained by triggers // Remove PostgreSQL notification (recipient = followed user) - if allowlist.cache.contains_did(&target_did) { - if let Err(e) = - crate::external::pg_notifications::remove_notification( - pool, - actor_id, // author_actor_id (follower) - "follow", // record_type - rkey, // record_rkey (i64) - ) - .await - { - tracing::warn!( - "Failed to remove follow notification from PostgreSQL: actor_id={}, rkey={}, error={}", - actor_id, - rkey, - e - ); - } + if let Err(e) = + crate::external::pg_notifications::remove_notification( + pool, + actor_id, // author_actor_id (follower) + "follow", // record_type + rkey, // record_rkey (i64) + ) + .await + { + tracing::warn!( + "Failed to remove follow notification from PostgreSQL: actor_id={}, rkey={}, error={}", + actor_id, + rkey, + e + ); } // Invalidate timeline cache when user unfollows someone @@ -945,7 +870,7 @@ pub async fn execute_operation( // Invalidate parent post cache since reply array changed // Construct URI using actor cache if let Ok(parent_did) = db::actor_did_from_id(conn, parent_actor_id).await { - let parent_uri = format!("at://{}/app.bsky.feed.post/{}", + let _parent_uri = format!("at://{}/app.bsky.feed.post/{}", parent_did, parakeet_db::models::i64_to_tid(parent_rkey)); // Database trigger handles cache invalidation @@ -966,7 +891,7 @@ pub async fn execute_operation( // Invalidate quoted post cache since quote array changed // Construct URI using actor cache if let Ok(embed_did) = db::actor_did_from_id(conn, embed_actor_id).await { - let embed_uri = format!("at://{}/app.bsky.feed.post/{}", + let _embed_uri = format!("at://{}/app.bsky.feed.post/{}", embed_did, parakeet_db::models::i64_to_tid(embed_rkey)); // Database trigger handles cache invalidation @@ -1098,31 +1023,5 @@ pub async fn execute_operation( db::postgate_maintain_detaches_cached(conn, &post_uri, &detached_uris, disable_effective).await?; Ok(()) } - DatabaseOperation::CachePdsMapping { - did, - host, - .. - } => { - // Cache PDS mapping in moka-based cache (in-memory, TTL: 24h) - pds_cache.set(did, host).await; - Ok(()) - } - DatabaseOperation::EnsureMinimumPostStats { - post_uri: _, - min_likes: _, - min_reposts: _, - min_quotes: _, - min_replies: _, - } => { - // DEPRECATED: EnsureMinimumPostStats is incompatible with array-only tracking. - // With arrays, we can't fabricate engagement entries to reach minimum counts - // (we'd need actual actor IDs and rkeys, not just synthetic counts). - // The Constellation enrichment feature that used this may need to be redesigned - // or removed entirely. - // - // For now, this operation is a no-op to avoid breaking callers. - // TODO: Remove this operation entirely once callers are updated. - Ok(()) - } } } diff --git a/consumer/src/database_writer/operations/handlers/block.rs b/consumer/src/database_writer/operations/handlers/block.rs index 1a1b2342..99abb5b4 100644 --- a/consumer/src/database_writer/operations/handlers/block.rs +++ b/consumer/src/database_writer/operations/handlers/block.rs @@ -6,13 +6,13 @@ use crate::types::records::AppBskyGraphBlock; pub fn handle_block( ctx: &super::RecordContext, record: AppBskyGraphBlock, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid block TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -34,5 +34,5 @@ pub fn handle_block( // Invalidate timeline cache when user blocks someone - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/bookmark.rs b/consumer/src/database_writer/operations/handlers/bookmark.rs index 47332bba..8cfe02d3 100644 --- a/consumer/src/database_writer/operations/handlers/bookmark.rs +++ b/consumer/src/database_writer/operations/handlers/bookmark.rs @@ -6,11 +6,11 @@ use lexica::community_lexicon::bookmarks::Bookmark; pub fn handle_bookmark( ctx: &super::RecordContext, record: Bookmark, -) -> (Vec, Vec) { +) -> Vec { // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid bookmark TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -23,5 +23,5 @@ pub fn handle_bookmark( record, }]; - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/declarations.rs b/consumer/src/database_writer/operations/handlers/declarations.rs index ba17f6ff..d7c02eb2 100644 --- a/consumer/src/database_writer/operations/handlers/declarations.rs +++ b/consumer/src/database_writer/operations/handlers/declarations.rs @@ -6,7 +6,7 @@ use crate::types::records::{AppBskyNotificationDeclaration, ChatBskyActorDeclara pub fn handle_notification_declaration( ctx: &super::RecordContext, record: AppBskyNotificationDeclaration, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); if ctx.rkey == "self" { @@ -16,13 +16,13 @@ pub fn handle_notification_declaration( }); } - (operations, vec![]) + operations } pub fn handle_chat_declaration( ctx: &super::RecordContext, record: ChatBskyActorDeclaration, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); if ctx.rkey == "self" { @@ -32,5 +32,5 @@ pub fn handle_chat_declaration( }); } - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/feedgen.rs b/consumer/src/database_writer/operations/handlers/feedgen.rs index 442a7c21..68db4daa 100644 --- a/consumer/src/database_writer/operations/handlers/feedgen.rs +++ b/consumer/src/database_writer/operations/handlers/feedgen.rs @@ -6,7 +6,7 @@ use crate::types::records::AppBskyFeedGenerator; pub fn handle_feedgen( ctx: &super::RecordContext, record: AppBskyFeedGenerator, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); let labels = record.labels.clone(); @@ -21,7 +21,7 @@ pub fn handle_feedgen( "Feedgen record missing service_actor_id - reference extraction may have filtered out the DID. \ This can happen if the DID field is empty, whitespace-only, or actor creation failed." ); - return (operations, vec![]); + return operations; }; operations.push(DatabaseOperation::UpsertFeedGenerator { @@ -44,5 +44,5 @@ pub fn handle_feedgen( // Note: Can't determine did_insert without database, so we queue cache invalidation // The batch writer will decide based on INSERT result - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/follow.rs b/consumer/src/database_writer/operations/handlers/follow.rs index bfae6c76..31700a75 100644 --- a/consumer/src/database_writer/operations/handlers/follow.rs +++ b/consumer/src/database_writer/operations/handlers/follow.rs @@ -6,13 +6,13 @@ use crate::types::records::AppBskyGraphFollow; pub fn handle_follow( ctx: &super::RecordContext, record: AppBskyGraphFollow, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid follow TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } let follow_created_at = crate::database_writer::timestamp::decode_tid_timestamp(&ctx.rkey) @@ -50,7 +50,5 @@ pub fn handle_follow( follow_created_at, )); - // Invalidate timeline cache when user follows someone - - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/labeler.rs b/consumer/src/database_writer/operations/handlers/labeler.rs index dd5279ce..1e278cb6 100644 --- a/consumer/src/database_writer/operations/handlers/labeler.rs +++ b/consumer/src/database_writer/operations/handlers/labeler.rs @@ -6,7 +6,7 @@ use crate::types::records::AppBskyLabelerService; pub fn handle_labeler( ctx: &super::RecordContext, record: AppBskyLabelerService, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); if ctx.rkey == "self" { @@ -29,5 +29,5 @@ pub fn handle_labeler( } - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/like.rs b/consumer/src/database_writer/operations/handlers/like.rs index 4d4866f7..9aa971f0 100644 --- a/consumer/src/database_writer/operations/handlers/like.rs +++ b/consumer/src/database_writer/operations/handlers/like.rs @@ -6,7 +6,7 @@ use crate::types::records::AppBskyFeedLike; pub fn handle_like( ctx: &super::RecordContext, record: AppBskyFeedLike, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); // Extract recipient DID from the liked post URI @@ -15,7 +15,7 @@ pub fn handle_like( // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid like TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } let like_created_at = crate::database_writer::timestamp::decode_tid_timestamp(&ctx.rkey) @@ -53,5 +53,5 @@ pub fn handle_like( } } - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/list.rs b/consumer/src/database_writer/operations/handlers/list.rs index d72b3663..49f8c455 100644 --- a/consumer/src/database_writer/operations/handlers/list.rs +++ b/consumer/src/database_writer/operations/handlers/list.rs @@ -7,7 +7,7 @@ use crate::utils::at_uri_is_by; pub fn handle_list( ctx: &super::RecordContext, record: AppBskyGraphList, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); // Lists can use both TID and arbitrary string rkeys (like "nfb", "bblock") @@ -15,7 +15,7 @@ pub fn handle_list( if parakeet_db::models::tid_to_i64(&ctx.rkey).is_ok() { if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid list TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } } @@ -39,17 +39,17 @@ pub fn handle_list( // Queue cache invalidation (batch writer decides based on insert result) - (operations, vec![]) + operations } pub fn handle_list_block( ctx: &super::RecordContext, record: AppBskyGraphListBlock, -) -> (Vec, Vec) { +) -> Vec { // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid list_block TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -63,22 +63,22 @@ pub fn handle_list_block( record, }]; - (operations, vec![]) + operations } pub fn handle_list_item( ctx: &super::RecordContext, record: AppBskyGraphListItem, -) -> (Vec, Vec) { +) -> Vec { if !at_uri_is_by(&record.list, &ctx.repo) { tracing::warn!("tried to create a listitem on a list we don't control!"); - return (vec![], vec![]); + return vec![]; } // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid list_item TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -95,5 +95,5 @@ pub fn handle_list_item( record, }]; - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/misc.rs b/consumer/src/database_writer/operations/handlers/misc.rs index 02914221..b7839bf3 100644 --- a/consumer/src/database_writer/operations/handlers/misc.rs +++ b/consumer/src/database_writer/operations/handlers/misc.rs @@ -6,11 +6,11 @@ use crate::types::records::FmTealAlpaActorStatus; pub fn handle_alpa_status( ctx: &super::RecordContext, _record: FmTealAlpaActorStatus, -) -> (Vec, Vec) { +) -> Vec { if ctx.rkey == "self" { // TODO: implement fm.teal.alpa.actor.status table } - (vec![], vec![]) + vec![] } diff --git a/consumer/src/database_writer/operations/handlers/mod.rs b/consumer/src/database_writer/operations/handlers/mod.rs index b51627ef..00fc7cce 100644 --- a/consumer/src/database_writer/operations/handlers/mod.rs +++ b/consumer/src/database_writer/operations/handlers/mod.rs @@ -28,7 +28,7 @@ pub struct RecordContext { pub at_uri: String, /// Record key (TID as string) pub rkey: String, - /// Event source (Jetstream or Backfill) + /// Event source (Tap or TapBackfill) pub source: EventSource, } diff --git a/consumer/src/database_writer/operations/handlers/post.rs b/consumer/src/database_writer/operations/handlers/post.rs index 3982d2f4..9cca3b85 100644 --- a/consumer/src/database_writer/operations/handlers/post.rs +++ b/consumer/src/database_writer/operations/handlers/post.rs @@ -10,7 +10,7 @@ use crate::utils::at_uri_is_by; pub fn handle_post( post_ctx: &super::PostContext, record: AppBskyFeedPost, -) -> (Vec, Vec) { +) -> Vec { let ctx = &post_ctx.ctx; let actor_id = ctx.actor_id; let cid = ctx.cid; @@ -34,7 +34,7 @@ pub fn handle_post( &record.embed.as_ref().and_then(|v| v.as_bsky()) { if !embed.media.record_with_media_allowed() { - return (vec![], vec![]); + return vec![]; } } @@ -68,7 +68,7 @@ pub fn handle_post( // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(rkey) { tracing::warn!("Invalid post TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } let post_created_at = crate::database_writer::timestamp::decode_tid_timestamp(rkey) @@ -185,7 +185,7 @@ pub fn handle_post( // Notify up to 3 additional levels of ancestors in the reply chain // PERFORMANCE: Skip during backfill - these deep notifications are low priority // and each one requires expensive database queries in a loop - if source != EventSource::Backfill { + if source != EventSource::TapBackfill { operations.push(DatabaseOperation::NotifyReplyChain { reply_uri: at_uri.clone(), author_actor_id: actor_id, @@ -212,16 +212,16 @@ pub fn handle_post( // Invalidate author feed cache when user creates a post - (operations, vec![]) + operations } pub fn handle_postgate( ctx: &super::RecordContext, record: AppBskyFeedPostgate, -) -> (Vec, Vec) { +) -> Vec { if !at_uri_is_by(&record.post, &ctx.repo) { tracing::warn!("tried to create a postgate on a post we don't control!"); - return (vec![], vec![]); + return vec![]; } let mut operations = Vec::new(); @@ -229,7 +229,7 @@ pub fn handle_postgate( // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid postgate TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -258,16 +258,16 @@ pub fn handle_postgate( .map(|dt| chrono::DateTime::::from_naive_utc_and_offset(dt, chrono::Utc)), }); - (operations, vec![]) + operations } pub fn handle_threadgate( ctx: &super::RecordContext, record: AppBskyFeedThreadgate, -) -> (Vec, Vec) { +) -> Vec { if !at_uri_is_by(&record.post, &ctx.repo) { tracing::warn!("tried to create a threadgate on a post we don't control!"); - return (vec![], vec![]); + return vec![]; } let mut operations = Vec::new(); @@ -275,7 +275,7 @@ pub fn handle_threadgate( // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid threadgate TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -290,5 +290,5 @@ pub fn handle_threadgate( }); - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/profile.rs b/consumer/src/database_writer/operations/handlers/profile.rs index b9537107..09be45c7 100644 --- a/consumer/src/database_writer/operations/handlers/profile.rs +++ b/consumer/src/database_writer/operations/handlers/profile.rs @@ -7,7 +7,7 @@ use crate::utils::at_uri_is_by; pub fn handle_profile( ctx: &super::RecordContext, mut record: AppBskyActorProfile, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); if ctx.rkey == "self" { @@ -37,13 +37,13 @@ pub fn handle_profile( } - (operations, vec![]) + operations } pub fn handle_status( ctx: &super::RecordContext, record: AppBskyActorStatus, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); if ctx.rkey == "self" { @@ -54,5 +54,5 @@ pub fn handle_status( }); } - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/repost.rs b/consumer/src/database_writer/operations/handlers/repost.rs index dae87897..0842c467 100644 --- a/consumer/src/database_writer/operations/handlers/repost.rs +++ b/consumer/src/database_writer/operations/handlers/repost.rs @@ -6,7 +6,7 @@ use crate::types::records::AppBskyFeedRepost; pub fn handle_repost( ctx: &super::RecordContext, record: AppBskyFeedRepost, -) -> (Vec, Vec) { +) -> Vec { let mut operations = Vec::new(); let subject_uri = record.subject.uri.clone(); @@ -14,7 +14,7 @@ pub fn handle_repost( // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid repost TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } let repost_created_at = crate::database_writer::timestamp::decode_tid_timestamp(&ctx.rkey) @@ -59,5 +59,5 @@ pub fn handle_repost( // Invalidate author feed cache when user creates a repost - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/starterpack.rs b/consumer/src/database_writer/operations/handlers/starterpack.rs index 5876a710..43b8772a 100644 --- a/consumer/src/database_writer/operations/handlers/starterpack.rs +++ b/consumer/src/database_writer/operations/handlers/starterpack.rs @@ -6,12 +6,12 @@ use crate::types::records::AppBskyGraphStarterPack; pub fn handle_starterpack( ctx: &super::RecordContext, record: AppBskyGraphStarterPack, -) -> (Vec, Vec) { +) -> Vec { // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid starter pack TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -26,5 +26,5 @@ pub fn handle_starterpack( }]; - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/handlers/verification.rs b/consumer/src/database_writer/operations/handlers/verification.rs index dd9ad3c0..038403d8 100644 --- a/consumer/src/database_writer/operations/handlers/verification.rs +++ b/consumer/src/database_writer/operations/handlers/verification.rs @@ -6,11 +6,11 @@ use crate::types::records::AppBskyGraphVerification; pub fn handle_verification( ctx: &super::RecordContext, record: AppBskyGraphVerification, -) -> (Vec, Vec) { +) -> Vec { // Validate TID timestamp is within acceptable range if let Err(e) = crate::database_writer::validate_tid_timestamp(&ctx.rkey) { tracing::warn!("Invalid verification TID timestamp: {}", e); - return (vec![], vec![]); + return vec![]; } // Convert TID string to i64 (safe because validation passed above) @@ -26,5 +26,5 @@ pub fn handle_verification( record, }]; - (operations, vec![]) + operations } diff --git a/consumer/src/database_writer/operations/mod.rs b/consumer/src/database_writer/operations/mod.rs index e93d8c6c..13a8cf30 100644 --- a/consumer/src/database_writer/operations/mod.rs +++ b/consumer/src/database_writer/operations/mod.rs @@ -13,8 +13,6 @@ //! //! ```text //! AT Protocol Record → Handler → DatabaseOperation[] → Database Writer -//! ↓ -//! Cache Invalidation[] //! ``` //! //! ## Module Structure @@ -33,7 +31,7 @@ //! ### `handlers/` - Record Type Handlers //! Specialized modules for each AT Protocol record type. Each handler: //! - Takes a parsed record and metadata (repo, actor_id, cid, etc.) -//! - Returns `(operations, cache_invalidations)` tuple +//! - Returns operations to be executed //! - Is pure - no database I/O, no side effects //! //! Available handlers: @@ -114,7 +112,6 @@ //! Each handler module includes comprehensive unit tests. Helper modules //! (notifications, cache) also have their own test suites. -pub mod cache; pub mod executor; pub mod handlers; pub mod notifications; @@ -122,9 +119,7 @@ pub mod processor; pub mod types; pub use processor::process_record_to_operations; -pub use types::DatabaseOperation; +pub use types::{DatabaseOperation, ResolvedActorIds}; // SelfLabels available via types::SelfLabels if needed, but most code imports from lexica directly // Notification helpers re-exported for convenience pub use notifications::*; -// Cache invalidation helpers re-exported for convenience -pub use cache::*; diff --git a/consumer/src/database_writer/operations/processor.rs b/consumer/src/database_writer/operations/processor.rs index 2b69e8ab..4845d334 100644 --- a/consumer/src/database_writer/operations/processor.rs +++ b/consumer/src/database_writer/operations/processor.rs @@ -18,13 +18,13 @@ use ipld_core::cid::Cid; /// - `actor_id`: The resolved actor ID for the repo (caller must resolve via ensure_actor_id) /// - `resolved_actor_ids`: All resolved actor IDs from the record (subject, parent, root, quoted, mentioned) /// -/// NOTE: With Jetstream server-side filtering, all events are from allowlisted DIDs. +/// NOTE: With Tap filtering, all events are from allowlisted DIDs. /// We always enqueue related posts (replies/quotes) for fetching to ensure completeness. #[expect(clippy::too_many_arguments, reason = "Comprehensive record processing requires multiple resolved IDs and metadata")] pub fn process_record_to_operations( repo: &str, actor_id: i32, - resolved_actor_ids: crate::database_writer::workers::ResolvedActorIds, + resolved_actor_ids: super::ResolvedActorIds, cid: Cid, record: RecordTypes, at_uri: String, @@ -46,7 +46,7 @@ pub fn process_record_to_operations( source, }; - let (operations, cache_invalidations) = match record { + let operations = match record { RecordTypes::AppBskyActorProfile(record) => handlers::handle_profile(&ctx, record), RecordTypes::AppBskyActorStatus(record) => handlers::handle_status(&ctx, record), RecordTypes::AppBskyFeedGenerator(record) => handlers::handle_feedgen(&ctx, record), @@ -80,6 +80,10 @@ pub fn process_record_to_operations( } RecordTypes::CommunityLexiconBookmark(record) => handlers::handle_bookmark(&ctx, record), RecordTypes::FmTealAlpaActorStatus(record) => handlers::handle_alpa_status(&ctx, record), + RecordTypes::Unknown(_value) => { + // Unknown record types are skipped + vec![] + } }; ProcessedEvent { diff --git a/consumer/src/database_writer/operations/types.rs b/consumer/src/database_writer/operations/types.rs index 65061037..fa9a258d 100644 --- a/consumer/src/database_writer/operations/types.rs +++ b/consumer/src/database_writer/operations/types.rs @@ -7,6 +7,41 @@ use crate::types::records::*; use chrono::{DateTime, Utc}; use ipld_core::cid::Cid; +/// Resolved actor IDs for record processing +/// +/// Contains all resolved FK references needed to process a record +/// without further database queries. +pub struct ResolvedActorIds { + /// Subject actor ID (for Follow, Block, ListItem, Verification) + pub subject_actor_id: Option, + + /// Via repost natural key (for likes/reposts that came via a repost) + pub via_repost_key: Option<(i32, i64)>, + + /// Service actor ID (for FeedGenerator - the DID hosting the feed) + pub service_actor_id: Option, + + /// Post-specific resolved actor IDs (for notifications) + pub parent_author_actor_id: Option, + pub root_author_actor_id: Option, + pub quoted_author_actor_id: Option, + pub mentioned_actor_ids: Vec, +} + +impl ResolvedActorIds { + pub fn empty() -> Self { + Self { + via_repost_key: None, + subject_actor_id: None, + service_actor_id: None, + parent_author_actor_id: None, + root_author_actor_id: None, + quoted_author_actor_id: None, + mentioned_actor_ids: Vec::new(), + } + } +} + /// Self-labels data for posts/profiles/etc (from AT Protocol) /// Re-export of lexica SelfLabels for convenience /// Used in DatabaseOperation::MaintainSelfLabels variant @@ -214,11 +249,6 @@ pub enum DatabaseOperation { rkey: i64, // TID converted to i64 (or 0 for non-TID collections like feedgens) }, - EnqueueFetch { - uri: String, - priority: i32, - }, - MaintainSelfLabels { actor_id: i32, cid: Option, @@ -231,23 +261,4 @@ pub enum DatabaseOperation { detached_uris: Vec, disable_effective: Option>, }, - - // PDS mapping cache - CachePdsMapping { - did: String, - host: String, - source: String, // "profile_fetch", "did_doc", etc. - }, - - // Constellation enrichment - /// Ensure post stats in RocksDB are at least the specified values (from Constellation) - /// This operation queries current stats and sends positive deltas to reach minimums. - /// Used for high-engagement posts where we don't want to fetch individual records. - EnsureMinimumPostStats { - post_uri: String, - min_likes: u64, - min_reposts: u64, - min_quotes: u64, - min_replies: u64, - }, } diff --git a/consumer/src/database_writer/routing.rs b/consumer/src/database_writer/routing.rs index 92e7692f..da60084d 100644 --- a/consumer/src/database_writer/routing.rs +++ b/consumer/src/database_writer/routing.rs @@ -1,29 +1,9 @@ //! Worker routing and load balancing //! -//! This module handles routing DatabaseOperation events to specialized worker pools -//! based on operation type and event source. Uses round-robin load balancing within -//! each worker pool for fair distribution. -//! -//! ## Architecture -//! -//! Workers are organized into 15 pools (3 sources × 5 types): -//! - **Jetstream** (real-time events - highest priority) -//! - **Backfill** (bulk historical data - low priority) -//! - **FetchQueue** (on-demand record fetching - medium priority) -//! -//! Each source has 5 worker types (3 workers per type = 45 total workers): -//! 1. **Actor** - Handles rare actor upserts (handle resolution, account events) -//! 2. **Like** - Handles likes (highest throughput operation) -//! 3. **Social** - Handles follows, reposts, blocks -//! 4. **Post** - Handles posts and embeds -//! 5. **Metadata** - Handles profiles, gates, lists, etc. +//! This module handles classification of DatabaseOperation events to determine +//! which worker type should handle them. use super::{DatabaseOperation, EventSource}; -use tokio::sync::mpsc::UnboundedSender; - -/// Number of workers per (source, type) combination -/// Hardcoded to 3 workers per pool for parallelism without excessive overhead -pub const WORKERS_PER_TYPE: usize = 3; /// Worker type for operation routing #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -46,8 +26,7 @@ pub fn classify_operation(op: &DatabaseOperation) -> WorkerType { | DatabaseOperation::InsertRepost { .. } | DatabaseOperation::InsertBlock { .. } => WorkerType::Social, - DatabaseOperation::InsertPost { .. } - | DatabaseOperation::EnsureMinimumPostStats { .. } => WorkerType::Post, + DatabaseOperation::InsertPost { .. } => WorkerType::Post, // Everything else goes to metadata worker _ => WorkerType::Metadata, @@ -58,138 +37,9 @@ pub fn classify_operation(op: &DatabaseOperation) -> WorkerType { pub struct PartitionedEvent { /// Operations for this worker pub operations: Vec, - /// Cursor update (only relevant for Jetstream events) - /// Contains Jetstream cursor timestamp in microseconds + /// Cursor update (only relevant for Tap events) + /// Contains cursor timestamp in microseconds pub cursor: Option, /// Event source pub source: EventSource, -} - -/// Worker pools organized by event source and operation type -/// Total: 45 workers (3 sources × 5 types × 3 workers each) -#[derive(Clone)] -pub struct WorkerPools { - // Jetstream workers (real-time events - highest priority) - jetstream_actor: Vec>, - jetstream_like: Vec>, - jetstream_social: Vec>, - jetstream_post: Vec>, - jetstream_metadata: Vec>, - - // Backfill workers (bulk historical data - low priority) - backfill_actor: Vec>, - backfill_like: Vec>, - backfill_social: Vec>, - backfill_post: Vec>, - backfill_metadata: Vec>, - - // Fetch workers (on-demand record fetching - medium priority) - fetch_actor: Vec>, - fetch_like: Vec>, - fetch_social: Vec>, - fetch_post: Vec>, - fetch_metadata: Vec>, - - // Round-robin counters for fair distribution across workers - counters: std::sync::Arc< - std::sync::Mutex>, - >, -} - -impl WorkerPools { - /// Create a new WorkerPools with the provided worker senders - #[expect(clippy::too_many_arguments, reason = "Constructor mirrors 15 worker pool structure (3 sources × 5 types)")] - pub fn new( - jetstream_actor: Vec>, - jetstream_like: Vec>, - jetstream_social: Vec>, - jetstream_post: Vec>, - jetstream_metadata: Vec>, - backfill_actor: Vec>, - backfill_like: Vec>, - backfill_social: Vec>, - backfill_post: Vec>, - backfill_metadata: Vec>, - fetch_actor: Vec>, - fetch_like: Vec>, - fetch_social: Vec>, - fetch_post: Vec>, - fetch_metadata: Vec>, - ) -> Self { - Self { - jetstream_actor, - jetstream_like, - jetstream_social, - jetstream_post, - jetstream_metadata, - backfill_actor, - backfill_like, - backfill_social, - backfill_post, - backfill_metadata, - fetch_actor, - fetch_like, - fetch_social, - fetch_post, - fetch_metadata, - counters: std::sync::Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), - } - } - - /// Get the next worker sender for a given source and type using round-robin - pub fn get_worker( - &self, - source: EventSource, - worker_type: WorkerType, - ) -> &UnboundedSender { - // Get next worker index (round-robin) - let mut map = self.counters.lock().unwrap(); - let counter = map.entry((source, worker_type)).or_insert(0); - let idx = *counter % WORKERS_PER_TYPE; - *counter = counter.wrapping_add(1); - drop(map); - - // Select worker pool based on source and type - let pool = match (source, worker_type) { - (EventSource::Jetstream, WorkerType::Actor) => &self.jetstream_actor, - (EventSource::Jetstream, WorkerType::Like) => &self.jetstream_like, - (EventSource::Jetstream, WorkerType::Social) => &self.jetstream_social, - (EventSource::Jetstream, WorkerType::Post) => &self.jetstream_post, - (EventSource::Jetstream, WorkerType::Metadata) => &self.jetstream_metadata, - - (EventSource::Backfill, WorkerType::Actor) => &self.backfill_actor, - (EventSource::Backfill, WorkerType::Like) => &self.backfill_like, - (EventSource::Backfill, WorkerType::Social) => &self.backfill_social, - (EventSource::Backfill, WorkerType::Post) => &self.backfill_post, - (EventSource::Backfill, WorkerType::Metadata) => &self.backfill_metadata, - - (EventSource::FetchQueue, WorkerType::Actor) => &self.fetch_actor, - (EventSource::FetchQueue, WorkerType::Like) => &self.fetch_like, - (EventSource::FetchQueue, WorkerType::Social) => &self.fetch_social, - (EventSource::FetchQueue, WorkerType::Post) => &self.fetch_post, - (EventSource::FetchQueue, WorkerType::Metadata) => &self.fetch_metadata, - }; - - &pool[idx] - } - - /// Get the first metadata worker for a given source (used for cursor/cache operations) - pub fn get_metadata_worker(&self, source: EventSource) -> &UnboundedSender { - match source { - EventSource::Jetstream => &self.jetstream_metadata[0], - EventSource::Backfill => &self.backfill_metadata[0], - EventSource::FetchQueue => &self.fetch_metadata[0], - } - } -} - -/// Convert partition string to Partition enum -pub fn partition_str_to_enum(s: &str) -> Option { - match s { - "likes" => Some(crate::sources::unified_consumer::Partition::Likes), - "posts" => Some(crate::sources::unified_consumer::Partition::Posts), - "reposts" => Some(crate::sources::unified_consumer::Partition::Reposts), - "social" => Some(crate::sources::unified_consumer::Partition::Social), - _ => None, - } -} +} \ No newline at end of file diff --git a/consumer/src/database_writer/workers.rs b/consumer/src/database_writer/workers.rs deleted file mode 100644 index 7790860c..00000000 --- a/consumer/src/database_writer/workers.rs +++ /dev/null @@ -1,1306 +0,0 @@ -//! Database writer task -//! -//! This module implements the background task that drains the event queue -//! and performs database writes asynchronously. - -use super::operations::executor; -use super::routing::{self, PartitionedEvent, WorkerPools, WORKERS_PER_TYPE}; -use super::ProcessedEvent; -use deadpool_postgres::Pool; -use eyre::Context as _; -use metrics::counter; - -/// Spawn a bulk processing worker -/// -/// Bulk workers handle UnresolvedBulk events asynchronously, preventing the dispatcher -/// from being blocked during bulk record processing (which can be 10k+ records per event). -fn spawn_bulk_worker( - name: &'static str, - pool: Pool, - mut rx: tokio::sync::mpsc::UnboundedReceiver, - allowlist: std::sync::Arc, - pds_cache: std::sync::Arc, -) { - tokio::spawn(async move { - tracing::info!(worker = name, "Bulk worker started"); - - while let Some(event) = rx.recv().await { - match event { - super::WriterEvent::UnresolvedBulk { repo, actor_id, records, source: event_source } => { - tracing::info!( - worker = name, - repo = %repo, - actor_id = actor_id, - records = records.len(), - "Processing bulk unresolved event" - ); - - // Get a connection from the pool - let conn = match pool.get().await { - Ok(conn) => conn, - Err(e) => { - tracing::error!(worker = name, error = ?e, "Failed to get database connection for bulk processing"); - continue; - } - }; - - // Process records into bulk operations - let bulk_ops = match super::bulk_processor::process_bulk_records( - &conn, - &repo, - actor_id, - records, - event_source, - ).await { - Ok(ops) => ops, - Err(e) => { - tracing::error!(worker = name, error = ?e, repo = %repo, "Failed to process bulk records"); - continue; - } - }; - - // Execute bulk COPY operations - let mut conn_mut = conn; - match execute_bulk_operations( - &pool, - &mut conn_mut, - &repo, - actor_id, - bulk_ops, - event_source, - &allowlist, - &pds_cache, - ).await { - Ok(()) => { - // All counts maintained by database triggers - } - Err(e) => { - tracing::error!(worker = name, error = ?e, repo = %repo, "Failed to execute bulk COPY operations"); - } - } - } - _ => { - tracing::warn!(worker = name, "Bulk worker received non-bulk event, ignoring"); - } - } - } - - tracing::info!(worker = name, "Bulk worker stopped"); - }); -} - -/// Execute bulk COPY operations -/// -/// This function: -/// 1. Executes COPY operations for each table type (likes, follows, reposts, blocks) -/// 2. Collects post stats deltas -/// 3. Returns accumulated deltas for batch updating -#[allow(clippy::too_many_arguments)] -async fn execute_bulk_operations( - pool: &deadpool_postgres::Pool, - conn: &mut deadpool_postgres::Object, - repo: &str, - actor_id: i32, - bulk_ops: super::BulkOperations, - _source: super::EventSource, - allowlist: &crate::db::Allowlist, - pds_cache: &crate::external::pds_cache::PdsHostCache, -) -> eyre::Result<()> { - - let start = std::time::Instant::now(); - let total_ops = bulk_ops.total_count(); - - tracing::info!( - repo = %repo, - actor_id = actor_id, - post_likes = bulk_ops.post_likes.len(), - feedgen_likes = bulk_ops.feedgen_likes.len(), - labeler_likes = bulk_ops.labeler_likes.len(), - follows = bulk_ops.follows.len(), - reposts = bulk_ops.reposts.len(), - blocks = bulk_ops.blocks.len(), - posts = bulk_ops.posts.len(), - individual = bulk_ops.individual_ops.len(), - total = total_ops, - "Executing bulk COPY operations" - ); - - // Begin transaction for all bulk operations - // This is critical - temporary tables created with ON COMMIT DROP - // must be in the same transaction as the data inserts - let txn = conn.transaction().await?; - - let mut total_inserted = 0; - - // Execute COPY for post likes (per-actor updates to respect decompression limits) - if !bulk_ops.post_likes.is_empty() { - let op_start = std::time::Instant::now(); - let count = bulk_ops.post_likes.len(); - match crate::db::bulk_copy::copy_post_likes(&txn, bulk_ops.post_likes).await { - Ok(inserted) => { - let op_elapsed = op_start.elapsed(); - total_inserted += inserted.len(); - - // Like arrays are updated directly in bulk_copy - - tracing::info!( - count = count, - inserted = total_inserted, - duration_ms = op_elapsed.as_millis(), - "Bulk COPY post_likes completed" - ); - if op_elapsed.as_millis() > 1000 { - tracing::warn!( - count = count, - duration_ms = op_elapsed.as_millis(), - "Slow post_likes COPY (>1s)" - ); - } - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY post_likes"); - } - } - } - - // Execute COPY for feedgen likes - if !bulk_ops.feedgen_likes.is_empty() { - match crate::db::bulk_copy::copy_feedgen_likes(&txn, bulk_ops.feedgen_likes).await { - Ok(inserted) => { - total_inserted += inserted.len(); - // Note: like_count is maintained automatically by trigger (update_feedgen_like_count) - tracing::info!(inserted = inserted.len(), "Bulk COPY feedgen_likes completed"); - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY feedgen_likes"); - } - } - } - - // Execute COPY for labeler likes - if !bulk_ops.labeler_likes.is_empty() { - match crate::db::bulk_copy::copy_labeler_likes(&txn, bulk_ops.labeler_likes).await { - Ok(inserted) => { - total_inserted += inserted.len(); - - // Update labeler like_count aggregates - let labeler_actor_ids: Vec = inserted.iter().map(|like| like.labeler_actor_id).collect(); - if let Err(e) = crate::db::operations::increment_labeler_like_counts(&txn, &labeler_actor_ids).await { - tracing::error!(error = ?e, "Failed to update labeler like_count aggregates"); - } - - tracing::info!(inserted = inserted.len(), "Bulk COPY labeler_likes completed"); - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY labeler_likes"); - } - } - } - - // Execute COPY for follows - // Follow counts are computed on-demand via ProfileStatsLoader COUNT queries - if !bulk_ops.follows.is_empty() { - match crate::db::bulk_copy::copy_follows(&txn, bulk_ops.follows).await { - Ok(inserted) => { - total_inserted += inserted.len(); - tracing::info!(inserted = inserted.len(), "Bulk COPY follows completed"); - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY follows"); - } - } - } - - // Execute COPY for reposts - if !bulk_ops.reposts.is_empty() { - match crate::db::bulk_copy::copy_reposts(&txn, bulk_ops.reposts).await { - Ok(inserted) => { - total_inserted += inserted.len(); - - // Repost arrays are updated directly in bulk_copy - - tracing::info!(inserted = inserted.len(), "Bulk COPY reposts completed"); - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY reposts"); - } - } - } - - // Execute COPY for blocks - if !bulk_ops.blocks.is_empty() { - match crate::db::bulk_copy::copy_blocks(&txn, bulk_ops.blocks).await { - Ok(inserted) => { - total_inserted += inserted.len(); - // Blocks don't generate aggregate deltas (no public counters) - tracing::info!(inserted = inserted.len(), "Bulk COPY blocks completed"); - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY blocks"); - } - } - } - - // Execute COPY for posts - if !bulk_ops.posts.is_empty() { - let op_start = std::time::Instant::now(); - let post_count = bulk_ops.posts.len(); - - match crate::db::bulk_copy::copy_posts(&txn, bulk_ops.posts.clone()).await { - Ok(inserted) => { - let op_elapsed = op_start.elapsed(); - tracing::info!( - total = post_count, - inserted = inserted.len(), - duration_ms = op_elapsed.as_millis(), - "Bulk COPY posts main table completed" - ); - if op_elapsed.as_millis() > 500 { - tracing::warn!( - count = post_count, - duration_ms = op_elapsed.as_millis(), - "Slow posts COPY (>500ms)" - ); - } - total_inserted += inserted.len(); - - // Generate deltas for posts - for post_data in &bulk_ops.posts { - // Reply/quote arrays are updated directly in bulk_copy - } - - // Execute child table inserts - if !inserted.is_empty() { - // Images - // NOTE: All embeds (images, videos, external, record), facets, and mentions - // are now stored as composite types directly in the posts table. - // No need for separate inserts - they're all handled by copy_posts - } - } - Err(e) => { - tracing::error!(error = ?e, "Failed to bulk COPY posts"); - } - } - } - - // Commit the bulk transaction - // All temporary tables (ON COMMIT DROP) will be cleaned up automatically - txn.commit().await?; - - // Execute individual operations through normal executor - // These are operations that can't be bulk-processed (posts, profiles, gates, etc) - if !bulk_ops.individual_ops.is_empty() { - tracing::debug!( - count = bulk_ops.individual_ops.len(), - "Executing individual operations from bulk batch" - ); - - for op in bulk_ops.individual_ops { - match super::operations::executor::execute_operation( - pool, - conn, - op, - allowlist, - pds_cache, - ).await { - Ok(()) => { - // Actor counts and cache maintained by triggers - } - Err(e) => { - tracing::error!(error = ?e, "Failed to execute individual operation from bulk batch"); - } - } - } - } - - let elapsed = start.elapsed(); - tracing::info!( - repo = %repo, - total_ops = total_ops, - total_inserted = total_inserted, - duration_ms = elapsed.as_millis(), - "Bulk COPY execution completed" - ); - - Ok(()) -} - -/// Batch update post stats in the database -/// -/// Takes accumulated deltas and updates denormalized stats columns in posts table. -/// Uses CASE-batched UPDATEs for efficiency (500 posts per UPDATE statement). -/// Now uses the consolidated PostUpdate infrastructure. -/// Batch update actor stats in the database -/// DEPRECATED: Counts are maintained by database triggers -/// All actor counts (followers_count, following_count, posts_count) are maintained -/// automatically by the update_actor_counts() trigger when arrays are modified. -#[allow(dead_code)] -async fn batch_update_actor_stats_deprecated( - _conn: &mut deadpool_postgres::Object, -) -> eyre::Result<()> { - // This function is deprecated and no longer used. - // All actor counts are maintained automatically by database triggers. - Ok(()) -} - -/// Resolve an UnresolvedEvent by creating actor/post stubs and getting their IDs -/// -/// This is the resolution phase that happens before operations are created. -/// It ensures all foreign key references exist in the database. -/// -/// For CreateUpdate events: -/// 1. Resolve actor_id (create actor stub if needed) -/// 2. Extract and resolve all referenced actors and records from the record -/// 3. Call process_record_to_operations with resolved IDs -/// 4. Return ProcessedEvent with operations ready for worker dispatch -/// -/// For Delete events: -/// 1. Resolve actor_id (create stub if needed) -/// 2. Determine cache invalidations for this delete -/// 3. Create DeleteRecord operation with resolved actor_id -/// 4. Return ProcessedEvent ready for executor -async fn resolve_event( - pool: &Pool, - unresolved: super::UnresolvedEvent, - actor_cache: ¶keet_db::id_cache::IdCache, -) -> eyre::Result { - use super::operations::DatabaseOperation; - use crate::relay::types::CollectionType; - - let conn = pool.get().await.wrap_err("Failed to get database connection for resolution")?; - let now = chrono::Utc::now(); - - // STEP 1: Resolve actor_id for the record owner (create stub if needed) - // Uses cache for fast lookups of frequently-accessed actors - let (actor_id, _is_allowlisted, actor_was_created) = crate::db::actor::ensure_actor_id_with_cache( - &conn, - &unresolved.repo, - None, // status (will be updated by UpsertActor if present) - None, // handle (will be updated by UpsertActor if present) - now, - actor_cache, - ) - .await - .wrap_err_with(|| format!("Failed to resolve actor_id for {}", unresolved.repo))?; - - // If we just created a new actor stub, enqueue their profile for fetching - // This ensures we get handle, display name, avatar, etc. for stub actors - if actor_was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", unresolved.repo); - if let Err(e) = crate::db::fetch_queue::enqueue(&conn, &profile_uri).await { - tracing::warn!(did = %unresolved.repo, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - - // STEP 2: Handle event type (CreateUpdate or Delete) - let mut processed = match unresolved.event_type { - super::UnresolvedEventType::CreateUpdate { record, cid } => { - // Extract and resolve all referenced actors and records from the record - // This prevents NULL FK violations when inserting records that reference other entities - let resolved_actor_ids = extract_and_resolve_references(&conn, &record, now, actor_cache).await?; - - // Call process_record_to_operations with all resolved actor IDs - counter!("resolution_create_update_resolved").increment(1); - super::process_record_to_operations( - &unresolved.repo, - actor_id, - resolved_actor_ids, - cid, - *record, - unresolved.at_uri, - unresolved.rkey, - unresolved.source, - ) - } - super::UnresolvedEventType::Delete { collection } => { - // Determine cache invalidations based on collection type - let mut cache_invalidations = Vec::new(); - - // URI-based cache invalidations - match collection { - CollectionType::BskyFeedGen => { - cache_invalidations.push(format!("feedgen#{}", unresolved.at_uri)); - } - CollectionType::BskyFeedPost => { - cache_invalidations.push(format!("post#{}", unresolved.at_uri)); - } - CollectionType::BskyFeedThreadgate => { - cache_invalidations.push(format!("post#{}", unresolved.at_uri)); - } - CollectionType::BskyList => { - cache_invalidations.push(format!("list#{}", unresolved.at_uri)); - } - CollectionType::BskyListItem => { - cache_invalidations.push(format!("list#{}", unresolved.at_uri)); - } - CollectionType::BskyStarterPack => { - cache_invalidations.push(format!("starterpacks#{}", unresolved.at_uri)); - } - CollectionType::BskyLabelerService => { - cache_invalidations.push(format!("labeler#{}", unresolved.repo)); - } - _ => { - // No URI-based cache invalidations for other types - // Actor-based invalidations (timeline, authorfeed, profile) will be handled in executor - } - } - - // Create DeleteRecord operation with resolved actor_id - // Convert rkey to i64 for TID-based collections - // Some collections support non-TID rkeys: - // - BskyProfile: always uses "self" - // - BskyFeedGen: arbitrary string rkeys - // - BskyList: can use TID or arbitrary strings (e.g., "nfb", "bblock") - let rkey_i64 = match collection { - crate::relay::types::CollectionType::BskyProfile | - crate::relay::types::CollectionType::BskyFeedGen => { - 0 // Non-TID collections, handled separately by their delete handlers - } - crate::relay::types::CollectionType::BskyList => { - // Lists can use either TID or arbitrary string rkeys - parakeet_db::models::tid_to_i64(&unresolved.rkey).unwrap_or(0) - } - _ => { - // All other collections must use TID rkeys - parakeet_db::models::tid_to_i64(&unresolved.rkey) - .unwrap_or_else(|e| { - tracing::warn!("Invalid TID in delete rkey for {:?}: {}, error: {}", - collection, unresolved.rkey, e); - 0 - }) - } - }; - - let operation = DatabaseOperation::DeleteRecord { - at_uri: unresolved.at_uri, - actor_id, // Resolved i32, not DID string - collection, - rkey: rkey_i64, - }; - - counter!("resolution_delete_resolved").increment(1); - - ProcessedEvent { - operations: vec![operation], - cursor: None, // Will be set below - source: unresolved.source, - } - } - }; - - // STEP 3: Attach cursor from unresolved event - processed.cursor = unresolved.cursor; - - Ok(processed) -} - -/// Resolved actor IDs from a record's references -#[derive(Debug, Clone)] -pub struct ResolvedActorIds { - /// Subject actor ID (for Follow, Block, ListItem, Verification) - pub subject_actor_id: Option, - - /// Via repost natural key (for likes/reposts that came via a repost) - pub via_repost_key: Option<(i32, i64)>, - - /// Service actor ID (for FeedGenerator - the DID hosting the feed) - pub service_actor_id: Option, - - /// Post-specific resolved actor IDs (for notifications) - pub parent_author_actor_id: Option, - pub root_author_actor_id: Option, - pub quoted_author_actor_id: Option, - pub mentioned_actor_ids: Vec, -} - -impl ResolvedActorIds { - fn empty() -> Self { - Self { - via_repost_key: None, - subject_actor_id: None, - service_actor_id: None, - parent_author_actor_id: None, - root_author_actor_id: None, - quoted_author_actor_id: None, - mentioned_actor_ids: Vec::new(), - } - } -} - -/// Extract all referenced actors and records from a record and ensure they exist -/// This creates stub actors and posts as needed to prevent FK violations -/// -/// Returns all resolved actor IDs (subject, parent, root, quoted, mentioned) -async fn extract_and_resolve_references( - conn: &deadpool_postgres::Object, - record: &crate::relay::types::RecordTypes, - now: chrono::DateTime, - actor_cache: ¶keet_db::id_cache::IdCache, -) -> eyre::Result { - // Extract references using the new extraction module - let refs = crate::database_writer::extract_references(record); - - let mut resolved = ResolvedActorIds::empty(); - - // Resolve subject actor if present (with caching) - // If we create a new actor, enqueue their profile for fetching - if let Some(subject_did) = refs.subject_did { - let (actor_id, _, was_created) = - crate::db::actor::ensure_actor_id_with_cache(conn, &subject_did, None, None, now, actor_cache).await?; - resolved.subject_actor_id = Some(actor_id); - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", subject_did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %subject_did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - - // Resolve via repost ID if present (for likes/reposts that came via a repost) - // This creates a repost stub if needed, ensuring the FK constraint will be satisfied - if let Some(via_uri) = refs.via_uri { - // We need the CID from the via field to create a proper stub - let via_cid_str = refs.via_cid.as_deref().unwrap_or("bafyreihxj5lhuip5iynyzqj6e4w2dzosfndbcvtqgwdfvpmzw4pj4v76fi"); - - // Extract components from via URI - if let Some(via_did) = parakeet_db::at_uri_util::extract_did(&via_uri) { - if let Some(via_rkey) = parakeet_db::at_uri_util::extract_rkey(&via_uri) { - let via_collection = via_uri.strip_prefix("at://") - .and_then(|s| s.split('/').nth(1)); - - if via_collection == Some("app.bsky.feed.repost") { - // Ensure via actor exists - let (via_actor_id, _, _) = crate::db::actor::ensure_actor_id_with_cache( - conn, via_did, None, None, now, actor_cache - ).await?; - - // Create repost stub using get_repost_id with the actual CID from the via field - // This ensures the stub has the correct CID, which will satisfy the FK constraint - match crate::db::operations::feed::get_repost_id( - conn, via_actor_id, via_rkey, via_cid_str - ).await { - Ok((repost_key, was_created)) => { - resolved.via_repost_key = Some(repost_key); - - // If we created a new stub, enqueue it for fetching - if was_created { - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &via_uri).await { - tracing::warn!(via_uri = %via_uri, error = ?e, "Failed to enqueue via repost"); - } - } - } - Err(e) => { - tracing::warn!(via_uri = %via_uri, error = ?e, "Failed to resolve via repost natural key"); - } - } - } - } - } - } - - // Resolve all additional referenced actors (with caching) - // Enqueue profiles for any newly created actors - // For FeedGenerator records, the first (and only) additional DID is the service actor - for (idx, did) in refs.additional_dids.iter().enumerate() { - match crate::db::actor::ensure_actor_id_with_cache(conn, did, None, None, now, actor_cache).await { - Ok((actor_id, _, was_created)) => { - // First additional DID is the service actor (for feedgen records) - if idx == 0 { - resolved.service_actor_id = Some(actor_id); - tracing::debug!( - did = %did, - actor_id = actor_id, - "Resolved service actor for feedgen" - ); - } - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - Err(e) => { - tracing::warn!( - did = %did, - error = ?e, - "Failed to ensure actor for additional DID reference - skipping but continuing with other references" - ); - // Continue processing other DIDs instead of failing entire batch - } - } - } - - // Resolve post-specific references (for notifications) - // Enqueue profiles for any newly created actors - if let Some(parent_did) = refs.parent_author_did { - let (actor_id, _, was_created) = - crate::db::actor::ensure_actor_id_with_cache(conn, &parent_did, None, None, now, actor_cache).await?; - resolved.parent_author_actor_id = Some(actor_id); - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", parent_did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %parent_did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - - if let Some(root_did) = refs.root_author_did { - let (actor_id, _, was_created) = - crate::db::actor::ensure_actor_id_with_cache(conn, &root_did, None, None, now, actor_cache).await?; - resolved.root_author_actor_id = Some(actor_id); - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", root_did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %root_did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - - if let Some(quoted_did) = refs.quoted_author_did { - let (actor_id, _, was_created) = - crate::db::actor::ensure_actor_id_with_cache(conn, "ed_did, None, None, now, actor_cache).await?; - resolved.quoted_author_actor_id = Some(actor_id); - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", quoted_did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %quoted_did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - - for mentioned_did in refs.mentioned_dids { - let (actor_id, _, was_created) = - crate::db::actor::ensure_actor_id_with_cache(conn, &mentioned_did, None, None, now, actor_cache).await?; - resolved.mentioned_actor_ids.push(actor_id); - - if was_created { - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", mentioned_did); - if let Err(e) = crate::db::fetch_queue::enqueue(conn, &profile_uri).await { - tracing::warn!(did = %mentioned_did, error = ?e, "Failed to enqueue profile for newly created actor"); - } - } - } - - Ok(resolved) -} - -/// Spawn a source-specific dispatcher task -/// -/// Each dispatcher: -/// - Receives events from its dedicated channel -/// - Resolves UnresolvedEvents (creates actor/post stubs) -/// - Routes operations to appropriate workers (5 types × 3 workers = 15 workers per source) -/// - Updates counters and metrics -/// -/// Workers are already specialized by source, so no priority management needed. -#[expect(clippy::too_many_arguments, reason = "Dispatcher requires multiple shared state components")] -fn spawn_source_dispatcher( - name: &'static str, - source: super::EventSource, - pool: Pool, - mut rx: tokio::sync::mpsc::Receiver, - worker_pools: std::sync::Arc, - bulk_workers: std::sync::Arc>>, - bulk_worker_counter: std::sync::Arc, - batch_events_processed: std::sync::Arc, - batch_operations_processed: std::sync::Arc, - fetch_queue_enqueued: std::sync::Arc, - actor_cache: parakeet_db::id_cache::IdCache, - _allowlist: std::sync::Arc, - _pds_cache: std::sync::Arc, - mut stop: tokio::sync::watch::Receiver, -) -> tokio::task::JoinHandle> { - tokio::spawn(async move { - tracing::info!(dispatcher = name, "Source dispatcher started"); - - loop { - tokio::select! { - _ = stop.changed() => { - tracing::info!(dispatcher = name, "Received stop signal, shutting down"); - break; - } - writer_event = rx.recv() => { - let Some(writer_event) = writer_event else { - tracing::info!(dispatcher = name, "Channel closed, shutting down"); - break; - }; - - // Handle different event types - match writer_event { - // Individual unresolved event - resolve and route - super::WriterEvent::Unresolved(unresolved) => { - let event = match resolve_event(&pool, *unresolved, &actor_cache).await { - Ok(resolved) => resolved, - Err(e) => { - tracing::error!(error=?e, "Failed to resolve event, skipping"); - continue; - } - }; - - let op_count = event.operations.len(); - let enqueue_count = event - .operations - .iter() - .filter(|op| matches!(op, super::DatabaseOperation::EnqueueFetch { .. })) - .count(); - - // Route operations to workers - for op in event.operations { - let worker_type = routing::classify_operation(&op); - let tx = worker_pools.get_worker(source, worker_type); - drop(tx.send(routing::PartitionedEvent { - operations: vec![op], - cursor: None, - source, - })); - } - - // Send cursor to metadata worker (if present) - if event.cursor.is_some() { - let metadata_tx = worker_pools.get_metadata_worker(source); - drop(metadata_tx.send(routing::PartitionedEvent { - operations: Vec::new(), - cursor: event.cursor, - source, - })); - } - - // Update counters - batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - batch_operations_processed.fetch_add(op_count as u64, std::sync::atomic::Ordering::Relaxed); - fetch_queue_enqueued.fetch_add(enqueue_count as u64, std::sync::atomic::Ordering::Relaxed); - counter!("batch_writer_events_dispatched").increment(1); - counter!("batch_writer_operations_dispatched").increment(op_count as u64); - } - - // Individual resolved event - route directly - super::WriterEvent::Resolved(event) => { - let event = *event; - let op_count = event.operations.len(); - let enqueue_count = event - .operations - .iter() - .filter(|op| matches!(op, super::DatabaseOperation::EnqueueFetch { .. })) - .count(); - - // Route operations to workers - for op in event.operations { - let worker_type = routing::classify_operation(&op); - let tx = worker_pools.get_worker(source, worker_type); - drop(tx.send(routing::PartitionedEvent { - operations: vec![op], - cursor: None, - source, - })); - } - - // Send cursor to metadata worker (if present) - if event.cursor.is_some() { - let metadata_tx = worker_pools.get_metadata_worker(source); - drop(metadata_tx.send(routing::PartitionedEvent { - operations: Vec::new(), - cursor: event.cursor, - source, - })); - } - - // Update counters - batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - batch_operations_processed.fetch_add(op_count as u64, std::sync::atomic::Ordering::Relaxed); - fetch_queue_enqueued.fetch_add(enqueue_count as u64, std::sync::atomic::Ordering::Relaxed); - counter!("batch_writer_events_dispatched").increment(1); - counter!("batch_writer_operations_dispatched").increment(op_count as u64); - } - - // Bulk unresolved event - route to bulk worker pool - super::WriterEvent::UnresolvedBulk { repo, actor_id, records, source: event_source } => { - tracing::debug!( - repo = %repo, - actor_id = actor_id, - records = records.len(), - "Routing bulk unresolved event to bulk worker" - ); - - // Round-robin across bulk workers - let worker_idx = bulk_worker_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) % bulk_workers.len(); - let tx = &bulk_workers[worker_idx]; - - // Send to bulk worker (non-blocking) - drop(tx.send(super::WriterEvent::UnresolvedBulk { - repo, - actor_id, - records, - source: event_source, - })); - - counter!("bulk_writer_events_dispatched").increment(1); - } - - // Bulk resolved event - execute COPY operations - super::WriterEvent::ResolvedBulk { repo, actor_id, operations: bulk_ops, source: _event_source } => { - // TODO: Implement bulk COPY execution - // For now, log and skip - tracing::warn!( - repo = %repo, - actor_id = actor_id, - total_ops = bulk_ops.total_count(), - "Bulk resolved event received but bulk execution not yet implemented, skipping" - ); - } - } - } - } - } - - tracing::info!(dispatcher = name, "Source dispatcher stopped gracefully"); - Ok(()) - }) -} - -/// Spawn the database writer task with multi-worker architecture -/// -/// This spawns 45 specialized workers organized by event source and operation type: -/// - Jetstream workers (15): Real-time events -/// - Backfill workers (15): Bulk historical data -/// - Fetch workers (15): On-demand record fetching -/// -/// Each source has 5 worker types, with 3 workers per type: -/// 1. ActorWorker - Handles rare actor upserts (handle resolution, account events) -/// 2. LikeWorker - Handles likes (highest throughput operation) -/// 3. SocialWorker - Handles follows, reposts, blocks -/// 4. PostWorker - Handles posts and embeds -/// 5. MetadataWorker - Handles profiles, gates, lists, etc. -/// -/// All workers run in parallel. CTEs in database insert functions atomically ensure -/// actors exist before dependent inserts, eliminating the need for synchronization. -/// -/// Three independent dispatchers route events to their respective worker pools. -/// Each source processes its own queue independently - no cross-source prioritization. -/// -/// Cursors are batched and saved to PostgreSQL every 10 seconds. -/// In-memory cursor is updated immediately for accurate logging. -#[expect(clippy::too_many_arguments, reason = "Database writer requires extensive configuration state")] -pub fn spawn_database_writer( - pool: Pool, - jetstream_rx: tokio::sync::mpsc::Receiver, - fetch_rx: tokio::sync::mpsc::Receiver, - backfill_rx: tokio::sync::mpsc::Receiver, - batch_events_processed: std::sync::Arc, - batch_operations_processed: std::sync::Arc, - fetch_queue_enqueued: std::sync::Arc, - cursor: std::sync::Arc>, - allowlist: crate::db::Allowlist, - actor_cache: parakeet_db::id_cache::IdCache, - stop: tokio::sync::watch::Receiver, -) -> tokio::task::JoinHandle> { - // Create cache invalidation worker with bounded channel - // Bounded at 1000 to provide backpressure - if channel fills, invalidations are dropped - // (graceful degradation - cache will expire naturally via TTL) - let (cache_tx, cache_rx) = tokio::sync::mpsc::channel::>(1000); - let cache_tx = std::sync::Arc::new(cache_tx); - - tokio::spawn(super::cache_worker::cache_invalidation_worker( - pool.clone(), - cache_rx, - "cache-invalidation".to_string(), - )); - - // Helper macro to create a worker pool for a specific (source, type) combination - macro_rules! create_worker_pool { - ($source:expr, $type:expr) => {{ - let mut senders = Vec::with_capacity(WORKERS_PER_TYPE); - - for i in 0..WORKERS_PER_TYPE { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - // Leak the worker name string to get 'static lifetime - // This is acceptable since we only create 45 workers once at startup - let worker_name: &'static str = - Box::leak(format!("{}-{}-{}", $source, $type, i).into_boxed_str()); - - spawn_worker(WorkerConfig { - name: worker_name, - pool: pool.clone(), - rx, - cursor: cursor.clone(), - allowlist: allowlist.clone(), - cache_tx: cache_tx.clone(), - }); - - senders.push(tx); - } - - senders - }}; - } - - // Create all 45 workers (3 sources × 5 types × 3 workers) - let pools = WorkerPools::new( - // Jetstream workers (real-time events) - create_worker_pool!("jetstream", "actor"), - create_worker_pool!("jetstream", "like"), - create_worker_pool!("jetstream", "social"), - create_worker_pool!("jetstream", "post"), - create_worker_pool!("jetstream", "metadata"), - // Backfill workers (bulk historical data) - create_worker_pool!("backfill", "actor"), - create_worker_pool!("backfill", "like"), - create_worker_pool!("backfill", "social"), - create_worker_pool!("backfill", "post"), - create_worker_pool!("backfill", "metadata"), - // Fetch workers (on-demand record fetching) - create_worker_pool!("fetch", "actor"), - create_worker_pool!("fetch", "like"), - create_worker_pool!("fetch", "social"), - create_worker_pool!("fetch", "post"), - create_worker_pool!("fetch", "metadata"), - ); - - // Share worker pools across all three dispatchers - let pools = std::sync::Arc::new(pools); - - // Create shared state for dispatchers - let allowlist = std::sync::Arc::new(allowlist); - let pds_cache = std::sync::Arc::new(crate::external::pds_cache::PdsHostCache::new()); - - // Create bulk worker pool (3 workers to handle large backfill repos asynchronously) - const BULK_WORKERS: usize = 3; - let mut bulk_workers = Vec::with_capacity(BULK_WORKERS); - - for i in 0..BULK_WORKERS { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let worker_name: &'static str = Box::leak(format!("bulk-{}", i).into_boxed_str()); - - spawn_bulk_worker( - worker_name, - pool.clone(), - rx, - allowlist.clone(), - pds_cache.clone(), - ); - - bulk_workers.push(tx); - } - - let bulk_workers = std::sync::Arc::new(bulk_workers); - let bulk_worker_counter = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); - - // Spawn three independent dispatchers (one per source) - // Each dispatcher routes events from its channel to its 15 workers - tracing::info!( - "Database writer starting with 45 workers (3 sources × 5 types × 3 workers), 3 bulk workers, 3 dispatchers, and 1 cache worker" - ); - - let jetstream_handle = spawn_source_dispatcher( - "jetstream", - super::EventSource::Jetstream, - pool.clone(), - jetstream_rx, - pools.clone(), - bulk_workers.clone(), - bulk_worker_counter.clone(), - batch_events_processed.clone(), - batch_operations_processed.clone(), - fetch_queue_enqueued.clone(), - actor_cache.clone(), - allowlist.clone(), - pds_cache.clone(), - stop.clone(), - ); - - let fetch_handle = spawn_source_dispatcher( - "fetch", - super::EventSource::FetchQueue, - pool.clone(), - fetch_rx, - pools.clone(), - bulk_workers.clone(), - bulk_worker_counter.clone(), - batch_events_processed.clone(), - batch_operations_processed.clone(), - fetch_queue_enqueued.clone(), - actor_cache.clone(), - allowlist.clone(), - pds_cache.clone(), - stop.clone(), - ); - - let backfill_handle = spawn_source_dispatcher( - "backfill", - super::EventSource::Backfill, - pool.clone(), - backfill_rx, - pools.clone(), - bulk_workers.clone(), - bulk_worker_counter.clone(), - batch_events_processed, - batch_operations_processed, - fetch_queue_enqueued, - actor_cache, - allowlist, - pds_cache, - stop, - ); - - // Return a handle that waits for all three dispatchers - tokio::spawn(async move { - let results = tokio::try_join!(jetstream_handle, fetch_handle, backfill_handle); - match results { - Ok((Ok(()), Ok(()), Ok(()))) => { - tracing::info!("All database writer dispatchers stopped successfully"); - Ok(()) - } - Ok((jetstream_result, fetch_result, backfill_result)) => { - // Report which dispatcher(s) failed - if let Err(e) = &jetstream_result { - tracing::error!(error=?e, dispatcher="jetstream", "Jetstream dispatcher failed"); - } - if let Err(e) = &fetch_result { - tracing::error!(error=?e, dispatcher="fetch", "Fetch dispatcher failed"); - } - if let Err(e) = &backfill_result { - tracing::error!(error=?e, dispatcher="backfill", "Backfill dispatcher failed"); - } - Err(eyre::eyre!("One or more database writer dispatchers failed")) - } - Err(e) => { - tracing::error!(error=?e, "Database writer dispatcher task panicked"); - Err(eyre::eyre!("Database writer dispatcher panicked: {}", e)) - } - } - }) -} - -/// Configuration for spawning a database writer worker -struct WorkerConfig { - name: &'static str, - pool: Pool, - rx: tokio::sync::mpsc::UnboundedReceiver, - cursor: std::sync::Arc>, - allowlist: crate::db::Allowlist, - cache_tx: std::sync::Arc>>, -} - -/// Spawn a specialized worker task -/// -/// Each worker processes operations from its channel and updates post stats in the database. -/// Workers run concurrently, allowing parallel database operations. -fn spawn_worker(config: WorkerConfig) -> tokio::task::JoinHandle> { - let WorkerConfig { - name, - pool, - mut rx, - cursor, - allowlist, - cache_tx, - } = config; - tokio::spawn(async move { - tracing::info!(worker = name, "Database writer worker started"); - - // Create PostgreSQL cursor manager for batched cursor saves - let cursor_manager = crate::external::pg_cursor_manager::PgCursorManager::new(pool.clone()); - let mut pending_cursor: Option = None; - let mut last_cursor_save = std::time::Instant::now(); - - // Create PDS cache (shared across all operations in this worker) - let pds_cache = crate::external::pds_cache::PdsHostCache::new(); - - while let Some(event) = rx.recv().await { - let event_start = std::time::Instant::now(); - let op_count = event.operations.len(); - - // Get database connection from pool - let conn_start = std::time::Instant::now(); - let mut conn = match pool.get().await { - Ok(c) => c, - Err(e) => { - tracing::error!(worker = name, "Failed to get database connection: {}", e); - counter!("batch_writer_db_connection_error").increment(1); - continue; - } - }; - - let conn_acquisition_ms = conn_start.elapsed().as_millis(); - if conn_acquisition_ms > 500 { - // Only warn on significantly slow acquisitions (>500ms) - // Brief waits (<500ms) are normal under load - tracing::warn!( - worker = name, - wait_ms = conn_acquisition_ms, - source = ?event.source, - "Connection pool under pressure" - ); - } - - - // Actor stats maintained by triggers - - let db_start = std::time::Instant::now(); - - // Execute operations individually (unified path for all event sources) - // Each operation handles its own actor creation atomically via get_actor_id() - let mut affected = 0; - for op in event.operations { - // Get operation type and URI for logging before moving `op` - let (op_type, op_uri) = executor::describe_operation(&op); - - match executor::execute_operation( - &pool, - &mut conn, - op, - &allowlist, - &pds_cache, - ) - .await - { - Ok(()) => { - affected += 1; - // Actor counts and cache maintained by triggers - } - Err(e) => { - // Use {:#} to show full error chain with causes - // This is critical for debugging SQL errors, TID validation failures, etc. - if let Some(uri) = op_uri { - tracing::error!( - worker = name, - operation = %op_type, - uri = %uri, - "Failed to execute database operation: {:#}", - e - ); - } else { - tracing::error!( - worker = name, - operation = %op_type, - "Failed to execute database operation: {:#}", - e - ); - } - counter!("batch_writer_operation_error").increment(1); - // Continue processing other operations - } - } - } - let affected_operations = affected; - - let db_execution_ms = db_start.elapsed().as_millis(); - - // Warn on slow operations, but use context-aware thresholds - // Large batches are expected to take longer - let slow_threshold_ms = if op_count > 1000 { - 5000 // 5s threshold for large batches - } else if op_count > 100 { - 2000 // 2s threshold for medium batches - } else { - 1000 // 1s threshold for small batches - }; - - // Only warn on slow operations with significant work (>10 operations) - // Single operations are often slow due to network latency, not actionable issues - if db_execution_ms > slow_threshold_ms && op_count > 10 { - let db_ops_per_sec = if db_execution_ms > 0 { - (op_count as u128 * 1000) / db_execution_ms - } else { - op_count as u128 - }; - - tracing::warn!( - worker = name, - op_count = op_count, - db_ms = db_execution_ms, - db_ops_per_sec = db_ops_per_sec, - source = ?event.source, - "Slow database operation detected" - ); - } - - // All counts maintained by database triggers: - // - Post stats (likes, replies, quotes, reposts) via update_post_counts trigger - // - Actor stats (followers, following, posts) via update_actor_counts trigger - // - No batch updates needed! - - // Cache invalidations are now handled by database triggers via pg_notify - // No need to send them from the application layer - - // Update cursor AFTER successful database operations - if let Some(cursor_timestamp) = event.cursor { - // Update in-memory cursor for accurate logging - if let Ok(mut c) = cursor.write() { - *c = cursor_timestamp; - } - - // Track for batched PostgreSQL save (happens every 10 seconds) - pending_cursor = Some(cursor_timestamp); - } - - // Save cursor to PostgreSQL every 10 seconds - if last_cursor_save.elapsed() >= std::time::Duration::from_secs(10) { - if let Some(timestamp) = pending_cursor.take() { - if let Err(e) = cursor_manager.save("jetstream", timestamp as i64).await { - tracing::error!(worker = name, "Failed to save cursor to PostgreSQL: {}", e); - counter!("batch_writer_cursor_save_error").increment(1); - } - } - last_cursor_save = std::time::Instant::now(); - counter!("batch_writer_cursor_batch_saves").increment(1); - } - - // Log event processing with appropriate level based on significance - let total_event_ms = event_start.elapsed().as_millis(); - - // Calculate throughput for informative logging - let ops_per_sec = if total_event_ms > 0 { - (op_count as u128 * 1000) / total_event_ms - } else { - op_count as u128 - }; - - // Log at different levels based on batch significance and performance - if total_event_ms > 10000 { - // Very slow batches (>10s) are always noteworthy - INFO level - tracing::info!( - worker = name, - op_count = op_count, - affected_operations = affected_operations, - total_ms = total_event_ms, - db_ms = db_execution_ms, - ops_per_sec = ops_per_sec, - source = ?event.source, - "Completed large batch" - ); - } else if op_count > 1000 { - // Large batches with good performance - DEBUG level - tracing::debug!( - worker = name, - op_count = op_count, - affected_operations = affected_operations, - total_ms = total_event_ms, - ops_per_sec = ops_per_sec, - source = ?event.source, - "Batch processed" - ); - } - - counter!("batch_writer_operations_processed").increment(op_count as u64); - } - - // Final cursor save to PostgreSQL before shutdown - if let Some(timestamp) = pending_cursor { - tracing::info!( - worker = name, - "Worker draining - saving final cursor to PostgreSQL" - ); - if let Err(e) = cursor_manager.save("jetstream", timestamp as i64).await { - tracing::error!(worker = name, "Failed to save final cursor: {}", e); - } - } - - tracing::info!(worker = name, "Database writer worker stopped"); - Ok(()) - }) -} diff --git a/consumer/src/database_writer/workers_tap.rs b/consumer/src/database_writer/workers_tap.rs new file mode 100644 index 00000000..6976fcb0 --- /dev/null +++ b/consumer/src/database_writer/workers_tap.rs @@ -0,0 +1,574 @@ +//! Simplified database writer for Tap-based ingestion +//! +//! This module implements a streamlined database writer that: +//! - Uses a single primary channel for Tap events +//! - Routes events based on their EventSource (Tap, TapBackfill) + +use super::operations::executor; +use super::routing::{PartitionedEvent, WorkerType}; +use super::{ProcessedEvent, WriterEvent, EventSource, DatabaseOperation}; +use crate::types::records::AppBskyEmbed; +use deadpool_postgres::Pool; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use tokio::sync::mpsc; + +const WORKERS_PER_TYPE: usize = 3; + +/// Simplified worker pools for Tap-based architecture +struct TapWorkerPools { + // Live event workers (high priority - from Tap live events) + live_actor: Vec>, + live_like: Vec>, + live_social: Vec>, + live_post: Vec>, + live_metadata: Vec>, + + // Backfill workers (medium priority - from Tap backfill) + backfill_actor: Vec>, + backfill_like: Vec>, + backfill_social: Vec>, + backfill_post: Vec>, + backfill_metadata: Vec>, + + // Round-robin counters for load balancing + counters: Arc>>, +} + +impl TapWorkerPools { + fn new( + live_actor: Vec>, + live_like: Vec>, + live_social: Vec>, + live_post: Vec>, + live_metadata: Vec>, + backfill_actor: Vec>, + backfill_like: Vec>, + backfill_social: Vec>, + backfill_post: Vec>, + backfill_metadata: Vec>, + ) -> Self { + Self { + live_actor, + live_like, + live_social, + live_post, + live_metadata, + backfill_actor, + backfill_like, + backfill_social, + backfill_post, + backfill_metadata, + counters: Arc::new(std::sync::Mutex::new(std::collections::HashMap::new())), + } + } + + /// Get the next worker for round-robin distribution + fn get_worker(&self, source: EventSource, worker_type: WorkerType) -> &mpsc::UnboundedSender { + let mut map = self.counters.lock().unwrap(); + let counter = map.entry((source, worker_type)).or_insert(0); + let idx = *counter % WORKERS_PER_TYPE; + *counter = counter.wrapping_add(1); + drop(map); + + let pool = match (source, worker_type) { + // Live Tap events (highest priority) + (EventSource::Tap, WorkerType::Actor) => &self.live_actor, + (EventSource::Tap, WorkerType::Like) => &self.live_like, + (EventSource::Tap, WorkerType::Social) => &self.live_social, + (EventSource::Tap, WorkerType::Post) => &self.live_post, + (EventSource::Tap, WorkerType::Metadata) => &self.live_metadata, + + // Tap backfill events (medium priority) + (EventSource::TapBackfill, WorkerType::Actor) => &self.backfill_actor, + (EventSource::TapBackfill, WorkerType::Like) => &self.backfill_like, + (EventSource::TapBackfill, WorkerType::Social) => &self.backfill_social, + (EventSource::TapBackfill, WorkerType::Post) => &self.backfill_post, + (EventSource::TapBackfill, WorkerType::Metadata) => &self.backfill_metadata, + }; + + &pool[idx] + } +} + +/// Spawn the simplified database writer for Tap +pub fn spawn_database_writer_tap( + pool: Pool, + tap_rx: mpsc::Receiver, + batch_events_processed: Arc, + batch_operations_processed: Arc, + id_cache: parakeet_db::id_cache::IdCache, + stop: tokio::sync::watch::Receiver, +) -> tokio::task::JoinHandle> { + tokio::spawn(async move { + tracing::info!("Starting Tap-based database writer"); + + // Create worker pools + macro_rules! create_worker_pool { + ($source:expr, $type:expr, $name:expr) => {{ + let mut senders = Vec::with_capacity(WORKERS_PER_TYPE); + + for i in 0..WORKERS_PER_TYPE { + let (tx, rx) = mpsc::unbounded_channel(); + senders.push(tx); + + let worker_name = format!("{}-{}-{}", $name, $type, i); + spawn_database_worker( + worker_name, + pool.clone(), + rx, + id_cache.clone(), + ); + } + + senders + }}; + } + + // Create all worker pools + let pools = TapWorkerPools::new( + // Live workers + create_worker_pool!(EventSource::Tap, "actor", "live"), + create_worker_pool!(EventSource::Tap, "like", "live"), + create_worker_pool!(EventSource::Tap, "social", "live"), + create_worker_pool!(EventSource::Tap, "post", "live"), + create_worker_pool!(EventSource::Tap, "metadata", "live"), + + // Backfill workers + create_worker_pool!(EventSource::TapBackfill, "actor", "backfill"), + create_worker_pool!(EventSource::TapBackfill, "like", "backfill"), + create_worker_pool!(EventSource::TapBackfill, "social", "backfill"), + create_worker_pool!(EventSource::TapBackfill, "post", "backfill"), + create_worker_pool!(EventSource::TapBackfill, "metadata", "backfill"), + ); + + // Create dispatcher channels + let (dispatcher_tx, mut dispatcher_rx) = mpsc::channel::(10_000); + + // Spawn receiver task that forwards to dispatcher + let tap_forwarder = tokio::spawn(forward_events(tap_rx, dispatcher_tx.clone(), "tap")); + + // Main dispatcher loop + while let Some(event) = dispatcher_rx.recv().await { + if *stop.borrow() { + break; + } + + match event { + WriterEvent::Unresolved(unresolved) => { + // Resolve and process unresolved event + match resolve_and_process_event( + pool.clone(), + *unresolved, + &pools, + &batch_events_processed, + &batch_operations_processed, + &id_cache, + ).await { + Ok(_) => {}, + Err(e) => { + tracing::error!("Failed to resolve event: {}", e); + } + } + } + WriterEvent::Resolved(resolved) => { + // Route resolved event to appropriate worker + route_processed_event(resolved, &pools, &batch_events_processed, &batch_operations_processed); + } + WriterEvent::UnresolvedBulk { repo, actor_id, records, source } => { + // Process bulk events using bulk processor + match process_bulk_event( + pool.clone(), + repo, + actor_id, + records, + source, + &pools, + &batch_events_processed, + &batch_operations_processed, + ).await { + Ok(_) => {}, + Err(e) => { + tracing::error!("Failed to process bulk event: {}", e); + } + } + } + WriterEvent::ResolvedBulk { .. } => { + // ResolvedBulk is not used in Tap architecture + tracing::debug!("ResolvedBulk event processing not implemented"); + } + } + } + + // Wait for forwarder to complete + tap_forwarder.await?; + + tracing::info!("Database writer shutdown complete"); + Ok(()) + }) +} + +/// Forward events from a source channel to the dispatcher +async fn forward_events( + mut rx: mpsc::Receiver, + tx: mpsc::Sender, + source_name: &str, +) -> () { + while let Some(event) = rx.recv().await { + if let Err(e) = tx.send(event).await { + tracing::error!("Failed to forward {} event: {}", source_name, e); + } + } + tracing::info!("{} forwarder stopped", source_name); +} + +/// Route a processed event to the appropriate worker +fn route_processed_event( + event: Box, + pools: &TapWorkerPools, + batch_events_processed: &Arc, + batch_operations_processed: &Arc, +) { + let num_operations = event.operations.len(); + let source = event.source; + let cursor = event.cursor; + + for op in event.operations { + let worker_type = super::routing::classify_operation(&op); + let worker = pools.get_worker(source, worker_type); + + let partitioned = PartitionedEvent { + operations: vec![op], + cursor, + source, + }; + + if let Err(e) = worker.send(partitioned) { + tracing::error!("Failed to send to worker: {}", e); + } + } + + batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + batch_operations_processed.fetch_add(num_operations as u64, std::sync::atomic::Ordering::Relaxed); +} + +/// Spawn a single database worker +fn spawn_database_worker( + name: String, + pool: Pool, + mut rx: mpsc::UnboundedReceiver, + _id_cache: parakeet_db::id_cache::IdCache, +) { + tokio::spawn(async move { + tracing::info!("Worker {} started", name); + + while let Some(event) = rx.recv().await { + // Get database connection + let mut conn = match pool.get().await { + Ok(c) => c, + Err(e) => { + tracing::error!(worker = name, "Failed to get database connection: {}", e); + continue; + } + }; + + // Execute operations + for op in event.operations { + match executor::execute_operation( + &pool, + &mut conn, + op, + ).await { + Ok(_) => { + // Operation succeeded + // TODO: Handle cache invalidations if needed + } + Err(e) => { + tracing::error!(worker = name, "Failed to execute operation: {}", e); + } + } + } + } + + tracing::info!("Worker {} stopped", name); + }); +} + +/// Resolve an unresolved event and route to appropriate worker +async fn resolve_and_process_event( + pool: Pool, + unresolved: super::UnresolvedEvent, + pools: &TapWorkerPools, + batch_events_processed: &Arc, + batch_operations_processed: &Arc, + _id_cache: ¶keet_db::id_cache::IdCache, +) -> eyre::Result<()> { + // Get database connection for resolution + let mut conn = pool.get().await?; + + // Resolve actor_id (create stub if needed) + let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, &unresolved.repo).await?; + + // Process based on event type + let operations = match unresolved.event_type { + super::UnresolvedEventType::CreateUpdate { record, cid } => { + // Extract references from the record + let refs = super::extract_references(&*record); + + // Resolve subject actor if present + let subject_actor_id = if let Some(subject_did) = refs.subject_did { + let (id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, &subject_did).await?; + Some(id) + } else { + None + }; + + // Resolve service actor if present (for feedgens) + let service_actor_id = if let Some(first_did) = refs.additional_dids.first() { + match crate::db::operations::feed::get_actor_id(&mut conn, first_did).await { + Ok((id, _, _)) => Some(id), + Err(e) => { + tracing::warn!("Failed to resolve service actor: {}", e); + None + } + } + } else { + None + }; + + // For posts, resolve parent/root/quoted authors + let (parent_author_actor_id, root_author_actor_id, quoted_author_actor_id, mentioned_actor_ids) = + if let crate::relay::types::RecordTypes::AppBskyFeedPost(ref post) = *record { + use parakeet_db::at_uri_util; + + // Resolve parent author + let parent_author = if let Some(ref reply) = post.reply { + if let Some(did) = at_uri_util::extract_did(&reply.parent.uri) { + let (aid, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, did).await?; + Some(aid) + } else { + None + } + } else { + None + }; + + // Resolve root author + let root_author = if let Some(ref reply) = post.reply { + if reply.root.uri != reply.parent.uri { + if let Some(did) = at_uri_util::extract_did(&reply.root.uri) { + let (aid, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, did).await?; + Some(aid) + } else { + None + } + } else { + parent_author + } + } else { + None + }; + + // Resolve quoted post author from embed + let quoted_author = if let Some(ref embed) = post.embed { + if let Some(bsky_embed) = embed.as_bsky() { + match bsky_embed { + AppBskyEmbed::Record(record_embed) => { + // Extract DID from the quoted post URI + if let Some(did) = parakeet_db::at_uri_util::extract_did(&record_embed.record.uri) { + let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, did).await?; + Some(actor_id) + } else { + None + } + }, + AppBskyEmbed::RecordWithMedia(record_with_media) => { + // Extract DID from the quoted post URI + if let Some(did) = parakeet_db::at_uri_util::extract_did(&record_with_media.record.record.uri) { + let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, did).await?; + Some(actor_id) + } else { + None + } + }, + _ => None, + } + } else { + None + } + } else { + None + }; + + // Resolve mentioned actors from facets + let mut mentions = Vec::new(); + if let Some(ref facets) = post.facets { + for facet_item in facets { + for feature in &facet_item.features { + if let lexica::app_bsky::richtext::FacetOuter::Bsky( + lexica::app_bsky::richtext::Facet::Mention { did } + ) = feature { + // Resolve mentioned actor + let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, did).await?; + mentions.push(actor_id); + } + } + } + } + + (parent_author, root_author, quoted_author, mentions) + } else { + (None, None, None, Vec::new()) + }; + + // Resolve via_repost if present + let via_repost_key = if let (Some(via_uri), Some(via_cid)) = (&refs.via_uri, &refs.via_cid) { + if let Some((via_did, via_rkey, _)) = parakeet_db::at_uri_util::parse_at_uri(via_uri) { + let (via_actor_id, _, _) = crate::db::operations::feed::get_actor_id(&mut conn, via_did).await?; + let (key, _) = crate::db::operations::feed::get_repost_id( + &mut conn, + via_actor_id, + via_rkey, + via_cid, + ).await?; + Some(key) + } else { + None + } + } else { + None + }; + + let resolved_actor_ids = super::operations::ResolvedActorIds { + subject_actor_id, + service_actor_id, + parent_author_actor_id, + root_author_actor_id, + quoted_author_actor_id, + mentioned_actor_ids, + via_repost_key, + }; + + // Process record to operations + let processed = super::process_record_to_operations( + &unresolved.repo, + actor_id, + resolved_actor_ids, + cid, + *record, + unresolved.at_uri, + unresolved.rkey, + unresolved.source, + ); + + processed.operations + } + super::UnresolvedEventType::Delete { collection } => { + // Create delete operation + // Need to convert rkey string to i64 for TID-based collections + let rkey_i64 = if matches!(collection, + crate::relay::types::CollectionType::BskyFeedLike | + crate::relay::types::CollectionType::BskyFeedPost | + crate::relay::types::CollectionType::BskyFeedRepost | + crate::relay::types::CollectionType::BskyFollow | + crate::relay::types::CollectionType::BskyBlock + ) { + // TID-based collection, convert to i64 + parakeet_db::models::tid_to_i64(&unresolved.rkey).unwrap_or(0) + } else { + // Non-TID collection (like profile, feedgen) + 0 + }; + + vec![DatabaseOperation::DeleteRecord { + at_uri: unresolved.at_uri.clone(), + actor_id, + rkey: rkey_i64, + collection, + }] + } + }; + + // Create processed event + let processed = ProcessedEvent { + operations, + cursor: unresolved.cursor, + source: unresolved.source, + }; + + // Route to appropriate worker + route_processed_event(Box::new(processed), pools, batch_events_processed, batch_operations_processed); + + Ok(()) +} + +/// Process bulk events using the bulk processor +async fn process_bulk_event( + pool: Pool, + repo: String, + mut actor_id: i32, + records: Vec, + source: EventSource, + pools: &TapWorkerPools, + batch_events_processed: &Arc, + batch_operations_processed: &Arc, +) -> eyre::Result<()> { + // Get database connection + let conn = pool.get().await?; + + // Resolve actor_id if not provided + if actor_id == 0 { + let (id, _, _) = crate::db::operations::feed::get_actor_id(&conn, &repo).await?; + actor_id = id; + } + + // Process bulk records + let bulk_ops = super::bulk_processor::process_bulk_records( + &conn, + &repo, + actor_id, + records, + source, + ).await?; + + // Convert bulk operations to individual operations and route them + // First handle COPY operations (bulk inserts) + let mut total_operations = 0; + + // Convert bulk data to individual operations (for now, until COPY is implemented) + // In the future, these would be sent directly to PostgreSQL COPY + + // TODO: Implement actual COPY operations for bulk data + // For now, convert to individual operations + for _post in bulk_ops.posts { + total_operations += 1; + // Post COPY operations would go here + } + + for _like in bulk_ops.post_likes { + total_operations += 1; + // Like COPY operations would go here + } + + // Route individual operations + for op in bulk_ops.individual_ops { + let worker_type = super::routing::classify_operation(&op); + let worker = pools.get_worker(source, worker_type); + + let partitioned = PartitionedEvent { + operations: vec![op], + cursor: None, + source, + }; + + if let Err(e) = worker.send(partitioned) { + tracing::error!("Failed to send to worker: {}", e); + } + total_operations += 1; + } + + batch_events_processed.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + batch_operations_processed.fetch_add(total_operations as u64, std::sync::atomic::Ordering::Relaxed); + + Ok(()) +} \ No newline at end of file diff --git a/consumer/src/db/actor.rs b/consumer/src/db/actor.rs index 82b1f78b..e00ab28e 100644 --- a/consumer/src/db/actor.rs +++ b/consumer/src/db/actor.rs @@ -156,7 +156,7 @@ pub async fn actor_get_repo_rev(conn: &C, did: &str) -> Result /// Uses INSERT ... ON CONFLICT to handle concurrent resolution workers safely. /// Returns existing actor_id if actor already exists. /// -/// TODO: Add caching here for frequently accessed actor_ids to reduce database load +/// Note: For cached version, use ensure_actor_id_with_cache or get_actor_id from feed::helpers. pub async fn ensure_actor_id( conn: &C, did: &str, diff --git a/consumer/src/db/allowlist.rs b/consumer/src/db/allowlist.rs deleted file mode 100644 index 0ff534ef..00000000 --- a/consumer/src/db/allowlist.rs +++ /dev/null @@ -1,220 +0,0 @@ -//! Allowlist functionality for DIDs with caching support -//! -//! This module provides tokio-postgres database operations for the allowlist, -//! wrapping the shared CachedAllowlist from parakeet-db. - -use deadpool_postgres::{Client, GenericClient}; -use eyre::Result; -use parakeet_db::allowlist::{clean_did, CachedAllowlist}; -use tracing::{debug, error}; - -/// A thread-safe cached allowlist that periodically refreshes from the database -/// -/// This is a thin wrapper around CachedAllowlist that provides tokio-postgres -/// database operations. All cache operations are delegated to the inner cache. -#[derive(Clone)] -pub struct Allowlist { - /// The cached allowlist state (use .cache.contains_did() for lookups) - pub cache: CachedAllowlist, -} - -impl Allowlist { - /// Create a new empty cached allowlist - pub fn new() -> Self { - Self { - cache: CachedAllowlist::new(), - } - } - - /// Initialize the cache by loading the allowlist from the database - /// - /// Returns true if the allowlist changed (additions or removals) - pub async fn initialize(&self, client: &Client) -> Result { - let dids = get_all(client).await?; - let changed = self.cache.update_cache(dids); - debug!( - "Allowlist cache initialized with {} entries", - self.cache.len() - ); - Ok(changed) - } - - /// Add a DID to the allowlist (both in the database and the cache) - pub async fn add_did( - &self, - client: &Client, - did: &str, - description: Option<&str>, - ) -> Result { - let Some(did) = clean_did(did) else { - return Err(eyre::eyre!("Invalid DID format")); - }; - - let result = add(client, &did, description).await?; - self.cache.add_did(&did); - debug!("Added DID {} to allowlist", did); - Ok(result > 0) - } - - /// Spawn a background task that periodically refreshes the allowlist cache - /// - /// Refreshes every 60 seconds. Allowlist changes are rare admin operations, - /// so this polling approach is acceptable. - /// - /// Returns a watch channel receiver that will be notified when the allowlist changes. - pub async fn spawn_periodic_refresh( - &self, - pool: deadpool_postgres::Pool, - ) -> ( - tokio::task::JoinHandle<()>, - tokio::sync::watch::Receiver, - ) { - let (tx, rx) = tokio::sync::watch::channel(false); - let cache = self.clone(); - - let handle = tokio::spawn(async move { - loop { - tokio::time::sleep(tokio::time::Duration::from_secs(60)).await; - - match pool.get().await { - Ok(client) => { - match cache.initialize(&client).await { - Ok(changed) => { - if changed { - tracing::info!( - "Allowlist changed ({} entries) - signaling reconnection", - cache.cache.len() - ); - let _ = tx.send(true); - } else { - debug!("Allowlist cache refreshed ({} entries)", cache.cache.len()); - } - } - Err(e) => { - error!("Failed to refresh allowlist cache: {}", e); - } - } - } - Err(e) => { - error!( - "Failed to get database connection for allowlist refresh: {}", - e - ); - } - } - } - }); - - (handle, rx) - } -} - -impl Default for Allowlist { - fn default() -> Self { - Self::new() - } -} - -/// Gets all DIDs in the allowlist -/// -/// An actor is considered "allowlisted" if their sync_state is Synced, Dirty, or Processing. -/// Partial actors are not allowlisted (they only interact with allowlisted users). -pub async fn get_all(client: &C) -> Result> { - let rows = client - .query( - "SELECT did FROM actors - WHERE sync_state IN ('synced', 'dirty', 'processing')", - &[], - ) - .await?; - - Ok(rows - .into_iter() - .map(|row| row.get::<_, String>(0)) - .collect()) -} - -/// Adds a DID to the allowlist -/// -/// Sets the actor's sync_state to 'dirty' to mark them as allowlisted and needing backfill. -/// Creates the actor if they don't exist (using get_actor_id() with advisory lock protection). -/// Returns the number of rows affected (1 if newly added/updated, 0 if already allowlisted) -pub async fn add( - client: &C, - did: &str, - _description: Option<&str>, -) -> Result { - // Note: description parameter kept for API compatibility but is no longer stored - // Admin notes should be kept in external documentation - - // Use get_actor_id() which has advisory lock protection to prevent duplicate actors - let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(client, did).await?; - - // Update sync_state to 'dirty' to mark as allowlisted (only if currently partial) - let rows_affected = client - .execute( - "UPDATE actors SET sync_state = 'dirty'::actor_sync_state - WHERE id = $1 AND sync_state = 'partial'::actor_sync_state", - &[&actor_id], - ) - .await?; - - Ok(rows_affected) -} - -/// Ensure all allowlisted DIDs are queued for backfill -/// Returns (actors_found, profiles_enqueued) -/// -/// Note: Profiles are enqueued separately because partial CAR backfills may not include profiles. -/// The profile will be fetched independently while backfill processes other records. -/// -/// IMPORTANT: This function does NOT modify actor sync_state. It only enqueues jobs for actors -/// that are already in 'dirty' or 'processing' state. Sync state is managed by: -/// - Allowlist add: partial -> dirty -/// - Backfill start: dirty -> processing -/// - Backfill completion: processing -> synced -pub async fn ensure_allowlist_actors( - client: &C, - pool: &deadpool_postgres::Pool, -) -> Result<(u64, u64)> { - let rows = client - .query( - "SELECT did FROM actors - WHERE sync_state IN ('dirty', 'processing')", - &[], - ) - .await?; - - let mut actors_found = 0_u64; - let mut profiles_enqueued = 0_u64; - - for row in rows { - let did: String = row.get(0); - actors_found += 1; - - // Enqueue profile for fetching (partial CAR backfills may not include profiles) - let profile_uri = format!("at://{}/app.bsky.actor.profile/self", did); - if let Err(e) = crate::db::fetch_queue::enqueue(client, &profile_uri).await { - tracing::warn!( - "Failed to enqueue profile for allowlisted DID {}: {}", - did, - e - ); - } else { - profiles_enqueued += 1; - } - - // Add to backfill queue if not already queued - if let Err(e) = crate::db::backfill_jobs::enqueue_job(pool, &did).await { - tracing::warn!( - "Failed to enqueue backfill for allowlisted DID {}: {}", - did, - e - ); - } else { - tracing::debug!("Enqueued backfill for allowlisted DID: {}", did); - } - } - - Ok((actors_found, profiles_enqueued)) -} diff --git a/consumer/src/db/backfill_jobs.rs b/consumer/src/db/backfill_jobs.rs deleted file mode 100644 index e22d0465..00000000 --- a/consumer/src/db/backfill_jobs.rs +++ /dev/null @@ -1,264 +0,0 @@ -//! PostgreSQL-based backfill job management -//! -//! Replaces the Redis-based backfill queue with PostgreSQL table using FOR UPDATE SKIP LOCKED -//! for efficient worker coordination. Supports priority scheduling with exponential backoff for retries. - -use chrono::{DateTime, Duration, Utc}; -use deadpool_postgres::Pool; -use eyre::Result; - -const MAX_ATTEMPTS: i32 = 3; -const PROCESSING_TIMEOUT_MINUTES: i64 = 10; - -#[derive(Debug, Clone)] -pub struct BackfillJob { - pub did: String, - pub status: String, - pub attempts: i32, - pub last_error: Option, - pub scheduled_at: DateTime, - pub started_at: Option>, -} - -/// Check if a job should be enqueued -/// -/// Returns true if the job needs to be enqueued, false if already queued/processing/complete -pub async fn should_enqueue(pool: &Pool, did: &str) -> Result { - let client = pool.get().await?; - - let row = client - .query_opt( - "SELECT status FROM backfill_jobs WHERE did = $1", - &[&did], - ) - .await?; - - match row { - None => Ok(true), // Doesn't exist - safe to enqueue - Some(row) => { - let status: String = row.get(0); - // Only reject if successful, permanently failed, or currently processing - Ok(status != "successful" && status != "failed.permanent" && status != "processing") - } - } -} - -/// Enqueue a new backfill job (or re-enqueue for immediate retry) -/// -/// If the job already exists and is not successful or permanently failed, resets it to pending with NOW() schedule. -/// Jobs with status 'successful' or 'failed.permanent' are never re-enqueued. -pub async fn enqueue_job(pool: &Pool, did: &str) -> Result<()> { - let client = pool.get().await?; - - client - .execute( - "INSERT INTO backfill_jobs (did, status, scheduled_at) - VALUES ($1, 'pending', NOW()) - ON CONFLICT (did) - DO UPDATE SET - status = 'pending', - scheduled_at = NOW() - WHERE backfill_jobs.status != 'successful' - AND backfill_jobs.status != 'failed.permanent'", - &[&did], - ) - .await?; - - Ok(()) -} - -/// Mark job as processing and get its current state -/// -/// Returns (attempts, max_attempts) or None if job doesn't exist -pub async fn start_processing(pool: &Pool, did: &str) -> Result> { - let client = pool.get().await?; - - // Check if job exists - let exists: bool = client - .query_one( - "SELECT EXISTS(SELECT 1 FROM backfill_jobs WHERE did = $1)", - &[&did], - ) - .await? - .get(0); - - if !exists { - return Ok(None); - } - - // Get current attempt count and update state - let row = client - .query_one( - "UPDATE backfill_jobs - SET status = 'processing', - attempts = attempts + 1, - started_at = NOW() - WHERE did = $1 - RETURNING attempts", - &[&did], - ) - .await?; - - let attempts: i32 = row.get(0); - - Ok(Some((attempts, MAX_ATTEMPTS))) -} - -/// Mark job as successful -pub async fn mark_successful(pool: &Pool, did: &str) -> Result<()> { - let client = pool.get().await?; - - client - .execute( - "UPDATE backfill_jobs - SET status = 'successful', completed_at = NOW() - WHERE did = $1", - &[&did], - ) - .await?; - - Ok(()) -} - -/// Mark job as failed and schedule retry if attempts remaining -/// -/// Returns the current attempt number -pub async fn mark_failed(pool: &Pool, did: &str, error: &str) -> Result { - let client = pool.get().await?; - - // Get current attempts - let row = client - .query_one( - "SELECT attempts FROM backfill_jobs WHERE did = $1", - &[&did], - ) - .await?; - let attempts: i32 = row.get(0); - - if attempts >= MAX_ATTEMPTS { - // Permanently failed - client - .execute( - "UPDATE backfill_jobs - SET status = 'failed.permanent', - last_error = $2, - completed_at = NOW() - WHERE did = $1", - &[&did, &error], - ) - .await?; - } else { - // Schedule retry with exponential backoff: 2^attempts minutes - let delay_minutes = 2_i64.pow(attempts as u32); - let retry_at = Utc::now() + Duration::minutes(delay_minutes); - - client - .execute( - "UPDATE backfill_jobs - SET status = 'failed.retry', - last_error = $2, - scheduled_at = $3 - WHERE did = $1", - &[&did, &error, &retry_at], - ) - .await?; - } - - Ok(attempts) -} - -/// Get jobs ready to process (score <= now) -/// -/// Includes jobs from queue and stale processing jobs. -/// Uses FOR UPDATE SKIP LOCKED for efficient worker coordination. -pub async fn dequeue(pool: &Pool) -> Result> { - let client = pool.get().await?; - - // First, recover any stale processing jobs - // Jobs that have been processing for more than PROCESSING_TIMEOUT_MINUTES are considered stale - let stale_cutoff = Utc::now() - Duration::minutes(PROCESSING_TIMEOUT_MINUTES); - - // Mark stale jobs as failed.permanent if they've exceeded MAX_ATTEMPTS - client - .execute( - "UPDATE backfill_jobs - SET status = 'failed.permanent', - last_error = 'Exceeded max attempts due to timeouts', - completed_at = NOW() - WHERE status = 'processing' - AND started_at < $1 - AND attempts >= $2", - &[&stale_cutoff, &MAX_ATTEMPTS], - ) - .await?; - - // Reset remaining stale jobs to pending for retry - client - .execute( - "UPDATE backfill_jobs - SET status = 'pending', scheduled_at = NOW() - WHERE status = 'processing' AND started_at < $1", - &[&stale_cutoff], - ) - .await?; - - // Now dequeue the next ready job - let row = client - .query_opt( - "UPDATE backfill_jobs - SET status = 'processing', - attempts = attempts + 1, - started_at = NOW() - WHERE did = ( - SELECT did FROM backfill_jobs - WHERE status IN ('pending', 'failed.retry') - AND scheduled_at <= NOW() - ORDER BY scheduled_at - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) - RETURNING did, status, attempts, last_error, scheduled_at, started_at", - &[], - ) - .await?; - - Ok(row.map(|r| BackfillJob { - did: r.get(0), - status: r.get(1), - attempts: r.get(2), - last_error: r.get(3), - scheduled_at: r.get(4), - started_at: r.get(5), - })) -} - -/// Get queue statistics -/// -/// Returns (pending_count, processing_count, successful_count, failed_count) -pub async fn get_stats(pool: &Pool) -> Result<(i64, i64, i64, i64)> { - let client = pool.get().await?; - - let row = client - .query_one( - "SELECT - COUNT(*) FILTER (WHERE status IN ('pending', 'failed.retry')) as pending, - COUNT(*) FILTER (WHERE status = 'processing') as processing, - COUNT(*) FILTER (WHERE status = 'successful') as successful, - COUNT(*) FILTER (WHERE status = 'failed.permanent') as failed - FROM backfill_jobs", - &[], - ) - .await?; - - Ok((row.get(0), row.get(1), row.get(2), row.get(3))) -} - -#[cfg(test)] -mod tests { - - - #[test] - fn test_backfill_jobs_module_compiles() { - // Basic compilation test - integration tests will verify actual behavior - } -} diff --git a/consumer/src/db/cursors.rs b/consumer/src/db/cursors.rs deleted file mode 100644 index 6eb78f3d..00000000 --- a/consumer/src/db/cursors.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! PostgreSQL-based cursor management for Jetstream partitions -//! -//! Replaces Redis cursor storage with a simple PostgreSQL table. -//! Only handles Jetstream cursors since backfill cursors are tracked via repo rev in the actors table. - -use deadpool_postgres::Pool; -use eyre::Result; - -/// Load a Jetstream partition cursor from PostgreSQL -/// -/// Returns None if the cursor doesn't exist. -/// Partition examples: "posts", "likes", "reposts", "social" -pub async fn load(pool: &Pool, partition: &str) -> Result> { - let client = pool.get().await?; - - let row = client - .query_opt( - "SELECT cursor_value FROM jetstream_cursors WHERE partition = $1", - &[&partition], - ) - .await?; - - Ok(row.map(|r| r.get::<_, i64>(0) as u64)) -} - -/// Save a Jetstream partition cursor to PostgreSQL -/// -/// This uses UPSERT (INSERT ... ON CONFLICT DO UPDATE) to handle both -/// initial creation and updates. -pub async fn save(pool: &Pool, partition: &str, timestamp_us: u64) -> Result<()> { - let client = pool.get().await?; - - client - .execute( - "INSERT INTO jetstream_cursors (partition, cursor_value) - VALUES ($1, $2) - ON CONFLICT (partition) - DO UPDATE SET cursor_value = $2, updated_at = NOW()", - &[&partition, &(timestamp_us as i64)], - ) - .await?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - - - #[test] - fn test_cursor_module_compiles() { - // Basic compilation test - integration tests will verify actual behavior - } -} diff --git a/consumer/src/db/fetch_queue.rs b/consumer/src/db/fetch_queue.rs deleted file mode 100644 index 9da29e5b..00000000 --- a/consumer/src/db/fetch_queue.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! PostgreSQL-based record fetch queue -//! -//! Replaces the Redis-based fetch queue with PostgreSQL table using FOR UPDATE SKIP LOCKED -//! for efficient worker coordination. - -use deadpool_postgres::{GenericClient, Pool}; -use eyre::Result; - -const MAX_ATTEMPTS: i32 = 3; - -#[derive(Debug, Clone)] -pub struct FetchQueueItem { - pub id: i64, - pub at_uri: String, - pub attempts: i32, -} - -/// Enqueue a record for fetching (idempotent) -/// -/// Returns true if the record was newly enqueued, false if it already exists -pub async fn enqueue(conn: &C, at_uri: &str) -> Result { - let result = conn - .execute( - "INSERT INTO fetch_queue (at_uri) - VALUES ($1) - ON CONFLICT (at_uri) DO NOTHING", - &[&at_uri], - ) - .await?; - - Ok(result > 0) -} - -/// Enqueue multiple records in a batch -/// -/// Returns the number of records successfully enqueued (excluding duplicates) -pub async fn enqueue_batch(conn: &C, at_uris: &[String]) -> Result { - if at_uris.is_empty() { - return Ok(0); - } - - // Build multi-row INSERT with ON CONFLICT - let mut query = String::from("INSERT INTO fetch_queue (at_uri) VALUES "); - let mut params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new(); - - for (i, uri) in at_uris.iter().enumerate() { - if i > 0 { - query.push_str(", "); - } - query.push_str(&format!("(${})", i + 1)); - params.push(uri); - } - - query.push_str(" ON CONFLICT (at_uri) DO NOTHING"); - - let result = conn.execute(&query, ¶ms).await?; - - Ok(result as usize) -} - -/// Dequeue the next pending item for processing -/// -/// Uses FOR UPDATE SKIP LOCKED to avoid contention between workers. -/// Returns None if queue is empty. -pub async fn dequeue(pool: &Pool) -> Result> { - let client = pool.get().await?; - - let row = client - .query_opt( - "UPDATE fetch_queue - SET status = 'processing', attempts = attempts + 1, updated_at = NOW() - WHERE id = ( - SELECT id FROM fetch_queue - WHERE status = 'pending' - ORDER BY created_at - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) - RETURNING id, at_uri, attempts", - &[], - ) - .await?; - - Ok(row.map(|r| FetchQueueItem { - id: r.get(0), - at_uri: r.get(1), - attempts: r.get(2), - })) -} - -/// Mark a fetch as complete and remove from queue -pub async fn mark_complete(pool: &Pool, id: i64) -> Result<()> { - let client = pool.get().await?; - - client - .execute("DELETE FROM fetch_queue WHERE id = $1", &[&id]) - .await?; - - Ok(()) -} - -/// Mark a fetch as failed and retry if attempts remain -/// -/// If max attempts reached, marks as permanently failed. -/// Otherwise, resets to pending for retry. -pub async fn mark_failed(pool: &Pool, id: i64, error: &str) -> Result<()> { - let client = pool.get().await?; - - // Get current attempts - let row = client - .query_one("SELECT attempts FROM fetch_queue WHERE id = $1", &[&id]) - .await?; - let attempts: i32 = row.get(0); - - if attempts >= MAX_ATTEMPTS { - // Permanently failed - client - .execute( - "UPDATE fetch_queue - SET status = 'failed', last_error = $2, updated_at = NOW() - WHERE id = $1", - &[&id, &error], - ) - .await?; - } else { - // Retry - reset to pending - client - .execute( - "UPDATE fetch_queue - SET status = 'pending', last_error = $2, updated_at = NOW() - WHERE id = $1", - &[&id, &error], - ) - .await?; - } - - Ok(()) -} - -/// Get queue statistics -/// -/// Returns (pending_count, processing_count, failed_count) -pub async fn get_stats(pool: &Pool) -> Result<(i64, i64, i64)> { - let client = pool.get().await?; - - let row = client - .query_one( - "SELECT - COUNT(*) FILTER (WHERE status = 'pending') as pending, - COUNT(*) FILTER (WHERE status = 'processing') as processing, - COUNT(*) FILTER (WHERE status = 'failed') as failed - FROM fetch_queue", - &[], - ) - .await?; - - Ok((row.get(0), row.get(1), row.get(2))) -} - -#[cfg(test)] -mod tests { - - - #[test] - fn test_fetch_queue_module_compiles() { - // Basic compilation test - integration tests will verify actual behavior - } -} diff --git a/consumer/src/db/handle_resolution_queue.rs b/consumer/src/db/handle_resolution_queue.rs deleted file mode 100644 index 428c0070..00000000 --- a/consumer/src/db/handle_resolution_queue.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! PostgreSQL-based handle resolution queue -//! -//! Replaces the Redis-based handle resolution queue with PostgreSQL table using FOR UPDATE SKIP LOCKED -//! for efficient worker coordination. - -use deadpool_postgres::Pool; -use eyre::Result; - -const MAX_ATTEMPTS: i32 = 3; - -#[derive(Debug, Clone)] -pub struct HandleResolutionItem { - pub id: i64, - pub did: String, - pub attempts: i32, -} - -/// Enqueue a DID for handle resolution (idempotent) -/// -/// Returns true if the DID was newly enqueued, false if it already exists -pub async fn enqueue(pool: &Pool, did: &str) -> Result { - let client = pool.get().await?; - - let result = client - .execute( - "INSERT INTO handle_resolution_queue (did) - VALUES ($1) - ON CONFLICT (did) DO NOTHING", - &[&did], - ) - .await?; - - Ok(result > 0) -} - -/// Dequeue the next pending item for processing -/// -/// Uses FOR UPDATE SKIP LOCKED to avoid contention between workers. -/// Returns None if queue is empty. -pub async fn dequeue(pool: &Pool) -> Result> { - let client = pool.get().await?; - - let row = client - .query_opt( - "UPDATE handle_resolution_queue - SET status = 'processing', attempts = attempts + 1, updated_at = NOW() - WHERE id = ( - SELECT id FROM handle_resolution_queue - WHERE status = 'pending' - ORDER BY created_at - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) - RETURNING id, did, attempts", - &[], - ) - .await?; - - Ok(row.map(|r| HandleResolutionItem { - id: r.get(0), - did: r.get(1), - attempts: r.get(2), - })) -} - -/// Mark a handle resolution as complete and remove from queue -pub async fn mark_complete(pool: &Pool, id: i64) -> Result<()> { - let client = pool.get().await?; - - client - .execute("DELETE FROM handle_resolution_queue WHERE id = $1", &[&id]) - .await?; - - Ok(()) -} - -/// Mark a handle resolution as failed and retry if attempts remain -/// -/// If max attempts reached, marks as permanently failed. -/// Otherwise, resets to pending for retry. -pub async fn mark_failed(pool: &Pool, id: i64, error: &str) -> Result<()> { - let client = pool.get().await?; - - // Get current attempts - let row = client - .query_one("SELECT attempts FROM handle_resolution_queue WHERE id = $1", &[&id]) - .await?; - let attempts: i32 = row.get(0); - - if attempts >= MAX_ATTEMPTS { - // Permanently failed - client - .execute( - "UPDATE handle_resolution_queue - SET status = 'failed', last_error = $2, updated_at = NOW() - WHERE id = $1", - &[&id, &error], - ) - .await?; - } else { - // Retry - reset to pending - client - .execute( - "UPDATE handle_resolution_queue - SET status = 'pending', last_error = $2, updated_at = NOW() - WHERE id = $1", - &[&id, &error], - ) - .await?; - } - - Ok(()) -} - -/// Get queue statistics -/// -/// Returns (pending_count, processing_count, failed_count) -pub async fn get_stats(pool: &Pool) -> Result<(i64, i64, i64)> { - let client = pool.get().await?; - - let row = client - .query_one( - "SELECT - COUNT(*) FILTER (WHERE status = 'pending') as pending, - COUNT(*) FILTER (WHERE status = 'processing') as processing, - COUNT(*) FILTER (WHERE status = 'failed') as failed - FROM handle_resolution_queue", - &[], - ) - .await?; - - Ok((row.get(0), row.get(1), row.get(2))) -} - -#[cfg(test)] -mod tests { - - - #[test] - fn test_handle_resolution_queue_module_compiles() { - // Basic compilation test - integration tests will verify actual behavior - } -} diff --git a/consumer/src/db/mod.rs b/consumer/src/db/mod.rs index b1a675a2..d0ea7490 100644 --- a/consumer/src/db/mod.rs +++ b/consumer/src/db/mod.rs @@ -1,17 +1,11 @@ use crate::error::Result; use deadpool_postgres::GenericClient; -use eyre::WrapErr; pub mod actor; -pub mod allowlist; -pub mod backfill_jobs; pub mod bulk_copy; pub mod bulk_resolve; pub mod composite_builders; -pub mod cursors; -pub mod fetch_queue; pub mod gates; -pub mod handle_resolution_queue; pub mod id_resolution; pub mod labels; pub mod operations; @@ -19,103 +13,11 @@ pub mod record_exists; pub mod workers; pub use actor::*; -pub use allowlist::Allowlist; pub use gates::*; pub use labels::*; pub use operations::*; pub use record_exists::record_exists; -/// Mark a stub record as 'missing' (permanently unfetchable) -/// -/// This is called when a fetch permanently fails after MAX_ATTEMPTS. -/// Parses the AT URI to determine the collection type and updates the appropriate table. -pub async fn mark_stub_as_missing( - conn: &C, - at_uri: &str, -) -> Result<()> { - // Parse AT URI: at://did:plc:xyz/collection/rkey - let uri_parts: Vec<&str> = at_uri - .strip_prefix("at://") - .unwrap_or(at_uri) - .split('/') - .collect(); - - if uri_parts.len() != 3 { - return Err(eyre::eyre!("Invalid AT URI format: {}", at_uri)); - } - - let (did, collection, rkey_tid) = (uri_parts[0], uri_parts[1], uri_parts[2]); - - // Resolve DID to actor_id using actor cache (no JOIN needed!) - let actor_id = match actor::actor_id_from_did(conn, did).await? { - Some(id) => id, - None => { - tracing::debug!("Actor not found for DID {}, cannot mark stub as missing", did); - return Ok(()); // Actor doesn't exist, nothing to update - } - }; - - // Determine which table to update based on collection - let result = match collection { - "app.bsky.feed.post" => { - // Convert TID to i64 in Rust, not SQL - let rkey_i64 = parakeet_db::models::tid_to_i64(rkey_tid) - .wrap_err_with(|| format!("Invalid TID in rkey: {}", rkey_tid))?; - - // Use consolidated PostUpdate API - let result = operations::PostUpdate { - target: operations::PostUpdateTarget::Individual { - actor_id, - rkey: rkey_i64, - }, - status: Some(parakeet_db::types::PostStatus::Missing), - ..Default::default() - } - .execute(conn) - .await?; - - match result { - operations::PostUpdateResult::Count(n) => n, - _ => unreachable!("PostUpdate with no RETURNING should return Count"), - } - } - "app.bsky.feed.generator" => { - // Feedgens use text rkey, not TID - conn.execute( - "UPDATE feedgens - SET status = 'missing'::feedgen_status - WHERE actor_id = $1 - AND rkey = $2 - AND status = 'stub'::feedgen_status", - &[&actor_id, &rkey_tid], - ) - .await? - } - "app.bsky.labeler.service" => { - // Labelers are stored in actors table with labeler_status column - // Labelers use "self" rkey - conn.execute( - "UPDATE actors - SET labeler_status = 'missing'::labeler_status - WHERE id = $1 - AND labeler_status = 'stub'::labeler_status", - &[&actor_id], - ) - .await? - } - _ => { - tracing::debug!("Collection type does not use stubs: {}", collection); - 0 - } - }; - - if result > 0 { - tracing::debug!("Marked stub as missing: {} (collection: {})", at_uri, collection); - } - - Ok(()) -} - /// Check if a recipient has muted a thread /// /// # Arguments diff --git a/consumer/src/db/operations/feed/post.rs b/consumer/src/db/operations/feed/post.rs index 371db391..b9903ae8 100644 --- a/consumer/src/db/operations/feed/post.rs +++ b/consumer/src/db/operations/feed/post.rs @@ -65,7 +65,7 @@ pub async fn post_insert( root, &repo, rec.created_at, - source == crate::database_writer::EventSource::Backfill, + source == crate::database_writer::EventSource::TapBackfill, ) .await? } diff --git a/consumer/src/db/operations/feed/post_update.rs b/consumer/src/db/operations/feed/post_update.rs index 67e7f83b..8076c273 100644 --- a/consumer/src/db/operations/feed/post_update.rs +++ b/consumer/src/db/operations/feed/post_update.rs @@ -404,7 +404,7 @@ impl PostUpdate { .map(|(a, r)| format!("({}, {})", a, r)) .collect(); - let expr = if let Some(ts) = disable_after { + let expr = if let Some(_ts) = disable_after { if tuples.is_empty() { format!("(tid_timestamp(rkey) > ${})", param_idx) } else { diff --git a/consumer/src/db/operations/feed/postgate.rs b/consumer/src/db/operations/feed/postgate.rs index 2065e900..71b68718 100644 --- a/consumer/src/db/operations/feed/postgate.rs +++ b/consumer/src/db/operations/feed/postgate.rs @@ -27,9 +27,9 @@ use ipld_core::cid::Cid; /// We only store the postgate data in the post record itself. pub async fn postgate_upsert( conn: &C, - actor_id: i32, - rkey: i64, - _cid: Cid, // No longer stored - postgates are denormalized + _actor_id: i32, // Not used - postgates are denormalized into posts + _rkey: i64, // Not used - postgates are denormalized into posts + _cid: Cid, // No longer stored - postgates are denormalized rec: &AppBskyFeedPostgate, ) -> Result { let rules = rec diff --git a/consumer/src/db/operations/feed/threadgate.rs b/consumer/src/db/operations/feed/threadgate.rs index 71f4ac6c..3571605f 100644 --- a/consumer/src/db/operations/feed/threadgate.rs +++ b/consumer/src/db/operations/feed/threadgate.rs @@ -79,9 +79,9 @@ pub async fn threadgate_get( /// We only store the threadgate data in the post record itself. pub async fn threadgate_upsert( conn: &C, - actor_id: i32, - rkey: i64, - _cid: Cid, // No longer stored - threadgates are denormalized + _actor_id: i32, // Not used - threadgates are denormalized into posts + _rkey: i64, // Not used - threadgates are denormalized into posts + _cid: Cid, // No longer stored - threadgates are denormalized rec: AppBskyFeedThreadgate, ) -> Result { // Extract allow rules as strings diff --git a/consumer/src/db/workers.rs b/consumer/src/db/workers.rs index 87c8f11b..72cf407e 100644 --- a/consumer/src/db/workers.rs +++ b/consumer/src/db/workers.rs @@ -1,13 +1,10 @@ //! Database operations for batch writer workers //! //! These functions support the batch writer workers with bulk operations -//! for actor ensuring, backfill status updates, and cache warming queries. +//! for actor ensuring. use super::Result; -use chrono::{DateTime, Utc}; use deadpool_postgres::GenericClient; -use eyre::Context as _; -use parakeet_db::types::ActorSyncState; /// Bulk ensure actors exist in the database /// @@ -33,109 +30,3 @@ pub async fn bulk_ensure_actors(conn: &C, dids: &[&str]) -> Re Ok(created_count) } -/// Update actor sync state and last_indexed after successful backfill -/// -/// Used by backfill workers to mark actors as successfully synced. -/// -/// # Arguments -/// * `conn` - Database connection -/// * `did` - Actor DID -/// * `sync_state` - New sync state (usually 'synced') -/// * `last_indexed` - Timestamp of completion -pub async fn backfill_update_actor_status( - conn: &C, - did: &str, - sync_state: &ActorSyncState, - last_indexed: DateTime, -) -> Result { - super::actor::actor_set_sync_status(conn, did, sync_state, last_indexed).await -} - -/// Mark actor as processing during backfill download -/// -/// Used by the backfill downloader to prevent concurrent downloads -/// of the same repo. -/// -/// This function uses get_actor_id() which has advisory lock protection -/// to prevent duplicate actor creation. -/// -/// # Arguments -/// * `conn` - Database connection -/// * `did` - Actor DID -pub async fn backfill_mark_processing(conn: &C, did: &str) -> Result { - // Use get_actor_id() which has advisory lock protection to prevent duplicates - let (actor_id, _, _) = crate::db::operations::feed::get_actor_id(conn, did).await?; - - // Use consolidated ActorUpdate API - use crate::db::operations::{ActorUpdate, ActorUpdateResult, ActorUpdateTarget}; - use chrono::Utc; - use parakeet_db::types::ActorSyncState; - - let result = ActorUpdate { - target: ActorUpdateTarget::ById(actor_id), - sync_state: Some(ActorSyncState::Processing), - last_indexed: Some(Utc::now()), - ..Default::default() - } - .execute(conn) - .await - .wrap_err_with(|| format!("Failed to mark actor {} as processing", did))?; - - match result { - ActorUpdateResult::Count(n) => Ok(n), - _ => unreachable!("ActorUpdate with Count returning should return Count"), - } -} - -/// Get pinned post URI for profile (used for constellation cache warming) -/// -/// Returns the AT URI of the pinned post if the actor has one. -/// -/// # Arguments -/// * `conn` - Database connection -/// * `did` - Actor DID -pub async fn get_pinned_post_uri(conn: &C, did: &str) -> Result> { - let row_opt = conn - .query_opt( - "SELECT 'at://' || a.did || '/app.bsky.feed.post/' || i64_to_tid(p.rkey) as pinned_uri - FROM actors a - LEFT JOIN posts p ON a.id = p.actor_id AND a.profile_pinned_post_rkey = p.rkey - WHERE a.did = $1 AND a.profile_pinned_post_rkey IS NOT NULL", - &[&did], - ) - .await?; - - Ok(row_opt.map(|row| row.get(0))) -} - -/// Get recent post URIs for actor (used for constellation cache warming) -/// -/// Returns up to 25 most recent post URIs for the actor. -/// -/// # Arguments -/// * `conn` - Database connection -/// * `did` - Actor DID -pub async fn get_recent_post_uris(conn: &C, did: &str) -> Result> { - let rows = conn - .query( - "SELECT 'at://' || a.did || '/app.bsky.feed.post/' || i64_to_tid(p.rkey) as post_uri - FROM posts p - INNER JOIN actors a ON p.actor_id = a.id - WHERE a.did = $1 - ORDER BY p.rkey DESC - LIMIT 25", - &[&did], - ) - .await?; - - Ok(rows.iter().map(|row| row.get(0)).collect()) -} - -// NOTE: PDS host tracking was moved to moka-based PDS cache (in-memory, TTL: 24h) -// The SQL below for pds_hosts and actor_pds_mapping tables is dead code -// that remains in database_writer/workers.rs but the tables don't exist -// in the current schema. PDS resolution now uses moka-based caching instead. -// -// Dead SQL locations in database_writer/workers.rs: -// - Lines ~2080: INSERT INTO pds_hosts (dead code) -// - Lines ~2111: INSERT INTO actor_pds_mapping (dead code) diff --git a/consumer/src/events.rs b/consumer/src/events.rs deleted file mode 100644 index 13e8df63..00000000 --- a/consumer/src/events.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Internal event types for the indexer -//! These types are used internally to process events from various transport layers (Jetstream, etc.) - -#![allow(unused)] - -use chrono::prelude::*; -use ipld_core::cid::Cid; -use serde::Deserialize; -use serde_bytes::ByteBuf; - -/// Raw Jetstream data types for parallel processing -#[derive(Debug)] -pub enum RawJetstreamData { - /// Raw text message - Text { - /// The content of the text message - content: String, - /// The partition this message came from - partition: crate::sources::unified_consumer::Partition, - }, - /// Raw binary message (potentially compressed) - Binary { - /// The raw binary data - data: Vec, - /// The partition this message came from - partition: crate::sources::unified_consumer::Partition, - }, -} - -/// Internal indexer event types - these are the events that the indexer processes -/// regardless of the transport layer (Jetstream, etc.) -#[derive(Debug)] -pub enum IndexerEvent { - Identity(AtpIdentityEvent), - Account(AtpAccountEvent), - Commit(Box), - Label(AtpLabelEvent), - Sync(AtpSyncEvent), - RawJetstream(RawJetstreamData), -} - -#[derive(Debug, Deserialize)] -pub struct AtpIdentityEvent { - pub seq: u64, - pub did: String, - pub time: DateTime, - pub handle: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum AtpAccountStatus { - Takendown, - Suspended, - Deleted, - Deactivated, - Throttled, - Desynchronized, -} - -impl AtpAccountStatus { - pub const fn as_str(&self) -> &'static str { - match self { - Self::Takendown => "takendown", - Self::Suspended => "suspended", - Self::Deleted => "deleted", - Self::Deactivated => "deactivated", - Self::Throttled => "throttled", - Self::Desynchronized => "desynchronized", - } - } -} - -impl From for parakeet_db::types::ActorStatus { - fn from(value: AtpAccountStatus) -> Self { - match value { - AtpAccountStatus::Takendown => Self::Takendown, - AtpAccountStatus::Suspended => Self::Suspended, - AtpAccountStatus::Deleted => Self::Deleted, - AtpAccountStatus::Deactivated => Self::Deactivated, - AtpAccountStatus::Throttled | AtpAccountStatus::Desynchronized => Self::Active, - } - } -} - -#[derive(Debug, Deserialize)] -pub struct AtpAccountEvent { - pub seq: u64, - pub did: String, - pub time: DateTime, - pub active: bool, - pub status: Option, -} - -#[derive(Debug, Deserialize)] -pub struct AtpCommitEvent { - pub seq: u64, - pub repo: String, - pub time: DateTime, - pub rev: String, - pub since: Option, - pub commit: Option, - #[serde(rename = "tooBig")] - #[deprecated(note = "Legacy AT Protocol field, no longer used")] - #[allow(dead_code, reason = "Required for deserialization of legacy relay events")] - pub too_big: bool, - #[serde(default)] - pub blocks: ByteBuf, - #[serde(default)] - pub ops: Vec, - #[serde(default)] - #[deprecated(note = "Legacy AT Protocol field, no longer used")] - #[allow(dead_code, reason = "Required for deserialization of legacy relay events")] - pub blobs: Vec, - #[serde(rename = "prevData")] - pub prev_data: Option, -} - -#[derive(Debug, Deserialize)] -pub struct CommitOp { - pub action: String, - pub cid: Option, - pub prev: Option, - pub path: String, - pub record: Option, -} - -#[derive(Debug, Deserialize)] -pub struct AtpLabel { - pub ver: i32, - pub src: String, - pub uri: String, - pub cid: Option, - pub val: String, - pub neg: Option, - pub cts: DateTime, - pub exp: Option>, - pub sig: Option, -} - -#[derive(Debug, Deserialize)] -pub struct AtpLabelEvent { - pub seq: u64, - pub labels: Vec, -} - -#[derive(Debug, Deserialize)] -pub struct AtpSyncEvent { - pub seq: u64, - pub did: String, - pub time: DateTime, - pub rev: String, - #[serde(default)] - pub blocks: ByteBuf, -} diff --git a/consumer/src/external/mod.rs b/consumer/src/external/mod.rs index d428f74f..b284437c 100644 --- a/consumer/src/external/mod.rs +++ b/consumer/src/external/mod.rs @@ -1,10 +1,6 @@ //! External service clients and integrations //! //! This module contains clients for external services: -//! - PostgreSQL cursor manager //! - PostgreSQL notifications -//! - Moka-based PDS cache -pub mod pds_cache; -pub mod pg_cursor_manager; pub mod pg_notifications; diff --git a/consumer/src/external/pds_cache.rs b/consumer/src/external/pds_cache.rs deleted file mode 100644 index 4fff23d5..00000000 --- a/consumer/src/external/pds_cache.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! Moka-based PDS host cache -//! -//! This module provides functions to cache PDS host mappings in memory using moka. -//! PDS hosts rarely change, so we cache them with a 24-hour TTL to reduce lookups. - -use moka::future::Cache; -use std::collections::HashMap; -use std::time::Duration; - -/// PDS host cache TTL: 24 hours -/// PDS hosts rarely change, so we can cache them for a long time -const PDS_HOST_CACHE_TTL_SECS: u64 = 86400; - -/// PDS host cache capacity: 100k entries -/// Each entry is small (~50 bytes), so 100k entries = ~5MB -const PDS_HOST_CACHE_CAPACITY: u64 = 100_000; - -/// A thread-safe in-memory PDS host cache -#[derive(Clone)] -pub struct PdsHostCache { - cache: Cache, -} - -impl PdsHostCache { - /// Create a new PDS host cache with 24h TTL and 100k capacity - pub fn new() -> Self { - Self { - cache: Cache::builder() - .max_capacity(PDS_HOST_CACHE_CAPACITY) - .time_to_live(Duration::from_secs(PDS_HOST_CACHE_TTL_SECS)) - .build(), - } - } - - /// Get a single PDS host for a DID from cache - pub async fn get(&self, did: &str) -> Option { - self.cache.get(did).await - } - - /// Get multiple PDS hosts for DIDs from cache (batch operation) - pub async fn get_batch(&self, dids: &[&str]) -> HashMap { - let mut result = HashMap::new(); - - for did in dids { - if let Some(host) = self.cache.get(*did).await { - result.insert(did.to_string(), host); - } - } - - result - } - - /// Cache a PDS host mapping - pub async fn set(&self, did: String, pds_host: String) { - self.cache.insert(did, pds_host).await; - } - - /// Cache multiple PDS host mappings (batch operation) - pub async fn set_batch(&self, mappings: &HashMap) { - for (did, host) in mappings { - self.cache.insert(did.clone(), host.clone()).await; - } - } -} - -impl Default for PdsHostCache { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_pds_cache_get_set() { - let cache = PdsHostCache::new(); - - // Initially empty - assert_eq!(cache.get("did:plc:test").await, None); - - // Set and get - cache - .set("did:plc:test".to_string(), "https://test.pds".to_string()) - .await; - assert_eq!( - cache.get("did:plc:test").await, - Some("https://test.pds".to_string()) - ); - } - - #[tokio::test] - async fn test_pds_cache_batch_operations() { - let cache = PdsHostCache::new(); - - // Set batch - let mut mappings = HashMap::new(); - mappings.insert("did:plc:test1".to_string(), "https://pds1.example".to_string()); - mappings.insert("did:plc:test2".to_string(), "https://pds2.example".to_string()); - cache.set_batch(&mappings).await; - - // Get batch - let dids = vec!["did:plc:test1", "did:plc:test2", "did:plc:nonexistent"]; - let result = cache.get_batch(&dids).await; - - assert_eq!(result.len(), 2); - assert_eq!( - result.get("did:plc:test1"), - Some(&"https://pds1.example".to_string()) - ); - assert_eq!( - result.get("did:plc:test2"), - Some(&"https://pds2.example".to_string()) - ); - assert_eq!(result.get("did:plc:nonexistent"), None); - } -} diff --git a/consumer/src/external/pg_cursor_manager.rs b/consumer/src/external/pg_cursor_manager.rs deleted file mode 100644 index eba17351..00000000 --- a/consumer/src/external/pg_cursor_manager.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! PostgreSQL-based cursor management for Jetstream partitions -//! -//! Provides simple, reliable cursor storage using PostgreSQL with ACID guarantees. -//! Only handles Jetstream cursors since backfill cursors are tracked via repo rev -//! in the actors table. - -use eyre::Result; - -/// Simple cursor manager using PostgreSQL -#[derive(Clone)] -pub struct PgCursorManager { - pool: deadpool_postgres::Pool, -} - -impl PgCursorManager { - /// Create a new PostgreSQL cursor manager - pub fn new(pool: deadpool_postgres::Pool) -> Self { - Self { pool } - } - - /// Load a Jetstream partition cursor from PostgreSQL - /// - /// Returns None if the cursor doesn't exist. - /// Partition examples: "posts", "likes", "reposts", "social" - pub async fn load(&self, partition: &str) -> Result> { - let client = self.pool.get().await?; - - let row = client - .query_opt( - "SELECT cursor_value FROM jetstream_cursors WHERE partition = $1", - &[&partition], - ) - .await?; - - Ok(row.map(|r| r.get(0))) - } - - /// Save a Jetstream partition cursor to PostgreSQL - /// - /// Uses UPSERT to create or update the cursor atomically. - pub async fn save(&self, partition: &str, timestamp_us: i64) -> Result<()> { - let client = self.pool.get().await?; - - client - .execute( - "INSERT INTO jetstream_cursors (partition, cursor_value, updated_at) - VALUES ($1, $2, NOW()) - ON CONFLICT (partition) - DO UPDATE SET cursor_value = $2, updated_at = NOW()", - &[&partition, ×tamp_us], - ) - .await?; - - Ok(()) - } -} diff --git a/consumer/src/indexer/conversion.rs b/consumer/src/indexer/conversion.rs deleted file mode 100644 index 9aa4e20f..00000000 --- a/consumer/src/indexer/conversion.rs +++ /dev/null @@ -1,170 +0,0 @@ -//! Conversion functions for transforming Jetstream events to internal indexer events - -use crate::events::{AtpCommitEvent, CommitOp}; -use ipld_core::cid::Cid; -use serde_bytes::ByteBuf; - -/// Convert a Jetstream commit event to an internal commit event (worker version) -#[expect(clippy::too_many_lines, reason = "Comprehensive handling of all Jetstream operation types")] -pub fn convert_jetstream_commit_worker( - commit_event: &crate::sources::jetstream::CommitEvent, -) -> Option { - use crate::sources::jetstream::CommitOperation; - - let did = &commit_event.did; - let commit = &commit_event.commit; - - // Handle different operation types with appropriate CID handling - let commit_cid = if commit.op == CommitOperation::Delete { - None - } else { - // For create/update operations, we need a valid CID - if let Some(cid_str) = &commit.cid { - match Cid::try_from(cid_str.as_str()) { - Ok(cid) => Some(cid), - Err(e) => { - tracing::error!("Invalid CID in commit.cid: {}, error: {}", cid_str, e); - return None; - } - } - } else { - tracing::error!("Create/Update operation missing commit.cid"); - return None; - } - }; - - // Convert microsecond timestamp to DateTime - let timestamp = - chrono::DateTime::::from_timestamp_micros(commit_event.time_us as i64) - .unwrap_or_else(chrono::Utc::now); - - // Map Jetstream operation to internal operation - let op = match commit.op { - CommitOperation::Create => { - if let Some(record) = &commit.record { - if let Some(cid_str) = &commit.cid { - // Convert the path to "collection/rkey" format - let path = format!("{}/{}", commit.collection, commit.rkey); - - // Try to parse the CID - Jetstream should provide valid CIDs - let cid = match Cid::try_from(cid_str.as_str()) { - Ok(cid) => Some(cid), - Err(e) => { - tracing::error!( - "Invalid CID in commit.cid for create operation: {}, error: {}", - cid_str, - e - ); - return None; - } - }; - - CommitOp { - action: "create".to_owned(), - path, - cid, - prev: None, - record: Some(record.clone()), - } - } else { - tracing::warn!("Create operation missing CID"); - return None; - } - } else { - tracing::warn!("Create operation missing record"); - return None; - } - } - CommitOperation::Update => { - if let Some(record) = &commit.record { - if let Some(cid_str) = &commit.cid { - // Convert the path to "collection/rkey" format - let path = format!("{}/{}", commit.collection, commit.rkey); - - // Try to parse the CID - Jetstream should provide valid CIDs - let cid = match Cid::try_from(cid_str.as_str()) { - Ok(cid) => { - tracing::debug!("Parsed valid CID in update operation: {}", cid_str); - Some(cid) - } - Err(e) => { - tracing::error!( - "Invalid CID in commit.cid for update operation: {}, error: {}", - cid_str, - e - ); - return None; - } - }; - - // Handle prev field if available (according to Jetstream docs, prev field exists for update operations) - let prev = if let Some(prev_str) = &commit.prev { - // For Jetstream, prev should be a proper CID - match Cid::try_from(prev_str.as_str()) { - Ok(prev_cid) => { - tracing::debug!("Parsed valid prev CID: {}", prev_str); - Some(prev_cid) - } - Err(e) => { - tracing::error!( - "Invalid CID in commit.prev for update operation: {}, error: {}", - prev_str, - e - ); - return None; - } - } - } else { - None - }; - - CommitOp { - action: "update".to_owned(), - path, - cid, - prev, - record: Some(record.clone()), - } - } else { - tracing::warn!("Update operation missing CID"); - return None; - } - } else { - tracing::warn!("Update operation missing record"); - return None; - } - } - CommitOperation::Delete => { - // Convert the path to "collection/rkey" format - let path = format!("{}/{}", commit.collection, commit.rkey); - - CommitOp { - action: "delete".to_owned(), - path, - cid: None, - prev: None, - record: None, - } - } - }; - - // Create the internal commit event - let event = AtpCommitEvent { - seq: 0, // Will be set correctly by caller - time: timestamp, - repo: did.clone(), - rev: commit.rev.clone(), // Original rev from Jetstream - commit: commit_cid, // Parsed CID from Jetstream event - ops: vec![op], - since: None, // Jetstream doesn't provide this - blocks: ByteBuf::new(), - // Fields are deprecated but still required by struct - #[expect(deprecated, reason = "Required by legacy AtpCommitEvent structure")] - too_big: false, - #[expect(deprecated, reason = "Required by legacy AtpCommitEvent structure")] - blobs: vec![], - prev_data: None, // Jetstream doesn't provide prev_cid - }; - - Some(event) -} diff --git a/consumer/src/indexer/mod.rs b/consumer/src/indexer/mod.rs deleted file mode 100644 index 777ef9b3..00000000 --- a/consumer/src/indexer/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Indexer module - shared operations and routing -//! -//! This module contains shared operations and routing logic. -//! The main relay indexer has been moved to the relay/ module. -//! Record types have been moved to the types/ module. - -// batch_writer moved to top-level database_writer module -// worker, jetstream_handler_dbfree, request_context moved to workers/jetstream module -// RelayIndexer moved to relay/ module -// records moved to types/ module -mod conversion; -pub mod operations; -pub(crate) mod routing; - -// Re-export commonly used items -pub(crate) use conversion::*; diff --git a/consumer/src/indexer/operations/decode.rs b/consumer/src/indexer/operations/decode.rs deleted file mode 100644 index e38b6584..00000000 --- a/consumer/src/indexer/operations/decode.rs +++ /dev/null @@ -1,84 +0,0 @@ -use crate::events::CommitOp; -use crate::relay::types::RecordTypes; -use crate::workers::fetch::json_types::RecordTypesJson; -use ipld_core::cid::Cid; -use std::collections::HashMap; - -pub fn decode_op(op: &CommitOp, blocks: &HashMap>) -> Option<(Cid, RecordTypes)> { - // First try to get the record directly from Jetstream if it exists - if let Some(record_value) = &op.record { - if let Some(cid) = op.cid { - // Try to deserialize the JSON record using RecordTypesJson which handles BlobJson - match serde_json::from_value::(record_value.clone()) { - Ok(record_json) => { - // Convert JSON types to standard types - let record: RecordTypes = record_json.into(); - return Some((cid, record)); - } - Err(err) => { - // Enhanced error reporting with record type and structure details - if let Some(type_value) = record_value.get("$type") { - let type_str = type_value.as_str().unwrap_or("unknown"); - - // Custom/unknown lexicons (not app.bsky.*) should be DEBUG, not WARN - // This includes: - // - app.bsky.actor.status (not yet implemented) - // - fm.teal.alpha.* (custom lexicons) - // - Any other non-standard lexicons - if !type_str.starts_with("app.bsky.") || type_str == "app.bsky.actor.status" { - tracing::debug!("Skipping unimplemented/unknown {} record", type_str); - return None; - } - - // Only WARN for official app.bsky lexicons that we should support - tracing::warn!("Failed to deserialize {} record: {}", type_str, err); - - // For actor profile records, provide detailed debugging - if type_str == "app.bsky.actor.profile" { - if let Ok(json_str) = serde_json::to_string(record_value) { - tracing::warn!( - "Failed profile record content (truncated): {:.200}...", - json_str - ); - - // Check for specific fields that might cause problems - if record_value.get("avatar").is_some() { - tracing::warn!( - "Profile has avatar field with structure: {:?}", - record_value.get("avatar") - ); - } - if record_value.get("banner").is_some() { - tracing::warn!( - "Profile has banner field with structure: {:?}", - record_value.get("banner") - ); - } - } - } - } else { - tracing::warn!( - "Failed to deserialize Jetstream record (no $type): {}", - err - ); - } - return None; - } - } - } - } - - // Traditional method using CBOR-encoded blocks - let cid = op.cid?; - let block = blocks.get(&cid)?; - - let reader = std::io::Cursor::new(block); - - match serde_ipld_dagcbor::from_reader(reader) { - Ok(data) => Some((cid, data)), - Err(err) => { - tracing::error!("Failed to decode record: {err}"); - None - } - } -} diff --git a/consumer/src/indexer/operations/mod.rs b/consumer/src/indexer/operations/mod.rs deleted file mode 100644 index 5ac80bbf..00000000 --- a/consumer/src/indexer/operations/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod decode; diff --git a/consumer/src/indexer/routing.rs b/consumer/src/indexer/routing.rs deleted file mode 100644 index 4c52e844..00000000 --- a/consumer/src/indexer/routing.rs +++ /dev/null @@ -1,89 +0,0 @@ -use crate::events::IndexerEvent; -use crate::sources::jetstream::RawJetstreamMessage; -use metrics::counter; -use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::sync::mpsc::Sender; - -/// Round-robin counter for worker selection -static ROUND_ROBIN_COUNTER: AtomicU64 = AtomicU64::new(0); - -/// Route a raw Jetstream message to a worker thread -/// -/// Uses round-robin for perfect load balancing across workers. -/// Non-blocking send, but counts when we would need to wait. -pub async fn route_raw_jetstream_message( - partition: crate::sources::unified_consumer::Partition, - raw_message: RawJetstreamMessage, - threads: u64, - submit: &[Sender], -) { - match raw_message { - RawJetstreamMessage::Text { content } => { - // Round-robin primary choice - let primary_idx = ROUND_ROBIN_COUNTER.fetch_add(1, Ordering::Relaxed) % threads; - - // Create raw event wrapper with partition info - let raw_event = IndexerEvent::RawJetstream(crate::events::RawJetstreamData::Text { - content, - partition, - }); - - // Try non-blocking send first - match submit[primary_idx as usize].try_send(raw_event) { - Ok(_) => { - counter!("jetstream_events.raw_text_processed").increment(1); - counter!("jetstream_events.send_immediate").increment(1); - } - Err(tokio::sync::mpsc::error::TrySendError::Full(event)) => { - // Worker channel full - need to wait (count this for visibility) - counter!("jetstream_events.worker_wait").increment(1); - - // Wait for space (this will block but ensures no events are dropped) - if let Err(e) = submit[primary_idx as usize].send(event).await { - tracing::error!("Error sending raw text event after wait: {e}"); - } else { - counter!("jetstream_events.raw_text_processed").increment(1); - counter!("jetstream_events.send_after_wait").increment(1); - } - } - Err(e) => { - tracing::error!("Error sending raw text event: {e}"); - } - } - } - RawJetstreamMessage::Binary { data } => { - // Round-robin primary choice - let primary_idx = ROUND_ROBIN_COUNTER.fetch_add(1, Ordering::Relaxed) % threads; - - // Create raw event wrapper with partition info - let raw_event = IndexerEvent::RawJetstream(crate::events::RawJetstreamData::Binary { - data, - partition, - }); - - // Try non-blocking send first - match submit[primary_idx as usize].try_send(raw_event) { - Ok(_) => { - counter!("jetstream_events.raw_binary_processed").increment(1); - } - Err(tokio::sync::mpsc::error::TrySendError::Full(event)) => { - // Worker channel full - need to wait - counter!("jetstream_events.worker_wait").increment(1); - - // Wait for space (ensures no events are dropped) - if let Err(e) = submit[primary_idx as usize].send(event).await { - tracing::error!("Error sending raw binary event after wait: {e}"); - } else { - counter!("jetstream_events.raw_binary_processed").increment(1); - } - } - Err(e) => { - tracing::error!("Error sending raw binary event: {e}"); - } - } - } - RawJetstreamMessage::Close => { - tracing::info!("Received WebSocket close message"); - } - } -} diff --git a/consumer/src/lib.rs b/consumer/src/lib.rs index 40149432..c94aed8f 100644 --- a/consumer/src/lib.rs +++ b/consumer/src/lib.rs @@ -9,11 +9,8 @@ mod config; pub mod database_writer; pub mod db; mod error; -mod events; mod external; -mod indexer; // mod label_indexer; // Disabled - will be reimplemented with new cursor system -pub mod parsing; mod relay; pub mod search; mod sources; @@ -25,7 +22,7 @@ pub mod workers; // Re-export key types and functions for binary use pub use cmd::{parse, Cli}; pub use config::{load_config, Config}; -pub use database_writer::spawn_database_writer; +pub use database_writer::spawn_database_writer_tap; pub use error::Result; // Re-export modules for testing @@ -42,7 +39,6 @@ pub mod external_services { } pub mod indexing { - pub use crate::indexer::*; pub use crate::relay::*; } @@ -56,7 +52,6 @@ pub mod streaming { } pub mod event_types { - pub use crate::events::*; pub use crate::types::*; } diff --git a/consumer/src/main.rs b/consumer/src/main.rs index 23125a43..395c7b95 100644 --- a/consumer/src/main.rs +++ b/consumer/src/main.rs @@ -1,12 +1,7 @@ -use consumer::db_modules as db; -use consumer::external_services as external; use consumer::indexing as relay; -use consumer::streaming::workers; use deadpool_postgres::Runtime; -use did_resolver::{Resolver, ResolverOpts}; use eyre::OptionExt as _; use metrics_exporter_prometheus::PrometheusBuilder; -use std::sync::Arc; use tokio::signal::ctrl_c; use tokio_postgres::NoTls; @@ -46,38 +41,26 @@ async fn main() -> eyre::Result<()> { let cli = consumer::parse(); let mut conf = consumer::load_config()?; - let user_agent = consumer::build_user_agent(conf.ua_contact.as_ref()); - // Configure database connection pool for 45-worker database writer architecture - // 45 workers + 15 headroom for burst traffic + other operations = 60 connections + // Configure database connection pool for 30-worker database writer architecture + // 30 workers + 15 headroom for burst traffic + other operations = 45 connections conf.database.pool = Some(deadpool_postgres::PoolConfig { - max_size: 60, + max_size: 45, ..Default::default() }); tracing::info!( - "Database connection pool configured with max_size=60 for 45-worker database writer" + "Database connection pool configured with max_size=45 for 30-worker database writer" ); let pool = conf.database.create_pool(Some(Runtime::Tokio1), NoTls)?; - // All queue operations use PostgreSQL. - - let resolver = Arc::new(Resolver::new(ResolverOpts { - plc_directory: conf.plc_directory, - user_agent: Some(user_agent.clone()), - ..Default::default() - })?); + // Note: DID/handle resolution is now handled by Tap let tracker = tokio_util::task::TaskTracker::new(); let (stop_tx, stop) = tokio::sync::watch::channel(false); - if cli.labels { - tracing::warn!("Label indexer is currently disabled and will be reimplemented with the new cursor system"); - } - // Calculate retention cutoff from config if retention_days is set - // This applies to both backfill and indexer modes - let retention_cutoff = conf.retention_days.map(|days| { + let _retention_cutoff = conf.retention_days.map(|days| { let cutoff = chrono::Utc::now() - chrono::Duration::days(i64::from(days)); tracing::info!( retention_days = days, @@ -87,48 +70,15 @@ async fn main() -> eyre::Result<()> { cutoff }); - // Create shared database writer infrastructure if backfill or indexer modes are enabled - let shared_database_writer = if cli.backfill || cli.indexer { - // Create and initialize the allowlist (used for Jetstream server-side filtering via wantedDids) - let allowlist = db::Allowlist::new(); - { - let conn = pool.get().await?; - if let Err(e) = allowlist.initialize(&conn).await { - tracing::warn!("Failed to initialize allowlist cache: {}", e); - } - // Initial load doesn't need to check for changes - } - - // Create three source-specific channels for database writer (bounded to provide backpressure) - // When channels fill up, workers will block, propagating backpressure - // Each source has its own dispatcher and 15 workers (5 types × 3 workers) - let (jetstream_tx, jetstream_rx) = tokio::sync::mpsc::channel::(10_000); - let (fetch_tx, fetch_rx) = tokio::sync::mpsc::channel::(5_000); - let (backfill_tx, backfill_rx) = tokio::sync::mpsc::channel::(5_000); - - // Load unified cursor for indexer mode - let cursor_arc = if cli.indexer { - // Load Jetstream cursor from PostgreSQL for indexer mode - let cursor_manager = external::pg_cursor_manager::PgCursorManager::new(pool.clone()); - - let cursor_value = if let Ok(Some(cursor)) = cursor_manager.load("jetstream").await { - tracing::info!("Loaded cursor from PostgreSQL: {}", cursor); - cursor as u64 - } else { - tracing::info!("No cursor found in PostgreSQL, starting from 0"); - 0 - }; - - std::sync::Arc::new(std::sync::RwLock::new(cursor_value)) - } else { - // Backfill-only mode: no cursor tracking needed - std::sync::Arc::new(std::sync::RwLock::new(0)) - }; + // Create shared database writer infrastructure if indexer mode is enabled + let shared_database_writer = if cli.indexer { + // Create channel for database writer (bounded to provide backpressure) + // When channel fills up, workers will block, propagating backpressure + let (tap_tx, tap_rx) = tokio::sync::mpsc::channel::(10_000); // Spawn single database writer task let batch_events_processed = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); let batch_operations_processed = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); - let fetch_queue_enqueued = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); // Create unified ID cache for fast DID→actor_id and URI→post_id lookups // Actor cache: 5min TTL, 50k capacity @@ -136,16 +86,12 @@ async fn main() -> eyre::Result<()> { let id_cache = parakeet_db::id_cache::IdCache::new(); tracing::info!("ID cache initialized (actors: 5min TTL/50k capacity, posts: 10min TTL/50k capacity)"); - let database_writer_handle = consumer::spawn_database_writer( + // Spawn database writer with Tap as primary source + let database_writer_handle = consumer::spawn_database_writer_tap( pool.clone(), - jetstream_rx, - fetch_rx, - backfill_rx, + tap_rx, batch_events_processed.clone(), batch_operations_processed.clone(), - fetch_queue_enqueued.clone(), - cursor_arc.clone(), - allowlist.clone(), id_cache, stop.clone(), ); @@ -154,192 +100,55 @@ async fn main() -> eyre::Result<()> { // Create worker supervisor (shared for all supervised workers) let worker_supervisor = consumer::worker_core::WorkerSupervisor::new(); - // Spawn handle resolution worker (PostgreSQL queue) - let handle_factory = workers::HandleResolutionManagerFactory::new(pool.clone(), resolver.clone(), 2); - worker_supervisor.spawn( - handle_factory, - stop.clone(), - consumer::worker_core::RestartPolicy::Always, // Critical worker - ); - - // Spawn stub resolution worker - // This finds stub records (status='stub') and enqueues them for fetching - // Respects retention_cutoff to only resolve stubs within the configured time range - let stub_factory = workers::StubResolutionWorkerFactory::new(pool.clone(), retention_cutoff); - worker_supervisor.spawn( - stub_factory, - stop.clone(), - consumer::worker_core::RestartPolicy::Backoff { max_retries: 3 }, - ); - - // Note: Worker supervisor manages its own tasks via TaskTracker + // Note: Handle resolution is now managed by Tap + // Worker supervisor manages its own tasks via TaskTracker // We don't need to explicitly track it in the main tracker Some(( - jetstream_tx, - fetch_tx, - backfill_tx, - cursor_arc, + tap_tx, batch_events_processed, batch_operations_processed, - fetch_queue_enqueued, worker_supervisor, )) } else { None }; - if cli.backfill { - let bf_cfg = conf - .backfill - .ok_or_eyre("Config item [backfill] must be specified when using --backfill")?; - - let ( - _jetstream_tx, - fetch_tx, - backfill_tx, - _cursor_arc, - _, - _, - _, - worker_supervisor, - ) = shared_database_writer.as_ref().unwrap(); - - // Start the record fetch worker manager (required for backfill) - let fetch_cfg = conf.record_fetch.clone().unwrap_or_default(); - let fetch_config = workers::fetch::types::RecordFetchConfig { - slingshot_url: fetch_cfg.slingshot_url.clone(), - bluesky_api_url: fetch_cfg.bluesky_api_url.clone(), - timeout_secs: fetch_cfg.timeout_secs, - }; - let record_fetcher = workers::fetch::RecordFetcher::new(&fetch_config, resolver.clone())?; - - let fetch_factory = workers::RecordFetchManagerFactory::new( - pool.clone(), - record_fetcher.clone(), - fetch_cfg, - fetch_tx.clone(), - ); - - // Run fetch manager with supervision - worker_supervisor.spawn( - fetch_factory, - stop.clone(), - consumer::worker_core::RestartPolicy::Always, // Critical worker - ); - - let backfill_factory = workers::BackfillManagerFactory::new( - pool.clone(), - resolver.clone(), - bf_cfg, - backfill_tx.clone(), - retention_cutoff, - ); - - worker_supervisor.spawn( - backfill_factory, - stop.clone(), - consumer::worker_core::RestartPolicy::Always, // Critical worker - ); - } - if cli.indexer { - let mut indexer_cfg = conf + let indexer_cfg = conf .indexer .ok_or_eyre("Config item [indexer] must be specified when using --indexer")?; - // Populate wantedDids from allowlist for server-side filtering - // Only override if not already set via config - if indexer_cfg.jetstream_wanted_dids.is_empty() { - let conn = pool.get().await?; - match db::allowlist::get_all(&conn).await { - Ok(allowlist_dids) => { - tracing::info!( - "Populating Jetstream wantedDids from allowlist ({} DIDs)", - allowlist_dids.len() - ); - indexer_cfg.jetstream_wanted_dids = allowlist_dids; - } - Err(e) => { - tracing::warn!("Failed to load allowlist for Jetstream filtering: {}", e); - } - } - } - // Get the shared infrastructure from the database writer setup let ( - jetstream_tx, - fetch_tx, - _backfill_tx, - cursor_arc, + tap_tx, batch_events_processed, batch_operations_processed, - fetch_queue_enqueued, worker_supervisor, ) = shared_database_writer.as_ref().unwrap(); - // UnifiedConsumer will be created by RelayIndexerFactory on each spawn/restart - // The factory ensures cursor_arc is shared for cursor tracking - - // Start the record fetch worker manager - let fetch_cfg = conf.record_fetch.clone().unwrap_or_default(); - let fetch_config = workers::fetch::types::RecordFetchConfig { - slingshot_url: fetch_cfg.slingshot_url.clone(), - bluesky_api_url: fetch_cfg.bluesky_api_url.clone(), - timeout_secs: fetch_cfg.timeout_secs, - }; - let record_fetcher = workers::fetch::RecordFetcher::new(&fetch_config, resolver.clone())?; - - let fetch_factory = workers::RecordFetchManagerFactory::new( - pool.clone(), - record_fetcher.clone(), - fetch_cfg, - fetch_tx.clone(), - ); - // Run fetch manager with supervision - worker_supervisor.spawn( - fetch_factory, - stop.clone(), - consumer::worker_core::RestartPolicy::Always, // Critical worker - ); - - // Ensure all allowlisted DIDs are queued for backfill - let conn = pool.get().await?; - match db::allowlist::ensure_allowlist_actors(&conn, &pool).await { - Ok((actors_found, profiles_enqueued)) => { - if actors_found > 0 { - tracing::info!( - "Found {} allowlisted actors needing backfill, enqueued {} profiles for fetching", - actors_found, - profiles_enqueued - ); - } - } - Err(e) => { - tracing::error!("Failed to ensure allowlist actors: {}", e); - } - } - - let indexer_opts = relay::RelayIndexerOpts { - skip_handle_validation: indexer_cfg.skip_handle_validation, - config: Some(indexer_cfg.clone()), + // Create Tap configuration + let tap_config = consumer::streaming::sources::tap::consumer::TapConfig { + websocket_url: indexer_cfg.tap_websocket_url.clone(), + admin_url: indexer_cfg.tap_admin_url.clone(), + admin_password: indexer_cfg.tap_admin_password.clone(), + max_pending_acks: indexer_cfg.max_pending_acks, + reconnect_backoff_ms: 1000, + reconnect_max_backoff_ms: 60000, }; - // Create relay indexer factory and spawn with supervision - let relay_indexer_factory = relay::RelayIndexerFactory::new( + // Create Tap indexer factory and spawn with supervision + let tap_indexer_factory = relay::TapIndexerFactory::new( pool.clone(), - indexer_opts, + tap_config, indexer_cfg.workers, - user_agent.clone(), - cursor_arc.clone(), - jetstream_tx.clone(), + tap_tx.clone(), batch_events_processed.clone(), batch_operations_processed.clone(), - fetch_queue_enqueued.clone(), ); worker_supervisor.spawn( - relay_indexer_factory, + tap_indexer_factory, stop.clone(), consumer::worker_core::RestartPolicy::Always, // Critical worker - always restart ); @@ -361,20 +170,14 @@ async fn main() -> eyre::Result<()> { // Tasks will drop their clones as they exit, and once all are dropped, // the database writer will drain and exit if let Some(( - jetstream_tx, - fetch_tx, - backfill_tx, - _cursor_arc, - _, - _, - _, + tap_tx, + _batch_events_processed, + _batch_operations_processed, _worker_supervisor, )) = shared_database_writer { tracing::info!("Closing database writer channels..."); - drop(jetstream_tx); - drop(fetch_tx); - drop(backfill_tx); + drop(tap_tx); } // Step 3: Close the tracker to prevent new tasks diff --git a/consumer/src/parsing/car.rs b/consumer/src/parsing/car.rs deleted file mode 100644 index 750c1bb4..00000000 --- a/consumer/src/parsing/car.rs +++ /dev/null @@ -1,41 +0,0 @@ -//! CAR (Content Addressable aRchive) parsing utilities -//! -//! This module provides shared utilities for parsing CAR data structures, -//! which are used to transport IPLD blocks (like AT Protocol commits). - -use futures::StreamExt; -use ipld_core::cid::Cid; -use std::collections::HashMap; - -/// Parse CAR blocks into a HashMap of CID -> bytes -/// -/// Takes raw CAR data and returns a mapping from Content Identifiers (CIDs) -/// to the corresponding block data. This is used for decoding AT Protocol -/// commit blocks from both Jetstream events and backfill CAR files. -/// -/// # Arguments -/// -/// * `car_data` - Raw CAR format data as bytes -/// -/// # Returns -/// -/// * `Ok(HashMap>)` - Mapping from CID to block data -/// * `Err(Box)` - If CAR parsing fails -/// -/// # Examples -/// -/// ```ignore -/// let car_data = &commit.blocks; -/// let blocks = parse_car_blocks(car_data).await?; -/// let block = blocks.get(&some_cid); -/// ``` -pub async fn parse_car_blocks( - car_data: &[u8], -) -> Result>, Box> { - let car_reader = iroh_car::CarReader::new(car_data).await?; - Ok(car_reader - .stream() - .filter_map(async |car| car.ok()) - .collect::>() - .await) -} diff --git a/consumer/src/parsing/mod.rs b/consumer/src/parsing/mod.rs deleted file mode 100644 index f5853d4b..00000000 --- a/consumer/src/parsing/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -//! Shared parsing utilities for the consumer crate -//! -//! This module contains parsing utilities that are used by multiple components, -//! such as Jetstream workers and Backfill workers. - -pub mod car; - -pub use car::parse_car_blocks; diff --git a/consumer/src/relay/indexer.rs b/consumer/src/relay/indexer.rs deleted file mode 100644 index 37fec84c..00000000 --- a/consumer/src/relay/indexer.rs +++ /dev/null @@ -1,765 +0,0 @@ -//! Relay indexer - coordinates Jetstream event consumption and worker pool -//! -//! The RelayIndexer manages: -//! - Single Jetstream consumer connection with logical partition tracking -//! - Worker pool coordination -//! - Progress tracking and logging -//! - Reconnection handling - -use crate::db; -use crate::worker_core::{Worker, WorkerFactory}; -use deadpool_postgres::Pool; -use eyre::Result; - -use metrics::counter; - -use tokio::sync::watch::Receiver as WatchReceiver; - -/// Configuration for creating a RelayIndexer -pub struct RelayIndexerConfig { - pub pool: Pool, - pub consumer: crate::sources::unified_consumer::UnifiedConsumer, - pub opts: RelayIndexerOpts, - pub threads: u8, - pub batch_writer_tx: tokio::sync::mpsc::Sender, - pub batch_events_processed: std::sync::Arc, - pub batch_operations_processed: std::sync::Arc, - pub fetch_queue_enqueued: std::sync::Arc, -} - -impl RelayIndexerConfig { - /// Create a new config with required parameters and default counters - pub fn new( - pool: Pool, - consumer: crate::sources::unified_consumer::UnifiedConsumer, - opts: RelayIndexerOpts, - threads: u8, - batch_writer_tx: tokio::sync::mpsc::Sender, - ) -> Self { - Self { - pool, - consumer, - opts, - threads, - batch_writer_tx, - batch_events_processed: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - batch_operations_processed: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - fetch_queue_enqueued: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - } - } - - /// Set custom batch events counter (for sharing with other components) - pub fn with_batch_events_processed(mut self, counter: std::sync::Arc) -> Self { - self.batch_events_processed = counter; - self - } - - /// Set custom batch operations counter (for sharing with other components) - pub fn with_batch_operations_processed(mut self, counter: std::sync::Arc) -> Self { - self.batch_operations_processed = counter; - self - } - - /// Set custom fetch queue counter (for sharing with other components) - pub fn with_fetch_queue_enqueued(mut self, counter: std::sync::Arc) -> Self { - self.fetch_queue_enqueued = counter; - self - } -} - -#[derive(Clone)] -pub(crate) struct RelayIndexerState { - /// Stores the last sequence number and timestamp for rate calculation (seq, timestamp) - pub(crate) last_seq_time: Option<(u64, chrono::DateTime)>, -} - -#[derive(Clone)] -pub struct RelayIndexerOpts { - pub skip_handle_validation: bool, - pub config: Option, -} - -pub struct RelayIndexer { - pool: Pool, - state: RelayIndexerState, - consumer: crate::sources::unified_consumer::UnifiedConsumer, - allowlist: db::Allowlist, - opts: RelayIndexerOpts, - threads: u8, // Number of worker threads to spawn - last_fetch_queue_size: u64, - event_count: std::sync::Arc, - worker_events_processed: std::sync::Arc, - worker_events_dropped: std::sync::Arc, - batch_events_processed: std::sync::Arc, - batch_operations_processed: std::sync::Arc, - fetch_queue_enqueued: std::sync::Arc, - // Long-term rate tracking for better ETA calculation - // Tracks cursor time progression: (wall_time, cursor_us, event_count_at_cursor) - rate_samples: Vec<(chrono::DateTime, u64, u64)>, - // Database writer channel (shared with backfill and fetch workers) - bounded for backpressure - batch_writer_tx: tokio::sync::mpsc::Sender, - // Channel to receive allowlist update notifications - allowlist_update_rx: tokio::sync::mpsc::UnboundedReceiver<()>, -} - -impl RelayIndexer { - pub async fn new(config: RelayIndexerConfig) -> eyre::Result { - let pool = config.pool; - let consumer = config.consumer; - let opts = config.opts; - let threads = config.threads; - let batch_writer_tx = config.batch_writer_tx; - let batch_events_processed = config.batch_events_processed; - let batch_operations_processed = config.batch_operations_processed; - let fetch_queue_enqueued = config.fetch_queue_enqueued; - - // Create and initialize the allowlist - let allowlist = db::Allowlist::new(); - { - let conn = pool.get().await?; - if let Err(e) = allowlist.initialize(&conn).await { - tracing::warn!("Failed to initialize allowlist cache: {}", e); - } - } - - // Spawn allowlist periodic refresh task - let (_refresh_handle, mut allowlist_changed_rx) = allowlist.spawn_periodic_refresh(pool.clone()).await; - tracing::info!("Allowlist cache periodic refresh started (60s interval)"); - - // Create a channel to signal allowlist changes to the main loop - let (allowlist_update_tx, allowlist_update_rx) = tokio::sync::mpsc::unbounded_channel(); - - // Spawn task to watch for allowlist changes and signal through channel - tokio::spawn(async move { - loop { - if allowlist_changed_rx.changed().await.is_ok() && *allowlist_changed_rx.borrow() { - tracing::info!("Allowlist changed detected - will send options update to Jetstream"); - let _ = allowlist_update_tx.send(()); - } - } - }); - - let indexer_opts = opts.clone(); - - let indexer = Self { - pool: pool.clone(), - consumer, - state: RelayIndexerState { - last_seq_time: None, - }, - allowlist: allowlist.clone(), - opts: indexer_opts, - threads, - last_fetch_queue_size: 0, - event_count: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - worker_events_processed: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - worker_events_dropped: std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)), - batch_events_processed, - batch_operations_processed, - fetch_queue_enqueued, - rate_samples: Vec::new(), - batch_writer_tx, - allowlist_update_rx, - }; - - if opts.skip_handle_validation { - tracing::warn!("Indexer running with handle validation DISABLED!"); - } - - Ok(indexer) - } - - pub async fn run(mut self, stop: WatchReceiver) -> eyre::Result<()> { - // Use the shared database writer (already spawned in main.rs) - // No need to spawn a new one here - - // Spawn worker threads with the shared database writer channel - let (submit, handles) = crate::workers::jetstream::spawn_workers( - self.threads, - &self.allowlist, - Some(self.batch_writer_tx.clone()), - self.worker_events_processed.clone(), - self.worker_events_dropped.clone(), - ); - - let mut join_set = tokio::task::JoinSet::from_iter(handles); - - let threads = u64::from(self.threads); - // timer to log the current seq every 10s. - let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(10)); - - 'outer: loop { - tokio::select! { - _ = timer.tick() => { - self.log_sequence_status().await; - - if stop.has_changed().unwrap_or(true) { - break; - } - }, - Some(()) = self.allowlist_update_rx.recv() => { - // Allowlist changed - send options update to Jetstream - if let Err(e) = self.send_allowlist_update().await { - tracing::error!("Failed to send allowlist update to Jetstream: {}", e); - } - }, - out = self.consumer.drive_raw() => { - match out { - Ok((partition, raw_message)) => { - self.event_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - // Route the message to workers (with partition info) - crate::indexer::routing::route_raw_jetstream_message( - partition, - raw_message, - threads, - &submit, - ).await; - }, - Err(err) => { - tracing::error!("Consumer drive error: {err:?}"); - - if Self::should_reconnect(&err) - && self.try_reconnect().await { - continue 'outer; - } - - // Check if error is non-fatal - if Self::is_nonfatal_error(&err).await { - continue 'outer; - } - - // If we couldn't reconnect or it wasn't a reconnectable error, exit - break; - } - } - } - } - } - - tracing::info!("Consumer closed - draining then exiting"); - - // When we get here, explicitly drop the senders - // The database writer is shared and will be closed by main.rs - drop(submit); - - // Wait for all workers to complete - loop { - let join_result = join_set.join_next().await; - match join_result { - Some(res) => match res { - Ok(Ok(())) => {} - Ok(Err(err)) | Err(err) => return Err(err.into()), - }, - None => break, - } - } - - tracing::info!("All workers completed"); - tracing::info!("indexer exiting."); - - Ok(()) - } - - /// Log the current sequence status with timing information - async fn log_sequence_status(&mut self) { - // Read and reset counters for this interval - let jetstream_received = self - .event_count - .swap(0, std::sync::atomic::Ordering::Relaxed); - let worker_processed = self - .worker_events_processed - .swap(0, std::sync::atomic::Ordering::Relaxed); - let batch_events = self - .batch_events_processed - .swap(0, std::sync::atomic::Ordering::Relaxed); - let batch_ops = self - .batch_operations_processed - .swap(0, std::sync::atomic::Ordering::Relaxed); - let fetch_enqueued = self - .fetch_queue_enqueued - .swap(0, std::sync::atomic::Ordering::Relaxed); - - // Get cursor position - let min_cursor = self.consumer.current_seq(); - - // Note: Cursors are now saved by the database writer every 10 seconds to PostgreSQL - - // Track long-term rate samples for better ETA calculation - // Store (wall_time, cursor_time_us, cumulative_event_count) to track cursor time progression - let now = chrono::Utc::now(); - - // Get cumulative event count for rate calculation - static CUMULATIVE_EVENTS: std::sync::atomic::AtomicU64 = - std::sync::atomic::AtomicU64::new(0); - let cumulative = CUMULATIVE_EVENTS - .fetch_add(jetstream_received, std::sync::atomic::Ordering::Relaxed) - + jetstream_received; - - self.rate_samples.push((now, min_cursor, cumulative)); - - // Keep samples spanning at least 2 hours of *cursor time* (not wall time) - // This means we keep samples until we have at least 2 hours of cursor progression - if let Some(&(_, oldest_cursor, _)) = self.rate_samples.first() { - let cursor_time_diff_us = min_cursor.saturating_sub(oldest_cursor); - let cursor_time_diff_hours = cursor_time_diff_us as f64 / 1_000_000.0 / 3600.0; - - // Once we have 2+ hours of cursor time, start removing old samples - if cursor_time_diff_hours >= 2.0 { - // Keep samples from the last 2 hours of cursor time - let two_cursor_hours_ago_us = min_cursor.saturating_sub(2 * 3600 * 1_000_000); - self.rate_samples - .retain(|(_, cursor, _)| *cursor >= two_cursor_hours_ago_us); - } - } - - counter!("consumer_seq").absolute(min_cursor); - - // Create status message - let log_message = { - let now = chrono::Utc::now(); - - // Convert cursor to event time - let event_time = - chrono::DateTime::::from_timestamp_micros(min_cursor as i64) - .unwrap_or(now); - - // Convert event time to ET (America/New_York timezone) - let et_tz = chrono_tz::America::New_York; - let event_time_et = event_time.with_timezone(&et_tz); - - // Calculate how far behind we are - let behind_duration = now.signed_duration_since(event_time); - - // Format lag as XdXhXmXs - let total_seconds = behind_duration.num_seconds(); - let days = total_seconds / 86400; - let hours = (total_seconds % 86400) / 3600; - let minutes = (total_seconds % 3600) / 60; - let seconds = total_seconds % 60; - - let (status_prefix, behind_formatted) = if total_seconds <= 10 { - ("", format!("{}s", seconds)) - } else if days > 0 { - ( - "Behind", - format!("{}d{}h{}m{}s", days, hours, minutes, seconds), - ) - } else if hours > 0 { - ("Behind", format!("{}h{}m{}s", hours, minutes, seconds)) - } else if minutes > 0 { - ("Behind", format!("{}m{}s", minutes, seconds)) - } else { - ("Behind", format!("{}s", seconds)) - }; - - // Calculate processing rate (how fast we're catching up) and ETA - // Use long-term averaging over cursor time for better estimates - let (rate_info, eta_info) = if self.rate_samples.len() >= 2 { - // Get oldest and newest samples for long-term rate calculation - let (oldest_wall_time, oldest_cursor, oldest_events) = self.rate_samples[0]; - let (newest_wall_time, newest_cursor, newest_events) = - *self.rate_samples.last().unwrap(); - - let wall_time_diff = - (newest_wall_time - oldest_wall_time).num_milliseconds() as f64 / 1000.0; - let cursor_time_diff_us = newest_cursor.saturating_sub(oldest_cursor) as f64; - let cursor_time_diff_sec = cursor_time_diff_us / 1_000_000.0; - let events_processed = newest_events.saturating_sub(oldest_events) as f64; - - if wall_time_diff > 0.0 && cursor_time_diff_sec > 0.0 { - // Rate = cursor_seconds / wall_seconds - // rate > 1.0 means we're catching up - // rate < 1.0 means we're falling behind - let rate = cursor_time_diff_sec / wall_time_diff; - - // Calculate events per cursor-hour for context - let _events_per_cursor_hour = if cursor_time_diff_sec > 0.0 { - (events_processed / cursor_time_diff_sec) * 3600.0 - } else { - 0.0 - }; - - // Calculate ETA if we're behind and catching up (rate > 1.0) - let eta = if total_seconds > 10 && rate > 1.0 { - // How many seconds of backlog are we processing per wall-clock second? - let catchup_rate = rate - 1.0; // rate=2.0 means we catch up 1s per wall-second - let eta_seconds = (total_seconds as f64 / catchup_rate) as i64; - - // Format ETA - let eta_hours = eta_seconds / 3600; - let eta_minutes = (eta_seconds % 3600) / 60; - - if eta_hours > 0 { - format!(", ETA {}h{}m", eta_hours, eta_minutes) - } else if eta_minutes > 0 { - format!(", ETA {}m", eta_minutes) - } else { - ", ETA <1m".to_string() - } - } else if total_seconds > 10 && rate > 0.0 && rate < 1.0 { - // We're falling further behind - ", falling behind".to_string() - } else { - String::new() - }; - - (format!(", x{:.1}", rate), eta) - } else { - (String::new(), String::new()) - } - } else { - // Fall back to short-term rate if we don't have enough samples yet - if let Some((prev_seq, prev_time)) = self.state.last_seq_time { - let wall_time_diff = (now - prev_time).num_milliseconds() as f64 / 1000.0; - if wall_time_diff > 0.0 { - // Calculate cursor time difference in seconds - let cursor_time_diff_us = min_cursor.saturating_sub(prev_seq) as f64; - let cursor_time_diff_sec = cursor_time_diff_us / 1_000_000.0; - - let rate = if cursor_time_diff_sec > 0.0 { - cursor_time_diff_sec / wall_time_diff - } else { - 0.0 - }; - - (format!(", x{:.1}", rate), String::new()) - } else { - (String::new(), String::new()) - } - } else { - (String::new(), String::new()) - } - }; - - // Query for queue sizes - // Backfill queue size tracking not implemented - let backfill_queue_size: u64 = 0; - - // Fetch queue is in PostgreSQL, query it - let fetch_queue_size: i64 = if let Ok(conn) = self.pool.get().await { - conn.query_one( - "SELECT COUNT(*) FROM fetch_queue WHERE status = 'pending'", - &[], - ) - .await - .ok() - .and_then(|row| row.try_get::<_, i64>(0).ok()) - .unwrap_or(0) - } else { - 0 - }; - - // Update last fetch queue size and sequence time for next calculation - self.last_fetch_queue_size = fetch_queue_size as u64; - self.state.last_seq_time = Some((min_cursor, now)); - - // Helper to format numbers - use K/M with one decimal place for readability - fn format_num(n: u64) -> String { - if n >= 1_000_000 { - let m = n as f64 / 1_000_000.0; - if m >= 10.0 { - format!("{:.0}M", m) - } else { - format!("{:.1}M", m) - } - } else if n >= 1_000 { - let k = n as f64 / 1_000.0; - if k >= 10.0 { - format!("{:.0}k", k) - } else { - format!("{:.1}k", k) - } - } else { - n.to_string() - } - } - - // Helper to format queue counts with underscores (smaller numbers that don't change as much) - fn format_queue(n: u64) -> String { - n.to_string() - .as_bytes() - .rchunks(3) - .rev() - .map(std::str::from_utf8) - .collect::, _>>() - .unwrap() - .join("_") - } - - let backfill_info = if backfill_queue_size > 0 { - format!(", {} repos in backfill", format_queue(backfill_queue_size)) - } else { - String::new() - }; - - // Show fetch queue size with K/M formatting for readability - let fetch_info = if fetch_queue_size > 0 { - format!(", {} fetches queued", format_num(fetch_queue_size as u64)) - } else { - String::new() - }; - - let fetch_enqueued_info = if fetch_enqueued > 0 { - let enqueued_per_sec = fetch_enqueued / 10; - format!(", {}/s fetch", format_num(enqueued_per_sec)) - } else { - String::new() - }; - - // Calculate per-second rates from the 10-second interval - let jetstream_per_sec = jetstream_received / 10; - let worker_per_sec = worker_processed / 10; - let batch_per_sec = batch_events / 10; - let ops_per_sec = batch_ops / 10; - - format!( - "{} {} ({}){}{}{}{} | JS:{}/s W:{}/s B:{}/s ({} ops/s){}", - status_prefix, - behind_formatted, - event_time_et.format("%-I:%M%p ET"), - rate_info, - eta_info, - backfill_info, - fetch_info, - format_num(jetstream_per_sec), - format_num(worker_per_sec), - format_num(batch_per_sec), - format_num(ops_per_sec), - fetch_enqueued_info - ) - }; - - tracing::info!("{}", log_message); - } - - /// Check if an error should trigger a reconnection attempt - fn should_reconnect(err: &eyre::Error) -> bool { - let error_str = err.to_string(); - let is_websocket_eof = - error_str.contains("WebSocket error") && error_str.contains("unexpected EOF"); - let is_websocket_reset = error_str.contains("WebSocket error") - && error_str.contains("Connection reset without closing handshake"); - let is_websocket_error = error_str.contains("WebSocket error"); - let is_parsing_error = error_str.contains("Failed to parse Jetstream event"); - - is_websocket_eof || is_websocket_reset || is_websocket_error || is_parsing_error - } - - /// Try to reconnect to Jetstream with exponential backoff - async fn try_reconnect(&mut self) -> bool { - if let Some(config) = &self.opts.config { - if let Some(jetstream_url) = &config.jetstream_source { - tracing::info!("Attempting to reconnect to Jetstream after error"); - - let ua = "Parakeet Indexer"; - - // Use exponential backoff for reconnection attempts - let mut backoff_ms = 1000; // Start with 1 second - let max_retries = 5; - - for attempt in 1..=max_retries { - let reconnect_result = self.consumer.reconnect(jetstream_url, ua).await; - match reconnect_result { - Ok(()) => { - tracing::info!( - "Successfully reconnected to Jetstream (attempt {}/{})", - attempt, - max_retries - ); - counter!("jetstream_reconnects_success").increment(1); - return true; - } - Err(e) => { - if attempt == max_retries { - tracing::error!( - "Failed to reconnect after {} attempts: {}", - max_retries, - e - ); - counter!("jetstream_reconnects_failed").increment(1); - return false; - } - - // Add jitter to backoff (±20%) - let jitter = - (backoff_ms as f64 * 0.2 * (rand::random::() - 0.5)) as u64; - let backoff_ms_u64: u64 = backoff_ms; - let sleep_ms = backoff_ms_u64.saturating_add(jitter); - - tracing::warn!( - "Reconnection attempt {} failed, retrying in {}ms: {}", - attempt, - sleep_ms, - e - ); - - tokio::time::sleep(std::time::Duration::from_millis(sleep_ms)).await; - backoff_ms = std::cmp::min(backoff_ms * 2, 30000); // Cap at 30 seconds - } - } - } - } - } - false - } - - /// Send an options update to Jetstream with the current allowlist - async fn send_allowlist_update(&mut self) -> eyre::Result<()> { - use crate::sources::jetstream::types::SubscriberOptions; - - tracing::info!("Sending allowlist update to Jetstream"); - - // Get all allowlisted DIDs from the database - let conn = self.pool.get().await?; - let dids = db::allowlist::get_all(&conn).await?; - - tracing::info!("Updating Jetstream with {} allowlisted DIDs", dids.len()); - - // Subscribe to all standard collections - let wanted_collections = Some(vec![ - "app.bsky.feed.post".to_string(), - "app.bsky.feed.like".to_string(), - "app.bsky.feed.repost".to_string(), - "app.bsky.graph.follow".to_string(), - "app.bsky.graph.block".to_string(), - "app.bsky.feed.threadgate".to_string(), - "app.bsky.feed.postgate".to_string(), - "app.bsky.graph.list".to_string(), - "app.bsky.graph.listitem".to_string(), - ]); - - // Create options with the updated allowlist - let options = SubscriberOptions { - wantedDids: Some(dids), - wantedCollections: wanted_collections, - maxMessageSizeBytes: None, - }; - - // Send the update - self.consumer.send_options_update(options).await?; - - tracing::info!("Successfully sent allowlist update to Jetstream"); - Ok(()) - } - - /// Check if an error is non-fatal and processing can continue - async fn is_nonfatal_error(err: &eyre::Error) -> bool { - let error_str = err.to_string(); - let is_parsing_error = error_str.contains("Failed to parse Jetstream event"); - let is_websocket_error = error_str.contains("WebSocket error"); - - if is_parsing_error { - tracing::warn!("Continuing after Jetstream parsing error"); - counter!("jetstream_events.parse_error_continued").increment(1); - return true; - } else if is_websocket_error { - // For websocket errors, we've already tried to reconnect but it failed - // Log this but continue trying rather than completely exiting - tracing::warn!("Continuing after failed reconnection attempt for WebSocket error"); - counter!("jetstream_events.websocket_error_continued").increment(1); - // Sleep briefly before retrying (non-blocking) - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - return true; - } - - false - } -} - -// Implement Worker trait - delegates to existing run() method -impl Worker for RelayIndexer { - fn name(&self) -> &'static str { - "relay_indexer" - } - - async fn run(self, stop: WatchReceiver) -> Result<()> { - RelayIndexer::run(self, stop).await - } -} - -/// Factory for creating RelayIndexer instances -#[derive(Clone)] -pub struct RelayIndexerFactory { - pool: Pool, - consumer_opts: RelayIndexerOpts, - threads: u8, - user_agent: String, - cursor_arc: std::sync::Arc>, - batch_writer_tx: tokio::sync::mpsc::Sender, - batch_events_processed: std::sync::Arc, - batch_operations_processed: std::sync::Arc, - fetch_queue_enqueued: std::sync::Arc, -} - -impl RelayIndexerFactory { - #[expect(clippy::too_many_arguments, reason = "Factory constructor mirrors RelayIndexer configuration")] - pub fn new( - pool: Pool, - consumer_opts: RelayIndexerOpts, - threads: u8, - user_agent: String, - cursor_arc: std::sync::Arc>, - batch_writer_tx: tokio::sync::mpsc::Sender, - batch_events_processed: std::sync::Arc, - batch_operations_processed: std::sync::Arc, - fetch_queue_enqueued: std::sync::Arc, - ) -> Self { - Self { - pool, - consumer_opts, - threads, - user_agent, - cursor_arc, - batch_writer_tx, - batch_events_processed, - batch_operations_processed, - fetch_queue_enqueued, - } - } -} - -impl WorkerFactory for RelayIndexerFactory { - type Worker = RelayIndexer; - - fn name(&self) -> &'static str { - "relay_indexer" - } - - async fn create(&self) -> Result { - // Create UnifiedConsumer for this instance with shared cursor_arc - // Get current cursor value from the shared Arc to pass to UnifiedConsumer - let cursor_to_pass = { - if let Ok(cursor) = self.cursor_arc.read() { - if *cursor == 0 { - None - } else { - Some(*cursor) - } - } else { - None - } - }; - - let consumer = crate::sources::unified_consumer::UnifiedConsumer::new( - self.consumer_opts.config.as_ref().ok_or_else(|| eyre::eyre!("IndexerConfig required"))?, - &self.user_agent, - cursor_to_pass, - Some(self.cursor_arc.clone()), - ) - .await?; - - let config = RelayIndexerConfig::new( - self.pool.clone(), - consumer, - self.consumer_opts.clone(), - self.threads, - self.batch_writer_tx.clone(), - ) - .with_batch_events_processed(self.batch_events_processed.clone()) - .with_batch_operations_processed(self.batch_operations_processed.clone()) - .with_fetch_queue_enqueued(self.fetch_queue_enqueued.clone()); - - RelayIndexer::new(config).await - } -} diff --git a/consumer/src/relay/mod.rs b/consumer/src/relay/mod.rs index 7d337b47..1c02d6b2 100644 --- a/consumer/src/relay/mod.rs +++ b/consumer/src/relay/mod.rs @@ -1,13 +1,13 @@ -//! High-level coordination for Jetstream event consumption +//! High-level coordination for event consumption //! //! The relay module coordinates: -//! - Jetstream connection and event consumption -//! - Worker pool management (via workers/jetstream) +//! - Tap WebSocket connection and event consumption +//! - Worker pool management (via workers/tap) //! - Progress tracking and status logging -//! - Reconnection handling +//! - Event acknowledgment -pub mod indexer; +pub mod tap_indexer; pub mod types; -pub use indexer::{RelayIndexer, RelayIndexerFactory, RelayIndexerOpts}; -// CollectionType and RecordTypes re-exports removed - use relay::types:: directly +pub use tap_indexer::{TapIndexer, TapIndexerFactory}; +// Legacy indexer removed - now using Tap-based indexer diff --git a/consumer/src/relay/tap_indexer.rs b/consumer/src/relay/tap_indexer.rs new file mode 100644 index 00000000..35a1c396 --- /dev/null +++ b/consumer/src/relay/tap_indexer.rs @@ -0,0 +1,251 @@ +//! Tap-based relay indexer - coordinates Tap event consumption and worker pool +//! +//! The TapIndexer manages: +//! - Single Tap WebSocket consumer connection +//! - Worker pool coordination +//! - Event acknowledgment after processing +//! - Progress tracking and logging +//! - Reconnection handling + +use crate::sources::tap::TapConsumer; +use crate::worker_core::{Worker, WorkerFactory}; +use deadpool_postgres::Pool; +use eyre::Result; +use metrics::counter; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use tokio::sync::watch::Receiver as WatchReceiver; + +/// Configuration for creating a TapIndexer +pub struct TapIndexerConfig { + pub pool: Pool, + pub tap_consumer: TapConsumer, + pub threads: u8, + pub batch_writer_tx: tokio::sync::mpsc::Sender, + pub batch_events_processed: Arc, + pub batch_operations_processed: Arc, +} + +pub struct TapIndexer { + _pool: Pool, + consumer: TapConsumer, + threads: u8, + event_count: Arc, + worker_events_processed: Arc, + worker_events_dropped: Arc, + batch_events_processed: Arc, + batch_operations_processed: Arc, + batch_writer_tx: tokio::sync::mpsc::Sender, + // Track events pending acknowledgment + pending_acks: std::collections::VecDeque, +} + +impl TapIndexer { + pub async fn new(config: TapIndexerConfig) -> Result { + let pool = config.pool; + let consumer = config.tap_consumer; + let threads = config.threads; + let batch_writer_tx = config.batch_writer_tx; + let batch_events_processed = config.batch_events_processed; + let batch_operations_processed = config.batch_operations_processed; + + let indexer = Self { + _pool: pool, + consumer, + threads, + event_count: Arc::new(AtomicU64::new(0)), + worker_events_processed: Arc::new(AtomicU64::new(0)), + worker_events_dropped: Arc::new(AtomicU64::new(0)), + batch_events_processed, + batch_operations_processed, + batch_writer_tx, + pending_acks: std::collections::VecDeque::new(), + }; + + Ok(indexer) + } + + pub async fn run(mut self, mut stop: WatchReceiver) -> Result<()> { + // Spawn worker threads with the shared database writer channel + let (submit, handles) = crate::workers::tap::spawn_workers( + self.threads, + self.batch_writer_tx.clone(), + self.worker_events_processed.clone(), + self.worker_events_dropped.clone(), + ); + + let mut join_set = tokio::task::JoinSet::from_iter(handles); + + // Timer to log status every 10s + let mut timer = tokio::time::interval(tokio::time::Duration::from_secs(10)); + + // Track worker pool size for round-robin routing + let threads = self.threads as usize; + let mut next_worker = 0; + + 'outer: loop { + tokio::select! { + _ = timer.tick() => { + self.log_status().await; + + if *stop.borrow() { + break; + } + }, + _ = stop.changed() => { + if *stop.borrow() { + break; + } + }, + event_result = self.consumer.next_event() => { + match event_result { + Ok(event) => { + self.event_count.fetch_add(1, Ordering::Relaxed); + + // Track event for acknowledgment + let event_id = event.id; + self.pending_acks.push_back(event_id); + + // Route to worker using round-robin + if let Err(e) = submit[next_worker].send(event).await { + tracing::error!("Failed to send event to worker: {}", e); + } + next_worker = (next_worker + 1) % threads; + + // Acknowledge events periodically (every 100 events) + if self.pending_acks.len() >= 100 { + self.acknowledge_pending().await; + } + } + Err(e) => { + tracing::error!("Error receiving Tap event: {}", e); + // The Tap consumer handles reconnection internally + // Just continue the loop + continue 'outer; + } + } + } + } + } + + tracing::info!("TapIndexer shutting down - acknowledging remaining events"); + + // Acknowledge any remaining pending events + self.acknowledge_pending().await; + + // Drop the senders to signal workers to stop + drop(submit); + + // Wait for all workers to complete + while join_set.join_next().await.is_some() { + // Workers shutting down + } + + tracing::info!("TapIndexer shutdown complete"); + Ok(()) + } + + /// Acknowledge all pending events + async fn acknowledge_pending(&mut self) { + while let Some(event_id) = self.pending_acks.pop_front() { + if let Err(e) = self.consumer.acknowledge(event_id).await { + tracing::error!("Failed to acknowledge event {}: {}", event_id, e); + // Re-add to front on failure + self.pending_acks.push_front(event_id); + break; + } + } + } + + /// Log current indexing status + async fn log_status(&self) { + let events = self.event_count.load(Ordering::Relaxed); + let worker_processed = self.worker_events_processed.load(Ordering::Relaxed); + let worker_dropped = self.worker_events_dropped.load(Ordering::Relaxed); + let batch_events = self.batch_events_processed.load(Ordering::Relaxed); + let batch_ops = self.batch_operations_processed.load(Ordering::Relaxed); + + tracing::info!( + "Tap status - Events: {}, Worker: {} processed/{} dropped, Batch: {} events/{} ops, Pending acks: {}", + events, + worker_processed, + worker_dropped, + batch_events, + batch_ops, + self.pending_acks.len() + ); + + // Update metrics + counter!("tap_events_total").absolute(events); + counter!("tap_worker_processed_total").absolute(worker_processed); + counter!("tap_worker_dropped_total").absolute(worker_dropped); + counter!("tap_batch_events_total").absolute(batch_events); + counter!("tap_batch_operations_total").absolute(batch_ops); + } +} + +/// Factory for creating TapIndexer instances (for worker supervision) +#[derive(Clone)] +pub struct TapIndexerFactory { + pool: Pool, + tap_config: crate::sources::tap::consumer::TapConfig, + threads: u8, + batch_writer_tx: tokio::sync::mpsc::Sender, + batch_events_processed: Arc, + batch_operations_processed: Arc, +} + +impl TapIndexerFactory { + pub fn new( + pool: Pool, + tap_config: crate::sources::tap::consumer::TapConfig, + threads: u8, + batch_writer_tx: tokio::sync::mpsc::Sender, + batch_events_processed: Arc, + batch_operations_processed: Arc, + ) -> Self { + Self { + pool, + tap_config, + threads, + batch_writer_tx, + batch_events_processed, + batch_operations_processed, + } + } +} + +impl WorkerFactory for TapIndexerFactory { + type Worker = TapIndexer; + + fn name(&self) -> &'static str { + "tap-indexer" + } + + async fn create(&self) -> Result { + // Connect to Tap + let tap_consumer = TapConsumer::connect(self.tap_config.clone()).await?; + + let config = TapIndexerConfig { + pool: self.pool.clone(), + tap_consumer, + threads: self.threads, + batch_writer_tx: self.batch_writer_tx.clone(), + batch_events_processed: self.batch_events_processed.clone(), + batch_operations_processed: self.batch_operations_processed.clone(), + }; + + let indexer = TapIndexer::new(config).await?; + Ok(indexer) + } +} + +impl Worker for TapIndexer { + fn name(&self) -> &'static str { + "tap-indexer" + } + + async fn run(self, stop: WatchReceiver) -> Result<()> { + self.run(stop).await + } +} \ No newline at end of file diff --git a/consumer/src/relay/types.rs b/consumer/src/relay/types.rs index b7fae8a7..8007eaeb 100644 --- a/consumer/src/relay/types.rs +++ b/consumer/src/relay/types.rs @@ -44,6 +44,8 @@ pub enum RecordTypes { CommunityLexiconBookmark(lexica::community_lexicon::bookmarks::Bookmark), #[serde(rename = "fm.team.alpa.actor.status")] FmTealAlpaActorStatus(records::FmTealAlpaActorStatus), + #[serde(untagged)] + Unknown(serde_json::Value), } #[derive(Debug, PartialOrd, PartialEq, Eq, Deserialize, Serialize)] @@ -70,31 +72,3 @@ pub enum CollectionType { FmTealAlpaActorStatus, Unsupported, } - -impl CollectionType { - pub(crate) fn from_str(input: &str) -> Self { - match input { - "app.bsky.actor.profile" => Self::BskyProfile, - "app.bsky.actor.status" => Self::BskyStatus, - "app.bsky.feed.generator" => Self::BskyFeedGen, - "app.bsky.feed.like" => Self::BskyFeedLike, - "app.bsky.feed.post" => Self::BskyFeedPost, - "app.bsky.feed.postgate" => Self::BskyFeedPostgate, - "app.bsky.feed.repost" => Self::BskyFeedRepost, - "app.bsky.feed.threadgate" => Self::BskyFeedThreadgate, - "app.bsky.graph.block" => Self::BskyBlock, - "app.bsky.graph.follow" => Self::BskyFollow, - "app.bsky.graph.list" => Self::BskyList, - "app.bsky.graph.listblock" => Self::BskyListBlock, - "app.bsky.graph.listitem" => Self::BskyListItem, - "app.bsky.graph.starterpack" => Self::BskyStarterPack, - "app.bsky.graph.verification" => Self::BskyVerification, - "app.bsky.labeler.service" => Self::BskyLabelerService, - "app.bsky.notification.declaration" => Self::BskyNotificationDeclaration, - "chat.bsky.actor.declaration" => Self::ChatActorDecl, - "community.lexicon.bookmarks.bookmark" => Self::CommunityLexiconBookmark, - "fm.team.alpa.actor.status" => Self::FmTealAlpaActorStatus, - _ => Self::Unsupported, - } - } -} diff --git a/consumer/src/sources/jetstream/consumer.rs b/consumer/src/sources/jetstream/consumer.rs deleted file mode 100644 index 5bf0cfcb..00000000 --- a/consumer/src/sources/jetstream/consumer.rs +++ /dev/null @@ -1,243 +0,0 @@ -use eyre::{eyre, Result}; -use futures::stream::SplitStream; -use futures::{SinkExt as _, StreamExt as _}; -use std::io::Read as _; -use std::time::Duration; -use tokio::net::TcpStream; -use tokio_tungstenite::tungstenite::client::IntoClientRequest as _; -use tokio_tungstenite::tungstenite::http::header::{CONTENT_ENCODING, USER_AGENT}; -use tokio_tungstenite::tungstenite::Message; -use tokio_tungstenite::{connect_async, MaybeTlsStream, WebSocketStream}; -use tracing::{debug, error, info, instrument, trace}; - -use super::types::{RawJetstreamMessage, SubscriberOptions, SubscriberSourcedMessage}; - -/// Handles connection to the Jetstream service and processes messages -#[derive(Debug)] -pub struct JetstreamConsumer { - seq: u64, - stream: SplitStream>>, - sink: futures::stream::SplitSink>, Message>, - options: Option, -} - -impl JetstreamConsumer { - /// Creates a new Jetstream consumer with optional starting cursor and options - #[instrument(skip(options))] - pub async fn new( - url: &str, - start_cursor: Option, - ua: &str, - options: Option, - use_compression: bool, - ) -> Result { - let cursor = start_cursor.unwrap_or(0); - - // Build URL with query parameters - let mut url_with_params = format!("{url}/subscribe?cursor={cursor}"); - - // Add compression parameter if enabled - if use_compression { - debug!("Requesting compression with parameter compress=true"); - url_with_params.push_str("&compress=true"); - } else { - debug!("Not requesting compression"); - } - - // Add requireHello if we have options to send after connection - if options.is_some() { - url_with_params.push_str("&requireHello=true"); - } - - debug!("Connecting to Jetstream: {}", url_with_params); - - // Create request - let mut request = url_with_params.into_client_request()?; - drop(request.headers_mut().insert(USER_AGENT, ua.parse()?)); - - if use_compression { - debug!("Adding Content-Encoding: zstd header"); - drop( - request - .headers_mut() - .insert(CONTENT_ENCODING, "zstd".parse()?), - ); - } - - // Connect to WebSocket - let (wss, response) = connect_async(request).await.map_err(|e| { - error!("Failed to connect to Jetstream: {:?}", e); - eyre!("WebSocket connection failed: {}", e) - })?; - - debug!("Connected to Jetstream: {:?}", response); - // Log the response headers to verify compression settings - for (name, value) in response.headers() { - debug!("Response header: {}: {:?}", name, value); - } - let (sink, stream) = wss.split(); - - let mut consumer = JetstreamConsumer { - seq: cursor, - stream, - sink, - options, - }; - - // If we have initial options, send them - if let Some(options) = &consumer.options { - consumer.send_options_update(options.clone()).await?; - } - - Ok(consumer) - } - - /// Returns the current cursor position (timestamp in microseconds) - pub const fn current_seq(&self) -> u64 { - self.seq - } - - /// Returns the current subscriber options - pub fn current_options(&self) -> Option { - self.options.clone() - } - - /// Updates the sequence number if the provided value is greater than the current one - pub const fn update_seq(&mut self, seq: u64) { - if seq > self.seq { - self.seq = seq; - } - } - - /// Update subscriber options after connection - /// - /// Sends an `options_update` message to the Jetstream server to dynamically - /// update filtering options (wantedDids, wantedCollections, maxMessageSizeBytes). - /// - /// This is useful for: - /// - Updating the allowlist without reconnecting - /// - Adjusting collection filters at runtime - /// - Changing message size limits dynamically - pub async fn send_options_update(&mut self, options: SubscriberOptions) -> Result<()> { - let message = SubscriberSourcedMessage { - message_type: "options_update".to_owned(), - payload: options.clone(), - }; - - let json = serde_json::to_string(&message)?; - self.sink.send(Message::Text(json)).await?; - - // Update our stored options - self.options = Some(options); - - debug!("Sent options update to Jetstream"); - Ok(()) - } - - /// Processes the next event from the Jetstream and returns a raw message for parallel processing - #[instrument(skip(self))] - pub async fn drive_raw(&mut self) -> Result { - let msg = match self.stream.next().await { - Some(Ok(msg)) => msg, - Some(Err(e)) => { - error!("WebSocket error: {}", e); - return Err(eyre!("WebSocket error: {}", e)); - } - None => { - error!("WebSocket stream closed unexpectedly"); - return Err(eyre!("WebSocket closed")); - } - }; - - match msg { - Message::Text(text) => { - trace!("Received text message of length {}", text.len()); - - // Send everything to workers - they'll handle all parsing - Ok(RawJetstreamMessage::Text { content: text }) - } - Message::Binary(data) => { - // Just check if it's zstd compressed - let is_compressed = data.len() >= 4 && data[0..4] == [0x28, 0xB5, 0x2F, 0xFD]; - - trace!( - "Received binary message of length {} (compressed: {})", - data.len(), - is_compressed - ); - - // Return the raw binary data for worker to process - Ok(RawJetstreamMessage::Binary { data }) - } - Message::Ping(_) | Message::Pong(_) | Message::Frame(_) => { - // Handle control frames as before - return Box::pin(self.drive_raw()).await; - } - Message::Close(frame) => { - info!("WebSocket closed: {:?}", frame); - return Ok(RawJetstreamMessage::Close); - } - } - } - - /// Decompress zstd data using the Jetstream dictionary - pub fn decompress_zstd(compressed: &[u8]) -> Result> { - // Load the zstd dictionary from the embedded byte array - // The dictionary is provided by Jetstream and necessary for decompression - static ZSTD_DICTIONARY: &[u8] = include_bytes!("zstd_dictionary"); - - // Create a zstd decoder with the dictionary - let mut decoder = match zstd::stream::read::Decoder::with_dictionary( - std::io::Cursor::new(compressed), - ZSTD_DICTIONARY, - ) { - Ok(decoder) => decoder, - Err(e) => return Err(eyre!("Failed to create zstd decoder: {}", e)), - }; - - // Decompress the data - let mut decompressed = Vec::new(); - let _ = decoder - .read_to_end(&mut decompressed) - .map_err(|e| eyre!("Failed to decompress data: {}", e))?; - - Ok(decompressed) - } - - /// Reconnect to the Jetstream server with the current cursor - #[instrument(skip(self, url, ua, options))] - pub async fn reconnect( - &mut self, - url: &str, - ua: &str, - options: Option, - use_compression: bool, - ) -> Result<()> { - debug!("Reconnecting to Jetstream with cursor {}", self.seq); - - // Add backoff before reconnecting - tokio::time::sleep(Duration::from_secs(1)).await; - - // Use the provided options or keep the current ones - let options_to_use = options.or_else(|| self.options.clone()); - - let new_consumer = Self::new( - url, - Some(self.seq), - ua, - options_to_use.clone(), - use_compression, - ) - .await?; - - // Replace our websocket components with the new ones - self.stream = new_consumer.stream; - self.sink = new_consumer.sink; - self.options = new_consumer.options; - - // Options are already sent during Self::new() when options are provided - // No need to send them again - - Ok(()) - } -} diff --git a/consumer/src/sources/jetstream/mod.rs b/consumer/src/sources/jetstream/mod.rs deleted file mode 100644 index 2fe13f25..00000000 --- a/consumer/src/sources/jetstream/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Similar to the firehose consumer, this consumes the jetstream. -//! -//! -//! -//! Hostname | Region -//! -------------------------------- | ------- -//! jetstream1.us-east.bsky.network | US-East -//! jetstream2.us-east.bsky.network | US-East -//! jetstream1.us-west.bsky.network | US-West -//! jetstream2.us-west.bsky.network | US-West - -mod consumer; -pub mod types; - -pub use consumer::JetstreamConsumer; -pub use types::{ - AccountEvent, CommitEvent, CommitOperation, IdentityEvent, JetstreamError, JetstreamEvent, - RawJetstreamMessage, SubscriberOptions, -}; diff --git a/consumer/src/sources/jetstream/types.rs b/consumer/src/sources/jetstream/types.rs deleted file mode 100644 index 10ab6ef0..00000000 --- a/consumer/src/sources/jetstream/types.rs +++ /dev/null @@ -1,233 +0,0 @@ -use serde::{Deserialize, Serialize}; - -// Jetstream event examples -// Jetstream events have 3 `kinds`s (so far): -// -// - `commit`: a Commit to a repo which involves either a create, update, or delete of a record -// - `identity`: an Identity update for a DID which indicates that you may want to purge an identity cache and revalidate the DID doc and handle -// - `account`: an Account event that indicates a change in account status i.e. from `active` to `deactivated`, or to `takendown` if the PDS has taken down the repo. -// -// Jetstream Commits have 3 `operations`: -// -// - `create`: Create a new record with the contents provided -// - `update`: Update an existing record and replace it with the contents provided -// - `delete`: Delete an existing record with the DID, Collection, and RKey provided -// -// Jetstream events end up looking something like: -// -// A like committed to a repo -// { -// "did": "did:plc:eygmaihciaxprqvxpfvl6flk", -// "time_us": 1725911162329308, -// "kind": "commit", -// "commit": { -// "rev": "3l3qo2vutsw2b", -// "operation": "create", -// "collection": "app.bsky.feed.like", -// "rkey": "3l3qo2vuowo2b", -// "record": { -// "$type": "app.bsky.feed.like", -// "createdAt": "2024-09-09T19:46:02.102Z", -// "subject": { -// "cid": "bafyreidc6sydkkbchcyg62v77wbhzvb2mvytlmsychqgwf2xojjtirmzj4", -// "uri": "at://did:plc:wa7b35aakoll7hugkrjtf3xf/app.bsky.feed.post/3l3pte3p2e325" -// } -// }, -// "cid": "bafyreidwaivazkwu67xztlmuobx35hs2lnfh3kolmgfmucldvhd3sgzcqi" -// } -// } -// A deleted follow record -// { -// "did": "did:plc:rfov6bpyztcnedeyyzgfq42k", -// "time_us": 1725516666833633, -// "kind": "commit", -// "commit": { -// "rev": "3l3f6nzl3cv2s", -// "operation": "delete", -// "collection": "app.bsky.graph.follow", -// "rkey": "3l3dn7tku762u" -// } -// } -// An identity update -// { -// "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", -// "time_us": 1725516665234703, -// "kind": "identity", -// "identity": { -// "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", -// "handle": "yohenrique.bsky.social", -// "seq": 1409752997, -// "time": "2024-09-05T06:11:04.870Z" -// } -// } -// An account becoming active -// { -// "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", -// "time_us": 1725516665333808, -// "kind": "account", -// "account": { -// "active": true, -// "did": "did:plc:ufbl4k27gp6kzas5glhz7fim", -// "seq": 1409753013, -// "time": "2024-09-05T06:11:04.870Z" -// } -// } - -/// Main Jetstream event structure that all events follow -#[derive(Debug, Clone, Deserialize)] -#[serde(tag = "kind")] -pub enum JetstreamEvent { - #[serde(rename = "commit", alias = "com")] - Commit(CommitEvent), - #[serde(rename = "identity")] - Identity(IdentityEvent), - #[serde(rename = "account")] - Account(AccountEvent), -} - -/// Base structure for all Jetstream events -#[derive(Debug, Clone, Deserialize)] -pub struct BaseEvent { - pub did: String, - pub time_us: u64, - #[serde(flatten)] - pub payload: T, -} - -/// Commit event with record creation, update, or deletion -#[derive(Debug, Clone, Deserialize)] -pub struct CommitEvent { - pub did: String, - pub time_us: u64, - pub commit: CommitPayload, -} - -/// Identity event with handle updates -#[derive(Debug, Clone, Deserialize)] -pub struct IdentityEvent { - // Required for JSON deserialization - pub did: String, - // Required for JSON deserialization - pub time_us: u64, - pub identity: IdentityPayload, -} - -/// Account event with status changes -#[derive(Debug, Clone, Deserialize)] -pub struct AccountEvent { - // Required for JSON deserialization - pub did: String, - // Required for JSON deserialization - pub time_us: u64, - pub account: AccountPayload, -} - -/// Payload of a commit event -#[derive(Debug, Clone, Deserialize)] -pub struct CommitPayload { - /// The revision hash of the commit - pub rev: String, - /// The operation type (create, update, delete) - #[serde(rename = "operation", alias = "type")] - pub op: CommitOperation, - /// Collection NSID (e.g. app.bsky.feed.post) - pub collection: String, - /// Record key - pub rkey: String, - /// Record content for create/update operations - #[serde(skip_serializing_if = "Option::is_none")] - pub record: Option, - /// CID of the record for create/update operations - #[serde(skip_serializing_if = "Option::is_none")] - pub cid: Option, - /// Previous record CID for update operations - #[serde(skip_serializing_if = "Option::is_none")] - pub prev: Option, -} - -/// Operation type for commits -#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum CommitOperation { - #[serde(alias = "c")] - Create, - #[serde(alias = "u")] - Update, - #[serde(alias = "d")] - Delete, -} - -/// Payload of an identity event -#[derive(Debug, Clone, Deserialize)] -pub struct IdentityPayload { - pub did: String, - /// Handle is optional - identity updates may not include a handle - #[serde(skip_serializing_if = "Option::is_none")] - pub handle: Option, - pub seq: u64, - pub time: String, -} - -/// Payload of an account event -#[derive(Debug, Clone, Deserialize)] -pub struct AccountPayload { - pub active: bool, - pub did: String, - pub seq: u64, - pub time: String, -} - -/// Error message that might be received from Jetstream -#[derive(Debug, Clone, Deserialize)] -pub struct JetstreamError { - pub error: String, - pub message: String, -} - -/// Options for filtering events from Jetstream -#[derive(Debug, Clone, Serialize)] -#[expect(non_snake_case, reason = "External Jetstream API requires camelCase field names")] -pub struct SubscriberOptions { - #[serde(skip_serializing_if = "Option::is_none")] - pub wantedCollections: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub wantedDids: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub maxMessageSizeBytes: Option, -} - -/// Options update message to send to Jetstream -#[derive(Debug, Clone, Serialize)] -pub struct SubscriberSourcedMessage { - #[serde(rename = "type")] - pub message_type: String, - pub payload: SubscriberOptions, -} - -/// Raw message types received from the Jetstream websocket -#[derive(Debug, Clone)] -pub enum RawJetstreamMessage { - /// Raw text message - Text { - /// The content of the text message - content: String, - }, - /// Raw binary message (potentially compressed) - Binary { - /// The raw binary data - data: Vec, - }, - /// Close message - Close, -} - -/// Output variants from the Jetstream consumer -#[derive(Debug)] -pub enum JetstreamOutput { - Commit(CommitEvent), - Identity(IdentityEvent), - Account(AccountEvent), - Error(JetstreamError), - /// Represents a parsing error - includes error message and a sample of the text that failed to parse - ParseError(String, String), -} diff --git a/consumer/src/sources/jetstream/zstd_dictionary b/consumer/src/sources/jetstream/zstd_dictionary deleted file mode 100644 index 106847e7007b4ac409fd58912fed85d69aedb352..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 112640 zcmXqCV(=!{O*BDG-~!hqzW*T5Jk#gUiDfQbJ}pxwiOl@Lf7aZ)EO^t-8w}jijK-`C z3=9m14kQ~GOgYX}%d)_sjh&^vm5G_9nW=j^D_hS54t5TnCWinKu@<(48xjmzN_bu} zSu!v%q?}e{Jji&MJ)=p2g^_{5Q9?mVpn;izfq{{Mfq{jAfq{d8fk7`VF*&tFIX5-0 zBr`uxNk=IqGeyY?L|PT(BwH1iR~eb7mZv2qrwtN|C5c5PN>)Y&MztW7If;4c#Y$GuN~w8Du{ug6sTCzk zR!WZ>`kYGh6nyf_GV{_OH}rFbXC#&=WF!_8q~;YXlxL*oDI_YCXXNLk7U$=tDr6+) zr0FIVrKYB&7AfSWrj`^dlxLP?C?qNr<>w~mm1HI>B$ejbD}<-!2DrDrBC?w{kC={2cCYDqx6sJ}wWELx=q^4!&WtODoR4U}>DU@WSDkNv5CTHhl z7MJMNaXFWkq$)U-=2M3WNJDvMGxGb%E&GEJ&V64O!>lZsN!jdF776El)aO>%Nglxl0aszFhL6fv0<1?eVcl?7Ef#@VH*iRFpq z#$^>56{$)(N+p@Osqv-7N>+yEMrIZk2Igj_7KRqaCgwUyC6xuKN>)nA`MF9uATl?z z1QdTosbxx5O2#=Rxn;&FMOn!vMj0S;po+mtb8=FX!70Q_DY2jc;>EPo)D%5X;40}T z6=kPZBAJnxSqe6zC^b31C4P(%r`G9F-tWzO0Ot3H?J%% zO--uGPs&U)Hcv@S%c)8$D>pI8F;UV{DlN)XvQkPcvC`L1$xH#K_o__e>=NV3UQqI^X~MMZmih161ol2nDn+PW=Z)4rDo-+#(7Ce#b$}cMJbv2<>tnP1r?PgW=1B(sihf}nW<(OmDs$W zl#*R(Txp(KXlhcKQdF6kXqH@7VpNia&HJVX=7tt#mZn5`zbrkcDAC9aWDdCr-KZ=* zy|fZ+2IUFe)XLOI&%g+8LeD5kFDx@EGd0f3GRa6XF*hnM$jr;hHLomAE6Xr9D$Fn` zEXgV_EzU|WDJsd$HA^%p0p)Zo3B97I$TZ6+KReGnwcMzxEV;b6qO{mJJqcGrFHOnF z%goL(D#J|Z$>wH>rr9QCM!ES(xke^t<&~wyImJ1p#VLjP$(81&8O9k&MwMpS1?f3? zrDf?AMLAhm6S`SNu}NxHWrmS)VPC#Hm zTr<-gqe_rDWX5!9rD;x8QVQ4%%46EZ%FtZTz`zn;k(OFemYMnr~K+lu~JK zQdVG`pIVZbTTx(MY*w0AnVFW8S7~HYQIVceU}BV!3@ZDv#dKw^NkxuHNlH#yWm#r= zWs*sONs@U{9wer%3{0$y%=L_ojqtf9xiGgpJ3l!mtF)>hH?=%5r?@mVxg@`=FsURf zs~|Z!!>BMVC#BRhuQI7LIV;nstSY?_djXqTUTI!sRG3m>T9#T_k!@UMR8*c;T2%&d z4XA1b*DCM=wzML(ptL9zTe?d(E;lbUs;VeW$E zWyTdrMtN0b*%f(}mBq&9=4Pgqr5Wa_n79(Aw8A94+z4DJ5h`KQixLYm^wRQka`MYj+fAjFrR5db`Cw!4 zL^hfMc$!WoRwl-J2F9p03aA|kZt7$e<(8D^ftpyRR>lT;1_pRbfDDs@q{Ni+#LA@X z^xUfClBB}a4AZhoqmtB|f+F+c+(MI-qQugyq_V=4f~>;a(p=-*Ozb5q zrU(NAJqrsHd}V#Qd4X|KQC@y-X-0*KsY#AWWnxycNt#i*SwUq%L2+?ea#ngtMqz4x zd1jhvL2-JPX$Fq6-l!-mCAloas4A}@C!@SPKQk*eC!?YQXJb0GINQ9cFxSW!Go~~0 zvkD3e(+di6vyDsBlk(Dwvr^O3%9C=-@{CF6w|D;*06z?4*L6{N!v<2M=Ed%*a5`z!0_8fR`dkrlv;brR61A zspSRPr5TxNDaA#_1^E>i{eh(Hg2c?Eg4Ck&;;Ouo{H$!FjN&TOiRv?8snDm@cy z2BA2BmbQ4}z!=mgGqW@V#R0CthorP*6VsHEybQC-^c=JDs)BMee)8edr^GwsS>=GlRG~=wyBJ=$6^1?LZ)SRr0(!{(%vr>}+6XR5@Jxr6_tn!M2 zyo~(R#O%C^!o;NVj5MP>Gpwb*si}#fk);7~Ln&2d#!1;mplBtSZqb@=$m3>4Riz~v z5M%HpT{Hvmq+U}iQ)4|tW7OInUUHXK7-yTMHg7*}RxBo~_$mggqsCYfQB8Y%fn zS!Mb8CMITP1^J2PsYWL0899Yn$vIhh8TpkJ#+gNlNg2t;nUxt8xmm@zrbVgXHatcv z%fvLLFfp+(BPTsSGdr)`+$1ljJU1f~dq2z45ImGcM9E!dT$quak!(~9Dt*XoWr55{ z%_#<(fhXvRY-O2Q8CvLBni$|KHBw45igMC3l1)-f3QbbXD~*$KOfpPOaw<)X)6DZy zbMsP+Q;jN3ij&ffOU$whs_W53;)?8Sqf&4_Gc7PpG|$h?F-^)#Og1V^%1+5PGs!AT&&VstE6z{OOEXT&Gb_Yp zb$)tLPI^&peo|hkc}0?0VNPyRs!3)g&MaO~TAoo_W>f&S8g&Rax3HipH`y$?EXT~W z&@|t;B+1CABGo7@Ik7k=CC$9b*rYPiD61;RC^OYG%dD^<(HLt}-aIwMthgXAwJ5)` zq#~ubyx26|v@kOTJj{nyHJO{48yFgy5H*NYW}I4PUQS^H&p5TrsH&7o4Lma|12a8y zBfOoq^i1QT(!7$AvdW6WjNHuPjQq+x)579(qly%heB+XmDl?PZ^wO%Nywb|7yt0zw z%=BbXS%@W(6_uu#ROV#l=BJjJC8lR47i8uZW)!EV;*9AW)BGf}B%@R;F<*Qf!i0T9907QdVLLnm{NmLY?lbEY2=0$;?VgF)J~y%uLV9Of@#oPfh_3VWH(7 zGh;IoLo-8DVk*eeipV^dtYC$BuY(!8h)+mJ_kNm7+*L20RJeo|?Ea#lfE zR&hpt0jS}em2X~@X_Q$|oNa7ml2uY#lA2SJnwwQn1j>ULxhFr>s3Il3vY;R1roPYy zJyQ~k%TtO|le5k9^E1=)a!gH2OA@n;3r$KgN|Ot6i_*$Ws`8A}it|&AO$w`WlGD?& zunu~rniiC185~}WGRrnCNK4HsEjGF#2OGeIZ2fn1;)u4rsZZu*%|qz1^H!Z>E+<5ZuBH? zY+z()Vs1=Kg;83OoKawwVq^-+I%Kv#L1tv6Ld>8%$s1dl8t56A;H@yy%uPzn3yl&@ za>_GO3oCB_+L$;rmK1x2MeW4gSeA}I&dl&VFaGEd2`%t|WEGfyqeN;R)4DoZyt zt4K9X%t*{kOv}ix$TKY|Pb$mKugoyZH!3$xEz8d=#M%hVt1>puFe)fW%&*KgEz3*J zHcKo^F-}hc#SmIdo0}P18W|Xy5;HzgW}H+}oNsDW05XSAOrv$vkZLlJF*zk^U}Nw^ zHktu=`gvwn2F7}3W~ieS@P1x-Nq%l>eqM@Ek*RTFYGz_`Mqy@om1&j|IP}rhKP0A< z=2s+^m6_z_B&U|-o0^oQml~&K72%A`ssiJbBqO6N%xs&HR#;h{TAWi-R#Z@0k(*(f zZJt_TmYCnpTpOZC;p|ZB~(;T4o zxu`O?AhoIpYdg=v#K6$d+?=?`EHln2Dl$qTCNgofNy?1Nj5A6hX5fiQWZih9(a2cO z&qLxYRhcxV*?T$tXR)GOHrD$TX`UsjQ?tBRL~I zu{^B`)W61(Z3`>&vx`#`D^rcqb4zp5GfmRXtBNX8DsaSfdbvqnib+-pwA+Z@a zHZe6eDabM|DL2k4GtSPeN-In+$OU--V`$d^6w?;QM72rEvof;sbB#dbRAjeF%Cj<( zOG?3JP#)99Rz^m8mX;Ry1_&}z%F@bms!|J+i%ZfHOH#^ms|rj~E0c|r%8N|Pa?GpJ zO)8R8%yW}7&60Aea*fh6^C2yIj5STg1tulQ<%LyA##LFBrs+A^rG=U48EGltmb=sY)#= z$;r>nt1!tar~-vImV{SSnx0mbmz$E2oLp6uQJzv#l%Hafl819#FuyP*zpx52ri9)^ z$}3OFEGe%r&8Nlhs!GdDHLG%+D{_j=&C&}p!Hrn7`ohA%#MIEp$dsrR04AAvnMR35-~>#l zzCbP{kShz5%)BJ?Ot2Z0Cpm{(AiW|W#y zRFPkrXUSZfQkgW_C(uiBWQ9 zO0sEsVnvBrftjgsKK4Na5=MYcEltb}jg3u+ng1&-Ps}VU0IgC%^pSART%c9#NRbUP zrl6$M2(%ypPh_JRfTv0^wlXx*Gc&_60$g5Zl0{%Ss7Zc$URi|^sAM6x=qxkIPtP>U z0GolwQ^@HbkB`hiZCgVNyko10g~e$m#wBJ+#)-xzd8wJHDOp7+=|<^Q#sx;EiKf}c z>7}_D#>Ewh#yN$VY3aE|=82#ziX{bCRvMY46qn~$nWSfBnHQ(0rIqER7iK}%PGMQC zRGOYrVGe2`Vzh0OGLy24OY<|$Gc)t^5=%;ovI=u6v-6D$6LX3RsMUFlHeuZ-~hDPS*#MDZqRVgNwWm!g`fGDX{|Yz3dpDuu%<6>>{6OY#*`a|`lQQi~E(5_J>`@{0;fQxy^w z(o^$FQsGiCl{twz3VEr;#i>B4G7=TiGIKJ(UP;c+OU^0HEY(xs zQgAFT&MYoTwBqu~FG|fzRPf9#D9SHMRLIL$2+B{+R>;g#2+9PF?IsmvrleQ;7iFer z<|XEY7Ud{eDP@$D6ck(O>ldY#fea}uP0iIy&&f~9*GtaN)lV%;%`3?-)-TFW&eqM$ z(=E!(*9T>;rl~n*rdj#PC23iv#aI^tWM||V7n>KA8E2Oy zrIZw=n;8|R6zAuIS^?-qtGT(Mk)eTwB{4&RWyYB$=~ao~c^|TyOU9WcNv4HhGbpb_ z&8!S8^h}Mwi->SlqKTEpRcS@#DT!Id85K#!+1VLM<&}v^rKXh`W%=nv6&dDLMyZu$ z$>lkw=DDUO>6PZV#uQSrD$^3P)68;BD+(%&i;QyiqPf`xi3QoEMc77< z60>v6GAfcva?-PkOv}nM^Nr0-GR$(aOEL;GQ!6skk_!s+(oCyLvT~~O(hJJ+3kym> z6WSP+XjMg8es*PHMpc=yQGr=nN=jy7UU^~`_Ca5BOG8sj;+Bh5WEGX9BpFp8noFR8 zq-3m1mC)Mj@CLteQfYp9QDS)^c+?xi=|xGE#TgZOrfKPgDXHnnB_?I2d1i%K;1z0U zy;3tXBXdI&L!wrzlvXBXWMvqGhjU9RGc(IUD{gd@it_U@H&&$-XD1h!m*ksdb6?=hC#>ld{8@~RG}cXD7`eVBwt4XRH^5JiVRRY zq%5^aq4@C1qRdo#dwY9@ki!c>1yylka$+8&jt1AUpbbwsCEx-TqzSb93ACTEI5!`o zyh$o5Pc6wSsH#XU&o$3XNzE=a%E(VC&QD9s%PlHSG)gJUFUl}AN;63*PfE*4HBZS+ z!I}z7vr7wd3X{r8jnk?!D=N)W%X10}l1#BirIER*p^>qv38E*7bGKe;g?VY3Sq^by zn7C4*NiN73vy>9BF?fm`Gy`zex`tK;##ScgdWIILBbo34>8ixk%<_sVQ?r~xGt=Cv ziq!0s(hRfWWX#Y{PR&j&HA_iPEy*j&uBu2WFDfmmFf-3AC`(N=POD0-Fv`g*F{`LD zD>o@AGO9FBHNiT(lUP|^Qc_T!S6otBT9%S&npu&WpIlIkb;+oqF{smTZbVGzmlx+I zmZln|Bgz2Kh!JUJfJts?d2z0Br3u&=DuuqWv7V_BxXlAE^WmYNUSOVIY*ta8SZZ#P zl%J|poLQBsWMye)X>M)-%4wM;IiN9t;KK`(5>xUON>YmsF9dA_R7lM$DN0qyD=hz1SzL3p~w`8m2^7wAGxrM7MJGe8tNIE8k-uMLl%-_Z`1fI_~jSnrxur_>VUdEsl^4Upe?%aEK{tdI+8{)3vR#X1TFMTa+6WP;lL`K1bp`3gm; zDfvYTDfz_;#s-IWIb)>fl$6ZWr0nFZ%oOv?yyX0nx)Qht7AD)vFFtRmyW^2CzDBIC-+^s<7?%*+gvoT40Ttrs%`V4D$vmN(*u`Gcq$u)AN(cb5b&^@^ULO%}q@*L8DoaB#N^4H#s>a&nUM% zrL3qdy{f`Iqqv~TxUAHy0$1zBtT4y4GOx_21as~yrJyRiDy^)nyr9H5rM$?vAhSFz zB_%)AEZMB8ILFv1BR#LQEUzFdx!fqZBD1O_IXMsODh<=rg5pARlaxFoqqH)!oWwli zN|TbrbZmn?<|Zb_h9(x4#LS4485NiqnU{c?0F|H)Q9)u+Y940opOKrDmQ_|#m6wu} zSy7czURaotX_}N;S(cKWn4FrPoS#!^Y*tX7o^GC(S!`Thl4+U_8dSg%v}R_>=4F|Q zX_aY7na1Xod4(Cdg(Zb0<~V{j+bG>At+=4V2wW#%Z<*w!W|o)c8Yd-{XQw0;m{t{* z7bX^#msFKimYJC6q*Rs`mYWslRuq}#n5UZ?C1vFr;~1qjDoV>t&&W(pD=#U|GBr=k zO3MSSmBO`%F{?P=tje^i2)xD>(K0Cn_3%@Za}tZ7!zn3=3I#>^1(}Ho0d5M#`AJ2o zR$Sn5q{M;({eq(Ww9K4T{gTS!3_U$PjFKX$z{ohaup~Xryf7stts*b0sv<4DusAoT zs46kNB*Q$@G`G|^JKZcRKc}G37&JkYjkV@WNl!}8s>;qSDNC=)FG@}{O{*|Ut1t(J z1bRQm)Y8)2(9q0?xRN3*)hNdtI)nr{Qa}lGpa3Zg^~=*zN(&0WW>8*I7=xGGLi(`q zIuC7?Pr7kRc3ELYqG@_oN>WC8hFPVNk$Gi#N=ja8L271}aambzu2EJ&K}uO=Syf?K zQfWmBs7S?C`D>oNm;&WQDu2)zL7~z zSx!k(W`QD$0IO1g=$QBiVcQI<)0VyUTVa!FcB9*#1!qN>8&xGX8JxWc3& zu{0??(=-IMQ%z-VtG}nQD#X=W_DIuSwW?7No7S+ zZenJ6W>reQsZphAdR|#(QF@taMm}hU4a-RxMdbx0Ny#NeNmUspMJ1*w6=_vwCOJuk zpjI1tH_Fh+%+So3sPzM-{SxF_C zm3c`fDV3?_DHX|;B}K*Nd1hq=MI|M9MLEeP>8Yi;#(6nOm64}lVyl@jiDKDcEAnk7Kp8U`}MG$R>o2BDaS>c-Q*GP5!^*0Zz(FYwBN z9j$}1HSEdmu4mJizTY$X+03TLjjLa5-Tt_-_i^L0$Bok;H|~4fxcG77{Kt*mkDH+9 zn3N_aW|ifES_Bx=8ObSS$t7j!Da9sLd1*POC1uG)1!g6QMR}#=X~q>fC8ZUaCPnE< z>1oDQrFjKKIeEp{V<*4J*gVZR+oU9;GS55(w7;psq&zJX+enPLp^2%XiKRJl6IaD0 z$)=D^cVy?os^a3xv<&d(KSHrXxYcTEWoW8rXl#yebw^5Cc4lfpRgPIkVNzjsa%ECl zYFb5#X@zM;VR1>NNqJdmR#sVYd46RX*19UdvjB~F`bm1o@H8IUYTf= zQe0+gYFd$1W?q$ATwGpWVU%iCmYQXpTA5pxo}QRhnV(Zqnw^tdf}?ItO*YOh&deyz zDmO|?&nPmkDk~_-NG`(JKPjz9H#bkoFsi_e>7>NM!ko%t(=-zk6FnmXa6y8z*_4`N zmSmi4o^EPdnpBWknO2^iQ<$G;YF1X9ZE94STvl38Rh(00mXlamo|9yhmy=t7quErR zTVS47nqHQkkyM(Qo}ZPOo>fwinT2zBxU@8>C@aGR+cNmn{KT~Ml>EwyETe+Na`W=! zLSwVk!kmh-#DWy_g3?SQv*gOOGE=jnvXo-uWYdg_#3HPV3rq4+a!o7qO;e1_E6t6~ zOAF1*@{E#;KzR*4shU|>T7cS3#4Ii>tujtd%%HFA6|OC1r^w6-L>ZNi{t$Gpne| zsMIJoEu*A7)7&I2Eu+%BJj*yKqr%8Iv%t7Azbq>;F}J9!%&e%e(kM5t9P5;iscE5! zSxHf*SxQDmR#{bYdWm_qafUH?EiYQ*z|6qV(9qb@h^Qgh^7MiN(Y5mu*{6_uwLm1JiYW>@6rWt3#4<|O5rSEhj)A?S(J+}PC6(9G0~sCJD} zdbTO@QEMik)j}4Qc(+O7np;##bQY#CMIQtS(%w>C7D&mMxbUvc}i(nNixo^LTRy)NoINu zc&-F}9iVxVX+e34QDtU9MV4_^PF`MWRc?Atg_%iqj&WjTQ7UNDLwcflPGM0=S$>Ln zzEKI*wKX}WX=%oVd8TP)X=R0_C7DH~rG?39*`NqT4`?$JOLIeWGvWr;DoU%W3d_KY zN1E(HaCdsCzsYTgY zMU_cLX&HIuSd)2%QI=^^N?B5>X>LiHS%y)5ijhf%aSFDw*xbOv*wD<#z?_KYdudf} zMOHRsxjn(q2RGlLO+5HKok=dpj8y2#e>_2ttQ$}B-4wK5-_Qi_ux3hSc6o_$Qd(k_ zNn$}-MMZj6T1l2^zIk50agtGDv2k90MS5ygPI^j3Nm7wfsYzvyF{nq2C7%_faZH%ltaEHFtkGA*qzFUZJA%B#$* z!qqk{EhsN3swl;F08El`MOHykN=j;Big{XkT1j42MrL|WWmZaxsZnJ~PI9?%PI7Xx zX=ZUrR;F21YJsU)2G$YP?9|+(!mMoboD|cXJo8kuBGVM(l!{!eeMB>JOG6U_6QV{` zjnj%tjVr->I*3bT&|V%=A}a%J=1R;2n?Wd%L3QIvWZ?b51}5Oq0C*n}WmQLZVo`pM zLON*jG~@8flGLJndyKrDUTU0VT54=+oRU>#R#lo`S)QJlQIThsW0ad!l~`JmZ=PRR zWn@~JR+?p8lv`SsYnqC+OJG!4WL%h?kzQ(&o1I%+SX@<=nw^zb1Zvx$m#pT-CI*J) zrpClVi%4Acc{1oFVlcI`}43q2%lj4f>6CNnEAPERi=GJ+ggnpjzoRFPqlRGOWilU7ht zl%JHGnwe8sm0?y=Vw8(@ca?E!L5@*crb&KbdI{)&5R=@*s@(iU@cuXSmXxWnrJ<35 zIWcirT2WbLY*vcglEPg}BA2vCFBw_8yXXcw0mFAk3q*j#YW||Zh z7?)OMC6^^9=b0xJCl?fz7UUHcnUxt;nI`2X=9;D%m*s&rpkk~*Nlr2{(S(cGsVxE~&R*;r$ zng|*x%d5!AG)^wgC^I%LPRh(Pt1!+oHOr_n%Q8zWG{u_FjB*PzD$PqPQ$by?^hDFV zJmbnl({gO5(peZ<7#W&am=V)sDKja{EioS7=(2SW;k= zV_aBPo>5qpo0M*xl$e@kY;Ka8fweuJU6Gobot$o*R+4IFo>gIgOHD$hzQ%gQj#$SN#0DmP0uH8(O(Nh{AU&Pgi*1r(N& zrZO!huPQCEG%F=9vnbE3*rY1o*wnZ@6<3X6USSG4YS<_pOYgkgEIBW;ASbOP$K2dJ zKhrcVu_U=7Kcz6KATJ}&xU4L*A}uL3E7LR|l%|Ij3V<~)3S7lS< zPQEH}1QATl6RaK=?zDZh;asZm*BQhr`#sZnYYXowd*VVj$nni`rL5jE^o zW>lD7o|10_T5UyK5r(bVTbN#$X$CffP_l;V#*?T`txQbyj4Z&NT6lj0a@|8wYC#UZ zbtXv_xrtRp=D8IaX?c~V##yGxNv0XeRhjvPr3K~L=9%d^IVQ!G>1HNL#ipt0<)!H< zxca)e<>h(CMW#79RmEwE#l`vMMX3cvg{3%894)OVOEE7k08KyRNc)LZ6{g8KW%-$D zl^G_f#+gNBxygAMDds5|$=O+%`T3O@$*K9d1xeY)1xCq5Mv2B{DLBSNNp)LERdRA! zWv-DCa@u#yQwYsV&Ph!yQV34YNKGltNma;CQ}E5tOHWNLPE{z+FU?6&NJ>>GE=>mA zjFFU>lapGckds-GT9jB+sgRdnmYSnb0$$_?-v1B2^`W%VJg+>z5`2UO!3WNw;SU~HO{ zYhqrJmtI+xmRXdBqogq}%_%L*$V@iLPb)|%ODd~M$*M>*FDbxL(&Uy_q!g#8lz`3| z$B`0JlalfZ@=6O#P4lyp64OodD$0_pj4JaB%2V@9jq}R$vx-vlOH)kp(@IUsj0*E} z^7C-izg4A0W=2I7MpeeSrI`g4MwzLZ$!Uc+_nPJAR+X4nm6aQrqof2!IKXIuW>ln? z7Z;gimzEWnres$YmK3I%7L;UUnP!(3RaK-FXOH{UACN? zlx1FAlwFXUn39{4V_cS;ZJLo}l83DYYG!O`X=q|X)M>e;6{*IBW@W^+ztEcRDAjVR zac&aC7(A&K%>X<_m@#;c#1hA{<=pI~;*84dl%&$+in8+bsd!PugpkJ zO3KPKNlMKvD@rY_$j>Xt$garDFDNZfF32dzEjCU|E6gv?$Sox=BVZ*2RCuhGxd524+OHevMMh^HZz9$Iz8jLiYcg80i^8 zGA#HyAkbAk)k!6{Ri`&AVVW#lzV~<4GmH634#1loB&RtNa!6^0Po2?sOEAQZus>ixd(S zlJb)i6;i+(40BWSQZm!?6+o-`K^yFnK|2b;3wet{%?0qLzf^_f#L{B${=37A@`^JP z6^g-I^zxI6GSd^m`;kC92$WIWk2ckinx3AVm}XQ7y0{=CB`>Gkv?M7fH@zq|v9hSD zDmT?UC%vjNC#kTg%qTe}KfT81Hq`30#t>8WPLnU%%KxQhLflGLoCiejT;aM6mg{wys$ zDKEP)BegKgJg=~*$~@P&FfT1VyRyRAG$lJDr?k>6H94a+-MF&Q*tFcVs=%le`=PT{ zrWyHpg%ugbMuqu_iK*$CnT3UBS(#WzoDB^u3{4FUh*=F$T2+uzo>K(wlt5d6pm{3n zlS<&Yf_F_2WoTL&`uV>Z8Ac^Z#pXt-RjKJ|skz4Km8N+a=0)a(6@@uvDMh)t#wN+8 z>ACskDdi#2bl%JlSS!$ALT2_)( zlxB-yQxJwvopH_*oa6ElqylZ+Eh za&t`*Q;W)zO-jnjD{~8Ts!G#~lGC!2Q&LhYOskC2j0&OtK|a8fRsc zrxd0ql@ui>n-`jtWM`KdC*#a#MTwau<|*J^f3-SFkow=yM9;_q?>Kk5QE5q9MOCFy ziFtlSWlCjwWl~ypa*=7ac}ZeMT6uQ4QCfa-R(YCfVMSTDDK7Vtcq>FsNm*H9 zS(=eiK4uY}l$vK!U}kKRpJ$q!n4Ml(k(5@HlW%Hfnwy!ERFPerT9sawpIBsUYMy0Y zlw(wAY+{BrWf~P%6dETQ6%>`3Rwm_D6;!3<78#j<&v-y+jhwPMDRaTyE23clIdCD{e-8^M#g4)7GOPQ&uWmy#&X8B1eiDh|Jrm3a*Ddpv5 zX^9yX1*T<1C28gvW(65V<+=H3g_(s(Ch27*xRwZK7ABXOB$|~MCRLOfr{$QImX?FOrj29h3m&#CK?&&nOmaHETE;ljJ&eMtOC;t@pn9z|6wRT%)RTqcrp6V^Wz_oSl7%)(g+&Ey)-E&w>&>1G2PT8H?gcNFA@8V zQh7yHRmo;mIhiT>iIwF=plj~3%}sM~oj{sXoReglpOp;ST?QTlLpoq0J>T5K$h@jB zqarcM%q-P7vC=#@%hWilqOveKF~1@?)ubRNy&$o;+^9G$DJQKmE2|Rgdk3aJ&kPEJk= zsS25SX*s2-dC8fHDWI+QDVgb+C5btZvqn-fK_~l^q!xj9%9drOB!bS#QAkWmEh+&W z8&m-rG)~k39isy}-liC2T5^7FehTym5YP^M$YCfcsR{+DIjMRI!HJnkpp#xQixo-| zlk)R(5)}eV!5#L(QqcJ*`FWsjI(Rpo0_f0?g8br4$mvBX`3i}}#gH=rKt}_mRutr9 zCMQ}c#3;FzDkLUnKqdnqr{#d%nss;~Xu=>VH5ck+Jq6cNh4lR567az#DXEziiA4$} zr4W-db3x+{xe7U;Gg%Z0iV_t-yN?xeOY>6l^%Me%KnL;UD&&F28V;{iC;`!FhZiPi z=73IfNi2fw08~g+NKDQH9ftxwHYv4KA+IzqIZ>e`vn*9l!4q_{Pg;JaLTRo-UTXT` z&B>V%M<*ud=ar-;gOr1;0iE{)@_`QMCRUIyN>V|`uoxLbj(bT-RmcSGq}G8Pc2KNP zkXl@vpIEGrmtT~dm;;_JQOL|q1fBJh3m#a>1X)~`nyZkPsF0Wfx^6l7@G4N8>nVWd zT0jPY?s-iuR!B@K%_&I)MNV$g;gz`xpp&3-!JDN)X6VHzDTF104tz>f@X0So1zD+3 zTmrgvIRzAvARADQ9#RNOOiN77QE)CTPAtjS4=hd11;tk$B={g9QIKDnoUf3QnRa+B zC^Qq3iZY8}=T3n_1QgPc<7LV+ixP_<$LAzMCU-!AlbD=eTvD2wnh8m?kn^9wu?jxa z33`GRIO0&{z{vqD4o(x`(9VUY;e5!166nC2!&?-RN{f@h$ut){vy@l_3W}iA#GK43 zuy9FHVhMPnDj$6KO};`R=%Aj&ybP#d?%|awnaTM&;4`p5v7HExnB3A7(9Uv&#GFh} zv_W^4>gp;KrIuuZcD#dAYH=y(2JDi|Jkar3$r<^{`8kQ8UF`+=MGC1EsksF?kOQ(H zsS^}s;Pa!B^K+9j^HQqPO*8ULN-Gk}3o?s~3QH=g%F`2#vU76F3-gn6t8&bZjkEI0 zO>9_Vhctdf$dq|}m(+*D(e(xS}5?2HPtl+^U3id-`zcUR<= zSLGDtnCB$u0KCMO#wX6NLXY`u zYGj_AnQKyDR$`WFUTj)Xl$~ajmsyfwUS(`*oMe_;g>_9tNk&eQNkwL|c~NOrj%iU! zxp_{RsaYQQ&MdTEw7Hq3k)g2#ahq$5GeGlK;06$)%~4B}t~H=4P2$W;r?LRh8KpMpY?BRTajG#udid#>R=s z>4^n7#d+Wc7nac~6VtR3lfr)N@PAsw$IA3b9>flaga% zVp3_A2wLToS(=n?nwyi7ZJr!$Amlb3s z85NnDWaj1;q!m}D8YP=o<`$V^TQdpTvubE!LEKrT6LfJgqX$G0%mVL3zS5 zu`;pLGcv*36-lo!E>BEJGc7MN%S<&lGA%LBPEJh9Oe#q$NKH&J%}>osG|4kbF3G9N zG)YRzN=wc#!7&VKTAX82n3|E4Wn7q)YFw3*Us03^%C-79Zj>+0H8x4fsl>KdJSjQR z#I(4oA~Q87Gd07!Dl^F>udtvr*}Sqm$GkKr&p5~2q&Uf}B+sn0q_iqEIlC$gYva_| zyf`(@#5^x4tuWENAU(@G%P6h1stRi-*uuyVbe0NnYoJR~jY+!h6yB9Uit94tl9X~H zFFb|o$I}%twK6f%voOTl6-hJ8Eyyj%GcGDJOEXF=HZ#h~OD`)c$}%ggtSHP*FUw6y zHO))TsZ6RgF-y0u)N~*{$C^RuQE=tVFD5y%# zNUh3DF3c;-$SK0Qh_kBFxFk6wIsVNq31 zMR8tgT2WGBig8s*d4)+)zFC!dR+ec}R-#dQd17`^a&k7-VV}&rO0yiJ?6e%y-0aN! zyrQIX^8&M^Vr=JcnHyS|8k$)UxwWJ`zbq*eI+aSyIT-L1iQX!@gD&64v z@uqk~3q2#q?r@wHXNGyHk!eX`V@LB2^@Nq!l$ftr(*nwpeYQI%JaZS-An z#>vU4X~j7e#i?0lW=0vtMMWt^X33SQr9~B~<;EFV`5C5VMunwm#<}@9$%VR@d9HC{Zf;(pxlx*NVtIaLR%xDbS%pbbSz>0gQC?YchPhEia!yWq zT6%J2VpdXV73dsT%mozXIaw81Cg$ZSg=V?=xkd#=#px-f#wFkzxY7GT=4Qr*7G?&f z#Ej${=M|S_WEg>#ZV?yTn9bEPg8)cYe6_lr!rezdlB_>srW)v0`rxuwNl%<+y=VzFh!*Az6X>+8N z<`k8crFLI1#rdg;g_)^FS*BH) z*?GAYMk!_mNvS!xRr%(o>4lj|#>quVDaKXVIr)X<8A+vB8v#`rNks)!#m1@GCTZ!J z<)&HY`9)bpkm4LI#h96ySr{4_84xx5UzwJiY?=X{w;?yhlvbuCn>!U)?+W$3UUcqBSEwFJCQ5PS@O zK6vO4G>)EGlvu1#T$-d{WT4=U4xr5|B}D(KG_R^4w;;dJq$ndlHKRN^EhoRADy=Fp zyD+)1Dl5GJbVf;~d4^e;S!RZLalT1rX)dJsj4V7GB-*rDJn|L&CbSp zNtU^Rk%6J5IguB4rkAG`C4z6jCN6TIH5YQiu1YUYEUg5aK`3&dy75Ggsg;4Dp1HXx zzEUi?EH5W5C9|jkbX}Oac~)kjNlI~9fk|GaX`WG1R&h#pK~Y&=eqKgVqG^tCRW>Ma zvCknEmY8LOhSUqq%&T&=a&WFy$jD4FFDj~puE|EJL^HC}QnSi3l1t0V zj1n`=t4d3fjg5`+Q%W+E%uLdAa#Jcwt1|M-%`-C!O|$be63eP`v93x<%`P#{E6+-; zDlIoHN-N9D%E>RQtgOH`(rad5W?^JtO5DQ1@?5jR+zi;l!Q^stW0UOEO5^;j@?x_} zvs`m*<3MJXhL(n=mP9RvD6KM0&M?acFQy_tDVZiGr5J(Dpgbv=fG!QPG{d_F6?80K zc5ZHFYEp?=d45uXd6s!rzG-4Xm1&lFNk(C@SwU_|mYKOpYJOr~R+@21W{w$-#&T96 z=pMW5l8WSf(~Nx6DigEv#G;f8Toqz&X;x{Od3rInq?DRdT9{;JTwGBcnx0*dla-Q}m{M9;l44qAT##y1oRXWCY@Th7eGDi+IXgMqG^?_rIL$1%urM>( z+_bzZu>k8vUeKlJ1}4O9udB?=E+~WCXij!aS7v5sSLK4oP^b~pMwWUe_!l0e6r~g! znHLn7r>5m(l;)(Q6q=i56{VF`W)+tul^SK5R1}sMR;8EZ=4P9km1b3#7USA_PwG{6 zr8xzKd8WBWDVW`&q>RGsDs%Is(h}nmljQR3oNRN`;`|EZ+@$Q{s`7%2inP+~;i_83L57C)x4?6`9;|8Oo22{ax#lcphn;=%E1>5f)^u!bb!|MmssiRr(~vB z733sa6`K`j7N!^Fl;u^H<>#B26y}s>nHp!7BSuXTw&@$^7#mlar01uX8XJN7_V7Lt zyv386o|#ctmY7^-T9sLzlx>uqSeRW=o@|0qC#U756&IUU6`5zJr6r}LXPV`trW7Qa zl%|!WCuLNcn59)D<(E~M7-eRqm>A_IXD63pA9*k}E>Fub$;dP}%}cM!$YMu#rJ`I|^KxH%bm93BvO3Tm5$p_UCND*3Em70~A zmjhnojVvvbQT zjEq3$5NaxeTWin;0g}7RjMK9-D+uTO^hl{%PR_sQj<%~ zaj6{BC{m(%Djq{)I_61@BzSy z#ir)QMoC5`g+{4aMFrVKrlooLW~oMHxk<&T<>l!W#TCgWMd`%_#buQ_W~NDr`M4(A z3kwT#jB^u9O|wi=i}ETmGt;U{l8Z|+5m^}=+n~0&zHyF8ZfRwrsd+^<_y#nzalxdt zEYkwhoZM3L%Ivh1tn5;=%#z$(6SI`OwDQctyp)onH1nj~yj-J<(v)KJid>^ybF9-{ zNvQ=T*~WRsm1*T=nTdsJs!K(bT}w(8S!3m^GiJ73C)7Wd-1_2sC$+ zzV5uD+@#P5a=#+wN!-NB&`{5Sz-f;8##KhrshW3MJ8!gRVHRe zCMEf0{N5}bkJeP7^{2J&5SDwO3li%Gczly%1lfOQYs5fQwq!~OR`Lj(kgPz%&Ia?@^eeG zjnndy@-ovhP0X=YeicbMrD?f^Nm)fX`FTYt#RVBfrP*b<*arU14Ghc-EzC@b+iH`R zlbsLQ5Jh%OSEc2omZX4N-IT|)sg;3&p0O#o;|j0;&_;hVa*KX5%`=PBlG2k)EAmpaKo=)rWWB1S((GK*^1O=Tf~>^6+ziw5iln5p z0<%)%)XKt)ioC>}!jzri=mI%x40xBw-yZZ9oQPb*GMH3FG|C+NX-1+-;= zC)^FKOilF+%q{WN?TJN26&dAaRVhivB`MieMa3zJx#ksSMQQ0q70KC2d1Xa8g-K~f zhuhgbo2b|vUe-DhWM1Rb%n)U$+aR6^US4;}vkU67TL z4;tt?yiy@wp*SPITp=Y|867$TAl&q@CoT~Jq{6h1z;NaOp}bN+(h%djH=YUirhTo(n3(19CK>N%&amwFRwB!4|Ix6rCE7Og;`-{2KXFN z^mJ=(VPI)!WJc8ZW0`TWX>M65xCo}8)BqWiXr2o;22au@zSJ-SpO;{S+A@Wg8io0} zMoA_WrK#mrnW@F)r8!jv#f8bqMoQ>|=V>WbY39jgDJ6y3IVF|JrYR|91$k*^rIlso zmHAoaNrl-NWl7m4>7`{Ux#q@INo6M3mz`CZCYP00WfT-8l@_I!rDYW+SEUx28iN+R zV1&N0k&&Ufr4cdBr81+G+(hG2BT(mrVCaMMJ*bTc>VsydRw`M6RgO3A56&IOx+ zC+LxNqm~+mX1az3x<-~Eh89-F#-QVFp?8Bqle@1cXmNF3esOW4LViw0BIF+2w9+Eb zvh$Qwg(T35^pt!B-$YOgBUQoAQNhL4%{AB&BQ`P;Qwz%ybBZcU(@HbVt8$H!vy0R6 zD)Y0=)AGxc@-wo`a?*0Dva-wzQ%X$BjEYQiDzWD5%Jkyo+|tVA9FxMr%IvbT)Re5s zoWgW$Iora}%+k=%%#xVH_R363GmMQ=jgl%V(oKys(lgCW%#%y<^UF=mt1@t%sZo|@ zTAWp0Qe>Q(Wl~g-WtN+2RGv{#o`j>tms^}tnwy&pIU}<~ioc zrb$Kxg~fSE$%%=@nZ~$Uqq+Iz*}3MKu%kwy(O;Yix;?cRwAMNmvRWFxv=_P^0JO|q zp&+riII&0}DX}Co1+-?n6qHu-!J7i0#}4GADx@kDXXcfFW(%_NA?xTtyA8nG7!EIn ztlLg40$op-S`50pI3-a5bmwcTLP};)YGz5I4!UaaN_6n5@#K-inVVsqlon)4jR+(p>kzJKvQk+`}x<<7uzbGXq zFVU#5veF2=Oao)ApgglOCojLExU@X6ETbSly(rh%JkvNA>kSU3CPs#47RJO(tCbm5 z7UkxpgC{?UD;c0I0_3p*qspR;A~UcVgh~deZq$+iCa5>)p*62lOEXP! z(~67=^Rmlw%~DIt(lX1-vvLb6Q?imWO!6u#3kx%|5|hm`bJHtQb1M>yQgPkjkeFDU zoS9-$WDdGxryx5sBiksmq9iQ~SGFpxN~x$a%>`ZCiBShsnWh<+R+bo>=A@)%7Z+q_ z87JnO6jW7JRpnI_mgJjfC7YM!7MJB`CKl(Wq?sFm7Wfn^Sw&YXCFYbUSt*6)DuAx! z%}mQo2HpNwmRVF%nwSIKv;fK_`3f;g&iT2ix%rU&2Zja;xtTc`hqtB{gA#LTap~cO zMVXLWhn;dtQ;V}Jbrh7%^o-05^h}J+bQGKt^AhtC6?_#QHLP+-0&{Zpz!zhI_aEgv zYFMMAkeQcW1lqMy3c8xl$O2??VtT$Fhz8v*9iybAqm&4`v9Ba3u_QAebcb0+YG!&y ziISC}nSp_hQh8=dNrsY@k%56ht&S3Cw@WHW5$qI`q?~-v*mqHC8dx+ZGcOx5QNZo?Mhxkeh6hol|UHkeHjElAlqO zUI;4xb2D>OLm=&8u!Z_r1*xDzQ6M)g8XK5cSb$DBsnt=!-qr%`wSngPBJg};N@i7R zkpg(j6FB$hCKl^KfP%jQG;kFX!FQs9$|>m4X9~EYL`NYNv|}i(G*clTDM}O+QGmjS z`IA~d%Mr)U-U&Jg*?J zB)2@PEX6dns@yc!I1TFnReEAos##ICNltQpN>X}JRfTzCdYWk#))hTQ=0=95#zc+N zlo^>-q?lEJr`yRdUd$@Yk_*6QP@{M;GSjm(N9`t}6)!18$>l~xB`Nupx#<;2rllsy z#Gz!cOpGRD7qI4wOdH>n~!B`v3@A|)rc zEVVc#J3Y71+_*R=x3nS&bVsgPZcar?xp`h$2I!Zasyz&EW;KJP0)W{Gt(GDND$OMhjXQrg)m1LHH z&%H|luj_`*cHl8PBQY-peEn)-UP*plW`0&_US@t#b{cX4sidP+oLZ=4WoT(^X=H9{ zU<}$OoS6%{v=t*3O-=O7L3e&&S>$14W?EX3o{?%?R$87|Rb`rPWLA`AWRBH`7NAYB zMnvr{DFYp7m7hk@s3^#cv|{2Gc@VD5%|NT6EDZ7PBT6bwF*mQOGB3?3D66Q*D9X=I zu1ZQSO0G0YGc!%hPb$vJC`d2OEGy1WHcL-3H_k4sz>(WZQ!Dezs!Gx-(^AXQ%S;N> z@^cEaG7E8?aG6`0np9aX}KjCnIDa@|%qYh=ue_?TEU&~QF|V>Lv!b9lzpM&08idgSHa9jjG&C}Y4yGtsDJ6rK$tCCK z=4O_F+7?BrWw6exaaK`PE@jJ~%8av$N(&*mi%>lT?LOma=9+=#3rrv}otm4JihZIN zETdPJnUb21r`2TuI<^$r>M}JqtJP70=&-6*BC64qk(iaAnv|25ZEBL2kyMs#lv($w;@!t^B5{JiXn zqN=Ql^7Qn~q^v^I-0YO3T;udyT$ewYj6;$PyV_iy>o|KeaZkk$>3);zCP?%I!oLrD!mWBQ3DKj%eQ&TgdCWgw(vXat3 zg*UwJ$2|pzHXw-9c`Yq3%QDF_qv{RtW_rdJs7vSIBZ|33`Q?S?g(ZcVl}R}nSt%8{ zX{G6&aE^_OEfZ1 z&aSM=&rT^fN-{Mvt1K?T8v14CS&1p-l{scSQl%v?|%g_7km z$})2^jPsKV(sPVUQVNsuDvC|3jEhYQ3$u)K&6A7MlgrK0ib^swjM6IeKo{4W;W~@a zC?&BdE#KIzBBj_QzbdaRv%tKvs_)=JSdyF8^R4RohkN=`vua$bH~ zVqsox38=_KF9*zwEsYH=3@t%@8$zL9T9IRJlvQ8^T8>FD^ucoj&`tw#5nPdDR+3)< zHUm%4BkRUf0vKDFTIiV>qm}?@9sIPMlDyoMoYc}Jv)syZ^VICjyt0Cnoc#QZ?2M$M zlI)7gvb2H%^U4Bqqr$3;O4E|0Qc%MNOA|WFI4iHHyeOkQDbXk|y`ng$qOda8Bn9{Q zVOd^QR$@Nr(DPayNN?Z3LeJa?Zx1QGvOFy=->AYk zF*&2CJgcHKD?L9mttj8jJPUMY5Dw>Nq-IoP8W$EDXC)@5mn5eYXC$SiCYj>C#xyrM zH^(FubUy+5aA#6^Wo5B(MQOfyNlHO_Qhs?wT1H`6l~HNH$PDMeHiCJ=a zdP;U_m2q}{64r#7QfOYDnUR`QWtx&yW}KT{mS~)uQw3faiIFgkj0_FU4Gf8D9#j|= z6_)2gZ(1Q{doZ?yX;hSLTn09S8VS?HP|wm7@6Mo9qwK1zk| zjC7;Y(ws`OqSV~7#GJg$eB-SA{ESTFoTB3V%p4psZCsIAT#{T|mR*!mWLi<0oo||& zm6e%{^Jc1ivy$v2lQe86J*JgrSC*P2mFHyUW>h7mCncp7=b5CI6(lF+=b0O4mSm-x znO782s?0U5gp}N57eQ5(IRz<@N`&&5HU&*88=2r=5|Nx(1R-hviEKw(*(Tc0o z%;enoL!pe<{1TGWAG#lGz0KdPo`F&^Jy(nr?=qE z0+U?hl&pNy6!Q|J^4#p?yu?%!Qxnt542)($Vs2q+idlI|N<~#+Zf;p#hH+6rdSz~z zSxI)bNkN%ua$;^#Mpa39WnxKkQK4}`X_6V%2C7MBd3kP1PI6+INv3&5X|`!fN@9VL zNfK!3FxptFnTe4FXh#!Kd9oruBPSWUzZ2SqP|{JtzIqy5u|r2pkb=InB0nQNJqx@? z6i?72>&6rACRPSUdZv*3LLO~!%N;(*Q@ z%PdP(aP#zY^zn7|3vpFQOioP7F9jWmR+Okv20GJTA+0newFt6yAvd)s1$0KfLQ;Nl z37C3JEZnW<@M73Qg#rN)^#iB%>g=Ek6* zFDyAZEhQL~J=ZEhRfE)2Of_E5oGJ zG%ckpJ+G=LztkkjtimL}#Hc*QtUR&GG{-2_D5tD2KO@VyDy;--O|u=^RmM2}@00UQudhoC-FBP+EfO z#*>zet&A=7Eby1v>A7WDl{uNIg%y=XW~nBb$!5uU<>rM|RawQx+38iKNf~*@re+yM z#YRS@X=W+O85Nm0YV!1)Ec4W=ax)Y2{8Z!Yq*RmiN)w~hRGejYX=z@SSuSYlK4|O> zrGJ^4RZ*3alvG)knpA3*Q3NX7vW$z<($Xr@i}Q_(%}R>%D$G(6P4jZiD~hwtKurv+ zF>RKSm|K`vY?hapTbP<|l%JTLUr=g>^(=RD&{^(AMn*)H8dZ6fpeCd#D8-N)(^Ywu zIaQF7hw_*<1>MqYXpXmb24%ctW6-EfdP;6#c2Q<}ZiQ)1K~{2^X>MM+S#Dx>l396v zYG#>vj#-XzX>xHvI`lXfJuHP@WubXhZYq`Tg)*@+HrKN-1~){|3Oz!%0VO3R8WkCv zo2DC8RuttGl;s-drx&HBmlmbugN~7}NUth1OD#<+G_A@=O-#(qGcv}tB`+l{KQ+-j zJ3G<1pe(g2tIFIsEv?inI|Emnqtv`4Gug}tbOa!d;|Y^e3M+EV)6=T5iVKShN(+p$ z@=MB$%JR}vDl_wqtFjY~l9Tf?64SDi%0Wld<(ieG;mBJhCE3YFSrxfO8KpVd$>pZG znOQ}N1?9Ni20EI%4BCBD0$l#S3TadUH1149#2qP9SR zdSiL1kj0jiG@*=hvoo?xsJCX*+{)Nc&%_d(sxeo37iXoF=b4oyXBFj{Bo(LTR3zsY znIz?#VuXHbYO+zTk#SyHQd*vIW?p%@v3a>^Vqs2URc>-oT4`~Lxmj^$dRlgNp|PoP zK}u?Jg&EeuFu5o@r!29+sJJTC$gHZkw7l5JI6Xfd`yC^u7KSFKum&ua$$v8|BO^Uy zBe35{&9OxpnaSy8sYxa|sm1B0rg>$VN##lD=DCUKrKx4bm6^sFNkv7;sm3OGl}4p` zrIlr6IC=?HIi+c-W+@r@$rY&?DMh6vd8TG&#o0Iq4@)vi3NnpyjM6}1T$G;=o_s^8 zc#@33-7C}N@+7k~Q}eQ-;@raY@}ktV#EdFq(~Oclfe`8RTLV zm!?#hWaZ>k6eNR2D65iFOf$=iO)83t^Q!U^D~ruatMZF0%W^9cbF#{d%t4oLR+V8* z73pcl85w3Kx#^WA<=LjmX6cz_rB%jhpfyS8!y4uWre=m_mgYo_w^d~)XBMP_w@nbt zli;Q?v^j&^u&&BZPA!I*L6s`fM9;_)?|55ENoitwiBWo1L9UTea)ohfVp>{3QA&kT zZfR+Gfk~Q4QkhwLZfTj3d2W7TX?A95X(EmSyRbB~EH|~hG%dp@CpobosjM(PxhyFq z9cN`yTv%mVRA`iqxm+bVqsTlfBiYQjFefJ|)uf`ZsKU71B+aY}G*)h2oRwFVo>-n{ zQf6*kQecu&SyE7*m5A+7h|1*BjEvIUg5rXLl+4W3?Bp^d^QuG>tfjAsp^>4XrI`_A z$_jo91GYHI&o@d-A*vk!)s3gY3U0=jSmGUe$}mk!GOa95G%Cm{O3JB9&oeSJ&Pp~l zs;tVWs>~}bHA^$jNjEc2FRL&sPdCX)PtGiWHe=A21E-{zSDBj>R%H~V85flq<(8DB zniQrO;ogQ=o|I&6Vv6kMPiCss| zVQNWvVPQd8c4<{{VwGuP9_UUUEK94hvW!a;vkQxIjf!(hj8fAIimQyOGO{bMwS|l= zjSbB$iCaKol%1JgOymL*w9z1>qPNT_t1LMwAG|`VD76H<3pXXrJgcf0YoMeRnidyV z6_llzl~yH}6jv2zn&f0BWf~c!CK;C+mzo%rXB8Gy80Y4K4s0sPOi9d50Z8e4O0MsLaGDH>VI{ z2Ia}$40P+3sS)1eg%gv~3d)o6Q_Ks~Qc}}X(~>fhQWH&*5{pvHsxm67s`5%rD>L&7 zQcFw=(<}1Iv-1mbakNE9yiF;$)Fe02ytn|=vq7ItNz2VF$TTT0E-3?zuVk9#Cnn{k zGR`q8H8;sJttc%q zP0z^Bz;>j71!yyxxv>Q?1znj*WnN|p?k%jH>%3X!Pe|ZF*PbqDz7L@%P)qn|4AxO zFECCn%*iOsFiI;dD=DkWHcCx5$w@9OsVX)}DmE@lE66EHEKg0%FeyvPD>tc31La*T z!-6EA2wPlWR+M3eZT(Mra*Ao9X=YANR%vdzxp75aaehH%Wl?HjNk(a2nsHL5d0MJz zae01aT1q)+omzfX71ktKQdD77XjYn0Xq=l@S&?N_S!SM_m0W~vv8$Phskx!0i5W3_ z+Da>mk_wE|i7R|?m&hPvvdzuF#^6bnXa?YEYnfOXnCclB;~3yiEGa1}E&;Wj3QLp9 zl2Xg6(vp)iGNA{;r6|FZL`rF5R#}=^MGok&-O@61v*hf8{KTBxG}D}tlFCA};*`9? ztiqzKtaM}JwB$;oLgOT?t1b!(l1eIz3zD)E%gb`Ba`H_Ii&G1$azMalz?}Tp!5PW3eq#p^URVeOR~#S@-vM~vy#j+^An3IlhVs` z^ApR9jH=3uOe@SQN{o{Wl1hxylCrT@ys5c~#U_=h*@b4N1)xnVIY!B8DLIwUH817) z<{4!<6`8526~-xL=EW&hnPxaQ%^4+So0e9X8&zTUWKxSu^OLet%L@w3%*_j{vNBBb zinFqeazXCQ%*{17GEFHf$S+PbOE1bSOis+ot0>0mT4S^1QqW-%Nd>6|S?NXDITd+j z>BZ)t+7F|dU|?x%XlOv(p7pA{(u#si60YLIdWdgTUTINDCD;@^85P{hhU>?ZQowy8 zLo>WB<&3nPto)+%oTSRUqO{z+qVh~Lld^Kts@(KKqmra@!SQsWe(Zh?9Ae{;u37*Ru+bamWJlwi=$z+KklX0CIy8iCMlF&FkWU-kXBd$ zS-*iNx=AS+Obzu+OmLJ8$(hNO#)&CL>E;zx#YJgZY5Cb@xrG@?7~?~!Wu=8hMp-3Q zW#*+OMODVh$r;8;CZ_q7iG@XZMagM}#Yx49IVq``x#_va+46l?=z7KSFqMy5t)M9uq~6dRcp7J+LNXj%oWb;UmOjFx(kDyHOKrk!eM49=4NJ%uUP;4UJ4JiE_G8VP$q< zA-E|5bvkHy9(JdL%Q@)C50bmfj0!7L%M!t6;K??~y788g#uj>}MyPEWw8be&SxM&R z#^!})={cp9=85S>xn{X$=E()6>4l(6ZBom!lTu77&5F#D)67c?Qp~b(cBe8;b2HO2 zvx?12Q}PO{Qj0T$&2g<=&oEBP%F0Lu9SMw4MyBT_nH85MCYP3GmQ-YBC+Aiq zrIn_D&Ra{aGO0+)HZRI4Gc7C2NK7)zEHNr9G0DZgEi%)ru)++K)02y`(u_d+3QDt* z3b2-uW}p(%#KeTSiB40KlyVcqA$(=Y=4mGRiD@b66~&nu*(S+Fl|~f>RXDH7Dy_`Q zOij+hc6CW=RbFv!YDR8FVPQ#OK}Jz(Mnz6hW?52hUJ7{P&Mc=WKP#muvohbT+$hmF zC8HnYlPVK43)0PV%gRhmD@}6?iz})!ii$FG%Tx2R&62Zll*DF9Wk#my<~c=KSs96G zg~eG$l~v|uNjMjb=9`wKrj>v)G)5v%D@#nvHA~7%&dRJxt}-brG|4O}sLHM?H7`gi zFs-UA%TG%#O*PG|%E>4wPAf4s%`C>+5GXgVFe%RlE!Qf@D$7eWGcHX^PRd9FZ5Tyw z_L*52ni(1!5O?`hWl2U}sj(4w4>6%G0=PheH`|a3w9?9wjP$A$uqlM%8m=Etk!E6L zVyI_eig*8SMpa=7_+Y<^^wP4j@{-i70;8(LqLPBD@;vkG?7aNMRI{{{q)Zdj?DCw7 zl+4l`@Wo(Q>fNmL)UpEe#EkNkbmOw@+%)rw#ER0g6k|xU57FC8%}FaZEy&9*C@MC} z%PLAWFE2AGtx76SF32}7C^bziHciS+E-6h;F-y+O$STP%$jJd^L@YK|WmFX#6;+jF znUh+Xmu_BCo|ToDW|o&% z3c7l^1k_u^64Qm|Nv1g_d3mO#Mi~`3CFaSdCRurji8(l9I^WE=DmfW6ON&1Gm{^f& zR+*KYmu!@oUzKB?m{gT&W|EU*oRgeioK~EXmYGpZBN>JEKlCsK5 z)6C+sl$6wxY)P=2C9Mp0^-ZH!VzNl8JmmA-y%VsS}+N`9VR zeqK&yUaEdcVtTQDZfZ$lT26jZVvLePa!z7#v2BbJ=oFpI{5*w>#Nv#S#Pk>?g`(6P z+ZZJ<8)Ahrm`W}#NX!GXQW55WoRqAOkO9l1xmCXaEDEY5l8Q1@(kuOoGC{Z6=7bjI zC|Tiju|sl3W>I2dy1t>Iv7xziSf=d#Mz=xk0*VgJFw%k(5N#OHQbd(?k04yPZ<695p!m9lI zL_N^%C>)Li%{3!C(%7sPq%tQlDK)1Ur}FQY@Sw_)0QVUX&Q_YLh%ZiIDj1qIq%Cd_pE0c}V@)A?i3QJQn(yJ=dQ;f>YObgPh z3W}4;D$~*sZbRGuZ)|01u4jaHh$UKwE2FG9wJg0TE!WH}HOHvDtfDeGGsU#9vNSWR zusl1{Bq`C@EF-VXv?|fGsKVSR!!#S$LCNVkIaS8xWhPm9M%gB1$!S$)+3DsbnYbqE zb8<~f%99h5jj}L@AX7>ci!*Zaij%7f(({b-%#+LtGEK6Q3yiaqtBew}b4v@$K_?2P z=O*Tro0pfQn-!H|tw9Pa&C7C<^NkD5DwFcfG7ItxO_EEpOtIA<<|gJAhDMea#C5HU zGs;r)jldVG5^9Tr6SW?;)4a=!N{fxtE5V%?Jk13(1Mrk;rdB4FdY0y>gOKpy>eT#- zbkpprib9jj#Io$7s)Dq%l0@_LB1r9lG+B^Zo|;&kn`By$)ADT{GlMO8szWm>XvL26lQc1ebrd0B2^ z3Dy~+m@*Ov}v9H>yOsNB~~qLwX*inPye_#@J@V z6VsB5jY@MXi?UNqQi>9DN-|6e(kjanqyLIo>yL!59*qu_iW8UlV28QrbMOZs+|11d{Cbpp7?Q( zF@ik}k1a%O8&#&47Nk@ZgWKa6LyTEfrAeu#6{Q8mrTO`#`GpxJxoHJC>EP)$^d-HP zW`>5w7DV;Q%S;mUO4E#p8=MEb9NHE}xH}hQMovyK*bF?`5Lq{#641=b*hJ6L0zAZs zy(OKLS(R8+l&=74`6}d==B4B-B!iB{1u+tniW0$x<>looBqbIpfKGGFOjIaTC@D?J zS4hh*E>2Y_0PX83C`!dFH8aYKi!$(?rb^9w+|WQ@4Vs7$NKE6B`FDN3(2HOfoO%gi%3E-nK#Xwg%dsimQr zp^2d>WC0-T_>Yw2WV7^i(>#;RbmP?Gk^$>rI_pks)NEAots zGt*PdGb{3N^a;((i;QwJ@(Qza%uL>)K=4Ip;DSVhev**M7{@s|NXjWEOP-32%lb%zUmzdootG=?lGxIPc$z|GcPPOH%`wjGA>BX z%1Nv$haQ!HGTvxzZfYGF=MVOCW^dR1CUX1Q^0Dzt^2l$Dwc+5?aR zy^5zO&8Q%;+}Jd&qRO6uvtDJ7)^CYi;? zNfk-v73pS$CRt_~*%cY(nTdtjNoBb?WvR(n^IUPdaeh^jd3j!CW>#8av3W{bNl|KL zA~?3tW8KKY$k4*jf~ZC66^S_}`9PPb)3Ytg19h$}mbw%}O&WswgT*%uG+mHf3pMZfRy{ zZfZtUOQ^CaD?8VixMe`NV!E;@E445OYzE~qZ35bxX^FP{3N5D7(lT>RbMo^utI7(p ziwXC)H3GFGbd*5*|IMrnE%c1>A6Jr+o1UAVZ&Z+y zlVn<6Zd8Q2{P*j?pR+X2V zl98NTRg_VlQc{$kVv>@F^DgOZ;~cY`B+${$7_(1_Rrv*`6-fnIX@yA@#f9lPRaxmq z1<3`)Nu}ill_|M}rIneL#)+w!iA9xF6=r5R=Ehi4W~F(05@_fqr7SHmF+VlAJh>#h zDkT%^4sR4z< zY6Yl#2al8{XXd6W6c;5U4d)gc=@yoz=IW*ABxUBNgGTZ6!*x?#joi$W0?p#XBg#DT z^1brCDx5t{Lh=l3^)vN~jgb|X8R|QRhIqs$8KfDcT9_Cam?vABSR^J{Bw3i7nZVU1_dD za$;U`YEB|(1RZixJm}huynKb+)XaPZP~RglH#09IUq>Mkq_iZzC{ZCVzo;Zt7w#?4 zN!g~R5L=2<6%q@IQj1gbN)i?FQbET66sJ~{BM*8JJobn&=rC;%$zkWELiy z>LQ^0LZ}%=0TV z@)J$+OwDrBK}Yr$CMV~VCMBB`C`ijOO)JPtDycFqOw2DXFfT1DFETT&Do?H| z&CkbL|CgsHW@V+Al%^S{8KoK*3fT6LX^aAm!%SNqL~1 z`oxaWVJ3^xa*Qv$BnnN=(u!Q!_L3&5DeYvT`z$i?MENFf+C=G$V3p zSY>iXS!Eu0y$f*#7qsPpl$=W|lQZ(oAZ8FMxS+c66kH}&M#g%k=6FWvN)mJPL1*A7 zBp+U=P>_!?M3kPGl$vHzQCMrvk$a#>ZL z63U)~b|s+ilofEqC}%I%z3@^O!JhY@}iVfBeUG{!qn2DJoC!p6^YqK$t7iG=4lxzS-BOa=>=6u z*a}fI@V+zy6XJ4tRe5%46>+0`xcZ;vRpn`wMPM@snH%8U4xeF? zTVhmDmR)6*kyo5uk(zI0YEo2Lnv|WGoL-QdSYA?5Rat0WkXKcdZdzDSUX-3`grk}- zE;g^qElw*dEjKMMuSiWRt1?Z?Nl(Ig3RG&ASyDk_F1C?|3}Z86qs)o|qpaN05@X|J zqau^EeDi#x;FLEL#P#QJWm1#eGLy85 z-11~_yAV%gqZxoFzRf`E2~BZrE2uIz$~4bNsm#mGHY+kRsxmUqFe@o5h0fqYkAcak zGAcDHDX%I@&M{5Ss?5kMt1`|`GR^=EKUSKS6qICUn3k4iCS@2`<>VEo7H1`;;Ybb@ zNyeF#X8A>CNrmYqRTafmrm4AAMdrBL_qm1Ti79F6$wr`xSqFBUjlV*$Q-}^|Dg|^Q z4(OVz)LhWjTcDm&szOO>PHI_Z@!`do#e7nEUUp7eMpkZdVs=VKvTBrQ{ z)J)IV6z`7Z6r;?lyxc^iB(vfYv$BG8lPvSB0@JGe%uMt0LenJk+^RfN5p zv)tr@M6)tb^9f7MP?cYmUu<5Qnx0mkky@CNn_5|wnNwC!fUEtNQ(9D3m6&T}j#)FL zmKdj{mXw#}XBB3qWtx}dXXfP=CFZAOmzN|Kl@{jYmX>5Bm1LwAmY5l3B&VblWu{}T z8O*aXvU0Ohvoop+aAQG2a}#1QF>12Q0T{Sr{9d7#P%| zuIkE41g+|V&J;dhv*yXJp2y9NkDHc1Zf<_uwEl7P+{aBz9yf1(+}!cFaoyvFo+mrj zzHHte?D&39$CDl1kDI#Q@9BKp(EGS?`Qyf!j~kkv_I5q(XnWb%_ZWocKW<#{xN-91 zrryVm-5}YX>5m(CJe@ER#BFSQzM}haW7pFqE1zwc{|S!s4oc5yn6o++st z*GtoqN=i-hvF(6KNioYTDoHFX%uNCvdTC~oR*;`&lvG-pWRjU|Y;K&Bl$n`cnrBpM znvtEJUY=N*lZ|yPTBS*8X=-U!MNxTCX0CZvYF<@lNlFfQYz;l8%`J>A42>*^Iy9!t zxU@XEq}(VSJpONDWnidhj{ml=lr*E9{EQOg!V>e$Y~vE6lH8oK(n?cfo$6T3lY5 zmY186US3*W25M$t)b{yF#wCSCNySMiWv1y#6=vC)>4iB-S=h$^&5aEV3{8wo361|7 z=YY!Yq`dMZGw^8>#N{Vw367kfjFR&5GRnbb5Xw(b-FWhoDd=nx6UfbTpu!*DjX$Xs zkO6xo9VIx^s#+-}wK%ybv!En1KTpX@DL6APJttM+&~}CJP&a=ce|P_25XZ>C$QVQ$ zDir7EL5`uzD9ue$BK9_*q%7mijO4W3^2C(<(#)!=4D+PS@>H{|LX%YEQnQ?5(D`v` zRYn!XiAClW$%SRc$)F8nI!gE}6H{YT3scit9i@`Y67Vf+jtXI!C6I+CVX27Z3R~fv zUyz!oS6q-^l9^Tso*Bu^E6LF(o$?=6D6dyR$5Vpi)11P@9OEpbytIn4oQlHa(u|TEW1J@jW@aU)WG5CFnSo;r zWf&yAsKmUW%)BH&E5kI+EZf95FSpb*$;8~qG&RF4w=%IZ-Q1)!+t@s_*fcLAuQ1mv zu@LLTg0Wd?aivLdSxQnyYDSV-u4zGqQ9)%{8J4{$<|am#hDPQVkX2cr%lz>4Lyhvv zjq@wObu9ViN?ut(1;h+OaZOCQVqyflSAg8KkG$_8I5n{-(LKMkC^xkP$JPhX1@=fO z*&Mv@fuziaY$0e~FEcMFGaoVs2%4G1VXuh^%8m;o6X;$BP%&o}tyE9~YMX+06M#xP z9D50pi;60Y(z2@ZbIbBGsw$I>iV8BTGV^mY^2>8eGjp>`lgz8kOe?Z-%_>aF3zL#c zjY~m|J1o5^)8w@D)ZEmng4F!X#GEX%;^cg@)bu2rM|dP>n3<&|few?e1)pq-mK{^7 zGL3SR3NrJIvQ3N2vdhyFQ?iPaQ}XgN%hFQJQp&PRk`jxuONvWODzXYQimFnw({W7m zlR7M9lvrYBR+)-zDQ>z+NpZe$W>#jQaYa^nSxS0ZX1ZBrp1Dy~UX^jNnOQ-4ab8Kj ziBWo)Nn*N5QgK=$_Pf+G^D0bJGqXy{$}+3+jEgfW(h3X9k}9yBHeqgVZfX=IXNmR4?*S&^PsVVam&l%H5$keid0QJI`;Y-*O1RaBOloNbnG zlwn$spOcrGl8v>Oos^nfZkAPAWt>r#n2}_boSA8wn3j_TK2sfirI)d>nW2T51u?Cx z(u%aaf|6q7p>qObD$srhO3jy+msJVc#t+&pl$>9Ly zmeG=olT%W%D${b(^2*Z_(~L|iDvUFeQ&Np`Dyve=$`Y$`^3u)C3UZ87%uK2>lS-1Z zGI7*-E1Tpe-SW7O3||p?M(1s4OWzyD-@}xi~2?D<#vs z5HwMokz!&58oo_0&oVVGDXK6rOH4@tT|1VSW||EeJ;PFj6(^e-=ci?trKc9BB$}C+ zmlvd#nUxjdthh@n3XL*N)4_Mtqjd<eqZ%wK&OENafNG&rmF*m6yE3Yss&nqa&$69fl zf=(eaH6!M{urg!Qoa9VWPS=1|y+}zFWQJJ|#0<(y3RBQcRR)&eCK~pVBCs?Oyvki6 zATcQ?U!k}(DYFQ)iUhQ(9lW!&pa`-GB@uk%M5;njW|2Z!Vx~e#QEF}~bd(uspHNa( zo^eugZb3<=SxH8IYI$l-ZboTIVy?MyPGMSUrAcK)d1gjNW_nIhnMqk%X;N`M_MUM@ zQE5qDnt5_|aj8j4X{k|GQE^q7nF-c?LKdc$hK3fP5iVHyjPvNA(#neL3X^Oj@NO@n z)@(t`1EeSdnNgTj3^s#M6hU?4DNRg3$B7Xb14%6`FgL5rF)t}jD$mQ!E=Wu(F)7MR z%Sq3#Dk&>W&o#=aGD^y<$V^YqEw9W^O3pLOgO7or=j-B}wCu9V?5eWLER*Dlw2}<- z!p!uFx+!qSw=+^mAid}EV>lFYJTwqk5o0(#qo0Xkq zRBmjPZCqBIo}X-1Zk(Qyo0y!AwWD8JX=Y+nR#lmwlWCG#nNen1oSa@(QU>w>#&C+U znUSHnks)ydy&2gBl_|tkL1^cLdZBr)S-w$Ha#eDvxlyulSy4f8SyFMLd8KJ~L0)-wMrlb&YOYyD z1&%I2c7}P0X@zNdWr0a~QI&aCYH4~&rLkEut_hXQ%#6IsqHM^7Nn?Y(F_lCq6&3BFChvG&89nBRi!yyDTr=sI)2#_vWGE^2)NJszRe2ES-3h zqSWG|j5O0S)1*ApG-Ko9tc5|2RZe0`RaUB5wn?5zVNw!kb_*lbnp#*I8XFl9)oiUaEduSD)IoYZ9KNHOY}3-IAXBp;R;Wt6An z7K2S8R0hNK<4Mn^R>o#}#wK`|!K5Z8m8X`Qr>C1GmKhtT8mAW(=clGrn4}k(rRh1`}mm6o9WhCY&!W&LXC?!|ASx#nUVM$qbZcc%zS#d^zd2(`@Svt;T zFr`V^*@>p6*bXvCGE2-cH7`yp&8;j*G%Ez{J1fY{&nikUE+{rDuShL0$}P%DH!>|D~yV9^j8ZD3Nn+DDk@Wrii!)(((-am zN;0!j(r_N1R$7pkl?+miS#o8Vl^UBAXIAAKflk{>E;7kXu1ZO%Fi*`+FD@x9%u7l$ zO*GA}G&3zMFHfliZSXC?n*S@)OH(Q`lFZ6YE6R#8)3cLHlgmnSQ$a;9X8s4AlxJvx zSiue&?9?@|)HN~;0j&l$1+6ABw8VcpSz)C~NmW6qX|idac|}rUQDI(UQf^tkNm`O| zc1dwkeqL%}WmR6OXmSw@n1W_D&e)^RE`6HCy2z{C`i zrIjhUrX^M2!-EMHl4xywl$?{2Q(}??HU>{AiDm%a%wue%XKIQ%*aPn*RAp6`XBJf@ zCz&NBXPRXvCY$GicK8)zbQ01F(#n!5vdWVx%d#s?vr3XoGP8=x)2d2Jt4y;CD?uBt zbIeLhP12I9DpJcbOLMc#uqNfSEYq}YGrwHc6Me_et}t@X?|%{u4!^%iLq&NZc1`RSx$;kPEJ8$VgcwZ zkQ6f$^E6ZATxjT*XXat<=uS0BOfSeb&&V*!Pckz%D@dzK&(2IYO)AX9n}H`AA?wDIhs;2CG+LN~w@_gp zMoY{DZ`J}WV9Zy@O)Se)$c6#T$TT(0Oau)}BxV#ARF#yKmlkDbXBp+2o0eAQ7G|VZ zC8npFWR&L>XO)#yWaXw77GuA0C^@Gn1vHS8lU-O?Ri08@TAY!YlT(OwL%oTak)f#t zXmSe{qPU01DvAm#Gb@pIyO1&9UQ}3`12Kb8@Do%28<^^uo8j$mCzoU;8<*v#r>2`^ z6{c0D8Wm?$nimwAS7n!H6zAup7x8=F@c zrDYp|4T5xCNRH|GdQb0W=EKyyaSCE=lmX@ArZeEp}Vq{vB zTy9!clvRnNb&^|JQIb<`nvZP=GtIctI3+VJ8?<~XFQuxq$|Nx>ImJ9B-84VD$RsPr z)FeMU#XQfXvLZj-xG=ZCG|LR@+P|`F6LV8@6O+WE#Hv)&iq!nveDjq2JaFd{txPqu zG&2L8l0npPL{(C;X-NgRJcN}1_`7h>Y>C`7sY)uU1g+i&9ebHd=`5NFXjQVQDc($- zR#;Y=o1dCuY@Cx)S(KZen~`m7Vw_h}m|SR{ZI*6gQe0S&mS&QiRal%{T$W{;1Ui}k z>%bGq7se$Q8)cc5VcV~gT9#~7l$}y+o?V%lYm{1^kyvJuRZwVVQkj@kP?VEgo|RdZ zmY8c)m{ysRTwIY|!*8EGb_<>pyw zCZ**`rC3{(d1=N4mAM&N<#|a}dHHEsxw!=ups55bt8gp~3{4G<&CQ5u&zG5`6`Ld` zL&LEERFHy(22i$aC1)g-=p}(pJWfn5$uH7NNlng4EJ_3qiRi$VUV)Cc#F}I9>rTwc z$uIZJOU?&vL07U;O3cYoLO)bDxx^^7Fe^8?)U>!FKfTx}wJ^V^w4lNy+r+H2+{i33 zFC*WyAUUJNG$*m7syrvHCZDa$hgA4`V)Tv7A1=h_Gp@{sx(&2o6LN@!b7@gYrUI8TczKv#K77#@M!HW+DacRH zNUACG`nc>_5US^!1Se|Esb(W_xwbV2# zFU7pPDk;k-Dc8u@xGX0pE2lEU*u1Q`vZ^GnvcN1YDK)7wDW#$;BfF|32b}J!L49l` zD-dZ_kdtg>o|9LeY*dwLY+g~AQ<|1umQ|UNY*dJ?vNba@wlp*{Hnv0*T6p?OCZNGm z@OS~im_{2lKyF!8n53H(rh$#Y6WM46;3?7rRNtW7v-81rRJMwmX=neXXR#BC$l2kB&{qt zrP8dxI6ohIOdBU=XI7OK6@jMYF)FIWZ1eP_irm67lgzB#3^OTYd^NP&0B9l~5MTMSWvyF2y3(Cz+%!~69Gct?{jWhFdlC!ZM zQ;!#FcNEw4N)F()xAtE?!`%%sF5+a#x|vN$)Zv@$=%EUBWT zpbE6+rL-usM9E4CZ34i=%EUy^$P91NN;5SsN;a>oGS163Eze0WO)kjHD6c3o%1KN$ zPpv33F09BZG&M=ANJ%y;G%n0bF3v3mO-5j87!>ALRhAc3W@M+O7#rm!nx_~Q8y8n4 z8)HveCb`AMX`oxP!ABgRcNmJ3%StngjdSzM(-KQ_N>Z!L(#$K9(#sN4va?I7ib~Rw ztBR7-OcP5h3Mw*6DnSSHVoO@N<;i8KMrOul8C6+{r5S03CAqoDCHYu~EiDa<3@wa^ zYVDSp7#md;LT=e6SQj8C4y2?7GNYu-m`az0m{}Pc=ouN}uk|yG%(9Y<%0V~&7G&iZ zr5I=DV zGAqt2&n+&_OEN0SFRMt-Da2aq7ntQ&Wv7>ASEVFXW>pzg=4E6TrQ~5f&BM&V(9+P@ z+!8u&2I_DngT}%VlaupHA=d>YCYNNEr7BsK6qTmxfLazv?F&3M6{i*|Ss7XyTN)V} zm>7f1&&&n&CegB`v6Ydbo-t@%4$GC}g*o}f1*KJ0C1oikCAno~6{h7zRk_LFaXa); zAyXqWLn9L-$XEgD`5%d8W@Sbx8Rb>xsU|sPCB~H|#>J`G$r)v76~<+GWkvRk=kOl__~ym079f#z`iLxkWiesV2raj{ztu%g9g4HiPb^ z#6E`Z=;`UH@L~RpwdfeL{a=xmoKakmn3R8gmX(Hmk!edUL`YbO)KC zF>yy=8y8h(8d12>*0`uL*{lq_35fDa#>~plQqK_o+Mf*5#H1>-io*1g!lVpS^ThO= ze6!s0%#!@F!jiQ7B-8RNbJLWna+9pef`Y8XQj-!BTqlDW8JU+Ql_#3ynB*GgB~@l+ z6{lC`l;z_*0w6Qls4N*=DPuGz(#ujSD+@BpQ!275({jslORCaJOU;VY5;KyGjZ&)e z(~EMmit@^nQ<5{wbBdFVjEgd`mIV3Pg#~$)#w8h<$;O#kW`#+q#mUB*Mp!S1vNSa| zG9_|3aaCeQL3SeY+8Vi|FV zuHdRQjY>?)($c_Y5DG!U4KhtGsg(LAB)^hmQi|1a&`&W3~I!*v8kScA$T7Z&X~?MDk(_ID9JUdOfk(e zuPQ4qDmAG{tSZf_N~_4MtVm5yDoroRG)c`Z2OVIPQC19II*O$wOzMirqLQSnv^+DT zbj)={X-TD~Nk*C3<(Y{Ui76Gur`KhKQI2NxNC#9$7n5Pt)7=f0Zp*6zON(z#*Q*(3k(@aZ^ zO>;8R3Mxv{lT)%yOOuSMax=;kD~<9~QjLm|a!t$gip)(?(knna`!V+9XXcof8WmO) znq?N|Cgta5r&m_w73G(KatV4OGcz_eHZ(RPcqT7sPkw1dT6(TI=;Q;$=nc*xXSDGY zq;@#Sn9O8|F?iA$ngMuM+U1FA5mStX* zR#lpmm5tHxNl8g8D9FvKG)^(kF3c;+si-hdD^JWQNij3ZD@--7D9<)BORvf=&o56h zEi2B<&CkrnI*vlZ&B*4429}0q#zw>x=A~5ypzCG8C#n1rQemliVM$eqQA%==k!fCWN=0Q^mPuu0MoxxVerZy9VrE)SLAj}MUPeJq zY9`iLFfGi_M%l$?sU?M}xtT?|CPtNIg=RTspx&WbNm@x+MSf0t znR!`a5~#+v6EAmX=0F1{TDeNMB}bW}awPY6M{Rf&nIX5ii!dhuvsXlh|- zY(!M?SY}d^Sz($5-iJ&ugMwRI@RisIm*;{^DNHT`n}VlwMAeTcdzyjTtY${2OBB%) zYFSlTT1G}@PMJw+W^txjd0AdgRZ(G*Nmh1Z2536I$~Yq{-^?T>r7AO}$ke3LtQY?NGLo|%|cmS0#|l97ljp%$hlW@V=oU^}2D&A1>ZHL=t< zqcpKJyC|zXHKoY3Br)5pD9a=_&nVL@BO^H{r=Y5=psXU_I5)8{yPyi|6n0K(nTc_p zS#Dm5k%>uOa&l@$eok&?BG&z=MxdPuCdBPrH!{N*{WAqGWitc!58#CYya+5YFU~DZ z%q&byEh(=^Nvt$8F3n0$&n&^Hh%%Cj(~a^?j4QL#QnFI>%5zMzOia_wlJY9eO4IXG z(^HI-(^4~%jf(QJ(zA1_jFZYzu{HzBDw7LJ%2G^?(kqgat5S;-Q;khBOpUS5Je!yq z8yZ^@cYRhxQBiVIiV-*jAz3sTXEOktPNDq*q*Pv7Q3P5U0XBnBDu?RElg3S~3{3UR zEb*rC^s1EPB-7l&%(A44#L8roiqibl%;NH_tZbvIid?hO{EV#9g6z_&!rV#|le`q8 zirfqwGtc>ng@u`gW|>(gc_~$8xfx}~`Bgc^skkm_$|=ptGcGeX!nWr+sWLseBsI;f zFe$%0t=POIH6tT8*{q_dAg3a)u(Bd0E3K@cG{e{&RGb+nrxll{nq%!hnpEberePRu~tjo9E>x78K@G6jv1G8|S7~RZ>A-Q4yAeWoBk!W@uz#M%0SA$_!)U^fDu`IhEk`KLm;p=->=;5mK39RF;zl zHiJ;Yg6hUwgczCYSz3SxCa7@=Pe@T>S!!`H_*PwPw-1BHs*#T1F|wfS3?ARayv#(G z)Uf>I!>e!{zGDbF-44l0L-5&uGmZ013iES8Ep7DHo|%P-rJ=EzAu$cr z(#mpZ{x?oFF3rl%GA~Kb&#E#?$tcb?HcvLsEGR5XFUv7aEK4;tugu6V$jwhGOE*c% zFflCxb^S3q?NvFZMg=+Lm8B(lg-N+-*(E9E*(O;j;4FeR4PkC%Vrgh%U_wl{vCPP% zu-q&IJbqU~R91u5w@6tHWJYlk#0)}N4XPVYRx`CSGSM@!1TXl9x029Wq#2;QbF)ng zDhji5Q*(;Tb4?11%_@o$%PLFDs&dT>D>4exD>GA5(ks)_Qp`$I^HY${phMYPNa_(U z#rc_~DY>Q4$rO}@7OBQ5#aT%vDFsHw<@t%3=4ScjSvg51DP_jR$%Sd4{aWQE1;!>- zIYsG-WhrT8MalVCyQVpn>B+g7*;&S!+39)d#>Q#sS!r3tX`m7iJw=+ESy~zySQ2%a zSDA4}VQD6$rX@S3jWY@hb0B6=9@F5h9p(fs$0*A)%}P$sD#%SQH%duMOe-%*OEk$! zNy^E{%*;1QPAbW$OixZNFfUI_E6q1e0?pB3-~XOnnp##^l$}wWS6G^ql4M+!XWyU#`S!oqU;6VwZ@+Q_Eqj64UYE}W*6hd(Z*N-QE zf=|FQGQ&G^n`~5+oK|3BTA6BER+Uy^T5Ot|n37eQ47%dMG&QxXD7m1@JSnZ*I5{h) zC?%`J*t8h5jtEN|E5)=lH!m&OJl`mL_yK26L` zDK0InEYGYgNi(Y|FwHDX$t+CC%F8z@sWizqOD|4Ot}4$cN;5GrOUcd7PfRJwz*=J% z<)D+%l7t zO!K4yurYX|8_fVb@of&895**Z-LwktwVD)~CFW(DC6%P5R~RK0B_&p76qr?3WMTAL z6Z6Uv%Z*YC^7BjzjPuizlCsi@jnYevGcwAPbJI&p3v$fO%T06g^76`(@=7vuii?b~ zZhXruD^II5D>p7EE-%fg03Du}TT)i zT4h!m*o>l7&}M}6?3}#P^vv8m)7B6-D`#S*Cf(8Of!2 z1?i>P`KI~VnMI&A9~i}mrIERzxuvNgF~x|9X@+rCnvpT6RZgfFK}#3NwY-UGdYM@s z*chrLEHf)(Lp@`2)LT2@3Cjp{m`PP;Np4w1Ze@~ra$;hMQAv(D<`Fs>$>vFUW@*M{ zsb&?WnK@aBRfSdgi6sSDxrIh$m5Jsl*{LO_1xaODNjZt8*%?MdA=Ea#N z<;JOn#yKUah34sHg~d6b+W|2`-`K*^(9+l#vC$h(C)p_5)GQ?pe4jSi3Ck$kq%xxn zYzCg7NA6#n%#z%Eh|0q&6$%coEC$Ud zDWobCAKt7`k_x%KJ3Te81a!t^u0lp)Nvc9#YFWo=KXqNm@xxZdqk{Io5VZW=djZ zQf_vZd2y<7RhD_8k#SC0a#jIo{sX;$GP5u+GPE=>CvLE%*fb}}9Na!6w}2|GEH=$Z zg_uEk0cB!kY^i5vhFUS8^#D`SQZiHWi*n18s)|jDj4D&gvNQ4$i!-W{Qp+6{IF*W~NkRWM?IVPZUBMw@A+@D>hHdFRx5bD@irYEJ)8zO-m}N zs?0J^FDuD5$}a~k$v4Y1%1^DxEi@@FD>cc&I&V>4l9ZU9m6?-SWRh8Ko?2K~Y@TmY zl7lrhm|Gecnwk)|1HH1aGO^s4xXr3)SqZ7JP+D17X|EKr;YOUNiwM%D1q< zkr&I%GIO#`vMMr4^YgRvs{PV$l~%c{zgD$>);vP-JWi^`Jn)ACZ% z!0T(!OF{!9b3mMyICMG7DRVJpJ<(gNR6d7fJ?)fXIFwU+_Hm@kn$}ur7$jK;9GR;XXGEFm2O3o?D zFfB^N-bXj8sLHA;tt?8fGE2$Lsx+!d$;e48PtUD3Zb4dRZgIJBQ3~jwBGV*uvx35sBvY)VZdPiBQF3aI znR$9;VN$tiZca*No_S&tcmf(Nrp-((O%06=Oc4DI=oMgzSs7(!W%-Hel?5p!Y3aGS zMy4sINma%s1=%Gnz04>H*W^-eLB4rbN^T0YAi_SD>sqRinwOlPS5gE%j{|cg0l3gq$Jal5|bjM+_c<`oWd;A(!3M{45g_^E{JGvq~cz119PDMTO>>#<|6rMQQ03MHR_r+1WXHIXI7e zE;KDqE-y95HeixeRG4p8Xp(J~W0Gl}W1N?il2nugI^!ulIVr8E+|<-G*T}3YE2+fD zI3?3OxiCL32Wu9}$*QW%GD<8;EiX<^&dn(_I`b2Y8d3Lrb*bK^J+QiDxM9&1an8IAo)Ch06nVe&uXl|CBTv?Qy zmztfJY?5SZRG5;OT2zpfTa=TSo>*E@l%8KunNggPmXw~8nO108ioM({EHTT@EK1BP zG&8Tt%_`2UOvx@Ut;oT(>oX^(vLe3%di5<@`y@57G^;W*Gc_X}w7M~+w9v#Hbfs2G zR(f%1acZ(zo~dz0MP*`kNqTmPnMq21MWrd$_DNYmQASx#iK$s~ab|95VR=D@d1`tx zc&ie6q?=oSR?!$B?f<~joi;8n%`Gc80?%EO8R=!l<)s-_`Cv0Bk90FDBhX4#{Cm2} ziqZ>nOmj+1OEYtdN-7Fd%QMWgQ_a$|j0!W;vvSG`^FY%VnNgNenXyrJrcqi>QE^6Mnpp*o8Zsx%q_oPYxWp&}v$dLDn3rWrS7>ZvZfIzR6w{fY25V+YYFCiu&N)LgUlobvRHf<#l(^6cbf z^V~cm)BK#uqST_C%G}Zv)BK8T)1=B$)9l>x!nDMsqMRfgD-v>wO3RYWOY+JpO3QMx zjZ4jvjH`+(ii&Zhj%@Rsth|C;=y}Dc`<0TjvJ+Fw@{B91%yY9$(kly73X)1P(^5*) z3i7k_603^xijpf+$_mVq&5SC`j7!YRv0v0vTxObHl3tpUnv{~1Q*4x$Zd6#6S(t%! z-qO_A1hm+gsI@0Xpv`Fo;L%9pTFTH#exy!HnNdZGNeXCz1mSJNP~CWXD5h4Xpo_GP zL3vNH0_GtA6#)6|ytd<))@&ChC=>=H(ZGuNNyQsPHNe^3yG-$ai#fbad8F zDN0SuO-?LH&PXlNMdc*sX6mJ9rrBp?*czIe$CMfv7@3uq+Zut7^+Q?^k(#Fz3%^!3 zr?LoS7zn2pft$|xc_me4kX=qh)l=}fHlvA$E`>+y z%_Zd*=B4CklvNld7v|(ws5DM4ElJJI%S%crOHIteQ5xlz86}mN=7L(b7<CDk#cN z%gh0V95lXho>hcXV`5oiNn#PG(o}&Kfk`>}pyiuIscB$Q@SS&7N=b=n*+r?D8HJf? zmAS@gMp=oeX_Yy7M(L^9MHTrOX(ojwCMMY`r6k9+)F`dQ$fP9GJlQNYH8D9SCpRxCBP+SIq$n$~Bqi0v z1l0FWPc%16&MnExOCq9xF)}wXGcW_4)l`(4Sdxl-1&@WUk!6UXrImphXn~0_WJm+` z%Ee6eoE(MX)KrDa{8EL?JcY!({F02+B88mHv{Z%ijMO}Z@>GSyqEv;X{E`fXBsHxdxhko&Dy68h%q-nJBR{7w%Oo?kv^XczI3u;f z7{};`d3sTPMW$(5dYVa5Ri$}(rnz}#Zdn1&eGa7==EjL>*+y6nwof!OHYq4gOEfPv zDK@W2&Pz4UF)lAJGOMg8FG_jG zD~pRWi@|0Psyv8^X=6)0Q^*QPT+6aj%t}&BOU=tGtEx&0iVDh2b1L&IbMp!_($h<_ z&2lpG%2JZEP4ZI;vWkq-^0RXi3o<|jAf)O>S(a6jmu*&1S!QCAnU`*sWSX0tm|>P` zmWJz0tK5=Q^CF|dbR$#DZcbWZWs*r!R(fGhc2aJYky%zoaRKPGg{)+=?3|3!vZQ?D zf*hlqG?S9#g0!l%;~tQ;Z7=Q_M5cD@(9c9cCs5riRAF zsbx5;x{{nCqmoQhqY_N_CgogA<`!nghNk8w z#7q~J8CRqfC#4#J7dMev27%1TO)CYPL3zpqcfif@mq8iXrj^-Q=0$16xrvD-iRGDz z#py=LX6cow$tGEaDWz4J`6b55S!rek<&{RoRmoY!#yGlIq#jY5QBa;$Zkl0~hZ)nU zdHEIPMI}W>Sthyp`DrO-g&8KLCPw9XxkiOK<2nWk7{ zIyX75$|yG}(KMqZCAYBHyr>v-C`}o71Qe}NV`gY+Xk=t&ju@iGJ=9WKUQwJvX>C(p zQEW=J+QtYp)M8$#^fU zgzLxCL^1)bpEEYWAKQ6}#)a9H*%f6O#+6xF`4#zD8Rf>Q#)awW$t9+#W`!n6Rc2+W zWw|-|$;qWfY2{{VIGRXVCFVvIRVCRurN${n$r)uiIYt?oRYhjF`dld$WhEJAIap`V z5{pYr(sGMUt165OG7EByOY+Q$Q%qBh%96~}3eC!@Dl&4?(m;FpDocy2s?v?K%dw6R znx|!z=4MnGC6=0}8s%1Im}lpjW~3EhJL%Ef5VYOgg1FeOD#|Y~PXq6SB)f@JRg|9! z>bl|I?Tp;O!V}Y`R)%JJ28OUPf9Uw1iMgJo3Eml^JE`mFJ~nJN+>!tE9BBFfGHpBqJp=uc|a9sl>D_J=M50GdsW1Jh!wY zx1_Kjt1{iBFr_rjD8H~Gvk>cItMb%bBV*I#%&N?a!rY|f@(j>zX`n;OQ;I8!igGGTl8p<^)6=VR(u;~sOLJ3-(kpS~bMu^ZqufH%w8FBI z;)3LqQlrA`qO8&!oHONF<%!0{rRhc$n5BAhK}AYIaz;){R&H8hc~x;~l5u`{QK?aK zig9^WK}u?Jrg>6wfpI}@YL#hnMq#3926#FdV~0wmS$=9-c4|p!W?7keR!U-eaao#C zT46bs2C}&Y=x{ev1ELOyG_I;l%t!{G#!6gFL&s{7%Jeehs!Ee=h#7=p8mb#lJ~y*6 zvd}ZL0Iz_@KBaDCppcPoU}S=H_m45C?P6$&WAm?Rc}cR7ab|W(3TVG_N};()S$d{< z1?J}8q%$(#)T)AvjP&y2N^{f1 z)bv!Wg>rI|Q9(*-c5X^)RfV}}aamGXiCKD95@_@YqY5@KGcg1$AtGX}acOx@aaDdU zc-I}l$U%;eIPoRVzQw4AIoqr#-B ze6!ptv!b%Xw5lwuqXwzv1^HP;M&NrWD|3pCjZ$*6iz`e)EjaYhH?=f1H?**@ByQ9o zDL19C1TxM#p>J$uV6JCsh&t>H5B+pwW25vG)8dlsiqeYI zRCCZlv-v5-$x5JMyiBC#XG)=2SwT@wcD}iBd0w)yX<}(snMrP0PHAaYUUFtpS#nVh zXoAipBR8wqxU{^YG9?jf=$jdvR+wd(=B1bAWF{Mzm?oJe8Jkz-S70lsOhH=+OpS3>wp~NG>$XFG)!^ zGDGq5lRv zCVWcE^9l>nvcSjc5iA>!n_MVKGp{f!s|sueo}`Ja8&9cV3@XTt@ZW=xSX7djQJ9xf zQks`pno(L-k)2UoV4PW*QdVx7lapAUnqHP^RGw0hT4SQB_7& zs##%Ca(b0%im6d+~D!a-kJFzU^$fP*4*tj^u zyfP&_**q%=Yb{`5XbIZdG9D8CmIOgl=pmGN#R}OfB`y zEm4<`qK{LXBv+K@n&hSCXXlm`Wu;`48s#RLRG22EmKA0gm82JBmKBzyXO`zARTP2l z-7`-IpTLJDrVDd(%rdhe`+t**iYkoKva0fP%knd-DwBZSicuQ))DYqd7-(A$sel2Q zVP<9uHiJ+B1J#YEfH4DAd`8CL-XHdsVo0F7LPlmDXtiZ>W`15`jzUVJf^&Y(;VrpI zpwj`+R%a!pq^ISVn3bg@7N-~)m6=zXn--ar7iVPW8|RivewC1#eGm1ZSX zrYBYvR$|>_oLF30R9R}2S!ixvo|J1=k(iNMU{+Cv^-NB43j;$#V`E}gWt3JHRHf#Z z7=hdTgc=~oSr;h=KxUYgq=3yJ6a&Qc%8bqRjLq=&%91KG$|@=gi}K7%%9FE=3rlkH zb8@RHD)W+4j4M-9bIOd;vU9SEO-;;F($b1D%~H}*K{I(+GJbYNMNUywxlw6)QigE> zWHl1lo1mnkZ=3@qztr2uvSM2>%|}$85J2O6{bcN8K!x8#hFQF~uf7|2B;o_Rr;`51*sT28TP zj(N7RaanOmc3MtJc9~hSnQ2i~YFTPwR+*V`d5KwNW^tBDRYqy5aZz?kVs19pg@#p$ zrpZNx#ii!iiRG!;xn>1r#yM%}N!U&rGqp4aEyXt@YA~@pCDXJh-v~TRMO0FT4+bJ9 z=JJ$`igGisDTER;TtA)y#l*_MM9t=o0^rBQdp6jQc_TymX}kWQJ8B~n3`Fbnq_QUXp&`=U6zuUf_1K=AU`*) zFh8#(sjAqtG&QL#xvV&^C_M|?E=@B_BMUaVGc8O^1RwrRT-5_kQ2Xuvml8#bhaY1TwNf79AGmsG(shQ~+B}!IC2B3R4$}>|+GL)yP%UR9NGMpdR zG&ip(%gX~Ta>~UzHd9bhRZwPHT$)^3Ra%*qQkk5domG^c0cww;kIk4Hnwc0Hni&uk z(^aMUDHSH*O-01TG_=5kr%;pJ(yG#YbE8tQ8Ab5S3@e9FV;ZU(YfKx27=rE!v@$i+ NGcW*;mlJ#98UX)HUhe<^ diff --git a/consumer/src/sources/mod.rs b/consumer/src/sources/mod.rs index e3572d43..9f3a410e 100644 --- a/consumer/src/sources/mod.rs +++ b/consumer/src/sources/mod.rs @@ -1,11 +1,6 @@ //! Event source clients //! //! This module contains clients for external event sources: -//! - Jetstream (Bluesky's firehose) -//! - Unified consumer (multi-partition coordinator) +//! - Tap (Bluesky's new synchronization utility) -pub mod jetstream; -pub mod unified_consumer; - -// JetstreamConsumer re-export removed - use sources::jetstream::JetstreamConsumer -pub use unified_consumer::{Partition, UnifiedConsumer}; +pub mod tap; diff --git a/consumer/src/sources/tap/consumer.rs b/consumer/src/sources/tap/consumer.rs new file mode 100644 index 00000000..e7590467 --- /dev/null +++ b/consumer/src/sources/tap/consumer.rs @@ -0,0 +1,258 @@ +//! WebSocket consumer for Tap events +//! +//! This module implements a WebSocket client that: +//! - Connects to a Tap instance +//! - Receives JSON events +//! - Sends acknowledgments for processed events +//! - Handles reconnection on failures + +use super::types::TapEvent; +use eyre::Result; +use futures_util::{StreamExt, SinkExt}; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Mutex; +use tokio::time::sleep; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; + +/// Configuration for Tap consumer +#[derive(Debug, Clone)] +pub struct TapConfig { + /// WebSocket URL for Tap channel (e.g., "ws://localhost:2480/channel") + pub websocket_url: String, + /// Admin API URL for Tap (e.g., "http://localhost:2480") + pub admin_url: String, + /// Admin password for authentication (if required) + pub admin_password: Option, + /// Maximum number of unacknowledged events to buffer + pub max_pending_acks: usize, + /// Reconnection backoff configuration + pub reconnect_backoff_ms: u64, + pub reconnect_max_backoff_ms: u64, +} + +impl Default for TapConfig { + fn default() -> Self { + Self { + websocket_url: "ws://localhost:2480/channel".to_string(), + admin_url: "http://localhost:2480".to_string(), + admin_password: None, + max_pending_acks: 1000, + reconnect_backoff_ms: 1000, + reconnect_max_backoff_ms: 60000, + } + } +} + +/// Internal state for TapConsumer +struct TapConsumerInner { + /// WebSocket connection + ws: WebSocketStream>, + /// Configuration + config: TapConfig, + /// Queue of event IDs pending acknowledgment + pending_acks: VecDeque, + /// Current reconnection backoff + current_backoff_ms: u64, +} + +/// WebSocket consumer for Tap events with acknowledgment support +#[derive(Clone)] +pub struct TapConsumer { + inner: Arc>, + config: TapConfig, +} + +impl TapConsumer { + /// Connect to Tap WebSocket + pub async fn connect(config: TapConfig) -> Result { + let (ws, _) = connect_async(&config.websocket_url).await?; + tracing::info!("Connected to Tap at {}", config.websocket_url); + + let inner = TapConsumerInner { + ws, + config: config.clone(), + pending_acks: VecDeque::new(), + current_backoff_ms: config.reconnect_backoff_ms, + }; + + Ok(Self { + inner: Arc::new(Mutex::new(inner)), + config, + }) + } + + /// Receive the next event from Tap + pub async fn next_event(&mut self) -> Result { + loop { + let mut inner = self.inner.lock().await; + + match inner.ws.next().await { + Some(Ok(Message::Text(text))) => { + // Parse the JSON event + let event: TapEvent = serde_json::from_str(&text)?; + + // Track event ID for acknowledgment + inner.pending_acks.push_back(event.id); + + // Reset backoff on successful receive + inner.current_backoff_ms = inner.config.reconnect_backoff_ms; + + return Ok(event); + } + Some(Ok(Message::Close(_))) => { + tracing::warn!("WebSocket closed by server, reconnecting..."); + Self::handle_reconnection(&mut inner).await?; + } + Some(Ok(Message::Ping(data))) => { + // Respond to ping with pong + inner.ws.send(Message::Pong(data)).await?; + } + Some(Ok(_)) => { + // Ignore other message types (binary, pong) + continue; + } + Some(Err(e)) => { + tracing::error!("WebSocket error: {}", e); + Self::handle_reconnection(&mut inner).await?; + } + None => { + tracing::error!("WebSocket stream ended, reconnecting..."); + Self::handle_reconnection(&mut inner).await?; + } + } + } + } + + /// Send acknowledgment for a processed event + pub async fn acknowledge(&mut self, event_id: u64) -> Result<()> { + let mut inner = self.inner.lock().await; + + // Remove from pending queue + if let Some(pos) = inner.pending_acks.iter().position(|&id| id == event_id) { + inner.pending_acks.remove(pos); + } + + // Send ack message to Tap + let ack_msg = serde_json::json!({ + "type": "ack", + "id": event_id, + }); + + inner.ws.send(Message::Text(ack_msg.to_string())).await?; + Ok(()) + } + + /// Acknowledge all events up to and including the specified ID + pub async fn acknowledge_up_to(&mut self, event_id: u64) -> Result<()> { + let mut inner = self.inner.lock().await; + + // Find all events up to this ID + let mut to_ack = Vec::new(); + while let Some(&id) = inner.pending_acks.front() { + if id <= event_id { + to_ack.push(inner.pending_acks.pop_front().unwrap()); + } else { + break; + } + } + + // Send acks for all + for id in to_ack { + let ack_msg = serde_json::json!({ + "type": "ack", + "id": id, + }); + inner.ws.send(Message::Text(ack_msg.to_string())).await?; + } + + Ok(()) + } + + /// Handle reconnection with exponential backoff + async fn handle_reconnection(inner: &mut TapConsumerInner) -> Result<()> { + loop { + // Wait with backoff + sleep(Duration::from_millis(inner.current_backoff_ms)).await; + + // Try to reconnect + match connect_async(&inner.config.websocket_url).await { + Ok((ws, _)) => { + inner.ws = ws; + tracing::info!("Reconnected to Tap"); + + // Clear pending acks as Tap will resend unacked events + inner.pending_acks.clear(); + + return Ok(()); + } + Err(e) => { + tracing::error!("Failed to reconnect: {}", e); + + // Increase backoff up to maximum + inner.current_backoff_ms = std::cmp::min( + inner.current_backoff_ms * 2, + inner.config.reconnect_max_backoff_ms + ); + } + } + } + } + + /// Add DIDs to track via the admin API + pub async fn add_dids(&self, dids: Vec) -> Result<()> { + let client = reqwest::Client::new(); + let mut request = client + .post(format!("{}/repos/add", self.config.admin_url)) + .json(&serde_json::json!({ "dids": dids })); + + // Add basic auth if password is configured + if let Some(ref password) = self.config.admin_password { + request = request.basic_auth("admin", Some(password)); + } + + let response = request.send().await?; + if !response.status().is_success() { + return Err(eyre::eyre!("Failed to add DIDs: {}", response.status())); + } + + tracing::info!("Added {} DIDs to Tap tracking", dids.len()); + Ok(()) + } + + /// Remove DIDs from tracking via the admin API + pub async fn remove_dids(&self, dids: Vec) -> Result<()> { + let client = reqwest::Client::new(); + let mut request = client + .post(format!("{}/repos/remove", self.config.admin_url)) + .json(&serde_json::json!({ "dids": dids })); + + // Add basic auth if password is configured + if let Some(ref password) = self.config.admin_password { + request = request.basic_auth("admin", Some(password)); + } + + let response = request.send().await?; + if !response.status().is_success() { + return Err(eyre::eyre!("Failed to remove DIDs: {}", response.status())); + } + + tracing::info!("Removed {} DIDs from Tap tracking", dids.len()); + Ok(()) + } + + /// Get Tap health status + pub async fn health_check(&self) -> Result { + let client = reqwest::Client::new(); + let mut request = client.get(format!("{}/health", self.config.admin_url)); + + // Add basic auth if password is configured + if let Some(ref password) = self.config.admin_password { + request = request.basic_auth("admin", Some(password)); + } + + let response = request.send().await?; + Ok(response.status().is_success()) + } +} \ No newline at end of file diff --git a/consumer/src/sources/tap/mod.rs b/consumer/src/sources/tap/mod.rs new file mode 100644 index 00000000..48fd7df7 --- /dev/null +++ b/consumer/src/sources/tap/mod.rs @@ -0,0 +1,11 @@ +//! Tap client for AT Protocol sync +//! +//! Tap is a synchronization utility that handles firehose connection, verification, +//! backfill, and filtering. This module provides a WebSocket client that connects +//! to a Tap instance and receives filtered events. + +pub mod consumer; +pub mod types; + +pub use consumer::TapConsumer; +pub use types::{TapEvent, RecordEvent, IdentityEvent, RecordAction}; \ No newline at end of file diff --git a/consumer/src/sources/tap/types.rs b/consumer/src/sources/tap/types.rs new file mode 100644 index 00000000..82e939f8 --- /dev/null +++ b/consumer/src/sources/tap/types.rs @@ -0,0 +1,102 @@ +//! Tap event type definitions +//! +//! These types match the JSON format that Tap delivers over WebSocket. + +use serde::{Deserialize, Serialize}; + +/// Main event envelope from Tap +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TapEvent { + /// Unique event ID for acknowledgment + pub id: u64, + /// Event type discriminator + #[serde(rename = "type")] + pub event_type: String, + /// Record event data (when type = "record") + #[serde(skip_serializing_if = "Option::is_none")] + pub record: Option, + /// Identity event data (when type = "identity") + #[serde(skip_serializing_if = "Option::is_none")] + pub identity: Option, +} + +/// Record event (create, update, delete) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct RecordEvent { + /// Whether this is a live event (true) or historical backfill (false) + pub live: bool, + /// Repository revision + pub rev: String, + /// DID of the repository + pub did: String, + /// AT Protocol collection (e.g., "app.bsky.feed.post") + pub collection: String, + /// Record key + pub rkey: String, + /// Action type: "create", "update", or "delete" + pub action: String, + /// Content ID (CID) of the record + #[serde(skip_serializing_if = "Option::is_none")] + pub cid: Option, + /// Record data (JSON value) + #[serde(skip_serializing_if = "Option::is_none")] + pub record: Option, +} + +/// Identity event (handle or status changes) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct IdentityEvent { + /// DID of the identity + pub did: String, + /// Current handle + #[serde(skip_serializing_if = "Option::is_none")] + pub handle: Option, + /// Whether the identity is active + #[serde(rename = "isActive")] + pub is_active: bool, + /// Status of the identity + pub status: String, +} + +/// Record action types +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordAction { + Create, + Update, + Delete, +} + +impl RecordAction { + /// Parse from string + pub fn from_str(s: &str) -> Option { + match s { + "create" => Some(Self::Create), + "update" => Some(Self::Update), + "delete" => Some(Self::Delete), + _ => None, + } + } +} + +impl TapEvent { + /// Check if this is a record event + pub fn is_record(&self) -> bool { + self.event_type == "record" && self.record.is_some() + } + + /// Check if this is an identity event + pub fn is_identity(&self) -> bool { + self.event_type == "identity" && self.identity.is_some() + } + + /// Extract the DID from the event + pub fn did(&self) -> Option<&str> { + if let Some(ref record) = self.record { + Some(&record.did) + } else if let Some(ref identity) = self.identity { + Some(&identity.did) + } else { + None + } + } +} \ No newline at end of file diff --git a/consumer/src/sources/unified_consumer.rs b/consumer/src/sources/unified_consumer.rs deleted file mode 100644 index 0a598d22..00000000 --- a/consumer/src/sources/unified_consumer.rs +++ /dev/null @@ -1,256 +0,0 @@ -//! Wrapper around `JetstreamConsumer` for the indexer -//! -//! Uses a single Jetstream connection that receives all events, -//! then classifies them into logical partitions for tracking: -//! 1. Likes -//! 2. Posts -//! 3. Reposts -//! 4. Everything else (social) - -use eyre::OptionExt as _; -use std::fmt::Debug; -use std::sync::{Arc, RwLock}; - -use crate::sources::jetstream::{JetstreamConsumer, RawJetstreamMessage}; - -/// Jetstream partition identifier -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Partition { - Likes, - Posts, - Reposts, - Social, -} - -impl Partition { - /// Get the partition name as a string slice - pub const fn as_str(&self) -> &'static str { - match self { - Self::Likes => "likes", - Self::Posts => "posts", - Self::Reposts => "reposts", - Self::Social => "social", - } - } - - /// All partitions in order - pub const fn all() -> [Partition; 4] { - [Self::Likes, Self::Posts, Self::Reposts, Self::Social] - } -} - -/// Single-connection Jetstream consumer with unified cursor tracking -pub struct UnifiedConsumer { - /// Single underlying Jetstream connection - consumer: JetstreamConsumer, - /// Unified cursor tracking (shared with database writer) - cursor: Arc>, -} - -impl UnifiedConsumer { - /// Classify a collection string into its partition - fn classify_collection(collection: &str) -> Partition { - match collection { - "app.bsky.feed.like" => Partition::Likes, - "app.bsky.feed.post" => Partition::Posts, - "app.bsky.feed.repost" => Partition::Reposts, - _ => Partition::Social, - } - } - - /// Create a new single-connection Jetstream consumer with unified cursor tracking - /// - /// # Arguments - /// * `config` - Indexer configuration - /// * `user_agent` - User agent string - /// * `start_cursor` - Optional cursor value to start from - /// * `cursor_arc` - Optional Arc to share with database writer for cursor tracking - /// If provided, this Arc will be used instead of creating a new one - pub async fn new( - config: &crate::config::IndexerConfig, - user_agent: &str, - start_cursor: Option, - cursor_arc: Option>>, - ) -> eyre::Result { - let jetstream_url = config - .jetstream_source - .as_ref() - .ok_or_eyre("Jetstream source URL must be provided")?; - - // Rewind cursor by 5 seconds for gapless playback (cursor is in microseconds) - const REWIND_SECONDS: u64 = 5; - const REWIND_MICROSECONDS: u64 = REWIND_SECONDS * 1_000_000; - - // Determine starting cursor with rewind for gapless playback - let rewound_cursor = if let Some(cursor) = start_cursor { - let rewound = cursor.saturating_sub(REWIND_MICROSECONDS); - tracing::info!( - "Using cursor with {} second rewind: {} (from {})", - REWIND_SECONDS, - rewound, - cursor - ); - Some(rewound) - } else if let Some(start) = config.start_timestamp { - let rewound = start.saturating_sub(REWIND_MICROSECONDS); - tracing::info!("Using start timestamp with {} second rewind: {}", REWIND_SECONDS, rewound); - Some(rewound) - } else { - tracing::info!("Starting from beginning (no cursor or timestamp provided)"); - None - }; - - // Initialize cursor tracking - // If an Arc was provided (shared with database writer), use it; otherwise create a new one - let cursor_arc = if let Some(arc) = cursor_arc { - // Update the shared Arc with our loaded cursor - if let Some(cursor_val) = start_cursor { - if let Ok(mut c) = arc.write() { - *c = cursor_val; - } - } - arc - } else { - Arc::new(RwLock::new(start_cursor.unwrap_or(0))) - }; - - // Create subscriber options - // wantedDids is populated from allowlist at startup for server-side filtering - let options = Some(crate::sources::jetstream::SubscriberOptions { - wantedCollections: None, // Receive all collections - wantedDids: if config.jetstream_wanted_dids.is_empty() { - None - } else { - Some(config.jetstream_wanted_dids.clone()) - }, - maxMessageSizeBytes: if config.jetstream_max_message_size == 0 { - None - } else { - Some(config.jetstream_max_message_size as i32) - }, - }); - - tracing::info!( - "Creating single Jetstream connection to: {}", - jetstream_url - ); - - // Create the single Jetstream consumer - let consumer = JetstreamConsumer::new( - jetstream_url, - rewound_cursor, - user_agent, - options, - config.jetstream_use_compression, - ) - .await?; - - tracing::info!("Connected to Jetstream at {}", jetstream_url); - - Ok(Self { - consumer, - cursor: cursor_arc, - }) - } - - /// Reconnect to the Jetstream server - pub async fn reconnect(&mut self, url: &str, user_agent: &str) -> eyre::Result<()> { - tracing::info!("Reconnecting to Jetstream..."); - - // Reconnect the underlying consumer (it maintains its own cursor) - self.consumer.reconnect( - url, - user_agent, - self.consumer.current_options(), - true, // use_compression (matches config from new()) - ).await?; - - tracing::info!("Reconnected successfully"); - Ok(()) - } - - /// Returns the current cursor position - pub fn current_seq(&self) -> u64 { - self.cursor - .read() - .ok() - .map(|c| *c) - .unwrap_or(0) - } - - /// Send an options update to the Jetstream server - /// - /// This allows dynamically updating the subscription options (wantedDids, wantedCollections) - /// without reconnecting. Useful for updating the allowlist at runtime. - pub async fn send_options_update( - &mut self, - options: super::jetstream::types::SubscriberOptions, - ) -> eyre::Result<()> { - self.consumer.send_options_update(options).await - } - - /// Extract collection from a Jetstream message for partition classification - /// This does minimal parsing - just enough to extract the "commit.collection" field - fn extract_collection(msg: &RawJetstreamMessage) -> Option { - match msg { - RawJetstreamMessage::Text { content } => { - // Do a quick regex-like search for "collection":"..." pattern - // This is much faster than full JSON parsing - if let Some(pos) = content.find(r#""collection":""#) { - let start = pos + r#""collection":""#.len(); - if let Some(end_pos) = content[start..].find('"') { - return Some(content[start..start + end_pos].to_string()); - } - } - None - } - RawJetstreamMessage::Binary { data } => { - // Decompress and then extract - if let Ok(decompressed) = JetstreamConsumer::decompress_zstd(data) { - if let Ok(text) = String::from_utf8(decompressed) { - if let Some(pos) = text.find(r#""collection":""#) { - let start = pos + r#""collection":""#.len(); - if let Some(end_pos) = text[start..].find('"') { - return Some(text[start..start + end_pos].to_string()); - } - } - } - } - None - } - RawJetstreamMessage::Close => None, - } - } - - /// Drive with raw message output for parallelized processing - /// Returns (partition, message) tuple - /// - /// Messages are classified into partitions based on their collection type: - /// - app.bsky.feed.like → Likes - /// - app.bsky.feed.post → Posts - /// - app.bsky.feed.repost → Reposts - /// - Everything else → Social - pub async fn drive_raw(&mut self) -> eyre::Result<(Partition, RawJetstreamMessage)> { - // Get the next message from the underlying consumer - let msg = self.consumer.drive_raw().await?; - - // Extract collection to determine partition - let partition = Self::extract_collection(&msg) - .map(|coll| Self::classify_collection(&coll)) - .unwrap_or(Partition::Social); // Default to Social if we can't parse - - // Track which partition this belongs to for debugging - metrics::counter!("jetstream.partition.events", "partition" => partition.as_str()) - .increment(1); - - Ok((partition, msg)) - } -} - -/// Debug implementation for `UnifiedConsumer` -impl Debug for UnifiedConsumer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let cursor = self.current_seq(); - write!(f, "UnifiedConsumer {{ cursor: {} }}", cursor) - } -} diff --git a/consumer/src/workers/backfill/car_processing.rs b/consumer/src/workers/backfill/car_processing.rs deleted file mode 100644 index 7d08313b..00000000 --- a/consumer/src/workers/backfill/car_processing.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! CAR file processing for repository backfills -//! -//! This module handles parsing CAR (Content Addressable aRchive) files from -//! ATP repository exports. It extracts commits, MST nodes, and records. - -use super::types::{CarCommitEntry, CarEntry, CarRecordEntry}; -use crate::relay::types::RecordTypes; -use ipld_core::cid::Cid; -use iroh_car::CarReader; -use std::collections::HashMap; -use tokio::io::BufReader; - -/// Record with metadata extracted from CAR file -#[derive(Debug)] -pub struct RecordWithMetadata { - pub cid: Cid, - pub path: String, // collection/rkey - pub record: RecordTypes, - #[expect(dead_code, reason = "Reserved for future timestamp-based filtering")] - pub created_at: Option, // microseconds since epoch -} - -/// Result of CAR file parsing -#[derive(Debug)] -pub struct CarParseResult { - pub commit: Option, - pub records: Vec, - pub stats: CarStats, -} - -/// Statistics about CAR parsing -#[derive(Debug, Default)] -pub struct CarStats { - pub total_records: usize, - pub record_type_counts: HashMap, - pub unknown_type_counts: HashMap, - pub orphaned_records: usize, - pub parse_failures: usize, -} - -/// Parse a CAR file and extract all records with metadata -pub async fn parse_car_file(car_file: tokio::fs::File) -> eyre::Result { - let mut car_stream = CarReader::new(BufReader::new(car_file)).await?; - - // The root should be the commit block - let root = car_stream.header().roots().first().copied().unwrap(); - - let mut commit = None; - let mut mst_nodes: HashMap = HashMap::new(); - let mut records: HashMap = HashMap::new(); - - // Track parsing statistics - let mut stats = CarStats::default(); - - // Parse all blocks from CAR stream - while let Some((cid, block)) = car_stream.next_block().await? { - let block = match serde_ipld_dagcbor::from_slice::(&block) { - Ok(b) => b, - Err(_e) => { - stats.parse_failures += 1; - continue; - } - }; - - if root == cid { - if let CarEntry::Commit(commit_entry) = block { - commit = Some(commit_entry); - } - continue; - } - - match block { - CarEntry::Commit(_) => { - // Found commit entry outside of root (unusual but not an error) - } - CarEntry::Record(record_entry) => match *record_entry { - CarRecordEntry::Known(record) => { - drop(records.insert(cid, *record)); - } - CarRecordEntry::Other { ty } => { - *stats.unknown_type_counts.entry(ty).or_insert(0) += 1; - } - }, - CarEntry::Mst(mst) => { - // Build MST nodes from entries - let mut out = Vec::with_capacity(mst.e.len()); - - for node in mst.e { - let ks = String::from_utf8_lossy(&node.k); - - let key = if node.p == 0 { - ks.to_string() - } else { - let (_, prev): &(Cid, String) = out.last().unwrap(); - let prefix = &prev[..node.p as usize]; - format!("{prefix}{ks}") - }; - - out.push((node.v, key.clone())); - } - - mst_nodes.extend(out); - } - } - } - - // Pair records with MST nodes to get paths - let mut records_with_metadata: Vec = Vec::new(); - - for (cid, record) in records { - if let Some(path) = mst_nodes.remove(&cid) { - let type_name = get_record_type_name(&record); - *stats - .record_type_counts - .entry(type_name.to_string()) - .or_insert(0) += 1; - - let created_at = get_record_created_at(&record); - records_with_metadata.push(RecordWithMetadata { - cid, - path, - record, - created_at, - }); - } else { - stats.orphaned_records += 1; - } - } - - stats.total_records = records_with_metadata.len(); - - Ok(CarParseResult { - commit, - records: records_with_metadata, - stats, - }) -} - -/// Get a human-readable name for a RecordType -pub fn get_record_type_name(record: &RecordTypes) -> &'static str { - match record { - RecordTypes::AppBskyActorProfile(_) => "app.bsky.actor.profile", - RecordTypes::AppBskyActorStatus(_) => "app.bsky.actor.status", - RecordTypes::AppBskyFeedGenerator(_) => "app.bsky.feed.generator", - RecordTypes::AppBskyFeedLike(_) => "app.bsky.feed.like", - RecordTypes::AppBskyFeedPost(_) => "app.bsky.feed.post", - RecordTypes::AppBskyFeedPostgate(_) => "app.bsky.feed.postgate", - RecordTypes::AppBskyFeedRepost(_) => "app.bsky.feed.repost", - RecordTypes::AppBskyFeedThreadgate(_) => "app.bsky.feed.threadgate", - RecordTypes::AppBskyGraphBlock(_) => "app.bsky.graph.block", - RecordTypes::AppBskyGraphFollow(_) => "app.bsky.graph.follow", - RecordTypes::AppBskyGraphList(_) => "app.bsky.graph.list", - RecordTypes::AppBskyGraphListBlock(_) => "app.bsky.graph.listblock", - RecordTypes::AppBskyGraphListItem(_) => "app.bsky.graph.listitem", - RecordTypes::AppBskyGraphStarterPack(_) => "app.bsky.graph.starterpack", - RecordTypes::AppBskyGraphVerification(_) => "app.bsky.graph.verification", - RecordTypes::AppBskyLabelerService(_) => "app.bsky.labeler.service", - RecordTypes::AppBskyNotificationDeclaration(_) => "app.bsky.notification.declaration", - RecordTypes::ChatBskyActorDeclaration(_) => "chat.bsky.actor.declaration", - RecordTypes::CommunityLexiconBookmark(_) => "community.lexicon.bookmarks.bookmark", - RecordTypes::FmTealAlpaActorStatus(_) => "fm.team.alpa.actor.status", - } -} - -/// Extract createdAt timestamp from a record (if it has one) -/// Returns microseconds since epoch for cursor comparison -pub fn get_record_created_at(record: &RecordTypes) -> Option { - let created_at = match record { - RecordTypes::AppBskyFeedLike(r) => Some(r.created_at), - RecordTypes::AppBskyFeedPost(r) => Some(r.created_at), - RecordTypes::AppBskyFeedRepost(r) => Some(r.created_at), - RecordTypes::AppBskyGraphFollow(r) => Some(r.created_at), - RecordTypes::AppBskyGraphBlock(r) => Some(r.created_at), - RecordTypes::AppBskyGraphListItem(r) => Some(r.created_at), - RecordTypes::AppBskyGraphVerification(r) => Some(r.created_at), - // Records without createdAt - _ => None, - }?; - - // Convert DateTime to microseconds since epoch - Some(created_at.timestamp_micros() as u64) -} diff --git a/consumer/src/workers/backfill/collection_fetch.rs b/consumer/src/workers/backfill/collection_fetch.rs deleted file mode 100644 index 6757f148..00000000 --- a/consumer/src/workers/backfill/collection_fetch.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! Collection-based record fetching for backfill supplemental to CAR files -//! -//! This module fetches specific collections via `com.atproto.repo.listRecords` -//! alongside the main CAR file backfill. This provides redundancy and ensures -//! completeness for critical record types (profiles, follows, generators, etc.). - -use crate::database_writer::{EventSource, ProcessedEvent, WriterEvent}; -use crate::relay::types::RecordTypes; -use crate::workers::fetch::json_types::RecordTypesJson; -use crate::workers::fetch::pds::PdsFetcher; -use deadpool_postgres::Object as PgObject; -use eyre::{Result, WrapErr}; -use std::collections::HashMap; -use tracing::{debug, info, warn}; - -/// Statistics about collection fetching -#[derive(Debug, Default)] -pub struct CollectionFetchStats { - pub total_collections: usize, - pub total_records: usize, - pub records_by_collection: HashMap, -} - -/// Critical collections to fetch alongside CAR files -/// -/// These collections are small but critical for functionality. -/// Fetching them separately ensures completeness even if CAR is partial. -pub const CRITICAL_COLLECTIONS: &[&str] = &[ - "app.bsky.actor.profile", - "app.bsky.feed.generator", - "app.bsky.graph.block", - "app.bsky.graph.follow", - "app.bsky.graph.list", - "app.bsky.graph.listitem", - "app.bsky.graph.starterpack", -]; - -/// Fetch and process critical collections for a repository -/// -/// This supplements the CAR file backfill by ensuring we have complete data -/// for critical collections. Records are streamed to the database writer -/// using the same hot path as CAR processing. -/// -/// # Parameters -/// - `conn`: Database connection (for resolving subject actors) -/// - `pds_fetcher`: PDS fetcher client -/// - `repo`: DID of the repository -/// - `actor_id`: Pre-resolved actor_id for the repo -/// - `event_tx`: Channel to database writer -/// - `is_allowed`: Whether actor is allowlisted (affects recursive fetching) -pub async fn fetch_and_process_collections( - conn: &PgObject, - pds_fetcher: &PdsFetcher, - repo: &str, - actor_id: i32, - event_tx: &tokio::sync::mpsc::Sender, - is_allowed: bool, -) -> Result { - let mut stats = CollectionFetchStats::default(); - - for &collection in CRITICAL_COLLECTIONS { - match fetch_and_process_collection( - conn, - pds_fetcher, - repo, - actor_id, - collection, - event_tx, - is_allowed, - ) - .await - { - Ok(count) => { - stats.total_collections += 1; - stats.total_records += count; - stats - .records_by_collection - .insert(collection.to_string(), count); - - if count > 0 { - debug!( - repo = %repo, - collection = %collection, - count = count, - "Fetched collection records" - ); - } - } - Err(e) => { - // Don't fail the entire backfill if one collection fails - // The CAR file should have already provided this data - warn!( - repo = %repo, - collection = %collection, - error = %e, - "Failed to fetch collection (CAR data should suffice)" - ); - } - } - } - - info!( - repo = %repo, - collections = stats.total_collections, - records = stats.total_records, - "Completed supplemental collection fetch" - ); - - Ok(stats) -} - -/// Fetch and process a single collection -async fn fetch_and_process_collection( - conn: &PgObject, - pds_fetcher: &PdsFetcher, - repo: &str, - actor_id: i32, - collection: &str, - event_tx: &tokio::sync::mpsc::Sender, - _is_allowed: bool, -) -> Result { - // Fetch all records from this collection - let records = pds_fetcher - .list_all_records(repo, collection) - .await - .wrap_err_with(|| format!("Failed to fetch collection {}", collection))?; - - let count = records.len(); - - for record in records { - // Extract at-uri and rkey from the record's URI before consuming the record - // Format: at://did/collection/rkey - let at_uri = extract_at_uri_from_fetched(&record, repo, collection)?; - let rkey = extract_rkey(&at_uri)?; - let cid = record.cid; // Copy CID before consuming record - - // Parse JSON value to RecordTypes using json_types (handles Blob conversion) - let record_type: RecordTypes = serde_json::from_value::(record.value) - .wrap_err("Failed to parse record from JSON")? - .into(); - - // Resolve subject_actor_id if needed (for follows, blocks, verifications, list items) - let subject_actor_id = resolve_subject_actor_id(conn, &record_type).await?; - - // Resolve service_actor_id for feedgen records - let service_actor_id = resolve_service_actor_id(conn, &record_type).await?; - - // Create resolved actor IDs (backfill doesn't need notification actor IDs since is_backfill=true) - let resolved_actor_ids = crate::database_writer::workers::ResolvedActorIds { - subject_actor_id, - parent_author_actor_id: None, - root_author_actor_id: None, - quoted_author_actor_id: None, - mentioned_actor_ids: Vec::new(), - service_actor_id, - via_repost_key: None, - }; - - // Use shared hot path (same as CAR processing and Jetstream!) - // EventSource::Backfill prevents notifications for historical data - let processed = crate::database_writer::operations::process_record_to_operations( - repo, - actor_id, - resolved_actor_ids, - cid, - record_type, - at_uri, - rkey, - EventSource::Backfill, - ); - - // Stream to database writer (same backpressure mechanism as CAR processing) - for operation in processed.operations { - event_tx - .send(WriterEvent::Resolved(Box::new(ProcessedEvent { - operations: vec![operation], - cursor: None, - source: EventSource::Backfill, - }))) - .await - .wrap_err("Failed to send operation to database writer")?; - } - } - - Ok(count) -} - -/// Extract at-uri from a FetchedRecord -fn extract_at_uri_from_fetched( - record: &crate::workers::fetch::types::FetchedRecord, - repo: &str, - collection: &str, -) -> Result { - // FetchedRecord now preserves the URI from listRecords response - record - .uri - .clone() - .ok_or_else(|| eyre::eyre!("Missing URI in FetchedRecord (from {}/{})", repo, collection)) -} - -/// Extract rkey from an at-uri -fn extract_rkey(at_uri: &str) -> Result { - let rkey = at_uri - .strip_prefix("at://") - .and_then(|s| s.split('/').nth(2)) - .ok_or_else(|| eyre::eyre!("Invalid at-uri format: {}", at_uri))?; - - Ok(rkey.to_string()) -} - -/// Resolve subject_actor_id for records that reference other actors -async fn resolve_subject_actor_id( - conn: &PgObject, - record: &RecordTypes, -) -> Result> { - let now = chrono::Utc::now(); - - // Extract references using the database_writer module - let refs = crate::database_writer::extract_references(record); - - // Resolve subject actor if present - let subject_actor_id = if let Some(subject_did) = refs.subject_did { - let actor_id = - crate::db::actor::ensure_actor_id(conn, &subject_did, None, None, now).await?; - Some(actor_id) - } else { - None - }; - - // Resolve all additional referenced actors - for did in refs.additional_dids { - crate::db::actor::ensure_actor_id(conn, &did, None, None, now).await?; - } - - Ok(subject_actor_id) -} - -/// Resolve service_actor_id for feedgen records -async fn resolve_service_actor_id( - conn: &PgObject, - record: &RecordTypes, -) -> Result> { - let now = chrono::Utc::now(); - - // Extract references using the database_writer module - let refs = crate::database_writer::extract_references(record); - - // For FeedGenerator records, the first (and only) additional DID is the service actor - let service_actor_id = if let Some(first_did) = refs.additional_dids.first() { - let actor_id = - crate::db::actor::ensure_actor_id(conn, first_did, None, None, now).await?; - Some(actor_id) - } else { - None - }; - - Ok(service_actor_id) -} diff --git a/consumer/src/workers/backfill/downloader.rs b/consumer/src/workers/backfill/downloader.rs deleted file mode 100644 index 2012a363..00000000 --- a/consumer/src/workers/backfill/downloader.rs +++ /dev/null @@ -1,288 +0,0 @@ -// NOTE: This worker is currently disabled and uses Redis. -// It will be migrated to PostgreSQL in a future update. -// For now, backfills are triggered via the backfill_jobs table directly. - -use super::{worker, DL_DONE_KEY, PDS_SERVICE_ID}; -use crate::db; -use chrono::prelude::*; -use deadpool_postgres::Pool; -use did_resolver::Resolver; -use parakeet_db::types::ActorSyncState; -use redis::aio::MultiplexedConnection; -use redis::AsyncTypedCommands as _; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::sync::watch::Receiver as WatchReceiver; -use tokio::time::Duration; -use tokio_util::task::TaskTracker; -use tracing::warn; - -/// Simple ingestion queue consumer -/// -/// Polls "backfill:ingest" list and automatically enqueues DIDs with proper Redis state. -/// This provides a convenient interface for manual backfill requests: -/// -/// ```bash -/// redis-cli LPUSH backfill:ingest "did:plc:example123" -/// ``` -pub async fn ingestion_queue_consumer(mut redis: MultiplexedConnection, stop: WatchReceiver) { - tracing::info!("Starting backfill ingestion queue consumer (backfill:ingest)"); - - loop { - if stop.has_changed().unwrap_or(true) { - tracing::info!("Stopping ingestion queue consumer"); - break; - } - - // Use blocking pop with 1s timeout for immediate processing + clean shutdown - let did: String = match redis.blpop("backfill:ingest", 1.0).await { - Ok(Some([_key, did])) => did, - Ok(None) => { - // Timeout - check stop signal on next iteration - continue; - } - Err(e) => { - tracing::error!("Failed to pop from backfill:ingest: {e}"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; - } - }; - - tracing::info!(did = %did, "Received DID from ingestion queue"); - - // Push directly to backfill_queue (list) for immediate processing - // Don't use enqueue_job() which uses the sorted set - that would require - // waiting for the retry processor to move it to the list - match redis.rpush::<_, _>("backfill_queue", &did).await { - Ok(_) => { - tracing::info!(did = %did, "Successfully enqueued backfill job"); - } - Err(e) => { - tracing::error!(did = %did, error = %e, "Failed to enqueue backfill job"); - } - } - } -} - -/// Configuration for the downloader orchestrator -pub struct DownloaderConfig { - pub redis: MultiplexedConnection, - pub pool: Pool, - pub resolver: Arc, - pub tmp_dir: PathBuf, - pub concurrency: usize, - pub buffer: usize, - pub tracker: TaskTracker, - pub stop: WatchReceiver, - /// Optional retention cutoff timestamp (for time-based backfill filtering) - pub retention_cutoff: Option>, -} - -/// Main downloader orchestrator -/// -/// Polls the backfill queue, resolves DIDs, checks allowlist, and dispatches -/// download jobs to worker threads. -#[expect(clippy::too_many_lines, reason = "Sequential orchestration logic for job processing")] -pub async fn downloader(config: DownloaderConfig) { - let mut rc = config.redis; - let pool = config.pool; - let resolver = config.resolver; - let tmp_dir = config.tmp_dir; - let concurrency = config.concurrency; - let buffer = config.buffer; - let tracker = config.tracker; - let stop = config.stop; - let retention_cutoff = config.retention_cutoff; - let (tx, rx) = flume::bounded(64); - let conn = pool.get().await.unwrap(); - - // Create allowlist with caching - let allowlist = db::Allowlist::new(); - - // Initialize it with the current data - if let Err(e) = allowlist.initialize(&conn).await { - warn!("Failed to initialize allowlist: {}", e); - // Continue anyway - will refresh on first use - } - - // Spawn the periodic refresh task for allowlist cache updates - let (_refresh_handle, _allowlist_changed_rx) = allowlist.spawn_periodic_refresh(pool.clone()).await; - tracing::info!("Backfill downloader: allowlist cache periodic refresh started (60s interval)"); - - let http = reqwest::Client::new(); - - // Spawn worker threads - for _ in 0..concurrency { - drop(tracker.spawn(worker::download_thread( - rc.clone(), - pool.clone(), - resolver.clone(), - http.clone(), - rx.clone(), - tmp_dir.clone(), - retention_cutoff, - ))); - } - - // No prepared statements needed - we use db::workers::backfill_mark_processing() which - // safely creates actors using get_actor_id() with advisory lock protection - - loop { - if stop.has_changed().unwrap_or(true) { - tracing::info!("stopping downloader"); - break; - } - - let len_result = rc.llen(DL_DONE_KEY).await; - if let Ok(count) = len_result { - if count > buffer { - tracing::info!("waiting due to full buffer"); - tokio::time::sleep(Duration::from_secs(5)).await; - continue; - } - } - - // Use blocking pop with 1s timeout for immediate processing + clean shutdown - let did: String = match rc.blpop("backfill_queue", 1.0).await { - Ok(Some([_key, did])) => did, - Ok(None) => { - // Timeout - check stop signal on next iteration - continue; - } - Err(e) => { - tracing::error!("Failed to pop from backfill queue: {e}"); - tokio::time::sleep(Duration::from_millis(100)).await; - continue; - } - }; - - tracing::info!(did = %did, "Popped from backfill queue, resolving DID and checking allowlist"); - - // Check if already processing to avoid concurrent downloads of the same repo - let status_result = db::actor_get_statuses(&conn, &did).await; - match status_result { - Ok(Some((_, state))) => { - if state == ActorSyncState::Processing { - tracing::debug!(did = %did, "Already processing, skipping duplicate"); - continue; - } - } - Ok(None) => {} - Err(e) => { - tracing::error!(did = %did, error = %e, "Failed to check actor status"); - if let Err(e) = crate::external::redis_backfill::write_status( - &mut rc, - &did, - "failed.status_check", - ) - .await - { - tracing::error!(did = %did, error = ?e, "Failed to mark job as failed"); - } - continue; - } - } - - // Check if the DID is in the allowlist, or add it automatically - let is_allowed = allowlist.cache.contains_did(&did); - if !is_allowed { - // Auto-add the DID to the allowlist - tracing::debug!(did = %did, "Auto-adding to allowlist"); - let add_result = allowlist - .add_did(&conn, &did, Some("Auto-added for backfill")) - .await; - match add_result { - Ok(_) => { - tracing::debug!(did = %did, "Added to allowlist"); - // Continue with backfill since now it's in the allowlist - } - Err(e) => { - tracing::error!(did = %did, error = %e, "Failed to add to allowlist"); - continue; - } - } - } - - let resolve_result = resolver.resolve_did(&did).await; - match resolve_result { - Ok(Some(did_doc)) => { - let service_opt = did_doc.find_service_by_id(PDS_SERVICE_ID); - let Some(service) = service_opt else { - tracing::warn!(did = %did, "DID doc missing PDS service endpoint"); - if let Err(e) = crate::external::redis_backfill::write_status( - &mut rc, - &did, - "failed.no_pds_endpoint", - ) - .await - { - tracing::error!(did = %did, error = ?e, "Failed to mark job as failed"); - } - continue; - }; - let service = service.service_endpoint.clone(); - - // Cache the PDS host for future record fetching (in Redis) - if let Ok(url) = reqwest::Url::parse(&service) { - if let Some(host) = url.host_str() { - if let Err(e) = - crate::external::redis_pds_cache::set_pds_host(&mut rc, &did, host) - .await - { - tracing::warn!(did = %did, pds_host = host, error = %e, "Failed to cache PDS mapping in Redis"); - } - } - } - - // Mark actor as processing using safe function (with advisory lock protection) - if let Err(e) = db::workers::backfill_mark_processing(&conn, &did).await { - tracing::error!(did = %did, error = %e, "Failed to mark actor as processing"); - continue; - } - - let handle = did_doc - .also_known_as - .and_then(|akas| akas.first().map(|v| v[5..].to_owned())); - - tracing::info!(did = %did, pds = %service, "Dispatching CAR download from PDS"); - if let Err(e) = tx.send_async((service, did, handle)).await { - tracing::error!("Failed to send to download worker: {e}"); - } - } - Ok(None) => { - tracing::warn!(did = %did, "DID resolution returned no document"); - if let Err(e) = - db::actor_set_sync_status(&conn, &did, &ActorSyncState::Dirty, Utc::now()).await - { - tracing::error!(did = %did, error = %e, "Failed to set actor to dirty"); - } - if let Err(e) = crate::external::redis_backfill::write_status( - &mut rc, - &did, - "failed.no_did_doc", - ) - .await - { - tracing::error!(did = %did, error = ?e, "Failed to mark job as failed"); - } - } - Err(e) => { - tracing::error!(did = %did, error = %e, "Failed to resolve DID"); - if let Err(e) = - db::actor_set_sync_status(&conn, &did, &ActorSyncState::Dirty, Utc::now()).await - { - tracing::error!(did = %did, error = %e, "Failed to set actor to dirty"); - } - if let Err(e) = crate::external::redis_backfill::write_status( - &mut rc, - &did, - "failed.resolve_error", - ) - .await - { - tracing::error!(did = %did, error = ?e, "Failed to mark job as failed"); - } - } - } - } -} diff --git a/consumer/src/workers/backfill/mod.rs b/consumer/src/workers/backfill/mod.rs deleted file mode 100644 index d86c66d6..00000000 --- a/consumer/src/workers/backfill/mod.rs +++ /dev/null @@ -1,451 +0,0 @@ -use crate::config::BackfillConfig; -use crate::db; -use crate::worker_core::{Worker, WorkerFactory}; -use chrono::prelude::*; -use deadpool_postgres::{Object, Pool}; -use did_resolver::Resolver; -use eyre::Result; -use metrics::counter; -use parakeet_db::types::ActorSyncState; -use reqwest::{Client, StatusCode}; -use std::path::PathBuf; -use std::str::FromStr as _; -use std::sync::Arc; -use tokio::sync::watch::Receiver as WatchReceiver; -use tokio::sync::Semaphore; -use tokio_util::task::TaskTracker; -use tracing::instrument; - -mod car_processing; -mod collection_fetch; -// Note: downloader and worker modules are no longer used after simplifying to single-phase architecture -// mod downloader; // Old two-phase download worker (replaced by integrated download+process) -// mod worker; // Old processing worker (replaced by backfill_actor_streaming) -mod ratelimit; -mod repo; -mod resolve_bulk; -mod types; - -const PDS_SERVICE_ID: &str = "#atproto_pds"; - -#[derive(Clone)] -pub struct BackfillManagerInner { - #[expect(dead_code, reason = "Stored for potential future stats aggregation integration")] - tmp_dir: PathBuf, - resolver: Arc, - retention_cutoff: Option>, - pds_fetcher: crate::workers::fetch::pds::PdsFetcher, -} - -pub struct BackfillManager { - pool: Pool, - #[expect(dead_code, reason = "Stored for potential future use in diagnostics")] - resolver: Arc, - semaphore: Arc, - #[expect(dead_code, reason = "Configuration stored for future extensibility")] - opts: BackfillConfig, - inner: BackfillManagerInner, - event_tx: tokio::sync::mpsc::Sender, - #[expect(dead_code, reason = "Stored for future timestamp-based filtering features")] - retention_cutoff: Option>, -} - -impl BackfillManager { - pub async fn new( - pool: Pool, - resolver: Arc, - opts: BackfillConfig, - event_tx: tokio::sync::mpsc::Sender, - retention_cutoff: Option>, - ) -> eyre::Result { - let semaphore = Arc::new(Semaphore::new(opts.workers as usize)); - - // Create PDS fetcher for supplemental collection fetching - // Uses default config (public Bluesky API + Slingshot) - let fetch_config = crate::workers::fetch::types::RecordFetchConfig::default(); - let pds_fetcher = crate::workers::fetch::pds::PdsFetcher::new(&fetch_config, resolver.clone())?; - - Ok(Self { - pool, - resolver: resolver.clone(), - semaphore, - inner: BackfillManagerInner { - tmp_dir: PathBuf::from_str(&opts.download_tmp_dir)?, - resolver, - retention_cutoff, - pds_fetcher, - }, - opts, - event_tx, - retention_cutoff, - }) - } - - pub async fn run(self, stop: WatchReceiver) -> eyre::Result<()> { - let tracker = TaskTracker::new(); - - // Note: Unlike the Redis-based system which separated download and processing, - // we now use PostgreSQL dequeue to get jobs and handle download+process in one task. - // This simplifies the architecture and eliminates the need for intermediate queues. - - loop { - if stop.has_changed().unwrap_or(true) { - let _ = tracker.close(); - tracing::info!("stopping backfiller"); - break; - } - - // Poll for ready jobs with a short sleep to avoid tight polling - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - - // Check semaphore availability before dequeuing - if self.semaphore.available_permits() == 0 { - continue; // No permits available, wait for workers to finish - } - - // Dequeue next ready job from PostgreSQL - let job_opt = match crate::db::backfill_jobs::dequeue(&self.pool).await { - Ok(job) => job, - Err(e) => { - tracing::error!("Failed to dequeue backfill job: {}", e); - continue; - } - }; - - let Some(job) = job_opt else { - // No jobs available - continue; - }; - - let p = self.semaphore.clone().acquire_owned().await?; - - let mut inner = self.inner.clone(); - let mut conn = self.pool.get().await?; - let event_tx = self.event_tx.clone(); - let pool = self.pool.clone(); - let did = job.did.clone(); - - drop(tracker.spawn(async move { - let _p = p; - - // Job is already marked as 'processing' by dequeue() - // No need for separate start_processing call - tracing::info!(did = &did, "Starting backfill: parsing CAR and streaming to database writer"); - - // Stream all chunks to database writer - // Bounded channel provides automatic backpressure - match backfill_actor_streaming( - &mut conn, - &mut inner, - &did, - &event_tx, - &pool, - ).await { - Ok(()) => { - // All chunks sent successfully - backfill completed - // Update actor sync state - if let Err(e) = crate::db::actor::actor_set_sync_status( - &conn, - &did, - &ActorSyncState::Synced, - Utc::now(), - ).await { - tracing::error!(did = &did, "Failed to update sync state: {e}"); - } - - counter!("backfill_success").increment(1); - - if let Err(e) = crate::db::backfill_jobs::mark_successful(&pool, &did).await { - tracing::error!(did = &did, error = ?e, "Failed to mark backfill as successful"); - } - - tracing::info!(did = &did, "Backfill completed successfully"); - } - Err(e) => { - // Backfill failed (CAR parsing, chunk sending, or other error) - tracing::error!(did = &did, "backfill failed: {e}"); - counter!("backfill_failure").increment(1); - - // Mark as failed with retry tracking - match crate::db::backfill_jobs::mark_failed(&pool, &did, &e.to_string()).await { - Ok(attempts) => { - tracing::info!(did = &did, attempts = attempts, "Backfill marked for retry"); - } - Err(db_err) => { - tracing::error!(did = &did, error = ?db_err, "Failed to update backfill job status"); - } - } - } - } - - // Clean up temp file (best effort - file may not exist for retried backfills) - let remove_result = tokio::fs::remove_file(inner.tmp_dir.join(&did)).await; - if let Err(e) = remove_result { - // File not found is expected for retries where file was already cleaned up - if e.kind() != std::io::ErrorKind::NotFound { - tracing::warn!(did = &did, error = %e, "Failed to clean up temp file"); - } - } - })); - } - - tracker.wait().await; - - Ok(()) - } -} - -/// Process CAR file and stream operations to database writer as records are parsed -/// Operations are sent one at a time with automatic backpressure from bounded channel -/// No buffering - operations flow directly from CAR parsing to database writer -/// Database writer's weighted dispatcher ensures fair interleaving with Jetstream events -#[instrument(skip(conn, inner, event_tx, _pool))] -async fn backfill_actor_streaming( - conn: &mut Object, - inner: &mut BackfillManagerInner, - did: &str, - event_tx: &tokio::sync::mpsc::Sender, - _pool: &Pool, -) -> eyre::Result<()> { - let backfill_start = std::time::Instant::now(); - tracing::info!(did = %did, "Starting CAR download and processing"); - - // Step 1: Resolve DID to get PDS endpoint - let did_doc = inner.resolver.resolve_did(did).await? - .ok_or_else(|| eyre::eyre!("DID resolution returned no document"))?; - - let service = did_doc.find_service_by_id(PDS_SERVICE_ID) - .ok_or_else(|| eyre::eyre!("DID doc missing PDS service endpoint"))?; - - let pds_url = service.service_endpoint.clone(); - tracing::debug!(did = %did, pds = %pds_url, "Resolved PDS endpoint"); - - // Step 2: Download CAR file from PDS - // Check for existing repo_rev for incremental backfill - let since_rev = db::actor_get_repo_rev(conn, did).await.ok().flatten(); - - // Construct URL with appropriate since parameter: - // 1. If we have a repo_rev, use it for commit-based incremental sync - // 2. Otherwise, if retention_cutoff is set, convert to TID for timestamp filter - // 3. Otherwise, request full repo - let url = if let Some(rev) = &since_rev { - format!("{}/xrpc/com.atproto.sync.getRepo?did={}&since={}", pds_url, did, rev) - } else if let Some(cutoff) = inner.retention_cutoff { - // Convert retention cutoff DateTime to TID format for since parameter - // TIDs encode timestamps and work as since filters for getRepo - let since_tid = parakeet_db::tid_util::timestamp_to_tid(cutoff); - format!("{}/xrpc/com.atproto.sync.getRepo?did={}&since={}", pds_url, did, since_tid) - } else { - format!("{}/xrpc/com.atproto.sync.getRepo?did={}", pds_url, did) - }; - - let download_start = std::time::Instant::now(); - - let http_client = reqwest::Client::new(); - let res = http_client.get(&url).send().await?.error_for_status()?; - - // Save CAR file to tmp_dir - let file_path = inner.tmp_dir.join(did); - - // Remove existing file if present to ensure idempotent retries - if file_path.exists() { - tracing::debug!(did = %did, "Removing existing CAR file for retry"); - tokio::fs::remove_file(&file_path).await?; - } - - let mut file = tokio::fs::File::create_new(&file_path).await?; - - use futures::TryStreamExt; - use tokio_util::io::StreamReader; - - let strm = res.bytes_stream().map_err(std::io::Error::other); - let mut reader = StreamReader::new(strm); - - let bytes_written = tokio::io::copy(&mut reader, &mut file).await?; - let download_elapsed = download_start.elapsed(); - - tracing::info!( - did = %did, - bytes = bytes_written, - duration_ms = download_elapsed.as_millis(), - incremental = since_rev.is_some(), - "CAR file downloaded" - ); - - if download_elapsed.as_secs() > 5 { - tracing::warn!( - did = %did, - duration_s = download_elapsed.as_secs(), - bytes_mb = bytes_written / 1_000_000, - "Slow CAR download (>5s)" - ); - } - - // Step 3: Parse CAR file and stream operations directly to database writer - // Operations are sent as each record is processed (no buffering) - // Bounded channel provides natural backpressure - let commit_opt = repo::insert_repo(conn, &inner.tmp_dir, did, &inner.resolver, event_tx, inner.retention_cutoff).await?; - - // Fetch critical collections via listRecords (only if retention is configured) - // When retention isn't configured, CAR file already has everything we need - if inner.retention_cutoff.is_some() { - let collection_start = std::time::Instant::now(); - - // Resolve actor_id for this repo - let actor_id = crate::db::actor::ensure_actor_id( - conn, - did, - None, - None, - chrono::Utc::now(), - ).await?; - - // TODO: Check allowlist if needed for recursive fetching behavior - let is_allowed = false; - - match collection_fetch::fetch_and_process_collections( - conn, - &inner.pds_fetcher, - did, - actor_id, - event_tx, - is_allowed, - ).await { - Ok(stats) => { - let collection_elapsed = collection_start.elapsed(); - tracing::info!( - did = %did, - collections = stats.total_collections, - records = stats.total_records, - duration_ms = collection_elapsed.as_millis(), - "Supplemental collection fetch completed" - ); - } - Err(e) => { - // Don't fail backfill if collection fetch fails - // CAR data should be sufficient - tracing::warn!( - did = %did, - error = %e, - "Collection fetch failed (CAR data should suffice)" - ); - } - } - } - - // Update actor repo state after sending operations - // Database writer will process them asynchronously - if let Some(commit) = commit_opt { - db::actor_set_repo_state(conn, did, &commit.rev, commit.data).await?; - tracing::debug!( - did = %did, - rev = %commit.rev, - "Updated repo state after successful backfill" - ); - } else { - tracing::debug!( - did = %did, - "Skipping repo state update (no commit in CAR file)" - ); - } - - let backfill_elapsed = backfill_start.elapsed(); - tracing::info!( - did = %did, - duration_ms = backfill_elapsed.as_millis(), - "Backfill streaming phase completed" - ); - - if backfill_elapsed.as_secs() > 10 { - tracing::warn!( - did = %did, - duration_s = backfill_elapsed.as_secs(), - "Slow backfill streaming (>10s)" - ); - } - - Ok(()) -} - -// Note: retry_processor() has been removed - PostgreSQL handles retries automatically -// The backfill_jobs table tracks attempts and uses exponential backoff via scheduled_at. -// The dequeue() function automatically recovers stale processing jobs and schedules retries. -// Failed jobs are marked with 'failed.retry' status and scheduled_at timestamp for next attempt. - -#[expect(dead_code, reason = "Infrastructure for future PDS health monitoring")] -async fn check_pds_repo_status( - client: &Client, - pds: &str, - repo: &str, -) -> eyre::Result> { - let res = client - .get(format!( - "{pds}/xrpc/com.atproto.sync.getRepoStatus?did={repo}" - )) - .send() - .await?; - - if [StatusCode::NOT_FOUND, StatusCode::BAD_REQUEST].contains(&res.status()) { - return Ok(None); - } - - Ok(res.json().await?) -} - -// Implement Worker trait - delegates to existing run() method -impl Worker for BackfillManager { - fn name(&self) -> &'static str { - "backfill" - } - - async fn run(self, stop: WatchReceiver) -> Result<()> { - BackfillManager::run(self, stop).await - } -} - -/// Factory for creating BackfillManager instances -#[derive(Clone)] -pub struct BackfillManagerFactory { - pool: Pool, - resolver: Arc, - opts: BackfillConfig, - event_tx: tokio::sync::mpsc::Sender, - retention_cutoff: Option>, -} - -impl BackfillManagerFactory { - pub fn new( - pool: Pool, - resolver: Arc, - opts: BackfillConfig, - event_tx: tokio::sync::mpsc::Sender, - retention_cutoff: Option>, - ) -> Self { - Self { - pool, - resolver, - opts, - event_tx, - retention_cutoff, - } - } -} - -impl WorkerFactory for BackfillManagerFactory { - type Worker = BackfillManager; - - fn name(&self) -> &'static str { - "backfill" - } - - async fn create(&self) -> Result { - BackfillManager::new( - self.pool.clone(), - self.resolver.clone(), - self.opts.clone(), - self.event_tx.clone(), - self.retention_cutoff, - ) - .await - } -} diff --git a/consumer/src/workers/backfill/ratelimit.rs b/consumer/src/workers/backfill/ratelimit.rs deleted file mode 100644 index 193b5e19..00000000 --- a/consumer/src/workers/backfill/ratelimit.rs +++ /dev/null @@ -1,20 +0,0 @@ -use reqwest::header::HeaderMap; - -/// Enforce rate limiting for a PDS endpoint -/// -/// Currently disabled - rate limiting is not implemented. -/// This could be reimplemented with PostgreSQL-based rate limit tracking in the future. -#[expect(dead_code, reason = "Infrastructure for future rate limiting feature")] -pub async fn enforce_ratelimit(_pds: &str) -> eyre::Result<()> { - // Rate limiting is currently disabled - Ok(()) -} - -/// Extract an integer value from an HTTP header -/// -/// Common use case is extracting rate limit headers like "ratelimit-remaining" or "ratelimit-reset" -#[expect(dead_code, reason = "Infrastructure for future rate limiting feature")] -pub fn header_to_int(headers: &HeaderMap, name: &str) -> Option { - let v = headers.get(name).and_then(|v| v.to_str().ok())?; - v.parse().ok() -} diff --git a/consumer/src/workers/backfill/repo.rs b/consumer/src/workers/backfill/repo.rs deleted file mode 100644 index 3f96feb9..00000000 --- a/consumer/src/workers/backfill/repo.rs +++ /dev/null @@ -1,380 +0,0 @@ -use super::car_processing::{self, CarParseResult}; -use super::types::CarCommitEntry; -use crate::database_writer::EventSource; -use crate::relay::types::RecordTypes; -use deadpool_postgres::Object as PgObject; -use std::path::Path; - -pub async fn insert_repo( - _conn: &PgObject, - tmp_dir: &Path, - repo: &str, - _resolver: &std::sync::Arc, - event_tx: &tokio::sync::mpsc::Sender, - retention_cutoff: Option>, -) -> eyre::Result> { - tracing::debug!(repo = %repo, "Processing CAR file"); - - // Parse CAR file using extracted module - let car_file = tokio::fs::File::open(tmp_dir.join(repo)).await?; - let CarParseResult { - commit, - records: records_with_metadata, - stats, - } = car_processing::parse_car_file(car_file).await?; - - // Log summary of what we parsed from CAR - let mut sorted_counts: Vec<_> = stats.record_type_counts.iter().collect(); - sorted_counts.sort_by_key(|(name, _)| *name); - - // Build concise summary with top record types - let mut summary_parts = Vec::new(); - for (type_name, count) in sorted_counts.iter().take(5) { - let short_name = type_name.rsplit('.').next().unwrap_or(type_name); - summary_parts.push(format!("{}={}", short_name, count)); - } - let summary = summary_parts.join(" "); - - tracing::debug!( - repo = %repo, - total = stats.total_records, - types = sorted_counts.len(), - "Parsed {} records: {}", - stats.total_records, - summary - ); - - if !stats.unknown_type_counts.is_empty() { - let unknown_total: usize = stats.unknown_type_counts.values().sum(); - tracing::debug!( - repo = %repo, - count = unknown_total, - types = ?stats.unknown_type_counts.keys().collect::>(), - "Skipped unknown record types" - ); - } - - if stats.orphaned_records > 0 { - tracing::debug!( - repo = %repo, - count = stats.orphaned_records, - "Found orphaned records without MST nodes" - ); - } - - if stats.parse_failures > 0 { - tracing::debug!( - repo = %repo, - count = stats.parse_failures, - "Failed to parse some blocks" - ); - } - - // Resolve actor_id once for all records - let actor_id = crate::db::actor::ensure_actor_id( - _conn, - repo, - None, // status - None, // handle - chrono::Utc::now(), - ) - .await?; - - let total_records = records_with_metadata.len(); - let mut processed_count = 0; - let mut skipped_count = 0; - - // Decision point: Use bulk COPY for large batches (50+ records) - let use_bulk = total_records >= crate::database_writer::bulk_types::BULK_THRESHOLD; - - if use_bulk { - tracing::info!( - repo = %repo, - total_records = total_records, - "Using bulk COPY path for large backfill" - ); - - // Convert records to UnresolvedRecord format - let mut unresolved_records = Vec::with_capacity(total_records); - - for record_meta in records_with_metadata { - let at_uri = format!("at://{}/{}", repo, record_meta.path); - let rkey = record_meta.path.split('/').next_back().unwrap_or("").to_string(); - - // Apply retention filtering - if let Some(cutoff) = retention_cutoff { - let mut has_old_tid = false; - - if parakeet_db::tid_util::is_valid_tid(&rkey) { - match parakeet_db::tid_util::is_tid_older_than(&rkey, cutoff) { - Ok(true) => { - has_old_tid = true; - } - Ok(false) => {} - Err(_) => {} - } - } - - if !has_old_tid { - let referenced_tids = extract_tids_from_record(&record_meta.record); - for tid in referenced_tids { - match parakeet_db::tid_util::is_tid_older_than(&tid, cutoff) { - Ok(true) => { - has_old_tid = true; - break; - } - Ok(false) => {} - Err(_) => {} - } - } - } - - if has_old_tid { - skipped_count += 1; - continue; - } - } - - unresolved_records.push(crate::database_writer::UnresolvedRecord { - at_uri, - rkey, - cid: record_meta.cid, - record: Box::new(record_meta.record), - }); - } - - // Resolve ALL stubs upfront (actors, posts, reposts, feedgens, labelers) - // This matches the individual path behavior and enables pure lookup in process_bulk_records - if !unresolved_records.is_empty() { - super::resolve_bulk::resolve_all_references_bulk(_conn, &unresolved_records).await?; - } - - // Send as bulk event - if !unresolved_records.is_empty() { - event_tx - .send(crate::database_writer::WriterEvent::UnresolvedBulk { - repo: repo.to_string(), - actor_id, - records: unresolved_records, - source: EventSource::Backfill, - }) - .await?; - - processed_count = total_records - skipped_count; - } - } else { - // Original individual processing path for small batches - for record_meta in records_with_metadata { - // Construct at_uri and rkey from path - let at_uri = format!("at://{}/{}", repo, record_meta.path); - let rkey = record_meta.path.split('/').next_back().unwrap_or("").to_string(); - - // Check retention policy if enabled - if let Some(cutoff) = retention_cutoff { - let mut has_old_tid = false; - - // Check the record's own rkey (TID) - if parakeet_db::tid_util::is_valid_tid(&rkey) { - match parakeet_db::tid_util::is_tid_older_than(&rkey, cutoff) { - Ok(true) => { - has_old_tid = true; - } - Ok(false) => {} // Recent TID, continue checking - Err(_) => {} // Invalid TID, continue processing - } - } - - // Check any referenced TIDs (like subjects, reply parents, quote embeds) - if !has_old_tid { - let referenced_tids = extract_tids_from_record(&record_meta.record); - for tid in referenced_tids { - match parakeet_db::tid_util::is_tid_older_than(&tid, cutoff) { - Ok(true) => { - has_old_tid = true; - break; - } - Ok(false) => {} // Recent reference - Err(_) => {} // Invalid TID - } - } - } - - // Skip this record if it or its references are too old - if has_old_tid { - skipped_count += 1; - continue; - } - } - - // Resolve subject_actor_id if needed - let subject_actor_id = resolve_subject_actor_id(_conn, &record_meta.record).await?; - - // Create resolved actor IDs (backfill doesn't need notification actor IDs since is_backfill=true) - let resolved_actor_ids = crate::database_writer::workers::ResolvedActorIds { - subject_actor_id, - parent_author_actor_id: None, - root_author_actor_id: None, - quoted_author_actor_id: None, - mentioned_actor_ids: Vec::new(), - service_actor_id: None, - via_repost_key: None, - }; - - // Use unified handler function (same as Jetstream uses) - // EventSource::Backfill prevents notifications for historical data - let processed = crate::database_writer::operations::process_record_to_operations( - repo, - actor_id, - resolved_actor_ids, - record_meta.cid, - record_meta.record, - at_uri, - rkey, - EventSource::Backfill, - ); - - // Stream each operation to database writer immediately - // Bounded channel provides natural backpressure - for operation in processed.operations { - event_tx.send(crate::database_writer::WriterEvent::Resolved(Box::new( - crate::database_writer::ProcessedEvent { - operations: vec![operation], - cursor: None, - source: EventSource::Backfill, - } - ))).await?; - } - - processed_count += 1; - - // Log progress periodically for large backfills - if total_records > 100 && (processed_count % 100 == 0 || processed_count == total_records) { - tracing::debug!( - repo = %repo, - processed = processed_count, - total = total_records, - "Backfill streaming progress" - ); - } - } - } // end of use_bulk if/else - - // Handle case where CAR file has no commit (e.g., incremental backfill with no changes) - if commit.is_none() { - tracing::info!( - repo = %repo, - "CAR file has no commit entry (empty incremental backfill with 0 changes)" - ); - } - - if skipped_count > 0 { - tracing::info!( - repo = %repo, - records_processed = processed_count, - records_skipped = skipped_count, - "Completed CAR processing with retention filtering" - ); - } else { - tracing::debug!( - repo = %repo, - records_processed = processed_count, - "Completed CAR processing and streaming" - ); - } - - Ok(commit) -} - -/// Resolve subject_actor_id and other referenced actors for a record -/// Uses database_writer::extract_references to identify which actors need to exist -async fn resolve_subject_actor_id( - conn: &PgObject, - record: &RecordTypes, -) -> eyre::Result> { - let now = chrono::Utc::now(); - - // Extract references using the database_writer module - let refs = crate::database_writer::extract_references(record); - - // Resolve subject actor if present - let subject_actor_id = if let Some(subject_did) = refs.subject_did { - let actor_id = - crate::db::actor::ensure_actor_id(conn, &subject_did, None, None, now).await?; - Some(actor_id) - } else { - None - }; - - // Resolve all additional referenced actors (like/repost targets, reply parents, etc.) - for did in refs.additional_dids { - crate::db::actor::ensure_actor_id(conn, &did, None, None, now).await?; - } - - Ok(subject_actor_id) -} - -/// Extract all TIDs from a record for retention validation -/// -/// Returns a Vec of TID strings found in: -/// - Like subjects (at://did/app.bsky.feed.post/TID) -/// - Repost subjects -/// - Post reply parents -/// - Post quote embeds -fn extract_tids_from_record(record: &RecordTypes) -> Vec { - let mut tids = Vec::new(); - - match record { - RecordTypes::AppBskyFeedLike(like) => { - // Extract TID from subject URI (at://did/collection/TID) - if let Some(tid) = extract_tid_from_uri(&like.subject.uri) { - tids.push(tid); - } - } - RecordTypes::AppBskyFeedRepost(repost) => { - // Extract TID from subject URI - if let Some(tid) = extract_tid_from_uri(&repost.subject.uri) { - tids.push(tid); - } - } - RecordTypes::AppBskyFeedPost(post) => { - // Extract TID from reply parent - if let Some(reply) = &post.reply { - if let Some(tid) = extract_tid_from_uri(&reply.parent.uri) { - tids.push(tid); - } - } - - // Extract TID from quote embed (app.bsky.embed.record) - if let Some(crate::types::records::EmbedOuter::Bsky(embed)) = &post.embed { - // Check for direct record embed (quote post) - if let crate::types::records::AppBskyEmbed::Record(record_embed) = embed { - if let Some(tid) = extract_tid_from_uri(&record_embed.record.uri) { - tids.push(tid); - } - } - // Check for recordWithMedia (quote post with image/video) - else if let crate::types::records::AppBskyEmbed::RecordWithMedia(rwm) = embed { - if let Some(tid) = extract_tid_from_uri(&rwm.record.record.uri) { - tids.push(tid); - } - } - } - } - _ => { - // Other record types (profiles, lists, follows, blocks) don't need retention filtering - } - } - - tids -} - -/// Extract TID from an AT Protocol URI -/// -/// Expects format: at://did:plc:xyz/app.bsky.feed.post/3l3qo2vuowo2b -/// Returns: Some("3l3qo2vuowo2b") or None -fn extract_tid_from_uri(uri: &str) -> Option { - // Split by '/' and get the last part (the TID) - uri.split('/').next_back() - .filter(|tid| parakeet_db::tid_util::is_valid_tid(tid)) - .map(|tid| tid.to_string()) -} diff --git a/consumer/src/workers/backfill/resolve_bulk.rs b/consumer/src/workers/backfill/resolve_bulk.rs deleted file mode 100644 index 39fb55eb..00000000 --- a/consumer/src/workers/backfill/resolve_bulk.rs +++ /dev/null @@ -1,281 +0,0 @@ -//! Bulk reference resolution for backfill operations -//! -//! This module resolves ALL stubs (actors, posts, reposts, feedgens, labelers) upfront -//! before sending UnresolvedBulk events to the database writer. -//! -//! This matches the individual path behavior where all stubs are resolved in the -//! backfill worker, making process_bulk_records() a pure lookup operation. - -use crate::database_writer::UnresolvedRecord; -use crate::db::bulk_resolve; -use deadpool_postgres::Object as PgObject; -use eyre::Result; -use std::collections::HashSet; - -/// Resolve all references (actors, posts, reposts, etc.) from a batch of records -/// -/// This creates stubs for ALL referenced entities so that process_bulk_records() -/// can perform pure lookups without any stub creation logic. -/// -/// Entities resolved: -/// - Actors (from all DIDs in records) -/// - Posts (from like/repost subjects, reply parents/roots, quote embeds) -/// - Reposts (from via fields) -/// - Feedgens (from feedgen likes) -/// - Labelers (from labeler likes) -pub async fn resolve_all_references_bulk( - conn: &PgObject, - records: &[UnresolvedRecord], -) -> Result<()> { - - // Step 1: Collect all unique actor DIDs - let mut actor_dids = HashSet::new(); - - for unresolved in records { - let refs = crate::database_writer::extract_references(&unresolved.record); - - if let Some(did) = refs.subject_did { - actor_dids.insert(did); - } - - for did in refs.additional_dids { - actor_dids.insert(did); - } - - if let Some(did) = refs.parent_author_did { - actor_dids.insert(did); - } - - if let Some(did) = refs.root_author_did { - actor_dids.insert(did); - } - - if let Some(did) = refs.quoted_author_did { - actor_dids.insert(did); - } - - for did in refs.mentioned_dids { - actor_dids.insert(did); - } - } - - // Step 2: Bulk resolve/create all actor stubs - if !actor_dids.is_empty() { - let dids: Vec<&str> = actor_dids.iter().map(|s| s.as_str()).collect(); - let mut resolved_actors = bulk_resolve::resolve_actor_dids_bulk(conn, &dids).await?; - - // Create stubs for missing actors - let missing_dids: Vec<&str> = dids - .iter() - .filter(|&&did| !resolved_actors.contains_key(did)) - .copied() - .collect(); - - if !missing_dids.is_empty() { - let created = bulk_resolve::create_actor_stubs_bulk(conn, &missing_dids).await?; - resolved_actors.extend(created); - } - - tracing::debug!( - total_dids = dids.len(), - created = missing_dids.len(), - "Bulk resolved actor stubs" - ); - } - - // Step 3: Collect all post URIs (from likes, reposts, reply parents/roots, quotes) - let mut post_uris_with_cids = Vec::new(); - - for unresolved in records { - use crate::relay::types::RecordTypes; - - match &*unresolved.record { - RecordTypes::AppBskyFeedLike(like) => { - // Subject post (if it's a post like) - if like.subject.uri.contains("/app.bsky.feed.post/") { - post_uris_with_cids.push(( - like.subject.uri.as_str(), - like.subject.cid.to_string(), - )); - } - } - RecordTypes::AppBskyFeedRepost(repost) => { - // Subject post - post_uris_with_cids.push(( - repost.subject.uri.as_str(), - repost.subject.cid.to_string(), - )); - } - RecordTypes::AppBskyFeedPost(post) => { - // Parent post - if let Some(ref reply) = post.reply { - post_uris_with_cids.push(( - reply.parent.uri.as_str(), - reply.parent.cid.to_string(), - )); - - // Root post - post_uris_with_cids.push(( - reply.root.uri.as_str(), - reply.root.cid.to_string(), - )); - } - - // Quoted post (from embed.record or embed.recordWithMedia) - // Note: Only add if it's actually a post - record embeds can be feedgens, lists, etc. - if let Some(crate::types::records::EmbedOuter::Bsky(embed)) = &post.embed { - use crate::types::records::AppBskyEmbed; - match embed { - AppBskyEmbed::Record(record_embed) => { - // Only resolve as post if it's actually a post URI (not feedgen, list, etc.) - if record_embed.record.uri.contains("/app.bsky.feed.post/") { - post_uris_with_cids.push(( - record_embed.record.uri.as_str(), - record_embed.record.cid.to_string(), - )); - } - } - AppBskyEmbed::RecordWithMedia(rwm) => { - // Only resolve as post if it's actually a post URI - if rwm.record.record.uri.contains("/app.bsky.feed.post/") { - post_uris_with_cids.push(( - rwm.record.record.uri.as_str(), - rwm.record.record.cid.to_string(), - )); - } - } - _ => {} - } - } - } - _ => {} - } - } - - // Step 4: Bulk resolve/create all post stubs - if !post_uris_with_cids.is_empty() { - // Deduplicate - let mut unique_posts: HashSet<(&str, String)> = HashSet::new(); - for (uri, cid) in &post_uris_with_cids { - unique_posts.insert((*uri, cid.clone())); - } - - let post_pairs: Vec<(&str, &str)> = unique_posts - .iter() - .map(|(uri, cid)| (*uri, cid.as_str())) - .collect(); - - drop(bulk_resolve::resolve_and_ensure_posts_bulk(conn, &post_pairs).await?); - - tracing::debug!( - total_posts = post_pairs.len(), - "Bulk resolved post stubs" - ); - } - - // Step 5: Collect all repost URIs (from via fields) - let mut repost_data = Vec::new(); - - for unresolved in records { - let refs = crate::database_writer::extract_references(&unresolved.record); - - if let (Some(via_uri), Some(via_cid)) = (refs.via_uri, refs.via_cid) { - // For via reposts, we need the full 4-tuple: - // (repost_uri, repost_cid, subject_post_uri, subject_post_cid) - use crate::relay::types::RecordTypes; - match &*unresolved.record { - RecordTypes::AppBskyFeedLike(like) => { - repost_data.push(( - via_uri.clone(), - via_cid.clone(), - like.subject.uri.clone(), - like.subject.cid.to_string(), - )); - } - RecordTypes::AppBskyFeedRepost(repost) => { - repost_data.push(( - via_uri.clone(), - via_cid.clone(), - repost.subject.uri.clone(), - repost.subject.cid.to_string(), - )); - } - _ => {} - } - } - } - - // Step 6: Bulk resolve/create all repost stubs - if !repost_data.is_empty() { - // Convert to required format with String for CIDs - let repost_tuples: Vec<(&str, &str, &str, &str)> = repost_data - .iter() - .map(|(uri, cid, subj_uri, subj_cid)| (uri.as_str(), cid.as_str(), subj_uri.as_str(), subj_cid.as_str())) - .collect(); - - drop(bulk_resolve::resolve_and_ensure_reposts_bulk(conn, &repost_tuples).await?); - - tracing::debug!( - total_reposts = repost_tuples.len(), - "Bulk resolved repost stubs (via fields)" - ); - } - - // Step 7: Collect all feedgen URIs (from likes) - let mut feedgen_uris_with_cids = Vec::new(); - - for unresolved in records { - use crate::relay::types::RecordTypes; - if let RecordTypes::AppBskyFeedLike(like) = &*unresolved.record { - if like.subject.uri.contains("/app.bsky.feed.generator/") { - feedgen_uris_with_cids.push(( - like.subject.uri.as_str(), - like.subject.cid.to_string(), - )); - } - } - } - - // Step 8: Bulk resolve feedgen URIs (no stub creation needed - feedgens are explicit) - if !feedgen_uris_with_cids.is_empty() { - let feedgen_uris: Vec<&str> = feedgen_uris_with_cids - .iter() - .map(|(uri, _)| *uri) - .collect(); - - drop(bulk_resolve::resolve_feedgen_uris_bulk(conn, &feedgen_uris).await?); - - tracing::debug!( - total_feedgens = feedgen_uris.len(), - "Bulk resolved feedgen URIs" - ); - } - - // Step 9: Collect all labeler DIDs (from likes) - let mut labeler_dids = Vec::new(); - - for unresolved in records { - use crate::relay::types::RecordTypes; - if let RecordTypes::AppBskyFeedLike(like) = &*unresolved.record { - // Labeler likes use AT URIs: at://did:plc:.../app.bsky.labeler.service/self - if like.subject.uri.contains("/app.bsky.labeler.service/") { - // Extract DID from AT URI - if let Some(did) = parakeet_db::at_uri_util::extract_did(&like.subject.uri) { - labeler_dids.push(did); - } - } - } - } - - // Step 10: Bulk resolve labeler DIDs (no stub creation - labelers are explicit) - if !labeler_dids.is_empty() { - drop(bulk_resolve::resolve_labeler_dids_bulk(conn, &labeler_dids).await?); - - tracing::debug!( - total_labelers = labeler_dids.len(), - "Bulk resolved labeler DIDs" - ); - } - - Ok(()) -} diff --git a/consumer/src/workers/backfill/types.rs b/consumer/src/workers/backfill/types.rs deleted file mode 100644 index 3e2d1d36..00000000 --- a/consumer/src/workers/backfill/types.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::relay::types::RecordTypes; -use ipld_core::cid::Cid; -use serde::Deserialize; -use serde_bytes::ByteBuf; - -/// CAR file entry types for backfill deserialization -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub enum CarEntry { - Mst(CarMstEntry), - Commit(CarCommitEntry), - Record(Box), -} - -/// Merkle Search Tree entry from CAR files -/// Fields marked dead_code are required for CBOR deserialization but not directly accessed -#[derive(Debug, Deserialize)] -pub struct CarMstEntry { - /// Left pointer (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub l: Option, - /// Entry nodes - pub e: Vec, -} - -/// MST entry node -#[derive(Debug, Deserialize)] -pub struct CarMstEntryNode { - pub p: i32, - pub k: ByteBuf, - pub v: Cid, - /// Tree pointer (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub t: Option, -} - -/// Commit entry from CAR files -/// Fields marked dead_code are required for CBOR deserialization but not directly accessed -#[derive(Debug, Deserialize)] -pub struct CarCommitEntry { - /// DID of the repository owner (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub did: String, - /// Repository version (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub version: i32, - /// CID of the MST root - pub data: Cid, - /// Repository revision string - pub rev: String, - /// Previous commit CID (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub prev: Option, - /// Commit signature (required for deserialization) - #[expect(dead_code, reason = "Required by CBOR format spec")] - pub sig: ByteBuf, -} - -#[derive(Debug, Deserialize)] -#[serde(untagged)] -pub enum CarRecordEntry { - Known(Box), - Other { - #[serde(rename = "$type")] - ty: String, - }, -} - -/// Response from com.atproto.sync.getRepoStatus -/// Fields marked dead_code are required for JSON deserialization but not directly accessed -#[derive(Debug, Deserialize)] -#[expect(dead_code, reason = "Required by JSON format spec")] -pub struct GetRepoStatusRes { - /// DID of the repository (required for deserialization) - pub did: String, - /// Whether the account is active - pub active: bool, - /// Account status - pub status: Option, - /// Repository revision (required for deserialization) - pub rev: Option, -} diff --git a/consumer/src/workers/backfill/worker.rs b/consumer/src/workers/backfill/worker.rs deleted file mode 100644 index 4135604f..00000000 --- a/consumer/src/workers/backfill/worker.rs +++ /dev/null @@ -1,252 +0,0 @@ -use super::{ratelimit, DL_DONE_KEY}; -use crate::db; -use deadpool_postgres::{Client as PgClient, Pool}; -use did_resolver::Resolver; -use futures::TryStreamExt as _; -use metrics::{counter, histogram}; -use parakeet_db::types::ActorStatus; -use reqwest::Client as HttpClient; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::time::Instant; -use tokio_util::io::StreamReader; -use tracing::instrument; - -/// Spawn and run a download worker thread -/// -/// This worker receives (PDS URL, DID, handle) tuples from the channel, -/// downloads the CAR file for each repo. -pub async fn download_thread( - pool: Pool, - resolver: Arc, - http: reqwest::Client, - rx: flume::Receiver<(String, String, Option)>, - tmp_dir: PathBuf, - retention_cutoff: Option>, -) { - tracing::debug!("spawning thread"); - - // this will return Err(_) and exit when all senders (only held above) are dropped - while let Ok((pds, did, maybe_handle)) = rx.recv_async().await { - // Rate limiting is currently disabled - if let Err(e) = ratelimit::enforce_ratelimit(&pds).await { - tracing::error!("ratelimiter error: {e}"); - continue; - } - - // Query stored repo_rev for incremental backfill, or use retention cutoff - let since_rev = { - tracing::trace!("getting DB conn..."); - let mut conn = pool.get().await.unwrap(); - tracing::trace!("got DB conn..."); - match check_and_update_repo_status(&http, &mut conn, &pds, &did).await { - Ok(true) => {} - Ok(false) => continue, - Err(e) => { - tracing::error!(pds, did, "failed to check repo status: {e}"); - // Job will be marked as failed by the main backfill error handler - continue; - } - } - - tracing::debug!("trying to resolve handle..."); - if let Some(handle) = maybe_handle { - let resolve_result = resolve_and_set_handle(&conn, &resolver, &did, &handle).await; - if let Err(e) = resolve_result { - tracing::error!(pds, did, "failed to resolve handle: {e}"); - // Non-fatal - continue with backfill even if handle resolution fails - } - } - - match db::actor_get_repo_rev(&conn, &did).await { - Ok(Some(rev)) => { - tracing::info!(did = %did, since_rev = %rev, "Using incremental backfill"); - counter!("backfill_incremental").increment(1); - Some(rev) - } - Ok(None) => { - // No stored repo_rev - check if retention is enabled - if let Some(cutoff) = retention_cutoff { - // Use TID-based retention cutoff for time-bounded backfill - let retention_tid = parakeet_db::tid_util::timestamp_to_tid(cutoff); - tracing::info!( - did = %did, - since_tid = %retention_tid, - cutoff_date = %cutoff.format("%Y-%m-%d"), - "Using retention-based backfill with TID since parameter" - ); - counter!("backfill_retention").increment(1); - Some(retention_tid) - } else { - tracing::debug!(did = %did, "No stored repo_rev, performing full backfill"); - counter!("backfill_full").increment(1); - None - } - } - Err(e) => { - tracing::warn!(did = %did, error = %e, "Failed to query repo_rev, falling back to full backfill"); - counter!("backfill_full").increment(1); - None - } - } - }; - - let start = Instant::now(); - - tracing::trace!("downloading repo {did}"); - - match download_car(&http, &tmp_dir, &pds, &did, since_rev.as_deref()).await { - Ok(Some((rem, reset))) => { - // Rate limit tracking not implemented (headers received but not stored) - let _ = (rem, reset); // Silence unused warning - } - Ok(_) => tracing::debug!(pds, "No ratelimit headers in response"), - Err(e) => { - tracing::error!(pds, did, "failed to download repo: {e}"); - continue; - } - } - - histogram!("backfill_download_dur", "pds" => pds).record(start.elapsed().as_secs_f64()); - - // Track successful downloads - counter!("backfill_downloaded").increment(1); - } - - tracing::debug!("thread exiting"); -} - -/// you wouldn't... -/// -/// Download a CAR file from a PDS and save it to the tmp directory -/// -/// If `since_rev` is provided, requests an incremental diff using the `since` parameter. -/// Returns the ratelimit-remaining and ratelimit-reset headers if present. -#[instrument(skip(http, tmp_dir, pds))] -async fn download_car( - http: &HttpClient, - tmp_dir: &Path, - pds: &str, - did: &str, - since_rev: Option<&str>, -) -> eyre::Result> { - let url = if let Some(rev) = since_rev { - format!("{pds}/xrpc/com.atproto.sync.getRepo?did={did}&since={rev}") - } else { - format!("{pds}/xrpc/com.atproto.sync.getRepo?did={did}") - }; - - let res = http.get(&url).send().await?.error_for_status()?; - - let file_path = tmp_dir.join(did); - - // Remove existing file if present to ensure idempotent retries - if file_path.exists() { - tracing::debug!(did, "removing existing CAR file for retry"); - tokio::fs::remove_file(&file_path).await?; - } - - let mut file = tokio::fs::File::create_new(&file_path).await?; - - let headers = res.headers(); - let ratelimit_rem = ratelimit::header_to_int(headers, "ratelimit-remaining"); - let ratelimit_reset = ratelimit::header_to_int(headers, "ratelimit-reset"); - - let strm = res.bytes_stream().map_err(std::io::Error::other); - let mut reader = StreamReader::new(strm); - - let bytes_written = tokio::io::copy(&mut reader, &mut file).await?; - - tracing::info!( - did = %did, - bytes = bytes_written, - incremental = since_rev.is_some(), - "CAR file downloaded, queuing for processing" - ); - - Ok(ratelimit_rem.zip(ratelimit_reset)) -} - -/// Check the repo status on the PDS and update the database accordingly -/// -/// Returns Ok(true) if the repo is active and should be downloaded, -/// Ok(false) if the repo is inactive/deleted and should be skipped. -#[instrument(skip(http, conn, pds))] -async fn check_and_update_repo_status( - http: &HttpClient, - conn: &mut PgClient, - pds: &str, - repo: &str, -) -> eyre::Result { - if let Some(status) = super::check_pds_repo_status(http, pds, repo).await? { - if status.active { - Ok(true) - } else { - tracing::debug!("repo is inactive"); - - let status = status - .status - .unwrap_or(crate::events::AtpAccountStatus::Deleted); - - // Use consolidated ActorUpdate API - use crate::db::operations::{ActorUpdate, ActorUpdateTarget}; - use parakeet_db::types::ActorSyncState; - - let _ = ActorUpdate { - target: ActorUpdateTarget::ByDid(repo.to_string()), - sync_state: Some(ActorSyncState::Dirty), - actor_status: Some(ActorStatus::from(status)), - ..Default::default() - } - .execute(conn) - .await?; - - Ok(false) - } - } else { - // this repo can't be found - set dirty and assume deleted. - tracing::debug!("repo was deleted"); - - // Use consolidated ActorUpdate API - use crate::db::operations::{ActorUpdate, ActorUpdateTarget}; - use parakeet_db::types::{ActorStatus, ActorSyncState}; - - let _ = ActorUpdate { - target: ActorUpdateTarget::ByDid(repo.to_string()), - sync_state: Some(ActorSyncState::Dirty), - actor_status: Some(ActorStatus::Deleted), - ..Default::default() - } - .execute(conn) - .await?; - - Ok(false) - } -} - -/// Resolve a handle and update the actor record if it matches the DID -async fn resolve_and_set_handle( - conn: &PgClient, - resolver: &Resolver, - did: &str, - handle: &str, -) -> eyre::Result<()> { - if let Some(handle_did) = resolver.resolve_handle(handle).await? { - if handle_did == did { - // Use consolidated ActorUpdate API - use crate::db::operations::{ActorUpdate, ActorUpdateTarget}; - - let _ = ActorUpdate { - target: ActorUpdateTarget::ByDid(did.to_string()), - handle: Some(handle.to_string()), - ..Default::default() - } - .execute(conn) - .await?; - } else { - tracing::warn!("requested DID ({did}) doesn't match handle"); - } - } - - Ok(()) -} diff --git a/consumer/src/workers/fetch/json_types.rs b/consumer/src/workers/fetch/json_types.rs deleted file mode 100644 index 2b6f8b5e..00000000 --- a/consumer/src/workers/fetch/json_types.rs +++ /dev/null @@ -1,375 +0,0 @@ -//! JSON-specific record types for Jetstream and Slingshot fetching -//! -//! These types mirror the CBOR types but use BlobJson instead of Blob, -//! allowing proper deserialization from JSON sources like Jetstream and Slingshot. -//! -//! After deserialization, convert these to the standard RecordTypes using .into() - -use crate::relay::types::RecordTypes; -use crate::types::records; -use crate::utils; -use chrono::{DateTime, Utc}; -use lexica::app_bsky::embed::AspectRatio; -use lexica::app_bsky::richtext::FacetMain; -use lexica::com_atproto::label::SelfLabels; -use lexica::{BlobJson, StrongRef}; -use serde::{Deserialize, Serialize}; -use serde_with::serde_as; - -/// JSON version of AppBskyActorProfile with BlobJson -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -#[serde_as] -pub struct AppBskyActorProfileJson { - #[serde_as(as = "utils::safe_string")] - pub display_name: Option, - #[serde_as(as = "utils::safe_string")] - pub description: Option, - pub avatar: Option, - pub banner: Option, - pub labels: Option, - pub joined_via_starter_pack: Option, - pub pinned_post: Option, - #[serde_as(as = "utils::safe_string")] - pub pronouns: Option, - #[serde_as(as = "utils::safe_string")] - pub website: Option, - pub created_at: Option>, -} - -impl From for records::AppBskyActorProfile { - fn from(json: AppBskyActorProfileJson) -> Self { - records::AppBskyActorProfile { - display_name: json.display_name, - description: json.description, - avatar: json.avatar.map(Into::into), - banner: json.banner.map(Into::into), - labels: json.labels, - joined_via_starter_pack: json.joined_via_starter_pack, - pinned_post: json.pinned_post, - pronouns: json.pronouns, - website: json.website, - created_at: json.created_at, - } - } -} - -/// JSON version of EmbedImage with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct EmbedImageJson { - pub image: BlobJson, - #[serde(deserialize_with = "utils::safe_string")] - pub alt: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub aspect_ratio: Option, -} - -impl From for records::EmbedImage { - fn from(json: EmbedImageJson) -> Self { - records::EmbedImage { - image: json.image.into(), - alt: json.alt, - aspect_ratio: json.aspect_ratio, - } - } -} - -/// JSON version of AppBskyEmbedImages with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct AppBskyEmbedImagesJson { - pub images: Vec, -} - -impl From for records::AppBskyEmbedImages { - fn from(json: AppBskyEmbedImagesJson) -> Self { - records::AppBskyEmbedImages { - images: json.images.into_iter().map(Into::into).collect(), - } - } -} - -/// JSON version of EmbedVideoCaptions with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct EmbedVideoCaptionsJson { - pub lang: String, - pub file: BlobJson, -} - -impl From for records::EmbedVideoCaptions { - fn from(json: EmbedVideoCaptionsJson) -> Self { - records::EmbedVideoCaptions { - lang: json.lang, - file: json.file.into(), - } - } -} - -/// JSON version of AppBskyEmbedVideo with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -#[serde_as] -pub struct AppBskyEmbedVideoJson { - pub video: BlobJson, - #[serde(skip_serializing_if = "Option::is_none")] - pub captions: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - #[serde_as(as = "utils::safe_string")] - pub alt: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub aspect_ratio: Option, -} - -impl From for records::AppBskyEmbedVideo { - fn from(json: AppBskyEmbedVideoJson) -> Self { - records::AppBskyEmbedVideo { - video: json.video.into(), - captions: json - .captions - .map(|caps| caps.into_iter().map(Into::into).collect()), - alt: json.alt, - aspect_ratio: json.aspect_ratio, - } - } -} - -/// JSON version of EmbedExternal with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct EmbedExternalJson { - pub uri: String, - #[serde(deserialize_with = "utils::safe_string")] - pub title: String, - #[serde(deserialize_with = "utils::safe_string")] - pub description: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub thumb: Option, -} - -impl From for records::EmbedExternal { - fn from(json: EmbedExternalJson) -> Self { - records::EmbedExternal { - uri: json.uri, - title: json.title, - description: json.description, - thumb: json.thumb.map(Into::into), - } - } -} - -/// JSON version of AppBskyEmbedExternal with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(tag = "$type")] -#[serde(rename = "app.bsky.embed.external")] -pub struct AppBskyEmbedExternalJson { - pub external: EmbedExternalJson, -} - -impl From for records::AppBskyEmbedExternal { - fn from(json: AppBskyEmbedExternalJson) -> Self { - records::AppBskyEmbedExternal { - external: json.external.into(), - } - } -} - -/// JSON version of AppBskyFeedGenerator with BlobJson -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -#[serde_as] -pub struct AppBskyFeedGeneratorJson { - pub did: String, - #[serde(deserialize_with = "utils::safe_string")] - pub display_name: String, - #[serde_as(as = "utils::safe_string")] - pub description: Option, - pub description_facets: Option>, - pub avatar: Option, - pub accepts_interactions: Option, - pub labels: Option, - pub content_mode: Option, - pub created_at: DateTime, -} - -impl From for records::AppBskyFeedGenerator { - fn from(json: AppBskyFeedGeneratorJson) -> Self { - records::AppBskyFeedGenerator { - did: json.did, - display_name: json.display_name, - description: json.description, - description_facets: json.description_facets, - avatar: json.avatar.map(Into::into), - accepts_interactions: json.accepts_interactions, - labels: json.labels, - content_mode: json.content_mode, - created_at: json.created_at, - } - } -} - -/// JSON version of AppBskyGraphList with BlobJson -#[derive(Debug, Deserialize, Serialize)] -#[serde(rename_all = "camelCase")] -#[serde_as] -pub struct AppBskyGraphListJson { - pub purpose: String, - #[serde(deserialize_with = "utils::safe_string")] - pub name: String, - #[serde_as(as = "utils::safe_string")] - pub description: Option, - pub description_facets: Option>, - pub avatar: Option, - pub labels: Option, - pub created_at: DateTime, -} - -impl From for records::AppBskyGraphList { - fn from(json: AppBskyGraphListJson) -> Self { - records::AppBskyGraphList { - purpose: json.purpose, - name: json.name, - description: json.description, - description_facets: json.description_facets, - avatar: json.avatar.map(Into::into), - labels: json.labels, - created_at: json.created_at, - } - } -} - -/// JSON version of AppBskyEmbed with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(tag = "$type")] -pub enum AppBskyEmbedJson { - #[serde(rename = "app.bsky.embed.images")] - Images(AppBskyEmbedImagesJson), - #[serde(rename = "app.bsky.embed.video")] - Video(AppBskyEmbedVideoJson), - #[serde(rename = "app.bsky.embed.external")] - External(AppBskyEmbedExternalJson), - #[serde(rename = "app.bsky.embed.record")] - Record(records::AppBskyEmbedRecord), - #[serde(rename = "app.bsky.embed.recordWithMedia")] - RecordWithMedia(AppBskyEmbedRecordWithMediaJson), -} - -impl From for records::AppBskyEmbed { - fn from(json: AppBskyEmbedJson) -> Self { - match json { - AppBskyEmbedJson::Images(images) => records::AppBskyEmbed::Images(images.into()), - AppBskyEmbedJson::Video(video) => records::AppBskyEmbed::Video(video.into()), - AppBskyEmbedJson::External(external) => { - records::AppBskyEmbed::External(external.into()) - } - AppBskyEmbedJson::Record(record) => records::AppBskyEmbed::Record(record), - AppBskyEmbedJson::RecordWithMedia(rwm) => { - records::AppBskyEmbed::RecordWithMedia(rwm.into()) - } - } - } -} - -/// JSON version of AppBskyEmbedRecordWithMedia with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct AppBskyEmbedRecordWithMediaJson { - pub record: records::AppBskyEmbedRecord, - pub media: Box, -} - -impl From for records::AppBskyEmbedRecordWithMedia { - fn from(json: AppBskyEmbedRecordWithMediaJson) -> Self { - records::AppBskyEmbedRecordWithMedia { - record: json.record, - media: Box::new((*json.media).into()), - } - } -} - -/// JSON version of EmbedOuter with BlobJson -#[derive(Clone, Debug, Deserialize, Serialize)] -#[serde(untagged)] -pub enum EmbedOuterJson { - Bsky(AppBskyEmbedJson), - Other(serde_json::Value), -} - -impl From for records::EmbedOuter { - fn from(json: EmbedOuterJson) -> Self { - match json { - EmbedOuterJson::Bsky(embed) => records::EmbedOuter::Bsky(embed.into()), - EmbedOuterJson::Other(value) => records::EmbedOuter::Other(value), - } - } -} - -/// JSON version of AppBskyFeedPost with BlobJson embeds -#[derive(Debug, Deserialize, Serialize)] -#[serde(tag = "$type")] -#[serde(rename = "app.bsky.feed.post")] -#[serde(rename_all = "camelCase")] -pub struct AppBskyFeedPostJson { - #[serde(deserialize_with = "utils::safe_string")] - pub text: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub facets: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub reply: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub embed: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub langs: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub labels: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub tags: Option>, - pub created_at: DateTime, -} - -impl From for records::AppBskyFeedPost { - fn from(json: AppBskyFeedPostJson) -> Self { - records::AppBskyFeedPost { - text: json.text, - facets: json.facets, - reply: json.reply, - embed: json.embed.map(Into::into), - langs: json.langs, - labels: json.labels, - tags: json.tags, - created_at: json.created_at, - } - } -} - -/// JSON version of RecordTypes enum - only handles records with blobs -#[derive(Debug, Deserialize, Serialize)] -#[serde(tag = "$type")] -pub enum RecordTypesJson { - #[serde(rename = "app.bsky.actor.profile")] - AppBskyActorProfile(AppBskyActorProfileJson), - #[serde(rename = "app.bsky.feed.post")] - AppBskyFeedPost(AppBskyFeedPostJson), - #[serde(rename = "app.bsky.feed.generator")] - AppBskyFeedGenerator(AppBskyFeedGeneratorJson), - #[serde(rename = "app.bsky.graph.list")] - AppBskyGraphList(AppBskyGraphListJson), - - // For records without blobs, fall back to standard deserialization - #[serde(untagged)] - Standard(RecordTypes), -} - -impl From for RecordTypes { - fn from(json: RecordTypesJson) -> Self { - match json { - RecordTypesJson::AppBskyActorProfile(profile) => { - RecordTypes::AppBskyActorProfile(profile.into()) - } - RecordTypesJson::AppBskyFeedPost(post) => RecordTypes::AppBskyFeedPost(post.into()), - RecordTypesJson::AppBskyFeedGenerator(generator) => { - RecordTypes::AppBskyFeedGenerator(generator.into()) - } - RecordTypesJson::AppBskyGraphList(list) => RecordTypes::AppBskyGraphList(list.into()), - RecordTypesJson::Standard(record) => record, - } - } -} diff --git a/consumer/src/workers/fetch/mod.rs b/consumer/src/workers/fetch/mod.rs deleted file mode 100644 index f578dafd..00000000 --- a/consumer/src/workers/fetch/mod.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! Record fetching with three-tier fallback strategy -//! -//! This module provides functionality to fetch individual records using a three-tier approach: -//! 1. Slingshot (preferred) - a fast edge cache for recent records -//! 2. PDS (fallback) - the owner's Personal Data Server (canonical but may be offline) -//! 3. Bluesky public API (final fallback) - distributed cache with high availability -//! -//! Records are fetched when allowlisted users interact with them (likes, replies, reposts) -//! to ensure we have complete data even if the records aren't from allowlisted users. - -pub mod json_types; -pub mod pds; -pub mod sources; -pub mod types; -pub mod worker; - -// Re-export worker factory -pub use worker::RecordFetchManagerFactory; - -use crate::relay::types::RecordTypes; -use did_resolver::Resolver; -use eyre::{Result, WrapErr as _}; -use metrics::{counter, histogram}; -use pds::PdsFetcher; -use sources::bluesky::BlueskyApiFetcher; -use sources::slingshot::SlingshotFetcher; -use std::sync::Arc; -use std::time::Instant; -use tracing::debug; -pub use types::{FetchedRecord, RecordFetchConfig}; - -/// Combined record fetcher with three-tier fallback strategy -#[derive(Clone)] -pub struct RecordFetcher { - slingshot: SlingshotFetcher, - pds: PdsFetcher, - bluesky_api: BlueskyApiFetcher, - pub resolver: Arc, -} - -impl RecordFetcher { - /// Create a new record fetcher with the given configuration - pub fn new(config: &RecordFetchConfig, resolver: Arc) -> Result { - let slingshot = SlingshotFetcher::new(config)?; - let pds = PdsFetcher::new(config, resolver.clone())?; - let bluesky_api = BlueskyApiFetcher::new(config)?; - - Ok(Self { - slingshot, - pds, - bluesky_api, - resolver, - }) - } - - /// Fetch a record by its at-uri using optimized fallback strategy - /// Tries: Bluesky API (fastest) → Slingshot (recent data) → PDS (canonical but slow) - pub async fn fetch_record(&self, at_uri: &str) -> Result { - // Track total fetch attempts - counter!("record_fetch.total").increment(1); - - // Tier 1: Try Bluesky public API first (fastest, high availability) - // Testing shows ~190ms for profiles vs ~518ms for Slingshot (2.7x faster) - let bluesky_start = Instant::now(); - match self.bluesky_api.fetch_by_uri(at_uri).await { - Ok(record) => { - let latency_ms = bluesky_start.elapsed().as_millis() as f64; - histogram!("fetch.bluesky_api_latency_ms").record(latency_ms); - counter!("record_fetch_bluesky_api_success").increment(1); - debug!("Successfully fetched record from Bluesky API: {}", at_uri); - return Ok(record); - } - Err(e) => { - let latency_ms = bluesky_start.elapsed().as_millis() as f64; - histogram!("fetch.bluesky_api_latency_ms").record(latency_ms); - counter!("record_fetch_bluesky_api_failure").increment(1); - debug!( - "Failed to fetch from Bluesky API, falling back to Slingshot for {}: {}", - at_uri, e - ); - } - } - - // Tier 2: Fall back to Slingshot (edge cache for recent records) - let slingshot_start = Instant::now(); - match self.slingshot.fetch_by_uri(at_uri).await { - Ok(record) => { - let latency_ms = slingshot_start.elapsed().as_millis() as f64; - histogram!("fetch.slingshot_latency_ms").record(latency_ms); - counter!("record_fetch_slingshot_success").increment(1); - debug!("Successfully fetched record from Slingshot: {}", at_uri); - return Ok(record); - } - Err(e) => { - let latency_ms = slingshot_start.elapsed().as_millis() as f64; - histogram!("fetch.slingshot_latency_ms").record(latency_ms); - counter!("record_fetch_slingshot_failure").increment(1); - debug!( - "Failed to fetch from Slingshot, falling back to PDS for {}: {}", - at_uri, e - ); - } - } - - // Tier 3: Final fallback to PDS (canonical source, may be offline) - let pds_start = Instant::now(); - match self.pds.fetch_by_uri(at_uri).await { - Ok(record) => { - let latency_ms = pds_start.elapsed().as_millis() as f64; - histogram!("fetch.pds_latency_ms").record(latency_ms); - counter!("record_fetch_pds_success").increment(1); - debug!("Successfully fetched record from PDS: {}", at_uri); - Ok(record) - } - Err(e) => { - let latency_ms = pds_start.elapsed().as_millis() as f64; - histogram!("fetch.pds_latency_ms").record(latency_ms); - counter!("record_fetch_pds_failure").increment(1); - Err(e).wrap_err("Failed to fetch record from Bluesky API, Slingshot, and PDS") - } - } - } - - /// Fetch a record skipping Slingshot (for retries) - /// Used for retry attempts where we already know Slingshot doesn't have the record - /// Tries: Bluesky API → PDS - pub async fn fetch_record_pds_only(&self, at_uri: &str) -> Result { - counter!("record_fetch.retry_skip_slingshot").increment(1); - - // Tier 1 (retry): Try Bluesky API first (fastest) - let bluesky_start = Instant::now(); - match self.bluesky_api.fetch_by_uri(at_uri).await { - Ok(record) => { - let latency_ms = bluesky_start.elapsed().as_millis() as f64; - histogram!("fetch.bluesky_api_latency_ms").record(latency_ms); - counter!("record_fetch_bluesky_api_success").increment(1); - debug!("Successfully fetched record from Bluesky API (retry): {}", at_uri); - return Ok(record); - } - Err(e) => { - let latency_ms = bluesky_start.elapsed().as_millis() as f64; - histogram!("fetch.bluesky_api_latency_ms").record(latency_ms); - counter!("record_fetch_bluesky_api_failure").increment(1); - debug!( - "Failed to fetch from Bluesky API (retry), falling back to PDS for {}: {}", - at_uri, e - ); - } - } - - // Tier 2 (retry): Final fallback to PDS - let pds_start = Instant::now(); - match self.pds.fetch_by_uri(at_uri).await { - Ok(record) => { - let latency_ms = pds_start.elapsed().as_millis() as f64; - histogram!("fetch.pds_latency_ms").record(latency_ms); - counter!("record_fetch_pds_success").increment(1); - debug!("Successfully fetched record from PDS (retry): {}", at_uri); - Ok(record) - } - Err(e) => { - let latency_ms = pds_start.elapsed().as_millis() as f64; - histogram!("fetch.pds_latency_ms").record(latency_ms); - counter!("record_fetch_pds_failure").increment(1); - Err(e).wrap_err("Failed to fetch record from Bluesky API and PDS (skipped Slingshot)") - } - } - } - - /// Parse a fetched record into a typed `RecordTypes` - pub fn parse_record(fetched: &FetchedRecord) -> Result { - // First try to parse as JSON type (handles BlobJson for Slingshot records) - let json_record: json_types::RecordTypesJson = - serde_json::from_value(fetched.value.clone()) - .wrap_err("Failed to parse fetched record")?; - - // Convert to standard RecordTypes - Ok(json_record.into()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use did_resolver::ResolverOpts; - - #[tokio::test] - #[expect(clippy::print_stdout, reason = "Test code uses println! for diagnostic output")] - async fn test_fetch_and_parse_profile() { - let config = RecordFetchConfig { - slingshot_url: "https://slingshot.microcosm.blue".to_owned(), - bluesky_api_url: "https://public.api.bsky.app".to_owned(), - timeout_secs: 10, - }; - - let resolver = - Arc::new(Resolver::new(ResolverOpts::default()).expect("Failed to create resolver")); - - let fetcher = RecordFetcher::new(&config, resolver).expect("Failed to create fetcher"); - - // Test fetching a known profile with avatar and banner blobs - let at_uri = "at://did:plc:q2tsdqmkgspo2b6jvahjszwq/app.bsky.actor.profile/self"; - - let result = fetcher.fetch_record(at_uri).await; - assert!(result.is_ok(), "Failed to fetch record: {:?}", result.err()); - - let fetched = result.unwrap(); - - // Test parsing the record - let parsed = RecordFetcher::parse_record(&fetched); - assert!(parsed.is_ok(), "Failed to parse record: {:?}", parsed.err()); - - // Verify it's a profile record - match parsed.unwrap() { - RecordTypes::AppBskyActorProfile(profile) => { - println!("✓ Successfully fetched and parsed profile from Slingshot!"); - println!(" Display name: {:?}", profile.display_name); - println!( - " Avatar: {:?}", - profile.avatar.as_ref().map(|b| b.cid.to_string()) - ); - println!( - " Banner: {:?}", - profile.banner.as_ref().map(|b| b.cid.to_string()) - ); - - // Verify blobs were parsed correctly - assert!(profile.avatar.is_some(), "Profile should have avatar"); - assert!(profile.banner.is_some(), "Profile should have banner"); - } - other => panic!("Expected profile record, got: {:?}", other), - } - } -} diff --git a/consumer/src/workers/fetch/pds.rs b/consumer/src/workers/fetch/pds.rs deleted file mode 100644 index 1d51426c..00000000 --- a/consumer/src/workers/fetch/pds.rs +++ /dev/null @@ -1,253 +0,0 @@ -use super::types::{FetchedRecord, GetRecordResponse, ListRecordsResponse, RecordFetchConfig}; -use did_resolver::Resolver; -use eyre::{Result, WrapErr as _}; -use reqwest::Client; -use std::sync::Arc; -use std::time::Duration; -use tracing::debug; - -/// Client for fetching records directly from a PDS -#[derive(Clone)] -pub struct PdsFetcher { - client: Client, - resolver: Arc, -} - -impl PdsFetcher { - pub fn new(config: &RecordFetchConfig, resolver: Arc) -> Result { - let client = Client::builder() - .timeout(Duration::from_secs(config.timeout_secs)) - .pool_max_idle_per_host(10) // Moderate pooling for PDS batches - .pool_idle_timeout(Some(Duration::from_secs(60))) - .tcp_keepalive(Some(Duration::from_secs(30))) - .connect_timeout(Duration::from_secs(3)) - .build() - .wrap_err("Failed to create HTTP client for PDS")?; - - Ok(Self { client, resolver }) - } - - /// Fetch a record from its original PDS - pub async fn fetch_by_uri(&self, at_uri: &str) -> Result { - // Parse the at-uri: at://did/collection/rkey - let parts: Vec<&str> = at_uri - .strip_prefix("at://") - .unwrap_or(at_uri) - .split('/') - .collect(); - if parts.len() != 3 { - return Err(eyre::eyre!("Invalid at-uri format: {}", at_uri)); - } - - let (repo, collection, rkey) = (parts[0], parts[1], parts[2]); - self.fetch_by_parts(repo, collection, rkey).await - } - - /// Fetch a record from its original PDS by parts - pub async fn fetch_by_parts( - &self, - repo: &str, - collection: &str, - rkey: &str, - ) -> Result { - // Resolve the DID to get the PDS endpoint - let did_doc = self - .resolver - .resolve_did(repo) - .await - .wrap_err_with(|| format!("Failed to resolve DID: {repo}"))? - .ok_or_else(|| eyre::eyre!("DID not found: {}", repo))?; - - // Find the PDS service endpoint - let pds_service = did_doc - .find_service_by_id("#atproto_pds") - .ok_or_else(|| eyre::eyre!("No atproto_pds service found for {}", repo))?; - - let pds_endpoint = &pds_service.service_endpoint; - - let url = format!("{pds_endpoint}/xrpc/com.atproto.repo.getRecord"); - - debug!( - "Fetching record from PDS {}: {}/{}/{}", - pds_endpoint, repo, collection, rkey - ); - - let response = self - .client - .get(&url) - .query(&[("repo", repo), ("collection", collection), ("rkey", rkey)]) - .send() - .await - .wrap_err_with(|| format!("Failed to send request to PDS: {pds_endpoint}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - debug!( - "PDS fetch failed for {}/{}/{}: {} - {}", - repo, collection, rkey, status, body - ); - return Err(eyre::eyre!("PDS returned error {}: {}", status, body)); - } - - let data: GetRecordResponse = response - .json() - .await - .wrap_err("Failed to parse PDS response")?; - - FetchedRecord::from_record_response(data) - } - - /// Fetch one page of records from a collection via listRecords - /// - /// # Parameters - /// - `repo`: DID of the repository - /// - `collection`: NSID of the collection (e.g., "app.bsky.graph.follow") - /// - `limit`: Number of records per page (1-100) - /// - `cursor`: Optional cursor for pagination - pub async fn list_records( - &self, - repo: &str, - collection: &str, - limit: u8, - cursor: Option<&str>, - ) -> Result { - // Resolve the DID to get the PDS endpoint - let did_doc = self - .resolver - .resolve_did(repo) - .await - .wrap_err_with(|| format!("Failed to resolve DID: {repo}"))? - .ok_or_else(|| eyre::eyre!("DID not found: {}", repo))?; - - // Find the PDS service endpoint - let pds_service = did_doc - .find_service_by_id("#atproto_pds") - .ok_or_else(|| eyre::eyre!("No atproto_pds service found for {}", repo))?; - - let pds_endpoint = &pds_service.service_endpoint; - - let url = format!("{pds_endpoint}/xrpc/com.atproto.repo.listRecords"); - - // Clamp limit to valid range (1-100) - let limit = limit.clamp(1, 100); - - debug!( - "Fetching records from PDS {}: repo={}, collection={}, limit={}", - pds_endpoint, repo, collection, limit - ); - - let mut request = self - .client - .get(&url) - .query(&[("repo", repo), ("collection", collection)]) - .query(&[("limit", limit.to_string())]); - - if let Some(cursor_val) = cursor { - request = request.query(&[("cursor", cursor_val)]); - } - - let response = request - .send() - .await - .wrap_err_with(|| format!("Failed to send listRecords request to PDS: {pds_endpoint}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - debug!( - "PDS listRecords failed for {}/{}: {} - {}", - repo, collection, status, body - ); - return Err(eyre::eyre!("PDS returned error {}: {}", status, body)); - } - - let data: ListRecordsResponse = response - .json() - .await - .wrap_err("Failed to parse PDS listRecords response")?; - - Ok(data) - } - - /// Fetch records from a collection (handles pagination automatically) - /// - /// This method will make multiple requests if needed, following cursors. - /// Defaults to fetching a maximum of 1000 records to avoid excessive HTTP requests - /// during backfill supplemental collection fetches (CAR file has most data already). - /// - /// # Parameters - /// - `repo`: DID of the repository - /// - `collection`: NSID of the collection (e.g., "app.bsky.graph.follow") - pub async fn list_all_records( - &self, - repo: &str, - collection: &str, - ) -> Result> { - self.list_all_records_with_limit(repo, collection, Some(1000)) - .await - } - - /// Fetch records from a collection with a configurable limit - /// - /// # Parameters - /// - `repo`: DID of the repository - /// - `collection`: NSID of the collection (e.g., "app.bsky.graph.follow") - /// - `max_records`: Maximum number of records to fetch (None for unlimited) - pub async fn list_all_records_with_limit( - &self, - repo: &str, - collection: &str, - max_records: Option, - ) -> Result> { - let mut all_records = Vec::new(); - let mut cursor: Option = None; - - loop { - // Check if we've hit the limit - if let Some(max) = max_records { - if all_records.len() >= max { - debug!( - "Reached limit of {} records for {}/{}", - max, repo, collection - ); - break; - } - } - - let response = self - .list_records(repo, collection, 100, cursor.as_deref()) - .await?; - - // Convert each list item to FetchedRecord - for item in response.records { - // Check limit before adding each record - if let Some(max) = max_records { - if all_records.len() >= max { - break; - } - } - let fetched = FetchedRecord::from_list_item(item)?; - all_records.push(fetched); - } - - // Check if there are more pages - match response.cursor { - Some(next_cursor) if !next_cursor.is_empty() => { - cursor = Some(next_cursor); - } - _ => break, // No more pages - } - } - - debug!( - "Fetched {} total records from {}/{} (limit: {:?})", - all_records.len(), - repo, - collection, - max_records - ); - - Ok(all_records) - } -} diff --git a/consumer/src/workers/fetch/sources/bluesky.rs b/consumer/src/workers/fetch/sources/bluesky.rs deleted file mode 100644 index bbf3433f..00000000 --- a/consumer/src/workers/fetch/sources/bluesky.rs +++ /dev/null @@ -1,95 +0,0 @@ -use super::super::types::{FetchedRecord, GetRecordResponse, RecordFetchConfig}; -use eyre::{Result, WrapErr as _}; -use reqwest::Client; -use std::time::Duration; -use tracing::debug; - -/// Client for fetching records from the public Bluesky API -/// -/// This acts as a third-tier fallback after Slingshot and PDS fail. -/// The public API is a distributed cache that provides high availability. -#[derive(Clone)] -pub struct BlueskyApiFetcher { - client: Client, - base_url: String, -} - -impl BlueskyApiFetcher { - pub fn new(config: &RecordFetchConfig) -> Result { - let client = Client::builder() - .timeout(Duration::from_secs(config.timeout_secs)) - .pool_max_idle_per_host(20) // Moderate pooling for public API - .pool_idle_timeout(Some(Duration::from_secs(60))) - .tcp_keepalive(Some(Duration::from_secs(30))) - .connect_timeout(Duration::from_secs(3)) - .build() - .wrap_err("Failed to create HTTP client for Bluesky API")?; - - Ok(Self { - client, - base_url: config.bluesky_api_url.clone(), - }) - } - - /// Fetch a record from the public Bluesky API - pub async fn fetch_by_uri(&self, at_uri: &str) -> Result { - // Parse the at-uri: at://did/collection/rkey - let parts: Vec<&str> = at_uri - .strip_prefix("at://") - .unwrap_or(at_uri) - .split('/') - .collect(); - if parts.len() != 3 { - return Err(eyre::eyre!("Invalid at-uri format: {}", at_uri)); - } - - let (repo, collection, rkey) = (parts[0], parts[1], parts[2]); - self.fetch_by_parts(repo, collection, rkey).await - } - - /// Fetch a record from the public Bluesky API by parts - pub async fn fetch_by_parts( - &self, - repo: &str, - collection: &str, - rkey: &str, - ) -> Result { - let url = format!("{}/xrpc/com.atproto.repo.getRecord", self.base_url); - - debug!( - "Fetching record from Bluesky API: {}/{}/{}", - repo, collection, rkey - ); - - let response = self - .client - .get(&url) - .query(&[("repo", repo), ("collection", collection), ("rkey", rkey)]) - .send() - .await - .wrap_err_with(|| { - format!("Failed to send request to Bluesky API: {}", self.base_url) - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - debug!( - "Bluesky API fetch failed for {}/{}/{}: {} - {}", - repo, collection, rkey, status, body - ); - return Err(eyre::eyre!( - "Bluesky API returned error {}: {}", - status, - body - )); - } - - let data: GetRecordResponse = response - .json() - .await - .wrap_err("Failed to parse Bluesky API response")?; - - FetchedRecord::from_record_response(data) - } -} diff --git a/consumer/src/workers/fetch/sources/mod.rs b/consumer/src/workers/fetch/sources/mod.rs deleted file mode 100644 index e5d8e0a9..00000000 --- a/consumer/src/workers/fetch/sources/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Record fetch sources -//! -//! Different backends for fetching AT Protocol records. - -pub mod bluesky; -pub mod slingshot; diff --git a/consumer/src/workers/fetch/sources/slingshot.rs b/consumer/src/workers/fetch/sources/slingshot.rs deleted file mode 100644 index 349747f4..00000000 --- a/consumer/src/workers/fetch/sources/slingshot.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::super::types::{FetchedRecord, GetUriRecordResponse, RecordFetchConfig}; -use eyre::{Result, WrapErr as _}; -use reqwest::Client; -use std::time::Duration; -use tracing::debug; - -/// Client for fetching records from Slingshot -#[derive(Clone)] -pub struct SlingshotFetcher { - client: Client, - base_url: String, -} - -impl SlingshotFetcher { - pub fn new(config: &RecordFetchConfig) -> Result { - let client = Client::builder() - .timeout(Duration::from_secs(config.timeout_secs)) - .pool_max_idle_per_host(50) // Keep many connections warm to single Slingshot host - .pool_idle_timeout(Some(Duration::from_secs(90))) - .tcp_keepalive(Some(Duration::from_secs(60))) - .http2_keep_alive_interval(Some(Duration::from_secs(30))) - .http2_keep_alive_while_idle(true) - .build() - .wrap_err("Failed to create HTTP client for Slingshot")?; - - Ok(Self { - client, - base_url: config.slingshot_url.clone(), - }) - } - - /// Fetch a record using the convenient at-uri endpoint - pub async fn fetch_by_uri(&self, at_uri: &str) -> Result { - let url = format!("{}/xrpc/com.bad-example.repo.getUriRecord", self.base_url); - - debug!("Fetching record from Slingshot: {}", at_uri); - - let response = self - .client - .get(&url) - .query(&[("at_uri", at_uri)]) - .send() - .await - .wrap_err("Failed to send request to Slingshot")?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - debug!( - "Slingshot fetch failed for {}: {} - {}", - at_uri, status, body - ); - return Err(eyre::eyre!("Slingshot returned error {}: {}", status, body)); - } - - let data: GetUriRecordResponse = response - .json() - .await - .wrap_err("Failed to parse Slingshot response")?; - - FetchedRecord::from_uri_response(data) - } -} diff --git a/consumer/src/workers/fetch/types.rs b/consumer/src/workers/fetch/types.rs deleted file mode 100644 index 0494d2d5..00000000 --- a/consumer/src/workers/fetch/types.rs +++ /dev/null @@ -1,98 +0,0 @@ -use eyre::Result; -use ipld_core::cid::Cid; -use serde::{Deserialize, Serialize}; - -/// Response from Slingshot's com.bad-example.repo.getUriRecord endpoint -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct GetUriRecordResponse { - pub uri: String, - pub cid: String, - pub value: serde_json::Value, -} - -/// Response from com.atproto.repo.getRecord (used by both Slingshot and PDS) -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct GetRecordResponse { - pub uri: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub cid: Option, - pub value: serde_json::Value, -} - -/// Response from com.atproto.repo.listRecords -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ListRecordsResponse { - pub records: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub cursor: Option, -} - -/// Individual record item from listRecords response -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ListRecordItem { - pub uri: String, - pub cid: String, - pub value: serde_json::Value, -} - -/// A fetched record with parsed components -#[derive(Debug, Clone)] -pub struct FetchedRecord { - pub cid: Cid, - pub value: serde_json::Value, - /// AT-URI of the record (optional, only present from listRecords) - pub uri: Option, -} - -impl FetchedRecord { - /// Parse from `GetUriRecordResponse` - pub fn from_uri_response(response: GetUriRecordResponse) -> Result { - let cid = Cid::try_from(response.cid.as_str())?; - Ok(Self { - cid, - value: response.value, - uri: Some(response.uri), - }) - } - - /// Parse from `GetRecordResponse` - pub fn from_record_response(response: GetRecordResponse) -> Result { - let cid_str = response - .cid - .ok_or_else(|| eyre::eyre!("Missing CID in response"))?; - let cid = Cid::try_from(cid_str.as_str())?; - Ok(Self { - cid, - value: response.value, - uri: Some(response.uri), - }) - } - - /// Parse from `ListRecordItem` (from listRecords response) - pub fn from_list_item(item: ListRecordItem) -> Result { - let cid = Cid::try_from(item.cid.as_str())?; - Ok(Self { - cid, - value: item.value, - uri: Some(item.uri), - }) - } -} - -/// Configuration for record fetching -#[derive(Debug, Clone)] -pub struct RecordFetchConfig { - pub slingshot_url: String, - pub bluesky_api_url: String, - pub timeout_secs: u64, -} - -impl Default for RecordFetchConfig { - fn default() -> Self { - Self { - slingshot_url: "https://slingshot.microcosm.blue".to_owned(), - bluesky_api_url: "https://public.api.bsky.app".to_owned(), - timeout_secs: 5, - } - } -} diff --git a/consumer/src/workers/fetch/worker.rs b/consumer/src/workers/fetch/worker.rs deleted file mode 100644 index 34a32df7..00000000 --- a/consumer/src/workers/fetch/worker.rs +++ /dev/null @@ -1,478 +0,0 @@ -use super::RecordFetcher; -use crate::config::RecordFetchConfig; -use crate::database_writer::ProcessedEvent; -use crate::worker_core::{Worker, WorkerFactory}; -use deadpool_postgres::Pool; -use eyre::Result; -use metrics::counter; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::mpsc::Sender; -use tokio::sync::watch::Receiver as WatchReceiver; -use tokio::sync::Semaphore; -use tokio_util::task::TaskTracker; -use tracing::{debug, error, info}; - -/// Manager for record fetch workers -pub struct RecordFetchManager { - pool: Pool, - fetcher: RecordFetcher, - config: RecordFetchConfig, - semaphore: Arc, - batch_writer_tx: Sender, -} - -impl RecordFetchManager { - pub async fn new( - pool: Pool, - fetcher: RecordFetcher, - config: RecordFetchConfig, - batch_writer_tx: Sender, - ) -> eyre::Result { - let semaphore = Arc::new(Semaphore::new(config.workers as usize)); - - Ok(Self { - pool, - fetcher, - config, - semaphore, - batch_writer_tx, - }) - } - - /// Run the fetch queue manager - /// - /// Uses PostgreSQL-based queue with FOR UPDATE SKIP LOCKED for worker coordination. - /// All database writes go through the batch writer - bounded channel provides backpressure. - pub async fn run( - mut self, - stop: WatchReceiver, - ) -> Result<()> { - let batch_writer_tx = self.batch_writer_tx.clone(); - let tracker = TaskTracker::new(); - let mut stats_timer = tokio::time::interval(Duration::from_secs(60)); // Log stats every minute - - info!( - "Record fetch manager started with {} workers (PostgreSQL-based queue)", - self.config.workers, - ); - - loop { - tokio::select! { - _ = stats_timer.tick() => { - // Log queue statistics - if let Err(e) = self.log_queue_stats().await { - error!("Failed to log queue stats: {}", e); - } - } - () = tokio::time::sleep(Duration::from_millis(1)) => { - if stop.has_changed().unwrap_or(true) { - info!("Stopping record fetch manager"); - let _ = tracker.close(); - break; - } - - // Spawn workers in a tight loop while permits and jobs are available - // PostgreSQL dequeue uses FOR UPDATE SKIP LOCKED for efficient coordination - loop { - // Check semaphore availability - if self.semaphore.available_permits() == 0 { - break; // No permits, exit inner loop - } - - // Get a job from PostgreSQL queue (non-blocking) - let job = match crate::db::fetch_queue::dequeue(&self.pool).await { - Ok(Some(item)) => Some(item), - Ok(None) => None, // No jobs available - Err(e) => { - error!("Failed to dequeue from PostgreSQL fetch queue: {}", e); - None - } - }; - - let Some(job) = job else { - break; // No jobs available, exit inner loop - }; - - // Acquire semaphore permit - let permit = match self.semaphore.clone().acquire_owned().await { - Ok(p) => p, - Err(e) => { - error!("Failed to acquire semaphore: {}", e); - break; - } - }; - - // Spawn worker task - let pool = self.pool.clone(); - let fetcher = self.fetcher.clone(); - let batch_writer_tx_clone = batch_writer_tx.clone(); - - drop(tracker.spawn(async move { - let _permit = permit; - let job_start = std::time::Instant::now(); - - debug!("Processing fetch job: {}", job.at_uri); - - let result = process_fetch_job( - &pool, - &fetcher, - &job, - &batch_writer_tx_clone, - ) - .await; - - match result { - Ok(()) => { - // Mark as complete in PostgreSQL (removes from queue) - // (Success logging happens inside process_fetch_job to distinguish skipped vs fetched) - let db_start = std::time::Instant::now(); - if let Err(e) = crate::db::fetch_queue::mark_complete(&pool, job.id).await { - error!("Failed to mark fetch as complete in PostgreSQL: {}", e); - } - let db_ms = db_start.elapsed().as_millis(); - let job_total_ms = job_start.elapsed().as_millis(); - - if job_total_ms > 500 { - debug!( - uri = %job.at_uri, - job_total_ms = job_total_ms, - db_complete_ms = db_ms, - "Slow job completion" - ); - } - } - Err(e) => { - debug!("Failed to fetch {}: {}", job.at_uri, e); - counter!("fetch_worker.failure").increment(1); - - // Mark as failed in PostgreSQL (will retry or permanently fail based on attempts) - if let Err(db_err) = crate::db::fetch_queue::mark_failed(&pool, job.id, &e.to_string()).await { - error!("Failed to mark fetch as failed in PostgreSQL: {}", db_err); - } else { - // If this was the final attempt (permanently failed), mark stub as missing in database - const MAX_ATTEMPTS: i32 = 3; - if job.attempts >= MAX_ATTEMPTS { - let pool_clone = pool.clone(); - let uri_clone = job.at_uri.clone(); - // Spawn async task to mark as missing (don't block worker) - tokio::spawn(async move { - match pool_clone.get().await { - Ok(conn) => { - if let Err(e) = crate::db::mark_stub_as_missing(&conn, &uri_clone).await { - error!("Failed to mark stub as missing: {}", e); - } - } - Err(e) => { - error!("Failed to get database connection to mark stub as missing: {}", e); - } - } - }); - } - } - } - } - })); - - // Continue inner loop to spawn more workers if permits/jobs available - } - - // Short sleep to allow stop signal checks and rate limit queue polling - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - } - - tracker.wait().await; - info!("Record fetch manager stopped"); - - Ok(()) - } - - /// Log queue statistics - async fn log_queue_stats(&mut self) -> Result<()> { - let (pending, processing, failed) = crate::db::fetch_queue::get_stats(&self.pool).await?; - - if pending > 0 || processing > 0 || failed > 0 { - info!( - "Fetch queue stats: pending={}, processing={}, failed={}", - pending, processing, failed - ); - } - - Ok(()) - } -} - -/// Process a single fetch job -/// -/// Fetches the record from the network and sends it to the batch writer for processing. -/// No database writes are done here - everything goes through the batch writer. -async fn process_fetch_job( - pool: &Pool, - fetcher: &RecordFetcher, - job: &crate::db::fetch_queue::FetchQueueItem, - batch_writer_tx: &Sender, -) -> Result<()> { - let total_start = std::time::Instant::now(); - let at_uri = &job.at_uri; - let attempts = job.attempts; - - // STEP 1: Check if record exists in the appropriate collection table (fast read query ~1ms) - // This prevents wasteful network fetches (50-200ms) - let pool_start = std::time::Instant::now(); - let conn = pool.get().await?; - let pool_ms = pool_start.elapsed().as_millis(); - - let exists_start = std::time::Instant::now(); - let exists = crate::db::record_exists(&conn, at_uri).await?; - let exists_ms = exists_start.elapsed().as_millis(); - - if exists { - let total_ms = total_start.elapsed().as_millis(); - if total_ms > 100 { - debug!( - uri = %at_uri, - total_ms = total_ms, - pool_ms = pool_ms, - exists_ms = exists_ms, - "Slow record_exists check" - ); - } - debug!("Record already exists, skipping fetch: {}", at_uri); - counter!("fetch_worker.skipped_exists").increment(1); - // Just return OK - caller will mark as complete in PostgreSQL - return Ok(()); - } - - // STEP 2: Parse URI - let parts: Vec<&str> = at_uri - .strip_prefix("at://") - .unwrap_or(at_uri) - .split('/') - .collect(); - - if parts.len() != 3 { - return Err(eyre::eyre!("Invalid at-uri format: {at_uri}")); - } - - let (repo, _collection, rkey) = (parts[0], parts[1], parts[2]); - - // STEP 3: Fetch record from network (expensive ~50-200ms) - let fetch_start = std::time::Instant::now(); - let fetched = if attempts == 1 { - // First attempt: Try Slingshot first - match fetcher.fetch_record(at_uri).await { - Ok(record) => record, - Err(_e) => { - // Slingshot failed - try PDS - fetcher.fetch_record_pds_only(at_uri).await? - } - } - } else { - // Retry: Go directly to PDS - fetcher.fetch_record_pds_only(at_uri).await? - }; - let fetch_ms = fetch_start.elapsed().as_millis(); - - // STEP 4: Parse the record - let record = RecordFetcher::parse_record(&fetched)?; - - // STEP 5: Resolve actor_id - // For profiles, resolve handle first, then create actor with handle - let actor_id = if matches!( - record, - crate::relay::types::RecordTypes::AppBskyActorProfile(_) - ) { - // Resolve handle using the RecordFetcher's resolver - let handle = match fetcher.resolver.resolve_did(repo).await { - Ok(Some(doc)) => { - // Extract handle from DID document's alsoKnownAs field - doc.also_known_as - .and_then(|aka| aka.first().cloned()) - .and_then(|uri| uri.strip_prefix("at://").map(|s| s.to_string())) - } - Ok(None) => { - debug!("DID document not found for {}", repo); - counter!("fetch_worker.handle_resolution_not_found").increment(1); - None - } - Err(e) => { - debug!("Failed to resolve handle for {}: {}", repo, e); - counter!("fetch_worker.handle_resolution_failure").increment(1); - None - } - }; - - if handle.is_some() { - counter!("fetch_worker.handle_resolution_success").increment(1); - } - - // Ensure actor exists with resolved handle - crate::db::actor::ensure_actor_id( - &conn, - repo, - None, // status - handle.as_deref(), - chrono::Utc::now(), - ) - .await? - } else { - // For non-profile records, just ensure actor exists - crate::db::actor::ensure_actor_id( - &conn, - repo, - None, // status - None, // handle - chrono::Utc::now(), - ) - .await? - }; - - // Extract references from record (subject DIDs, service DIDs, etc.) - let refs = crate::database_writer::extract_references(&record); - - // Resolve service actor for feedgens - // For FeedGenerator records, the first (and only) additional DID is the service actor - let service_actor_id = if let Some(first_did) = refs.additional_dids.first() { - match crate::db::operations::feed::get_actor_id(&conn, first_did).await { - Ok((actor_id, _, _)) => Some(actor_id), - Err(e) => { - tracing::warn!( - at_uri = %at_uri, - service_did = %first_did, - error = ?e, - "Failed to resolve service actor ID for feedgen - skipping record" - ); - None - } - } - } else { - None - }; - - let resolved_actor_ids = crate::database_writer::workers::ResolvedActorIds { - subject_actor_id: None, - parent_author_actor_id: None, - root_author_actor_id: None, - quoted_author_actor_id: None, - mentioned_actor_ids: Vec::new(), - service_actor_id, - via_repost_key: None, - }; - let mut processed = crate::database_writer::process_record_to_operations( - repo, - actor_id, - resolved_actor_ids, - fetched.cid, - record, - at_uri.to_string(), - rkey.to_string(), - crate::database_writer::EventSource::FetchQueue, - ); - - let mut operations = Vec::new(); - - // Merge operations - operations.append(&mut processed.operations); - - // Create ProcessedEvent for batch writer - let event = ProcessedEvent { - operations, - - cursor: None, - source: crate::database_writer::EventSource::FetchQueue, - }; - - // STEP 6: Send to batch writer (bounded channel with backpressure) - // Fetch workers already resolve actor_ids, so send as Resolved - let send_start = std::time::Instant::now(); - if let Err(e) = batch_writer_tx - .send(crate::database_writer::WriterEvent::Resolved(Box::new(event))) - .await - { - error!("Failed to send fetch event to batch writer: {}", e); - return Err(e.into()); - } - let send_ms = send_start.elapsed().as_millis(); - - counter!("fetch_worker.events_sent_to_batch_writer").increment(1); - - let total_ms = total_start.elapsed().as_millis(); - - // Log success after actually fetching from network - if total_ms > 500 || send_ms > 100 { - // Log detailed timing for slow fetches (>500ms) or slow sends (>100ms) - tracing::warn!( - uri = %at_uri, - total_ms = total_ms, - pool_ms = pool_ms, - exists_ms = exists_ms, - fetch_ms = fetch_ms, - send_ms = send_ms, - "Slow fetch detected" - ); - } - - info!( - "Successfully fetched from network: {} ({}ms)", - at_uri, total_ms - ); - counter!("fetch_worker.success").increment(1); - - Ok(()) -} - -// Implement Worker trait - delegates to existing run() method -impl Worker for RecordFetchManager { - fn name(&self) -> &'static str { - "record_fetch" - } - - async fn run(self, stop: WatchReceiver) -> Result<()> { - RecordFetchManager::run(self, stop).await - } -} - -/// Factory for creating RecordFetchManager instances -#[derive(Clone)] -pub struct RecordFetchManagerFactory { - pool: Pool, - fetcher: RecordFetcher, - config: RecordFetchConfig, - batch_writer_tx: Sender, -} - -impl RecordFetchManagerFactory { - pub fn new( - pool: Pool, - fetcher: RecordFetcher, - config: RecordFetchConfig, - batch_writer_tx: Sender, - ) -> Self { - Self { - pool, - fetcher, - config, - batch_writer_tx, - } - } -} - -impl WorkerFactory for RecordFetchManagerFactory { - type Worker = RecordFetchManager; - - fn name(&self) -> &'static str { - "record_fetch" - } - - async fn create(&self) -> Result { - RecordFetchManager::new( - self.pool.clone(), - self.fetcher.clone(), - self.config.clone(), - self.batch_writer_tx.clone(), - ) - .await - } -} diff --git a/consumer/src/workers/handle_resolver.rs b/consumer/src/workers/handle_resolver.rs deleted file mode 100644 index 67d6b77f..00000000 --- a/consumer/src/workers/handle_resolver.rs +++ /dev/null @@ -1,284 +0,0 @@ -//! Handle resolution worker -//! -//! This module provides a background worker that resolves DID handles asynchronously. -//! It processes DIDs from a PostgreSQL queue and updates actor handles in the database. - -use deadpool_postgres::Pool; -use did_resolver::Resolver; -use eyre::Result; -use metrics::counter; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::watch::Receiver as WatchReceiver; -use tokio::sync::Semaphore; -use tokio_util::task::TaskTracker; -use tracing::{debug, error, info}; - -use crate::worker_core::{Worker, WorkerFactory}; - -/// Manager for handle resolution workers -pub struct HandleResolutionManager { - pool: Pool, - resolver: Arc, - semaphore: Arc, - workers: u32, -} - -impl HandleResolutionManager { - pub async fn new( - pool: Pool, - resolver: Arc, - workers: u32, - ) -> eyre::Result { - let semaphore = Arc::new(Semaphore::new(workers as usize)); - - Ok(Self { - pool, - resolver, - semaphore, - workers, - }) - } - - /// Run the handle resolution manager - /// - /// Uses PostgreSQL-based queue with FOR UPDATE SKIP LOCKED for worker coordination. - /// Implements exponential backoff when queue is empty to reduce database load. - pub async fn run( - mut self, - stop: WatchReceiver, - ) -> Result<()> { - let tracker = TaskTracker::new(); - let mut stats_timer = tokio::time::interval(Duration::from_secs(60)); - - // Exponential backoff: 100ms -> 200ms -> 400ms -> 800ms -> 1.6s -> 3.2s -> 5s (max) - let mut backoff_ms = 100u64; - const MAX_BACKOFF_MS: u64 = 5000; - const MIN_BACKOFF_MS: u64 = 100; - - info!( - "Handle resolution manager started with {} workers (exponential backoff polling)", - self.workers, - ); - - loop { - tokio::select! { - _ = stats_timer.tick() => { - if let Err(e) = self.log_queue_stats().await { - error!("Failed to log queue stats: {}", e); - } - } - () = tokio::time::sleep(Duration::from_millis(backoff_ms)) => { - if stop.has_changed().unwrap_or(true) { - info!("Stopping handle resolution manager"); - tracker.close(); - break; - } - - let mut jobs_processed = 0; - - // Process all available jobs - loop { - if self.semaphore.available_permits() == 0 { - break; - } - - let job = match crate::db::handle_resolution_queue::dequeue(&self.pool).await { - Ok(Some(item)) => item, - Ok(None) => break, - Err(e) => { - error!("Failed to dequeue from handle resolution queue: {}", e); - break; - } - }; - - jobs_processed += 1; - - let permit = match self.semaphore.clone().acquire_owned().await { - Ok(p) => p, - Err(e) => { - error!("Failed to acquire semaphore: {}", e); - break; - } - }; - - let pool = self.pool.clone(); - let resolver = self.resolver.clone(); - - drop(tracker.spawn(async move { - let _permit = permit; - debug!("Processing handle resolution job: {}", job.did); - - let result = process_handle_resolution_job(&pool, &resolver, &job).await; - - match result { - Ok(()) => { - if let Err(e) = crate::db::handle_resolution_queue::mark_complete(&pool, job.id).await { - error!("Failed to mark handle resolution as complete: {}", e); - } - counter!("handle_resolution.success").increment(1); - } - Err(e) => { - debug!("Failed to resolve handle for {}: {}", job.did, e); - counter!("handle_resolution.failure").increment(1); - - if let Err(db_err) = crate::db::handle_resolution_queue::mark_failed(&pool, job.id, &e.to_string()).await { - error!("Failed to mark handle resolution as failed: {}", db_err); - } - } - } - })); - } - - // Adjust backoff based on whether we found work - if jobs_processed > 0 { - backoff_ms = MIN_BACKOFF_MS; // Reset to minimum when work found - } else { - backoff_ms = (backoff_ms * 2).min(MAX_BACKOFF_MS); // Double up to max - } - } - } - } - - tracker.wait().await; - info!("Handle resolution manager stopped"); - Ok(()) - } - - /// Log queue statistics - async fn log_queue_stats(&mut self) -> Result<()> { - let (pending, processing, failed) = crate::db::handle_resolution_queue::get_stats(&self.pool).await?; - - if pending > 0 || processing > 0 || failed > 0 { - info!( - "Handle resolution queue stats: pending={}, processing={}, failed={}", - pending, processing, failed - ); - } - - Ok(()) - } -} - -/// Process a single handle resolution job -/// -/// Resolves the DID to a handle and updates the actor record in the database. -/// For did:plc DIDs, also fetches and stores the account creation timestamp. -async fn process_handle_resolution_job( - pool: &Pool, - resolver: &Arc, - job: &crate::db::handle_resolution_queue::HandleResolutionItem, -) -> Result<()> { - let did = &job.did; - - // Resolve DID to get handle from DID document - let handle = match resolver.resolve_did(did).await { - Ok(Some(doc)) => { - // Extract handle from DID document's alsoKnownAs field - doc.also_known_as - .and_then(|aka| aka.first().cloned()) - .and_then(|uri| uri.strip_prefix("at://").map(|s| s.to_string())) - } - Ok(None) => { - debug!("DID document not found for {}", did); - counter!("handle_resolution.did_not_found").increment(1); - return Err(eyre::eyre!("DID document not found")); - } - Err(e) => { - debug!("Failed to resolve DID for {}: {}", did, e); - counter!("handle_resolution.did_resolution_failure").increment(1); - return Err(e.into()); - } - }; - - let Some(handle) = handle else { - debug!("No handle found in DID document for {}", did); - counter!("handle_resolution.no_handle_in_did_doc").increment(1); - return Err(eyre::eyre!("No handle in DID document")); - }; - - // For did:plc, fetch account creation timestamp from PLC audit log - let account_created_at = if did.starts_with("did:plc:") { - match resolver.get_plc_creation_time(did).await { - Ok(Some(created_at)) => { - debug!("Got PLC creation time for {}: {}", did, created_at); - Some(created_at) - } - Ok(None) => { - debug!("No PLC creation time found for {}", did); - None - } - Err(e) => { - debug!("Failed to get PLC creation time for {}: {}", did, e); - None // Don't fail the whole job if we can't get creation time - } - } - } else { - None // did:web and other methods don't have authoritative creation timestamps - }; - - // Update actor handle and account_created_at in database - // Only update account_created_at if it's NULL (don't overwrite existing values) - let conn = pool.get().await?; - - // Use consolidated ActorUpdate API - use crate::db::operations::{ActorUpdate, ActorUpdateResult, ActorUpdateTarget}; - - let result = ActorUpdate { - target: ActorUpdateTarget::ByDid(did.to_string()), - handle: Some(handle.clone()), - account_created_at, - account_created_at_coalesce: true, // Don't overwrite existing values - ..Default::default() - } - .execute(&conn) - .await?; - - match result { - ActorUpdateResult::Count(_) => {}, - _ => unreachable!("ActorUpdate with Count returning should return Count"), - } - - debug!("Resolved handle for {}: {}", did, handle); - counter!("handle_resolution.success").increment(1); - - Ok(()) -} - -// Implement Worker trait -impl Worker for HandleResolutionManager { - fn name(&self) -> &'static str { - "handle_resolution" - } - - async fn run(self, stop: WatchReceiver) -> Result<()> { - // Delegate to the main run method - self.run(stop).await - } -} - -/// Factory for creating HandleResolutionManager instances -#[derive(Clone)] -pub struct HandleResolutionManagerFactory { - pool: Pool, - resolver: Arc, - workers: u32, -} - -impl HandleResolutionManagerFactory { - pub fn new(pool: Pool, resolver: Arc, workers: u32) -> Self { - Self { pool, resolver, workers } - } -} - -impl WorkerFactory for HandleResolutionManagerFactory { - type Worker = HandleResolutionManager; - - fn name(&self) -> &'static str { - "handle_resolution" - } - - async fn create(&self) -> Result { - HandleResolutionManager::new(self.pool.clone(), self.resolver.clone(), self.workers).await - } -} diff --git a/consumer/src/workers/jetstream/decompression.rs b/consumer/src/workers/jetstream/decompression.rs deleted file mode 100644 index 5708235c..00000000 --- a/consumer/src/workers/jetstream/decompression.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! Jetstream message decompression -//! -//! This module handles decompression of binary Jetstream messages. -//! Jetstream can send zstd-compressed messages to reduce bandwidth. - -/// Decompress a binary Jetstream message -/// -/// Checks for zstd compression magic bytes (0x28 0xB5 0x2F 0xFD) and decompresses -/// if present. Returns an error if the data appears compressed but decompression fails. -/// -/// # Arguments -/// -/// * `data` - Raw binary data from Jetstream websocket -/// -/// # Returns -/// -/// * `Ok(String)` - Decompressed UTF-8 text -/// * `Err(&'static str)` - Error message if decompression or UTF-8 conversion fails -/// -/// # Examples -/// -/// ```ignore -/// match decompress_jetstream_message(&binary_data) { -/// Ok(text) => process_text(text), -/// Err(e) => tracing::error!("Decompression failed: {}", e), -/// } -/// ``` -pub fn decompress_jetstream_message(data: &[u8]) -> Result { - // Check if data is zstd compressed (magic bytes: 0x28 0xB5 0x2F 0xFD) - if data.len() >= 4 && data[0..4] == [0x28, 0xB5, 0x2F, 0xFD] { - // Decompress using JetstreamConsumer's helper - let decompressed = crate::sources::jetstream::JetstreamConsumer::decompress_zstd(data) - .map_err(|_| "Failed to decompress zstd message")?; - - // Convert to UTF-8 string - String::from_utf8(decompressed) - .map_err(|_| "Failed to convert decompressed data to UTF-8") - } else { - Err("Binary message is not zstd compressed") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_rejects_uncompressed_binary() { - let uncompressed = b"not compressed"; - let result = decompress_jetstream_message(uncompressed); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "Binary message is not zstd compressed"); - } - - #[test] - fn test_rejects_invalid_magic_bytes() { - let invalid = b"\x00\x00\x00\x00some data"; - let result = decompress_jetstream_message(invalid); - assert!(result.is_err()); - assert_eq!(result.unwrap_err(), "Binary message is not zstd compressed"); - } - - #[test] - fn test_rejects_too_short() { - let too_short = b"\x28\xB5"; // Only 2 bytes - let result = decompress_jetstream_message(too_short); - assert!(result.is_err()); - } -} diff --git a/consumer/src/workers/jetstream/filtering.rs b/consumer/src/workers/jetstream/filtering.rs deleted file mode 100644 index 3579cbb4..00000000 --- a/consumer/src/workers/jetstream/filtering.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Jetstream message utilities -//! -//! This module provides utilities for working with Jetstream JSON events. - -/// Fast timestamp extraction from JSON string without full parsing -/// -/// Looks for "time_us":12345 pattern and extracts the timestamp. -/// This is faster than parsing the entire JSON just to get the timestamp. -/// -/// # Arguments -/// -/// * `content` - JSON string from Jetstream -/// -/// # Returns -/// -/// * `Some(u64)` - Timestamp in microseconds if found -/// * `None` - If timestamp field not found or invalid -pub fn extract_time_us(content: &str) -> Option { - if let Some(start) = content.find("\"time_us\":") { - let rest = &content[start + 10..]; - if let Some(end) = rest.find(&[',', '}'][..]) { - if let Ok(time_us) = rest[..end].trim().parse::() { - return Some(time_us); - } - } - } - None -} diff --git a/consumer/src/workers/jetstream/handler.rs b/consumer/src/workers/jetstream/handler.rs deleted file mode 100644 index 3ca8aa5e..00000000 --- a/consumer/src/workers/jetstream/handler.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! Database-free Jetstream event processing -//! -//! This module processes Jetstream events into `ProcessedEvent` structures -//! without touching the database. Workers become pure functions. - -use super::decompression::decompress_jetstream_message; -use super::filtering::extract_time_us; -use super::processing::{ - process_account_event, process_commit_to_unresolved_events, process_identity_event, -}; -use super::RequestContext; -use crate::events::RawJetstreamData; -use crate::indexer::convert_jetstream_commit_worker; - -/// Process a raw Jetstream event (text or binary) without database operations -/// -/// Returns (WriterEvents, is_commit, timestamp) tuple. -/// WriterEvents contains all events to send to the database writer, or empty vec if no operations produced. -/// is_commit indicates whether this was a Commit event (true) vs Identity/Account event (false). -/// timestamp (time_us) is always extracted for cursor tracking. -pub async fn process_raw_jetstream_dbfree( - raw_data: RawJetstreamData, - req_ctx: &mut RequestContext, -) -> (Vec, bool, Option) { - match raw_data { - RawJetstreamData::Text { content, .. } => { - process_jetstream_text_dbfree(content, req_ctx).await - } - RawJetstreamData::Binary { data, .. } => { - process_jetstream_binary_dbfree(data, req_ctx).await - } - } -} - -/// Process a text Jetstream message without database operations -/// -/// Returns (WriterEvents, was_allowlisted, timestamp) tuple -async fn process_jetstream_text_dbfree( - content: String, - req_ctx: &mut RequestContext, -) -> (Vec, bool, Option) { - // Extract timestamp first (before any early returns) - let time_us = extract_time_us(&content); - - // Check if it's an error message first - if let Ok(error) = serde_json::from_str::(&content) { - tracing::warn!("Jetstream error received by worker: {:?}", error); - return (Vec::new(), false, time_us); - } - - // Parse the JSON event - match serde_json::from_str::(&content) { - Ok(event) => { - // Process event and check allowlist - let (events, allowlisted) = process_jetstream_event_dbfree(event, req_ctx).await; - (events, allowlisted, time_us) - } - Err(e) => { - // Log parsing error with event content for debugging - // Truncate very long events to avoid log spam - let preview = if content.len() > 500 { - format!("{}... ({} bytes total)", &content[..500], content.len()) - } else { - content.clone() - }; - tracing::warn!( - error = %e, - event_preview = %preview, - "Failed to parse Jetstream text event (compression requested, text event received?)" - ); - (Vec::new(), false, time_us) - } - } -} - -/// Process a binary (potentially compressed) Jetstream message without database operations -/// -/// Returns (WriterEvents, was_allowlisted, timestamp) tuple -async fn process_jetstream_binary_dbfree( - data: Vec, - req_ctx: &mut RequestContext, -) -> (Vec, bool, Option) { - // Decompress directly - zstd decompression is fast (<1-2ms per message) - // and we're already in a worker task, so no need for spawn_blocking overhead - match decompress_jetstream_message(&data) { - Ok(text) => process_jetstream_text_dbfree(text, req_ctx).await, - Err(e) => { - tracing::warn!("Worker failed to decompress binary message: {}", e); - (Vec::new(), false, None) - } - } -} - -/// Process a parsed Jetstream event (Commit, Identity, or Account) without database operations -/// -/// Returns (WriterEvents, is_commit) tuple where is_commit indicates if this was a Commit event. -/// With server-side filtering, all Commit events are from allowlisted DIDs. -async fn process_jetstream_event_dbfree( - event: crate::sources::jetstream::JetstreamEvent, - _req_ctx: &mut RequestContext, -) -> (Vec, bool) { - match event { - crate::sources::jetstream::JetstreamEvent::Commit(commit) => { - // NOTE: Jetstream filters events server-side using wantedDids subscription option - // We only receive commits from allowlisted DIDs - - // Convert the commit to internal event format - let converted = match convert_jetstream_commit_worker(&commit) { - Some(c) => c, - None => return (Vec::new(), false), - }; - - // Process commit operations into UnresolvedEvents - let events = process_commit_to_unresolved_events(converted).await; - (events, true) // All commits from Jetstream are allowlisted - } - crate::sources::jetstream::JetstreamEvent::Identity(identity) => { - // Process identity events (handle updates) - // Identity events are always processed regardless of allowlist - let event = process_identity_event(identity); - // Identity events are simple UpsertActor operations, wrap in Resolved - let events = event.into_iter().map(|e| crate::database_writer::WriterEvent::Resolved(Box::new(e))).collect(); - (events, false) // Not a commit event - } - crate::sources::jetstream::JetstreamEvent::Account(account) => { - // Process account events (status changes) - // Account events are always processed regardless of allowlist - let event = process_account_event(account); - // Account events are simple UpsertActor operations, wrap in Resolved - let events = event.into_iter().map(|e| crate::database_writer::WriterEvent::Resolved(Box::new(e))).collect(); - (events, false) // Not a commit event - } - } -} - diff --git a/consumer/src/workers/jetstream/mod.rs b/consumer/src/workers/jetstream/mod.rs deleted file mode 100644 index 60136437..00000000 --- a/consumer/src/workers/jetstream/mod.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Jetstream event processing workers -//! -//! Database-free workers that process Jetstream events and produce DatabaseOperation structures. - -pub mod decompression; -pub mod filtering; -pub mod handler; -pub mod processing; -pub mod request_context; -pub mod worker; - -// process_raw_jetstream_dbfree is internal to workers, not exposed publicly -pub use request_context::RequestContext; -pub use worker::spawn_workers; diff --git a/consumer/src/workers/jetstream/processing.rs b/consumer/src/workers/jetstream/processing.rs deleted file mode 100644 index 7b43a377..00000000 --- a/consumer/src/workers/jetstream/processing.rs +++ /dev/null @@ -1,186 +0,0 @@ -//! Jetstream event processing logic -//! -//! This module contains the core business logic for converting parsed Jetstream events -//! into database operations. It handles: -//! - Commit events (create/update/delete operations) -//! - Identity events (handle updates) -//! - Account events (status changes) - -use crate::database_writer::{EventSource, ProcessedEvent}; -use crate::parsing::parse_car_blocks; -use std::collections::HashMap; - -/// Process a commit event into UnresolvedEvents (async for CAR parsing) -/// -/// Each create/update operation becomes an UnresolvedEvent. -/// Delete operations are processed immediately since they don't need actor_id resolution. -/// -/// NOTE: With Jetstream server-side filtering, all commits are from allowlisted DIDs. -pub async fn process_commit_to_unresolved_events( - commit: crate::events::AtpCommitEvent, -) -> Vec { - use crate::indexer::operations::decode::decode_op; - - if commit.ops.is_empty() { - return Vec::new(); - } - - // Parse blocks into a map - let blocks = if commit.blocks.is_empty() { - HashMap::new() - } else { - // Parse CAR data - match parse_car_blocks(&commit.blocks).await { - Ok(b) => b, - Err(e) => { - tracing::warn!("Failed to parse CAR blocks: {}", e); - return Vec::new(); - } - } - }; - - // Collect all events from this commit - let mut events = Vec::new(); - - for op in &commit.ops { - let Some((collection_raw, rkey)) = op.path.split_once('/') else { - tracing::warn!("op contained invalid path {}", op.path); - continue; - }; - - let full_path = format!("at://{}/{}", commit.repo, &op.path); - - if op.action == "create" || op.action == "update" { - let Some((cid, decoded)) = decode_op(op, &blocks) else { - // decode_op already logs warnings for actual errors, skip logging here - // (avoids spam from unimplemented record types like app.bsky.actor.status) - continue; - }; - - // Jetstream workers stay database-free: create UnresolvedEvent - // The database writer will resolve actor_id and process the record - // NOTE: All commits from Jetstream are from allowlisted DIDs (server-side filtered) - let unresolved = crate::database_writer::UnresolvedEvent { - repo: commit.repo.clone(), - event_type: crate::database_writer::UnresolvedEventType::CreateUpdate { - record: Box::new(decoded), - cid, - }, - at_uri: full_path.clone(), - rkey: rkey.to_string(), - source: EventSource::Jetstream, - cursor: None, // Cursor will be attached by caller - }; - - events.push(crate::database_writer::WriterEvent::Unresolved(Box::new(unresolved))); - } else if op.action == "delete" { - // Handle delete operations - create UnresolvedEvent with Delete type - use crate::relay::types::CollectionType; - - let unresolved = crate::database_writer::UnresolvedEvent { - repo: commit.repo.clone(), - event_type: crate::database_writer::UnresolvedEventType::Delete { - collection: CollectionType::from_str(collection_raw), - }, - at_uri: full_path.clone(), - rkey: rkey.to_string(), - source: EventSource::Jetstream, - cursor: None, // Cursor will be attached by caller - }; - - events.push(crate::database_writer::WriterEvent::Unresolved(Box::new(unresolved))); - } - } - - events -} - - -/// Process an identity event (handle update) -pub fn process_identity_event( - identity: crate::sources::jetstream::IdentityEvent, -) -> Option { - use crate::database_writer::DatabaseOperation; - use parakeet_db::types::ActorSyncState; - - let did = &identity.identity.did; - let handle = identity.identity.handle; // Already Option - - // Parse timestamp from ISO string - let timestamp = match chrono::DateTime::parse_from_rfc3339(&identity.identity.time) { - Ok(dt) => dt.with_timezone(&chrono::Utc), - Err(_) => chrono::Utc::now(), // Fallback to current time if parsing fails - }; - - // Always pass Partial - the SQL ON CONFLICT clause preserves existing sync_state - // New actors get Partial, existing actors keep their current state - let sync_state = ActorSyncState::Partial; - - // Note: Handle validation against DID doc is skipped in database-free mode - // This would require async DID resolution which we want to avoid in workers - // The legacy code had a do_handle_res flag to control this - - let operations = vec![DatabaseOperation::UpsertActor { - did: did.to_string(), - status: None, - sync_state, - handle, // Pass the Option directly - account_created_at: None, // Jetstream doesn't provide creation time, enriched during handle resolution - timestamp, - }]; - - Some(ProcessedEvent { - operations, - - cursor: None, // Cursor is set by caller - source: EventSource::Jetstream, - }) -} - -/// Process an account event (status change) -pub fn process_account_event( - account: crate::sources::jetstream::AccountEvent, -) -> Option { - use crate::database_writer::DatabaseOperation; - use parakeet_db::types::{ActorStatus, ActorSyncState}; - - let did = &account.account.did; - let status = if account.account.active { - ActorStatus::Active - } else { - // When active=false, we don't know if it's Deactivated or Takendown - // Default to Deactivated (the legacy code had more context for this) - ActorStatus::Deactivated - }; - - // Parse timestamp from ISO string - let timestamp = match chrono::DateTime::parse_from_rfc3339(&account.account.time) { - Ok(dt) => dt.with_timezone(&chrono::Utc), - Err(_) => chrono::Utc::now(), - }; - - // Always pass Partial - the SQL ON CONFLICT clause preserves existing sync_state - // New actors get Partial, existing actors keep their current state - let sync_state = ActorSyncState::Partial; - - let operations = vec![DatabaseOperation::UpsertActor { - did: did.to_string(), - status: Some(status), - sync_state, - handle: None, - account_created_at: None, // Jetstream doesn't provide creation time, enriched during handle resolution - timestamp, - }]; - - // Note: Backfill triggering logic is skipped in database-free mode - // The legacy code would check if actor was coming out of inactive state - // and trigger backfill. This could be added as an EnqueueBackfill - // operation in the future if needed. - - Some(ProcessedEvent { - operations, - - cursor: None, // Cursor is set by caller - source: EventSource::Jetstream, - }) -} diff --git a/consumer/src/workers/jetstream/request_context.rs b/consumer/src/workers/jetstream/request_context.rs deleted file mode 100644 index 54684e10..00000000 --- a/consumer/src/workers/jetstream/request_context.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Request-scoped context for event processing -//! -//! This module provides a RequestContext that holds a reference to the allowlist -//! for checking allowlist status during event processing. - -use crate::db; - -/// Request-scoped context for event processing -/// -/// Holds a reference to the allowlist for checking allowlist status -/// during a single event's processing lifetime. -pub struct RequestContext { - /// Reference to allowlist for checking allowlist status - allowlist: db::Allowlist, - /// Optional retention cutoff timestamp (records older than this are filtered) - retention_cutoff: Option>, -} - -impl RequestContext { - /// Create a new request context - pub fn new(allowlist: db::Allowlist) -> Self { - Self { - allowlist, - retention_cutoff: None, - } - } - - /// Create a new request context with retention cutoff - pub fn with_retention( - allowlist: db::Allowlist, - retention_cutoff: Option>, - ) -> Self { - Self { - allowlist, - retention_cutoff, - } - } - - /// Get the retention cutoff timestamp - pub fn retention_cutoff(&self) -> Option> { - self.retention_cutoff - } - - /// Check if a DID is fully allowed on the allowlist - /// - /// This is the primary method for checking allowlist status during event processing. - /// It checks the actual allowlist table (via the cached allowlist). - /// Returns true only for DIDs that are in the allowlist table. - pub fn get_allowlist_status(&self, did: &str) -> bool { - self.allowlist.cache.contains_did(did) - } - - /// Emit metrics for this request (currently a no-op) - pub fn emit_metrics(self) { - // No metrics to emit for now - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_allowlist_consistent_results() { - // Question: Does the allowlist cache work correctly for repeated checks? - let allowlist = db::Allowlist::new(); - let ctx = RequestContext::new(allowlist); - - // Test that allowlist status returns consistent results - let did = "did:plc:test123"; - - let result1 = ctx.get_allowlist_status(did); - let result2 = ctx.get_allowlist_status(did); - let result3 = ctx.get_allowlist_status(did); - - // Results should be consistent (all false since allowlist is empty) - assert_eq!(result1, result2); - assert_eq!(result2, result3); - } - - #[test] - fn test_different_dids() { - // Question: Does the allowlist check work for different DIDs? - let allowlist = db::Allowlist::new(); - let ctx = RequestContext::new(allowlist); - - // Check three different DIDs - let r1 = ctx.get_allowlist_status("did:plc:user1"); - let r2 = ctx.get_allowlist_status("did:plc:user2"); - let r3 = ctx.get_allowlist_status("did:plc:user3"); - - // All should be false (allowlist is empty) - assert!(!r1); - assert!(!r2); - assert!(!r3); - } - - #[test] - fn test_allowlist_checks_consistent() { - // Question: Are repeated allowlist checks consistent? - let allowlist = db::Allowlist::new(); - let ctx = RequestContext::new(allowlist); - - // Check allowlist multiple times - let a1 = ctx.get_allowlist_status("did:plc:user1"); - let a2 = ctx.get_allowlist_status("did:plc:user1"); - let a3 = ctx.get_allowlist_status("did:plc:user1"); - - // All should be consistent - assert_eq!(a1, a2); - assert_eq!(a2, a3); - } -} diff --git a/consumer/src/workers/jetstream/worker.rs b/consumer/src/workers/jetstream/worker.rs deleted file mode 100644 index 1c872771..00000000 --- a/consumer/src/workers/jetstream/worker.rs +++ /dev/null @@ -1,156 +0,0 @@ -use super::RequestContext; -use crate::db; -use crate::events::IndexerEvent; -use metrics::counter; -use tokio::sync::mpsc::{channel, Receiver, Sender}; -use tokio::task::JoinHandle; - -struct WorkerContext { - allowlist: db::Allowlist, - batch_writer_tx: - Option>, - worker_events_processed: std::sync::Arc, - worker_events_dropped: std::sync::Arc, -} - -/// Spawn worker threads for processing indexer events -pub fn spawn_workers( - threads: u8, - allowlist: &db::Allowlist, - batch_writer_tx: Option< - tokio::sync::mpsc::Sender, - >, - worker_events_processed: std::sync::Arc, - worker_events_dropped: std::sync::Arc, -) -> (Vec>, Vec>) { - let result = (0..threads) - .map(|idx| { - let (tx, mut rx) = channel(1000); - let mut ctx = WorkerContext { - allowlist: allowlist.clone(), - batch_writer_tx: batch_writer_tx.clone(), - worker_events_processed: worker_events_processed.clone(), - worker_events_dropped: worker_events_dropped.clone(), - }; - - let handle = tokio::spawn(async move { - run_worker(idx, &mut ctx, &mut rx).await; - }); - - (tx, handle) - }) - .unzip(); - - tracing::info!("Started {} Jetstream workers", threads); - result -} - -/// Run a single worker thread, processing events from the channel -async fn run_worker(idx: u8, ctx: &mut WorkerContext, rx: &mut Receiver) { - tracing::debug!("Worker {} started", idx); - - while let Some(event) = rx.recv().await { - // DATABASE-FREE ARCHITECTURE: Workers produce ProcessedEvent structures - // containing all data needed for database writes, but never touch the database - match event { - IndexerEvent::RawJetstream(raw_data) => { - // Create request context for allowlist checking - let mut req_ctx = RequestContext::new(ctx.allowlist.clone()); - - // Process the event without database operations - let (mut events, _is_allowlisted, time_us) = - super::handler::process_raw_jetstream_dbfree(raw_data, &mut req_ctx).await; - - if events.is_empty() { - // Event produced no operations (Identity/Account event or parse failure) - // NOTE: Jetstream filters commits server-side, so this is NOT an allowlist filter - // Still need to update cursor to track our position - if let Some(time_us) = time_us { - if let Some(ref batch_tx) = ctx.batch_writer_tx { - // Send empty ProcessedEvent with just cursor update - let cursor_only_event = crate::database_writer::ProcessedEvent { - operations: vec![], - cursor: Some(time_us), - source: crate::database_writer::EventSource::Jetstream, - }; - - // Bounded channel: send().await provides backpressure - if let Err(e) = batch_tx.send(crate::database_writer::WriterEvent::Resolved(Box::new(cursor_only_event))).await { - tracing::error!("Failed to send cursor update to batch writer: {}", e); - } - } - } - - ctx.worker_events_dropped - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - counter!("worker_event_skipped_dbfree", "thread" => idx.to_string()) - .increment(1); - } else { - // Attach cursor to the LAST event only (to ensure cursor is updated after all operations) - if let Some(time_us) = time_us { - if let Some(last_event) = events.last_mut() { - match last_event { - crate::database_writer::WriterEvent::Unresolved(unresolved) => { - unresolved.cursor = Some(time_us); - } - crate::database_writer::WriterEvent::Resolved(resolved) => { - resolved.cursor = Some(time_us); - } - // Bulk events don't come from Jetstream, so this shouldn't happen - crate::database_writer::WriterEvent::UnresolvedBulk { .. } - | crate::database_writer::WriterEvent::ResolvedBulk { .. } => { - tracing::warn!("Unexpected bulk event in Jetstream worker"); - } - } - } - } - - // Increment worker events processed counter - ctx.worker_events_processed - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - // Emit cache metrics - req_ctx.emit_metrics(); - - // Send all events to batch writer queue (if configured) - if let Some(ref batch_tx) = ctx.batch_writer_tx { - let event_count = events.len(); - - for event in events { - // Bounded channel: send().await provides backpressure - // If database writer is overloaded, this blocks, propagating backpressure to Jetstream - if let Err(e) = batch_tx.send(event).await { - tracing::error!("Failed to send event to batch writer (channel closed): {}", e); - counter!("worker_batch_writer_send_error", "thread" => idx.to_string()) - .increment(1); - } - } - - counter!( - "worker_processed_event_dbfree", - "thread" => idx.to_string() - ) - .increment(1); - - counter!( - "worker_db_operations_queued", - "thread" => idx.to_string() - ) - .increment(event_count as u64); - } else { - // No batch writer configured - just count for testing - counter!( - "worker_processed_event_no_batch_writer", - "thread" => idx.to_string() - ) - .increment(1); - } - } - } - _ => { - // Other event types (legacy - should not happen with Jetstream-only mode) - counter!("worker_events_other", "thread" => idx.to_string()).increment(1); - } - } - } -} diff --git a/consumer/src/workers/mod.rs b/consumer/src/workers/mod.rs index 3493d58e..5cee2fb2 100644 --- a/consumer/src/workers/mod.rs +++ b/consumer/src/workers/mod.rs @@ -3,14 +3,4 @@ //! All workers that process events without directly accessing the database. //! Workers produce DatabaseOperation structures that are sent to the database writer. -pub mod backfill; -pub mod fetch; -pub mod handle_resolver; -pub mod jetstream; -pub mod stub_resolution; - -// Re-export worker factories for ergonomic main.rs -pub use backfill::BackfillManagerFactory; -pub use fetch::RecordFetchManagerFactory; -pub use handle_resolver::HandleResolutionManagerFactory; -pub use stub_resolution::factory::StubResolutionWorkerFactory; +pub mod tap; diff --git a/consumer/src/workers/stub_resolution/factory.rs b/consumer/src/workers/stub_resolution/factory.rs deleted file mode 100644 index 6792e268..00000000 --- a/consumer/src/workers/stub_resolution/factory.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Factory for creating StubResolutionWorker instances - -use deadpool_postgres::Pool; -use eyre::Result; - -use crate::worker_core::WorkerFactory; -use super::StubResolutionWorker; - -/// Factory for creating StubResolutionWorker instances -#[derive(Clone)] -pub struct StubResolutionWorkerFactory { - pool: Pool, - retention_cutoff: Option>, -} - -impl StubResolutionWorkerFactory { - pub fn new(pool: Pool, retention_cutoff: Option>) -> Self { - Self { pool, retention_cutoff } - } -} - -impl WorkerFactory for StubResolutionWorkerFactory { - type Worker = StubResolutionWorker; - - fn name(&self) -> &'static str { - "stub_resolution" - } - - async fn create(&self) -> Result { - Ok(StubResolutionWorker::new(self.pool.clone(), self.retention_cutoff)) - } -} diff --git a/consumer/src/workers/stub_resolution/mod.rs b/consumer/src/workers/stub_resolution/mod.rs deleted file mode 100644 index 039ea870..00000000 --- a/consumer/src/workers/stub_resolution/mod.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Stub resolution worker -//! -//! This module provides a background worker that finds stub records (with status='stub') -//! and enqueues them for fetching. Stub records are created when: -//! - A reference arrives before the target (e.g., a reply before its parent post) -//! - A like/repost references a post that doesn't exist yet -//! - An embed references a post, feedgen, or labeler that doesn't exist yet -//! -//! The stub pattern creates placeholder records immediately with status='stub', then -//! this worker finds them and enqueues them for fetching. Once fetched and processed, -//! the normal upsert operations will update status from 'stub' to 'complete'. - -use deadpool_postgres::Pool; -use eyre::Result; -use metrics::counter; -use std::time::Duration; -use tokio::sync::watch::Receiver as WatchReceiver; -use tracing::{debug, error, info}; - -use crate::worker_core::Worker; - -pub mod factory; -pub mod queries; - -/// Stub resolution worker -/// -/// This worker acts as a safety net to catch stubs that weren't immediately enqueued. -/// Most stubs are enqueued immediately during Jetstream processing, but this worker -/// backs that up by checking for stubs from the last 30 minutes every 60 seconds. -/// -/// This worker: -/// 1. Queries database for records with status='stub' created in last 30 minutes (every 60 seconds) -/// 2. Constructs AT URIs for stub records -/// 3. Enqueues AT URIs to PostgreSQL fetch queue for fetching (atomic deduplication via ON CONFLICT) -/// 4. Once fetched, the normal processing pipeline updates status to 'complete' -/// -/// Note: This worker performs direct database reads and PostgreSQL fetch queue writes. -/// It does not write to the database - resolution happens when fetches succeed. -pub struct StubResolutionWorker { - pool: Pool, - retention_cutoff: Option>, -} - -impl StubResolutionWorker { - pub fn new(pool: Pool, _retention_cutoff: Option>) -> Self { - // Stub resolution worker is a safety net that catches stubs from the last 30 minutes - // Runs every 60 seconds to back up the immediate enqueueing that happens during Jetstream processing - let retention_cutoff = Some(chrono::Utc::now() - chrono::Duration::minutes(30)); - Self { pool, retention_cutoff } - } - - /// Process all types of stub records - async fn process_stubs(&mut self) -> Result<()> { - // Process each type of stub record - self.resolve_stub_posts().await?; - self.resolve_stub_feedgens().await?; - self.resolve_stub_labelers().await?; - - Ok(()) - } - - /// Find stub posts and enqueue them for fetching - async fn resolve_stub_posts(&mut self) -> Result<()> { - let conn = self.pool.get().await?; - let uris = queries::find_stub_posts(&**conn, self.retention_cutoff).await?; - - if !uris.is_empty() { - debug!("Found {} stub posts to fetch", uris.len()); - - // enqueue_batch does atomic deduplication with ON CONFLICT DO NOTHING - let enqueued = crate::db::fetch_queue::enqueue_batch(&conn, &uris).await?; - counter!("stub_resolution.posts_enqueued").increment(enqueued as u64); - } - - Ok(()) - } - - /// Find stub feedgens and enqueue them for fetching - async fn resolve_stub_feedgens(&mut self) -> Result<()> { - let conn = self.pool.get().await?; - let uris = queries::find_stub_feedgens(&**conn).await?; - - if !uris.is_empty() { - debug!("Found {} stub feedgens to fetch", uris.len()); - - // enqueue_batch does atomic deduplication with ON CONFLICT DO NOTHING - let enqueued = crate::db::fetch_queue::enqueue_batch(&conn, &uris).await?; - counter!("stub_resolution.feedgens_enqueued").increment(enqueued as u64); - } - - Ok(()) - } - - /// Find stub labelers and enqueue them for fetching - async fn resolve_stub_labelers(&mut self) -> Result<()> { - let conn = self.pool.get().await?; - let uris = queries::find_stub_labelers(&**conn).await?; - - if !uris.is_empty() { - debug!("Found {} stub labelers to fetch", uris.len()); - - // enqueue_batch does atomic deduplication with ON CONFLICT DO NOTHING - let enqueued = crate::db::fetch_queue::enqueue_batch(&conn, &uris).await?; - counter!("stub_resolution.labelers_enqueued").increment(enqueued as u64); - } - - Ok(()) - } -} - -// Implement Worker trait -impl Worker for StubResolutionWorker { - fn name(&self) -> &'static str { - "stub_resolution" - } - - async fn run(mut self, mut stop: WatchReceiver) -> Result<()> { - info!("Stub resolution worker started"); - - loop { - tokio::select! { - _ = stop.changed() => { - info!("Stub resolution worker stopping"); - break; - } - _ = tokio::time::sleep(Duration::from_secs(60)) => { - if let Err(e) = self.process_stubs().await { - error!("Failed to process stubs: {}", e); - } - } - } - } - - info!("Stub resolution worker stopped"); - Ok(()) - } -} diff --git a/consumer/src/workers/stub_resolution/queries.rs b/consumer/src/workers/stub_resolution/queries.rs deleted file mode 100644 index 525d5825..00000000 --- a/consumer/src/workers/stub_resolution/queries.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! SQL queries for stub resolution -//! -//! These queries find stub records (status='stub') and construct AT URIs for fetching. - -use tokio_postgres::GenericClient; - -pub type QueryResult = Result; - -/// Find stub posts and return their AT URIs for fetching -/// -/// This query finds posts with status='stub', constructs their AT URIs, -/// and returns them for enqueuing to the fetch queue. -/// Limits to 100 records per batch to avoid overwhelming the fetch queue. -/// -/// If retention_cutoff is provided, only returns posts newer than the cutoff. -pub async fn find_stub_posts( - conn: &C, - retention_cutoff: Option>, -) -> QueryResult> { - // Convert retention cutoff to rkey (i64) if present - let min_rkey = retention_cutoff.map(|cutoff| { - let tid_str = parakeet_db::tid_util::timestamp_to_tid(cutoff); - parakeet_db::models::tid_to_i64(&tid_str) - .expect("timestamp_to_tid always produces valid TID") - }); - - let rows = if let Some(min_rkey) = min_rkey { - conn.query( - "SELECT 'at://' || a.did || '/app.bsky.feed.post/' || i64_to_tid(p.rkey) as uri - FROM posts p - INNER JOIN actors a ON p.actor_id = a.id - WHERE p.status = 'stub'::post_status - AND p.rkey >= $1 - ORDER BY p.rkey ASC - LIMIT 100", - &[&min_rkey], - ) - .await? - } else { - conn.query( - "SELECT 'at://' || a.did || '/app.bsky.feed.post/' || i64_to_tid(p.rkey) as uri - FROM posts p - INNER JOIN actors a ON p.actor_id = a.id - WHERE p.status = 'stub'::post_status - ORDER BY p.rkey ASC - LIMIT 100", - &[], - ) - .await? - }; - - Ok(rows.into_iter().map(|row| row.get(0)).collect()) -} - -/// Find stub feedgens and return their AT URIs for fetching -/// -/// This query finds feedgens with status='stub', constructs their AT URIs, -/// and returns them for enqueuing to the fetch queue. -/// Limits to 100 records per batch to avoid overwhelming the fetch queue. -pub async fn find_stub_feedgens(conn: &C) -> QueryResult> { - let rows = conn - .query( - "SELECT 'at://' || a.did || '/app.bsky.feed.generator/' || f.rkey::text as uri - FROM feedgens f - INNER JOIN actors a ON f.actor_id = a.id - WHERE f.status = 'stub'::feedgen_status - ORDER BY f.created_at ASC - LIMIT 100", - &[], - ) - .await?; - - Ok(rows.into_iter().map(|row| row.get(0)).collect()) -} - -/// Find stub labelers and return their AT URIs for fetching -/// -/// DENORMALIZED: Labelers are now stored directly on actors table (labeler_status, labeler_cid, etc) -/// -/// This query finds labelers with status='stub', constructs their AT URIs, -/// and returns them for enqueuing to the fetch queue. -/// Limits to 100 records per batch to avoid overwhelming the fetch queue. -pub async fn find_stub_labelers(conn: &C) -> QueryResult> { - let rows = conn - .query( - "SELECT 'at://' || did || '/app.bsky.labeler.service/self' as uri - FROM actors - WHERE labeler_status = 'stub'::labeler_status - ORDER BY labeler_created_at ASC - LIMIT 100", - &[], - ) - .await?; - - Ok(rows.into_iter().map(|row| row.get(0)).collect()) -} diff --git a/consumer/src/workers/tap/mod.rs b/consumer/src/workers/tap/mod.rs new file mode 100644 index 00000000..402e8c7d --- /dev/null +++ b/consumer/src/workers/tap/mod.rs @@ -0,0 +1,10 @@ +//! Tap event processing workers +//! +//! These workers convert Tap events into database operations that are sent +//! to the database writer. + +pub mod processor; +pub mod worker; + +pub use processor::process_tap_event; +pub use worker::spawn_workers; \ No newline at end of file diff --git a/consumer/src/workers/tap/processor.rs b/consumer/src/workers/tap/processor.rs new file mode 100644 index 00000000..aaecdb58 --- /dev/null +++ b/consumer/src/workers/tap/processor.rs @@ -0,0 +1,256 @@ +//! Tap event processor +//! +//! Converts Tap events into database operations without performing any database access. +//! This maintains the database-free architecture of the worker layer. + +use crate::database_writer::{UnresolvedEvent, UnresolvedEventType, UnresolvedRecord, WriterEvent, EventSource}; +use crate::relay::types::{CollectionType, RecordTypes}; +use crate::sources::tap::{TapEvent, RecordAction}; +use eyre::Result; +use std::str::FromStr; + +/// Process a Tap event and convert it to database operations +/// +/// Returns a vector of WriterEvents to be sent to the database writer. +/// Most events produce a single WriterEvent, but some may produce multiple. +pub async fn process_tap_event(event: TapEvent) -> Result> { + let mut writer_events = Vec::new(); + + if let Some(record) = event.record { + // Parse record action + let action = RecordAction::from_str(&record.action) + .ok_or_else(|| eyre::eyre!("Unknown record action: {}", record.action))?; + + // Build AT URI + let at_uri = format!("at://{}/{}/{}", record.did, record.collection, record.rkey); + + // Determine event type + let event_type = match action { + RecordAction::Delete => { + // Parse collection type + let collection = parse_collection_type(&record.collection)?; + UnresolvedEventType::Delete { collection } + } + RecordAction::Create | RecordAction::Update => { + // Parse and deserialize the record + if let Some(record_data) = record.record { + let record_type = parse_record(&record.collection, record_data)?; + let cid = if let Some(cid_str) = record.cid { + ipld_core::cid::Cid::from_str(&cid_str)? + } else { + return Err(eyre::eyre!("Missing CID for create/update operation")); + }; + + UnresolvedEventType::CreateUpdate { + record: Box::new(record_type), + cid, + } + } else { + return Err(eyre::eyre!("Missing record data for create/update operation")); + } + } + }; + + // Create unresolved event for the database writer + let unresolved = UnresolvedEvent { + repo: record.did.clone(), + event_type, + at_uri, + rkey: record.rkey.clone(), + source: if record.live { EventSource::Tap } else { EventSource::TapBackfill }, + cursor: None, // Will be set by the relay indexer + }; + + // For backfill events, we could potentially batch them + // For now, treat them the same as live events but with different source + writer_events.push(WriterEvent::Unresolved(Box::new(unresolved))); + } else if let Some(identity) = event.identity { + // Handle identity events as resolved events (no FK resolution needed) + // These update actor handle and status + use crate::database_writer::DatabaseOperation; + use chrono::Utc; + + let status = if identity.is_active { + Some(parakeet_db::types::ActorStatus::Active) + } else { + Some(parakeet_db::types::ActorStatus::Deactivated) + }; + + let operation = DatabaseOperation::UpsertActor { + did: identity.did.clone(), + status, + sync_state: parakeet_db::types::ActorSyncState::Synced, + handle: identity.handle.clone(), + account_created_at: None, // Not provided by Tap identity events + timestamp: Utc::now(), + }; + + let processed = crate::database_writer::ProcessedEvent { + operations: vec![operation], + cursor: None, + source: EventSource::Tap, + }; + + writer_events.push(WriterEvent::Resolved(Box::new(processed))); + } + + Ok(writer_events) +} + +/// Parse collection string into CollectionType enum +fn parse_collection_type(collection: &str) -> Result { + match collection { + "app.bsky.actor.profile" => Ok(CollectionType::BskyProfile), + "app.bsky.actor.status" => Ok(CollectionType::BskyStatus), + "app.bsky.feed.generator" => Ok(CollectionType::BskyFeedGen), + "app.bsky.feed.like" => Ok(CollectionType::BskyFeedLike), + "app.bsky.feed.post" => Ok(CollectionType::BskyFeedPost), + "app.bsky.feed.repost" => Ok(CollectionType::BskyFeedRepost), + "app.bsky.feed.threadgate" => Ok(CollectionType::BskyFeedThreadgate), + "app.bsky.graph.block" => Ok(CollectionType::BskyBlock), + "app.bsky.graph.follow" => Ok(CollectionType::BskyFollow), + "app.bsky.graph.list" => Ok(CollectionType::BskyList), + "app.bsky.graph.listblock" => Ok(CollectionType::BskyListBlock), + "app.bsky.graph.listitem" => Ok(CollectionType::BskyListItem), + "app.bsky.graph.starterpack" => Ok(CollectionType::BskyStarterPack), + "app.bsky.labeler.service" => Ok(CollectionType::BskyLabelerService), + _ => Ok(CollectionType::Unsupported), + } +} + +/// Parse JSON record into RecordTypes enum +fn parse_record(collection: &str, record_data: serde_json::Value) -> Result { + // Parse the record with the $type field + let mut record_with_type = record_data; + + // Add the $type field if not present (for deserialization) + if !record_with_type.get("$type").is_some() { + if let Some(obj) = record_with_type.as_object_mut() { + obj.insert("$type".to_string(), serde_json::Value::String(collection.to_string())); + } + } + + // Deserialize using serde's tagged enum support + let record_type: RecordTypes = serde_json::from_value(record_with_type)?; + Ok(record_type) +} + +/// Process a batch of Tap events +/// +/// This function batches backfill events by actor for bulk processing. +/// Live events are processed immediately, while backfill events are collected +/// into batches by actor and sent as UnresolvedBulk or ResolvedBulk events. +pub async fn process_tap_events_batch(events: Vec) -> Result> { + let mut writer_events = Vec::new(); + let mut backfill_by_actor: std::collections::HashMap> = std::collections::HashMap::new(); + + for event in events { + if let Some(record) = event.record { + // Check if this is a backfill event + if !record.live { + // This is a backfill event - batch it by actor + let actor_did = record.did.clone(); + + // Parse record action + let action = RecordAction::from_str(&record.action) + .ok_or_else(|| eyre::eyre!("Unknown record action: {}", record.action))?; + + // Build AT URI + let at_uri = format!("at://{}/{}/{}", record.did, record.collection, record.rkey); + + // For backfill, we only batch create/update operations + // Deletes are processed individually + match action { + RecordAction::Create | RecordAction::Update => { + if let Some(record_data) = record.record { + let record_type = parse_record(&record.collection, record_data)?; + let cid = if let Some(cid_str) = record.cid { + ipld_core::cid::Cid::from_str(&cid_str)? + } else { + return Err(eyre::eyre!("Missing CID for create/update operation")); + }; + + // Create unresolved record for batching + let unresolved_record = UnresolvedRecord { + at_uri: at_uri.clone(), + rkey: record.rkey.clone(), + cid, + record: Box::new(record_type), + }; + + backfill_by_actor.entry(actor_did) + .or_default() + .push(unresolved_record); + } + } + RecordAction::Delete => { + // Process deletes immediately (they're rare in backfill) + let collection = parse_collection_type(&record.collection)?; + let unresolved = UnresolvedEvent { + repo: record.did.clone(), + event_type: UnresolvedEventType::Delete { collection }, + at_uri, + rkey: record.rkey.clone(), + source: EventSource::TapBackfill, + cursor: None, + }; + writer_events.push(WriterEvent::Unresolved(Box::new(unresolved))); + } + } + } else { + // Live event - process immediately + let live_events = process_tap_event(TapEvent { + id: event.id, + event_type: event.event_type, + record: Some(record), + identity: None, + }).await?; + writer_events.extend(live_events); + } + } else if let Some(identity) = event.identity { + // Identity events are always processed immediately + let identity_events = process_tap_event(TapEvent { + id: event.id, + event_type: event.event_type, + record: None, + identity: Some(identity), + }).await?; + writer_events.extend(identity_events); + } + } + + // Send batched backfill events as UnresolvedBulk + // Batch size threshold: 50+ records per actor + const BULK_THRESHOLD: usize = 50; + + for (actor_did, records) in backfill_by_actor { + if records.len() >= BULK_THRESHOLD { + // Send as bulk event for efficient processing + // Note: We don't have actor_id yet, it will be resolved by the database writer + writer_events.push(WriterEvent::UnresolvedBulk { + repo: actor_did, + actor_id: 0, // Will be resolved by database writer + records, + source: EventSource::TapBackfill, + }); + } else { + // Small batch - send as individual unresolved events + for unresolved_record in records { + let unresolved = UnresolvedEvent { + repo: actor_did.clone(), + event_type: UnresolvedEventType::CreateUpdate { + record: unresolved_record.record, + cid: unresolved_record.cid, + }, + at_uri: unresolved_record.at_uri, + rkey: unresolved_record.rkey, + source: EventSource::TapBackfill, + cursor: None, + }; + writer_events.push(WriterEvent::Unresolved(Box::new(unresolved))); + } + } + } + + Ok(writer_events) +} \ No newline at end of file diff --git a/consumer/src/workers/tap/worker.rs b/consumer/src/workers/tap/worker.rs new file mode 100644 index 00000000..5fd88a9d --- /dev/null +++ b/consumer/src/workers/tap/worker.rs @@ -0,0 +1,89 @@ +//! Tap worker threads +//! +//! These workers receive Tap events from a channel and process them into +//! database operations that are sent to the database writer. + +use crate::database_writer::WriterEvent; +use crate::sources::tap::TapEvent; +use metrics::counter; +use tokio::sync::mpsc::{channel, Receiver, Sender}; +use tokio::task::JoinHandle; + +struct WorkerContext { + batch_writer_tx: tokio::sync::mpsc::Sender, + worker_events_processed: std::sync::Arc, + worker_events_dropped: std::sync::Arc, +} + +/// Spawn worker threads for processing Tap events +pub fn spawn_workers( + threads: u8, + batch_writer_tx: tokio::sync::mpsc::Sender, + worker_events_processed: std::sync::Arc, + worker_events_dropped: std::sync::Arc, +) -> (Vec>, Vec>) { + let result = (0..threads) + .map(|idx| { + let (tx, mut rx) = channel(1000); + let mut ctx = WorkerContext { + batch_writer_tx: batch_writer_tx.clone(), + worker_events_processed: worker_events_processed.clone(), + worker_events_dropped: worker_events_dropped.clone(), + }; + + let handle = tokio::spawn(async move { + run_worker(idx, &mut ctx, &mut rx).await; + }); + + (tx, handle) + }) + .unzip(); + + tracing::info!("Started {} Tap workers", threads); + result +} + +/// Run a single worker thread, processing events from the channel +async fn run_worker(idx: u8, ctx: &mut WorkerContext, rx: &mut Receiver) { + tracing::debug!("Tap worker {} started", idx); + + while let Some(event) = rx.recv().await { + // Track the event ID for acknowledgment + let event_id = event.id; + + // Process the event without database operations + match super::processor::process_tap_event(event).await { + Ok(writer_events) => { + if writer_events.is_empty() { + // Event produced no operations + ctx.worker_events_dropped + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + counter!("tap_worker_event_skipped", "thread" => idx.to_string()) + .increment(1); + } else { + // Send all writer events to the database writer + for writer_event in writer_events { + // Bounded channel: send().await provides backpressure + if let Err(e) = ctx.batch_writer_tx.send(writer_event).await { + tracing::error!("Failed to send event to batch writer: {}", e); + } + } + + ctx.worker_events_processed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + counter!("tap_worker_event_processed", "thread" => idx.to_string()) + .increment(1); + } + } + Err(e) => { + tracing::error!("Failed to process Tap event {}: {}", event_id, e); + ctx.worker_events_dropped + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + counter!("tap_worker_event_error", "thread" => idx.to_string()) + .increment(1); + } + } + } + + tracing::debug!("Tap worker {} stopped", idx); +} \ No newline at end of file diff --git a/consumer/tests/allowlist_test.rs b/consumer/tests/allowlist_test.rs deleted file mode 100644 index fcfd07cf..00000000 --- a/consumer/tests/allowlist_test.rs +++ /dev/null @@ -1,231 +0,0 @@ -//! Integration tests for allowlist database operations -//! -//! These tests verify that allowlist SQL operations work with the current schema. - -mod common; -use chrono::Utc; -use common::*; -use consumer::db::allowlist; -use parakeet_db::types::{ActorStatus, ActorSyncState}; -use eyre::WrapErr; - -/// Test get_all returns DIDs with allowlisted sync states -#[tokio::test] -async fn test_get_all() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Create synced actor (should be in allowlist) - consumer::db::actor::actor_upsert( - &tx, - "did:plc:synced", - Some(&ActorStatus::Active), - &ActorSyncState::Synced, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - // Create partial actor (should NOT be in allowlist) - consumer::db::actor::actor_upsert( - &tx, - "did:plc:partial", - Some(&ActorStatus::Active), - &ActorSyncState::Partial, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - // Test get_all - let result = allowlist::get_all(&tx).await; - - assert!(result.is_ok(), "get_all should work: {:?}", result.err()); - - let dids = result.unwrap(); - assert!( - dids.contains(&"did:plc:synced".to_string()), - "Should contain synced actor" - ); - assert!( - !dids.contains(&"did:plc:partial".to_string()), - "Should NOT contain partial actor" - ); - Ok(()) -} - -/// Test add creates or updates actors with dirty sync state -#[tokio::test] -async fn test_add() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Test adding new DID - let result = allowlist::add(&tx, "did:plc:test_add", None).await; - - assert!(result.is_ok(), "add should work: {:?}", result.err()); - - let rows_affected = result.unwrap(); - assert_eq!(rows_affected, 1, "Should insert 1 row for new DID"); - - // Verify the actor was created with dirty sync state - let status_result = consumer::db::actor::actor_get_statuses(&tx, "did:plc:test_add").await; - assert!(status_result.is_ok()); - if let Some((status, sync_state)) = status_result.unwrap() { - assert_eq!(status, ActorStatus::Active); - assert_eq!(sync_state, ActorSyncState::Dirty); - } else { - panic!("Actor should exist after add"); - } - Ok(()) -} - -/// Test add updates partial actors to dirty -#[tokio::test] -async fn test_add_updates_partial() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // First create a partial actor - consumer::db::actor::actor_upsert( - &tx, - "did:plc:test_partial", - Some(&ActorStatus::Active), - &ActorSyncState::Partial, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - // Test adding it to allowlist (should update to dirty) - let result = allowlist::add(&tx, "did:plc:test_partial", Some("Test description")).await; - - assert!(result.is_ok(), "add should work: {:?}", result.err()); - - let rows_affected = result.unwrap(); - assert_eq!( - rows_affected, 1, - "Should update 1 row when converting partial to dirty" - ); - - // Verify the actor was updated to dirty - let status_result = consumer::db::actor::actor_get_statuses(&tx, "did:plc:test_partial").await; - assert!(status_result.is_ok()); - if let Some((_status, sync_state)) = status_result.unwrap() { - assert_eq!(sync_state, ActorSyncState::Dirty); - } else { - panic!("Actor should exist after add"); - } - Ok(()) -} - -/// Test add does nothing for already-allowlisted actors -#[tokio::test] -async fn test_add_no_op_for_synced() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // First create a synced actor (already allowlisted) - consumer::db::actor::actor_upsert( - &tx, - "did:plc:test_synced", - Some(&ActorStatus::Active), - &ActorSyncState::Synced, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - // Test adding it to allowlist (should be no-op) - let result = allowlist::add(&tx, "did:plc:test_synced", None).await; - - assert!(result.is_ok(), "add should work: {:?}", result.err()); - - let rows_affected = result.unwrap(); - assert_eq!( - rows_affected, 0, - "Should not update already-allowlisted (synced) actor" - ); - - // Verify the actor is still synced - let status_result = consumer::db::actor::actor_get_statuses(&tx, "did:plc:test_synced").await; - assert!(status_result.is_ok()); - if let Some((_status, sync_state)) = status_result.unwrap() { - assert_eq!(sync_state, ActorSyncState::Synced); - } else { - panic!("Actor should exist"); - } - Ok(()) -} - -/// Test ensure_allowlist_actors query is valid SQL -/// Note: This function also interacts with Redis and calls actor_upsert, -/// so we're just testing the SELECT query portion here. -#[tokio::test] -async fn test_ensure_allowlist_actors_query() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Create some actors with different sync states - consumer::db::actor::actor_upsert( - &tx, - "did:plc:test_synced2", - Some(&ActorStatus::Active), - &ActorSyncState::Synced, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - consumer::db::actor::actor_upsert( - &tx, - "did:plc:test_processing", - Some(&ActorStatus::Active), - &ActorSyncState::Processing, - None, - None, - Utc::now(), - ) - .await - .unwrap(); - - // Test the SELECT query used in ensure_allowlist_actors - let result = tx - .query( - "SELECT did FROM actors - WHERE sync_state IN ('synced', 'processing') - AND sync_state != 'dirty'::actor_sync_state", - &[], - ) - .await; - - assert!( - result.is_ok(), - "ensure_allowlist_actors query should be valid SQL: {:?}", - result.err() - ); - - let rows = result.unwrap(); - // The test runs in isolation, so we should find exactly the 2 actors we created - assert!( - rows.len() >= 2, - "Should find at least 2 actors needing backfill, found {}", - rows.len() - ); - Ok(()) -} diff --git a/consumer/tests/backfill_jobs_test.rs b/consumer/tests/backfill_jobs_test.rs deleted file mode 100644 index 817d2489..00000000 --- a/consumer/tests/backfill_jobs_test.rs +++ /dev/null @@ -1,434 +0,0 @@ -//! Integration tests for backfill_jobs database operations -//! -//! These tests verify that backfill job queue SQL operations work with the current schema. -//! Tests run in isolation using unique DIDs to avoid conflicts. - -mod common; -use chrono::Utc; -use common::*; -use consumer::db::backfill_jobs; -use eyre::WrapErr; - -/// Test enqueue_job SQL is valid and runs without errors -#[tokio::test] -async fn test_enqueue_job_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the raw SQL used by enqueue_job - let test_did = format!("did:plc:test_enqueue_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - let result = conn - .execute( - "INSERT INTO backfill_jobs (did, status, scheduled_at) - VALUES ($1, 'pending', NOW()) - ON CONFLICT (did) - DO UPDATE SET - status = 'pending', - scheduled_at = NOW() - WHERE backfill_jobs.status != 'successful'", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "enqueue_job SQL should work: {:?}", result.err()); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test should_enqueue query SQL is valid -#[tokio::test] -async fn test_should_enqueue_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the SQL used by should_enqueue - let test_did = format!("did:plc:test_should_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - let result = conn - .query_opt( - "SELECT status FROM backfill_jobs WHERE did = $1", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "should_enqueue SQL should work: {:?}", result.err()); - - Ok(()) -} - -/// Test start_processing EXISTS check SQL is valid -#[tokio::test] -async fn test_start_processing_exists_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the EXISTS query used in start_processing - let test_did = format!("did:plc:test_exists_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - let result = conn - .query_one( - "SELECT EXISTS(SELECT 1 FROM backfill_jobs WHERE did = $1)", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "EXISTS SQL should work: {:?}", result.err()); - - let exists: bool = result.unwrap().get(0); - assert!(!exists, "Should return false for non-existent job"); - - Ok(()) -} - -/// Test start_processing UPDATE SQL is valid -#[tokio::test] -async fn test_start_processing_update_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Create a test job first - let test_did = format!("did:plc:test_start_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - conn.execute( - "INSERT INTO backfill_jobs (did, status, attempts, scheduled_at) - VALUES ($1, 'pending', 0, NOW())", - &[&test_did], - ) - .await?; - - // Test the UPDATE...RETURNING used in start_processing - let result = conn - .query_one( - "UPDATE backfill_jobs - SET status = 'processing', - attempts = attempts + 1, - started_at = NOW() - WHERE did = $1 - RETURNING attempts", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "start_processing UPDATE SQL should work: {:?}", result.err()); - - let attempts: i32 = result.unwrap().get(0); - assert_eq!(attempts, 1, "Should increment attempts"); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test mark_successful SQL is valid -#[tokio::test] -async fn test_mark_successful_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Create a test job first - let test_did = format!("did:plc:test_success_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - conn.execute( - "INSERT INTO backfill_jobs (did, status, attempts, scheduled_at) - VALUES ($1, 'processing', 1, NOW())", - &[&test_did], - ) - .await?; - - // Test the SQL used in mark_successful - let result = conn - .execute( - "UPDATE backfill_jobs - SET status = 'successful', completed_at = NOW() - WHERE did = $1", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "mark_successful SQL should work: {:?}", result.err()); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test mark_failed SELECT attempts SQL is valid -#[tokio::test] -async fn test_mark_failed_select_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Create a test job first - let test_did = format!("did:plc:test_fail_select_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - conn.execute( - "INSERT INTO backfill_jobs (did, status, attempts, scheduled_at) - VALUES ($1, 'processing', 1, NOW())", - &[&test_did], - ) - .await?; - - // Test the SELECT used in mark_failed - let result = conn - .query_one( - "SELECT attempts FROM backfill_jobs WHERE did = $1", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "mark_failed SELECT SQL should work: {:?}", result.err()); - - let attempts: i32 = result.unwrap().get(0); - assert_eq!(attempts, 1); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test mark_failed retry SQL is valid -#[tokio::test] -async fn test_mark_failed_retry_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Create a test job first - let test_did = format!("did:plc:test_fail_retry_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - conn.execute( - "INSERT INTO backfill_jobs (did, status, attempts, scheduled_at) - VALUES ($1, 'processing', 1, NOW())", - &[&test_did], - ) - .await?; - - // Test the retry SQL used in mark_failed - let retry_at = Utc::now() + chrono::Duration::minutes(2); - let result = conn - .execute( - "UPDATE backfill_jobs - SET status = 'failed.retry', - last_error = $2, - scheduled_at = $3 - WHERE did = $1", - &[&test_did, &"Test error", &retry_at], - ) - .await; - - assert!(result.is_ok(), "mark_failed retry SQL should work: {:?}", result.err()); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test mark_failed permanent SQL is valid -#[tokio::test] -async fn test_mark_failed_permanent_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Create a test job first - let test_did = format!("did:plc:test_fail_perm_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - conn.execute( - "INSERT INTO backfill_jobs (did, status, attempts, scheduled_at) - VALUES ($1, 'processing', 3, NOW())", - &[&test_did], - ) - .await?; - - // Test the permanent failure SQL used in mark_failed - let result = conn - .execute( - "UPDATE backfill_jobs - SET status = 'failed.permanent', - last_error = $2, - completed_at = NOW() - WHERE did = $1", - &[&test_did, &"Final error"], - ) - .await; - - assert!(result.is_ok(), "mark_failed permanent SQL should work: {:?}", result.err()); - - // Cleanup - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} - -/// Test dequeue stale recovery SQL is valid -#[tokio::test] -async fn test_dequeue_stale_recovery_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the stale recovery SQL used in dequeue - let stale_cutoff = Utc::now() - chrono::Duration::minutes(10); - let result = conn - .execute( - "UPDATE backfill_jobs - SET status = 'pending', scheduled_at = NOW() - WHERE status = 'processing' AND started_at < $1", - &[&stale_cutoff], - ) - .await; - - assert!(result.is_ok(), "dequeue stale recovery SQL should work: {:?}", result.err()); - - Ok(()) -} - -/// Test dequeue complex UPDATE...WHERE...SELECT SQL is valid -#[tokio::test] -async fn test_dequeue_select_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the complex dequeue SQL - let result = conn - .query_opt( - "UPDATE backfill_jobs - SET status = 'processing', - attempts = attempts + 1, - started_at = NOW() - WHERE did = ( - SELECT did FROM backfill_jobs - WHERE status IN ('pending', 'failed.retry') - AND scheduled_at <= NOW() - ORDER BY scheduled_at - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) - RETURNING did, status, attempts, last_error, scheduled_at, started_at", - &[], - ) - .await; - - // This may return None if no jobs are available, but the SQL should be valid - assert!(result.is_ok(), "dequeue SQL should be valid: {:?}", result.err()); - - Ok(()) -} - -/// Test get_stats aggregation SQL is valid -#[tokio::test] -async fn test_get_stats_sql() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - // Test the aggregation SQL used in get_stats - let result = conn - .query_one( - "SELECT - COUNT(*) FILTER (WHERE status IN ('pending', 'failed.retry')) as pending, - COUNT(*) FILTER (WHERE status = 'processing') as processing, - COUNT(*) FILTER (WHERE status = 'successful') as successful, - COUNT(*) FILTER (WHERE status = 'failed.permanent') as failed - FROM backfill_jobs", - &[], - ) - .await; - - assert!(result.is_ok(), "get_stats SQL should work: {:?}", result.err()); - - // Verify we got counts - let row = result.unwrap(); - let pending: i64 = row.get(0); - let processing: i64 = row.get(1); - let successful: i64 = row.get(2); - let failed: i64 = row.get(3); - - // All counts should be non-negative - assert!(pending >= 0); - assert!(processing >= 0); - assert!(successful >= 0); - assert!(failed >= 0); - - Ok(()) -} - -/// Test actor EXISTS query (used in admin API check_actor_exists) -#[tokio::test] -async fn test_actor_exists_query() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Create an actor - let test_did = format!("did:plc:test_exists_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - consumer::db::actor::actor_upsert( - &tx, - &test_did, - Some(¶keet_db::types::ActorStatus::Active), - ¶keet_db::types::ActorSyncState::Partial, - None, - None, - Utc::now(), - ) - .await?; - - // Test EXISTS query (used in check_actor_exists and start_processing) - let result = tx - .query_one( - "SELECT EXISTS(SELECT 1 FROM actors WHERE did = $1)", - &[&test_did], - ) - .await; - - assert!(result.is_ok(), "EXISTS query should work: {:?}", result.err()); - - let exists: bool = result.unwrap().get(0); - assert!(exists, "Should return true for existing actor"); - - // Test with non-existent DID - let nonexistent_did = format!("did:plc:nonexistent_{}", Utc::now().timestamp_nanos_opt().unwrap()); - let result = tx - .query_one( - "SELECT EXISTS(SELECT 1 FROM actors WHERE did = $1)", - &[&nonexistent_did], - ) - .await; - - assert!(result.is_ok(), "EXISTS query should work: {:?}", result.err()); - - let exists: bool = result.unwrap().get(0); - assert!(!exists, "Should return false for non-existent actor"); - - Ok(()) -} - -/// Test backfill_jobs module functions compile and run -#[tokio::test] -async fn test_backfill_jobs_functions_compile() -> eyre::Result<()> { - let pool = test_pool(); - - // Test that all the public functions exist and can be called - let test_did = format!("did:plc:test_compile_{}", Utc::now().timestamp_nanos_opt().unwrap()); - - // enqueue_job - drop(backfill_jobs::enqueue_job(&pool, &test_did).await); - - // should_enqueue - drop(backfill_jobs::should_enqueue(&pool, &test_did).await); - - // get_stats - let stats_result = backfill_jobs::get_stats(&pool).await; - assert!(stats_result.is_ok(), "get_stats should work"); - - // Cleanup - let conn = pool.get().await?; - conn.execute("DELETE FROM backfill_jobs WHERE did = $1", &[&test_did]).await?; - - Ok(()) -} diff --git a/consumer/tests/community_starterpack_operations_test.rs b/consumer/tests/community_starterpack_operations_test.rs index 80dd3058..d1e3d8fb 100644 --- a/consumer/tests/community_starterpack_operations_test.rs +++ b/consumer/tests/community_starterpack_operations_test.rs @@ -59,7 +59,7 @@ async fn test_bookmark_upsert_post_subject() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(post_uri)).unwrap(), test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for bookmark test")?; @@ -135,7 +135,7 @@ async fn test_bookmark_upsert_idempotent() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(post_uri)).unwrap(), test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for bookmark idempotency test")?; @@ -217,7 +217,7 @@ async fn test_bookmark_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(post_uri)).unwrap(), test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for bookmark delete test")?; diff --git a/consumer/tests/cursor_manager_test.rs b/consumer/tests/cursor_manager_test.rs deleted file mode 100644 index 996d4645..00000000 --- a/consumer/tests/cursor_manager_test.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Integration tests for PostgreSQL cursor manager -//! -//! Tests the pg_cursor_manager module which stores Jetstream partition cursors -//! in PostgreSQL instead of Redis. - -mod common; - -use consumer::external_services::pg_cursor_manager::PgCursorManager; -use eyre::WrapErr; - -#[tokio::test] -async fn test_cursor_save_and_load() -> eyre::Result<()> { - common::ensure_test_db_ready().await; - let pool = common::test_pool(); - - // Create cursor manager using the test pool - let manager = PgCursorManager::new(pool); - - let partition = "test-posts"; - let cursor_value: i64 = 1234567890123456; - - // Test save operation - manager - .save(partition, cursor_value) - .await - .wrap_err("Failed to save cursor")?; - - // Test load operation - let loaded = manager - .load(partition) - .await - .wrap_err("Failed to load cursor")?; - - assert_eq!( - loaded, - Some(cursor_value), - "Loaded cursor should match saved cursor" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_cursor_load_nonexistent() -> eyre::Result<()> { - common::ensure_test_db_ready().await; - let pool = common::test_pool(); - - let manager = PgCursorManager::new(pool); - let partition = "nonexistent-partition"; - - // Test load operation for partition that doesn't exist - let loaded = manager - .load(partition) - .await - .wrap_err("Failed to load cursor")?; - - assert_eq!(loaded, None, "Should return None for nonexistent partition"); - - Ok(()) -} - -#[tokio::test] -async fn test_cursor_update() -> eyre::Result<()> { - common::ensure_test_db_ready().await; - let pool = common::test_pool(); - - let manager = PgCursorManager::new(pool); - let partition = "test-likes"; - let initial_cursor: i64 = 1000000000000000; - let updated_cursor: i64 = 2000000000000000; - - // Insert initial cursor - manager - .save(partition, initial_cursor) - .await - .wrap_err("Failed to insert initial cursor")?; - - // Update cursor (UPSERT should update existing row) - manager - .save(partition, updated_cursor) - .await - .wrap_err("Failed to update cursor")?; - - // Verify cursor was updated - let loaded = manager - .load(partition) - .await - .wrap_err("Failed to load cursor")?; - - assert_eq!( - loaded, - Some(updated_cursor), - "Cursor should be updated to new value" - ); - - Ok(()) -} - -#[tokio::test] -async fn test_cursor_multiple_partitions() -> eyre::Result<()> { - common::ensure_test_db_ready().await; - let pool = common::test_pool(); - - let manager = PgCursorManager::new(pool); - - let partitions = vec![ - ("posts", 1111111111111111_i64), - ("likes", 2222222222222222_i64), - ("reposts", 3333333333333333_i64), - ("social", 4444444444444444_i64), - ]; - - // Save cursors for multiple partitions - for (partition, cursor_value) in &partitions { - manager - .save(partition, *cursor_value) - .await - .wrap_err("Failed to save cursor")?; - } - - // Verify all cursors can be loaded independently - for (partition, expected_cursor) in &partitions { - let loaded = manager - .load(partition) - .await - .wrap_err("Failed to load cursor")?; - - assert_eq!( - loaded, - Some(*expected_cursor), - "Cursor for partition {} should match", - partition - ); - } - - Ok(()) -} diff --git a/consumer/tests/feed_operations_test.rs b/consumer/tests/feed_operations_test.rs index 50b4302e..2451f7c2 100644 --- a/consumer/tests/feed_operations_test.rs +++ b/consumer/tests/feed_operations_test.rs @@ -55,7 +55,7 @@ async fn test_post_insert_normal() -> eyre::Result<()> { actor_id, rkey_i64, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await; @@ -126,7 +126,7 @@ async fn test_post_insert_with_reply() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk235")?, test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -154,7 +154,7 @@ async fn test_post_insert_with_reply() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk236")?, test_cid(), reply_post, - EventSource::Jetstream, + EventSource::Tap, ) .await; @@ -256,7 +256,7 @@ async fn test_post_insert_with_facets() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk25a")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await; @@ -405,7 +405,7 @@ async fn test_post_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk237")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post")?; @@ -481,7 +481,7 @@ async fn test_like_insert_post_subject() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23a")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post")?; @@ -501,7 +501,7 @@ async fn test_like_insert_post_subject() -> eyre::Result<()> { liker_actor_id, like, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await; @@ -572,7 +572,7 @@ async fn test_like_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23c")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post")?; @@ -591,7 +591,7 @@ async fn test_like_delete() -> eyre::Result<()> { liker3_actor_id, like, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert like")?; @@ -682,7 +682,7 @@ async fn test_repost_insert() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23d")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert original post")?; @@ -701,7 +701,7 @@ async fn test_repost_insert() -> eyre::Result<()> { test_cid(), repost, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await; @@ -778,7 +778,7 @@ async fn test_repost_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23f")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert original post")?; @@ -796,7 +796,7 @@ async fn test_repost_delete() -> eyre::Result<()> { test_cid(), repost, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert repost")?; @@ -868,7 +868,7 @@ async fn test_repost_insert_idempotent() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23g")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert original post")?; @@ -887,7 +887,7 @@ async fn test_repost_insert_idempotent() -> eyre::Result<()> { test_cid(), repost, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await; assert_eq!(result1.unwrap(), 1, "First repost insert should succeed"); @@ -906,7 +906,7 @@ async fn test_repost_insert_idempotent() -> eyre::Result<()> { test_cid(), repost2, None, // via_repost_id - EventSource::Jetstream, + EventSource::Tap, ) .await; assert_eq!( @@ -952,7 +952,7 @@ async fn test_postgate_upsert_insert() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23h")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for postgate")?; @@ -1020,7 +1020,7 @@ async fn test_postgate_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23k")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for postgate")?; @@ -1097,7 +1097,7 @@ async fn test_threadgate_upsert_insert() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23n")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for threadgate")?; @@ -1165,7 +1165,7 @@ async fn test_threadgate_delete() -> eyre::Result<()> { parakeet_db::models::tid_to_i64("3l7mkz4lmk23r")?, test_cid(), post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert post for threadgate")?; diff --git a/consumer/tests/notification_test.rs b/consumer/tests/notification_test.rs index 339efb05..cc4e5341 100644 --- a/consumer/tests/notification_test.rs +++ b/consumer/tests/notification_test.rs @@ -89,7 +89,7 @@ async fn test_is_thread_muted_muted() -> eyre::Result<()> { rkey_i64, test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -178,7 +178,7 @@ async fn test_reply_chain_walker_single_level() -> eyre::Result<()> { rkey_i64, test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -210,7 +210,7 @@ async fn test_reply_chain_walker_single_level() -> eyre::Result<()> { rkey_i64, test_cid(), reply_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert reply")?; @@ -283,7 +283,7 @@ async fn test_reply_chain_walker_multi_level() -> eyre::Result<()> { rkey_i64, test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -315,7 +315,7 @@ async fn test_reply_chain_walker_multi_level() -> eyre::Result<()> { rkey_i64, test_cid(), reply1, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert reply1")?; @@ -347,7 +347,7 @@ async fn test_reply_chain_walker_multi_level() -> eyre::Result<()> { rkey_i64, test_cid(), reply2, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert reply2")?; @@ -379,7 +379,7 @@ async fn test_reply_chain_walker_multi_level() -> eyre::Result<()> { rkey_i64, test_cid(), reply3, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert reply3")?; diff --git a/consumer/tests/pds_cache_test.rs b/consumer/tests/pds_cache_test.rs deleted file mode 100644 index 209ec00f..00000000 --- a/consumer/tests/pds_cache_test.rs +++ /dev/null @@ -1,114 +0,0 @@ -//! Integration tests for PDS host cache -//! -//! Tests the pds_cache module which provides in-memory caching of PDS host -//! mappings using moka. These tests verify cache operations including TTL -//! and batch operations. - -mod common; - -use consumer::external_services::pds_cache::PdsHostCache; -use std::collections::HashMap; - -#[tokio::test] -async fn test_pds_cache_get_set() { - let cache = PdsHostCache::new(); - - // Initially empty - assert_eq!(cache.get("did:plc:test").await, None); - - // Set and get - cache - .set("did:plc:test".to_string(), "https://test.pds".to_string()) - .await; - assert_eq!( - cache.get("did:plc:test").await, - Some("https://test.pds".to_string()) - ); -} - -#[tokio::test] -async fn test_pds_cache_batch_operations() { - let cache = PdsHostCache::new(); - - // Set batch - let mut mappings = HashMap::new(); - mappings.insert( - "did:plc:test1".to_string(), - "https://pds1.example".to_string(), - ); - mappings.insert( - "did:plc:test2".to_string(), - "https://pds2.example".to_string(), - ); - cache.set_batch(&mappings).await; - - // Get batch - let dids = vec!["did:plc:test1", "did:plc:test2", "did:plc:nonexistent"]; - let result = cache.get_batch(&dids).await; - - assert_eq!(result.len(), 2); - assert_eq!( - result.get("did:plc:test1"), - Some(&"https://pds1.example".to_string()) - ); - assert_eq!( - result.get("did:plc:test2"), - Some(&"https://pds2.example".to_string()) - ); - assert_eq!(result.get("did:plc:nonexistent"), None); -} - -#[tokio::test] -async fn test_pds_cache_overwrite() { - let cache = PdsHostCache::new(); - - // Set initial value - cache - .set( - "did:plc:user".to_string(), - "https://old.pds".to_string(), - ) - .await; - assert_eq!( - cache.get("did:plc:user").await, - Some("https://old.pds".to_string()) - ); - - // Overwrite with new value - cache - .set( - "did:plc:user".to_string(), - "https://new.pds".to_string(), - ) - .await; - assert_eq!( - cache.get("did:plc:user").await, - Some("https://new.pds".to_string()) - ); -} - -#[tokio::test] -async fn test_pds_cache_multiple_entries() { - let cache = PdsHostCache::new(); - - // Add multiple entries - let entries = vec![ - ("did:plc:alice", "https://alice.pds"), - ("did:plc:bob", "https://bob.pds"), - ("did:plc:charlie", "https://charlie.pds"), - ]; - - for (did, host) in &entries { - cache.set(did.to_string(), host.to_string()).await; - } - - // Verify all entries are cached independently - for (did, expected_host) in &entries { - assert_eq!( - cache.get(did).await, - Some(expected_host.to_string()), - "Host for {} should match", - did - ); - } -} diff --git a/consumer/tests/stub_resolution_queries_test.rs b/consumer/tests/stub_resolution_queries_test.rs deleted file mode 100644 index d2b87bd5..00000000 --- a/consumer/tests/stub_resolution_queries_test.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Tests for stub resolution SQL queries -//! -//! These tests validate that the SQL queries used by the stub_resolution worker -//! are syntactically correct against the current database schema. -//! -//! Note: Behavioral testing of stub creation and resolution is covered by -//! feed_operations_test.rs (test_like_insert_stub_subject, test_repost_insert_stub_post) -//! which verify that stubs are created correctly and contain the expected data. - -mod common; -use common::*; -use consumer::workers::stub_resolution::queries; -use eyre::WrapErr; - -#[tokio::test] -async fn test_find_stub_posts_query() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - let result = queries::find_stub_posts(&**conn, None).await; - - assert!( - result.is_ok(), - "find_stub_posts query should be valid SQL: {:?}", - result.err() - ); - Ok(()) -} - -#[tokio::test] -async fn test_find_stub_feedgens_query() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - let result = queries::find_stub_feedgens(&**conn).await; - - assert!( - result.is_ok(), - "find_stub_feedgens query should be valid SQL: {:?}", - result.err() - ); - Ok(()) -} - -#[tokio::test] -async fn test_find_stub_labelers_query() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - let result = queries::find_stub_labelers(&**conn).await; - - assert!( - result.is_ok(), - "find_stub_labelers query should be valid SQL: {:?}", - result.err() - ); - Ok(()) -} diff --git a/consumer/tests/threadgate_enforcement_test.rs b/consumer/tests/threadgate_enforcement_test.rs index c0617b22..3a8073ae 100644 --- a/consumer/tests/threadgate_enforcement_test.rs +++ b/consumer/tests/threadgate_enforcement_test.rs @@ -63,7 +63,7 @@ async fn test_threadgate_enforcement_no_threadgate() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -121,7 +121,7 @@ async fn test_threadgate_enforcement_same_author() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -198,7 +198,7 @@ async fn test_threadgate_enforcement_empty_allow_list() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -294,7 +294,7 @@ async fn test_threadgate_enforcement_following_rule_allows() -> eyre::Result<()> parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -377,7 +377,7 @@ async fn test_threadgate_enforcement_following_rule_blocks() -> eyre::Result<()> parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -473,7 +473,7 @@ async fn test_threadgate_enforcement_follower_rule_allows() -> eyre::Result<()> parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -553,7 +553,7 @@ async fn test_threadgate_enforcement_follower_rule_blocks() -> eyre::Result<()> parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -643,7 +643,7 @@ async fn test_threadgate_enforcement_mention_rule_allows() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -723,7 +723,7 @@ async fn test_threadgate_enforcement_mention_rule_blocks() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -843,7 +843,7 @@ async fn test_threadgate_enforcement_list_rule_allows() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -947,7 +947,7 @@ async fn test_threadgate_enforcement_list_rule_blocks() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; @@ -1048,7 +1048,7 @@ async fn test_threadgate_enforcement_multiple_rules() -> eyre::Result<()> { parakeet_db::models::tid_to_i64(&extract_rkey(root_post_uri)).unwrap(), test_cid(), root_post, - EventSource::Jetstream, + EventSource::Tap, ) .await .wrap_err("Failed to insert root post")?; diff --git a/consumer/tests/workers_test.rs b/consumer/tests/workers_test.rs index 73106dcc..a59586b3 100644 --- a/consumer/tests/workers_test.rs +++ b/consumer/tests/workers_test.rs @@ -1,24 +1,14 @@ //! Integration tests for database writer worker operations //! -//! These tests verify the operations used by batch writer workers: -//! - Actor ensuring (bulk INSERT with ON CONFLICT) -//! - Backfill status updates (UPDATE sync_state) -//! - Cache warming queries (pinned posts, recent posts) +//! These tests verify the bulk_ensure_actors operation used by batch writer workers +//! for ensuring actor entries exist in the database. //! -//! Note: PDS host tracking was moved to Redis (see redis_pds_cache.rs). -//! The pds_hosts and actor_pds_mapping tables don't exist in the schema. -//! -//! Tests call the actual functions in src/db/workers.rs (no SQL duplication). +//! Tests call the actual function in src/db/workers.rs (no SQL duplication). mod common; -use chrono::{Duration, Utc}; use common::*; -use consumer::database_writer::EventSource; -use consumer::db::{operations, workers}; -use consumer::types::records::{AppBskyActorProfile, AppBskyFeedPost}; +use consumer::db::workers; use eyre::WrapErr; -use lexica::StrongRef; -use parakeet_db::types::ActorSyncState; /// Test bulk_ensure_actors with new actors #[tokio::test] @@ -106,307 +96,3 @@ async fn test_bulk_ensure_actors_empty() -> eyre::Result<()> { Ok(()) } -/// Test backfill_update_actor_status updates sync_state and timestamp -#[tokio::test] -async fn test_backfill_update_actor_status() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // First create an actor - let dids = vec!["did:plc:backfill_test"]; - workers::bulk_ensure_actors(&tx, &dids).await.wrap_err("Failed to bulk ensure actors")?; - - // Update to synced state - let now = Utc::now(); - let result = workers::backfill_update_actor_status( - &tx, - "did:plc:backfill_test", - &ActorSyncState::Synced, - now, - ) - .await; - - assert!( - result.is_ok(), - "backfill_update_actor_status should work: {:?}", - result.err() - ); - - let rows_updated = result.wrap_err("Operation failed")?; - assert_eq!(rows_updated, 1, "Should update 1 actor"); - - // Verify the update worked - let row = tx - .query_one( - "SELECT sync_state FROM actors WHERE did = $1", - &[&"did:plc:backfill_test"], - ) - .await - .wrap_err("Query failed")?; - let state: ActorSyncState = row.get(0); - assert_eq!(state, ActorSyncState::Synced); -Ok(()) -} - -/// Test backfill_update_actor_status with non-existent DID (should be no-op) -#[tokio::test] -async fn test_backfill_update_actor_status_nonexistent() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - let now = Utc::now(); - let result = workers::backfill_update_actor_status( - &tx, - "did:plc:does_not_exist", - &ActorSyncState::Synced, - now, - ) - .await; - - assert!( - result.is_ok(), - "backfill_update_actor_status with non-existent DID should not error: {:?}", - result.err() - ); - - let rows_updated = result.wrap_err("Operation failed")?; - assert_eq!( - rows_updated, 0, - "Should update 0 actors (DID doesn't exist)" - ); -Ok(()) -} - -/// Test backfill_mark_processing inserts new actor as processing -#[tokio::test] -async fn test_backfill_mark_processing_new() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - let result = workers::backfill_mark_processing(&tx, "did:plc:processing_test").await; - - assert!( - result.is_ok(), - "backfill_mark_processing should work: {:?}", - result.err() - ); - - let rows_inserted = result.wrap_err("Operation failed")?; - assert_eq!(rows_inserted, 1, "Should insert 1 new actor"); - - // Verify the actor was created with processing state - let row = tx - .query_one( - "SELECT sync_state FROM actors WHERE did = $1", - &[&"did:plc:processing_test"], - ) - .await - .wrap_err("Query failed")?; - let state: ActorSyncState = row.get(0); - assert_eq!(state, ActorSyncState::Processing); -Ok(()) -} - -/// Test backfill_mark_processing updates existing actor to processing -#[tokio::test] -async fn test_backfill_mark_processing_existing() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // First create an actor with different state - let dids = vec!["did:plc:existing_processing"]; - workers::bulk_ensure_actors(&tx, &dids).await.wrap_err("Failed to bulk ensure actors")?; - - // Mark as processing - let result = workers::backfill_mark_processing(&tx, "did:plc:existing_processing").await; - - assert!( - result.is_ok(), - "backfill_mark_processing should work on existing actor: {:?}", - result.err() - ); - - // PostgreSQL returns 1 for ON CONFLICT DO UPDATE (counts as affected row) - let rows_affected = result.wrap_err("Operation failed")?; - assert_eq!( - rows_affected, 1, - "Should affect 1 row (update existing actor)" - ); - - // Verify the state was updated - let row = tx - .query_one( - "SELECT sync_state FROM actors WHERE did = $1", - &[&"did:plc:existing_processing"], - ) - .await - .wrap_err("Query failed")?; - let state: ActorSyncState = row.get(0); - assert_eq!(state, ActorSyncState::Processing); -Ok(()) -} - -/// Test get_pinned_post_uri returns None when no pinned post -#[tokio::test] -async fn test_get_pinned_post_uri_none() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - let result = workers::get_pinned_post_uri(&conn, "did:plc:no_profile").await; - - assert!( - result.is_ok(), - "get_pinned_post_uri should work: {:?}", - result.err() - ); - - let pinned_uri = result.wrap_err("Failed to get pinned post URI")?; - assert!(pinned_uri.is_none(), - "Should return None when no profile exists"); - Ok(()) -} - -/// Test get_pinned_post_uri returns URI when pinned post exists -#[tokio::test] -async fn test_get_pinned_post_uri_exists() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Create actor using production function - let (actor_id, _, _) = operations::feed::get_actor_id(&tx, "did:plc:with_pinned") - .await - .wrap_err("Failed to ensure actor")?; - - // Insert a post using production function - let post_uri = "at://did:plc:with_pinned/app.bsky.feed.post/3l7mkz4lmk235"; - let rkey = post_uri.split('/').next_back().unwrap_or(""); - let post = AppBskyFeedPost { - text: "test post".to_string(), - facets: None, - reply: None, - embed: None, - langs: None, - labels: None, - tags: None, - created_at: Utc::now(), - }; - let rkey_i64 = parakeet_db::models::tid_to_i64(rkey)?; - operations::feed::post_insert(&tx, actor_id, rkey_i64, test_cid(), post, EventSource::Jetstream) - .await - .wrap_err("Failed to insert post")?; - - // Create profile with pinned post using production function - let profile = AppBskyActorProfile { - display_name: Some("Test User".to_string()), - description: None, - avatar: None, - banner: None, - labels: None, - joined_via_starter_pack: None, - created_at: Some(Utc::now()), - pinned_post: Some(StrongRef { - uri: post_uri.to_string(), - cid: test_cid(), - }), - pronouns: None, - website: None, - }; - operations::actor::profile_upsert(&tx, actor_id, "did:plc:with_pinned", test_cid(), profile) - .await - .wrap_err("Failed to insert profile")?; - - let result = workers::get_pinned_post_uri(&tx, "did:plc:with_pinned").await; - - assert!( - result.is_ok(), - "get_pinned_post_uri should work: {:?}", - result.err() - ); - - let pinned_uri = result.wrap_err("Expected value")?; - assert!( - pinned_uri.is_some(), - "Should return Some when pinned post exists" - ); - - let uri = pinned_uri.ok_or_else(|| eyre::eyre!("Expected pinned URI to be Some"))?; - assert!(uri.starts_with("at://did:plc:with_pinned/app.bsky.feed.post/")); -Ok(()) -} - -/// Test get_recent_post_uris returns empty list when no posts -#[tokio::test] -async fn test_get_recent_post_uris_empty() -> eyre::Result<()> { - let pool = test_pool(); - let conn = pool.get().await.wrap_err("Failed to get connection")?; - - let result = workers::get_recent_post_uris(&conn, "did:plc:no_posts").await; - - assert!( - result.is_ok(), - "get_recent_post_uris should work: {:?}", - result.err() - ); - - let uris = result.wrap_err("Query failed")?; - assert!( - uris.is_empty(), - "Should return empty list when no posts exist" - ); - Ok(()) -} - -/// Test get_recent_post_uris returns posts ordered by created_at DESC -#[tokio::test] -async fn test_get_recent_post_uris_ordered() -> eyre::Result<()> { - let pool = test_pool(); - let mut conn = pool.get().await.wrap_err("Failed to get connection")?; - let tx = conn.transaction().await.wrap_err("Failed to start transaction")?; - - // Create actor using production function - let (actor_id, _, _) = operations::feed::get_actor_id(&tx, "did:plc:with_posts") - .await - .wrap_err("Failed to ensure actor")?; - - // Insert 3 posts with different timestamps using production function - let base_tids = ["3l7mkz4lmk23a", "3l7mkz4lmk23b", "3l7mkz4lmk23c"]; - for (i, tid) in base_tids.iter().enumerate() { - let created_at = Utc::now() - Duration::hours(i as i64); - let post = AppBskyFeedPost { - text: "test post".to_string(), - facets: None, - reply: None, - embed: None, - langs: None, - labels: None, - tags: None, - created_at, - }; - let tid_i64 = parakeet_db::models::tid_to_i64(tid)?; - operations::feed::post_insert(&tx, actor_id, tid_i64, test_cid(), post, EventSource::Jetstream) - .await - .wrap_err("Failed to insert post")?; - } - - let result = workers::get_recent_post_uris(&tx, "did:plc:with_posts").await; - - assert!( - result.is_ok(), - "get_recent_post_uris should work: {:?}", - result.err() - ); - - let uris = result.wrap_err("Query failed")?; - assert_eq!(uris.len(), 3, "Should return 3 posts"); - - // Verify all URIs are properly formatted - for uri in &uris { - assert!(uri.starts_with("at://did:plc:with_posts/app.bsky.feed.post/")); - } -Ok(()) -}