From abace9d6ce66ee1625b0601aa6a58d3f04049452 Mon Sep 17 00:00:00 2001 From: dawn <90008@gaze.systems> Date: Tue, 31 Mar 2026 02:38:35 +0300 Subject: [PATCH] [all] internal relay backed indexer --- Cargo.toml | 6 +- README.md | 2 +- src/api/debug.rs | 19 +- src/api/filter.rs | 6 +- src/api/mod.rs | 4 +- src/api/xrpc/get_latest_commit.rs | 3 + src/api/xrpc/get_repo.rs | 3 + src/api/xrpc/get_repo_status.rs | 31 +- src/api/xrpc/list_repos.rs | 6 +- src/api/xrpc/mod.rs | 18 +- src/backfill/manager.rs | 103 ++-- src/backfill/mod.rs | 104 ++-- src/control/filter.rs | 7 +- src/control/mod.rs | 115 ++-- src/control/repos.rs | 218 +++++-- src/control/stream.rs | 6 +- src/crawler/mod.rs | 4 +- src/crawler/worker.rs | 24 +- src/db/ephemeral.rs | 73 ++- src/db/keys/mod.rs | 24 +- src/db/migration/mod.rs | 2 + src/db/migration/v2.rs | 1 + src/db/migration/v4.rs | 89 +++ src/db/mod.rs | 77 +-- src/filter.rs | 11 +- src/ingest/firehose.rs | 30 +- src/ingest/indexer.rs | 653 ++++++++++++++++++++ src/ingest/mod.rs | 125 +--- src/ingest/relay.rs | 872 +++++++++++++++++++++++++++ src/ingest/relay_worker.rs | 529 ---------------- src/ingest/stream.rs | 18 +- src/ingest/worker.rs | 962 ------------------------------ src/lib.rs | 13 +- src/ops.rs | 146 ++--- src/types.rs | 136 +++-- tests/authenticated_stream.nu | 2 +- tests/backlinks.nu | 2 +- tests/by_collection.nu | 4 +- tests/run_all.nu | 9 +- tests/signal_filter.nu | 2 +- 40 files changed, 2428 insertions(+), 2031 deletions(-) create mode 100644 src/db/migration/v4.rs create mode 100644 src/ingest/indexer.rs create mode 100644 src/ingest/relay.rs delete mode 100644 src/ingest/relay_worker.rs delete mode 100644 src/ingest/worker.rs diff --git a/Cargo.toml b/Cargo.toml index ad1af1f..61f9c0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,11 +4,11 @@ version = "0.1.0" edition = "2024" [features] -default = ["events"] -sync_all = [] +default = ["indexer"] +__persist_sync_all = [] backlinks = [] relay = [] -events = [] +indexer = [] [dependencies] tokio = { version = "1.0", features = ["full"] } diff --git a/README.md b/README.md index a1bf1f5..8f5e4e6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ `hydrant` is an AT Protocol indexer built on the `fjall` database. it's built to be flexible, supporting both full-network indexing and filtered indexing (e.g., by DID), allowing querying with XRPCs (not only `com.atproto.*`!), providing an -ordered event stream, etc. +ordered event stream, etc. oh and it can also act as a relay! you can see [random.wisp.place](https://tangled.org/did:plc:dfl62fgb7wtjj3fcbb72naae/random.wisp.place) diff --git a/src/api/debug.rs b/src/api/debug.rs index 2d6590f..d494648 100644 --- a/src/api/debug.rs +++ b/src/api/debug.rs @@ -205,7 +205,7 @@ pub async fn handle_debug_iter( "invalid_u64".to_string() } } else if partition == "blocks" { - // key is col|cid_bytes — show as "col|" + // key is col|cid_bytes, show as "col|" if let Some(sep) = k.iter().position(|&b| b == keys::SEP) { let col = String::from_utf8_lossy(&k[..sep]); match cid::Cid::read_bytes(&k[sep + 1..]) { @@ -303,7 +303,11 @@ pub async fn handle_debug_ephemeral_ttl_tick( State(state): State>, ) -> Result { tokio::task::spawn_blocking(move || { - crate::db::ephemeral::ephemeral_ttl_tick(&state.db, &state.ephemeral_ttl) + #[cfg(feature = "indexer")] + let res = crate::db::ephemeral::ephemeral_ttl_tick(&state.db, &state.ephemeral_ttl); + #[cfg(feature = "relay")] + let res = crate::db::ephemeral::relay_events_ttl_tick(&state.db, &state.ephemeral_ttl); + res }) .await .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? @@ -316,7 +320,7 @@ pub async fn handle_debug_ephemeral_ttl_tick( pub struct DebugSeedWatermarkRequest { /// unix timestamp (seconds) to write the watermark at pub ts: u64, - /// event_id the watermark points to — all events before this id will be pruned + /// event_id the watermark points to, all events before this id will be pruned pub event_id: u64, } @@ -328,13 +332,14 @@ pub async fn handle_debug_seed_watermark( Query(req): Query, ) -> Result { tokio::task::spawn_blocking(move || { + #[cfg(feature = "indexer")] + let key = crate::db::keys::event_watermark_key(req.ts); + #[cfg(feature = "relay")] + let key = crate::db::keys::relay_event_watermark_key(req.ts); state .db .cursors - .insert( - crate::db::keys::event_watermark_key(req.ts), - req.event_id.to_be_bytes(), - ) + .insert(key, req.event_id.to_be_bytes()) .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) }) .await diff --git a/src/api/filter.rs b/src/api/filter.rs index f2d9fe0..5bf9312 100644 --- a/src/api/filter.rs +++ b/src/api/filter.rs @@ -16,6 +16,7 @@ pub fn router() -> Router { .route("/filter", patch(handle_patch_filter)) } +use tracing::error; pub async fn handle_get_filter( State(hydrant): State, ) -> Result, (StatusCode, String)> { @@ -46,6 +47,9 @@ pub async fn handle_patch_filter( p.excludes = body.excludes; p.apply() .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string())) + .map_err(|e| { + error!(err = %e, "failed to patch filter"); + (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()) + }) .map(Json) } diff --git a/src/api/mod.rs b/src/api/mod.rs index c829ee9..f4299bb 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -13,7 +13,7 @@ mod firehose; mod ingestion; mod repos; mod stats; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] mod stream; mod xrpc; @@ -22,7 +22,7 @@ pub async fn serve(hydrant: Hydrant, port: u16) -> miette::Result<()> { let mut app = Router::new() .route("/health", get(|| async { "OK" })) .route("/stats", get(stats::get_stats)); - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] let app = app.nest("/stream", stream::router()); let app = app .merge(xrpc::router()) diff --git a/src/api/xrpc/get_latest_commit.rs b/src/api/xrpc/get_latest_commit.rs index 62dffb3..3cf8eb1 100644 --- a/src/api/xrpc/get_latest_commit.rs +++ b/src/api/xrpc/get_latest_commit.rs @@ -31,6 +31,9 @@ pub async fn handle( RepoStatus::Takendown => Some(GetLatestCommitError::RepoTakendown(None)), RepoStatus::Suspended => Some(GetLatestCommitError::RepoSuspended(None)), RepoStatus::Deactivated => Some(GetLatestCommitError::RepoDeactivated(None)), + RepoStatus::Deleted => Some(GetLatestCommitError::RepoNotFound(Some(CowStr::Borrowed( + "deleted", + )))), _ => None, }; if let Some(err) = xrpc_err { diff --git a/src/api/xrpc/get_repo.rs b/src/api/xrpc/get_repo.rs index f4aca5c..4146080 100644 --- a/src/api/xrpc/get_repo.rs +++ b/src/api/xrpc/get_repo.rs @@ -32,6 +32,9 @@ pub async fn handle( RepoStatus::Takendown => Some(GetRepoError::RepoTakendown(None)), RepoStatus::Suspended => Some(GetRepoError::RepoSuspended(None)), RepoStatus::Deactivated => Some(GetRepoError::RepoDeactivated(None)), + RepoStatus::Deleted => Some(GetRepoError::RepoNotFound(Some(CowStr::Borrowed( + "deleted", + )))), _ => None, }; if let Some(err) = xrpc_err { diff --git a/src/api/xrpc/get_repo_status.rs b/src/api/xrpc/get_repo_status.rs index 3f28534..72bb73e 100644 --- a/src/api/xrpc/get_repo_status.rs +++ b/src/api/xrpc/get_repo_status.rs @@ -29,13 +29,16 @@ pub async fn handle( }); }; - let (active, status) = repo_status_to_api(state.status); + let status = repo_status_to_api(state.status); // rev is only meaningful when the repo is active and has been synced at least once - let rev = active.then(|| state.root.map(|c| c.rev.to_tid())).flatten(); + let rev = state + .active + .then(|| state.root.map(|c| c.rev.to_tid())) + .flatten(); Ok(Json(GetRepoStatusOutput { - active, + active: state.active, did: req.did, rev, status: status.map(|s| match s { @@ -51,18 +54,16 @@ pub async fn handle( })) } -pub(super) fn repo_status_to_api(status: RepoStatus) -> (bool, Option>) { +pub(super) fn repo_status_to_api(status: RepoStatus) -> Option> { match status { - RepoStatus::Synced => (true, None), - RepoStatus::Deactivated => (false, Some(ApiRepoStatus::Deactivated)), - RepoStatus::Takendown => (false, Some(ApiRepoStatus::Takendown)), - RepoStatus::Suspended => (false, Some(ApiRepoStatus::Suspended)), - // we lost sync with this repo! report desynchronized - // technicalllyyyy backfilling can mean the repo is active - // because we are syncing it from the pds, but like also it is currently - // desync'ed so... - RepoStatus::Backfilling | RepoStatus::Error(_) => { - (false, Some(ApiRepoStatus::Desynchronized)) - } + RepoStatus::Synced => None, + RepoStatus::Deactivated => Some(ApiRepoStatus::Deactivated), + RepoStatus::Takendown => Some(ApiRepoStatus::Takendown), + RepoStatus::Suspended => Some(ApiRepoStatus::Suspended), + RepoStatus::Deleted => Some(ApiRepoStatus::Deleted), + // per spec, desynchronized and throttled have active=may-be-true + RepoStatus::Desynchronized => Some(ApiRepoStatus::Desynchronized), + RepoStatus::Throttled => Some(ApiRepoStatus::Throttled), + RepoStatus::Error(_) => Some(ApiRepoStatus::Desynchronized), } } diff --git a/src/api/xrpc/list_repos.rs b/src/api/xrpc/list_repos.rs index 0745fb2..da8b96d 100644 --- a/src/api/xrpc/list_repos.rs +++ b/src/api/xrpc/list_repos.rs @@ -31,7 +31,7 @@ pub async fn handle( let mut next_cursor: Option> = None; for item in hydrant.repos.iter_states(cursor.as_ref()) { - let (did, state) = item?; + let (did, state, _metadata) = item?; // skip repos that haven't been synced at least once let Some(commit) = state.root else { @@ -48,9 +48,9 @@ pub async fn handle( continue; }; - let (active, status) = repo_status_to_api(state.status); + let status = repo_status_to_api(state.status); repos.push(Repo { - active: Some(active), + active: Some(state.active), did: did.clone(), head: Cid::from(commit_cid), rev: atp_commit.rev, diff --git a/src/api/xrpc/mod.rs b/src/api/xrpc/mod.rs index c278e40..b098376 100644 --- a/src/api/xrpc/mod.rs +++ b/src/api/xrpc/mod.rs @@ -27,6 +27,11 @@ use jacquard_common::{ use serde::{Deserialize, Serialize}; use smol_str::ToSmolStr; use std::fmt::Display; +#[cfg(feature = "relay")] +use { + jacquard_api::com_atproto::sync::subscribe_repos::SubscribeReposEndpoint, + jacquard_common::xrpc::SubscriptionEndpoint, +}; mod com_atproto_describe_repo; mod count_records; @@ -43,8 +48,7 @@ mod list_repos; mod subscribe_repos; pub fn router() -> Router { - #[allow(unused_mut)] - let mut r = Router::new() + let r = Router::new() .route(GetRecordRequest::PATH, get(get_record::handle)) .route(ListRecordsRequest::PATH, get(list_records::handle)) .route(CountRecords::PATH, get(count_records::handle)) @@ -61,12 +65,10 @@ pub fn router() -> Router { .route(ListReposRequest::PATH, get(list_repos::handle)); #[cfg(feature = "relay")] - { - r = r.route( - "/xrpc/com.atproto.sync.subscribeRepos", - axum::routing::get(subscribe_repos::handle), - ); - } + let r = r.route( + SubscribeReposEndpoint::PATH, + axum::routing::get(subscribe_repos::handle), + ); r } diff --git a/src/backfill/manager.rs b/src/backfill/manager.rs index 7fd100a..a5e91df 100644 --- a/src/backfill/manager.rs +++ b/src/backfill/manager.rs @@ -1,12 +1,12 @@ use crate::db::types::TrimmedDid; -use crate::db::{self, deser_repo_state}; -use crate::ops; +use crate::db::{self, keys}; use crate::state::AppState; -use crate::types::{GaugeState, RepoStatus, ResyncState}; +use crate::types::{GaugeState, ResyncState}; use miette::{IntoDiagnostic, Result}; + use std::sync::Arc; use std::time::Duration; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info}; pub fn queue_gone_backfills(state: &Arc) -> Result<()> { debug!("scanning for deactivated/takendown repos to retry..."); @@ -28,20 +28,38 @@ pub fn queue_gone_backfills(state: &Arc) -> Result<()> { if matches!(resync_state, ResyncState::Gone { .. }) { debug!(did = %did, "queuing retry for gone repo"); - let Some(state_bytes) = state.db.repos.get(&key).into_diagnostic()? else { - warn!(did = %did, "repo state not found"); - continue; + let metadata_key = keys::repo_metadata_key(&did); + let metadata_bytes = match state + .db + .repo_metadata + .get(&metadata_key) + .map(|b| b.ok_or_else(|| miette::miette!("repo metadata not found"))) + .into_diagnostic() + .flatten() + { + Ok(b) => b, + Err(e) => { + error!(did = %did, err = %e, "failed to get repo metadata"); + continue; + } }; - - // update repo state back to backfilling - let repo_state = deser_repo_state(&state_bytes)?; - ops::update_repo_status( - &mut batch, - &state.db, - &did, - repo_state, - RepoStatus::Backfilling, - )?; + let mut metadata = crate::db::deser_repo_metadata(&metadata_bytes)?; + + // move from resync back into pending + batch.remove(&state.db.resync, key.clone()); + let old_pending = keys::pending_key(metadata.index_id); + batch.remove(&state.db.pending, old_pending); + metadata.index_id = rand::random::(); + batch.insert( + &state.db.pending, + keys::pending_key(metadata.index_id), + key.clone(), + ); + batch.insert( + &state.db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, + ); transitions.push((GaugeState::Resync(None), GaugeState::Pending)); } @@ -100,36 +118,49 @@ pub fn retry_worker(state: Arc) { if next_retry <= now { debug!(did = %did, "retrying backfill"); - let state_bytes = match state.db.repos.get(&key).into_diagnostic() { + let metadata_key = keys::repo_metadata_key(&did); + let metadata_bytes = match state + .db + .repo_metadata + .get(&metadata_key) + .map(|b| b.ok_or_else(|| miette::miette!("repo metadata not found"))) + .into_diagnostic() + .flatten() + { Ok(b) => b, - Err(err) => { - error!(did = %did, err = %err, "failed to get repo state"); + Err(e) => { + error!(did = %did, err = %e, "failed to get repo metadata"); continue; } }; - let Some(state_bytes) = state_bytes else { - error!(did = %did, "repo state not found"); - continue; + let mut metadata = match crate::db::deser_repo_metadata( + metadata_bytes.as_ref(), + ) { + Ok(m) => m, + Err(e) => { + error!(did = %did, err = %e, "failed to deserialize repo metadata"); + continue; + } }; - let repo_state = match deser_repo_state(&state_bytes) { + // move from resync back into pending + batch.remove(&state.db.resync, key.clone()); + let old_pending = keys::pending_key(metadata.index_id); + batch.remove(&state.db.pending, old_pending); + metadata.index_id = rand::random::(); + batch.insert( + &state.db.pending, + keys::pending_key(metadata.index_id), + key.clone(), + ); + let serialized_metadata = match crate::db::ser_repo_metadata(&metadata) { Ok(s) => s, Err(e) => { - error!(did = %did, err = %e, "failed to deserialize repo state"); + error!(did = %did, err = %e, "failed to serialize repo metadata"); continue; } }; - let res = ops::update_repo_status( - &mut batch, - &state.db, - &did, - repo_state, - RepoStatus::Backfilling, - ); - if let Err(e) = res { - error!(did = %did, err = %e, "failed to update repo status"); - continue; - } + batch.insert(&state.db.repo_metadata, &metadata_key, serialized_metadata); transitions.push((GaugeState::Resync(Some(kind)), GaugeState::Pending)); } diff --git a/src/backfill/mod.rs b/src/backfill/mod.rs index ddd75f5..92a80ac 100644 --- a/src/backfill/mod.rs +++ b/src/backfill/mod.rs @@ -31,12 +31,12 @@ use tracing::{Instrument, debug, error, info, trace, warn}; pub mod manager; -use crate::ingest::{BufferTx, IngestMessage}; +use crate::ingest::indexer::{IndexerMessage, IndexerTx}; use crate::util::{WatchEnabledExt, url_to_fluent_uri}; pub struct BackfillWorker { state: Arc, - buffer_tx: BufferTx, + buffer_tx: IndexerTx, http: reqwest::Client, semaphore: Arc, verify_signatures: bool, @@ -48,7 +48,7 @@ pub struct BackfillWorker { impl BackfillWorker { pub fn new( state: Arc, - buffer_tx: BufferTx, + buffer_tx: IndexerTx, timeout: Duration, concurrency_limit: usize, verify_signatures: bool, @@ -182,7 +182,7 @@ impl BackfillWorker { async fn did_task( state: &Arc, http: reqwest::Client, - buffer_tx: BufferTx, + buffer_tx: IndexerTx, did: &Did<'static>, pending_key: Slice, _permit: tokio::sync::OwnedSemaphorePermit, @@ -192,30 +192,24 @@ async fn did_task( let db = &state.db; match process_did(&state, &http, &did, verify_signatures, ephemeral).await { - Ok(Some(repo_state)) => { + Ok(Some(_repo_state)) => { let did_key = keys::repo_key(&did); // determine old gauge state // if it was error/suspended etc, we need to know which error kind it was to decrement correctly. - // we have to peek at the resync state. - let old_gauge = state.db.repo_gauge_state_async(&repo_state, &did_key).await; - let mut batch = db.inner.batch(); - // remove from pending - if old_gauge == GaugeState::Pending { - batch.remove(&db.pending, pending_key); - } - // remove from resync - if old_gauge.is_resync() { - batch.remove(&db.resync, &did_key); - } + // unconditionally remove from pending + batch.remove(&db.pending, pending_key); + // remove from resync, just in case + batch.remove(&db.resync, &did_key); + tokio::task::spawn_blocking(move || batch.commit().into_diagnostic()) .await .into_diagnostic()??; state .db - .update_gauge_diff_async(&old_gauge, &GaugeState::Synced) + .update_gauge_diff_async(&GaugeState::Pending, &GaugeState::Synced) .await; let state = state.clone(); @@ -229,7 +223,10 @@ async fn did_task( .await .into_diagnostic()??; - if let Err(e) = buffer_tx.send(IngestMessage::BackfillFinished(did.clone())) { + if let Err(e) = buffer_tx + .send(IndexerMessage::BackfillFinished(did.clone())) + .await + { error!(err = %e, "failed to send BackfillFinished"); } Ok(()) @@ -310,6 +307,7 @@ async fn did_task( { let mut state: RepoState = rmp_serde::from_slice(&state_bytes).into_diagnostic()?; + state.active = true; state.status = RepoStatus::Error(error_string.into()); Some(rmp_serde::to_vec(&state).into_diagnostic()?) } else { @@ -422,22 +420,23 @@ async fn process_did<'i>( ); state.update_from_doc(doc); - let emit_identity = |status: &RepoStatus| { + let emit_identity = |status: &RepoStatus, active: bool| { + let status = match status { + RepoStatus::Deactivated => "deactivated", + RepoStatus::Takendown => "takendown", + RepoStatus::Suspended => "suspended", + RepoStatus::Deleted => "deleted", + RepoStatus::Desynchronized => "desynchronized", + RepoStatus::Throttled => "throttled", + _ => "", + }; let evt = AccountEvt { did: did.clone(), - active: !matches!( - status, - RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended - ), - status: Some( - match status { - RepoStatus::Deactivated => "deactivated", - RepoStatus::Takendown => "takendown", - RepoStatus::Suspended => "suspended", - _ => "active", - } - .into(), - ), + active, + status: status + .is_empty() + .then_some(None) + .unwrap_or_else(|| Some(status.into())), }; let _ = app_state.db.event_tx.send(ops::make_account_event(db, evt)); }; @@ -472,7 +471,7 @@ async fn process_did<'i>( if let Some(status) = inactive_status { warn!(?status, "repo is inactive, stopping backfill"); - emit_identity(&status); + emit_identity(&status, false); let resync_state = ResyncState::Gone { status: status.clone(), @@ -483,6 +482,7 @@ async fn process_did<'i>( app_state .db .update_repo_state_async(did, move |state, (key, batch)| { + state.active = false; state.status = status; batch.insert(&app_state_clone.db.resync, key, resync_bytes); Ok((true, ())) @@ -498,8 +498,13 @@ async fn process_did<'i>( Err(e) => Err(e).into_diagnostic()?, }; - // emit identity event so any consumers know - emit_identity(&state.status); + // emit identity event so any consumers know, but only if something changed + if state.active != previous_state.active + || state.status != previous_state.status + || previous_state.pds.is_none() + { + emit_identity(&state.status, state.active); + } trace!( bytes = car_bytes.body.len(), @@ -721,7 +726,6 @@ async fn process_did<'i>( } // 6. update data, status is updated in worker shard - state.tracked = true; state.root = Some(root_commit); state.touch(); @@ -731,6 +735,21 @@ async fn process_did<'i>( ser_repo_state(&state)?, ); + let metadata_key = keys::repo_metadata_key(&did); + let metadata_bytes = app_state + .db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let mut metadata = crate::db::deser_repo_metadata(&metadata_bytes)?; + metadata.tracked = true; + batch.insert( + &app_state.db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, + ); + // add the counts if !ephemeral { for (col, cnt) in collection_counts { @@ -746,14 +765,23 @@ async fn process_did<'i>( .into_diagnostic()?? }; + let metadata_key = keys::repo_metadata_key(did); + let metadata_bytes = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let metadata = crate::db::deser_repo_metadata(metadata_bytes.as_ref())?; + let Some((_state, records_cnt_delta, added_blocks, count)) = result else { - // signal mode: no signal-matching records found — clean up the optimistically-added repo + // signal mode: no signal-matching records found, clean up the optimistically-added repo let did_key = keys::repo_key(did); - let backfill_pending_key = keys::pending_key(previous_state.index_id); + let backfill_pending_key = keys::pending_key(metadata.index_id); let app_state = app_state.clone(); tokio::task::spawn_blocking(move || { let mut batch = app_state.db.inner.batch(); batch.remove(&app_state.db.repos, &did_key); + batch.remove(&app_state.db.repo_metadata, &metadata_key); batch.remove(&app_state.db.pending, backfill_pending_key); batch.commit().into_diagnostic() }) diff --git a/src/control/filter.rs b/src/control/filter.rs index be710a4..93f6419 100644 --- a/src/control/filter.rs +++ b/src/control/filter.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use tracing::error; use miette::{IntoDiagnostic, Result}; @@ -284,7 +285,11 @@ impl FilterPatch { db_filter::load(&filter_ks) }) .await - .into_diagnostic()??; + .into_diagnostic()? + .map_err(|e| { + error!(err = %e, "failed to apply filter patch"); + e + })?; let exclude_list = { let filter_ks = self.state.db.filter.clone(); diff --git a/src/control/mod.rs b/src/control/mod.rs index 265588e..3ae9010 100644 --- a/src/control/mod.rs +++ b/src/control/mod.rs @@ -1,3 +1,5 @@ +#![allow(unused_imports)] + pub(crate) mod crawler; pub(crate) mod filter; pub(crate) mod firehose; @@ -22,7 +24,7 @@ use miette::{IntoDiagnostic, Result, WrapErr}; use tokio::sync::{mpsc, watch}; use tracing::{debug, error, info}; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] use crate::backfill::BackfillWorker; use crate::config::{Config, SignatureVerification}; use crate::db::{ @@ -30,14 +32,14 @@ use crate::db::{ load_persisted_firehose_sources, }; use crate::filter::FilterMode; -#[cfg(feature = "events")] -use crate::ingest::worker::FirehoseWorker; +#[cfg(feature = "indexer")] +use crate::ingest::indexer::FirehoseWorker; use crate::state::AppState; use crate::types::MarshallableEvt; use crawler::{CrawlerShared, spawn_crawler_producer}; use firehose::{FirehoseShared, spawn_firehose_ingestor}; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] use stream::event_stream_thread; #[cfg(feature = "relay")] use stream::relay_stream_thread; @@ -215,16 +217,21 @@ impl Hydrant { } let fut = async move { - // internal buffered channel between ingestors / backfill and the firehose worker - let (buffer_tx, buffer_rx) = mpsc::unbounded_channel(); + // raw firehose events from pds/relay to RelayWorker + let (buffer_tx, buffer_rx) = mpsc::channel::(500); + + // validated IndexerMessages from RelayWorker/backfill to FirehoseWorker + #[cfg(feature = "indexer")] + let (indexer_tx, indexer_rx) = + mpsc::channel::(500); // 5. spawn the backfill worker (not used in relay mode) - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] tokio::spawn({ let state = state.clone(); BackfillWorker::new( state.clone(), - buffer_tx.clone(), + indexer_tx.clone(), config.repo_fetch_timeout, config.backfill_concurrency_limit, matches!( @@ -238,7 +245,7 @@ impl Hydrant { }); // 6. re-queue any repos that lost their backfill state, then start the retry worker - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] { if let Err(e) = tokio::task::spawn_blocking({ let state = state.clone(); @@ -258,7 +265,7 @@ impl Hydrant { } // 7. ephemeral GC thread (not used in relay mode) - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] if config.ephemeral { let state = state.clone(); std::thread::Builder::new() @@ -267,6 +274,16 @@ impl Hydrant { .into_diagnostic()?; } + // relay events TTL: relay_events keyspace grows unbounded without pruning + #[cfg(feature = "relay")] + { + let state = state.clone(); + std::thread::Builder::new() + .name("relay-events-gc".into()) + .spawn(move || crate::db::ephemeral::relay_events_ttl_worker(state)) + .into_diagnostic()?; + } + // 8. cursor / counts persist thread std::thread::spawn({ let state = state.clone(); @@ -298,17 +315,23 @@ impl Hydrant { }); // 9. events/sec stats ticker - #[cfg(feature = "events")] tokio::spawn({ let state = state.clone(); - let mut last_id = state.db.next_event_id.load(Ordering::Relaxed); + let get_id = |state: &AppState| { + #[cfg(feature = "indexer")] + let id = state.db.next_event_id.load(Ordering::Relaxed); + #[cfg(feature = "relay")] + let id = state.db.next_relay_seq.load(Ordering::Relaxed); + id + }; + let mut last_id = get_id(&state); let mut last_time = std::time::Instant::now(); let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); async move { loop { interval.tick().await; - let current_id = state.db.next_event_id.load(Ordering::Relaxed); + let current_id = get_id(&state); let current_time = std::time::Instant::now(); let delta = current_id.saturating_sub(last_id); @@ -410,7 +433,8 @@ impl Hydrant { .await; } - // 11. spawn crawler infrastructure (always, to support dynamic source management) + // 11. spawn crawler infrastructure + #[cfg(feature = "indexer")] { use crate::crawler::throttle::Throttler; use crate::crawler::{ @@ -536,16 +560,21 @@ impl Hydrant { } } - // 12. spawn the firehose worker on a blocking thread (fatal task) - let handle = tokio::runtime::Handle::current(); - let firehose_worker = std::thread::spawn({ + // 12. spawn the relay worker + let relay_worker = std::thread::spawn({ let state = state.clone(); - let handle = handle.clone(); + let handle = tokio::runtime::Handle::current(); + let config = config.clone(); + + #[cfg(feature = "indexer")] + let hook = indexer_tx.clone(); + move || { - #[cfg(feature = "relay")] - return crate::ingest::relay_worker::RelayWorker::new( + crate::ingest::relay::RelayWorker::new( state, buffer_rx, + #[cfg(feature = "indexer")] + hook, matches!(config.verify_signatures, SignatureVerification::Full), config.firehose_workers, crate::ingest::validation::ValidationOptions { @@ -553,30 +582,48 @@ impl Hydrant { rev_clock_skew_secs: config.rev_clock_skew_secs, }, ) - .run(handle); - #[cfg(feature = "events")] - return FirehoseWorker::new( + .run(handle) + } + }); + + let tx = Arc::clone(&fatal_tx); + tokio::spawn( + tokio::task::spawn_blocking(move || { + relay_worker + .join() + .map_err(|e| miette::miette!("relay worker died: {e:?}")) + }) + .map(move |r| { + let result = r.into_diagnostic().flatten().flatten(); + let _ = tx.send(Some(result.map_err(|e| e.to_string()))); + }), + ); + + // 13. spawn the firehose worker (if enabled) + #[cfg(feature = "indexer")] + let firehose_worker = std::thread::spawn({ + let state = state.clone(); + let handle = tokio::runtime::Handle::current(); + let config = config.clone(); + move || { + FirehoseWorker::new( state, - buffer_rx, - matches!(config.verify_signatures, SignatureVerification::Full), + indexer_rx, config.ephemeral, config.firehose_workers, - crate::ingest::validation::ValidationOptions { - verify_mst: config.verify_mst, - rev_clock_skew_secs: config.rev_clock_skew_secs, - }, ) - .run(handle); + .run(handle) } }); + #[cfg(feature = "indexer")] { let tx = Arc::clone(&fatal_tx); tokio::spawn( tokio::task::spawn_blocking(move || { firehose_worker .join() - .map_err(|e| miette::miette!("buffer processor died: {e:?}")) + .map_err(|e| miette::miette!("firehose worker died: {e:?}")) }) .map(move |r| { let result = r.into_diagnostic().flatten().flatten(); @@ -619,7 +666,7 @@ impl Hydrant { /// /// multiple concurrent subscribers each receive a full independent copy of the stream. /// the stream ends when the `EventStream` is dropped. - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] pub fn subscribe(&self, cursor: Option) -> EventStream { let (tx, rx) = mpsc::channel(500); let state = self.state.clone(); @@ -835,10 +882,10 @@ impl axum::extract::FromRef for Arc { /// implements [`futures::Stream`] and can be used with `StreamExt::next`, /// `while let Some(evt) = stream.next().await`, `forward`, etc. /// the stream terminates when the underlying channel closes (i.e. hydrant shuts down). -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] pub struct EventStream(mpsc::Receiver); -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] impl Stream for EventStream { type Item = Event; diff --git a/src/control/repos.rs b/src/control/repos.rs index e445999..893db0f 100644 --- a/src/control/repos.rs +++ b/src/control/repos.rs @@ -17,9 +17,9 @@ use smol_str::ToSmolStr; use url::Url; use crate::db::types::{DbRkey, DidKey, TrimmedDid}; -use crate::db::{self, Db, keys, ser_repo_state}; +use crate::db::{self, Db, keys}; use crate::state::AppState; -use crate::types::{GaugeState, RepoState, RepoStatus}; +use crate::types::{GaugeState, RepoMetadata, RepoState, RepoStatus}; use crate::util::invalid_handle; /// information about a tracked or known repository. returned by [`ReposControl`] methods. @@ -77,7 +77,8 @@ impl ReposControl { pub(crate) fn iter_states( &self, cursor: Option<&Did<'_>>, - ) -> impl Iterator, RepoState<'static>)>> { + ) -> impl Iterator, RepoState<'static>, crate::types::RepoMetadata)>> + { let start_bound = if let Some(cursor) = cursor { let did_key = keys::repo_key(cursor); std::ops::Bound::Excluded(did_key) @@ -85,22 +86,30 @@ impl ReposControl { std::ops::Bound::Unbounded }; + let db = self.0.db.clone(); self.0 .db .repos .range((start_bound, std::ops::Bound::Unbounded)) - .map(|g| { + .map(move |g| { let (k, v) = g.into_inner().into_diagnostic()?; let repo_state = crate::db::deser_repo_state(&v)?.into_static(); let did = TrimmedDid::try_from(k.as_ref())?.to_did(); - Ok((did, repo_state)) + let metadata_key = keys::repo_metadata_key(&did); + let metadata = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let metadata = crate::db::deser_repo_metadata(metadata.as_ref())?; + Ok((did, repo_state, metadata)) }) } /// iterates through all repositories, returning their state. pub fn iter(&self, cursor: Option<&Did<'_>>) -> impl Iterator> { self.iter_states(cursor) - .map(|r| r.map(|(did, s)| repo_state_to_info(did, s))) + .map(|r| r.map(|(did, s, m)| repo_state_to_info(did, s, m.tracked))) } #[allow(dead_code)] @@ -113,6 +122,7 @@ impl ReposControl { }; let repos = self.0.db.repos.clone(); + let db = self.0.db.clone(); self.0 .db .pending @@ -131,9 +141,19 @@ impl ReposControl { tracing::warn!(id, did = ?did_key, "stale pending???"); return Ok(None); }; - let repo_state = crate::db::deser_repo_state(&bytes)?; + let repo_state = crate::db::deser_repo_state(bytes.as_ref())?; let did = TrimmedDid::try_from(did_key.as_ref())?.to_did(); - Ok(Some((id, repo_state_to_info(did, repo_state)))) + let metadata_key = keys::repo_metadata_key(&did); + let metadata = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let metadata = crate::db::deser_repo_metadata(metadata.as_ref())?; + Ok(Some(( + id, + repo_state_to_info(did, repo_state.into_static(), metadata.tracked), + ))) }) .map(|b| b.transpose()) .flatten() @@ -149,6 +169,7 @@ impl ReposControl { }; let repos = self.0.db.repos.clone(); + let db = self.0.db.clone(); self.0 .db .resync @@ -160,9 +181,20 @@ impl ReposControl { tracing::warn!(did = ?did_key, "stale resync???"); return Ok(None); }; - let repo_state = crate::db::deser_repo_state(&bytes)?; + let repo_state = crate::db::deser_repo_state(bytes.as_ref())?; let did = TrimmedDid::try_from(did_key.as_ref())?.to_did(); - Ok(Some(repo_state_to_info(did, repo_state))) + let metadata_key = keys::repo_metadata_key(&did); + let metadata = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let metadata = crate::db::deser_repo_metadata(metadata.as_ref())?; + Ok(Some(repo_state_to_info( + did, + repo_state.into_static(), + metadata.tracked, + ))) }) .map(|b| b.transpose()) .flatten() @@ -199,28 +231,46 @@ impl ReposControl { transitions: &mut Vec<(GaugeState, GaugeState)>, ) -> Result { let did_key = keys::repo_key(did); + let metadata_key = keys::repo_metadata_key(did); + let repo_bytes = db.repos.get(&did_key).into_diagnostic()?; let existing = repo_bytes .as_deref() .map(db::deser_repo_state) .transpose()?; - if let Some(mut repo_state) = existing - && repo_state.status != RepoStatus::Backfilling - { - let resync = db.resync.get(&did_key).into_diagnostic()?; - let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); - repo_state.tracked = true; - repo_state.status = RepoStatus::Backfilling; - batch.insert(&db.repos, &did_key, ser_repo_state(&repo_state)?); - batch.insert( - &db.pending, - keys::pending_key(repo_state.index_id), - &did_key, - ); - batch.remove(&db.resync, &did_key); - transitions.push((old, GaugeState::Pending)); - return Ok(true); + if let Some(repo_state) = existing { + let metadata_bytes = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let mut metadata = crate::db::deser_repo_metadata(&metadata_bytes)?; + + // skip if already in pending queue + let is_pending = db + .pending + .get(keys::pending_key(metadata.index_id)) + .into_diagnostic()? + .is_some(); + if !is_pending { + let resync = db.resync.get(&did_key).into_diagnostic()?; + let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); + metadata.tracked = true; + // insert into pending with new index_id + let old_pending = keys::pending_key(metadata.index_id); + batch.remove(&db.pending, &old_pending); + metadata.index_id = rand::Rng::next_u64(&mut rand::rng()); + batch.insert(&db.pending, keys::pending_key(metadata.index_id), &did_key); + batch.remove(&db.resync, &did_key); + batch.insert( + &db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, + ); + transitions.push((old, GaugeState::Pending)); + return Ok(true); + } } Ok(false) @@ -291,26 +341,27 @@ impl ReposControl { for did in dids { let did_key = keys::repo_key(&did); - let repo_bytes = db.repos.get(&did_key).into_diagnostic()?; - let existing = repo_bytes - .as_deref() - .map(db::deser_repo_state) + let metadata_key = keys::repo_metadata_key(&did); + + let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; + let existing_metadata = metadata_bytes + .map(|b| crate::db::deser_repo_metadata(&b)) .transpose()?; - if let Some(repo_state) = existing { - // the double read here is an ok tradeoff, the block will be in read-cache anyway - if !repo_state.tracked && Self::_resync(db, &did, &mut batch, &mut transitions)? - { + if let Some(metadata) = existing_metadata { + if !metadata.tracked && Self::_resync(db, &did, &mut batch, &mut transitions)? { queued.push(did); } } else { - let repo_state = RepoState::backfilling(rng.next_u64()); - batch.insert(&db.repos, &did_key, ser_repo_state(&repo_state)?); + let repo_state = RepoState::backfilling(); + let metadata = RepoMetadata::backfilling(rng.next_u64()); + batch.insert(&db.repos, &did_key, crate::db::ser_repo_state(&repo_state)?); batch.insert( - &db.pending, - keys::pending_key(repo_state.index_id), - &did_key, + &db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, ); + batch.insert(&db.pending, keys::pending_key(metadata.index_id), &did_key); added += 1; queued.push(did); transitions.push((GaugeState::Synced, GaugeState::Pending)); @@ -351,6 +402,8 @@ impl ReposControl { for did in dids { let did_key = keys::repo_key(&did); + let metadata_key = keys::repo_metadata_key(&did); + let repo_bytes = db.repos.get(&did_key).into_diagnostic()?; let existing = repo_bytes .as_deref() @@ -358,18 +411,28 @@ impl ReposControl { .transpose()?; if let Some(repo_state) = existing { - if repo_state.tracked { - let resync = db.resync.get(&did_key).into_diagnostic()?; - let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); - let mut repo_state = repo_state.into_static(); - repo_state.tracked = false; - batch.insert(&db.repos, &did_key, ser_repo_state(&repo_state)?); - batch.remove(&db.pending, keys::pending_key(repo_state.index_id)); - batch.remove(&db.resync, &did_key); - if old != GaugeState::Synced { - gauge_decrements.push(old); + let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; + let existing_metadata = metadata_bytes + .map(|b| crate::db::deser_repo_metadata(&b)) + .transpose()?; + + if let Some(mut metadata) = existing_metadata { + if metadata.tracked { + let resync = db.resync.get(&did_key).into_diagnostic()?; + let old = db::Db::repo_gauge_state(&repo_state, resync.as_deref()); + metadata.tracked = false; + batch.insert( + &db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, + ); + batch.remove(&db.pending, keys::pending_key(metadata.index_id)); + batch.remove(&db.resync, &did_key); + if old != GaugeState::Synced { + gauge_decrements.push(old); + } + untracked.push(did); } - untracked.push(did); } } } @@ -390,7 +453,7 @@ impl ReposControl { } } -pub(crate) fn repo_state_to_info(did: Did<'static>, s: RepoState<'_>) -> RepoInfo { +pub(crate) fn repo_state_to_info(did: Did<'static>, s: RepoState<'_>, tracked: bool) -> RepoInfo { let (rev, data) = s .root .map(|c| (Some(c.rev.to_tid()), Some(c.data))) @@ -398,7 +461,7 @@ pub(crate) fn repo_state_to_info(did: Did<'static>, s: RepoState<'_>) -> RepoInf RepoInfo { did, status: s.status, - tracked: s.tracked, + tracked, rev, data, handle: s.handle.map(|h| h.into_static()), @@ -479,7 +542,29 @@ impl<'i> RepoHandle<'i> { /// returns `None` if hydrant has never seen this repository. pub async fn info(&self) -> Result> { let did = self.did.clone().into_static(); - Ok(self.state().await?.map(|s| repo_state_to_info(did, s))) + let did_key = keys::repo_key(&did); + let metadata_key = keys::repo_metadata_key(&did); + let app_state = self.state.clone(); + + tokio::task::spawn_blocking(move || { + let state_bytes = app_state.db.repos.get(&did_key).into_diagnostic()?; + let Some(state_bytes) = state_bytes else { + return Ok(None); + }; + let repo_state = crate::db::deser_repo_state(&state_bytes)?; + + let metadata_bytes = app_state + .db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .ok_or_else(|| miette::miette!("repo metadata not found for {}", did))?; + let metadata = crate::db::deser_repo_metadata(&metadata_bytes)?; + + Ok(Some(repo_state_to_info(did, repo_state, metadata.tracked))) + }) + .await + .into_diagnostic()? } /// returns the collections of this repository and the number of records it has in each. @@ -518,7 +603,32 @@ impl<'i> RepoHandle<'i> { return Err(MiniDocError::RepoNotFound); }; - if info.status == RepoStatus::Backfilling { + // check if repo is still backfilling (in pending) + let metadata_key = keys::repo_metadata_key(&self.did); + let app_state = self.state.clone(); + + let is_pending = tokio::task::spawn_blocking(move || { + let metadata_bytes = app_state + .db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()?; + let Some(metadata_bytes) = metadata_bytes else { + return Ok::<_, miette::Report>(false); + }; + let metadata = crate::db::deser_repo_metadata(metadata_bytes.as_ref())?; + Ok(app_state + .db + .pending + .get(crate::db::keys::pending_key(metadata.index_id)) + .into_diagnostic()? + .is_some()) + }) + .await + .map_err(|e| MiniDocError::Other(miette::miette!(e)))? + .map_err(MiniDocError::Other)?; + + if is_pending { return Err(MiniDocError::NotSynced); } diff --git a/src/control/stream.rs b/src/control/stream.rs index af9508c..88e4b74 100644 --- a/src/control/stream.rs +++ b/src/control/stream.rs @@ -7,7 +7,7 @@ use crate::db::keys; use crate::state::AppState; use std::sync::atomic::Ordering; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] use { super::Event, crate::db, @@ -20,7 +20,7 @@ use { sha2::{Digest, Sha256}, }; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] pub(super) fn event_stream_thread( state: Arc, tx: mpsc::Sender, @@ -156,7 +156,7 @@ pub(super) fn relay_stream_thread( } } -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] fn stored_to_event(state: &AppState, id: u64, stored: StoredEvent<'_>) -> Option { let StoredEvent { live, diff --git a/src/crawler/mod.rs b/src/crawler/mod.rs index 5852b48..b496722 100644 --- a/src/crawler/mod.rs +++ b/src/crawler/mod.rs @@ -1,3 +1,5 @@ +#![allow(dead_code, unused_imports)] + use crate::state::AppState; use futures::future::join_all; use jacquard_common::types::string::Did; @@ -121,7 +123,7 @@ impl CrawlerStats { if delta_processed == 0 && delta_crawled == 0 { if is_throttled { info!("throttled: pending queue full"); - } else { + } else if *self.0.state.crawler_enabled.borrow() { info!("idle: no repos crawled or processed in 60s"); } continue; diff --git a/src/crawler/worker.rs b/src/crawler/worker.rs index 7f44320..2242db6 100644 --- a/src/crawler/worker.rs +++ b/src/crawler/worker.rs @@ -1,6 +1,6 @@ use crate::db::{keys, ser_repo_state}; use crate::state::AppState; -use crate::types::RepoState; +use crate::types::{RepoMetadata, RepoState}; use miette::{IntoDiagnostic, Result}; use rand::Rng; use rand::rngs::SmallRng; @@ -142,25 +142,33 @@ impl CrawlerWorker { BLOCKING_TASK_TIMEOUT, tokio::task::spawn_blocking(move || -> Result> { let mut rng: SmallRng = rand::make_rng(); - let mut write_batch = db.inner.batch(); + let mut batch = db.inner.batch(); let mut surviving = Vec::new(); for guard in guards { let did_key = keys::repo_key(&*guard); + let metadata_key = keys::repo_metadata_key(&*guard); if db.repos.contains_key(&did_key).into_diagnostic()? { continue; } - let state = RepoState::untracked(rng.next_u64()); - write_batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); - write_batch.insert(&db.pending, keys::pending_key(state.index_id), &did_key); + let state = RepoState::backfilling(); + let metadata = RepoMetadata::backfilling(rng.next_u64()); + batch.insert(&db.repos, &did_key, ser_repo_state(&state)?); + batch.insert( + &db.repo_metadata, + &metadata_key, + crate::db::ser_repo_metadata(&metadata)?, + ); + batch.insert(&db.pending, keys::pending_key(metadata.index_id), &did_key); // clear any stale retry entry, this DID is confirmed and being enqueued - write_batch.remove(&db.crawler, keys::crawler_retry_key(&*guard)); + batch.remove(&db.crawler, keys::crawler_retry_key(&*guard)); trace!(did = %*guard, "enqueuing repo"); surviving.push(guard); } if let Some(cursor) = cursor_update { - write_batch.insert(&db.cursors, cursor.key, cursor.value); + batch.insert(&db.cursors, cursor.key, cursor.value); } - write_batch.commit().into_diagnostic()?; + // todo: repo state overwrites here are acceptable? + batch.commit().into_diagnostic()?; Ok(surviving) }), ) diff --git a/src/db/ephemeral.rs b/src/db/ephemeral.rs index 65fbb37..bb53abd 100644 --- a/src/db/ephemeral.rs +++ b/src/db/ephemeral.rs @@ -1,10 +1,12 @@ use crate::db::{Db, keys}; +use fjall::Keyspace; use miette::{IntoDiagnostic, WrapErr}; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::Duration; use tracing::{debug, error, info}; +#[cfg(feature = "indexer")] pub fn ephemeral_ttl_worker(state: Arc) { info!("ephemeral TTL worker started"); loop { @@ -15,59 +17,96 @@ pub fn ephemeral_ttl_worker(state: Arc) { } } +#[cfg(feature = "relay")] +pub fn relay_events_ttl_worker(state: Arc) { + info!("relay events TTL worker started"); + loop { + std::thread::sleep(Duration::from_secs(60)); + if let Err(e) = relay_events_ttl_tick(&state.db, &state.ephemeral_ttl) { + error!(err = %e, "relay events TTL tick failed"); + } + } +} + +#[cfg(feature = "indexer")] pub fn ephemeral_ttl_tick(db: &Db, ttl: &Duration) -> miette::Result<()> { + let current_seq = db.next_event_id.load(Ordering::SeqCst); + ttl_tick_inner( + db, + ttl, + keys::EVENT_WATERMARK_PREFIX, + keys::event_watermark_key, + &db.events, + current_seq, + ) +} + +#[cfg(feature = "relay")] +pub fn relay_events_ttl_tick(db: &Db, ttl: &Duration) -> miette::Result<()> { + let current_seq = db.next_relay_seq.load(Ordering::SeqCst); + ttl_tick_inner( + db, + ttl, + keys::RELAY_EVENT_WATERMARK_PREFIX, + keys::relay_event_watermark_key, + &db.relay_events, + current_seq, + ) +} + +fn ttl_tick_inner( + db: &Db, + ttl: &Duration, + watermark_prefix: &'static [u8], + watermark_key: fn(u64) -> Vec, + events_ks: &Keyspace, + current_seq: u64, +) -> miette::Result<()> { let now = chrono::Utc::now().timestamp() as u64; let cutoff_ts = now.saturating_sub(ttl.as_secs()); // write current watermark - #[cfg(feature = "events")] - let current_event_id = db.next_event_id.load(Ordering::SeqCst); - #[cfg(not(feature = "events"))] - let current_event_id = 0u64; db.cursors - .insert( - keys::event_watermark_key(now), - current_event_id.to_be_bytes(), - ) + .insert(watermark_key(now), current_seq.to_be_bytes()) .into_diagnostic()?; // find the watermark entry closest to and <= cutoff_ts - let cutoff_key = keys::event_watermark_key(cutoff_ts); - let cutoff_event_id = db + let cutoff_key = watermark_key(cutoff_ts); + let cutoff_seq = db .cursors .range(..=cutoff_key.as_slice()) .next_back() .map(|g| g.into_inner().into_diagnostic()) .transpose()? - .filter(|(k, _)| k.starts_with(keys::EVENT_WATERMARK_PREFIX)) + .filter(|(k, _)| k.starts_with(watermark_prefix)) .map(|(_, v)| { v.as_ref() .try_into() .into_diagnostic() - .wrap_err("expected cutoff event id to be u64") + .wrap_err("expected cutoff seq to be u64") }) .transpose()? .map(u64::from_be_bytes); - let Some(cutoff_event_id) = cutoff_event_id else { + let Some(cutoff_seq) = cutoff_seq else { // no watermark old enough yet, nothing to prune return Ok(()); }; - let cutoff_key_events = keys::event_key(cutoff_event_id); + let cutoff_key_events = keys::event_key(cutoff_seq); let mut batch = db.inner.batch(); let mut pruned = 0usize; - for guard in db.events.range(..cutoff_key_events) { + for guard in events_ks.range(..cutoff_key_events) { let k = guard.key().into_diagnostic()?; - batch.remove(&db.events, k); + batch.remove(events_ks, k); pruned += 1; } // clean up consumed watermark entries (everything up to and including cutoff_ts) for guard in db.cursors.range(..=cutoff_key) { let k = guard.key().into_diagnostic()?; - if k.starts_with(keys::EVENT_WATERMARK_PREFIX) { + if k.starts_with(watermark_prefix) { batch.remove(&db.cursors, k); } } diff --git a/src/db/keys/mod.rs b/src/db/keys/mod.rs index c92a90e..0b4501c 100644 --- a/src/db/keys/mod.rs +++ b/src/db/keys/mod.rs @@ -10,8 +10,12 @@ pub use v1::{firehose_cursor_key, firehose_cursor_key_from_url}; /// separator used for composite keys pub const SEP: u8 = b'|'; +#[cfg(feature = "indexer")] pub const EVENT_WATERMARK_PREFIX: &[u8] = b"ewm|"; +#[cfg(feature = "relay")] +pub const RELAY_EVENT_WATERMARK_PREFIX: &[u8] = b"rwm|"; + /// THIS SHOULD ALWAYS BE STABLE. DO NOT CHANGE pub const VERSIONING_KEY: &[u8] = b"db_version"; @@ -22,10 +26,20 @@ pub fn repo_key<'a>(did: &'a Did) -> Vec { vec } +pub const REPO_METADATA_PREFIX: &[u8] = b"rm|"; + +pub fn repo_metadata_key<'a>(did: &'a Did) -> Vec { + let mut vec = Vec::with_capacity(REPO_METADATA_PREFIX.len() + 32); + vec.extend_from_slice(REPO_METADATA_PREFIX); + TrimmedDid::from(did).write_to_vec(&mut vec); + vec +} + pub fn pending_key(id: u64) -> [u8; 8] { id.to_be_bytes() } +#[cfg(feature = "indexer")] pub fn event_watermark_key(timestamp_secs: u64) -> Vec { let mut key = Vec::with_capacity(EVENT_WATERMARK_PREFIX.len() + 8); key.extend_from_slice(EVENT_WATERMARK_PREFIX); @@ -33,6 +47,14 @@ pub fn event_watermark_key(timestamp_secs: u64) -> Vec { key } +#[cfg(feature = "relay")] +pub fn relay_event_watermark_key(timestamp_secs: u64) -> Vec { + let mut key = Vec::with_capacity(RELAY_EVENT_WATERMARK_PREFIX.len() + 8); + key.extend_from_slice(RELAY_EVENT_WATERMARK_PREFIX); + key.extend_from_slice(×tamp_secs.to_be_bytes()); + key +} + // prefix format: {DID}| (DID trimmed) pub fn record_prefix_did(did: &Did) -> Vec { let repo = TrimmedDid::from(did); @@ -209,8 +231,8 @@ pub fn firehose_source_key(url: &str) -> Vec { key } -/// key format: {SEQ} (u64 big-endian), mirroring event_key #[cfg(feature = "relay")] +/// key format: {SEQ} (u64 big-endian), mirroring event_key pub fn relay_event_key(seq: u64) -> [u8; 8] { seq.to_be_bytes() } diff --git a/src/db/migration/mod.rs b/src/db/migration/mod.rs index 0a73648..cbdd43a 100644 --- a/src/db/migration/mod.rs +++ b/src/db/migration/mod.rs @@ -7,6 +7,7 @@ use crate::db::keys::VERSIONING_KEY; mod v1; mod v2; mod v3; +mod v4; type MigrationFn = fn(&Db, &mut OwnedWriteBatch) -> Result<()>; @@ -15,6 +16,7 @@ const MIGRATIONS: &[(&str, MigrationFn)] = &[ ("stable_firehose_cursors", v1::stable_firehose_cursors), ("repo_state_root_commit", v2::repo_state_root_commit), ("firehose_source_is_pds", v3::firehose_source_is_pds), + ("repo_state_active", v4::repo_state_active), ]; fn read_version(db: &Db) -> Result { diff --git a/src/db/migration/v2.rs b/src/db/migration/v2.rs index d9ee23e..4ebb5ae 100644 --- a/src/db/migration/v2.rs +++ b/src/db/migration/v2.rs @@ -9,6 +9,7 @@ use crate::db::{ Db, types::{DbTid, DidKey}, }; +use crate::types::Commit; use crate::types::v2::*; #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src/db/migration/v4.rs b/src/db/migration/v4.rs new file mode 100644 index 0000000..bb28b1d --- /dev/null +++ b/src/db/migration/v4.rs @@ -0,0 +1,89 @@ +use fjall::OwnedWriteBatch; +use miette::{Context, IntoDiagnostic, Result}; + +use crate::db::Db; +use crate::types::v4; + +#[derive(serde::Deserialize)] +#[serde(bound(deserialize = "'i: 'de"))] +pub(crate) struct OldRepoState<'i> { + pub status: crate::types::v2::RepoStatus, + pub root: Option, + pub last_message_time: Option, + pub last_updated_at: i64, + pub tracked: bool, + pub index_id: u64, + #[serde(borrow)] + pub signing_key: Option>, + #[serde(borrow)] + pub pds: Option>, + #[serde(borrow)] + pub handle: Option>, +} + +pub(super) fn repo_state_active(db: &Db, batch: &mut OwnedWriteBatch) -> Result<()> { + for item in db.repos.iter() { + let (k, v) = item.into_inner().into_diagnostic()?; + let old: OldRepoState = rmp_serde::from_slice(&v) + .into_diagnostic() + .wrap_err("invalid repo state")?; + + // derive active from the old status: accounts in any inactive state had active=false; + // everything else (synced, backfilling, error) was active from the upstream's perspective. + let active = !matches!( + old.status, + crate::types::v2::RepoStatus::Deactivated + | crate::types::v2::RepoStatus::Takendown + | crate::types::v2::RepoStatus::Suspended + ); + + let status = match old.status { + crate::types::v2::RepoStatus::Backfilling => v4::RepoStatus::Desynchronized, + crate::types::v2::RepoStatus::Synced => v4::RepoStatus::Synced, + crate::types::v2::RepoStatus::Error(s) => match s.as_str() { + "desynchronized" => v4::RepoStatus::Desynchronized, + "throttled" => v4::RepoStatus::Throttled, + _ => v4::RepoStatus::Error(s), + }, + crate::types::v2::RepoStatus::Deactivated => v4::RepoStatus::Deactivated, + crate::types::v2::RepoStatus::Takendown => v4::RepoStatus::Takendown, + crate::types::v2::RepoStatus::Suspended => v4::RepoStatus::Suspended, + }; + + let new_state = v4::RepoState { + active, + status, + root: old.root, + last_message_time: old.last_message_time, + last_updated_at: old.last_updated_at, + signing_key: old.signing_key, + pds: old.pds, + handle: old.handle, + }; + + let new_metadata = v4::RepoMetadata { + tracked: old.tracked, + index_id: old.index_id, + }; + + batch.insert( + &db.repos, + k.clone(), + rmp_serde::to_vec(&new_state) + .into_diagnostic() + .wrap_err("cant serialize new repo state")?, + ); + + let did = crate::db::types::TrimmedDid::try_from(k.as_ref())?.to_did(); + let metadata_key = crate::db::keys::repo_metadata_key(&did); + batch.insert( + &db.repo_metadata, + metadata_key, + rmp_serde::to_vec(&new_metadata) + .into_diagnostic() + .wrap_err("cant serialize new repo metadata")?, + ); + } + + Ok(()) +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 3859920..73c9558 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,10 +1,10 @@ use crate::config::Compression; use crate::db::compaction::DropPrefixFilterFactory; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] use crate::types::BroadcastEvent; #[cfg(feature = "relay")] use crate::types::RelayBroadcast; -use crate::types::RepoState; +use crate::types::{RepoMetadata, RepoState}; use fjall::config::{BlockSizePolicy, CompressionPolicy, RestartIntervalPolicy}; use fjall::{ @@ -49,15 +49,16 @@ pub struct Db { pub pending: Keyspace, pub resync: Keyspace, pub resync_buffer: Keyspace, + pub repo_metadata: Keyspace, pub events: Keyspace, pub counts: Keyspace, pub filter: Keyspace, pub crawler: Keyspace, #[cfg(feature = "backlinks")] pub backlinks: Keyspace, - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] pub(crate) event_tx: broadcast::Sender, - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] pub next_event_id: Arc, #[cfg(feature = "relay")] pub(crate) relay_events: Keyspace, @@ -297,6 +298,16 @@ impl Db { .data_block_compression_policy(CompressionPolicy::disabled()) .data_block_restart_interval_policy(RestartIntervalPolicy::all(16)), )?; + let repo_metadata = open_ks( + "repo_metadata", + opts() + // point reads for tracking check + .expect_point_read_hits(true) + .max_memtable_size(mb(8)) + .data_block_size_policy(BlockSizePolicy::all(kb(4))) + .data_block_compression_policy(CompressionPolicy::disabled()) + .data_block_restart_interval_policy(RestartIntervalPolicy::all(4)), + )?; let events = open_ks( "events", opts() @@ -404,9 +415,12 @@ impl Db { // when adding new keyspaces, make sure to add them to the /stats endpoint // and also update any relevant /debug/* endpoints - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] let (event_tx, _) = broadcast::channel(10000); + #[cfg(feature = "relay")] + let (relay_broadcast_tx, _) = broadcast::channel(10000); + let this = Self { inner: db, path: cfg.database_path.clone(), @@ -417,26 +431,24 @@ impl Db { pending, resync, resync_buffer, + repo_metadata, events, counts, filter, crawler, #[cfg(feature = "backlinks")] backlinks, - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] event_tx, counts_map: HashMap::new(), - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] next_event_id: Arc::new(AtomicU64::new(0)), #[cfg(feature = "relay")] relay_events, #[cfg(feature = "relay")] next_relay_seq: Arc::new(AtomicU64::new(0)), #[cfg(feature = "relay")] - relay_broadcast_tx: { - let (tx, _) = broadcast::channel(10000); - tx - }, + relay_broadcast_tx, }; migration::run(&this)?; @@ -457,7 +469,7 @@ impl Db { .store(last_relay_seq + 1, std::sync::atomic::Ordering::Relaxed); } - #[cfg(feature = "events")] + #[cfg(feature = "indexer")] { let mut last_id = 0; if let Some(guard) = this.events.iter().next_back() { @@ -572,9 +584,9 @@ impl Db { } pub fn persist(&self) -> Result<()> { - #[cfg(not(feature = "sync_all"))] + #[cfg(not(feature = "__persist_sync_all"))] const MODE: PersistMode = PersistMode::Buffer; - #[cfg(feature = "sync_all")] + #[cfg(feature = "__persist_sync_all")] const MODE: PersistMode = PersistMode::SyncAll; self.inner.persist(MODE).into_diagnostic()?; Ok(()) @@ -594,15 +606,16 @@ impl Db { compact(self.pending.clone()), compact(self.resync.clone()), compact(self.resync_buffer.clone()), + compact(self.repo_metadata.clone()), compact(self.events.clone()), compact(self.counts.clone()), compact(self.filter.clone()), compact(self.crawler.clone()), )?; - #[cfg(feature = "backlinks")] - compact(self.backlinks.clone()).await?; #[cfg(feature = "relay")] compact(self.relay_events.clone()).await?; + #[cfg(feature = "backlinks")] + compact(self.backlinks.clone()).await?; Ok(()) } @@ -762,11 +775,13 @@ impl Db { ) -> crate::types::GaugeState { match repo_state.status { crate::types::RepoStatus::Synced => crate::types::GaugeState::Synced, - crate::types::RepoStatus::Backfilling => crate::types::GaugeState::Pending, crate::types::RepoStatus::Error(_) | crate::types::RepoStatus::Deactivated | crate::types::RepoStatus::Takendown - | crate::types::RepoStatus::Suspended => { + | crate::types::RepoStatus::Suspended + | crate::types::RepoStatus::Deleted + | crate::types::RepoStatus::Desynchronized + | crate::types::RepoStatus::Throttled => { if let Some(resync_bytes) = resync_bytes { if let Ok(crate::types::ResyncState::Error { kind, .. }) = rmp_serde::from_slice::(resync_bytes) @@ -781,24 +796,6 @@ impl Db { } } } - - pub(crate) async fn repo_gauge_state_async( - &self, - repo_state: &RepoState<'_>, - did_key: &[u8], - ) -> crate::types::GaugeState { - let repo_state = repo_state.clone().into_static(); - let did_key = did_key.to_vec(); - - let db_resync = self.resync.clone(); - - tokio::task::spawn_blocking(move || { - let resync_bytes_opt = db_resync.get(&did_key).ok().flatten(); - Self::repo_gauge_state(&repo_state, resync_bytes_opt.as_deref()) - }) - .await - .unwrap_or(crate::types::GaugeState::Resync(None)) - } } pub fn set_firehose_cursor(db: &Db, relay: &Url, cursor: i64) -> Result<()> { @@ -825,6 +822,14 @@ pub async fn get_firehose_cursor(db: &Db, relay: &Url) -> Result> { .transpose() } +pub fn ser_repo_metadata(state: &RepoMetadata) -> Result> { + rmp_serde::to_vec(&state).into_diagnostic() +} + +pub fn deser_repo_metadata(bytes: &[u8]) -> Result { + rmp_serde::from_slice(bytes).into_diagnostic() +} + pub fn ser_repo_state(state: &RepoState) -> Result> { rmp_serde::to_vec(&state).into_diagnostic() } diff --git a/src/filter.rs b/src/filter.rs index b0ef21e..ee3d837 100644 --- a/src/filter.rs +++ b/src/filter.rs @@ -57,10 +57,9 @@ impl FilterConfig { } } -fn nsid_matches(pattern: &str, collection: &str) -> bool { - if let Some(prefix) = pattern.strip_suffix(".*") { - collection == prefix || collection.starts_with(prefix) - } else { - collection == pattern - } +fn nsid_matches(pattern: &str, col: &str) -> bool { + pattern + .strip_suffix(".*") + .map(|prefix| col == prefix || col.starts_with(prefix)) + .unwrap_or_else(|| col == pattern) } diff --git a/src/ingest/firehose.rs b/src/ingest/firehose.rs index 9b57917..f8035d3 100644 --- a/src/ingest/firehose.rs +++ b/src/ingest/firehose.rs @@ -1,4 +1,3 @@ -use crate::db::deser_repo_state; use crate::filter::{FilterHandle, FilterMode}; use crate::ingest::stream::{FirehoseError, FirehoseStream, SubscribeReposMessage, decode_frame}; use crate::ingest::{BufferTx, IngestMessage}; @@ -147,11 +146,15 @@ impl FirehoseIngestor { } trace!(did = %did, "forwarding message to ingest buffer"); - if let Err(e) = self.buffer_tx.send(IngestMessage::Firehose { - relay: self.relay_host.clone(), - is_pds: self.is_pds, - msg: msg.into_static(), - }) { + if let Err(e) = self + .buffer_tx + .send(IngestMessage::Firehose { + url: self.relay_host.clone(), + is_pds: self.is_pds, + msg: msg.into_static(), + }) + .await + { error!(err = %e, "failed to send message to buffer processor"); } } @@ -174,11 +177,16 @@ impl FirehoseIngestor { match filter.mode { FilterMode::Full => Ok(true), FilterMode::Filter => { - let repo_key = crate::db::keys::repo_key(&did); - if let Some(bytes) = state.db.repos.get(&repo_key).into_diagnostic()? { - let repo_state = deser_repo_state(&bytes)?; - - if repo_state.tracked { + let metadata_key = crate::db::keys::repo_metadata_key(&did); + if let Some(bytes) = state + .db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + { + let metadata = crate::db::deser_repo_metadata(bytes.as_ref())?; + + if metadata.tracked { trace!(did = %did, "tracked repo, processing"); return Ok(true); } else { diff --git a/src/ingest/indexer.rs b/src/ingest/indexer.rs new file mode 100644 index 0000000..d36fe80 --- /dev/null +++ b/src/ingest/indexer.rs @@ -0,0 +1,653 @@ +use super::*; +use crate::db::{self, keys, ser_repo_metadata}; +use crate::ingest::stream::{Account, Commit, Identity}; +use crate::ingest::validation; +use crate::ops; +use crate::resolver::{NoSigningKeyError, ResolverError}; +use crate::state::AppState; +use crate::types::{ + AccountEvt, BroadcastEvent, GaugeState, IdentityEvt, RepoMetadata, RepoState, RepoStatus, +}; + +use fjall::OwnedWriteBatch; + +use jacquard_common::IntoStatic; +use jacquard_common::cowstr::ToCowStr; +use jacquard_common::types::did::Did; +use jacquard_repo::error::CommitError; +use miette::{Diagnostic, IntoDiagnostic, Result}; +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +use std::sync::atomic::Ordering::SeqCst; +use thiserror::Error; +use tokio::runtime::Handle as TokioHandle; +use tokio::sync::mpsc; +use tracing::{debug, error, info, warn}; + +#[derive(Debug)] +pub struct IndexerCommitData { + pub commit: stream::Commit<'static>, + /// true if the relay detected a gap (missing seq) before this commit + /// and the indexer should trigger a backfill. + pub chain_break: bool, + /// result of parse_car_bytes, already done by relay so indexer does not re-parse. + pub parsed_blocks: jacquard_repo::car::reader::ParsedCar, +} + +#[derive(Debug)] +pub struct IndexerIdentityData { + pub identity: stream::Identity<'static>, + /// whether the identity actually changed (handle or key). + pub changed: bool, +} + +#[derive(Debug)] +pub struct IndexerAccountData { + pub account: stream::Account<'static>, + /// whether the repo was active prior to this event. + pub was_active: bool, + /// whether any state actually changed (active or status). + pub changed: bool, +} + +#[derive(Debug)] +pub enum IndexerEventData { + Commit(IndexerCommitData), + Identity(IndexerIdentityData), + Account(IndexerAccountData), + Sync(Did<'static>), +} + +#[derive(Debug)] +pub struct IndexerEvent { + pub seq: i64, + pub firehose: Url, + pub data: IndexerEventData, +} + +/// message sent from `relay_worker` to the indexer (`FirehoseWorker`) after +/// validation and repo-state management are done. +#[derive(Debug)] +pub enum IndexerMessage { + /// a firehose event that passed relay-side validation. + Event(Box), + /// a new repo was discovered and needs backfill. + NewRepo(Did<'static>), + /// backfill for this DID has completed; drain the resync buffer. + BackfillFinished(Did<'static>), +} + +pub type IndexerTx = mpsc::Sender; +pub type IndexerRx = mpsc::Receiver; + +#[derive(Debug, Diagnostic, Error)] +enum IngestError { + #[error("{0}")] + Generic(miette::Report), + + #[error(transparent)] + #[diagnostic(transparent)] + Resolver(#[from] ResolverError), + + #[error(transparent)] + #[diagnostic(transparent)] + Commit(#[from] CommitError), + + #[error(transparent)] + #[diagnostic(transparent)] + NoSigningKey(#[from] NoSigningKeyError), +} + +impl From for IngestError { + fn from(report: miette::Report) -> Self { + IngestError::Generic(report) + } +} + +#[derive(Debug)] +enum RepoProcessResult<'s, 'c> { + // message processed successfully, here is the (possibly updated) state + Ok(RepoState<'s>), + // repo was deleted as part of processing + Deleted, + // needs backfill; carries the triggering commit to buffer (None when already in the buffer) + NeedsBackfill(Option<&'c Commit<'c>>), +} + +pub struct FirehoseWorker { + state: Arc, + rx: IndexerRx, + ephemeral: bool, + num_shards: usize, +} + +struct WorkerContext<'a> { + ephemeral: bool, + state: &'a AppState, + batch: OwnedWriteBatch, + added_blocks: &'a mut i64, + records_delta: &'a mut i64, + broadcast_events: &'a mut Vec, +} + +impl FirehoseWorker { + pub fn new(state: Arc, rx: IndexerRx, ephemeral: bool, num_shards: usize) -> Self { + Self { + state, + rx, + ephemeral, + num_shards, + } + } + + // starts the worker threads and the main dispatch loop + // the dispatch loop reads from the firehose channel and + // distributes messages to shards based on the hash of the DID + pub fn run(mut self, handle: TokioHandle) -> Result<()> { + let mut shards = Vec::with_capacity(self.num_shards); + + for i in 0..self.num_shards { + // unbounded here so we dont block other shards potentially + // if one has a small lag or something + let (tx, rx) = mpsc::unbounded_channel(); + shards.push(tx); + + let state = self.state.clone(); + let ephemeral = self.ephemeral; + let handle = handle.clone(); + std::thread::Builder::new() + .name(format!("ingest-shard-{i}")) + .spawn(move || { + Self::shard(i, rx, state, ephemeral, handle); + }) + .into_diagnostic()?; + } + + info!(num = self.num_shards, "started shards"); + + while let Some(msg) = self.rx.blocking_recv() { + let did = match &msg { + IndexerMessage::Event(e) => match &e.data { + IndexerEventData::Commit(m) => &m.commit.repo, + IndexerEventData::Identity(m) => &m.identity.did, + IndexerEventData::Account(m) => &m.account.did, + IndexerEventData::Sync(did) => did, + }, + IndexerMessage::NewRepo(did) => did, + IndexerMessage::BackfillFinished(did) => did, + }; + + let mut hasher = DefaultHasher::new(); + did.hash(&mut hasher); + let hash = hasher.finish(); + let shard_idx = (hash as usize) % self.num_shards; + + if let Err(e) = shards[shard_idx].send(msg) { + error!(shard = shard_idx, err = %e, "failed to send message to shard, shard panicked?"); + break; + } + } + + Err(miette::miette!( + "firehose worker dispatcher shutting down, shard died?" + )) + } + + #[inline(always)] + fn shard( + id: usize, + mut rx: mpsc::UnboundedReceiver, + state: Arc, + ephemeral: bool, + handle: TokioHandle, + ) { + let _guard = handle.enter(); + debug!(shard = id, "shard started"); + + let mut broadcast_events = Vec::new(); + + while let Some(msg) = rx.blocking_recv() { + let batch = state.db.inner.batch(); + broadcast_events.clear(); + + let mut added_blocks = 0; + let mut records_delta = 0; + + let mut ctx = WorkerContext { + state: &state, + batch, + added_blocks: &mut added_blocks, + records_delta: &mut records_delta, + broadcast_events: &mut broadcast_events, + ephemeral, + }; + + match msg { + IndexerMessage::BackfillFinished(did) => { + let _span = tracing::info_span!("ingest", did = %did).entered(); + debug!("backfill finished, verifying state and draining buffer"); + + let repo_key = keys::repo_key(&did); + if let Ok(Some(state_bytes)) = state.db.repos.get(&repo_key).into_diagnostic() { + match crate::db::deser_repo_state(&state_bytes) { + Ok(repo_state) => { + let repo_state = repo_state.into_static(); + + match Self::drain_resync_buffer(&mut ctx, &did, repo_state) { + Ok(RepoProcessResult::Ok(s)) => { + let res = ops::transition_repo( + &mut ctx.batch, + &state.db, + &did, + s, + RepoStatus::Synced, + ); + if let Err(e) = res { + error!(err = %e, "failed to transition to synced"); + } + } + Ok(RepoProcessResult::NeedsBackfill(_)) => {} + Ok(RepoProcessResult::Deleted) => {} + Err(e) => { + error!(err = %e, "failed to drain resync buffer") + } + }; + } + Err(e) => error!(err = %e, "failed to deser repo state"), + } + } + } + IndexerMessage::NewRepo(did) => { + let _span = tracing::info_span!("ingest", did = %did).entered(); + debug!("new repo discovered, triggering backfill"); + + let repo_key = keys::repo_key(&did); + if let Ok(Some(state_bytes)) = state.db.repos.get(&repo_key).into_diagnostic() { + match crate::db::deser_repo_state(&state_bytes) { + Ok(repo_state) => { + if let Err(e) = Self::trigger_backfill(&mut ctx, &did, repo_state) { + error!(err = %e, "failed to trigger backfill for new repo"); + } + } + Err(e) => error!(err = %e, "failed to deser repo state"), + } + } + } + IndexerMessage::Event(e) => { + let IndexerEvent { + seq, + firehose, + data, + } = *e; + let _span = tracing::info_span!("ingest", hose = %firehose, did = tracing::field::Empty).entered(); + + let repo_bytes = { + let did = match &data { + IndexerEventData::Commit(m) => &m.commit.repo, + IndexerEventData::Identity(m) => &m.identity.did, + IndexerEventData::Account(m) => &m.account.did, + IndexerEventData::Sync(did) => did, + }; + _span.record("did", did.as_ref()); + + let repo_key = keys::repo_key(did); + match state.db.repos.get(&repo_key).into_diagnostic() { + Ok(Some(b)) => b, + Ok(None) => { + state + .firehose_cursors + .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); + continue; + } + Err(e) => { + error!(err = %e, "failed to get repo state"); + state + .firehose_cursors + .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); + continue; + } + } + }; + let repo_state = match crate::db::deser_repo_state(&repo_bytes) { + Ok(s) => s, + Err(e) => { + error!(err = %e, "failed to deser repo state"); + state + .firehose_cursors + .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); + continue; + } + }; + + match data { + IndexerEventData::Commit(msg) => { + let IndexerCommitData { + commit, + chain_break, + parsed_blocks, + } = msg; + + let try_persist = |commit: &Commit| { + if let Err(e) = + ops::persist_to_resync_buffer(&state.db, &commit.repo, commit) + { + error!(err = %e, "failed to persist commit to resync_buffer"); + } + }; + + match Self::handle_commit( + &mut ctx, + repo_state, + &commit, + chain_break, + parsed_blocks, + ) { + Ok(RepoProcessResult::Ok(_)) => {} + Ok(RepoProcessResult::Deleted) => { + state.db.update_count("repos", -1); + } + Ok(RepoProcessResult::NeedsBackfill(Some(commit))) => { + try_persist(commit); + } + Ok(RepoProcessResult::NeedsBackfill(None)) => {} + Err(e) => { + if let IngestError::Generic(ref r) = e { + db::check_poisoned_report(r); + } + error!(err = %e, "error processing commit"); + try_persist(&commit); + } + } + } + IndexerEventData::Identity(msg) => { + let IndexerIdentityData { identity, changed } = msg; + + if let Err(e) = + Self::handle_identity(&mut ctx, repo_state, &identity, changed) + { + error!(err = %e, "error processing identity"); + } + } + IndexerEventData::Account(msg) => { + let IndexerAccountData { + account, + was_active, + changed, + } = msg; + + if let Err(e) = Self::handle_account( + &mut ctx, repo_state, changed, &account, was_active, + ) { + error!(err = %e, "error processing account"); + } + } + IndexerEventData::Sync(did) => { + warn!("sync event, triggering backfill"); + if let Err(e) = Self::trigger_backfill(&mut ctx, &did, repo_state) { + error!(err = %e, "failed to trigger backfill on sync"); + } + } + } + state + .firehose_cursors + .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); + } + } + + if let Err(e) = ctx.batch.commit() { + error!(shard = id, err = %e, "failed to commit batch"); + } + + if added_blocks > 0 { + state.db.update_count("blocks", added_blocks); + } + if records_delta != 0 { + state.db.update_count("records", records_delta); + } + for evt in broadcast_events.drain(..) { + let _ = state.db.event_tx.send(evt); + } + + // state.db.inner.persist(fjall::PersistMode::Buffer).ok(); + } + } + + // don't retry commit or sync on key fetch errors + // since we'll just try again later if we get commit or sync again + fn check_if_retriable_failure(e: &IngestError) -> bool { + matches!( + e, + IngestError::Generic(_) + | IngestError::Resolver(ResolverError::Ratelimited) + | IngestError::Resolver(ResolverError::Transport(_)) + ) + } +} + +impl FirehoseWorker { + fn handle_commit<'s, 'c>( + ctx: &mut WorkerContext, + mut repo_state: RepoState<'s>, + commit: &'c Commit<'c>, + chain_break: bool, + parsed_blocks: jacquard_repo::car::reader::ParsedCar, + ) -> Result, IngestError> { + let db = &ctx.state.db; + let did = &commit.repo; + repo_state.advance_message_time(commit.time.0.timestamp_millis()); + + let metadata_key = keys::repo_metadata_key(did); + let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; + let is_backfilling = if let Some(metadata_bytes) = metadata_bytes { + let metadata = crate::db::deser_repo_metadata(metadata_bytes.as_ref())?; + db.pending + .get(keys::pending_key(metadata.index_id)) + .into_diagnostic()? + .is_some() + } else { + false + }; + + if chain_break { + warn!("chain break detected, triggering backfill"); + Self::trigger_backfill(ctx, did, repo_state)?; + return Ok(RepoProcessResult::NeedsBackfill(Some(commit))); + } + + if is_backfilling { + return Ok(RepoProcessResult::NeedsBackfill(Some(commit))); + } + + let root_bytes = parsed_blocks + .blocks + .get(&parsed_blocks.root) + .ok_or_else(|| IngestError::Generic(miette::miette!("root block missing from CAR")))?; + + let commit_obj = jacquard_repo::commit::Commit::from_cbor(root_bytes) + .map_err(|e| IngestError::Generic(miette::miette!("invalid commit object: {e}")))? + .into_static(); + + let validated = validation::ValidatedCommit { + commit, + parsed_blocks, + commit_obj, + chain_break: validation::ChainBreak::default(), // not used by apply_commit + }; + + let res = ops::apply_commit( + &mut ctx.batch, + db, + repo_state, + validated, + &ctx.state.filter.load(), + ctx.ephemeral, + )?; + let repo_state = res.repo_state; + *ctx.added_blocks += res.blocks_count; + *ctx.records_delta += res.records_delta; + ctx.broadcast_events + .push(BroadcastEvent::Persisted(db.next_event_id.load(SeqCst) - 1)); + + Ok(RepoProcessResult::Ok(repo_state)) + } + + fn handle_identity<'s>( + ctx: &mut WorkerContext, + repo_state: RepoState<'s>, + identity: &Identity<'_>, + changed: bool, + ) -> Result, IngestError> { + let db = &ctx.state.db; + let did = &identity.did; + if changed { + let evt = IdentityEvt { + did: did.clone().into_static(), + handle: repo_state.handle.clone().map(IntoStatic::into_static), + }; + ctx.broadcast_events.push(ops::make_identity_event(db, evt)); + } + + Ok(RepoProcessResult::Ok(repo_state)) + } + + fn handle_account<'s, 'c>( + ctx: &mut WorkerContext, + repo_state: RepoState<'s>, + changed: bool, + account: &'c Account<'c>, + was_active: bool, + ) -> Result, IngestError> { + let db = &ctx.state.db; + let did = &account.did; + let is_inactive = !account.active; + let evt = AccountEvt { + did: did.clone().into_static(), + active: account.active, + status: account.status.as_ref().map(|s| s.to_cowstr().into_static()), + }; + + if is_inactive { + use crate::ingest::stream::AccountStatus; + match &account.status { + Some(AccountStatus::Deleted) => { + debug!("account deleted, wiping data"); + crate::ops::delete_repo(&mut ctx.batch, db, did, &repo_state)?; + return Ok(RepoProcessResult::Deleted); + } + _ => { + // status update logic is now handled in RelayWorker; + // FirehoseWorker just needs to update gauges if status changed. + if changed && was_active { + db.update_gauge_diff(&GaugeState::Synced, &GaugeState::Resync(None)); + } + } + } + } else { + // if account became active, update gauges + if !was_active { + db.update_gauge_diff(&GaugeState::Resync(None), &GaugeState::Synced); + } + } + + if changed { + ctx.broadcast_events.push(ops::make_account_event(db, evt)); + } + + Ok(RepoProcessResult::Ok(repo_state)) + } + + fn drain_resync_buffer<'s>( + ctx: &mut WorkerContext, + did: &Did, + mut repo_state: RepoState<'s>, + ) -> Result, IngestError> { + let db = &ctx.state.db; + let prefix = keys::resync_buffer_prefix(did); + + for guard in db.resync_buffer.prefix(&prefix) { + let (key, value) = guard.into_inner().into_diagnostic()?; + let commit: Commit = rmp_serde::from_slice(&value).into_diagnostic()?; + + let parsed_blocks = TokioHandle::current() + .block_on(jacquard_repo::car::reader::parse_car_bytes( + commit.blocks.as_ref(), + )) + .map_err(|e| IngestError::Generic(miette::miette!("malformed CAR: {e}")))?; + + // buffered commits have already been source-checked on arrival; skip host check + let res = Self::handle_commit(ctx, repo_state, &commit, false, parsed_blocks); + let res = match res { + Ok(r) => r, + Err(e) => { + if !Self::check_if_retriable_failure(&e) { + ctx.batch.remove(&db.resync_buffer, key); + } + return Err(e); + } + }; + match res { + RepoProcessResult::Ok(rs) => { + ctx.batch.remove(&db.resync_buffer, key); + repo_state = rs; + } + RepoProcessResult::NeedsBackfill(_) => { + // commit is already in the buffer, leave it there for the next backfill + return Ok(RepoProcessResult::NeedsBackfill(None)); + } + RepoProcessResult::Deleted => { + ctx.batch.remove(&db.resync_buffer, key); + return Ok(RepoProcessResult::Deleted); + } + } + } + + Ok(RepoProcessResult::Ok(repo_state)) + } + + fn trigger_backfill<'s>( + ctx: &mut WorkerContext, + did: &Did, + repo_state: RepoState<'s>, + ) -> Result, IngestError> { + let db = &ctx.state.db; + let mut batch = db.inner.batch(); + let repo_key = keys::repo_key(did); + let meta_key = keys::repo_metadata_key(did); + + let resync_bytes = db.resync.get(&repo_key).into_diagnostic()?; + let old_gauge = crate::db::Db::repo_gauge_state(&repo_state, resync_bytes.as_deref()); + + let existing_metadata = db + .repo_metadata + .get(&meta_key) + .into_diagnostic()? + .map(|b| crate::db::deser_repo_metadata(&b)) + .transpose()?; + let had_metadata = existing_metadata.is_some(); + let mut metadata = existing_metadata.unwrap_or_else(|| RepoMetadata { + index_id: 0, // this is set later + tracked: true, + }); + + let old_pkey = keys::pending_key(metadata.index_id); + let was_pending = had_metadata && db.pending.get(&old_pkey).into_diagnostic()?.is_some(); + // remove old pending entry and insert new one with fresh index_id + if had_metadata { + // only remove if we had one so we dont delete a random entry + batch.remove(&db.pending, old_pkey); + } + + metadata.index_id = rand::random::(); + batch.insert(&db.pending, keys::pending_key(metadata.index_id), &repo_key); + batch.insert(&db.repo_metadata, &meta_key, ser_repo_metadata(&metadata)?); + batch.commit().into_diagnostic()?; + + if !was_pending { + db.update_gauge_diff(&old_gauge, &crate::types::GaugeState::Pending); + ctx.state.notify_backfill(); + } + + Ok(repo_state) + } +} diff --git a/src/ingest/mod.rs b/src/ingest/mod.rs index b073683..2134026 100644 --- a/src/ingest/mod.rs +++ b/src/ingest/mod.rs @@ -1,136 +1,25 @@ use tokio::sync::mpsc; -use tracing::warn; pub mod firehose; -#[cfg(feature = "relay")] -pub mod relay_worker; +#[cfg(feature = "indexer")] +pub mod indexer; +pub mod relay; pub mod stream; pub mod validation; -#[cfg(feature = "events")] -pub mod worker; - -use jacquard_common::types::crypto::PublicKey; -use jacquard_common::types::did::Did; -use miette::Result; -use smol_str::{SmolStr, ToSmolStr}; use url::Url; -use crate::ingest::stream::{AccountStatus, SubscribeReposMessage}; -use crate::resolver::Resolver; -use crate::types::{RepoState, RepoStatus}; +use crate::ingest::stream::SubscribeReposMessage; #[derive(Debug)] pub enum IngestMessage { Firehose { - relay: Url, + url: Url, /// true when `relay` is a direct PDS connection (not an aggregating relay). /// enables host authority enforcement in the worker. is_pds: bool, msg: SubscribeReposMessage<'static>, }, - BackfillFinished(Did<'static>), -} - -pub type BufferTx = mpsc::UnboundedSender; -pub type BufferRx = mpsc::UnboundedReceiver; - -/// outcome of a host authority check. -enum AuthorityOutcome { - /// stored pds matched the source host immediately. - Authorized, - /// pds migrated: doc now points to this host, but our stored state was stale. - WasStale, - /// host did not match even after doc resolution. - WrongHost { expected: SmolStr }, -} - -fn pds_host(pds: Option<&str>) -> Option { - // todo: add faster host parsing since we only need that - pds.and_then(|pds| Url::parse(pds).ok()).map(|u| { - u.host_str() - .map(SmolStr::new) - .expect("that there is host in pds url") - }) -} - -/// invalidates the resolver cache for `did`, fetches a fresh document, and updates `repo_state`. -/// -/// panics if called outside a tokio runtime context. -fn refresh_doc(resolver: &Resolver, did: &Did, repo_state: &mut RepoState) -> Result<()> { - resolver.invalidate_sync(did); - let doc = tokio::runtime::Handle::current() - .block_on(resolver.resolve_doc(did)) - .map_err(|e| miette::miette!("{e}"))?; - repo_state.update_from_doc(doc); - repo_state.touch(); - Ok(()) -} - -/// checks that `source_host` is the authoritative PDS for `did`. -/// -/// updates `repo_state` in place when a doc refresh is performed (i.e. on any outcome other than -/// `Authorized`). callers that persist state (e.g. the indexer worker) should write `repo_state` -/// to their batch after this call when the outcome is not `Authorized`. -/// -/// panics if called outside a tokio runtime context. -fn check_host_authority( - resolver: &Resolver, - did: &Did, - repo_state: &mut RepoState, - source_host: &str, -) -> Result { - let expected = pds_host(repo_state.pds.as_deref()); - if expected.as_deref() == Some(source_host) { - return Ok(AuthorityOutcome::Authorized); - } - - // try again once - refresh_doc(resolver, did, repo_state)?; - let Some(expected) = pds_host(repo_state.pds.as_deref()) else { - miette::bail!("can't get pds host???"); - }; - if expected.as_str() == source_host { - Ok(AuthorityOutcome::WasStale) - } else { - Ok(AuthorityOutcome::WrongHost { expected }) - } } -/// resolves the signing key for `did` if `verify_signatures` is true. -/// -/// panics if called outside a tokio runtime context. -fn fetch_key( - resolver: &Resolver, - verify_signatures: bool, - did: &Did, -) -> Result>> { - if verify_signatures { - let key = tokio::runtime::Handle::current() - .block_on(resolver.resolve_signing_key(did)) - .map_err(|e| miette::miette!("{e}"))?; - Ok(Some(key)) - } else { - Ok(None) - } -} - -/// maps an inactive account status to the corresponding `RepoStatus`. -/// panics on `AccountStatus::Deleted`, caller must handle that -fn inactive_account_repo_status(did: &Did, status: &Option>) -> RepoStatus { - match status { - Some(AccountStatus::Takendown) => RepoStatus::Takendown, - Some(AccountStatus::Suspended) => RepoStatus::Suspended, - Some(AccountStatus::Deactivated) => RepoStatus::Deactivated, - Some(AccountStatus::Throttled) => RepoStatus::Error("throttled".into()), - Some(AccountStatus::Desynchronized) => RepoStatus::Error("desynchronized".into()), - Some(AccountStatus::Other(s)) => { - warn!(did = %did, status = %s, "unknown account status"); - RepoStatus::Error(s.to_smolstr()) - } - Some(AccountStatus::Deleted) => unreachable!("deleted is handled before status mapping"), - None => { - warn!(did = %did, "account inactive but no status provided"); - RepoStatus::Error("unknown".into()) - } - } -} +pub type BufferTx = mpsc::Sender; +pub type BufferRx = mpsc::Receiver; diff --git a/src/ingest/relay.rs b/src/ingest/relay.rs new file mode 100644 index 0000000..d38a0b0 --- /dev/null +++ b/src/ingest/relay.rs @@ -0,0 +1,872 @@ +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; +#[cfg(feature = "relay")] +use std::sync::atomic::Ordering; + +use fjall::OwnedWriteBatch; + +use jacquard_api::com_atproto::sync::get_repo_status::{ + GetRepoStatus, GetRepoStatusError, GetRepoStatusOutputStatus, +}; +use jacquard_common::types::crypto::PublicKey; +use jacquard_common::types::did::Did; +use jacquard_common::xrpc::{XrpcError, XrpcExt}; +use jacquard_common::{CowStr, IntoStatic}; +use miette::{IntoDiagnostic, Result}; +use tokio::runtime::Handle; +use tokio::sync::mpsc; +use tracing::{debug, error, info, info_span, trace, warn}; +use url::Url; + +use crate::db::{self, keys}; +use crate::ingest::stream::AccountStatus; +#[cfg(feature = "relay")] +use crate::ingest::stream::encode_frame; +use crate::ingest::stream::{Account, Commit, Identity, InfoName, SubscribeReposMessage, Sync}; +use crate::ingest::validation::{ + CommitValidationError, SyncValidationError, ValidatedCommit, ValidatedSync, ValidationContext, + ValidationOptions, +}; +use crate::ingest::{BufferRx, IngestMessage}; +use crate::state::AppState; +#[cfg(feature = "relay")] +use crate::types::RelayBroadcast; +use crate::types::{RepoState, RepoStatus}; +use smol_str::{SmolStr, ToSmolStr}; + +struct WorkerContext<'a> { + verify_signatures: bool, + state: &'a AppState, + vctx: ValidationContext<'a>, + batch: OwnedWriteBatch, + #[cfg(feature = "relay")] + pending_broadcasts: Vec, + #[cfg(feature = "indexer")] + pending_hook_messages: Vec, + #[cfg(feature = "indexer")] + hook: crate::ingest::indexer::IndexerTx, + http: reqwest::Client, +} + +struct WorkerMessage { + is_pds: bool, + firehose: Url, + msg: SubscribeReposMessage<'static>, +} + +pub struct RelayWorker { + state: Arc, + rx: BufferRx, + #[cfg(feature = "indexer")] + hook: crate::ingest::indexer::IndexerTx, + verify_signatures: bool, + num_shards: usize, + validation_opts: Arc, + http: reqwest::Client, +} + +impl RelayWorker { + pub fn new( + state: Arc, + rx: BufferRx, + #[cfg(feature = "indexer")] hook: crate::ingest::indexer::IndexerTx, + verify_signatures: bool, + num_shards: usize, + validation_opts: ValidationOptions, + ) -> Self { + Self { + state, + rx, + #[cfg(feature = "indexer")] + hook, + verify_signatures, + num_shards, + validation_opts: Arc::new(validation_opts), + http: reqwest::Client::new(), + } + } + + pub fn run(mut self, handle: Handle) -> Result<()> { + let mut shards = Vec::with_capacity(self.num_shards); + + for i in 0..self.num_shards { + let (tx, rx) = mpsc::unbounded_channel(); + shards.push(tx); + + let state = self.state.clone(); + #[cfg(feature = "indexer")] + let hook = self.hook.clone(); + let verify = self.verify_signatures; + let h = handle.clone(); + let opts = self.validation_opts.clone(); + let http = self.http.clone(); + + std::thread::Builder::new() + .name(format!("relay-shard-{i}")) + .spawn(move || { + Self::shard( + i, + rx, + state, + #[cfg(feature = "indexer")] + hook, + verify, + h, + opts, + http, + ); + }) + .into_diagnostic()?; + } + + info!(num = self.num_shards, "relay worker: started shards"); + + let _g = handle.enter(); + + while let Some(msg) = self.rx.blocking_recv() { + let IngestMessage::Firehose { url, is_pds, msg } = msg; + + // #info only pertains to us, the direct consumer + if let SubscribeReposMessage::Info(inf) = msg { + match inf.name { + InfoName::OutdatedCursor => { + // todo: handle + } + InfoName::Other(name) => { + let message = inf + .message + .unwrap_or_else(|| CowStr::Borrowed("")); + info!(name = %name, "relay sent info: {message}"); + } + } + continue; + } + + let shard_idx = { + let did = match &msg { + SubscribeReposMessage::Commit(c) => &c.repo, + SubscribeReposMessage::Identity(i) => &i.did, + SubscribeReposMessage::Account(a) => &a.did, + SubscribeReposMessage::Sync(s) => &s.did, + _ => continue, + }; + let mut hasher = DefaultHasher::new(); + did.hash(&mut hasher); + let idx = (hasher.finish() as usize) % self.num_shards; + idx + }; + + if let Err(e) = shards[shard_idx].send(WorkerMessage { + firehose: url, + is_pds, + msg, + }) { + error!(shard = shard_idx, err = %e, "relay worker: failed to send to shard"); + break; + } + } + + Err(miette::miette!("relay worker dispatcher shutting down")) + } + + fn shard( + id: usize, + mut rx: mpsc::UnboundedReceiver, + state: Arc, + #[cfg(feature = "indexer")] hook: crate::ingest::indexer::IndexerTx, + verify_signatures: bool, + handle: Handle, + validation_opts: Arc, + http: reqwest::Client, + ) { + let _guard = handle.enter(); + let span = info_span!("worker_shard", shard = id); + let _entered = span.clone().entered(); + debug!("relay shard started"); + + let mut ctx = WorkerContext { + verify_signatures, + state: &state, + vctx: ValidationContext { + opts: &validation_opts, + }, + batch: state.db.inner.batch(), + #[cfg(feature = "relay")] + pending_broadcasts: Vec::with_capacity(2), + #[cfg(feature = "indexer")] + pending_hook_messages: Vec::with_capacity(2), + #[cfg(feature = "indexer")] + hook, + http, + }; + + while let Some(msg) = rx.blocking_recv() { + let (did, seq) = match &msg.msg { + SubscribeReposMessage::Commit(c) => (c.repo.clone(), c.seq), + SubscribeReposMessage::Identity(i) => (i.did.clone(), i.seq), + SubscribeReposMessage::Account(a) => (a.did.clone(), a.seq), + SubscribeReposMessage::Sync(s) => (s.did.clone(), s.seq), + _ => continue, + }; + + let firehose = msg.firehose.clone(); + let _span = info_span!("relay", did = %did, firehose = %firehose, seq = %seq).entered(); + + if let Err(e) = Self::process_message(&mut ctx, msg) { + error!(did = %did, err = %e, "relay shard: error processing message"); + } + + let res = std::mem::replace(&mut ctx.batch, ctx.state.db.inner.batch()).commit(); + if let Err(e) = res { + error!(shard = id, err = %e, "relay shard: failed to commit batch"); + continue; + } + + #[cfg(feature = "relay")] + for broadcast in ctx.pending_broadcasts.drain(..) { + let _ = state.db.relay_broadcast_tx.send(broadcast); + } + #[cfg(feature = "indexer")] + for msg in ctx.pending_hook_messages.drain(..) { + let _ = ctx.hook.blocking_send(msg); + } + + // advance cursor for this firehose only if we are the terminal consumer (relay mode) + // in events mode, FirehoseWorker will advance the cursor after processing + #[cfg(feature = "relay")] + { + ctx.state + .firehose_cursors + .peek_with(&firehose, |_, c| c.store(seq, Ordering::SeqCst)); + } + } + } + + fn process_message(ctx: &mut WorkerContext, msg: WorkerMessage) -> Result<()> { + let Some(mut repo_state) = ctx.load_repo_state(&msg)? else { + return Ok(()); + }; + let did = msg.msg.did().expect("already checked for did"); + + if let Some(host) = msg.firehose.host_str() + && msg.is_pds + { + let outcome = ctx.check_host_authority(did, &mut repo_state, host)?; + if let AuthorityOutcome::WrongHost { expected } = outcome { + warn!(got = host, expected = %expected, "message rejected: wrong host"); + return Ok(()); + } + } + + match msg.msg { + SubscribeReposMessage::Commit(commit) => { + trace!("processing commit"); + Self::handle_commit(ctx, &mut repo_state, &msg.firehose, *commit) + } + SubscribeReposMessage::Sync(sync) => { + debug!("processing sync"); + Self::handle_sync(ctx, &mut repo_state, &msg.firehose, *sync) + } + SubscribeReposMessage::Identity(identity) => { + debug!("processing identity"); + Self::handle_identity(ctx, &mut repo_state, &msg.firehose, *identity, msg.is_pds) + } + SubscribeReposMessage::Account(account) => { + debug!("processing account"); + Self::handle_account(ctx, &mut repo_state, &msg.firehose, *account) + } + _ => Ok(()), + } + } + + fn handle_commit( + ctx: &mut WorkerContext, + repo_state: &mut RepoState, + #[allow(unused_variables)] firehose: &Url, + #[allow(unused_mut)] mut commit: Commit<'static>, + ) -> Result<()> { + if !repo_state.active { + return Ok(()); + } + + repo_state.advance_message_time(commit.time.0.timestamp_millis()); + + let Some(validated) = ctx.validate_commit(repo_state, &commit)? else { + return Ok(()); + }; + let ValidatedCommit { + chain_break, + commit_obj, + parsed_blocks, + .. + } = validated; + + if chain_break.is_broken() { + // chain breaks are not grounds for blocking in relay mode + warn!(broken = ?chain_break, "relay: chain break, forwarding anyway"); + } + + let repo_key = keys::repo_key(&commit.repo); + + #[cfg(feature = "indexer")] + { + ctx.pending_hook_messages + .push(crate::ingest::indexer::IndexerMessage::Event(Box::new( + crate::ingest::indexer::IndexerEvent { + seq: commit.seq, + firehose: firehose.clone(), + data: crate::ingest::indexer::IndexerEventData::Commit( + crate::ingest::indexer::IndexerCommitData { + commit, + chain_break: chain_break.is_broken(), + parsed_blocks, + }, + ), + }, + ))); + } + #[cfg(feature = "relay")] + { + ctx.queue_emit(|seq| { + commit.seq = seq; + encode_frame("#commit", &commit) + })?; + } + + repo_state.root = Some(commit_obj.into()); + repo_state.touch(); + ctx.batch.insert( + &ctx.state.db.repos, + repo_key, + db::ser_repo_state(repo_state)?, + ); + + Ok(()) + } + + fn handle_sync( + ctx: &mut WorkerContext, + repo_state: &mut RepoState, + #[allow(unused_variables)] firehose: &Url, + #[allow(unused_mut)] mut sync: Sync<'static>, + ) -> Result<()> { + if !repo_state.active { + return Ok(()); + } + + repo_state.advance_message_time(sync.time.0.timestamp_millis()); + + let Some(validated) = ctx.validate_sync(repo_state, &sync)? else { + return Ok(()); + }; + + let repo_key = keys::repo_key(&sync.did); + + #[cfg(feature = "indexer")] + { + ctx.pending_hook_messages + .push(crate::ingest::indexer::IndexerMessage::Event(Box::new( + crate::ingest::indexer::IndexerEvent { + seq: sync.seq, + firehose: firehose.clone(), + data: crate::ingest::indexer::IndexerEventData::Sync( + sync.did.into_static(), + ), + }, + ))); + } + #[cfg(feature = "relay")] + { + ctx.queue_emit(|seq| { + sync.seq = seq; + encode_frame("#sync", &sync) + })?; + } + + repo_state.root = Some(validated.commit_obj.into()); + repo_state.touch(); + ctx.batch.insert( + &ctx.state.db.repos, + repo_key, + db::ser_repo_state(repo_state)?, + ); + + Ok(()) + } + + fn handle_identity( + ctx: &mut WorkerContext, + repo_state: &mut RepoState, + #[allow(unused_variables)] firehose: &Url, + mut identity: Identity<'static>, + is_pds: bool, + ) -> Result<()> { + let event_ms = identity.time.0.timestamp_millis(); + if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { + debug!("skipping stale/duplicate identity event"); + return Ok(()); + } + repo_state.advance_message_time(event_ms); + + #[cfg(feature = "indexer")] + let (was_handle, was_signing_key) = ( + repo_state.handle.clone().map(IntoStatic::into_static), + repo_state.signing_key.clone().map(IntoStatic::into_static), + ); + + // refresh did doc if a pds sent this event + // or if there is no handle specified + if is_pds || identity.handle.is_none() { + ctx.state.resolver.invalidate_sync(&identity.did); + let doc = Handle::current().block_on(ctx.state.resolver.resolve_doc(&identity.did)); + match doc { + Ok(doc) => { + repo_state.update_from_doc(doc); + } + Err(err) => { + warn!(err = %err, "couldnt fetch identity"); + } + } + } + + // don't pass handle through if it doesnt match ours for pds events + if is_pds && repo_state.handle != identity.handle { + identity.handle = None; + } + + let repo_key = keys::repo_key(&identity.did); + + #[cfg(feature = "indexer")] + { + let changed = + repo_state.handle != was_handle || repo_state.signing_key != was_signing_key; + ctx.pending_hook_messages + .push(crate::ingest::indexer::IndexerMessage::Event(Box::new( + crate::ingest::indexer::IndexerEvent { + seq: identity.seq, + firehose: firehose.clone(), + data: crate::ingest::indexer::IndexerEventData::Identity( + crate::ingest::indexer::IndexerIdentityData { identity, changed }, + ), + }, + ))); + } + #[cfg(feature = "relay")] + { + ctx.queue_emit(|seq| { + identity.seq = seq; + encode_frame("#identity", &identity) + })?; + } + + ctx.batch.insert( + &ctx.state.db.repos, + repo_key, + db::ser_repo_state(repo_state)?, + ); + + Ok(()) + } + + fn handle_account( + ctx: &mut WorkerContext, + repo_state: &mut RepoState, + #[allow(unused_variables)] firehose: &Url, + #[allow(unused_mut)] mut account: Account<'static>, + ) -> Result<()> { + let event_ms = account.time.0.timestamp_millis(); + if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { + debug!("skipping stale/duplicate account event"); + return Ok(()); + } + + repo_state.advance_message_time(event_ms); + + #[cfg(feature = "indexer")] + let (was_active, was_status) = (repo_state.active, repo_state.status.clone()); + + repo_state.active = account.active; + if !account.active { + use crate::ingest::stream::AccountStatus; + match &account.status { + Some(AccountStatus::Deleted) => { + // keep a Deleted tombstone so any stale commits that arrive later + // (e.g. from the upstream backfill window) are not forwarded. + // per spec: "if any further #commit messages are emitted for the repo, + // all downstream services should ignore the event and not pass it through." + repo_state.status = RepoStatus::Deleted; + } + status => { + repo_state.status = ctx.inactive_account_repo_status(&account.did, status); + } + } + } else { + // active=true: desynchronized/throttled may still carry active=true per spec. + // anything else (including unknown statuses) is treated as synced. + use crate::ingest::stream::AccountStatus; + repo_state.status = match &account.status { + Some(AccountStatus::Desynchronized) => RepoStatus::Desynchronized, + Some(AccountStatus::Throttled) => RepoStatus::Throttled, + _ => RepoStatus::Synced, + }; + } + + let repo_key = keys::repo_key(&account.did); + + #[cfg(feature = "indexer")] + { + let changed = repo_state.active != was_active || repo_state.status != was_status; + ctx.pending_hook_messages + .push(crate::ingest::indexer::IndexerMessage::Event(Box::new( + crate::ingest::indexer::IndexerEvent { + seq: account.seq, + firehose: firehose.clone(), + data: crate::ingest::indexer::IndexerEventData::Account( + crate::ingest::indexer::IndexerAccountData { + account, + was_active, + changed, + }, + ), + }, + ))); + } + #[cfg(feature = "relay")] + { + ctx.queue_emit(|seq| { + account.seq = seq; + encode_frame("#account", &account) + })?; + } + + repo_state.touch(); + ctx.batch.insert( + &ctx.state.db.repos, + repo_key, + db::ser_repo_state(repo_state)?, + ); + + Ok(()) + } +} + +impl WorkerContext<'_> { + fn check_host_authority( + &mut self, + did: &Did, + repo_state: &mut RepoState, + source_host: &str, + ) -> Result { + let expected = pds_host(repo_state.pds.as_deref()); + if expected.as_deref() == Some(source_host) { + return Ok(AuthorityOutcome::Authorized); + } + + // try again once + self.refresh_doc(did, repo_state)?; + let Some(expected) = pds_host(repo_state.pds.as_deref()) else { + miette::bail!("can't get pds host???"); + }; + + if expected.as_str() == source_host { + Ok(AuthorityOutcome::WasStale) + } else { + Ok(AuthorityOutcome::WrongHost { expected }) + } + } + + fn refresh_doc(&mut self, did: &Did, repo_state: &mut RepoState) -> Result<()> { + let db = &self.state.db; + self.state.resolver.invalidate_sync(did); + let doc = Handle::current() + .block_on(self.state.resolver.resolve_doc(did)) + .map_err(|e| miette::miette!("{e}"))?; + repo_state.update_from_doc(doc); + repo_state.touch(); + + self.batch.insert( + &db.repos, + keys::repo_key(did), + db::ser_repo_state(repo_state)?, + ); + Ok(()) + } + + fn validate_commit<'c>( + &mut self, + repo_state: &mut RepoState, + commit: &'c Commit<'c>, + ) -> Result>> { + let did = &commit.repo; + let key = self.fetch_key(did)?; + match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { + Ok(v) => return Ok(Some(v)), + Err(CommitValidationError::StaleRev) => { + trace!("skipping replayed commit"); + return Ok(None); + } + Err(CommitValidationError::SigFailure) => {} + Err(e) => { + warn!(err = %e, "commit rejected"); + return Ok(None); + } + } + + self.refresh_doc(did, repo_state)?; + let key = self.fetch_key(did)?; + match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { + Ok(v) => Ok(Some(v)), + Err(e) => { + warn!(err = %e, "commit rejected after key refresh"); + Ok(None) + } + } + } + + fn validate_sync( + &mut self, + repo_state: &mut RepoState, + sync: &Sync<'_>, + ) -> Result> { + let did = &sync.did; + let key = self.fetch_key(did)?; + match self.vctx.validate_sync(sync, key.as_ref()) { + Ok(v) => return Ok(Some(v)), + Err(SyncValidationError::SigFailure) => {} + Err(e) => { + warn!(err = %e, "sync rejected"); + return Ok(None); + } + } + + self.refresh_doc(did, repo_state)?; + let key = self.fetch_key(did)?; + match self.vctx.validate_sync(sync, key.as_ref()) { + Ok(v) => Ok(Some(v)), + Err(e) => { + warn!(err = %e, "sync rejected after key refresh"); + Ok(None) + } + } + } + + fn fetch_key(&self, did: &Did) -> Result>> { + if self.verify_signatures { + let key = Handle::current() + .block_on(self.state.resolver.resolve_signing_key(did)) + .map_err(|e| miette::miette!("{e}"))?; + Ok(Some(key)) + } else { + Ok(None) + } + } + + /// maps an inactive account status to the corresponding `RepoStatus`. + /// panics on `AccountStatus::Deleted`, caller must handle that + fn inactive_account_repo_status( + &self, + did: &Did, + status: &Option>, + ) -> RepoStatus { + match status { + Some(AccountStatus::Takendown) => RepoStatus::Takendown, + Some(AccountStatus::Suspended) => RepoStatus::Suspended, + Some(AccountStatus::Deactivated) => RepoStatus::Deactivated, + Some(AccountStatus::Throttled) => RepoStatus::Throttled, + Some(AccountStatus::Desynchronized) => RepoStatus::Desynchronized, + Some(AccountStatus::Other(s)) => { + warn!(did = %did, status = %s, "unknown account status"); + RepoStatus::Error(s.to_smolstr()) + } + Some(AccountStatus::Deleted) => { + unreachable!("deleted is handled before status mapping") + } + None => { + warn!(did = %did, "account inactive but no status provided"); + RepoStatus::Error("unknown".into()) + } + } + } + + async fn check_repo_status( + &self, + did: &Did<'_>, + pds: &Url, + ) -> Result>> { + let req = GetRepoStatus::new().did(did.clone().into_static()).build(); + let resp = self + .http + .xrpc(crate::util::url_to_fluent_uri(pds)) + .send(&req) + .await; + + let output = match resp { + Err(_) => return Ok(None), + Ok(r) => match r.into_output() { + Ok(o) => o, + Err(XrpcError::Xrpc(GetRepoStatusError::RepoNotFound(_))) => { + // pds explicitly says it doesn't have this repo + // we shouldnt really get here unless the pds is buggy? + // or somehow the repo gets gon right after we receive the event + let mut repo_state = RepoState::backfilling(); + repo_state.active = false; + repo_state.status = RepoStatus::Error("not_found".into()); + return Ok(Some(repo_state)); + } + Err(_) => return Ok(None), + }, + }; + + let mut repo_state = RepoState::backfilling(); + repo_state.active = output.active; + repo_state.status = match output.status { + Some(GetRepoStatusOutputStatus::Takendown) => RepoStatus::Takendown, + Some(GetRepoStatusOutputStatus::Suspended) => RepoStatus::Suspended, + Some(GetRepoStatusOutputStatus::Deactivated) => RepoStatus::Deactivated, + Some(GetRepoStatusOutputStatus::Deleted) => RepoStatus::Deleted, + Some(GetRepoStatusOutputStatus::Desynchronized) => RepoStatus::Desynchronized, + Some(GetRepoStatusOutputStatus::Throttled) => RepoStatus::Throttled, + Some(GetRepoStatusOutputStatus::Other(s)) => RepoStatus::Error(s.into()), + None => output + .active + .then_some(RepoStatus::Synced) + .unwrap_or_else(|| RepoStatus::Error("unknown".into())), + }; + + Ok(Some(repo_state)) + } + + fn load_repo_state(&mut self, msg: &WorkerMessage) -> Result>> { + let db = &self.state.db; + let did = msg.msg.did().expect("we checked if valid"); + let repo_key = keys::repo_key(did); + let metadata_key = keys::repo_metadata_key(did); + + let metadata = db + .repo_metadata + .get(&metadata_key) + .into_diagnostic()? + .map(|bytes| db::deser_repo_metadata(&bytes)) + .transpose()?; + + if metadata.map_or(false, |m| !m.tracked) { + trace!(did = %did, "ignoring message, repo is explicitly untracked"); + return Ok(None); + } + + let repo_state_opt = db + .repos + .get(&repo_key) + .into_diagnostic()? + .map(|bytes| db::deser_repo_state(bytes.as_ref()).map(|s| s.into_static())) + .transpose()?; + + if let Some(repo_state) = repo_state_opt { + return Ok(Some(repo_state)); + } + + #[cfg(feature = "indexer")] + { + let filter = self.state.filter.load(); + if filter.mode == crate::filter::FilterMode::Filter && !filter.signals.is_empty() { + let commit = match &msg.msg { + SubscribeReposMessage::Commit(c) => c, + _ => return Ok(None), + }; + let touches_signal = commit.ops.iter().any(|op| { + op.path + .split_once('/') + .map(|(col, _)| { + let m = filter.matches_signal(col); + debug!( + did = %did, path = %op.path, col = %col, + signals = ?filter.signals, matched = m, + "signal check" + ); + m + }) + .unwrap_or(false) + }); + if !touches_signal { + trace!(did = %did, "dropping commit, no signal-matching ops"); + return Ok(None); + } + } + } + + debug!(did = %did, "discovered new account from firehose, queueing backfill"); + + // resolve doc to initialize repo state + self.state.resolver.invalidate_sync(did); + let doc = tokio::runtime::Handle::current() + .block_on(self.state.resolver.resolve_doc(did)) + .into_diagnostic()?; + + // if it's a PDS, verify it's the authoritative one + if msg.is_pds { + let pds_host = doc.pds.host_str().map(|h| h.to_string()); + if pds_host.as_deref() != msg.firehose.host_str() { + warn!(did = %did, got = ?pds_host, expected = ?msg.firehose.host_str(), "message rejected: wrong host for new account"); + return Ok(None); + } + } + + // try to get upstream status + let mut repo_state = tokio::runtime::Handle::current() + .block_on(self.check_repo_status(did, &doc.pds)) + .ok() + .flatten() + .unwrap_or_else(RepoState::backfilling); + + repo_state.update_from_doc(doc); + + self.batch.insert( + &db.repos, + &repo_key, + crate::db::ser_repo_state(&repo_state)?, + ); + + #[cfg(feature = "indexer")] + { + self.pending_hook_messages + .push(crate::ingest::indexer::IndexerMessage::NewRepo( + did.clone().into_static(), + )); + } + + db.update_count("repos", 1); + + Ok(Some(repo_state)) + } + + #[cfg(feature = "relay")] + fn queue_emit(&mut self, make_frame: impl FnOnce(i64) -> Result) -> Result<()> { + let db = &self.state.db; + let seq = db.next_relay_seq.fetch_add(1, Ordering::SeqCst); + let frame = make_frame(seq as i64)?; + self.batch + .insert(&db.relay_events, keys::relay_event_key(seq), frame.as_ref()); + self.pending_broadcasts.push(RelayBroadcast::Persisted(seq)); + Ok(()) + } +} + +/// outcome of a host authority check. +enum AuthorityOutcome { + /// stored pds matched the source host immediately. + Authorized, + /// pds migrated: doc now points to this host, but our stored state was stale. + WasStale, + /// host did not match even after doc resolution. + WrongHost { expected: SmolStr }, +} + +fn pds_host(pds: Option<&str>) -> Option { + // todo: add faster host parsing since we only need that + pds.and_then(|pds| Url::parse(pds).ok()).map(|u| { + u.host_str() + .map(SmolStr::new) + .expect("that there is host in pds url") + }) +} diff --git a/src/ingest/relay_worker.rs b/src/ingest/relay_worker.rs deleted file mode 100644 index b803156..0000000 --- a/src/ingest/relay_worker.rs +++ /dev/null @@ -1,529 +0,0 @@ -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; -use std::sync::atomic::Ordering; - -use fjall::OwnedWriteBatch; - -use jacquard_common::types::crypto::PublicKey; -use jacquard_common::types::did::Did; -use jacquard_common::{CowStr, IntoStatic}; -use miette::{IntoDiagnostic, Result}; -use tokio::runtime::Handle; -use tokio::sync::mpsc; -use tracing::{debug, error, info, info_span, trace, warn}; -use url::Url; - -use crate::db::{self, keys}; -use crate::ingest::stream::{ - Account, Commit, Identity, InfoName, SubscribeReposMessage, Sync, encode_frame, -}; -use crate::ingest::validation::{ - CommitValidationError, SyncValidationError, ValidatedCommit, ValidatedSync, ValidationContext, - ValidationOptions, -}; -use crate::ingest::{BufferRx, IngestMessage}; -use crate::state::AppState; -use crate::types::{RelayBroadcast, RepoState, RepoStatus}; - -struct WorkerContext<'a> { - verify_signatures: bool, - state: &'a AppState, - vctx: ValidationContext<'a>, - batch: OwnedWriteBatch, - pending_broadcasts: Vec, -} - -struct WorkerMessage { - is_pds: bool, - firehose: Url, - msg: SubscribeReposMessage<'static>, -} - -pub struct RelayWorker { - state: Arc, - rx: BufferRx, - verify_signatures: bool, - num_shards: usize, - validation_opts: Arc, -} - -impl RelayWorker { - pub fn new( - state: Arc, - rx: BufferRx, - verify_signatures: bool, - num_shards: usize, - validation_opts: ValidationOptions, - ) -> Self { - Self { - state, - rx, - verify_signatures, - num_shards, - validation_opts: Arc::new(validation_opts), - } - } - - pub fn run(mut self, handle: Handle) -> Result<()> { - let mut shards = Vec::with_capacity(self.num_shards); - - for i in 0..self.num_shards { - let (tx, rx) = mpsc::unbounded_channel(); - shards.push(tx); - - let state = self.state.clone(); - let verify = self.verify_signatures; - let h = handle.clone(); - let opts = self.validation_opts.clone(); - - std::thread::Builder::new() - .name(format!("relay-shard-{i}")) - .spawn(move || { - Self::shard(i, rx, state, verify, h, opts); - }) - .into_diagnostic()?; - } - - info!(num = self.num_shards, "relay worker: started shards"); - - let _g = handle.enter(); - - while let Some(msg) = self.rx.blocking_recv() { - let IngestMessage::Firehose { - relay: firehose, - is_pds, - msg, - } = msg - else { - continue; - }; - - // #info only pertains to us, the direct consumer - if let SubscribeReposMessage::Info(inf) = msg { - match inf.name { - InfoName::OutdatedCursor => { - // todo: handle - } - InfoName::Other(name) => { - let message = inf - .message - .unwrap_or_else(|| CowStr::Borrowed("")); - info!(name = %name, "relay sent info: {message}"); - } - } - continue; - } - - let shard_idx = { - let did = match &msg { - SubscribeReposMessage::Commit(c) => &c.repo, - SubscribeReposMessage::Identity(i) => &i.did, - SubscribeReposMessage::Account(a) => &a.did, - SubscribeReposMessage::Sync(s) => &s.did, - _ => continue, - }; - let mut hasher = DefaultHasher::new(); - did.hash(&mut hasher); - let idx = (hasher.finish() as usize) % self.num_shards; - idx - }; - - if let Err(e) = shards[shard_idx].send(WorkerMessage { - firehose, - is_pds, - msg, - }) { - error!(shard = shard_idx, err = %e, "relay worker: failed to send to shard"); - break; - } - } - - Err(miette::miette!("relay worker dispatcher shutting down")) - } - - fn shard( - id: usize, - mut rx: mpsc::UnboundedReceiver, - state: Arc, - verify_signatures: bool, - handle: Handle, - validation_opts: Arc, - ) { - let _guard = handle.enter(); - let span = info_span!("worker_shard", shard = id, did = tracing::field::Empty); - let _entered = span.clone().entered(); - debug!("relay shard started"); - - let mut ctx = WorkerContext { - verify_signatures, - state: &state, - vctx: ValidationContext { - opts: &validation_opts, - }, - batch: state.db.inner.batch(), - pending_broadcasts: Vec::with_capacity(1), - }; - - while let Some(msg) = rx.blocking_recv() { - let (did, seq) = match &msg.msg { - SubscribeReposMessage::Commit(c) => (&c.repo, c.seq), - SubscribeReposMessage::Identity(i) => (&i.did, i.seq), - SubscribeReposMessage::Account(a) => (&a.did, a.seq), - SubscribeReposMessage::Sync(s) => (&s.did, s.seq), - _ => continue, - }; - - span.record("did", &**did); - - let firehose = msg.firehose.clone(); - if let Err(e) = Self::process_message(&mut ctx, msg) { - error!(err = %e, "relay shard: error processing message"); - } - - let res = std::mem::replace(&mut ctx.batch, ctx.state.db.inner.batch()).commit(); - if let Err(e) = res { - error!(shard = id, err = %e, "relay shard: failed to commit batch"); - continue; - } - - for broadcast in ctx.pending_broadcasts.drain(..) { - let _ = state.db.relay_broadcast_tx.send(broadcast); - } - - // advance cursor for this firehose - ctx.state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, Ordering::SeqCst)); - } - } - - fn process_message(ctx: &mut WorkerContext, msg: WorkerMessage) -> Result<()> { - let did = msg - .msg - .did() - .expect("that we checked if we are in valid commit"); - let mut repo_state = ctx.load_repo_state(did)?; - - if let Some(host) = msg.firehose.host_str() - && msg.is_pds - { - let outcome = ctx.check_host_authority(did, &mut repo_state, host)?; - if let super::AuthorityOutcome::WrongHost { expected } = outcome { - warn!(got = host, expected = %expected, "message rejected: wrong host"); - return Ok(()); - } - } - - match msg.msg { - SubscribeReposMessage::Commit(commit) => { - trace!("processing commit"); - Self::handle_commit(ctx, &mut repo_state, *commit) - } - SubscribeReposMessage::Sync(sync) => { - debug!("processing sync"); - Self::handle_sync(ctx, &mut repo_state, *sync) - } - SubscribeReposMessage::Identity(identity) => { - debug!("processing identity"); - Self::handle_identity(ctx, &mut repo_state, *identity, msg.is_pds) - } - SubscribeReposMessage::Account(account) => { - debug!("processing account"); - Self::handle_account(ctx, &mut repo_state, *account) - } - _ => Ok(()), - } - } - - fn handle_commit( - ctx: &mut WorkerContext, - repo_state: &mut RepoState, - mut commit: Commit<'static>, - ) -> Result<()> { - if repo_state.status != RepoStatus::Synced { - return Ok(()); - } - - let Some(validated) = ctx.validate_commit(repo_state, &commit)? else { - return Ok(()); - }; - let ValidatedCommit { - chain_break, - commit_obj, - .. - } = validated; - - if chain_break.is_broken() { - warn!(broken = ?chain_break, "out of sync"); - // todo: we need Desynchronized on RepoStatus (and Throttled) - repo_state.status = RepoStatus::Error("desynchronized".into()); - } - - let repo_key = keys::repo_key(&commit.repo); - ctx.queue_emit(|seq| { - commit.seq = seq; - encode_frame("#commit", &commit) - })?; - - repo_state.root = Some(commit_obj.into()); - repo_state.touch(); - ctx.batch.insert( - &ctx.state.db.repos, - repo_key, - db::ser_repo_state(repo_state)?, - ); - - Ok(()) - } - - fn handle_sync( - ctx: &mut WorkerContext, - repo_state: &mut RepoState, - mut sync: Sync<'static>, - ) -> Result<()> { - if repo_state.status != RepoStatus::Synced { - return Ok(()); - } - - let Some(validated) = ctx.validate_sync(repo_state, &sync)? else { - return Ok(()); - }; - - let repo_key = keys::repo_key(&sync.did); - ctx.queue_emit(|seq| { - sync.seq = seq; - encode_frame("#sync", &sync) - })?; - - repo_state.root = Some(validated.commit_obj.into()); - repo_state.touch(); - ctx.batch.insert( - &ctx.state.db.repos, - repo_key, - db::ser_repo_state(repo_state)?, - ); - - Ok(()) - } - - fn handle_identity( - ctx: &mut WorkerContext, - repo_state: &mut RepoState, - mut identity: Identity<'static>, - is_pds: bool, - ) -> Result<()> { - let event_ms = identity.time.0.timestamp_millis(); - if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { - debug!("skipping stale/duplicate identity event"); - return Ok(()); - } - repo_state.advance_message_time(event_ms); - - // refresh did doc if a pds sent this event - // or if there is no handle specified - if is_pds || identity.handle.is_none() { - ctx.state.resolver.invalidate_sync(&identity.did); - let doc = Handle::current().block_on(ctx.state.resolver.resolve_doc(&identity.did)); - match doc { - Ok(doc) => { - repo_state.update_from_doc(doc); - } - Err(err) => { - warn!(err = %err, "couldnt fetch identity"); - } - } - } - - // don't pass handle through if it doesnt match ours for pds events - if is_pds && repo_state.handle != identity.handle { - identity.handle = None; - } - - let repo_key = keys::repo_key(&identity.did); - ctx.queue_emit(|seq| { - identity.seq = seq; - encode_frame("#identity", &identity) - })?; - - ctx.batch.insert( - &ctx.state.db.repos, - repo_key, - db::ser_repo_state(repo_state)?, - ); - - Ok(()) - } - - fn handle_account( - ctx: &mut WorkerContext, - repo_state: &mut RepoState, - mut account: Account<'static>, - ) -> Result<()> { - let event_ms = account.time.0.timestamp_millis(); - if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { - debug!("skipping stale/duplicate account event"); - return Ok(()); - } - repo_state.advance_message_time(event_ms); - - if !account.active { - use crate::ingest::stream::AccountStatus; - match &account.status { - Some(AccountStatus::Deleted) => { - // todo: dont remove repo state? - // forward the event and remove repo state - let repo_key = keys::repo_key(&account.did); - ctx.queue_emit(|seq| { - account.seq = seq; - encode_frame("#account", &account) - })?; - ctx.batch.remove(&ctx.state.db.repos, repo_key); - return Ok(()); - } - status => { - repo_state.status = super::inactive_account_repo_status(&account.did, status); - } - } - } else { - repo_state.status = RepoStatus::Synced; - } - - let repo_key = keys::repo_key(&account.did); - ctx.queue_emit(|seq| { - account.seq = seq; - encode_frame("#account", &account) - })?; - - repo_state.touch(); - ctx.batch.insert( - &ctx.state.db.repos, - repo_key, - db::ser_repo_state(repo_state)?, - ); - - Ok(()) - } -} - -impl WorkerContext<'_> { - fn check_host_authority( - &mut self, - did: &Did, - repo_state: &mut RepoState, - source_host: &str, - ) -> Result { - let outcome = - super::check_host_authority(&self.state.resolver, did, repo_state, source_host)?; - if !matches!(outcome, super::AuthorityOutcome::Authorized) { - self.batch.insert( - &self.state.db.repos, - keys::repo_key(did), - db::ser_repo_state(repo_state)?, - ); - } - Ok(outcome) - } - - fn refresh_doc(&mut self, did: &Did, repo_state: &mut RepoState) -> Result<()> { - super::refresh_doc(&self.state.resolver, did, repo_state)?; - self.batch.insert( - &self.state.db.repos, - keys::repo_key(did), - db::ser_repo_state(repo_state)?, - ); - Ok(()) - } - - fn validate_commit<'c>( - &mut self, - repo_state: &mut RepoState, - commit: &'c Commit<'c>, - ) -> Result>> { - let did = &commit.repo; - let key = self.fetch_key(did)?; - match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { - Ok(v) => return Ok(Some(v)), - Err(CommitValidationError::StaleRev) => { - trace!("skipping replayed commit"); - return Ok(None); - } - Err(CommitValidationError::SigFailure) => {} - Err(e) => { - warn!(err = %e, "commit rejected"); - return Ok(None); - } - } - - self.refresh_doc(did, repo_state)?; - let key = self.fetch_key(did)?; - match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { - Ok(v) => Ok(Some(v)), - Err(e) => { - warn!(err = %e, "commit rejected after key refresh"); - Ok(None) - } - } - } - - fn validate_sync( - &mut self, - repo_state: &mut RepoState, - sync: &Sync<'_>, - ) -> Result> { - let did = &sync.did; - let key = self.fetch_key(did)?; - match self.vctx.validate_sync(sync, key.as_ref()) { - Ok(v) => return Ok(Some(v)), - Err(SyncValidationError::SigFailure) => {} - Err(e) => { - warn!(err = %e, "sync rejected"); - return Ok(None); - } - } - - self.refresh_doc(did, repo_state)?; - let key = self.fetch_key(did)?; - match self.vctx.validate_sync(sync, key.as_ref()) { - Ok(v) => Ok(Some(v)), - Err(e) => { - warn!(err = %e, "sync rejected after key refresh"); - Ok(None) - } - } - } - - fn fetch_key(&self, did: &Did) -> Result>> { - super::fetch_key(&self.state.resolver, self.verify_signatures, did) - } - - fn load_repo_state(&self, did: &Did) -> Result> { - let key = keys::repo_key(did); - let Some(bytes) = self.state.db.repos.get(&key).into_diagnostic()? else { - return Ok(RepoState { - status: RepoStatus::Synced, - root: None, - last_updated_at: chrono::Utc::now().timestamp(), - index_id: 0, - tracked: true, - handle: None, - pds: None, - signing_key: None, - last_message_time: None, - }); - }; - Ok(db::deser_repo_state(&bytes)?.into_static()) - } - - fn queue_emit(&mut self, make_frame: impl FnOnce(i64) -> Result) -> Result<()> { - let seq = self.state.db.next_relay_seq.fetch_add(1, Ordering::SeqCst); - let frame = make_frame(seq as i64)?; - self.batch.insert( - &self.state.db.relay_events, - keys::relay_event_key(seq), - frame.as_ref(), - ); - self.pending_broadcasts.push(RelayBroadcast::Persisted(seq)); - Ok(()) - } -} diff --git a/src/ingest/stream.rs b/src/ingest/stream.rs index 2f6583c..4793284 100644 --- a/src/ingest/stream.rs +++ b/src/ingest/stream.rs @@ -12,6 +12,7 @@ use jacquard_common::{ }, }; use miette::Diagnostic; +use serde::{Deserialize, Serialize}; use smol_str::format_smolstr; use thiserror::Error; use tokio::net::TcpStream; @@ -388,7 +389,7 @@ pub struct Account<'a> { pub time: Datetime, } -#[derive(serde::Deserialize, serde::Serialize, Debug, Clone, jacquard_derive::IntoStatic)] +#[derive(Deserialize, Serialize, Debug, Clone, jacquard_derive::IntoStatic)] #[serde(rename_all = "camelCase")] pub struct Sync<'a> { #[serde(with = "jacquard_common::serde_bytes_helper")] @@ -515,9 +516,6 @@ impl<'i> SubscribeReposMessage<'i> { } } -use serde::Deserialize; -use serde_ipld_dagcbor::de::Deserializer; - // some relays send `""` for `since` when there is no previous revision instead of null fn deserialize_tid_or_empty<'de, D>(deserializer: D) -> Result, D::Error> where @@ -530,11 +528,11 @@ where tracing::warn!("received since with empty string instead of null"); Ok(None) } - Some(s) => s.parse::().map(Some).map_err(serde::de::Error::custom), + Some(s) => s.parse().map(Some).map_err(serde::de::Error::custom), } } -#[derive(Debug, Deserialize, serde::Serialize)] +#[derive(Debug, Deserialize, Serialize)] struct EventHeader { op: i64, t: Option, @@ -547,7 +545,7 @@ struct ErrorFrame { } pub fn decode_frame<'i>(bytes: &'i [u8]) -> Result, FirehoseError> { - let mut de = Deserializer::from_slice(bytes); + let mut de = serde_ipld_dagcbor::de::Deserializer::from_slice(bytes); let header = EventHeader::deserialize(&mut de)?; match header.op { @@ -579,18 +577,18 @@ pub fn decode_frame<'i>(bytes: &'i [u8]) -> Result, Fi } #[cfg(feature = "relay")] -#[derive(serde::Serialize)] +#[derive(Serialize)] struct EncodeHeader<'a> { op: i64, t: &'a str, } #[cfg(feature = "relay")] -pub fn encode_frame(t: &str, body: &T) -> miette::Result { +pub fn encode_frame(t: &str, msg: &T) -> miette::Result { let mut buf = serde_ipld_dagcbor::to_vec(&EncodeHeader { op: 1, t }) .map_err(|e| miette::miette!("encode_frame header: {e}"))?; buf.extend_from_slice( - &serde_ipld_dagcbor::to_vec(body).map_err(|e| miette::miette!("encode_frame body: {e}"))?, + &serde_ipld_dagcbor::to_vec(msg).map_err(|e| miette::miette!("encode_frame body: {e}"))?, ); Ok(bytes::Bytes::from(buf)) } diff --git a/src/ingest/worker.rs b/src/ingest/worker.rs deleted file mode 100644 index 814630d..0000000 --- a/src/ingest/worker.rs +++ /dev/null @@ -1,962 +0,0 @@ -use super::*; -use crate::db::{self, keys}; -use crate::filter::FilterMode; -use crate::ingest::stream::{Account, Commit, Identity, SubscribeReposMessage, Sync}; -use crate::ingest::validation::{ - CommitValidationError, SyncValidationError, ValidatedCommit, ValidatedSync, ValidationContext, - ValidationOptions, -}; -use crate::ops; -use crate::resolver::{NoSigningKeyError, ResolverError}; -use crate::state::AppState; -use crate::types::{AccountEvt, BroadcastEvent, GaugeState, IdentityEvt, RepoState, RepoStatus}; - -use fjall::OwnedWriteBatch; - -use jacquard_common::IntoStatic; -use jacquard_common::cowstr::ToCowStr; -use jacquard_common::types::did::Did; -use jacquard_repo::error::CommitError; -use miette::{Diagnostic, IntoDiagnostic, Result}; -use rand::Rng; -use std::collections::hash_map::DefaultHasher; -use std::hash::{Hash, Hasher}; -use std::sync::Arc; -use std::sync::atomic::Ordering::SeqCst; -use thiserror::Error; -use tokio::runtime::Handle; -use tokio::sync::mpsc; -use tracing::{debug, error, info, trace, warn}; - -#[derive(Debug, Diagnostic, Error)] -enum IngestError { - #[error("{0}")] - Generic(miette::Report), - - #[error(transparent)] - #[diagnostic(transparent)] - Resolver(#[from] ResolverError), - - #[error(transparent)] - #[diagnostic(transparent)] - Commit(#[from] CommitError), - - #[error(transparent)] - #[diagnostic(transparent)] - NoSigningKey(#[from] NoSigningKeyError), -} - -impl From for IngestError { - fn from(report: miette::Report) -> Self { - IngestError::Generic(report) - } -} - -// gate returned by check_repo_state, tells the shard loop what to do with the message -enum ProcessGate<'s, 'c> { - // did not exist in db, newly queued for backfill, drop - NewRepo, - // explicitly untracked, backfilling, or in error, drop - Drop, - // inactive repo receiving a non-account message, buffer the commit if present, drop otherwise - Buffer(Option<&'c Commit<'c>>), - // ready to process with the latest state - Ready(RepoState<'s>), -} - -// result returned by a message handler after the gate has been resolved -#[derive(Debug)] -enum RepoProcessResult<'s, 'c> { - // message processed successfully, here is the (possibly updated) state - Ok(RepoState<'s>), - // repo was deleted as part of processing - Deleted, - // needs backfill; carries the triggering commit to buffer (None when already in the buffer) - NeedsBackfill(Option<&'c Commit<'c>>), -} - -pub struct FirehoseWorker { - state: Arc, - rx: BufferRx, - verify_signatures: bool, - ephemeral: bool, - num_shards: usize, - validation_opts: Arc, -} - -struct WorkerContext<'a> { - verify_signatures: bool, - ephemeral: bool, - state: &'a AppState, - batch: OwnedWriteBatch, - added_blocks: &'a mut i64, - records_delta: &'a mut i64, - broadcast_events: &'a mut Vec, - vctx: ValidationContext<'a>, -} - -impl FirehoseWorker { - pub fn new( - state: Arc, - rx: BufferRx, - verify_signatures: bool, - ephemeral: bool, - num_shards: usize, - validation_opts: ValidationOptions, - ) -> Self { - Self { - state, - rx, - verify_signatures, - ephemeral, - num_shards, - validation_opts: Arc::new(validation_opts), - } - } - - // starts the worker threads and the main dispatch loop - // the dispatch loop reads from the firehose channel and - // distributes messages to shards based on the hash of the DID - pub fn run(mut self, handle: Handle) -> Result<()> { - let mut shards = Vec::with_capacity(self.num_shards); - - for i in 0..self.num_shards { - // unbounded here so we dont block other shards potentially - // if one has a small lag or something - let (tx, rx) = mpsc::unbounded_channel(); - shards.push(tx); - - let state = self.state.clone(); - let verify = self.verify_signatures; - let ephemeral = self.ephemeral; - let handle = handle.clone(); - let validation_opts = self.validation_opts.clone(); - - std::thread::Builder::new() - .name(format!("ingest-shard-{i}")) - .spawn(move || { - Self::shard(i, rx, state, verify, ephemeral, handle, validation_opts); - }) - .into_diagnostic()?; - } - - info!(num = self.num_shards, "started shards"); - - let _g = handle.enter(); - - // dispatch loop - while let Some(msg) = self.rx.blocking_recv() { - let did = match &msg { - IngestMessage::Firehose { msg: m, .. } => match m { - SubscribeReposMessage::Commit(c) => &c.repo, - SubscribeReposMessage::Identity(i) => &i.did, - SubscribeReposMessage::Account(a) => &a.did, - SubscribeReposMessage::Sync(s) => &s.did, - _ => continue, - }, - IngestMessage::BackfillFinished(did) => did, - }; - - // todo: consider using a different hasher? - let mut hasher = DefaultHasher::new(); - did.hash(&mut hasher); - let hash = hasher.finish(); - let shard_idx = (hash as usize) % self.num_shards; - - if let Err(e) = shards[shard_idx].send(msg) { - error!(shard = shard_idx, err = %e, "failed to send message to shard, shard panicked?"); - break; - } - } - - Err(miette::miette!( - "firehose worker dispatcher shutting down, shard died?" - )) - } - - #[inline(always)] - fn shard( - id: usize, - mut rx: mpsc::UnboundedReceiver, - state: Arc, - verify_signatures: bool, - ephemeral: bool, - handle: Handle, - validation_opts: Arc, - ) { - let _guard = handle.enter(); - debug!(shard = id, "shard started"); - - let mut broadcast_events = Vec::new(); - - while let Some(msg) = rx.blocking_recv() { - let batch = state.db.inner.batch(); - broadcast_events.clear(); - - let mut added_blocks = 0; - let mut records_delta = 0; - - let mut ctx = WorkerContext { - state: &state, - batch, - added_blocks: &mut added_blocks, - records_delta: &mut records_delta, - broadcast_events: &mut broadcast_events, - vctx: ValidationContext { - opts: &validation_opts, - }, - verify_signatures, - ephemeral, - }; - - match msg { - IngestMessage::BackfillFinished(did) => { - debug!(did = %did, "backfill finished, verifying state and draining buffer"); - - let repo_key = keys::repo_key(&did); - if let Ok(Some(state_bytes)) = state.db.repos.get(&repo_key).into_diagnostic() { - match crate::db::deser_repo_state(&state_bytes) { - Ok(repo_state) => { - let repo_state = repo_state.into_static(); - - match Self::drain_resync_buffer(&mut ctx, &did, repo_state) { - Ok(RepoProcessResult::Ok(s)) => { - // TODO: there might be a race condition here where we get a new commit - // while the resync buffer is being drained, we should handle that probably - // but also it should still be fine since we'll sync eventually anyway - let res = ops::update_repo_status( - &mut ctx.batch, - &state.db, - &did, - s, - RepoStatus::Synced, - ); - if let Err(e) = res { - // this can only fail if serde retry fails which would be really weird - error!(did = %did, err = %e, "failed to transition to synced"); - } - } - // we don't have to handle this since drain_resync_buffer doesn't delete - // the commits from the resync buffer so they will get retried later - Ok(RepoProcessResult::NeedsBackfill(_)) => {} - Ok(RepoProcessResult::Deleted) => {} - Err(e) => { - error!(did = %did, err = %e, "failed to drain resync buffer") - } - }; - } - Err(e) => error!(did = %did, err = %e, "failed to deser repo state"), - } - } - } - IngestMessage::Firehose { - relay: firehose, - is_pds, - msg, - } => { - let _span = tracing::info_span!("firehose", relay = %firehose).entered(); - let (did, seq) = match &msg { - SubscribeReposMessage::Commit(c) => (&c.repo, c.seq), - SubscribeReposMessage::Identity(i) => (&i.did, i.seq), - SubscribeReposMessage::Account(a) => (&a.did, a.seq), - SubscribeReposMessage::Sync(s) => (&s.did, s.seq), - _ => continue, - }; - - let gate = match Self::check_repo_state(&mut ctx, did, &msg) { - Ok(g) => g, - Err(e) => { - if let IngestError::Generic(ref r) = e { - db::check_poisoned_report(r); - } - error!(did = %did, err = %e, "error in check_repo_state"); - state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); - continue; - } - }; - - match gate { - ProcessGate::NewRepo | ProcessGate::Drop => {} - ProcessGate::Buffer(commit) => { - if let Some(commit) = commit { - if let Err(e) = - ops::persist_to_resync_buffer(&state.db, did, commit) - { - error!( - did = %did, err = %e, - "failed to persist commit to resync_buffer" - ); - } - } - } - ProcessGate::Ready(mut repo_state) => { - // first validate the pds host - if let Some(host) = firehose.host_str() - && is_pds - { - let authority = match Self::check_host_authority( - &mut ctx, - did, - &mut repo_state, - host, - ) { - Ok(a) => a, - Err(e) => { - error!(did = %did, err = %e, "failed to check host authority"); - state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); - continue; - } - }; - match authority { - AuthorityOutcome::Authorized => {} - AuthorityOutcome::WasStale => { - // pds migrated: our data may be stale, backfill from the new host - warn!(did = %did, source_host = host, "pds migration detected, triggering backfill"); - if let Err(e) = - Self::trigger_backfill(&mut ctx, did, repo_state) - { - error!(did = %did, err = %e, "failed to trigger backfill"); - } else if let SubscribeReposMessage::Commit(commit) = &msg { - if let Err(e) = ops::persist_to_resync_buffer( - &state.db, did, commit, - ) { - error!( - did = %did, err = %e, - "failed to persist commit to resync_buffer" - ); - } - } - state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); - continue; - } - // todo: ideally ban pds - AuthorityOutcome::WrongHost { expected } => { - warn!(did = %did, got = host, expected = %expected, "commit rejected: wrong host"); - state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); - continue; - } - } - } - - let pre_status = repo_state.status.clone(); - - // if it was in deactivated/takendown/suspended state, we can mark it - // as synced because we are receiving an active=true account event now. - // we do this before dispatching so handle_account sees pre_status correctly - if matches!( - pre_status, - RepoStatus::Deactivated - | RepoStatus::Suspended - | RepoStatus::Takendown - ) { - if let SubscribeReposMessage::Account(acc) = &msg { - if acc.active { - match ops::update_repo_status( - &mut ctx.batch, - &ctx.state.db, - did, - repo_state, - RepoStatus::Synced, - ) { - Ok(rs) => { - repo_state = rs; - ctx.state.db.update_gauge_diff( - &GaugeState::Resync(None), - &GaugeState::Synced, - ); - } - Err(e) => { - error!( - did = %did, err = %e, - "failed to transition inactive repo to synced" - ); - state - .firehose_cursors - .peek_with(&firehose, |_, c| { - c.store(seq, SeqCst) - }); - continue; - } - } - } - } - } - - match Self::process_message(&mut ctx, &msg, did, repo_state, pre_status) - { - Ok(RepoProcessResult::Ok(_)) => {} - Ok(RepoProcessResult::Deleted) => { - state.db.update_count("repos", -1); - } - Ok(RepoProcessResult::NeedsBackfill(Some(commit))) => { - if let Err(e) = - ops::persist_to_resync_buffer(&state.db, did, commit) - { - error!( - did = %did, err = %e, - "failed to persist commit to resync_buffer" - ); - } - } - Ok(RepoProcessResult::NeedsBackfill(None)) => {} - Err(e) => { - if let IngestError::Generic(ref r) = e { - db::check_poisoned_report(r); - } - error!(did = %did, err = %e, "error processing message"); - if Self::check_if_retriable_failure(&e) { - if let SubscribeReposMessage::Commit(commit) = &msg { - if let Err(e) = ops::persist_to_resync_buffer( - &state.db, did, commit, - ) { - error!( - did = %did, err = %e, - "failed to persist commit to resync_buffer" - ); - } - } - } - } - } - } - } - - state - .firehose_cursors - .peek_with(&firehose, |_, c| c.store(seq, SeqCst)); - } - } - - if let Err(e) = ctx.batch.commit() { - error!(shard = id, err = %e, "failed to commit batch"); - } - - if added_blocks > 0 { - state.db.update_count("blocks", added_blocks); - } - if records_delta != 0 { - state.db.update_count("records", records_delta); - } - for evt in broadcast_events.drain(..) { - let _ = state.db.event_tx.send(evt); - } - - // state.db.inner.persist(fjall::PersistMode::Buffer).ok(); - } - } - - // don't retry commit or sync on key fetch errors - // since we'll just try again later if we get commit or sync again - fn check_if_retriable_failure(e: &IngestError) -> bool { - matches!( - e, - IngestError::Generic(_) - | IngestError::Resolver(ResolverError::Ratelimited) - | IngestError::Resolver(ResolverError::Transport(_)) - ) - } - - fn process_message<'s, 'c>( - ctx: &mut WorkerContext, - msg: &'c SubscribeReposMessage<'static>, - did: &Did, - repo_state: RepoState<'s>, - pre_status: RepoStatus, - ) -> Result, IngestError> { - match msg { - SubscribeReposMessage::Commit(commit) => { - trace!(did = %did, "processing commit"); - Self::handle_commit(ctx, did, repo_state, commit) - } - SubscribeReposMessage::Sync(sync) => { - debug!(did = %did, "processing sync"); - Self::handle_sync(ctx, did, repo_state, sync) - } - SubscribeReposMessage::Identity(identity) => { - debug!(did = %did, "processing identity"); - Self::handle_identity(ctx, did, repo_state, identity) - } - SubscribeReposMessage::Account(account) => { - debug!(did = %did, "processing account"); - Self::handle_account(ctx, did, repo_state, pre_status, account) - } - _ => { - warn!(did = %did, "unknown message type in buffer"); - Ok(RepoProcessResult::Ok(repo_state)) - } - } - } - - fn handle_commit<'s, 'c>( - ctx: &mut WorkerContext, - did: &Did, - mut repo_state: RepoState<'s>, - commit: &'c Commit<'c>, - ) -> Result, IngestError> { - repo_state.advance_message_time(commit.time.0.timestamp_millis()); - - let Some(validated) = ctx.validate_commit(did, &mut repo_state, commit)? else { - return Ok(RepoProcessResult::Ok(repo_state)); - }; - - if validated.chain_break.is_broken() { - warn!( - did = %did, - broken = ?validated.chain_break, - "chain break detected, triggering backfill" - ); - Self::trigger_backfill(ctx, did, repo_state)?; - // not updating repo state root commit since we are backfilling anyway - return Ok(RepoProcessResult::NeedsBackfill(Some(commit))); - } - - let res = ops::apply_commit( - &mut ctx.batch, - &ctx.state.db, - repo_state, - validated, - &ctx.state.filter.load(), - ctx.ephemeral, - )?; - let repo_state = res.repo_state; - *ctx.added_blocks += res.blocks_count; - *ctx.records_delta += res.records_delta; - ctx.broadcast_events.push(BroadcastEvent::Persisted( - ctx.state.db.next_event_id.load(SeqCst) - 1, - )); - - Ok(RepoProcessResult::Ok(repo_state)) - } - - fn handle_sync<'s, 'c>( - ctx: &mut WorkerContext, - did: &Did, - mut repo_state: RepoState<'s>, - sync: &'c Sync<'c>, - ) -> Result, IngestError> { - repo_state.advance_message_time(sync.time.0.timestamp_millis()); - - let Some(validated) = ctx.validate_sync(did, &mut repo_state, sync)? else { - return Ok(RepoProcessResult::Ok(repo_state)); - }; - - // skip noop syncs (data CID unchanged) - if let Some(current_commit) = &repo_state.root { - if current_commit.data == validated.commit_obj.data { - debug!(did = %did, "skipping noop sync"); - return Ok(RepoProcessResult::Ok(repo_state)); - } - - if validated.commit_obj.rev.as_str() <= current_commit.rev.to_tid().as_str() { - debug!(did = %did, "skipping replayed sync"); - return Ok(RepoProcessResult::Ok(repo_state)); - } - } - // not updating repo state root commit since we are backfilling anyway - - warn!(did = %did, "sync event, triggering backfill"); - let repo_state = Self::trigger_backfill(ctx, did, repo_state)?; - Ok(RepoProcessResult::Ok(repo_state)) - } - - fn handle_identity<'s>( - ctx: &mut WorkerContext, - did: &Did, - mut repo_state: RepoState<'s>, - identity: &Identity<'_>, - ) -> Result, IngestError> { - let event_ms = identity.time.0.timestamp_millis(); - if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { - debug!(did = %did, "skipping stale/duplicate identity event"); - return Ok(RepoProcessResult::Ok(repo_state)); - } - repo_state.advance_message_time(event_ms); - - // todo: make this match relay sync behaviour - let changed = if identity.handle.is_none() { - // no handle sent is basically "invalidate your caches" - ctx.state.resolver.invalidate_sync(did); - let doc = Handle::current().block_on(ctx.state.resolver.resolve_doc(did))?; - repo_state.update_from_doc(doc) - } else { - let old_handle = repo_state.handle.clone(); - repo_state.handle = identity - .handle - .clone() - .map(IntoStatic::into_static) - .or(repo_state.handle); - repo_state.handle != old_handle - }; - - repo_state.touch(); - ctx.batch.insert( - &ctx.state.db.repos, - keys::repo_key(did), - crate::db::ser_repo_state(&repo_state)?, - ); - - if changed { - let evt = IdentityEvt { - did: did.clone().into_static(), - handle: repo_state.handle.clone().map(IntoStatic::into_static), - }; - ctx.broadcast_events - .push(ops::make_identity_event(&ctx.state.db, evt)); - } - - Ok(RepoProcessResult::Ok(repo_state)) - } - - fn handle_account<'s, 'c>( - ctx: &mut WorkerContext, - did: &Did, - mut repo_state: RepoState<'s>, - pre_status: RepoStatus, - account: &'c Account<'c>, - ) -> Result, IngestError> { - let event_ms = account.time.0.timestamp_millis(); - if repo_state.last_message_time.is_some_and(|t| event_ms <= t) { - debug!(did = %did, "skipping stale/duplicate account event"); - return Ok(RepoProcessResult::Ok(repo_state)); - } - repo_state.advance_message_time(event_ms); - - // get active before we do any mutations - let was_inactive = matches!( - pre_status, - RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended - ); - let is_inactive = !account.active; - let evt = AccountEvt { - did: did.clone().into_static(), - active: account.active, - status: account.status.as_ref().map(|s| s.to_cowstr().into_static()), - }; - - ctx.refresh_doc(&mut repo_state, did)?; - - if !account.active { - use crate::ingest::stream::AccountStatus; - match &account.status { - Some(AccountStatus::Deleted) => { - debug!(did = %did, "account deleted, wiping data"); - crate::ops::delete_repo(&mut ctx.batch, &ctx.state.db, did, &repo_state)?; - return Ok(RepoProcessResult::Deleted); - } - status => { - let target_status = inactive_account_repo_status(did, status); - - if repo_state.status == target_status { - debug!(did = %did, ?target_status, "account status unchanged"); - ctx.batch.insert( - &ctx.state.db.repos, - keys::repo_key(did), - crate::db::ser_repo_state(&repo_state)?, - ); - return Ok(RepoProcessResult::Ok(repo_state)); - } - - repo_state = ops::update_repo_status( - &mut ctx.batch, - &ctx.state.db, - did, - repo_state, - target_status, - )?; - ctx.state - .db - .update_gauge_diff(&GaugeState::Synced, &GaugeState::Resync(None)); - } - } - } else { - // active=true: transition to synced is handled in the shard dispatch before calling this - } - - if was_inactive != is_inactive || repo_state.status != pre_status { - ctx.broadcast_events - .push(ops::make_account_event(&ctx.state.db, evt)); - } - - // persist last_message_time for paths that don't go through update_repo_status - // (active=true and already synced). harmless double-write for the status-changed path - ctx.batch.insert( - &ctx.state.db.repos, - keys::repo_key(did), - crate::db::ser_repo_state(&repo_state)?, - ); - - Ok(RepoProcessResult::Ok(repo_state)) - } - - // checks the current state of the repo in the database and returns a gate - // indicating what the shard loop should do with the message. - // if the repo is new, creates initial state and triggers backfill - // for synced repos with buffered commits, drains the buffer first - // so events are applied in order - fn check_repo_state<'s, 'c>( - ctx: &mut WorkerContext, - did: &Did<'_>, - msg: &'c SubscribeReposMessage<'static>, - ) -> Result, IngestError> { - let repo_key = keys::repo_key(&did); - let Some(state_bytes) = ctx.state.db.repos.get(&repo_key).into_diagnostic()? else { - let filter = ctx.state.filter.load(); - - if filter.mode == FilterMode::Filter && !filter.signals.is_empty() { - let commit = match msg { - SubscribeReposMessage::Commit(c) => c, - _ => return Ok(ProcessGate::NewRepo), - }; - let touches_signal = commit.ops.iter().any(|op| { - op.path - .split_once('/') - .map(|(col, _)| { - let m = filter.matches_signal(col); - debug!( - did = %did, path = %op.path, col = %col, signals = ?filter.signals, matched = m, - "signal check" - ); - m - }) - .unwrap_or(false) - }); - if !touches_signal { - trace!(did = %did, "dropping commit, no signal-matching ops"); - return Ok(ProcessGate::NewRepo); - } - } - - debug!(did = %did, "discovered new account from firehose, queueing backfill"); - - let repo_state = RepoState::untracked(rand::rng().next_u64()); - let mut batch = ctx.state.db.inner.batch(); - batch.insert( - &ctx.state.db.repos, - &repo_key, - crate::db::ser_repo_state(&repo_state)?, - ); - batch.insert( - &ctx.state.db.pending, - keys::pending_key(repo_state.index_id), - &repo_key, - ); - batch.commit().into_diagnostic()?; - - ctx.state.db.update_count("repos", 1); - ctx.state - .db - .update_gauge_diff(&GaugeState::Synced, &GaugeState::Pending); - - ctx.state.notify_backfill(); - - return Ok(ProcessGate::NewRepo); - }; - - let repo_state = crate::db::deser_repo_state(&state_bytes)?.into_static(); - - if !repo_state.tracked && repo_state.status != RepoStatus::Backfilling { - trace!(did = %did, "ignoring message, repo is explicitly untracked"); - return Ok(ProcessGate::Drop); - } - - match &repo_state.status { - RepoStatus::Synced => { - // lazy drain: if there are buffered commits, drain them now before - // applying the live message so events are applied in order - if ops::has_buffered_commits(&ctx.state.db, did) { - return match Self::drain_resync_buffer(ctx, did, repo_state)? { - RepoProcessResult::Ok(rs) => Ok(ProcessGate::Ready(rs)), - // gap triggered during drain, so drop the live message - RepoProcessResult::NeedsBackfill(_) => Ok(ProcessGate::Drop), - RepoProcessResult::Deleted => Ok(ProcessGate::Drop), - }; - } - Ok(ProcessGate::Ready(repo_state)) - } - RepoStatus::Backfilling | RepoStatus::Error(_) => { - debug!( - did = %did, status = ?repo_state.status, - "ignoring message, repo is backfilling or in error state" - ); - Ok(ProcessGate::Drop) - } - RepoStatus::Deactivated | RepoStatus::Suspended | RepoStatus::Takendown => { - // account events always pass through because the - // shard dispatch handles the active=true transition - if let SubscribeReposMessage::Account(_) = msg { - return Ok(ProcessGate::Ready(repo_state)); - } - // buffer commits and drop everything else until we get an active=true message - let commit = match msg { - SubscribeReposMessage::Commit(c) => Some(c.as_ref()), - _ => None, - }; - Ok(ProcessGate::Buffer(commit)) - } - } - } - - fn drain_resync_buffer<'s>( - ctx: &mut WorkerContext, - did: &Did, - mut repo_state: RepoState<'s>, - ) -> Result, IngestError> { - let prefix = keys::resync_buffer_prefix(did); - - for guard in ctx.state.db.resync_buffer.prefix(&prefix) { - let (key, value) = guard.into_inner().into_diagnostic()?; - let commit: Commit = rmp_serde::from_slice(&value).into_diagnostic()?; - - // buffered commits have already been source-checked on arrival; skip host check - let res = Self::handle_commit(ctx, did, repo_state, &commit); - let res = match res { - Ok(r) => r, - Err(e) => { - if !Self::check_if_retriable_failure(&e) { - ctx.batch.remove(&ctx.state.db.resync_buffer, key); - } - return Err(e); - } - }; - match res { - RepoProcessResult::Ok(rs) => { - ctx.batch.remove(&ctx.state.db.resync_buffer, key); - repo_state = rs; - } - RepoProcessResult::NeedsBackfill(_) => { - // commit is already in the buffer, leave it there for the next backfill - return Ok(RepoProcessResult::NeedsBackfill(None)); - } - RepoProcessResult::Deleted => { - ctx.batch.remove(&ctx.state.db.resync_buffer, key); - return Ok(RepoProcessResult::Deleted); - } - } - } - - Ok(RepoProcessResult::Ok(repo_state)) - } - - // transitions repo to Backfilling, commits the status change immediately (separate from - // ctx.batch), updates the gauge, and pings the backfill worker. returns the updated state. - fn trigger_backfill<'s>( - ctx: &mut WorkerContext, - did: &Did, - repo_state: RepoState<'s>, - ) -> Result, IngestError> { - let mut batch = ctx.state.db.inner.batch(); - let repo_state = ops::update_repo_status( - &mut batch, - &ctx.state.db, - did, - repo_state, - RepoStatus::Backfilling, - )?; - batch.commit().into_diagnostic()?; - ctx.state - .db - .update_gauge_diff(&GaugeState::Synced, &GaugeState::Pending); - ctx.state.notify_backfill(); - Ok(repo_state) - } - - fn check_host_authority( - ctx: &mut WorkerContext, - did: &Did, - repo_state: &mut RepoState, - source_host: &str, - ) -> Result { - let outcome = - super::check_host_authority(&ctx.state.resolver, did, repo_state, source_host)?; - if !matches!(outcome, AuthorityOutcome::Authorized) { - ctx.batch.insert( - &ctx.state.db.repos, - keys::repo_key(did), - crate::db::ser_repo_state(repo_state)?, - ); - } - Ok(outcome) - } -} - -impl WorkerContext<'_> { - fn refresh_doc(&mut self, repo_state: &mut RepoState, did: &Did) -> Result<(), IngestError> { - super::refresh_doc(&self.state.resolver, did, repo_state)?; - self.batch.insert( - &self.state.db.repos, - keys::repo_key(did), - crate::db::ser_repo_state(repo_state)?, - ); - Ok(()) - } - - fn fetch_key(&self, did: &Did) -> Result>> { - super::fetch_key(&self.state.resolver, self.verify_signatures, did) - } - - fn validate_commit<'s, 'c>( - &mut self, - did: &Did, - repo_state: &mut RepoState<'s>, - commit: &'c Commit<'c>, - ) -> Result>, IngestError> { - let key = self.fetch_key(did)?; - match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { - Ok(v) => return Ok(Some(v)), - Err(CommitValidationError::StaleRev) => { - debug!(did = %did, commit_rev = %commit.rev, "skipping replayed commit"); - return Ok(None); - } - Err(CommitValidationError::SigFailure) => {} - Err(e) => { - warn!(did = %did, err = %e, "commit rejected"); - return Ok(None); - } - } - - self.refresh_doc(repo_state, did)?; - let key = self.fetch_key(did)?; - match self.vctx.validate_commit(commit, repo_state, key.as_ref()) { - Ok(v) => Ok(Some(v)), - Err(e) => { - warn!(did = %did, err = %e, "commit rejected after key refresh"); - Ok(None) - } - } - } - - fn validate_sync<'s>( - &mut self, - did: &Did, - repo_state: &mut RepoState<'s>, - sync: &Sync<'_>, - ) -> Result, IngestError> { - let key = self.fetch_key(did)?; - match self.vctx.validate_sync(sync, key.as_ref()) { - Ok(v) => return Ok(Some(v)), - Err(SyncValidationError::SigFailure) => {} - Err(e) => { - warn!(did = %did, err = %e, "sync rejected"); - return Ok(None); - } - } - - self.refresh_doc(repo_state, did)?; - let key = self.fetch_key(did)?; - match self.vctx.validate_sync(sync, key.as_ref()) { - Ok(v) => Ok(Some(v)), - Err(e) => { - warn!(did = %did, err = %e, "sync rejected after key refresh"); - Ok(None) - } - } - } -} diff --git a/src/lib.rs b/src/lib.rs index b33b5d2..7f119b5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,21 +3,20 @@ pub mod control; pub mod filter; pub mod types; -#[cfg(all(feature = "relay", feature = "events", not(debug_assertions)))] -compile_error!("`relay` and `events` features are mutually exclusive"); - -#[cfg(all(feature = "relay", feature = "backlinks", not(debug_assertions)))] -compile_error!("`relay` and `backlinks` features are mutually exclusive"); +#[cfg(all(feature = "relay", feature = "indexer"))] +compile_error!("can't be relay and indexer at the same time"); +#[cfg(all(feature = "relay", feature = "backlinks"))] +compile_error!("can't index backlinks while running as a relay"); pub(crate) mod api; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] pub(crate) mod backfill; #[cfg(feature = "backlinks")] pub(crate) mod backlinks; pub(crate) mod crawler; pub(crate) mod db; pub(crate) mod ingest; -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] pub(crate) mod ops; pub(crate) mod resolver; pub(crate) mod state; diff --git a/src/ops.rs b/src/ops.rs index 63722f9..638980b 100644 --- a/src/ops.rs +++ b/src/ops.rs @@ -6,13 +6,12 @@ use jacquard_common::CowStr; use jacquard_common::Data; use jacquard_common::types::did::Did; use miette::{Context, IntoDiagnostic, Result}; -use rand::{Rng, rng}; use std::collections::HashMap; use std::sync::atomic::Ordering; use tracing::debug; use crate::db::types::{DbAction, DbRkey, DbTid, TrimmedDid}; -use crate::db::{self, Db, keys, ser_repo_state}; +use crate::db::{self, Db, keys}; use crate::filter::FilterConfig; use crate::ingest::stream::Commit; use crate::ingest::validation::ValidatedCommit; @@ -34,11 +33,6 @@ pub fn persist_to_resync_buffer(db: &Db, did: &Did, commit: &Commit) -> Result<( Ok(()) } -pub fn has_buffered_commits(db: &Db, did: &Did) -> bool { - let prefix = keys::resync_buffer_prefix(did); - db.resync_buffer.prefix(&prefix).next().is_some() -} - // emitting identity is ephemeral // we dont replay these, consumers can just fetch identity themselves if they need it pub fn make_identity_event(db: &Db, evt: IdentityEvt<'static>) -> BroadcastEvent { @@ -69,25 +63,25 @@ pub fn delete_repo( batch: &mut OwnedWriteBatch, db: &Db, did: &Did, - repo_state: &RepoState, + _repo_state: &RepoState, ) -> Result<()> { debug!(did = %did, "deleting repo"); let repo_key = keys::repo_key(did); - let pending_key = keys::pending_key(repo_state.index_id); - - // 1. delete from repos, pending, resync - batch.remove(&db.repos, &repo_key); - match repo_state.status { - RepoStatus::Synced => {} - RepoStatus::Backfilling => { - batch.remove(&db.pending, &pending_key); - } - _ => { - batch.remove(&db.resync, &repo_key); - } + let metadata_key = keys::repo_metadata_key(did); + + let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; + if let Some(metadata_bytes) = metadata_bytes { + let metadata = db::deser_repo_metadata(&metadata_bytes)?; + batch.remove(&db.pending, keys::pending_key(metadata.index_id)); } + // 1. delete from resync, and metadata + // we don't delete from repos, relay uses it as a tombstone + // todo: we should still delete it after some time + batch.remove(&db.resync, &repo_key); + batch.remove(&db.repo_metadata, &metadata_key); + // 2. delete from resync buffer let resync_prefix = keys::resync_buffer_prefix(did); for guard in db.resync_buffer.prefix(&resync_prefix) { @@ -123,7 +117,7 @@ pub fn delete_repo( Ok(()) } -pub fn update_repo_status<'batch, 's>( +pub fn transition_repo<'batch, 's>( batch: &'batch mut OwnedWriteBatch, db: &Db, did: &Did, @@ -133,63 +127,73 @@ pub fn update_repo_status<'batch, 's>( debug!(did = %did, status = ?new_status, "updating repo status"); let repo_key = keys::repo_key(did); - let pending_key = keys::pending_key(repo_state.index_id); - - // manage queues - match &new_status { - RepoStatus::Synced => { - batch.remove(&db.pending, &pending_key); - // we dont have to remove from resync here because it has to transition resync -> pending first - } - RepoStatus::Backfilling => { - // if we are coming from an error state, remove from resync - if !matches!(repo_state.status, RepoStatus::Synced) { + let metadata_key = keys::repo_metadata_key(did); + + let metadata_bytes = db.repo_metadata.get(&metadata_key).into_diagnostic()?; + if let Some(metadata_bytes) = metadata_bytes { + let metadata = db::deser_repo_metadata(&metadata_bytes)?; + let pending_key = keys::pending_key(metadata.index_id); + + // manage queues + match &new_status { + RepoStatus::Synced => { + batch.remove(&db.pending, &pending_key); + // we dont have to remove from resync here because it has to transition resync -> pending first + } + RepoStatus::Error(msg) => { + tracing::warn!("transitioning to error: {msg}"); + batch.remove(&db.pending, &pending_key); + // TODO: we need to make errors have kind instead of "message" in repo status + // and then pass it to resync error kind + let resync_state = crate::types::ResyncState::Error { + kind: crate::types::ResyncErrorKind::Generic, + retry_count: 0, + next_retry: chrono::Utc::now().timestamp(), + }; + batch.insert( + &db.resync, + &repo_key, + rmp_serde::to_vec(&resync_state).into_diagnostic()?, + ); + } + RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended => { + // this shouldnt be needed since a repo wont be in a pending state when it gets to any of these states + // batch.remove(&db.pending, &pending_key); + let resync_state = ResyncState::Gone { + status: new_status.clone(), + }; + batch.insert( + &db.resync, + &repo_key, + rmp_serde::to_vec(&resync_state).into_diagnostic()?, + ); + } + RepoStatus::Deleted => { + // terminal state: remove from queues, no resync entry needed + batch.remove(&db.pending, &pending_key); batch.remove(&db.resync, &repo_key); } - // remove the old entry - batch.remove(&db.pending, &pending_key); - // add as new entry - repo_state.index_id = rng().next_u64(); - batch.insert( - &db.pending, - keys::pending_key(repo_state.index_id), - &repo_key, - ); - } - RepoStatus::Error(_msg) => { - batch.remove(&db.pending, &pending_key); - // TODO: we need to make errors have kind instead of "message" in repo status - // and then pass it to resync error kind - let resync_state = crate::types::ResyncState::Error { - kind: crate::types::ResyncErrorKind::Generic, - retry_count: 0, - next_retry: chrono::Utc::now().timestamp(), - }; - batch.insert( - &db.resync, - &repo_key, - rmp_serde::to_vec(&resync_state).into_diagnostic()?, - ); - } - RepoStatus::Deactivated | RepoStatus::Takendown | RepoStatus::Suspended => { - // this shouldnt be needed since a repo wont be in a pending state when it gets to any of these states - // batch.remove(&db.pending, &pending_key); - let resync_state = ResyncState::Gone { - status: new_status.clone(), - }; - batch.insert( - &db.resync, - &repo_key, - rmp_serde::to_vec(&resync_state).into_diagnostic()?, - ); + RepoStatus::Desynchronized | RepoStatus::Throttled => { + // like an error: remove from pending and schedule a resync attempt + batch.remove(&db.pending, &pending_key); + let resync_state = crate::types::ResyncState::Error { + kind: crate::types::ResyncErrorKind::Generic, + retry_count: 0, + next_retry: chrono::Utc::now().timestamp(), + }; + batch.insert( + &db.resync, + &repo_key, + rmp_serde::to_vec(&resync_state).into_diagnostic()?, + ); + } } } + repo_state.active = matches!(new_status, RepoStatus::Synced | RepoStatus::Error(_)); repo_state.status = new_status; repo_state.touch(); - batch.insert(&db.repos, &repo_key, ser_repo_state(&repo_state)?); - Ok(repo_state) } @@ -215,8 +219,6 @@ pub fn apply_commit<'s>( repo_state.root = Some(validated.commit_obj.into()); repo_state.touch(); - batch.insert(&db.repos, keys::repo_key(did), ser_repo_state(&repo_state)?); - // 2. iterate ops and update records index let mut records_delta = 0; let mut blocks_count = 0; diff --git a/src/types.rs b/src/types.rs index 1079d1c..4a1759d 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,5 +1,6 @@ use std::fmt::{Debug, Display}; +use bytes::Bytes; use jacquard_common::types::cid::IpldCid; use jacquard_common::types::nsid::Nsid; use jacquard_common::types::string::{Did, Rkey}; @@ -16,7 +17,6 @@ use crate::resolver::MiniDoc; pub(crate) mod v2 { use super::*; - // todo: add desynchronized and throttled fields #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub enum RepoStatus { Backfilling, @@ -40,22 +40,72 @@ pub(crate) mod v2 { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(bound(deserialize = "'i: 'de"))] pub(crate) struct RepoState<'i> { - // todo: add active field pub status: RepoStatus, pub root: Option, - // todo: is this actually valid? the spec says this is informal and intermadiate - // services may change it. we should probably document it. if we cant use this - // then how do we dedup account / identity ops? + pub last_message_time: Option, + pub last_updated_at: i64, + pub tracked: bool, + pub index_id: u64, + #[serde(borrow)] + pub signing_key: Option>, + #[serde(borrow)] + pub pds: Option>, + #[serde(borrow)] + pub handle: Option>, + } +} + +pub(crate) mod v4 { + use super::*; + pub(crate) use v2::Commit; + + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] + pub enum RepoStatus { + /// repo is synced to latest commit from what we know of + Synced, + /// some unclassified fatal error + Error(SmolStr), + /// user has temporarily paused their overall account. content should + /// not be displayed or redistributed, but does not need to be deleted + /// from infrastructure. implied time-limited. also the initial state + /// for an account after migrating to another pds instance. + Deactivated, + /// host or service has takendown the account. implied permanent or + /// long-term, though may be reverted. + Takendown, + /// host or service has temporarily paused the account. implied + /// time-limited. + Suspended, + /// user or host has deleted the account, and content should be removed + /// from the network. implied permanent or long-term, though may be + /// reverted (deleted accounts may reactivate on the same or another + /// host). + /// + /// account is deleted; kept as a tombstone so stale commits arriving from the upstream + /// backfill window are not forwarded. active=false per spec. + Deleted, + /// host detected a repo sync problem. active may be true or false per spec; + /// the `active` field on `RepoState` is authoritative. + Desynchronized, + /// resource rate-limit exceeded. active may be true or false per spec; + /// the `active` field on `RepoState` is authoritative. + Throttled, + } + + #[derive(Debug, Clone, Serialize, Deserialize)] + #[serde(bound(deserialize = "'i: 'de"))] + pub(crate) struct RepoState<'i> { + /// whether the upstream considers this account active. + /// services should use the `active` flag to control overall account visibility + pub active: bool, + pub status: RepoStatus, + pub root: Option, /// ms since epoch of the last firehose message we processed for this repo. /// used to deduplicate identity / account events that can arrive from multiple relays at /// different wall-clock times but represent the same underlying PDS event. pub last_message_time: Option, /// this is when we *ingested* any last updates pub last_updated_at: i64, // unix timestamp - /// whether we are ingesting events for this repo - pub tracked: bool, - /// index id in pending keyspace - pub index_id: u64, #[serde(borrow)] pub signing_key: Option>, #[serde(borrow)] @@ -63,22 +113,18 @@ pub(crate) mod v2 { #[serde(borrow)] pub handle: Option>, } -} -pub(crate) use v2::*; -impl Display for RepoStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - RepoStatus::Backfilling => write!(f, "backfilling"), - RepoStatus::Synced => write!(f, "synced"), - RepoStatus::Error(e) => write!(f, "error({e})"), - RepoStatus::Deactivated => write!(f, "deactivated"), - RepoStatus::Takendown => write!(f, "takendown"), - RepoStatus::Suspended => write!(f, "suspended"), - } + #[derive(Debug, Clone, Serialize, Deserialize)] + pub(crate) struct RepoMetadata { + /// whether we are ingesting events for this repo + pub tracked: bool, + /// index id in pending keyspace (if backfilling) + pub index_id: u64, } } +pub(crate) use v4::*; + impl<'c> From> for Commit { fn from(value: AtpCommit<'c>) -> Self { Self { @@ -93,7 +139,7 @@ impl<'c> From> for Commit { impl Commit { pub(crate) fn into_atp_commit<'i>(self, did: Did<'i>) -> Option> { - // from a migration + // version < 0 is a sentinel used in v2 migration for repos with no commit data if self.version < 0 { return None; } @@ -108,26 +154,41 @@ impl Commit { } } -impl<'i> RepoState<'i> { +impl Display for RepoStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RepoStatus::Synced => write!(f, "synced"), + RepoStatus::Error(e) => write!(f, "error({e})"), + RepoStatus::Deactivated => write!(f, "deactivated"), + RepoStatus::Takendown => write!(f, "takendown"), + RepoStatus::Suspended => write!(f, "suspended"), + RepoStatus::Deleted => write!(f, "deleted"), + RepoStatus::Desynchronized => write!(f, "desynchronized"), + RepoStatus::Throttled => write!(f, "throttled"), + } + } +} + +impl RepoMetadata { pub fn backfilling(index_id: u64) -> Self { Self { - status: RepoStatus::Backfilling, - root: None, - last_updated_at: chrono::Utc::now().timestamp(), index_id, tracked: true, - handle: None, - pds: None, - signing_key: None, - last_message_time: None, } } +} - /// backfilling, but not tracked yet - pub fn untracked(index_id: u64) -> Self { +impl<'i> RepoState<'i> { + pub fn backfilling() -> Self { Self { - tracked: false, - ..Self::backfilling(index_id) + active: true, + status: RepoStatus::Desynchronized, + root: None, + last_updated_at: chrono::Utc::now().timestamp(), + handle: None, + pds: None, + signing_key: None, + last_message_time: None, } } @@ -158,11 +219,10 @@ impl<'i> IntoStatic for RepoState<'i> { fn into_static(self) -> Self::Output { RepoState { + active: self.active, status: self.status, root: self.root, last_updated_at: self.last_updated_at, - index_id: self.index_id, - tracked: self.tracked, handle: self.handle.map(IntoStatic::into_static), pds: self.pds.map(IntoStatic::into_static), signing_key: self.signing_key.map(IntoStatic::into_static), @@ -244,7 +304,7 @@ pub struct MarshallableEvt<'i> { pub account: Option>, } -#[cfg(feature = "events")] +#[cfg(feature = "indexer")] #[derive(Clone, Debug)] pub(crate) enum BroadcastEvent { #[allow(dead_code)] @@ -285,8 +345,6 @@ pub struct AccountEvt<'i> { pub status: Option>, } -use bytes::Bytes; - #[derive(Serialize, Deserialize, Clone)] pub(crate) enum StoredData { Nothing, diff --git a/tests/authenticated_stream.nu b/tests/authenticated_stream.nu index c089fef..7373ef7 100644 --- a/tests/authenticated_stream.nu +++ b/tests/authenticated_stream.nu @@ -106,7 +106,7 @@ def run-auth-test [did: string, password: string, pds_url: string, relays: strin $e | select id type | insert value $value }) print $"captured ($events | length) events" - $display_events | table -e | print + $display_events | to text | print # filter live events for the relevant entities let relevant_events = ($events | where { |it| diff --git a/tests/backlinks.nu b/tests/backlinks.nu index 2b295ab..a8f9de5 100644 --- a/tests/backlinks.nu +++ b/tests/backlinks.nu @@ -157,7 +157,7 @@ def check-source-filter [url: string, likes: list] { # verify that reverse=true actually inverts the order using a subject with 2+ backlinks. # returns an error string on failure, or empty string on success. def check-reverse-ordering [url: string, subject: string, expected_count: int] { - print $"checking reverse ordering — subject has ($expected_count) backlinks..." + print $"checking reverse ordering... subject has ($expected_count) backlinks..." print $" subject: ($subject)" let fwd = (http get $"($url)/xrpc/blue.microcosm.links.getBacklinks?subject=($subject | url encode)&limit=50") diff --git a/tests/by_collection.nu b/tests/by_collection.nu index a6228eb..ace8327 100644 --- a/tests/by_collection.nu +++ b/tests/by_collection.nu @@ -57,7 +57,7 @@ def main [] { let filter = (http get $"($url)/filter") print $"filter state: ($filter | to json)" if not ($filter.signals | any { |s| $s == $collection }) { - print $"FAILED: ($collection) not in signals — filter not configured" + print $"FAILED: ($collection) not in signals, filter not configured" try { kill -9 $instance.pid } rm -rf $db_path exit 1 @@ -99,7 +99,7 @@ def main [] { print $"FAILED: ($did) not found in repos API" $all_found = false } else { - print $"ok: ($did) — status: ($repo.status)" + print $"ok: ($did), status: ($repo.status)" } } diff --git a/tests/run_all.nu b/tests/run_all.nu index 3803a60..b318f0a 100644 --- a/tests/run_all.nu +++ b/tests/run_all.nu @@ -37,16 +37,19 @@ def run-test [] { } } -def main [--only: list = []] { +def main [--only: list = [], --skip-creds] { print "building hydrant..." - # build defaults features + # build default features cargo build # build backlinks cargo build --features backlinks print "" # discover all test scripts, excluding infrastructure files - let excluded = ["common", "mock_relay", "run_all"] + mut excluded = ["common", "mock_relay", "run_all"] + if $skip_creds { + $excluded = ($excluded | append ["authenticated_stream", "repo_sync_integrity"]) + } let discovered = ( ls tests/*.nu | get name diff --git a/tests/signal_filter.nu b/tests/signal_filter.nu index f0189e0..26b9cbd 100644 --- a/tests/signal_filter.nu +++ b/tests/signal_filter.nu @@ -15,7 +15,7 @@ def main [] { let url = $"http://localhost:($port)" let db_path = (mktemp -d -t hydrant_signal_test.XXXXXX) - let random_str = (random chars -l 6) + let random_str = ("a" + (random chars -l 5)) let collection = $"systems.hydrant.test.($random_str)" print $"database path: ($db_path)" -- 2.51.2