From 485657997b901f08725385921798b26c253b6e70 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 12:08:23 -0500 Subject: [PATCH] feat: make verbose event logs configurable Signed-off-by: Trezy --- src/admin/settings.rs | 1 + src/lib.rs | 1 + src/lua/atproto_api.rs | 1 + src/lua/db_api.rs | 1 + src/lua/execute.rs | 38 ++++---- src/lua/http_api.rs | 1 + src/lua/xrpc_api.rs | 1 + src/main.rs | 34 ++++++- src/record_handler.rs | 96 +++++-------------- tests/common/app.rs | 1 + tests/lua_atproto_api.rs | 1 + tests/lua_db_api.rs | 1 + .../app/dashboard/settings/general/page.tsx | 50 +++++++++- web/src/lib/api.ts | 4 + 14 files changed, 136 insertions(+), 95 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index f730f7b..c3c1a1b 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -31,6 +31,7 @@ const ENV_FALLBACKS: &[(&str, &str)] = &[ ("logo_uri", "LOGO_URI"), ("tos_uri", "TOS_URI"), ("policy_uri", "POLICY_URI"), + ("verbose_event_logging", "VERBOSE_EVENT_LOGGING"), ]; /// Resolve a setting value: check the DB first, then fall back to env var. diff --git a/src/lib.rs b/src/lib.rs index 3b3a061..39a44ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,6 +80,7 @@ pub struct AppState { pub official_registry_config: RegistryConfig, pub proxy_config: Arc>, pub backfill_events_tx: tokio::sync::broadcast::Sender, + pub verbose_event_logging: std::sync::Arc, } impl axum::extract::FromRef for axum_extra::extract::cookie::Key { diff --git a/src/lua/atproto_api.rs b/src/lua/atproto_api.rs index 40af1b0..aeb047c 100644 --- a/src/lua/atproto_api.rs +++ b/src/lua/atproto_api.rs @@ -584,6 +584,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index e8d95b8..5e7f062 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -792,6 +792,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/execute.rs b/src/lua/execute.rs index bf50cf3..0e0037d 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -4,6 +4,7 @@ use mlua::LuaSerdeExt; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::Instant; use crate::AppState; @@ -913,23 +914,25 @@ pub async fn execute_hook_script(event: &HookEvent<'_>) -> Option { match run_hook_once(event).await { Ok(hook_result) => { - log_event( - &event.state.db, - EventLog { - event_type: "hook.executed".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(event.uri.to_string()), - detail: serde_json::json!({ - "lexicon_id": event.lexicon_id, - "action": event.action, - "collection": event.collection, - "attempts": attempt + 1, - }), - }, - backend, - ) - .await; + if event.state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + &event.state.db, + EventLog { + event_type: "hook.executed".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(event.uri.to_string()), + detail: serde_json::json!({ + "lexicon_id": event.lexicon_id, + "action": event.action, + "collection": event.collection, + "attempts": attempt + 1, + }), + }, + backend, + ) + .await; + } return hook_result; } Err(e) => { @@ -1182,6 +1185,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/http_api.rs b/src/lua/http_api.rs index 12b3716..cce9b54 100644 --- a/src/lua/http_api.rs +++ b/src/lua/http_api.rs @@ -190,6 +190,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 82c6ed7..6a5b930 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -294,6 +294,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/main.rs b/src/main.rs index 9a3bf7c..f6a78b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,8 +28,11 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "happyview=debug,tower_http=debug".parse().unwrap()), + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + "happyview=debug,tower_http=debug,sqlx=warn" + .parse() + .unwrap() + }), ) .init(); @@ -615,6 +618,15 @@ async fn main() { let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let verbose_event_logging = { + let enabled = + happyview::admin::settings::get_setting(&db_pool, "verbose_event_logging", db_backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(enabled)) + }; + let state = AppState { config: config.clone(), http, @@ -636,6 +648,7 @@ async fn main() { official_registry_config, proxy_config, backfill_events_tx, + verbose_event_logging, }; jetstream::spawn(state.clone(), collections_rx); @@ -649,6 +662,23 @@ async fn main() { state.db_backend, )); + { + let db = state.db.clone(); + let flag = state.verbose_event_logging.clone(); + let backend = state.db_backend; + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + let enabled = + happyview::admin::settings::get_setting(&db, "verbose_event_logging", backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + flag.store(enabled, std::sync::atomic::Ordering::Relaxed); + } + }); + } + happyview::admin::backfill::resume_backfill_jobs(&state).await; { diff --git a/src/record_handler.rs b/src/record_handler.rs index e59f75f..34dacee 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::atomic::Ordering; use serde_json::Value; @@ -74,27 +75,7 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; match hook_result { - None => { - // Hook returned nil — skip indexing this record. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - state.db_backend, - ) - .await; - return; - } + None => return, Some(v) => v, } } else { @@ -138,23 +119,6 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { ) .await; - log_event( - db, - EventLog { - event_type: "record.created".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), uri.clone()); } Err(e) => { @@ -198,48 +162,32 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; if hook_result.is_none() { - // Hook returned nil — skip the delete. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - backend, - ) - .await; + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "hook returned nil", + }), + }, + backend, + ) + .await; + } return; } } let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); match sqlx::query(&delete_sql).bind(&uri).execute(db).await { - Ok(_) => { - log_event( - db, - EventLog { - event_type: "record.deleted".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - } + Ok(_) => {} Err(e) => { tracing::warn!(uri = %uri, "failed to delete record: {e}"); log_event( diff --git a/tests/common/app.rs b/tests/common/app.rs index bac7ff8..cd08de7 100644 --- a/tests/common/app.rs +++ b/tests/common/app.rs @@ -169,6 +169,7 @@ impl TestApp { ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let router = server::router(state.clone()).layer(axum::middleware::from_fn( diff --git a/tests/lua_atproto_api.rs b/tests/lua_atproto_api.rs index f0a24c1..396a7ba 100644 --- a/tests/lua_atproto_api.rs +++ b/tests/lua_atproto_api.rs @@ -103,6 +103,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index bbf17f2..d32bb90 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -106,6 +106,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index a6292ef..47ebe58 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -18,6 +18,7 @@ import { SiteHeader } from "@/components/site-header"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; const SETTING_KEYS = [ "app_name", @@ -29,6 +30,7 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", + "verbose_event_logging", ] as const; type FieldKey = (typeof SETTING_KEYS)[number]; @@ -90,6 +92,7 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", + verbose_event_logging: "", }); const [sources, setSources] = useState< Record @@ -103,6 +106,7 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", + verbose_event_logging: "unset", }); const [logoUploaded, setLogoUploaded] = useState(false); const [dbInfo, setDbInfo] = useState(null); @@ -137,6 +141,7 @@ export default function GeneralSettingsPage() { logo_uri: val("logo_uri", ""), tos_uri: val("tos_uri", ""), policy_uri: val("policy_uri", ""), + verbose_event_logging: val("verbose_event_logging", ""), }); setSources({ app_name: src("app_name"), @@ -150,6 +155,7 @@ export default function GeneralSettingsPage() { logo_uri: src("logo_uri"), tos_uri: src("tos_uri"), policy_uri: src("policy_uri"), + verbose_event_logging: src("verbose_event_logging"), }); setLogoUploaded(byKey.has("logo_data")); try { @@ -181,13 +187,14 @@ export default function GeneralSettingsPage() { await upsertSetting(field.key, value); } } - const backfillKeys = [ + const extraKeys = [ "backfill_concurrent_dids_per_pds", "backfill_concurrent_pds", "backfill_concurrent_resolution", "backfill_retention_days", + "verbose_event_logging", ] as const; - for (const key of backfillKeys) { + for (const key of extraKeys) { const value = values[key]; if (value === "") { if (sources[key] === "database") { @@ -442,6 +449,45 @@ export default function GeneralSettingsPage() { ))} +
+

Logging

+

+ Configure event log verbosity. +

+
+ +
+
+
+ + {sources["verbose_event_logging"] === "env" && ( + + from env var + + )} +
+

+ Log every record index, hook execution, and hook skip to the event + log. Generates high write volume and will cause issues if + you're indexing high-traffic collections. Recommended to only use + for debugging. +

+
+ + setValues((v) => ({ + ...v, + verbose_event_logging: checked ? "true" : "", + })) + } + disabled={!canManage} + /> +
+