From 18a01fdf436a71b2f4b6bd73672f2008ae05f880 Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 20 May 2026 19:42:24 -0500 Subject: [PATCH 01/64] feat: add `filter` to `db.query` Signed-off-by: Trezy --- src/lua/db_api.rs | 241 ++++++++++++++++++++++++++++------------------ 1 file changed, 149 insertions(+), 92 deletions(-) diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 1937267..6ab7a2a 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -7,11 +7,114 @@ use std::sync::Arc; use crate::AppState; use crate::db::{DatabaseBackend, adapt_sql, decode_cursor, encode_cursor}; +const MAX_FILTER_DEPTH: u8 = 5; +const ALLOWED_OPS: &[&str] = &["=", "!=", "<", ">", "<=", ">=", "LIKE", "NOT LIKE"]; + +enum FilterNode { + Condition { + field: String, + op: String, + value: String, + }, + Group { + combine: String, + children: Vec, + }, +} + +fn parse_filter_node(table: &mlua::Table, depth: u8) -> LuaResult { + if depth > MAX_FILTER_DEPTH { + return Err(mlua::Error::runtime( + "filter nesting too deep (max 5 levels)", + )); + } + + if let Ok(field) = table.get::("field") { + let valid = field.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); + if !valid || field.is_empty() { + return Err(mlua::Error::runtime(format!( + "invalid filter field '{field}': only alphanumeric characters and underscores are allowed", + ))); + } + + let op: String = table + .get::("op") + .unwrap_or_else(|_| "=".to_string()); + let op_upper = op.to_uppercase(); + if !ALLOWED_OPS.contains(&op_upper.as_str()) { + return Err(mlua::Error::runtime(format!( + "invalid filter op '{op}': must be one of {ALLOWED_OPS:?}", + ))); + } + + let val: mlua::Value = table.get("value")?; + let value = match val { + mlua::Value::String(s) => s.to_str()?.to_string(), + mlua::Value::Integer(n) => n.to_string(), + mlua::Value::Number(n) => n.to_string(), + mlua::Value::Boolean(b) => (if b { "true" } else { "false" }).to_string(), + other => { + return Err(mlua::Error::runtime(format!( + "unsupported filter value type for '{field}': {}", + other.type_name() + ))); + } + }; + + return Ok(FilterNode::Condition { + field, + op: op_upper, + value, + }); + } + + let combine: String = table + .get::("combine") + .unwrap_or_else(|_| "AND".to_string()) + .to_uppercase(); + if combine != "AND" && combine != "OR" { + return Err(mlua::Error::runtime(format!( + "invalid filter combine '{combine}': must be 'AND' or 'OR'", + ))); + } + + let mut children = Vec::new(); + for child in table.sequence_values::() { + children.push(parse_filter_node(&child?, depth + 1)?); + } + + if children.is_empty() { + return Err(mlua::Error::runtime("filter group has no conditions")); + } + + Ok(FilterNode::Group { combine, children }) +} + +fn build_filter_sql(node: &FilterNode, binds: &mut Vec) -> String { + match node { + FilterNode::Condition { field, op, value } => { + binds.push(value.clone()); + format!("json_extract(record, '$.value.{field}') {op} ?") + } + FilterNode::Group { combine, children } => { + let parts: Vec = children + .iter() + .map(|c| build_filter_sql(c, binds)) + .collect(); + if parts.len() == 1 { + parts.into_iter().next().unwrap() + } else { + format!("({})", parts.join(&format!(" {combine} "))) + } + } + } +} + /// Register the `db` table with database query functions. pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { let db_table = lua.create_table()?; - // db.query({ collection, did?, limit?, offset?, cursor?, sort?, sortDirection? }) -> { records, cursor? } + // db.query({ collection, did?, limit?, offset?, cursor?, sort?, sortDirection?, filter? }) -> { records, cursor? } let state_query = state.clone(); let query_fn = lua.create_async_function(move |lua, opts: mlua::Table| { let state = state_query.clone(); @@ -45,6 +148,16 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { } }; + let filter_table: Option = opts.get("filter").ok(); + let mut filter_binds: Vec = Vec::new(); + let filter_clause = if let Some(ref tbl) = filter_table { + let node = parse_filter_node(tbl, 0)?; + let sql = build_filter_sql(&node, &mut filter_binds); + format!(" AND {sql}") + } else { + String::new() + }; + let result_table = lua.create_table()?; if let Some(ref sort_field) = sort { @@ -68,32 +181,20 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { } }; - let rows: Vec<(String, String, String)> = if let Some(ref did) = did { - let sql = adapt_sql( - &format!("SELECT uri, did, record FROM records WHERE collection = ? AND did = ? ORDER BY {order_expr} LIMIT ? OFFSET ?"), - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(did) + let did_clause = if did.is_some() { " AND did = ?" } else { "" }; + let sql = adapt_sql( + &format!("SELECT uri, did, record FROM records WHERE collection = ?{did_clause}{filter_clause} ORDER BY {order_expr} LIMIT ? OFFSET ?"), + backend, + ); + let mut q = sqlx::query_as(&sql).bind(&collection); + if let Some(ref did) = did { q = q.bind(did); } + for val in &filter_binds { q = q.bind(val); } + let rows: Vec<(String, String, String)> = q .bind(limit) .bind(offset) .fetch_all(&state.db) .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - } else { - let sql = adapt_sql( - &format!("SELECT uri, did, record FROM records WHERE collection = ? ORDER BY {order_expr} LIMIT ? OFFSET ?"), - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(limit) - .bind(offset) - .fetch_all(&state.db) - .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - }; + .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))?; let has_next = rows.len() as i64 == limit; @@ -126,76 +227,32 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { type RowType = (String, String, String, String); - let rows_raw: Vec = match (&did, &cursor_parts) { - (Some(did), Some((cursor_ts, cursor_uri))) => { - let sql = adapt_sql( - "SELECT uri, did, record, created_at FROM records \ - WHERE collection = ? AND did = ? AND (created_at < ? OR (created_at = ? AND uri < ?)) \ - ORDER BY created_at DESC, uri DESC \ - LIMIT ?", - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(did) - .bind(cursor_ts) - .bind(cursor_ts) - .bind(cursor_uri) - .bind(limit) - .fetch_all(&state.db) - .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - } - (Some(did), None) => { - let sql = adapt_sql( - "SELECT uri, did, record, created_at FROM records \ - WHERE collection = ? AND did = ? \ - ORDER BY created_at DESC, uri DESC \ - LIMIT ?", - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(did) - .bind(limit) - .fetch_all(&state.db) - .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - } - (None, Some((cursor_ts, cursor_uri))) => { - let sql = adapt_sql( - "SELECT uri, did, record, created_at FROM records \ - WHERE collection = ? AND (created_at < ? OR (created_at = ? AND uri < ?)) \ - ORDER BY created_at DESC, uri DESC \ - LIMIT ?", - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(cursor_ts) - .bind(cursor_ts) - .bind(cursor_uri) - .bind(limit) - .fetch_all(&state.db) - .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - } - (None, None) => { - let sql = adapt_sql( - "SELECT uri, did, record, created_at FROM records \ - WHERE collection = ? \ - ORDER BY created_at DESC, uri DESC \ - LIMIT ?", - backend, - ); - sqlx::query_as(&sql) - .bind(&collection) - .bind(limit) - .fetch_all(&state.db) - .await - .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))? - } + let did_clause = if did.is_some() { " AND did = ?" } else { "" }; + let cursor_clause = if cursor_parts.is_some() { + " AND (created_at < ? OR (created_at = ? AND uri < ?))" + } else { + "" }; + let sql = adapt_sql( + &format!( + "SELECT uri, did, record, created_at FROM records \ + WHERE collection = ?{did_clause}{cursor_clause}{filter_clause} \ + ORDER BY created_at DESC, uri DESC \ + LIMIT ?" + ), + backend, + ); + let mut q = sqlx::query_as::<_, RowType>(&sql).bind(&collection); + if let Some(ref did) = did { q = q.bind(did); } + if let Some((cursor_ts, cursor_uri)) = &cursor_parts { + q = q.bind(cursor_ts).bind(cursor_ts).bind(cursor_uri); + } + for val in &filter_binds { q = q.bind(val); } + let rows_raw: Vec = q + .bind(limit) + .fetch_all(&state.db) + .await + .map_err(|e| mlua::Error::runtime(format!("DB query failed: {e}")))?; let has_next = rows_raw.len() as i64 == limit; -- 2.51.2 From 950594bb5d12eb7fefe3cc21a76ade00c242f675 Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 20 May 2026 20:32:14 -0500 Subject: [PATCH 02/64] feat: add support for nexted syntax in field selectors Signed-off-by: Trezy --- src/db.rs | 53 +++++++++++++++++---- src/lua/db_api.rs | 119 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 144 insertions(+), 28 deletions(-) diff --git a/src/db.rs b/src/db.rs index 19f571a..114ced1 100644 --- a/src/db.rs +++ b/src/db.rs @@ -100,22 +100,39 @@ pub fn adapt_sql(sql: &str, backend: DatabaseBackend) -> String { } /// Convert `json_extract(col, '$.seg1.seg2.leaf')` to Postgres `col::jsonb->'seg1'->'seg2'->>'leaf'`. +/// Handles array indices: `seg[0].leaf` becomes `->seg->0->>'leaf'`. fn adapt_json_extract_to_postgres(sql: &str) -> String { JSON_EXTRACT_RE .replace_all(sql, |caps: ®ex::Captures| { let col = &caps[1]; - let path = &caps[2]; // e.g. "defs.main.type" or "title" + let path = &caps[2]; + + let mut parts: Vec<(String, bool)> = Vec::new(); + for segment in path.split('.') { + let bracket_start = segment.find('[').unwrap_or(segment.len()); + let field_name = &segment[..bracket_start]; + if !field_name.is_empty() { + parts.push((field_name.to_string(), false)); + } + let mut rest = &segment[bracket_start..]; + while rest.starts_with('[') { + if let Some(close) = rest.find(']') { + parts.push((rest[1..close].to_string(), true)); + rest = &rest[close + 1..]; + } else { + break; + } + } + } - let segments: Vec<&str> = path.split('.').collect(); let mut chain = format!("{col}::jsonb"); - - for (i, seg) in segments.iter().enumerate() { - if i == segments.len() - 1 { - // Last segment uses ->> (text extraction) - chain.push_str(&format!("->>'{seg}'")); + let last = parts.len().saturating_sub(1); + for (i, (text, is_index)) in parts.iter().enumerate() { + let arrow = if i == last { "->>" } else { "->" }; + if *is_index { + chain.push_str(&format!("{arrow}{text}")); } else { - // Intermediate segments use -> (JSON traversal) - chain.push_str(&format!("->'{seg}'")); + chain.push_str(&format!("{arrow}'{text}'")); } } @@ -388,6 +405,24 @@ mod tests { ); } + #[test] + fn adapt_sql_postgres_converts_array_index_json_extract() { + let sql = "WHERE json_extract(record, '$.value.websites[0].url') = ?"; + assert_eq!( + adapt_sql(sql, DatabaseBackend::Postgres), + "WHERE record::jsonb->'value'->'websites'->0->>'url' = $1" + ); + } + + #[test] + fn adapt_sql_sqlite_keeps_array_index_json_extract() { + let sql = "WHERE json_extract(record, '$.value.tags[0]') = ?"; + assert_eq!( + adapt_sql(sql, DatabaseBackend::Sqlite), + "WHERE json_extract(record, '$.value.tags[0]') = ?" + ); + } + #[test] fn adapt_sql_multiple_json_expressions() { let sql = diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 6ab7a2a..c51c10c 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -10,6 +10,38 @@ use crate::db::{DatabaseBackend, adapt_sql, decode_cursor, encode_cursor}; const MAX_FILTER_DEPTH: u8 = 5; const ALLOWED_OPS: &[&str] = &["=", "!=", "<", ">", "<=", ">=", "LIKE", "NOT LIKE"]; +fn is_valid_json_field_path(path: &str) -> bool { + if path.is_empty() { + return false; + } + for segment in path.split('.') { + if segment.is_empty() { + return false; + } + let bracket_start = segment.find('[').unwrap_or(segment.len()); + let ident = &segment[..bracket_start]; + if ident.is_empty() || !ident.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return false; + } + let mut rest = &segment[bracket_start..]; + while !rest.is_empty() { + if !rest.starts_with('[') { + return false; + } + let close = match rest.find(']') { + Some(i) => i, + None => return false, + }; + let idx = &rest[1..close]; + if idx.is_empty() || !idx.chars().all(|c| c.is_ascii_digit()) { + return false; + } + rest = &rest[close + 1..]; + } + } + true +} + enum FilterNode { Condition { field: String, @@ -30,10 +62,9 @@ fn parse_filter_node(table: &mlua::Table, depth: u8) -> LuaResult { } if let Ok(field) = table.get::("field") { - let valid = field.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); - if !valid || field.is_empty() { + if !is_valid_json_field_path(&field) { return Err(mlua::Error::runtime(format!( - "invalid filter field '{field}': only alphanumeric characters and underscores are allowed", + "invalid filter field '{field}': use alphanumeric names with optional dot notation and array indices (e.g. 'name', 'author.handle', 'tags[0]')", ))); } @@ -127,14 +158,12 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { let sort_direction: Option = opts.get("sortDirection").ok(); let cursor_str: Option = opts.get("cursor").ok(); - // Validate sort field name to prevent SQL injection - if let Some(ref field) = sort { - let valid = field.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); - if !valid || field.is_empty() { - return Err(mlua::Error::runtime( - "invalid sort field: only alphanumeric characters and underscores are allowed", - )); - } + if let Some(ref field) = sort + && !is_valid_json_field_path(field) + { + return Err(mlua::Error::runtime( + "invalid sort field: use alphanumeric names with optional dot notation and array indices (e.g. 'name', 'author.handle', 'tags[0]')", + )); } let direction = match sort_direction.as_deref() { @@ -175,10 +204,7 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { let order_expr = if top_level_columns.contains(&sort_field.as_str()) { format!("{sort_field} {direction}") } else { - match backend { - DatabaseBackend::Sqlite => format!("json_extract(record, '$.value.{sort_field}') {direction}"), - DatabaseBackend::Postgres => format!("record::jsonb->'value'->>'{sort_field}' {direction}"), - } + format!("json_extract(record, '$.value.{sort_field}') {direction}") }; let did_clause = if did.is_some() { " AND did = ?" } else { "" }; @@ -326,11 +352,9 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { let query: String = opts.get("query")?; let limit: i64 = opts.get::("limit").unwrap_or(10).min(100); - // Validate field name to prevent SQL injection - let valid = field.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'); - if !valid || field.is_empty() { + if !is_valid_json_field_path(&field) { return Err(mlua::Error::runtime( - "invalid search field: only alphanumeric characters and underscores are allowed", + "invalid search field: use alphanumeric names with optional dot notation and array indices (e.g. 'name', 'author.handle', 'tags[0]')", )); } @@ -809,6 +833,63 @@ mod tests { } } + #[test] + fn valid_json_field_paths() { + assert!(super::is_valid_json_field_path("name")); + assert!(super::is_valid_json_field_path("author_name")); + assert!(super::is_valid_json_field_path("author.handle")); + assert!(super::is_valid_json_field_path("tags[0]")); + assert!(super::is_valid_json_field_path("data[0][1]")); + assert!(super::is_valid_json_field_path("author.websites[0].url")); + assert!(super::is_valid_json_field_path("a.b.c.d.e")); + } + + #[test] + fn invalid_json_field_paths() { + assert!(!super::is_valid_json_field_path("")); + assert!(!super::is_valid_json_field_path(".name")); + assert!(!super::is_valid_json_field_path("name.")); + assert!(!super::is_valid_json_field_path("name..foo")); + assert!(!super::is_valid_json_field_path("[0]")); + assert!(!super::is_valid_json_field_path("name[]")); + assert!(!super::is_valid_json_field_path("name[abc]")); + assert!(!super::is_valid_json_field_path("name; DROP TABLE")); + assert!(!super::is_valid_json_field_path("name'OR 1=1")); + assert!(!super::is_valid_json_field_path("na-me")); + } + + #[tokio::test] + async fn query_accepts_nested_sort_field() { + let state = test_state(); + let lua = setup(&state); + let result: Result = lua + .load(r#"return db.query({ collection = "test", sort = "author.handle" })"#) + .eval_async() + .await; + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + !err.contains("invalid sort field"), + "nested sort field should be accepted, got: {err}" + ); + } + + #[tokio::test] + async fn query_accepts_array_index_sort_field() { + let state = test_state(); + let lua = setup(&state); + let result: Result = lua + .load(r#"return db.query({ collection = "test", sort = "tags[0]" })"#) + .eval_async() + .await; + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + !err.contains("invalid sort field"), + "array index sort field should be accepted, got: {err}" + ); + } + #[tokio::test] async fn query_rejects_invalid_sort_field() { let state = test_state(); -- 2.51.2 From 698c099211eb053bf2d4f24cb60c0fed107b3a23 Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 20 May 2026 20:33:10 -0500 Subject: [PATCH 03/64] fix: filter nesting was allowed to 6 levels instead of 5 Signed-off-by: Trezy --- src/lua/db_api.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index c51c10c..5899bbd 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -55,10 +55,10 @@ enum FilterNode { } fn parse_filter_node(table: &mlua::Table, depth: u8) -> LuaResult { - if depth > MAX_FILTER_DEPTH { - return Err(mlua::Error::runtime( - "filter nesting too deep (max 5 levels)", - )); + if depth >= MAX_FILTER_DEPTH { + return Err(mlua::Error::runtime(format!( + "filter nesting too deep (max {MAX_FILTER_DEPTH} levels)", + ))); } if let Ok(field) = table.get::("field") { -- 2.51.2 From a0edded64dd7cf5ef8fe69da7acc0f2040355443 Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 20 May 2026 20:34:20 -0500 Subject: [PATCH 04/64] docs: add `db.query`s new `filter` prop to the docs Signed-off-by: Trezy --- .../docs/api-reference/lua/database-api.md | 94 +++++++++++++++++-- 1 file changed, 84 insertions(+), 10 deletions(-) diff --git a/packages/docs/content/docs/api-reference/lua/database-api.md b/packages/docs/content/docs/api-reference/lua/database-api.md index 64e4b4d..96c78b3 100644 --- a/packages/docs/content/docs/api-reference/lua/database-api.md +++ b/packages/docs/content/docs/api-reference/lua/database-api.md @@ -14,6 +14,7 @@ local result = db.query({ cursor = params.cursor, -- optional: opaque cursor from a previous response sort = "name", -- optional: field to sort by, default "indexed_at" sortDirection = "asc", -- optional: "asc" or "desc", default "desc" + filter = { field = "status", value = "active" }, -- optional: filter conditions }) -- result.records — array of record tables (each includes a "uri" field) @@ -22,7 +23,80 @@ local result = db.query({ The `cursor` is an opaque string returned in a previous response. Pass it through directly — don't parse or modify it. When no `sort` field is specified, `db.query` uses keyset pagination (based on `created_at` and `uri`), which is stable even when records are inserted between pages. When a custom `sort` field is specified, offset-based pagination is used instead. -The `sort` field can be a top-level column (`indexed_at`, `did`, `uri`) or any field inside the record's `value` object (e.g. `name`, `createdAt`). Field names must contain only alphanumeric characters and underscores. +The `sort` field can be a top-level column (`indexed_at`, `did`, `uri`) or any field inside the record (e.g. `name`, `createdAt`). Nested paths are supported with dot notation and array indices (e.g. `author.handle`, `scores[0]`). + +### Filtering + +The `filter` option lets you restrict results by record field values. Field names correspond to the fields defined in your lexicon schema (e.g. `streamer`, `status`, `viewers`). + +**Simple condition** — match a single field (operator defaults to `=`): + +```lua +db.query({ + collection = "xyz.statusphere.status", + filter = { field = "streamer", value = "did:plc:abc" }, +}) +``` + +**With operator** — specify a comparison operator: + +```lua +db.query({ + collection = "xyz.statusphere.status", + filter = { field = "viewers", op = ">", value = 100 }, +}) +``` + +Supported operators: `=`, `!=`, `<`, `>`, `<=`, `>=`, `LIKE`, `NOT LIKE`. + +**Combining conditions** — group multiple conditions with `AND` or `OR`: + +```lua +db.query({ + collection = "xyz.statusphere.status", + filter = { + combine = "AND", + { field = "streamer", value = "did:plc:abc" }, + { field = "viewers", op = ">", value = 50 }, + }, +}) +``` + +When `combine` is omitted it defaults to `"AND"`. + +**Nesting** — groups can contain other groups, up to 5 levels deep: + +```lua +db.query({ + collection = "xyz.statusphere.status", + filter = { + combine = "AND", + { field = "streamer", value = "did:plc:abc" }, + { + combine = "OR", + { field = "status", value = "live" }, + { field = "viewers", op = ">=", value = 100 }, + }, + }, +}) +``` + +This matches records where `streamer` is `did:plc:abc` **and** either `status` is `live` **or** `viewers` is at least 100. + +Field names support dot notation for nested objects and bracket syntax for array indices: + +```lua +-- Nested object field +filter = { field = "author.handle", value = "alice.bsky.social" } + +-- Array index +filter = { field = "tags[0]", value = "gaming" } + +-- Combined +filter = { field = "links[0].url", op = "LIKE", value = "%twitch.tv%" } +``` + +Each path segment must be alphanumeric or underscores. Values can be strings, numbers, or booleans. ## db.get @@ -101,15 +175,15 @@ Write SQL in **SQLite syntax** — HappyView translates it to Postgres at runtim ### Column type mapping -| SQLite type | Postgres type | Lua type | -| ---------------------- | ---------------------- | -------- | -| `TEXT` | `TEXT`, `VARCHAR` | string | -| `INTEGER` | `INT4`, `INT8` | integer | -| `REAL` | `FLOAT4`, `FLOAT8` | number | -| `INTEGER` (0/1) | `BOOL` | boolean | -| `TEXT` (JSON) | `JSON`, `JSONB` | table | -| `TEXT` (ISO 8601) | `TIMESTAMPTZ` | string (ISO 8601) | -| Other | Other | string (fallback) | +| SQLite type | Postgres type | Lua type | +| ----------------- | ------------------ | ----------------- | +| `TEXT` | `TEXT`, `VARCHAR` | string | +| `INTEGER` | `INT4`, `INT8` | integer | +| `REAL` | `FLOAT4`, `FLOAT8` | number | +| `INTEGER` (0/1) | `BOOL` | boolean | +| `TEXT` (JSON) | `JSON`, `JSONB` | table | +| `TEXT` (ISO 8601) | `TIMESTAMPTZ` | string (ISO 8601) | +| Other | Other | string (fallback) | ## db.backend -- 2.51.2 From 60cf8868a6a5566786f9ad42eed44b4299e2e00b Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 20 May 2026 20:57:24 -0500 Subject: [PATCH 05/64] ci: publish Rust binaries to Github Releases Signed-off-by: Trezy --- .github/workflows/ci.yml | 76 +++++++++++++++++++++++++++++++++++++--- Dockerfile | 2 +- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 427c506..b9b48b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,60 @@ jobs: GITHUB_TOKEN: ${{ secrets.DISPATCH_GH_TOKEN }} run: npx -p semantic-release -p semantic-release-gha-output semantic-release + # --------------------------------------------------------------------------- + # Build binaries — compile per-platform and upload to GitHub release + # --------------------------------------------------------------------------- + build-binary: + needs: release + if: needs.release.outputs.version != '' + runs-on: ${{ matrix.runs-on }} + permissions: + contents: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: depot-ubuntu-24.04 + arch: amd64 + - platform: linux/arm64 + runs-on: depot-ubuntu-24.04-arm + arch: arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-release-${{ matrix.arch }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + cargo-release-${{ matrix.arch }}- + + - name: Build release binary + run: | + docker run --rm \ + -v "$PWD:/app" \ + -v "$HOME/.cargo/registry:/usr/local/cargo/registry" \ + -v "$HOME/.cargo/git:/usr/local/cargo/git" \ + -w /app \ + -e SQLX_OFFLINE=true \ + -e HAPPYVIEW_VERSION="v${{ needs.release.outputs.version }}" \ + rust:1.93-bookworm \ + cargo build --release + + - name: Upload binary to release + env: + GH_TOKEN: ${{ secrets.DISPATCH_GH_TOKEN }} + run: | + cp target/release/happyview happyview-linux-${{ matrix.arch }} + gh release upload "v${{ needs.release.outputs.version }}" \ + happyview-linux-${{ matrix.arch }} \ + --clobber + # --------------------------------------------------------------------------- # SDK — per-package tests + releases # --------------------------------------------------------------------------- @@ -509,7 +563,7 @@ jobs: # Docker — main app image (triggered by GitHub release) # --------------------------------------------------------------------------- docker: - needs: release + needs: [release, build-binary] if: needs.release.outputs.version != '' runs-on: ${{ matrix.runs-on }} permissions: @@ -522,8 +576,10 @@ jobs: include: - platform: linux/amd64 runs-on: depot-ubuntu-24.04 + arch: amd64 - platform: linux/arm64 runs-on: depot-ubuntu-24.04-arm + arch: arm64 env: GHCR_IMAGE: ghcr.io/${{ github.repository }} ATCR_IMAGE: atcr.io/${{ secrets.ATCR_NAMESPACE }}/happyview @@ -531,6 +587,16 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Download pre-built binary + env: + GH_TOKEN: ${{ secrets.DISPATCH_GH_TOKEN }} + run: | + gh release download "v${{ needs.release.outputs.version }}" \ + --pattern "happyview-linux-${{ matrix.arch }}" \ + --dir . + mkdir -p .builder-override/app/target/release + cp happyview-linux-${{ matrix.arch }} .builder-override/app/target/release/happyview + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -590,8 +656,8 @@ jobs: context: . platforms: ${{ matrix.platform }} labels: ${{ steps.meta-ghcr.outputs.labels }} - build-args: | - HAPPYVIEW_VERSION=v${{ needs.release.outputs.version }} + build-contexts: | + builder=.builder-override cache-from: type=gha,scope=build-${{ matrix.platform }} cache-to: type=gha,scope=build-${{ matrix.platform }},mode=max,ignore-error=true outputs: type=image,"name=${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true @@ -605,8 +671,8 @@ jobs: # context: . # platforms: ${{ matrix.platform }} # labels: ${{ steps.meta-atcr.outputs.labels }} - # build-args: | - # HAPPYVIEW_VERSION=${{ github.event.release.tag_name }} + # build-contexts: | + # builder=.builder-override # cache-from: type=gha,scope=build-${{ matrix.platform }} # outputs: type=image,"name=${{ env.ATCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true diff --git a/Dockerfile b/Dockerfile index 87aee8b..97108be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app COPY --from=builder /app/target/release/happyview /usr/local/bin/happyview -COPY --from=builder /app/migrations /app/migrations +COPY migrations/ /app/migrations COPY --from=frontend /app/web/out /srv/static COPY entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh && touch /srv/static/.base-path-pending -- 2.51.2 From 8f21474dbdefd550d4a39c2d4afdc9576779d217 Mon Sep 17 00:00:00 2001 From: Trezy Date: Thu, 21 May 2026 09:22:40 -0500 Subject: [PATCH 06/64] feat: parallelize PDS resolution and record retrieval during backfills Signed-off-by: Trezy --- .github/workflows/ci.yml | 176 +++++++ .../20260520000000_add_resolved_repos.sql | 1 + .../20260520000000_add_resolved_repos.sql | 1 + src/admin/backfill.rs | 474 ++++++++++++++++-- src/admin/types.rs | 1 + web/src/app/dashboard/backfill/page.tsx | 24 +- web/src/types/backfill.ts | 1 + 7 files changed, 626 insertions(+), 52 deletions(-) create mode 100644 migrations/postgres/20260520000000_add_resolved_repos.sql create mode 100644 migrations/sqlite/20260520000000_add_resolved_repos.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9b48b2..2ee8050 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,6 +159,182 @@ jobs: - name: Clippy run: cargo clippy --all-targets -- -D warnings + # --------------------------------------------------------------------------- + # PR binaries — build on every PR push so reviewers can test + # --------------------------------------------------------------------------- + pr-binary: + needs: changes + if: >- + github.event_name == 'pull_request' + && needs.changes.outputs.server == 'true' + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + include: + - runs-on: depot-ubuntu-24.04 + arch: amd64 + - runs-on: depot-ubuntu-24.04-arm + arch: arm64 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-release-${{ matrix.arch }}-${{ hashFiles('Cargo.lock') }} + restore-keys: | + cargo-release-${{ matrix.arch }}- + + - name: Build release binary + run: | + docker run --rm \ + -v "$PWD:/app" \ + -v "$HOME/.cargo/registry:/usr/local/cargo/registry" \ + -v "$HOME/.cargo/git:/usr/local/cargo/git" \ + -w /app \ + -e SQLX_OFFLINE=true \ + rust:1.93-bookworm \ + cargo build --release + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: happyview-linux-${{ matrix.arch }} + path: target/release/happyview + + pr-docker: + needs: pr-binary + runs-on: ${{ matrix.runs-on }} + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runs-on: depot-ubuntu-24.04 + arch: amd64 + - platform: linux/arm64 + runs-on: depot-ubuntu-24.04-arm + arch: arm64 + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download pre-built binary + uses: actions/download-artifact@v4 + with: + name: happyview-linux-${{ matrix.arch }} + path: .binary + + - name: Prepare binary override + run: | + mkdir -p .builder-override/app/target/release + cp .binary/happyview .builder-override/app/target/release/happyview + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_IMAGE }} + tags: | + type=raw,value=pr-${{ github.event.pull_request.number }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + build-contexts: | + builder=.builder-override + cache-from: type=gha,scope=pr-${{ matrix.platform }} + cache-to: type=gha,scope=pr-${{ matrix.platform }},mode=max,ignore-error=true + outputs: type=image,"name=${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + - name: Create manifest + run: | + TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta.outputs.json }}') + docker buildx imagetools create --append $TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} 2>/dev/null || \ + docker buildx imagetools create $TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + + pr-build-comment: + needs: pr-docker + runs-on: depot-ubuntu-24.04 + permissions: + pull-requests: write + steps: + - name: Comment on PR + uses: actions/github-script@v7 + with: + script: | + const sha = context.payload.pull_request.head.sha.substring(0, 7); + const prNumber = context.payload.pull_request.number; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const image = `ghcr.io/${context.repo.owner}/${context.repo.repo}`; + const body = [ + '### PR Build', + '', + `Builds for \`${sha}\`:`, + '', + '**Docker**', + '```sh', + `docker pull ${image}:pr-${prNumber}`, + '```', + '', + '**Binaries**', + '```sh', + `gh run download ${context.runId} -n happyview-linux-amd64`, + `gh run download ${context.runId} -n happyview-linux-arm64`, + '```', + '', + `Or download binaries from the [workflow run](${runUrl}).`, + ].join('\n'); + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body.startsWith('### PR Build')); + + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + # --------------------------------------------------------------------------- # Server release # --------------------------------------------------------------------------- diff --git a/migrations/postgres/20260520000000_add_resolved_repos.sql b/migrations/postgres/20260520000000_add_resolved_repos.sql new file mode 100644 index 0000000..da79bf0 --- /dev/null +++ b/migrations/postgres/20260520000000_add_resolved_repos.sql @@ -0,0 +1 @@ +ALTER TABLE backfill_jobs ADD COLUMN resolved_repos INTEGER; diff --git a/migrations/sqlite/20260520000000_add_resolved_repos.sql b/migrations/sqlite/20260520000000_add_resolved_repos.sql new file mode 100644 index 0000000..da79bf0 --- /dev/null +++ b/migrations/sqlite/20260520000000_add_resolved_repos.sql @@ -0,0 +1 @@ +ALTER TABLE backfill_jobs ADD COLUMN resolved_repos INTEGER; diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 553c3a5..26b7253 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -5,9 +5,11 @@ use std::sync::atomic::{AtomicBool, AtomicI32, Ordering}; use axum::Json; use axum::extract::{Path, State}; use axum::http::StatusCode; -use futures_util::stream::{self, StreamExt}; +use futures_util::FutureExt; +use futures_util::stream::{self, FuturesUnordered, StreamExt}; use serde::Deserialize; use serde_json::Value; +use tokio::sync::mpsc; use uuid::Uuid; use crate::AppState; @@ -69,6 +71,7 @@ async fn set_stage(state: &AppState, job_id: &str, stage: &str) { async fn update_job_counter(state: &AppState, job_id: &str, column: &str, value: i32) { let query = match column { "total_repos" => "UPDATE backfill_jobs SET total_repos = ? WHERE id = ?", + "resolved_repos" => "UPDATE backfill_jobs SET resolved_repos = ? WHERE id = ?", "processed_repos" => "UPDATE backfill_jobs SET processed_repos = ? WHERE id = ?", "total_records" => "UPDATE backfill_jobs SET total_records = ? WHERE id = ?", other => { @@ -317,69 +320,448 @@ async fn discover_repos_from_relay( } // --------------------------------------------------------------------------- -// Phase 2: Resolve PDS endpoints +// Pipelined Phase 2+3: Resolve PDS endpoints and fetch records concurrently // --------------------------------------------------------------------------- -async fn run_resolution_phase(state: &AppState, job_id: &str) { - set_stage(state, job_id, "resolving_pds").await; +async fn run_pipelined_resolve_and_fetch( + state: &AppState, + job_id: &str, + collections: &[String], +) -> (i32, i32) { + set_stage(state, job_id, "resolving_and_fetching").await; - let sql = adapt_sql( - "SELECT did FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NULL", + // Count already-resolved and already-completed repos for accurate progress + let already_resolved: i32 = { + let sql = adapt_sql( + "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL", + state.db_backend, + ); + sqlx::query_as::<_, (i32,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c) + .unwrap_or(0) + }; + + let already_completed: i32 = { + let sql = adapt_sql( + "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND status = 'completed'", + state.db_backend, + ); + sqlx::query_as::<_, (i32,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c) + .unwrap_or(0) + }; + + update_job_counter(state, job_id, "resolved_repos", already_resolved).await; + update_job_counter(state, job_id, "processed_repos", already_completed).await; + + let existing_records: i32 = { + let sql = adapt_sql( + "SELECT total_records FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + sqlx::query_as::<_, (Option,)>(&sql) + .bind(job_id) + .fetch_one(&state.db) + .await + .map(|(c,)| c.unwrap_or(0)) + .unwrap_or(0) + }; + + // Shared atomics for lock-free counter updates + let resolved_repos = Arc::new(AtomicI32::new(already_resolved)); + let processed_repos = Arc::new(AtomicI32::new(already_completed)); + let total_records = Arc::new(AtomicI32::new(existing_records)); + let cancelled = Arc::new(AtomicBool::new(false)); + + let (tx, mut rx) = mpsc::channel::<(String, String)>(256); + let tx_resolver = tx.clone(); + let tx_backlog = tx.clone(); + + // --- Resolver task --- + let resolver_state = state.clone(); + let resolver_job_id = job_id.to_string(); + let resolver_resolved = Arc::clone(&resolved_repos); + let resolver_cancelled = Arc::clone(&cancelled); + + let resolver_handle = tokio::spawn(async move { + let sql = adapt_sql( + "SELECT did FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NULL", + resolver_state.db_backend, + ); + let unresolved: Vec<(String,)> = sqlx::query_as(&sql) + .bind(&resolver_job_id) + .fetch_all(&resolver_state.db) + .await + .unwrap_or_default(); + + let mut attempted: i32 = 0; + for (did,) in &unresolved { + if resolver_cancelled.load(Ordering::Relaxed) { + break; + } + + match profile::resolve_pds_endpoint( + &resolver_state.http, + &resolver_state.config.plc_url, + did, + ) + .await + { + Ok(pds) => { + let sql = adapt_sql( + "UPDATE backfill_repos SET pds_endpoint = ? WHERE job_id = ? AND did = ?", + resolver_state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(&pds) + .bind(&resolver_job_id) + .bind(did) + .execute(&resolver_state.db) + .await; + + let count = resolver_resolved.fetch_add(1, Ordering::Relaxed) + 1; + if count % 100 == 0 { + update_job_counter( + &resolver_state, + &resolver_job_id, + "resolved_repos", + count, + ) + .await; + } + + if tx_resolver.send((did.clone(), pds)).await.is_err() { + break; + } + } + Err(e) => { + tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); + } + } + + attempted += 1; + if attempted % 100 == 0 && is_cancelled(&resolver_state, &resolver_job_id).await { + resolver_cancelled.store(true, Ordering::Relaxed); + break; + } + } + + // Persist final resolved count + let final_resolved = resolver_resolved.load(Ordering::Relaxed); + update_job_counter( + &resolver_state, + &resolver_job_id, + "resolved_repos", + final_resolved, + ) + .await; + // tx is dropped here, signalling the fetcher that no more DIDs are coming + }); + + // --- Also send already-resolved-but-unfetched DIDs to the fetcher --- + let pending_sql = adapt_sql( + "SELECT did, pds_endpoint FROM backfill_repos WHERE job_id = ? AND status = 'pending' AND pds_endpoint IS NOT NULL", state.db_backend, ); - let unresolved: Vec<(String,)> = sqlx::query_as(&sql) + let pending_rows: Vec<(String, String)> = sqlx::query_as(&pending_sql) .bind(job_id) .fetch_all(&state.db) .await .unwrap_or_default(); + let backlog_cancelled = Arc::clone(&cancelled); + let backlog_handle = tokio::spawn(async move { + for (did, pds) in pending_rows { + if backlog_cancelled.load(Ordering::Relaxed) { + break; + } + if tx_backlog.send((did, pds)).await.is_err() { + break; + } + } + }); + + // Drop our copy of tx so the channel closes when both senders finish + drop(tx); + + // --- Fetcher: receive (did, pds) pairs and dispatch to PDS workers --- + // Each PDS gets its own worker with a DID channel. Workers acquire a + // semaphore permit before starting, limiting concurrent PDS connections. + // We never hold the workers lock across an `.await` — use `try_send` to + // avoid blocking when a worker's channel is full (overflow goes to a + // retry queue drained on each iteration). + let state = Arc::new(state.clone()); + let collections = Arc::new(collections.to_vec()); + let job_id_arc = Arc::new(job_id.to_string()); + + let pds_semaphore = Arc::new(tokio::sync::Semaphore::new(10)); + let mut pds_workers: HashMap> = HashMap::new(); + let mut worker_handles = FuturesUnordered::new(); + let mut overflow: Vec<(String, String)> = Vec::new(); + + while let Some((did, pds_endpoint)) = rx.recv().await { + // Also drain any overflow from previous iterations + overflow.push((did, pds_endpoint)); + + let mut still_pending = Vec::new(); + for (did, pds_endpoint) in overflow.drain(..) { + if cancelled.load(Ordering::Relaxed) { + break; + } + + // Try to send to an existing PDS worker + if let Some(pds_tx) = pds_workers.get(&pds_endpoint) { + match pds_tx.try_send(did.clone()) { + Ok(()) => continue, + Err(mpsc::error::TrySendError::Full(_)) => { + still_pending.push((did, pds_endpoint)); + continue; + } + Err(mpsc::error::TrySendError::Closed(_)) => { + // Worker finished, will be removed below + } + } + } + + // Remove stale workers whose channels have closed + pds_workers.retain(|_, tx| !tx.is_closed()); + + // Spawn a new PDS worker + let permit = Arc::clone(&pds_semaphore); + let (pds_tx, pds_rx) = mpsc::channel::(64); + let _ = pds_tx.try_send(did); + pds_workers.insert(pds_endpoint.clone(), pds_tx); + + let ctx = FetchContext { + state: Arc::clone(&state), + job_id: Arc::clone(&job_id_arc), + collections: Arc::clone(&collections), + processed_repos: Arc::clone(&processed_repos), + total_records: Arc::clone(&total_records), + cancelled: Arc::clone(&cancelled), + }; + + worker_handles.push(tokio::spawn(async move { + let _permit = permit + .acquire() + .await + .expect("semaphore should not be closed"); + + run_pds_worker(ctx, pds_endpoint, pds_rx).await; + })); + } + overflow = still_pending; + + // Drain any completed worker handles to avoid unbounded accumulation + while let Some(result) = worker_handles.next().now_or_never() { + if let Some(Err(e)) = result { + tracing::warn!(error = %e, "PDS worker task panicked"); + } + } + } + + // Drain remaining overflow after channel closes + for (did, pds_endpoint) in overflow.drain(..) { + if cancelled.load(Ordering::Relaxed) { + break; + } + + // Remove stale workers + pds_workers.retain(|_, tx| !tx.is_closed()); + + if let Some(pds_tx) = pds_workers.get(&pds_endpoint) { + // Channel is bounded; this can block, but all senders are done so it's fine + let _ = pds_tx.send(did).await; + continue; + } + + let permit = Arc::clone(&pds_semaphore); + let (pds_tx, pds_rx) = mpsc::channel::(64); + let _ = pds_tx.try_send(did); + pds_workers.insert(pds_endpoint.clone(), pds_tx); + + let ctx = FetchContext { + state: Arc::clone(&state), + job_id: Arc::clone(&job_id_arc), + collections: Arc::clone(&collections), + processed_repos: Arc::clone(&processed_repos), + total_records: Arc::clone(&total_records), + cancelled: Arc::clone(&cancelled), + }; + + worker_handles.push(tokio::spawn(async move { + let _permit = permit + .acquire() + .await + .expect("semaphore should not be closed"); + + run_pds_worker(ctx, pds_endpoint.clone(), pds_rx).await; + })); + } + + // Drop all PDS senders so workers know no more DIDs are coming + drop(pds_workers); + + // Wait for all PDS workers to finish + while let Some(result) = worker_handles.next().await { + if let Err(e) = result { + tracing::warn!(error = %e, "PDS worker task panicked"); + } + } + + // Wait for resolver and backlog tasks + let _ = resolver_handle.await; + let _ = backlog_handle.await; + + let final_repos = processed_repos.load(Ordering::Relaxed); + let final_records = total_records.load(Ordering::Relaxed); + + // Persist final counts let sql = adapt_sql( - "SELECT COUNT(*) FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL", + "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", state.db_backend, ); - let already_resolved: i32 = sqlx::query_as::<_, (i32,)>(&sql) + let _ = sqlx::query(&sql) + .bind(final_repos) + .bind(final_records) .bind(job_id) - .fetch_one(&state.db) - .await - .map(|(c,)| c) - .unwrap_or(0); + .execute(&state.db) + .await; + + (final_repos, final_records) +} + +struct FetchContext { + state: Arc, + job_id: Arc, + collections: Arc>, + processed_repos: Arc, + total_records: Arc, + cancelled: Arc, +} - let mut resolved_count = already_resolved; +async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::Receiver) { + let FetchContext { + state, + job_id, + collections, + processed_repos, + total_records, + cancelled, + } = ctx; + let mut fetches = FuturesUnordered::new(); + let mut rx_open = true; - let mut attempted = already_resolved; - for (did,) in &unresolved { - match profile::resolve_pds_endpoint(&state.http, &state.config.plc_url, did).await { - Ok(pds) => { + loop { + tokio::select! { + biased; + + Some(result) = fetches.next(), if !fetches.is_empty() => { + let (did, records): (String, i32) = result; + total_records.fetch_add(records, Ordering::Relaxed); + + // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET pds_endpoint = ? WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) - .bind(&pds) - .bind(job_id) - .bind(did) + .bind(job_id.as_str()) + .bind(&did) .execute(&state.db) .await; - resolved_count += 1; - } - Err(e) => { - tracing::warn!(did, error = %e, "failed to resolve PDS endpoint, skipping DID"); + + let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; + if repos % 10 == 0 { + let records = total_records.load(Ordering::Relaxed); + let sql = adapt_sql( + "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(repos) + .bind(records) + .bind(job_id.as_str()) + .execute(&state.db) + .await; + + if is_cancelled(&state, job_id.as_str()).await { + cancelled.store(true, Ordering::Relaxed); + break; + } + } } - } - attempted += 1; - if attempted % 100 == 0 { - update_job_counter(state, job_id, "processed_repos", resolved_count).await; - if is_cancelled(state, job_id).await { - return; + + did = rx.recv(), if rx_open && fetches.len() < 3 => { + match did { + Some(did) if !cancelled.load(Ordering::Relaxed) => { + let state = Arc::clone(&state); + let collections = collections.clone(); + let pds_endpoint = pds_endpoint.clone(); + + fetches.push(async move { + let mut count: i32 = 0; + for collection in collections.iter() { + match fetch_records_from_pds( + &state, + &pds_endpoint, + &did, + collection, + ) + .await + { + Ok(c) => count += c as i32, + Err(e) => { + tracing::warn!( + did, + collection, + pds = %pds_endpoint, + error = %e, + "failed to fetch records from PDS" + ); + } + } + } + (did, count) + }); + } + _ => { + rx_open = false; + } + } } + + else => break, } } - update_job_counter(state, job_id, "processed_repos", resolved_count).await; + // Drain any remaining fetches + while let Some(result) = fetches.next().await { + let (did, records): (String, i32) = result; + total_records.fetch_add(records, Ordering::Relaxed); + + let sql = adapt_sql( + "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id.as_str()) + .bind(&did) + .execute(&state.db) + .await; + + processed_repos.fetch_add(1, Ordering::Relaxed); + } } // --------------------------------------------------------------------------- -// Phase 3: Fetch records from PDS instances +// Phase 3: Fetch records from PDS instances (legacy, for resumed jobs) // --------------------------------------------------------------------------- async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[String]) -> (i32, i32) { @@ -713,20 +1095,15 @@ async fn run_backfill_job(state: AppState, job_id: String) { } } - if matches!( + let (final_processed, final_records) = if matches!( stage.as_str(), - "pending" | "discovering_repos" | "resolving_pds" + "pending" | "discovering_repos" | "resolving_pds" | "resolving_and_fetching" ) { - run_resolution_phase(&state, &job_id).await; - - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; - } - } - - let (final_processed, final_records) = run_fetching_phase(&state, &job_id, &collections).await; + run_pipelined_resolve_and_fetch(&state, &job_id, &collections).await + } else { + // stage == "fetching_records": resolution already done (legacy or resumed) + run_fetching_phase(&state, &job_id, &collections).await + }; if is_cancelled(&state, &job_id).await { tracing::info!(job_id, "backfill job cancelled"); @@ -871,7 +1248,7 @@ pub(super) async fn backfill_status( let backend = state.db_backend; let sql = adapt_sql( - "SELECT id, collection, did, status, stage, total_repos, processed_repos, total_records, error, started_at, completed_at, created_at FROM backfill_jobs ORDER BY created_at DESC", + "SELECT id, collection, did, status, stage, total_repos, resolved_repos, processed_repos, total_records, error, started_at, completed_at, created_at FROM backfill_jobs ORDER BY created_at DESC", backend, ); #[allow(clippy::type_complexity)] @@ -884,6 +1261,7 @@ pub(super) async fn backfill_status( Option, Option, Option, + Option, Option, Option, Option, @@ -903,6 +1281,7 @@ pub(super) async fn backfill_status( status, stage, total_repos, + resolved_repos, processed_repos, total_records, error, @@ -917,6 +1296,7 @@ pub(super) async fn backfill_status( status, stage, total_repos, + resolved_repos, processed_repos, total_records, error, diff --git a/src/admin/types.rs b/src/admin/types.rs index 4871b2e..3a5c0d1 100644 --- a/src/admin/types.rs +++ b/src/admin/types.rs @@ -76,6 +76,7 @@ pub(crate) struct BackfillJob { pub(crate) status: String, pub(crate) stage: String, pub(crate) total_repos: Option, + pub(crate) resolved_repos: Option, pub(crate) processed_repos: Option, pub(crate) total_records: Option, pub(crate) error: Option, diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index f2ed0b6..e3f1060 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -90,6 +90,7 @@ function statusBadge(job: BackfillJob) { } function phaseIndex(stage: string): number { + if (stage === "resolving_and_fetching") return 1; const idx = PROGRESS_PHASES.indexOf( stage as (typeof PROGRESS_PHASES)[number], ); @@ -229,6 +230,9 @@ function JobDetail({ function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; + if (job.stage === "resolving_and_fetching") { + return phase === "discovering_repos" || phase === "resolving_pds" || phase === "fetching_records"; + } return current >= phaseIndex(phase); } @@ -307,26 +311,36 @@ function JobDetail({ /> Date: Thu, 21 May 2026 11:32:30 -0500 Subject: [PATCH 07/64] feat: add more detailed information to backfill UI Signed-off-by: Trezy --- Cargo.lock | 39 +- Cargo.toml | 1 + .../20260521000000_backfill_diagnostics.sql | 2 + .../20260521000000_backfill_diagnostics.sql | 2 + src/admin/backfill.rs | 396 +++++++++++- src/admin/mod.rs | 14 + src/admin/settings.rs | 1 + src/admin/types.rs | 71 ++ src/lib.rs | 1 + src/lua/atproto_api.rs | 1 + src/lua/db_api.rs | 1 + src/lua/execute.rs | 1 + src/lua/http_api.rs | 1 + src/lua/xrpc_api.rs | 1 + src/main.rs | 10 + tests/common/app.rs | 1 + tests/lua_atproto_api.rs | 1 + tests/lua_db_api.rs | 1 + web/package-lock.json | 28 + web/package.json | 1 + web/src/app/dashboard/backfill/page.tsx | 610 ++++++++++++++++-- .../app/dashboard/settings/general/page.tsx | 44 ++ web/src/lib/api.ts | 32 +- web/src/types/backfill.ts | 45 ++ 24 files changed, 1231 insertions(+), 74 deletions(-) create mode 100644 migrations/postgres/20260521000000_backfill_diagnostics.sql create mode 100644 migrations/sqlite/20260521000000_backfill_diagnostics.sql diff --git a/Cargo.lock b/Cargo.lock index 29026e6..69eeee1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,6 +157,28 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -483,7 +505,7 @@ dependencies = [ "cap-primitives", "cap-std", "io-lifetimes", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -512,7 +534,7 @@ dependencies = [ "maybe-owned", "rustix 1.1.3", "rustix-linux-procfs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winx", ] @@ -1293,7 +1315,7 @@ checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", "rustix 1.1.3", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1377,7 +1399,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" dependencies = [ "io-lifetimes", "rustix 1.1.3", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1626,6 +1648,7 @@ dependencies = [ "aes-gcm", "anyhow", "arc-swap", + "async-stream", "atrium-api", "atrium-common", "atrium-identity", @@ -2095,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2285ddfe3054097ef4b2fe909ef8c3bcd1ea52a8f0d274416caebeef39f04a65" dependencies = [ "io-lifetimes", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3264,7 +3287,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3975,7 +3998,7 @@ dependencies = [ "fd-lock", "io-lifetimes", "rustix 0.38.44", - "windows-sys 0.52.0", + "windows-sys 0.59.0", "winx", ] @@ -5544,7 +5567,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" dependencies = [ "bitflags", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index aaf858d..40bc34f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ wasmtime = { version = "29", features = ["async"] } wasmtime-wasi = "29" regex = "1.12.3" semver = "1.0" +async-stream = "0.3.6" [[bin]] name = "migrate-lua-sql" diff --git a/migrations/postgres/20260521000000_backfill_diagnostics.sql b/migrations/postgres/20260521000000_backfill_diagnostics.sql new file mode 100644 index 0000000..f80f7c7 --- /dev/null +++ b/migrations/postgres/20260521000000_backfill_diagnostics.sql @@ -0,0 +1,2 @@ +ALTER TABLE backfill_repos ADD COLUMN records_fetched INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_backfill_repos_job_status ON backfill_repos (job_id, status); diff --git a/migrations/sqlite/20260521000000_backfill_diagnostics.sql b/migrations/sqlite/20260521000000_backfill_diagnostics.sql new file mode 100644 index 0000000..f80f7c7 --- /dev/null +++ b/migrations/sqlite/20260521000000_backfill_diagnostics.sql @@ -0,0 +1,2 @@ +ALTER TABLE backfill_repos ADD COLUMN records_fetched INTEGER NOT NULL DEFAULT 0; +CREATE INDEX IF NOT EXISTS idx_backfill_repos_job_status ON backfill_repos (job_id, status); diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 26b7253..c9c3d8c 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -12,6 +12,8 @@ use serde_json::Value; use tokio::sync::mpsc; use uuid::Uuid; +use rand::Rng; + use crate::AppState; use crate::db::{adapt_sql, now_rfc3339}; use crate::error::AppError; @@ -66,6 +68,13 @@ async fn set_stage(state: &AppState, job_id: &str, stage: &str) { .bind(job_id) .execute(&state.db) .await; + publish_event( + state, + super::types::BackfillEvent::JobStageChanged { + job_id: job_id.to_string(), + stage: stage.to_string(), + }, + ); } async fn update_job_counter(state: &AppState, job_id: &str, column: &str, value: i32) { @@ -103,12 +112,13 @@ async fn count_repos(state: &AppState, job_id: &str) -> i32 { .unwrap_or(0) } -async fn cleanup_repos(state: &AppState, job_id: &str) { - let sql = adapt_sql( - "DELETE FROM backfill_repos WHERE job_id = ?", - state.db_backend, - ); - let _ = sqlx::query(&sql).bind(job_id).execute(&state.db).await; +fn publish_event(state: &AppState, event: super::types::BackfillEvent) { + let _ = state.backfill_events_tx.send(event); +} + +fn random_batch_threshold(base: i32) -> i32 { + let low = base - base / 10; + rand::rng().random_range(low..=base) } async fn fail_job(state: &AppState, job_id: &str, error: &str) { @@ -123,7 +133,14 @@ async fn fail_job(state: &AppState, job_id: &str, error: &str) { .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "failed".to_string(), + error: Some(error.to_string()), + }, + ); } async fn is_cancelled(state: &AppState, job_id: &str) -> bool { @@ -159,7 +176,14 @@ async fn finalise_cancel(state: &AppState, job_id: &str) { .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "cancelled".to_string(), + error: None, + }, + ); } async fn complete_job( @@ -182,7 +206,14 @@ async fn complete_job( .bind(job_id) .execute(&state.db) .await; - cleanup_repos(state, job_id).await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "completed".to_string(), + error: error.map(|e| e.to_string()), + }, + ); } // --------------------------------------------------------------------------- @@ -207,6 +238,13 @@ async fn run_discovery_phase( .bind(did) .execute(&state.db) .await; + publish_event( + state, + super::types::BackfillEvent::RepoDiscovered { + job_id: job_id.to_string(), + did: did.to_string(), + }, + ); } else { for collection in collections { if is_cancelled(state, job_id).await { @@ -301,6 +339,15 @@ async fn discover_repos_from_relay( if let Ok(result) = query.execute(&state.db).await { running_total += result.rows_affected() as i32; } + for repo in chunk { + publish_event( + state, + super::types::BackfillEvent::RepoDiscovered { + job_id: job_id.to_string(), + did: repo.did.clone(), + }, + ); + } } } @@ -401,6 +448,8 @@ async fn run_pipelined_resolve_and_fetch( .unwrap_or_default(); let mut attempted: i32 = 0; + let mut next_flush = random_batch_threshold(100); + let mut next_cancel_check = random_batch_threshold(100); for (did,) in &unresolved { if resolver_cancelled.load(Ordering::Relaxed) { break; @@ -425,8 +474,17 @@ async fn run_pipelined_resolve_and_fetch( .execute(&resolver_state.db) .await; + publish_event( + &resolver_state, + super::types::BackfillEvent::RepoResolved { + job_id: resolver_job_id.clone(), + did: did.clone(), + pds_endpoint: pds.clone(), + }, + ); + let count = resolver_resolved.fetch_add(1, Ordering::Relaxed) + 1; - if count % 100 == 0 { + if count >= next_flush { update_job_counter( &resolver_state, &resolver_job_id, @@ -434,7 +492,18 @@ async fn run_pipelined_resolve_and_fetch( count, ) .await; + next_flush = count + random_batch_threshold(100); } + publish_event( + &resolver_state, + super::types::BackfillEvent::JobCounters { + job_id: resolver_job_id.clone(), + total_repos: None, + resolved_repos: Some(count), + processed_repos: None, + total_records: None, + }, + ); if tx_resolver.send((did.clone(), pds)).await.is_err() { break; @@ -446,9 +515,12 @@ async fn run_pipelined_resolve_and_fetch( } attempted += 1; - if attempted % 100 == 0 && is_cancelled(&resolver_state, &resolver_job_id).await { - resolver_cancelled.store(true, Ordering::Relaxed); - break; + if attempted >= next_cancel_check { + if is_cancelled(&resolver_state, &resolver_job_id).await { + resolver_cancelled.store(true, Ordering::Relaxed); + break; + } + next_cancel_check = attempted + random_batch_threshold(100); } } @@ -657,6 +729,7 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R } = ctx; let mut fetches = FuturesUnordered::new(); let mut rx_open = true; + let mut next_flush = random_batch_threshold(10); loop { tokio::select! { @@ -668,18 +741,26 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; + publish_event(&state, super::types::BackfillEvent::RepoFetched { + job_id: job_id.to_string(), + did: did.clone(), + pds_endpoint: pds_endpoint.clone(), + records_fetched: records, + }); + let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; - if repos % 10 == 0 { - let records = total_records.load(Ordering::Relaxed); + let records = total_records.load(Ordering::Relaxed); + if repos >= next_flush { let sql = adapt_sql( "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", state.db_backend, @@ -695,7 +776,15 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R cancelled.store(true, Ordering::Relaxed); break; } + next_flush = repos + random_batch_threshold(10); } + publish_event(&state, super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: None, + resolved_repos: None, + processed_repos: Some(repos), + total_records: Some(records), + }); } did = rx.recv(), if rx_open && fetches.len() < 3 => { @@ -747,15 +836,26 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R total_records.fetch_add(records, Ordering::Relaxed); let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; + publish_event( + &state, + super::types::BackfillEvent::RepoFetched { + job_id: job_id.to_string(), + did: did.clone(), + pds_endpoint: pds_endpoint.clone(), + records_fetched: records, + }, + ); + processed_repos.fetch_add(1, Ordering::Relaxed); } } @@ -815,6 +915,9 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::new(AtomicI32::new(already_completed)); let total_records = Arc::new(AtomicI32::new(existing_records)); let cancelled = Arc::new(AtomicBool::new(false)); + let next_flush = Arc::new(AtomicI32::new( + already_completed + random_batch_threshold(10), + )); let state = Arc::new(state.clone()); let collections = Arc::new(collections.to_vec()); let job_id_arc = Arc::new(job_id.to_string()); @@ -828,6 +931,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::clone(&processed_repos); let total_records = Arc::clone(&total_records); let cancelled = Arc::clone(&cancelled); + let next_flush = Arc::clone(&next_flush); let job_id = Arc::clone(&job_id_arc); async move { @@ -838,6 +942,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin let processed_repos = Arc::clone(&processed_repos); let total_records = Arc::clone(&total_records); let cancelled = Arc::clone(&cancelled); + let next_flush = Arc::clone(&next_flush); let pds_endpoint = pds_endpoint.clone(); let job_id = Arc::clone(&job_id); @@ -846,6 +951,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin return; } + let mut did_records: i32 = 0; for collection in collections.iter() { match fetch_records_from_pds( &state, @@ -856,6 +962,7 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin .await { Ok(count) => { + did_records += count as i32; total_records .fetch_add(count as i32, Ordering::Relaxed); } @@ -873,19 +980,23 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin // Mark DID as completed let sql = adapt_sql( - "UPDATE backfill_repos SET status = 'completed' WHERE job_id = ? AND did = ?", + "UPDATE backfill_repos SET status = 'completed', records_fetched = ? WHERE job_id = ? AND did = ?", state.db_backend, ); let _ = sqlx::query(&sql) + .bind(did_records) .bind(job_id.as_str()) .bind(&did) .execute(&state.db) .await; let repos = processed_repos.fetch_add(1, Ordering::Relaxed) + 1; + let records = total_records.load(Ordering::Relaxed); - if repos % 10 == 0 { - let records = total_records.load(Ordering::Relaxed); + let threshold = next_flush.load(Ordering::Relaxed); + if repos >= threshold + && next_flush.compare_exchange(threshold, repos + random_batch_threshold(10), Ordering::Relaxed, Ordering::Relaxed).is_ok() + { let backend = state.db_backend; let sql = adapt_sql( "UPDATE backfill_jobs SET processed_repos = ?, total_records = ? WHERE id = ?", @@ -902,6 +1013,14 @@ async fn run_fetching_phase(state: &AppState, job_id: &str, collections: &[Strin cancelled.store(true, Ordering::Relaxed); } } + + publish_event(&state, super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: None, + resolved_repos: None, + processed_repos: Some(repos), + total_records: Some(records), + }); } }) .await; @@ -1311,6 +1430,243 @@ pub(super) async fn backfill_status( Ok(Json(jobs)) } +// --------------------------------------------------------------------------- +// SSE events endpoint +// --------------------------------------------------------------------------- + +pub(super) async fn backfill_events( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result< + axum::response::sse::Sse< + impl futures_util::Stream>, + >, + AppError, +> { + auth.require(Permission::BackfillRead).await?; + + let mut rx = state.backfill_events_tx.subscribe(); + + let stream = async_stream::stream! { + loop { + match rx.recv().await { + Ok(event) => { + let event_job_id = match &event { + super::types::BackfillEvent::RepoDiscovered { job_id, .. } + | super::types::BackfillEvent::RepoResolved { job_id, .. } + | super::types::BackfillEvent::RepoFetched { job_id, .. } + | super::types::BackfillEvent::JobCounters { job_id, .. } + | super::types::BackfillEvent::JobStageChanged { job_id, .. } + | super::types::BackfillEvent::JobCompleted { job_id, .. } => job_id, + }; + if *event_job_id != job_id { + continue; + } + if let Ok(json) = serde_json::to_string(&event) { + yield Ok(axum::response::sse::Event::default().event("event").data(json)); + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => { + tracing::warn!(job_id, skipped = n, "SSE client lagged behind"); + continue; + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } + }; + + Ok(axum::response::sse::Sse::new(stream).keep_alive(axum::response::sse::KeepAlive::default())) +} + +// --------------------------------------------------------------------------- +// REST detail endpoints +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +pub(super) struct ReposQuery { + phase: Option, + cursor: Option, + limit: Option, +} + +pub(super) async fn backfill_repos( + State(state): State, + Path(job_id): Path, + auth: UserAuth, + axum::extract::Query(query): axum::extract::Query, +) -> Result, AppError> { + auth.require(Permission::BackfillRead).await?; + + let limit = query.limit.unwrap_or(50).min(100); + let phase_filter = match query.phase.as_deref() { + Some("resolved") => " AND pds_endpoint IS NOT NULL", + Some("fetched") => " AND status = 'completed'", + _ => "", + }; + let cursor_filter = if query.cursor.is_some() { + " AND did > ?" + } else { + "" + }; + + let sql_str = format!( + "SELECT did, pds_endpoint, status, records_fetched FROM backfill_repos WHERE job_id = ?{phase_filter}{cursor_filter} ORDER BY did ASC LIMIT ?", + ); + let sql = adapt_sql(&sql_str, state.db_backend); + + let mut q = sqlx::query_as::<_, (String, Option, String, i32)>(&sql).bind(&job_id); + if let Some(ref cursor) = query.cursor { + q = q.bind(cursor); + } + q = q.bind(limit + 1); + + let rows: Vec<(String, Option, String, i32)> = q + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill repos: {e}")))?; + + let has_more = rows.len() > limit as usize; + let repos: Vec = rows + .into_iter() + .take(limit as usize) + .map( + |(did, pds_endpoint, status, records_fetched)| super::types::BackfillRepoEntry { + did, + pds_endpoint, + status, + records_fetched, + }, + ) + .collect(); + + let cursor = if has_more { + repos.last().map(|r| r.did.clone()) + } else { + None + }; + + Ok(Json(super::types::BackfillReposResponse { repos, cursor })) +} + +pub(super) async fn backfill_pds_summary( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result, AppError> { + auth.require(Permission::BackfillRead).await?; + + let sql = adapt_sql( + "SELECT pds_endpoint, COUNT(*) as total_repos, SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed_repos, SUM(records_fetched) as total_records FROM backfill_repos WHERE job_id = ? AND pds_endpoint IS NOT NULL GROUP BY pds_endpoint ORDER BY COUNT(*) DESC", + state.db_backend, + ); + + let rows: Vec<(String, i32, i32, i64)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query PDS summary: {e}")))?; + + let pds_endpoints: Vec = rows + .into_iter() + .map( + |(pds_endpoint, total_repos, completed_repos, total_records)| { + super::types::PdsSummaryEntry { + pds_endpoint, + total_repos, + completed_repos, + total_records: total_records as i32, + } + }, + ) + .collect(); + + Ok(Json(super::types::PdsSummaryResponse { pds_endpoints })) +} + +// --------------------------------------------------------------------------- +// Flush endpoints +// --------------------------------------------------------------------------- + +pub(super) async fn flush_backfill_details( + State(state): State, + Path(job_id): Path, + auth: UserAuth, +) -> Result { + auth.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql).bind(&job_id).execute(&state.db).await; + + Ok(StatusCode::NO_CONTENT) +} + +pub(super) async fn flush_all_backfill_details( + State(state): State, + auth: UserAuth, +) -> Result { + auth.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE status IN ('completed', 'cancelled', 'failed'))", + state.db_backend, + ); + let _ = sqlx::query(&sql).execute(&state.db).await; + + Ok(StatusCode::NO_CONTENT) +} + +// --------------------------------------------------------------------------- +// Retention cleanup +// --------------------------------------------------------------------------- + +pub async fn run_backfill_retention_cleanup(state: &AppState) { + use super::settings::get_setting; + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(86400)); + interval.tick().await; // first tick is immediate — run once on startup + + loop { + interval.tick().await; + + let retention_days: i64 = + get_setting(&state.db, "backfill_retention_days", state.db_backend) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(28); + + if retention_days == 0 { + continue; + } + + let cutoff = chrono::Utc::now() - chrono::Duration::days(retention_days); + let cutoff_str = cutoff.to_rfc3339(); + + let sql = adapt_sql( + "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE completed_at IS NOT NULL AND completed_at < ?)", + state.db_backend, + ); + match sqlx::query(&sql).bind(&cutoff_str).execute(&state.db).await { + Ok(result) => { + let deleted = result.rows_affected(); + if deleted > 0 { + tracing::info!( + deleted, + retention_days, + "cleaned up old backfill detail rows" + ); + } + } + Err(e) => { + tracing::warn!(error = %e, "backfill retention cleanup failed"); + } + } + } +} + // --------------------------------------------------------------------------- // Startup resumption // --------------------------------------------------------------------------- diff --git a/src/admin/mod.rs b/src/admin/mod.rs index fa7722a..5cba8b4 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -37,7 +37,21 @@ pub fn admin_routes(_state: AppState) -> Router { .route("/stats", get(stats::stats)) .route("/backfill", post(backfill::create_backfill)) .route("/backfill/status", get(backfill::backfill_status)) + .route( + "/backfill/details", + delete(backfill::flush_all_backfill_details), + ) .route("/backfill/{id}/cancel", post(backfill::cancel_backfill)) + .route("/backfill/{id}/events", get(backfill::backfill_events)) + .route("/backfill/{id}/repos", get(backfill::backfill_repos)) + .route( + "/backfill/{id}/pds-summary", + get(backfill::backfill_pds_summary), + ) + .route( + "/backfill/{id}/details", + delete(backfill::flush_backfill_details), + ) .route("/events", get(events::list_events)) .route("/users", post(users::create_user).get(users::list_users)) .route("/users/transfer-super", post(users::transfer_super)) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index f86c065..8c6a138 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -16,6 +16,7 @@ use super::types::{SettingEntry, UpsertSettingBody}; const ENV_FALLBACKS: &[(&str, &str)] = &[ ("app_name", "APP_NAME"), + ("backfill_retention_days", "BACKFILL_RETENTION_DAYS"), ("client_uri", "CLIENT_URI"), ("feature.spaces_enabled", "FEATURE_SPACES_ENABLED"), ("logo_uri", "LOGO_URI"), diff --git a/src/admin/types.rs b/src/admin/types.rs index 3a5c0d1..2e903a8 100644 --- a/src/admin/types.rs +++ b/src/admin/types.rs @@ -85,6 +85,77 @@ pub(crate) struct BackfillJob { pub(crate) created_at: String, } +// --------------------------------------------------------------------------- +// Backfill event types +// --------------------------------------------------------------------------- + +#[derive(Clone, Debug, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BackfillEvent { + RepoDiscovered { + job_id: String, + did: String, + }, + RepoResolved { + job_id: String, + did: String, + pds_endpoint: String, + }, + RepoFetched { + job_id: String, + did: String, + pds_endpoint: String, + records_fetched: i32, + }, + JobCounters { + job_id: String, + total_repos: Option, + resolved_repos: Option, + processed_repos: Option, + total_records: Option, + }, + JobStageChanged { + job_id: String, + stage: String, + }, + JobCompleted { + job_id: String, + status: String, + error: Option, + }, +} + +// --------------------------------------------------------------------------- +// Backfill detail response types +// --------------------------------------------------------------------------- + +#[derive(Serialize)] +pub(crate) struct BackfillRepoEntry { + pub(crate) did: String, + pub(crate) pds_endpoint: Option, + pub(crate) status: String, + pub(crate) records_fetched: i32, +} + +#[derive(Serialize)] +pub(crate) struct BackfillReposResponse { + pub(crate) repos: Vec, + pub(crate) cursor: Option, +} + +#[derive(Serialize)] +pub(crate) struct PdsSummaryEntry { + pub(crate) pds_endpoint: String, + pub(crate) total_repos: i32, + pub(crate) completed_repos: i32, + pub(crate) total_records: i32, +} + +#[derive(Serialize)] +pub(crate) struct PdsSummaryResponse { + pub(crate) pds_endpoints: Vec, +} + // --------------------------------------------------------------------------- // Network lexicon types // --------------------------------------------------------------------------- diff --git a/src/lib.rs b/src/lib.rs index fe19b38..9ea8e48 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,6 +78,7 @@ pub struct AppState { pub official_registry: SharedRegistry, pub official_registry_config: RegistryConfig, pub proxy_config: Arc>, + pub backfill_events_tx: tokio::sync::broadcast::Sender, } impl axum::extract::FromRef for axum_extra::extract::cookie::Key { diff --git a/src/lua/atproto_api.rs b/src/lua/atproto_api.rs index e70e488..560e638 100644 --- a/src/lua/atproto_api.rs +++ b/src/lua/atproto_api.rs @@ -582,6 +582,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 5899bbd..f22725b 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -790,6 +790,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 2e32cb9..97f5e6b 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -1180,6 +1180,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/http_api.rs b/src/lua/http_api.rs index 39b7a22..2c20331 100644 --- a/src/lua/http_api.rs +++ b/src/lua/http_api.rs @@ -188,6 +188,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 308be59..6cb7860 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -292,6 +292,7 @@ mod tests { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( crate::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/src/main.rs b/src/main.rs index 16f92fe..595bb0a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -612,6 +612,8 @@ async fn main() { std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new(config))) }; + let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let state = AppState { config: config.clone(), http, @@ -631,6 +633,7 @@ async fn main() { official_registry, official_registry_config, proxy_config, + backfill_events_tx, }; jetstream::spawn(state.clone(), collections_rx); @@ -646,6 +649,13 @@ async fn main() { happyview::admin::backfill::resume_backfill_jobs(&state).await; + { + let state = state.clone(); + tokio::spawn(async move { + happyview::admin::backfill::run_backfill_retention_cleanup(&state).await; + }); + } + let app = server::router(state); let addr = config.listen_addr(); diff --git a/tests/common/app.rs b/tests/common/app.rs index f39914b..2331947 100644 --- a/tests/common/app.rs +++ b/tests/common/app.rs @@ -167,6 +167,7 @@ impl TestApp { proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, }; let router = server::router(state.clone()).layer(axum::middleware::from_fn( diff --git a/tests/lua_atproto_api.rs b/tests/lua_atproto_api.rs index 4946ced..459a9b6 100644 --- a/tests/lua_atproto_api.rs +++ b/tests/lua_atproto_api.rs @@ -101,6 +101,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index 1eb8de3..cd6a621 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -104,6 +104,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( happyview::proxy_config::ProxyConfig::default(), ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, } } diff --git a/web/package-lock.json b/web/package-lock.json index ed30f1a..df10472 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,6 +13,7 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", + "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -4143,6 +4144,23 @@ "react-dom": ">=16.8" } }, + "node_modules/@tanstack/react-virtual": { + "version": "3.13.25", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.25.tgz", + "integrity": "sha512-bmNoqMu6gcAW9JGrKVB0Q1tN1i5RONZF8r1fW0bbE4Oyf3DwEGnzzQJ2OW+Ozg1P4s8PyugkHg2ULZoFQN+cqw==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.15.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/@tanstack/table-core": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", @@ -4156,6 +4174,16 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tanstack/virtual-core": { + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.15.0.tgz", + "integrity": "sha512-0AwPGx0I8QxPYjAxShT/+z+ZOe9u8mW5rsXvivCTjRfRmz9a43+3mRyi4wwlyoUqOC56q/jatKa0Bh9M99BEHQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", diff --git a/web/package.json b/web/package.json index 6286c50..517f9d7 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", + "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index e3f1060..3331075 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -1,18 +1,44 @@ "use client"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCurrentUser } from "@/hooks/use-current-user"; import { cancelBackfillJob, createBackfillJob, getBackfillJobs, + getBackfillRepos, + getBackfillPdsSummary, + flushBackfillDetails, + flushAllBackfillDetails, getLexicons, } from "@/lib/api"; -import type { BackfillJob } from "@/types/backfill"; -import { CheckCircle2, Circle, Loader2 } from "lucide-react"; +import type { + BackfillJob, + BackfillRepoEntry, + PdsSummaryEntry, + BackfillEvent, + BlueskyProfile, +} from "@/types/backfill"; +import { CheckCircle2, ChevronRight, Circle, Loader2 } from "lucide-react"; import { SiteHeader } from "@/components/site-header"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; import { Badge } from "@/components/ui/badge"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; import { Button } from "@/components/ui/button"; import { Combobox, @@ -97,6 +123,257 @@ function phaseIndex(stage: string): number { return idx; } +// SSE hook for backfill events +function useBackfillSSE(jobId: string | null, active: boolean): BackfillEvent[] { + const [events, setEvents] = useState([]); + + useEffect(() => { + if (!jobId || !active) { + setEvents([]); + return; + } + + const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ""; + const es = new EventSource(`${basePath}/admin/backfill/${jobId}/events`, { + withCredentials: true, + }); + + es.addEventListener("event", (e) => { + try { + const event: BackfillEvent = JSON.parse((e as MessageEvent).data); + setEvents((prev) => [...prev, event]); + } catch { /* ignore parse errors */ } + }); + + return () => es.close(); + }, [jobId, active]); + + return events; +} + +// Batch Bluesky profile resolution hook +function useBlueskyProfiles(dids: string[]): Map { + const [profiles, setProfiles] = useState>(new Map()); + const resolvedRef = useRef>(new Set()); + const pendingRef = useRef(false); + + useEffect(() => { + const unresolved = dids.filter((d) => !resolvedRef.current.has(d)); + if (unresolved.length === 0 || pendingRef.current) return; + + pendingRef.current = true; + + const batches: string[][] = []; + for (let i = 0; i < unresolved.length; i += 25) { + batches.push(unresolved.slice(i, i + 25)); + } + + // Mark all as resolved immediately to prevent re-fetching + for (const did of unresolved) { + resolvedRef.current.add(did); + } + + Promise.all( + batches.map(async (batch) => { + const params = batch.map((d) => `actors=${encodeURIComponent(d)}`).join("&"); + try { + const resp = await fetch( + `https://public.api.bsky.app/xrpc/app.bsky.actor.getProfiles?${params}` + ); + if (!resp.ok) return []; + const data = await resp.json(); + return (data.profiles || []) as BlueskyProfile[]; + } catch { + return []; + } + }) + ).then((results) => { + setProfiles((prev) => { + const newProfiles = new Map(prev); + for (const batch of results) { + for (const p of batch) { + newProfiles.set(p.did, p); + } + } + return newProfiles; + }); + pendingRef.current = false; + }); + }, [dids]); + + return profiles; +} + +const BSKY_PDS_SUFFIX = ".bsky.network"; +const BSKY_PDS_HOSTNAMES = ["bsky.social", "staging.bsky.dev"]; +const failedFaviconUrls = new Set(); + +function isBskyPds(pdsEndpoint: string): boolean { + try { + const hostname = new URL(pdsEndpoint).hostname; + return BSKY_PDS_HOSTNAMES.includes(hostname) || hostname.endsWith(BSKY_PDS_SUFFIX); + } catch { + return false; + } +} + +function PdsFavicon({ pdsEndpoint }: { pdsEndpoint: string }) { + let hostname: string; + try { + hostname = new URL(pdsEndpoint).hostname; + } catch { + return ; + } + + if (isBskyPds(pdsEndpoint)) { + return ( + + + + ); + } + + const faviconUrl = `https://twenty-icons.com/${hostname}`; + + if (failedFaviconUrls.has(faviconUrl)) { + return ; + } + + return ; +} + +function PdsFaviconImg({ url }: { url: string }) { + const [failed, setFailed] = useState(false); + + if (failed) return ; + + return ( + { failedFaviconUrls.add(url); setFailed(true); }} + /> + ); +} + +function PdsPlaceholderIcon() { + return ( + + + + + + + + ); +} + +function ScrollSentinel({ onVisible }: { onVisible: () => void }) { + const ref = useRef(null); + const onVisibleRef = useRef(onVisible); + onVisibleRef.current = onVisible; + + useEffect(() => { + const el = ref.current; + if (!el) return; + const observer = new IntersectionObserver( + ([entry]) => { if (entry.isIntersecting) onVisibleRef.current(); }, + { rootMargin: "100px" }, + ); + observer.observe(el); + return () => observer.disconnect(); + }, []); + + return
; +} + +function AnimatedNumber({ value }: { value: number }) { + const targetRef = useRef(value); + const displayedRef = useRef(value); + const [displayed, setDisplayed] = useState(value); + const rafRef = useRef(0); + + targetRef.current = value; + + useEffect(() => { + cancelAnimationFrame(rafRef.current); + + function tick() { + const current = displayedRef.current; + const target = targetRef.current; + const diff = target - current; + + if (Math.abs(diff) < 0.5) { + displayedRef.current = target; + setDisplayed(target); + return; + } + + const next = current + diff * 0.06; + displayedRef.current = next; + setDisplayed(next); + rafRef.current = requestAnimationFrame(tick); + } + + tick(); + return () => cancelAnimationFrame(rafRef.current); + }, [value]); + + return <>{Math.round(displayed).toLocaleString()}; +} + +function CompactRepoRow({ did, profile }: { + did: string; + profile?: BlueskyProfile; +}) { + return ( +
+
+ {profile?.avatar && ( + + )} +
+ + {profile?.handle ? `@${profile.handle}` : {did}} + +
+ ); +} + +function ProfileRow({ did, profile, suffix }: { + did: string; + profile?: BlueskyProfile; + suffix?: React.ReactNode; +}) { + return ( +
+
+ {profile?.avatar && ( + + )} +
+
+

+ {profile?.displayName || profile?.handle || did} +

+ {profile?.handle && ( +

@{profile.handle}

+ )} + {!profile?.handle && ( +

{did}

+ )} +
+ {suffix && ( + {suffix} + )} +
+ ); +} + export default function BackfillPage() { const { hasPermission } = useCurrentUser(); const [jobs, setJobs] = useState([]); @@ -119,6 +396,7 @@ export default function BackfillPage() { }, [load]); const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + const canFlush = hasPermission("backfill:create"); return ( <> @@ -128,9 +406,32 @@ export default function BackfillPage() {

Backfill Jobs

- {hasPermission("backfill:create") && ( - - )} +
+ {canFlush && ( + + + + + + + Clear all job details? + + This will permanently delete per-repo detail data for all backfill jobs. + + + + Cancel + { + await flushAllBackfillDetails(); + }}>Clear + + + + )} + {hasPermission("backfill:create") && ( + + )} +
@@ -201,6 +502,7 @@ export default function BackfillPage() { { await cancelBackfillJob(selectedJob.id); load(); @@ -217,10 +519,12 @@ export default function BackfillPage() { function JobDetail({ job, canCancel, + canFlush, onCancel, }: { job: BackfillJob; canCancel: boolean; + canFlush: boolean; onCancel: () => Promise; }) { const [cancelling, setCancelling] = useState(false); @@ -228,6 +532,21 @@ function JobDetail({ const allDone = job.status === "completed"; const isActive = job.status === "running" || job.status === "cancelling"; + // Detail data state + const [discoveredRepos, setDiscoveredRepos] = useState([]); + const [discoveredCursor, setDiscoveredCursor] = useState(null); + const [discoveredLoaded, setDiscoveredLoaded] = useState(false); + + const [pdsSummary, setPdsSummary] = useState([]); + const [pdsLoaded, setPdsLoaded] = useState(false); + + const [fetchedRepos, setFetchedRepos] = useState([]); + const [fetchedCursor, setFetchedCursor] = useState(null); + const [fetchedLoaded, setFetchedLoaded] = useState(false); + + // SSE events for active jobs + const sseEvents = useBackfillSSE(job.id, isActive); + function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; if (job.stage === "resolving_and_fetching") { @@ -245,6 +564,102 @@ function JobDetail({ } } + // Auto-load detail data when phases are reached + const discoveredReached = hasReached("discovering_repos"); + const pdsReached = hasReached("resolving_pds"); + const fetchedReached = hasReached("fetching_records") || job.stage === "resolving_and_fetching"; + + useEffect(() => { + if (discoveredReached && !discoveredLoaded) { + getBackfillRepos(job.id, { phase: "discovered", limit: 50 }) + .then((resp) => { setDiscoveredRepos(resp.repos); setDiscoveredCursor(resp.cursor); setDiscoveredLoaded(true); }) + .catch(() => {}); + } + }, [job.id, discoveredReached, discoveredLoaded]); + + useEffect(() => { + if (pdsReached && !pdsLoaded) { + getBackfillPdsSummary(job.id) + .then((resp) => { setPdsSummary(resp.pds_endpoints); setPdsLoaded(true); }) + .catch(() => {}); + } + }, [job.id, pdsReached, pdsLoaded]); + + useEffect(() => { + if (fetchedReached && !fetchedLoaded) { + getBackfillRepos(job.id, { phase: "fetched", limit: 50 }) + .then((resp) => { setFetchedRepos(resp.repos); setFetchedCursor(resp.cursor); setFetchedLoaded(true); }) + .catch(() => {}); + } + }, [job.id, fetchedReached, fetchedLoaded]); + + const loadMoreDiscovered = useCallback(async () => { + if (!discoveredCursor) return; + try { + const resp = await getBackfillRepos(job.id, { phase: "discovered", cursor: discoveredCursor, limit: 50 }); + setDiscoveredRepos((prev) => [...prev, ...resp.repos]); + setDiscoveredCursor(resp.cursor); + } catch { /* ignore */ } + }, [job.id, discoveredCursor]); + + const loadMoreFetched = useCallback(async () => { + if (!fetchedCursor) return; + try { + const resp = await getBackfillRepos(job.id, { phase: "fetched", cursor: fetchedCursor, limit: 50 }); + setFetchedRepos((prev) => [...prev, ...resp.repos]); + setFetchedCursor(resp.cursor); + } catch { /* ignore */ } + }, [job.id, fetchedCursor]); + + // Process SSE events + useEffect(() => { + for (const event of sseEvents) { + if (event.type === "repo_discovered" && event.did) { + setDiscoveredRepos((prev) => { + if (prev.some((r) => r.did === event.did)) return prev; + return [{ did: event.did!, pds_endpoint: null, status: "pending", records_fetched: 0 }, ...prev]; + }); + } + if (event.type === "repo_resolved" && event.did && event.pds_endpoint) { + setDiscoveredRepos((prev) => + prev.map((r) => r.did === event.did ? { ...r, pds_endpoint: event.pds_endpoint! } : r) + ); + setPdsSummary((prev) => { + const idx = prev.findIndex((p) => p.pds_endpoint === event.pds_endpoint); + if (idx >= 0) { + const updated = [...prev]; + updated[idx] = { ...updated[idx], total_repos: updated[idx].total_repos + 1 }; + return updated; + } + return [...prev, { pds_endpoint: event.pds_endpoint!, total_repos: 1, completed_repos: 0, total_records: 0 }]; + }); + } + if (event.type === "repo_fetched" && event.did) { + setFetchedRepos((prev) => { + if (prev.some((r) => r.did === event.did)) return prev; + return [{ did: event.did!, pds_endpoint: event.pds_endpoint ?? null, status: "completed", records_fetched: event.records_fetched ?? 0 }, ...prev]; + }); + setPdsSummary((prev) => { + if (!event.pds_endpoint) return prev; + return prev.map((p) => p.pds_endpoint === event.pds_endpoint + ? { ...p, completed_repos: p.completed_repos + 1, total_records: p.total_records + (event.records_fetched ?? 0) } + : p + ); + }); + } + } + }, [sseEvents]); + + // Collect all visible DIDs for profile resolution + const allVisibleDids = useMemo(() => { + const dids = new Set(); + for (const r of discoveredRepos) dids.add(r.did); + for (const r of fetchedRepos) dids.add(r.did); + return Array.from(dids); + }, [discoveredRepos, fetchedRepos]); + + const profiles = useBlueskyProfiles(allVisibleDids); + return ( <> @@ -306,9 +721,27 @@ function JobDetail({ label="Discovering repos" active={isActive && job.stage === "discovering_repos"} reached={hasReached("discovering_repos")} - value={job.total_repos?.toLocaleString()} + value={job.total_repos != null ? : undefined} suffix="repos found" - /> + loading={discoveredReached && !discoveredLoaded} + > + {discoveredRepos.length > 0 ? ( +
+ {discoveredRepos.map((repo) => ( + + ))} + {discoveredCursor && ( + + )} +
+ ) : discoveredLoaded ? ( +

No repos discovered yet.

+ ) : null} + / : undefined } suffix="resolved" - /> + loading={pdsReached && !pdsLoaded} + > + {pdsSummary.length > 0 ? ( +
+ {pdsSummary + .sort((a, b) => b.total_repos - a.total_repos) + .map((pds) => ( +
+ + {new URL(pds.pds_endpoint).hostname} + + / repos · records + +
+ ))} +
+ ) : pdsLoaded ? ( +

No PDS data yet.

+ ) : null} + / repos : undefined } suffix={ hasReached("fetching_records") || job.stage === "resolving_and_fetching" - ? `${job.total_records?.toLocaleString() ?? "0"} records` + ? <> records : undefined } - /> + loading={fetchedReached && !fetchedLoaded} + > + {fetchedRepos.filter((r) => r.records_fetched > 0).length > 0 ? ( +
+ {fetchedRepos.filter((r) => r.records_fetched > 0).map((repo) => ( + records} + /> + ))} + {fetchedCursor && ( + + )} +
+ ) : fetchedLoaded ? ( +

No repos fetched yet.

+ ) : null} +
- {canCancel && isActive && ( - + + {canFlush && !isActive && ( + + + + + + + Clear job details? + + This will permanently delete per-repo detail data for this backfill job. + + + + Cancel + { + await flushBackfillDetails(job.id); + setDiscoveredRepos([]); + setDiscoveredCursor(null); + setDiscoveredLoaded(false); + setPdsSummary([]); + setPdsLoaded(false); + setFetchedRepos([]); + setFetchedCursor(null); + setFetchedLoaded(false); + }}>Clear + + + + )} + {canCancel && isActive && ( - - )} + )} + ); } @@ -370,42 +870,64 @@ function ProgressRow({ reached, value, suffix, + loading, + children, }: { label: string; active: boolean; reached: boolean; - value?: string; - suffix?: string; + value?: React.ReactNode; + suffix?: React.ReactNode; + loading?: boolean; + children?: React.ReactNode; }) { + const [open, setOpen] = useState(false); const done = reached && !active; + const expandable = reached; return ( -
- - {active ? ( - - ) : done ? ( - - ) : ( - - )} - - {label} - {reached && value && ( - - {value} - {suffix ? ` · ${suffix}` : ""} - + + +
+ + {active ? ( + + ) : done ? ( + + ) : ( + + )} + + {label} + {reached && value && ( + + {value} + {suffix && <> · {suffix}} + + )} + {expandable && ( + + )} +
+ + {expandable && ( + +
+ {loading ? ( +
+ +
+ ) : children} +
+ )} -
+ ); } diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index 731ac34..9d04415 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -23,6 +23,7 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", + "backfill_retention_days", ] as const type FieldKey = (typeof SETTING_KEYS)[number] @@ -80,6 +81,7 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", + backfill_retention_days: "28", }) const [sources, setSources] = useState>({ app_name: "unset", @@ -87,6 +89,7 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", + backfill_retention_days: "unset", }) const [logoUploaded, setLogoUploaded] = useState(false) const [error, setError] = useState(null) @@ -104,6 +107,7 @@ export default function GeneralSettingsPage() { logo_uri: byKey.get("logo_uri")?.value ?? "", tos_uri: byKey.get("tos_uri")?.value ?? "", policy_uri: byKey.get("policy_uri")?.value ?? "", + backfill_retention_days: byKey.get("backfill_retention_days")?.value ?? "28", }) setSources({ app_name: (byKey.get("app_name")?.source as "database" | "env" | undefined) ?? "unset", @@ -111,6 +115,7 @@ export default function GeneralSettingsPage() { logo_uri: (byKey.get("logo_uri")?.source as "database" | "env" | undefined) ?? "unset", tos_uri: (byKey.get("tos_uri")?.source as "database" | "env" | undefined) ?? "unset", policy_uri: (byKey.get("policy_uri")?.source as "database" | "env" | undefined) ?? "unset", + backfill_retention_days: (byKey.get("backfill_retention_days")?.source as "database" | "env" | undefined) ?? "unset", }) setLogoUploaded(byKey.has("logo_data")) } catch (e: unknown) { @@ -137,6 +142,14 @@ export default function GeneralSettingsPage() { await upsertSetting(field.key, value) } } + const retentionValue = values["backfill_retention_days"] + if (retentionValue === "") { + if (sources["backfill_retention_days"] === "database") { + await deleteSetting("backfill_retention_days") + } + } else { + await upsertSetting("backfill_retention_days", retentionValue) + } setNotice("Settings saved.") await load() } catch (e: unknown) { @@ -252,6 +265,37 @@ export default function GeneralSettingsPage() { +
+

Data Retention

+

+ Configure how long HappyView retains detailed data from completed backfill jobs. +

+
+ +
+
+ + {sources["backfill_retention_days"] === "env" && ( + + from env var + + )} +
+ + setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) + } + placeholder="28" + disabled={!canManage} + /> +

+ How long to keep per-repo detail data from completed backfill jobs. Set to 0 to keep indefinitely. +

+
+
+
+

Backfill Performance

+

+ Tune concurrency limits for backfill jobs. Changes apply to the next job started. +

+
+ + {([ + { key: "backfill_concurrent_resolution" as const, id: "backfill_concurrent_resolution", label: "Concurrent PLC Resolutions", placeholder: "100", description: "How many DID document lookups to run in parallel during PDS resolution." }, + { key: "backfill_concurrent_pds" as const, id: "backfill_concurrent_pds", label: "Concurrent PDS Hosts", placeholder: "10", description: "How many PDS servers to fetch records from simultaneously." }, + { key: "backfill_concurrent_dids_per_pds" as const, id: "backfill_concurrent_dids_per_pds", label: "Concurrent DIDs per PDS", placeholder: "3", description: "How many repos to fetch concurrently from each PDS host." }, + ]).map((field) => ( +
+
+ + {sources[field.key] === "env" && ( + from env var + )} +
+ setValues((v) => ({ ...v, [field.key]: e.target.value }))} + placeholder={field.placeholder} + disabled={!canManage} + /> +

{field.description}

+
+ ))} +
{([ @@ -355,7 +391,7 @@ export default function GeneralSettingsPage() {
diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 4438ace..1c43284 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -365,6 +365,17 @@ export function getSettings() { return apiFetch("/admin/settings"); } +export type DbInfo = { + backend: "sqlite" | "postgres"; + server_max_connections: number | null; + main_pool_size: number; + backfill_pool_size: number; +}; + +export function getDbInfo() { + return apiFetch("/admin/settings/db-info"); +} + export function upsertSetting(key: string, value: string) { return apiFetch(`/admin/settings/${encodeURIComponent(key)}`, { method: "PUT", -- 2.51.2 From 7ff501afe91b8f45e0c7c34bf016a989ae0d36f5 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 01:20:23 -0500 Subject: [PATCH 11/64] feat: increase backfill record batch inserts Signed-off-by: Trezy --- src/admin/backfill.rs | 131 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 9 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 97b4a61..27b2b01 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -20,7 +20,6 @@ use crate::error::AppError; use crate::event_log::{EventLog, Severity, log_event}; use crate::http_retry::parse_retry_after; use crate::profile; -use crate::record_handler::{self, RecordEvent}; use super::auth::UserAuth; use super::permissions::Permission; @@ -1108,6 +1107,95 @@ async fn run_fetching_phase( (final_repos, final_records) } +struct PreparedRecord { + uri: String, + did: String, + collection: String, + rkey: String, + record_json: String, + cid: String, +} + +async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { + if batch.is_empty() { + return; + } + + let backend = state.db_backend; + let now = now_rfc3339(); + + // Build multi-row INSERT. 8 params per row; ON CONFLICT uses EXCLUDED. + let placeholders: Vec = (0..batch.len()) + .map(|_| "(?, ?, ?, ?, ?, ?, ?, ?)".to_string()) + .collect(); + let raw_sql = format!( + "INSERT INTO records (uri, did, collection, rkey, record, cid, indexed_at, created_at) VALUES {} ON CONFLICT (uri) DO UPDATE SET record = EXCLUDED.record, cid = EXCLUDED.cid, indexed_at = EXCLUDED.indexed_at", + placeholders.join(", ") + ); + let sql = adapt_sql(&raw_sql, backend); + + let mut query = sqlx::query(&sql); + for rec in batch { + query = query + .bind(&rec.uri) + .bind(&rec.did) + .bind(&rec.collection) + .bind(&rec.rkey) + .bind(&rec.record_json) + .bind(&rec.cid) + .bind(&now) + .bind(&now); + } + + if let Err(e) = query.execute(&state.db).await { + tracing::warn!(batch_size = batch.len(), "batch record upsert failed: {e}"); + } + + // Batch sync_refs: delete old refs for all URIs, then insert new ones. + let uris: Vec<&str> = batch.iter().map(|r| r.uri.as_str()).collect(); + let delete_placeholders: Vec<&str> = (0..uris.len()).map(|_| "?").collect(); + let delete_raw = format!( + "DELETE FROM record_refs WHERE source_uri IN ({})", + delete_placeholders.join(", ") + ); + let delete_sql = adapt_sql(&delete_raw, backend); + let mut del_query = sqlx::query(&delete_sql); + for uri in &uris { + del_query = del_query.bind(*uri); + } + let _ = del_query.execute(&state.db).await; + + // Collect all new refs and batch insert them + let mut all_refs: Vec<(&str, String, &str)> = Vec::new(); + for rec in batch { + let record_val: serde_json::Value = + serde_json::from_str(&rec.record_json).unwrap_or_default(); + for target_uri in crate::record_refs::extract_at_uris(&record_val) { + all_refs.push((&rec.uri, target_uri, &rec.collection)); + } + } + + // Insert refs in chunks to stay within SQLite's param limit (3 params per ref) + for chunk in all_refs.chunks(300) { + let ref_placeholders: Vec<&str> = (0..chunk.len()).map(|_| "(?, ?, ?)").collect(); + let ref_raw = format!( + "INSERT INTO record_refs (source_uri, target_uri, collection) VALUES {} ON CONFLICT DO NOTHING", + ref_placeholders.join(", ") + ); + let ref_sql = adapt_sql(&ref_raw, backend); + let mut ref_query = sqlx::query(&ref_sql); + for (source, target, collection) in chunk { + ref_query = ref_query.bind(*source).bind(target).bind(*collection); + } + let _ = ref_query.execute(&state.db).await; + } + + // Queue label backfill for each record + for rec in batch { + crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), rec.uri.clone()); + } +} + /// Fetch all records for a given DID and collection from a PDS via /// `com.atproto.repo.listRecords`, paginating and handling rate limits. async fn fetch_records_from_pds( @@ -1153,22 +1241,47 @@ async fn fetch_records_from_pds( let page_count = body.records.len(); + let mut batch: Vec = Vec::with_capacity(page_count); for entry in &body.records { let rkey = entry.uri.rsplit('/').next().unwrap_or_default().to_string(); + let uri = format!("at://{did}/{collection}/{rkey}"); + + let rec_to_store = if let Some(script) = state.lexicons.get_index_hook(collection).await + { + let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { + state, + lexicon_id: collection, + script: &script, + action: "create", + uri: &uri, + did, + collection, + rkey: &rkey, + record: Some(&entry.value), + }) + .await; + + match hook_result { + None => continue, + Some(v) => v, + } + } else { + entry.value.clone() + }; - let event = RecordEvent { + batch.push(PreparedRecord { + uri, did: did.to_string(), collection: collection.to_string(), rkey, - action: "create".to_string(), - record: Some(entry.value.clone()), - cid: Some(entry.cid.clone()), - }; - - record_handler::handle_record_event(state, &event).await; - count += 1; + record_json: serde_json::to_string(&rec_to_store).unwrap_or_default(), + cid: entry.cid.clone(), + }); } + count += batch.len() as u32; + batch_upsert_records(state, &batch).await; + match body.cursor { Some(c) if page_count > 0 => cursor = Some(c), _ => break, -- 2.51.2 From 060cba303f392233554f1744ea4714ec09c51fa5 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 01:23:16 -0500 Subject: [PATCH 12/64] feat: make backfill collection discovery concurrent Signed-off-by: Trezy --- src/admin/backfill.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 27b2b01..ba27f63 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -282,14 +282,16 @@ async fn run_discovery_phase( }, ); } else { - for collection in collections { - if is_cancelled(state, job_id).await { - return; - } - if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { - tracing::warn!(collection, error = %e, "failed to discover repos, skipping"); - } - } + stream::iter(collections.iter()) + .for_each_concurrent(collections.len(), |collection| async move { + if is_cancelled(state, job_id).await { + return; + } + if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { + tracing::warn!(collection, error = %e, "failed to discover repos, skipping"); + } + }) + .await; } let total = count_repos(state, job_id).await; -- 2.51.2 From 66b4804a7db626aabc27e354b5fb6da1c0ed4a9e Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:37:18 -0500 Subject: [PATCH 13/64] fix: get max connections from Postgres instead of current connections Signed-off-by: Trezy --- src/admin/settings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index de001a3..b91318e 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -200,8 +200,8 @@ pub(super) async fn db_info( None }; - let main_pool_size = state.db.size() as i64; - let backfill_pool_size = state.backfill_db.size() as i64; + let main_pool_size = state.db.options().get_max_connections() as i64; + let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; Ok(Json(serde_json::json!({ "backend": match state.db_backend { -- 2.51.2 From 22f3ba92acecba1c84b4355871eb617003f7a7ea Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:39:07 -0500 Subject: [PATCH 14/64] fix: use the correct connection pool for record upserts Signed-off-by: Trezy --- src/admin/backfill.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index ba27f63..73f5955 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1149,7 +1149,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { .bind(&now); } - if let Err(e) = query.execute(&state.db).await { + if let Err(e) = query.execute(&state.backfill_db).await { tracing::warn!(batch_size = batch.len(), "batch record upsert failed: {e}"); } @@ -1165,7 +1165,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { for uri in &uris { del_query = del_query.bind(*uri); } - let _ = del_query.execute(&state.db).await; + let _ = del_query.execute(&state.backfill_db).await; // Collect all new refs and batch insert them let mut all_refs: Vec<(&str, String, &str)> = Vec::new(); @@ -1189,7 +1189,7 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { for (source, target, collection) in chunk { ref_query = ref_query.bind(*source).bind(target).bind(*collection); } - let _ = ref_query.execute(&state.db).await; + let _ = ref_query.execute(&state.backfill_db).await; } // Queue label backfill for each record @@ -1700,7 +1700,7 @@ pub(super) async fn backfill_repos( q = q.bind(limit + 1); let rows: Vec<(String, Option, String, i32)> = q - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to query backfill repos: {e}")))?; @@ -1741,7 +1741,7 @@ pub(super) async fn backfill_pds_summary( let rows: Vec<(String, i32, i32, i64)> = sqlx::query_as(&sql) .bind(&job_id) - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to query PDS summary: {e}")))?; @@ -1777,7 +1777,10 @@ pub(super) async fn flush_backfill_details( "DELETE FROM backfill_repos WHERE job_id = ?", state.db_backend, ); - let _ = sqlx::query(&sql).bind(&job_id).execute(&state.db).await; + let _ = sqlx::query(&sql) + .bind(&job_id) + .execute(&state.backfill_db) + .await; Ok(StatusCode::NO_CONTENT) } @@ -1792,7 +1795,7 @@ pub(super) async fn flush_all_backfill_details( "DELETE FROM backfill_repos WHERE job_id IN (SELECT id FROM backfill_jobs WHERE status IN ('completed', 'cancelled', 'failed'))", state.db_backend, ); - let _ = sqlx::query(&sql).execute(&state.db).await; + let _ = sqlx::query(&sql).execute(&state.backfill_db).await; Ok(StatusCode::NO_CONTENT) } -- 2.51.2 From 9c2847ae2e3d3ff80117ee8cdd69341a9a17f471 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:39:50 -0500 Subject: [PATCH 15/64] fix: clamp backfill concurrency values Signed-off-by: Trezy --- src/admin/backfill.rs | 9 ++++++--- src/db.rs | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 73f5955..2e85569 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -134,11 +134,13 @@ async fn load_concurrency(state: &AppState) -> BackfillConcurrency { ) .await .and_then(|v| v.parse().ok()) - .unwrap_or(100); + .unwrap_or(100usize) + .max(1); let pds = super::settings::get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) .await .and_then(|v| v.parse().ok()) - .unwrap_or(10); + .unwrap_or(10usize) + .max(1); let dids_per_pds = super::settings::get_setting( &state.db, "backfill_concurrent_dids_per_pds", @@ -146,7 +148,8 @@ async fn load_concurrency(state: &AppState) -> BackfillConcurrency { ) .await .and_then(|v| v.parse().ok()) - .unwrap_or(3); + .unwrap_or(3usize) + .max(1); BackfillConcurrency { resolution, pds, diff --git a/src/db.rs b/src/db.rs index 638fc02..6eb5cbc 100644 --- a/src/db.rs +++ b/src/db.rs @@ -286,7 +286,7 @@ pub async fn connect(url: &str, backend: DatabaseBackend) -> AnyPool { pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { let max_connections: u32 = std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") .ok() - .and_then(|v| v.parse().ok()) + .and_then(|v| v.parse::().ok()) .unwrap_or_else(|| { let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") .ok() @@ -308,7 +308,8 @@ pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPo DatabaseBackend::Postgres => 256, }; needed.min(ceiling) - }); + }) + .max(1); tracing::info!(max_connections, "backfill pool sized"); -- 2.51.2 From bd745ae7d501251c40398065a40c7b9e69698502 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:41:12 -0500 Subject: [PATCH 16/64] fix: remove cleanup delay caused by tick eater Signed-off-by: Trezy --- src/admin/backfill.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 2e85569..5e8c230 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1811,7 +1811,6 @@ pub async fn run_backfill_retention_cleanup(state: &AppState) { use super::settings::get_setting; let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(86400)); - interval.tick().await; // first tick is immediate — run once on startup loop { interval.tick().await; -- 2.51.2 From 149d61eda9e81e4f38f9c3c05d6eb1080ef9658a Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:44:15 -0500 Subject: [PATCH 17/64] chore: remove unused module Signed-off-by: Trezy --- web/package-lock.json | 59 ++++++++++++++++++------------------------- web/package.json | 1 - 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/web/package-lock.json b/web/package-lock.json index df10472..65bd0ba 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -13,7 +13,6 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -117,6 +116,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -774,6 +774,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -2044,6 +2045,7 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -4144,23 +4146,6 @@ "react-dom": ">=16.8" } }, - "node_modules/@tanstack/react-virtual": { - "version": "3.13.25", - "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.13.25.tgz", - "integrity": "sha512-bmNoqMu6gcAW9JGrKVB0Q1tN1i5RONZF8r1fW0bbE4Oyf3DwEGnzzQJ2OW+Ozg1P4s8PyugkHg2ULZoFQN+cqw==", - "license": "MIT", - "dependencies": { - "@tanstack/virtual-core": "3.15.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - }, - "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" - } - }, "node_modules/@tanstack/table-core": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.21.3.tgz", @@ -4174,16 +4159,6 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, - "node_modules/@tanstack/virtual-core": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.15.0.tgz", - "integrity": "sha512-0AwPGx0I8QxPYjAxShT/+z+ZOe9u8mW5rsXvivCTjRfRmz9a43+3mRyi4wwlyoUqOC56q/jatKa0Bh9M99BEHQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/tannerlinsley" - } - }, "node_modules/@ts-morph/common": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.27.0.tgz", @@ -4368,6 +4343,7 @@ "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4377,6 +4353,7 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4387,6 +4364,7 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", + "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4410,8 +4388,7 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true, - "peer": true + "optional": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -4477,6 +4454,7 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -4990,6 +4968,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5376,6 +5355,7 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -5480,6 +5460,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6441,7 +6422,6 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", - "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -6780,6 +6760,7 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6920,6 +6901,7 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -7398,6 +7380,7 @@ "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -8196,6 +8179,7 @@ "integrity": "sha512-Eaw2YTGM6WOxA6CXbckaEvslr2Ne4NFsKrvc0v97JD5awbmeBLO5w9Ho9L9kmKonrwF9RJlW6BxT1PVv/agBHQ==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -9621,7 +9605,6 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", - "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -11778,6 +11761,7 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", + "peer": true, "engines": { "node": ">=0.10.0" } @@ -11808,6 +11792,7 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", + "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11854,6 +11839,7 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", + "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -11992,7 +11978,8 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -13187,7 +13174,8 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.0.tgz", "integrity": "sha512-yYzTZ4++b7fNYxFfpnberEEKu43w44aqDMNM9MHMmcKuCH7lL8jJ4yJ7LGHv7rSwiqM0nkiobF9I6cLlpS2P7Q==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/tapable": { "version": "2.3.0", @@ -13260,6 +13248,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -13537,6 +13526,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14286,6 +14276,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/web/package.json b/web/package.json index 517f9d7..6286c50 100644 --- a/web/package.json +++ b/web/package.json @@ -14,7 +14,6 @@ "@tabler/icons-react": "^3.36.1", "@tailwindcss/typography": "^0.5.19", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.25", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", -- 2.51.2 From ce68c6694b2cc0f5d2789b938133577c5bc1de6b Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:44:56 -0500 Subject: [PATCH 18/64] fix: prevent reprocessing of sse Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 3331075..b3e59ce 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -546,6 +546,7 @@ function JobDetail({ // SSE events for active jobs const sseEvents = useBackfillSSE(job.id, isActive); + const sseProcessedRef = useRef(0); function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; @@ -611,9 +612,14 @@ function JobDetail({ } catch { /* ignore */ } }, [job.id, fetchedCursor]); - // Process SSE events + // Process SSE events — only handle events added since last render useEffect(() => { - for (const event of sseEvents) { + const start = sseProcessedRef.current; + if (start >= sseEvents.length) return; + sseProcessedRef.current = sseEvents.length; + + for (let i = start; i < sseEvents.length; i++) { + const event = sseEvents[i]; if (event.type === "repo_discovered" && event.did) { setDiscoveredRepos((prev) => { if (prev.some((r) => r.did === event.did)) return prev; -- 2.51.2 From 3d31877d207916c2170150357e423d237f793eac Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:45:41 -0500 Subject: [PATCH 19/64] fix: prevent concurrency settings from being set to invalid values Signed-off-by: Trezy --- web/src/app/dashboard/settings/general/page.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index c7924a8..bc0180e 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -336,6 +336,8 @@ export default function GeneralSettingsPage() { setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) @@ -379,6 +381,8 @@ export default function GeneralSettingsPage() { setValues((v) => ({ ...v, [field.key]: e.target.value }))} placeholder={field.placeholder} -- 2.51.2 From f83f15009187196dadb81fbf86f213ec2981ec9b Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 09:46:39 -0500 Subject: [PATCH 20/64] fix: make backfill progress headings more accessible Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index b3e59ce..c556d9c 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -894,8 +894,9 @@ function ProgressRow({ return ( -
)} -
+ {expandable && ( -- 2.51.2 From bd65caf32eafcc42ef65aae68559bae03b9f22a4 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 20:14:07 -0500 Subject: [PATCH 21/64] feat: allow backfill jobs to be paused and resumed Signed-off-by: Trezy --- src/admin/backfill.rs | 296 +++++++++++++++--- src/admin/mod.rs | 2 + src/admin/settings.rs | 24 ++ web/package-lock.json | 31 +- web/src/app/dashboard/backfill/page.tsx | 107 ++++++- .../app/dashboard/settings/general/page.tsx | 248 +++++++++------ web/src/lib/api.ts | 14 + 7 files changed, 558 insertions(+), 164 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 5e8c230..4e1165d 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -179,23 +179,32 @@ async fn fail_job(state: &AppState, job_id: &str, error: &str) { ); } -async fn is_cancelled(state: &AppState, job_id: &str) -> bool { +async fn should_stop(state: &AppState, job_id: &str) -> Option<&'static str> { let sql = adapt_sql( "SELECT status FROM backfill_jobs WHERE id = ?", state.db_backend, ); - sqlx::query_as::<_, (String,)>(&sql) + let status = sqlx::query_as::<_, (String,)>(&sql) .bind(job_id) .fetch_optional(&state.backfill_db) .await .ok() .flatten() - .is_some_and(|(status,)| status == "cancelling") + .map(|(s,)| s); + match status.as_deref() { + Some("cancelling") => Some("cancelling"), + Some("pausing") => Some("pausing"), + _ => None, + } +} + +async fn should_stop_worker(state: &AppState, job_id: &str) -> bool { + should_stop(state, job_id).await.is_some() } async fn request_cancel(state: &AppState, job_id: &str) { let sql = adapt_sql( - "UPDATE backfill_jobs SET status = 'cancelling' WHERE id = ? AND status = 'running'", + "UPDATE backfill_jobs SET status = 'cancelling' WHERE id = ? AND status IN ('running', 'paused')", state.db_backend, ); let _ = sqlx::query(&sql) @@ -225,6 +234,36 @@ async fn finalise_cancel(state: &AppState, job_id: &str) { ); } +async fn request_pause(state: &AppState, job_id: &str) { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'pausing' WHERE id = ? AND status = 'running'", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id) + .execute(&state.backfill_db) + .await; +} + +async fn finalise_pause(state: &AppState, job_id: &str) { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'paused' WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(job_id) + .execute(&state.backfill_db) + .await; + publish_event( + state, + super::types::BackfillEvent::JobCompleted { + job_id: job_id.to_string(), + status: "paused".to_string(), + error: None, + }, + ); +} + async fn complete_job( state: &AppState, job_id: &str, @@ -287,7 +326,7 @@ async fn run_discovery_phase( } else { stream::iter(collections.iter()) .for_each_concurrent(collections.len(), |collection| async move { - if is_cancelled(state, job_id).await { + if should_stop_worker(state, job_id).await { return; } if let Err(e) = discover_repos_from_relay(state, job_id, collection).await { @@ -394,7 +433,7 @@ async fn discover_repos_from_relay( update_job_counter(state, job_id, "total_repos", running_total).await; - if is_cancelled(state, job_id).await { + if should_stop_worker(state, job_id).await { return Ok(()); } @@ -492,7 +531,7 @@ async fn run_pipelined_resolve_and_fetch( let mut attempted: i32 = 0; let mut next_flush = random_batch_threshold(100); - let mut next_cancel_check = random_batch_threshold(100); + let mut next_cancel_check = random_batch_threshold(10); let stream_state = resolver_state.clone(); let stream_cancelled = Arc::clone(&resolver_cancelled); @@ -572,11 +611,11 @@ async fn run_pipelined_resolve_and_fetch( attempted += 1; if attempted >= next_cancel_check { - if is_cancelled(&resolver_state, &resolver_job_id).await { + if should_stop_worker(&resolver_state, &resolver_job_id).await { resolver_cancelled.store(true, Ordering::Relaxed); break; } - next_cancel_check = attempted + random_batch_threshold(100); + next_cancel_check = attempted + random_batch_threshold(10); } } @@ -633,7 +672,31 @@ async fn run_pipelined_resolve_and_fetch( let mut worker_handles = FuturesUnordered::new(); let mut overflow: Vec<(String, String)> = Vec::new(); - while let Some((did, pds_endpoint)) = rx.recv().await { + loop { + if cancelled.load(Ordering::Relaxed) { + break; + } + + let poll_state = Arc::clone(&state); + let poll_job_id = Arc::clone(&job_id_arc); + let poll_cancelled = Arc::clone(&cancelled); + let pair = tokio::select! { + biased; + result = rx.recv() => result, + _ = async move { + loop { + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + if poll_cancelled.load(Ordering::Relaxed) || should_stop_worker(&poll_state, &poll_job_id).await { + poll_cancelled.store(true, Ordering::Relaxed); + return; + } + } + } => None, + }; + let Some((did, pds_endpoint)) = pair else { + break; + }; + // Also drain any overflow from previous iterations overflow.push((did, pds_endpoint)); @@ -831,13 +894,12 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R .bind(job_id.as_str()) .execute(&state.backfill_db) .await; - - if is_cancelled(&state, job_id.as_str()).await { - cancelled.store(true, Ordering::Relaxed); - break; - } next_flush = repos + random_batch_threshold(10); } + if cancelled.load(Ordering::Relaxed) || should_stop_worker(&state, job_id.as_str()).await { + cancelled.store(true, Ordering::Relaxed); + break; + } publish_event(&state, super::types::BackfillEvent::JobCounters { job_id: job_id.to_string(), total_repos: None, @@ -853,15 +915,20 @@ async fn run_pds_worker(ctx: FetchContext, pds_endpoint: String, mut rx: mpsc::R let state = Arc::clone(&state); let collections = collections.clone(); let pds_endpoint = pds_endpoint.clone(); + let cancelled = Arc::clone(&cancelled); fetches.push(async move { let mut count: i32 = 0; for collection in collections.iter() { + if cancelled.load(Ordering::Relaxed) { + break; + } match fetch_records_from_pds( &state, &pds_endpoint, &did, collection, + &cancelled, ) .await { @@ -1019,11 +1086,15 @@ async fn run_fetching_phase( let mut did_records: i32 = 0; for collection in collections.iter() { + if cancelled.load(Ordering::Relaxed) { + break; + } match fetch_records_from_pds( &state, &pds_endpoint, &did, collection, + &cancelled, ) .await { @@ -1075,7 +1146,7 @@ async fn run_fetching_phase( .execute(&state.backfill_db) .await; - if is_cancelled(&state, job_id.as_str()).await { + if should_stop_worker(&state, job_id.as_str()).await { cancelled.store(true, Ordering::Relaxed); } } @@ -1208,12 +1279,17 @@ async fn fetch_records_from_pds( pds_endpoint: &str, did: &str, collection: &str, + cancelled: &AtomicBool, ) -> Result { let base = pds_endpoint.trim_end_matches('/'); let mut cursor: Option = None; let mut count: u32 = 0; loop { + if cancelled.load(Ordering::Relaxed) { + break; + } + let mut url = format!( "{base}/xrpc/com.atproto.repo.listRecords?repo={did}&collection={collection}&limit=100" ); @@ -1365,10 +1441,18 @@ async fn run_backfill_job(state: AppState, job_id: String) { if matches!(stage.as_str(), "pending" | "discovering_repos") { run_discovery_phase(&state, &job_id, &collections, did.as_deref()).await; - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; + match should_stop(&state, &job_id).await { + Some("cancelling") => { + tracing::info!(job_id, "backfill job cancelled"); + finalise_cancel(&state, &job_id).await; + return; + } + Some("pausing") => { + tracing::info!(job_id, "backfill job paused"); + finalise_pause(&state, &job_id).await; + return; + } + _ => {} } let total = count_repos(&state, &job_id).await; @@ -1405,10 +1489,18 @@ async fn run_backfill_job(state: AppState, job_id: String) { run_fetching_phase(&state, &job_id, &collections, &concurrency).await }; - if is_cancelled(&state, &job_id).await { - tracing::info!(job_id, "backfill job cancelled"); - finalise_cancel(&state, &job_id).await; - return; + match should_stop(&state, &job_id).await { + Some("cancelling") => { + tracing::info!(job_id, "backfill job cancelled"); + finalise_cancel(&state, &job_id).await; + return; + } + Some("pausing") => { + tracing::info!(job_id, "backfill job paused"); + finalise_pause(&state, &job_id).await; + return; + } + _ => {} } complete_job(&state, &job_id, final_processed, final_records, None).await; @@ -1515,6 +1607,24 @@ pub(super) async fn cancel_backfill( Some((ref status,)) if status == "cancelling" || status == "cancelled" => { Ok(Json(serde_json::json!({ "id": job_id, "status": status }))) } + Some((ref status,)) if status == "paused" => { + finalise_cancel(&state, &job_id).await; + log_event( + &state.db, + EventLog { + event_type: "backfill.cancelled".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "cancelled" }), + )) + } Some((status,)) if status != "running" => Err(AppError::BadRequest(format!( "job is not running (status: {status})" ))), @@ -1539,6 +1649,111 @@ pub(super) async fn cancel_backfill( } } +/// POST /admin/backfill/{id}/pause — pause a running backfill job. +pub(super) async fn pause_backfill( + State(state): State, + admin: UserAuth, + Path(job_id): Path, +) -> Result, AppError> { + admin.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "SELECT status FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_optional(&state.backfill_db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill job: {e}")))?; + + match row { + None => Err(AppError::NotFound("backfill job not found".into())), + Some((ref status,)) if status == "pausing" || status == "paused" => { + Ok(Json(serde_json::json!({ "id": job_id, "status": status }))) + } + Some((status,)) if status != "running" => Err(AppError::BadRequest(format!( + "job is not running (status: {status})" + ))), + Some(_) => { + request_pause(&state, &job_id).await; + log_event( + &state.db, + EventLog { + event_type: "backfill.pausing".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "pausing" }), + )) + } + } +} + +/// POST /admin/backfill/{id}/resume — resume a paused backfill job. +pub(super) async fn resume_backfill( + State(state): State, + admin: UserAuth, + Path(job_id): Path, +) -> Result, AppError> { + admin.require(Permission::BackfillCreate).await?; + + let sql = adapt_sql( + "SELECT status FROM backfill_jobs WHERE id = ?", + state.db_backend, + ); + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(&job_id) + .fetch_optional(&state.backfill_db) + .await + .map_err(|e| AppError::Internal(format!("failed to query backfill job: {e}")))?; + + match row { + None => Err(AppError::NotFound("backfill job not found".into())), + Some((status,)) if status != "paused" => Err(AppError::BadRequest(format!( + "job is not paused (status: {status})" + ))), + Some(_) => { + let sql = adapt_sql( + "UPDATE backfill_jobs SET status = 'running' WHERE id = ?", + state.db_backend, + ); + let _ = sqlx::query(&sql) + .bind(&job_id) + .execute(&state.backfill_db) + .await; + + let spawn_state = state.clone(); + let spawn_job_id = job_id.clone(); + tokio::spawn(async move { + run_backfill_job(spawn_state, spawn_job_id).await; + }); + + log_event( + &state.db, + EventLog { + event_type: "backfill.resumed".to_string(), + severity: Severity::Info, + actor_did: Some(admin.did.clone()), + subject: None, + detail: serde_json::json!({ "job_id": job_id }), + }, + state.db_backend, + ) + .await; + Ok(Json( + serde_json::json!({ "id": job_id, "status": "running" }), + )) + } + } +} + /// GET /admin/backfill/status — list all backfill jobs. pub(super) async fn backfill_status( State(state): State, @@ -1865,7 +2080,7 @@ pub async fn run_backfill_retention_cleanup(state: &AppState) { /// Jobs stuck in `cancelling` are finalised immediately. pub async fn resume_backfill_jobs(state: &AppState) { let sql = adapt_sql( - "SELECT id, status FROM backfill_jobs WHERE status IN ('running', 'cancelling')", + "SELECT id, status FROM backfill_jobs WHERE status IN ('running', 'cancelling', 'pausing')", state.db_backend, ); let rows: Vec<(String, String)> = sqlx::query_as(&sql) @@ -1874,18 +2089,25 @@ pub async fn resume_backfill_jobs(state: &AppState) { .unwrap_or_default(); for (job_id, status) in rows { - if status == "cancelling" { - tracing::info!( - job_id, - "finalising cancelled backfill job from previous run" - ); - finalise_cancel(state, &job_id).await; - } else { - tracing::info!(job_id, "resuming interrupted backfill job"); - let spawn_state = state.clone(); - tokio::spawn(async move { - run_backfill_job(spawn_state, job_id).await; - }); + match status.as_str() { + "cancelling" => { + tracing::info!( + job_id, + "finalising cancelled backfill job from previous run" + ); + finalise_cancel(state, &job_id).await; + } + "pausing" => { + tracing::info!(job_id, "finalising paused backfill job from previous run"); + finalise_pause(state, &job_id).await; + } + _ => { + tracing::info!(job_id, "resuming interrupted backfill job"); + let spawn_state = state.clone(); + tokio::spawn(async move { + run_backfill_job(spawn_state, job_id).await; + }); + } } } } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 0bb7d4f..e0f50fa 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -42,6 +42,8 @@ pub fn admin_routes(_state: AppState) -> Router { delete(backfill::flush_all_backfill_details), ) .route("/backfill/{id}/cancel", post(backfill::cancel_backfill)) + .route("/backfill/{id}/pause", post(backfill::pause_backfill)) + .route("/backfill/{id}/resume", post(backfill::resume_backfill)) .route("/backfill/{id}/events", get(backfill::backfill_events)) .route("/backfill/{id}/repos", get(backfill::backfill_repos)) .route( diff --git a/src/admin/settings.rs b/src/admin/settings.rs index b91318e..f730f7b 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -203,6 +203,29 @@ pub(super) async fn db_info( let main_pool_size = state.db.options().get_max_connections() as i64; let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; + let pds: i64 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let dids: i64 = get_setting( + &state.db, + "backfill_concurrent_dids_per_pds", + state.db_backend, + ) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + let resolution: i64 = get_setting( + &state.db, + "backfill_concurrent_resolution", + state.db_backend, + ) + .await + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + let needed_backfill_pool = (pds * dids) + resolution + 4; + let restart_recommended = needed_backfill_pool > backfill_pool_size; + Ok(Json(serde_json::json!({ "backend": match state.db_backend { DatabaseBackend::Sqlite => "sqlite", @@ -211,6 +234,7 @@ pub(super) async fn db_info( "server_max_connections": server_max, "main_pool_size": main_pool_size, "backfill_pool_size": backfill_pool_size, + "restart_recommended": restart_recommended, }))) } diff --git a/web/package-lock.json b/web/package-lock.json index 65bd0ba..ed30f1a 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -116,7 +116,6 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -774,7 +773,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -2045,7 +2043,6 @@ "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": "^14.21.3 || >=16" }, @@ -4343,7 +4340,6 @@ "integrity": "sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -4353,7 +4349,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -4364,7 +4359,6 @@ "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", "devOptional": true, "license": "MIT", - "peer": true, "peerDependencies": { "@types/react": "^19.2.0" } @@ -4388,7 +4382,8 @@ "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", "license": "MIT", - "optional": true + "optional": true, + "peer": true }, "node_modules/@types/unist": { "version": "3.0.3", @@ -4454,7 +4449,6 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -4968,7 +4962,6 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -5355,7 +5348,6 @@ "integrity": "sha512-Ixm8tFfoKKIPYdCCKYTsqv+Fd4IJ0DQqMyEimo+pxUOMUR9cVPlwTrFt9Avu+3cb6Zp3mAzl+t1MrG2fxxKsxw==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/types": "^7.26.0" } @@ -5460,7 +5452,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -6422,6 +6413,7 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.2.7.tgz", "integrity": "sha512-WhL/YuveyGXJaerVlMYGWhvQswa7myDG17P7Vu65EWC05o8vfeNbvNf4d/BOvH99+ZW+LlQsc1GDKMa1vNK6dw==", "license": "(MPL-2.0 OR Apache-2.0)", + "peer": true, "optionalDependencies": { "@types/trusted-types": "^2.0.7" } @@ -6760,7 +6752,6 @@ "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -6901,7 +6892,6 @@ "integrity": "sha512-whOE1HFo/qJDyX4SnXzP4N6zOWn79WhnCUY/iDR0mPfQZO8wcYE4JClzI2oZrhBnnMUCBCHZhO6VQyoBU95mZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@rtsao/scc": "^1.1.0", "array-includes": "^3.1.9", @@ -7380,7 +7370,6 @@ "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -8179,7 +8168,6 @@ "integrity": "sha512-Eaw2YTGM6WOxA6CXbckaEvslr2Ne4NFsKrvc0v97JD5awbmeBLO5w9Ho9L9kmKonrwF9RJlW6BxT1PVv/agBHQ==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -9605,6 +9593,7 @@ "resolved": "https://registry.npmjs.org/marked/-/marked-14.0.0.tgz", "integrity": "sha512-uIj4+faQ+MgHgwUW1l2PsPglZLOLOT1uErt06dAPtx2kjteLAkbsd/0FiYg/MGS+i7ZKLb7w2WClxHkzOOuryQ==", "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -11761,7 +11750,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -11792,7 +11780,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.4.tgz", "integrity": "sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -11839,7 +11826,6 @@ "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.2.0.tgz", "integrity": "sha512-ROY9fvHhwOD9ySfrF0wmvu//bKCQ6AeZZq1nJNtbDC+kk5DuSuNX/n6YWYF/SYy7bSba4D4FSz8DJeKY/S/r+g==", "license": "MIT", - "peer": true, "dependencies": { "@types/use-sync-external-store": "^0.0.6", "use-sync-external-store": "^1.4.0" @@ -11978,8 +11964,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/redux-thunk": { "version": "3.1.0", @@ -13174,8 +13159,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.2.0.tgz", "integrity": "sha512-yYzTZ4++b7fNYxFfpnberEEKu43w44aqDMNM9MHMmcKuCH7lL8jJ4yJ7LGHv7rSwiqM0nkiobF9I6cLlpS2P7Q==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/tapable": { "version": "2.3.0", @@ -13248,7 +13232,6 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -13526,7 +13509,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -14276,7 +14258,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index c556d9c..d344bcc 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -5,6 +5,8 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useCurrentUser } from "@/hooks/use-current-user"; import { cancelBackfillJob, + pauseBackfillJob, + resumeBackfillJob, createBackfillJob, getBackfillJobs, getBackfillRepos, @@ -20,7 +22,7 @@ import type { BackfillEvent, BlueskyProfile, } from "@/types/backfill"; -import { CheckCircle2, ChevronRight, Circle, Loader2 } from "lucide-react"; +import { CheckCircle2, ChevronRight, Circle, Loader2, PauseCircle } from "lucide-react"; import { SiteHeader } from "@/components/site-header"; import { AlertDialog, @@ -104,6 +106,18 @@ function statusBadge(job: BackfillJob) { cancelling ); + case "pausing": + return ( + + pausing + + ); + case "paused": + return ( + + paused + + ); case "running": return ( @@ -507,6 +521,14 @@ export default function BackfillPage() { await cancelBackfillJob(selectedJob.id); load(); }} + onPause={async () => { + await pauseBackfillJob(selectedJob.id); + load(); + }} + onResume={async () => { + await resumeBackfillJob(selectedJob.id); + load(); + }} /> )} @@ -521,16 +543,23 @@ function JobDetail({ canCancel, canFlush, onCancel, + onPause, + onResume, }: { job: BackfillJob; canCancel: boolean; canFlush: boolean; onCancel: () => Promise; + onPause: () => Promise; + onResume: () => Promise; }) { const [cancelling, setCancelling] = useState(false); + const [pausing, setPausing] = useState(false); + const [resuming, setResuming] = useState(false); const current = phaseIndex(job.stage); const allDone = job.status === "completed"; - const isActive = job.status === "running" || job.status === "cancelling"; + const isActive = job.status === "running" || job.status === "cancelling" || job.status === "pausing"; + const isPaused = job.status === "paused" || job.status === "pausing"; // Detail data state const [discoveredRepos, setDiscoveredRepos] = useState([]); @@ -556,6 +585,23 @@ function JobDetail({ return current >= phaseIndex(phase); } + function isPhasePaused(phase: (typeof PROGRESS_PHASES)[number]): boolean { + if (!isPaused) return false; + if (job.stage === "resolving_and_fetching") { + return phase === "resolving_pds" || phase === "fetching_records"; + } + if (job.stage === "discovering_repos") { + return phase === "discovering_repos"; + } + if (job.stage === "resolving_pds") { + return phase === "resolving_pds"; + } + if (job.stage === "fetching_records") { + return phase === "fetching_records"; + } + return false; + } + async function handleCancel() { setCancelling(true); try { @@ -565,6 +611,24 @@ function JobDetail({ } } + async function handlePause() { + setPausing(true); + try { + await onPause(); + } finally { + setPausing(false); + } + } + + async function handleResume() { + setResuming(true); + try { + await onResume(); + } finally { + setResuming(false); + } + } + // Auto-load detail data when phases are reached const discoveredReached = hasReached("discovering_repos"); const pdsReached = hasReached("resolving_pds"); @@ -727,6 +791,7 @@ function JobDetail({ label="Discovering repos" active={isActive && job.stage === "discovering_repos"} reached={hasReached("discovering_repos")} + paused={isPhasePaused("discovering_repos")} value={job.total_repos != null ? : undefined} suffix="repos found" loading={discoveredReached && !discoveredLoaded} @@ -756,6 +821,7 @@ function JobDetail({ job.stage === "resolving_and_fetching") } reached={hasReached("resolving_pds")} + paused={isPhasePaused("resolving_pds")} value={ hasReached("resolving_pds") ? <> / @@ -790,6 +856,7 @@ function JobDetail({ job.stage === "resolving_and_fetching") } reached={hasReached("fetching_records")} + paused={isPhasePaused("fetching_records")} value={ hasReached("fetching_records") || job.stage === "resolving_and_fetching" @@ -855,6 +922,26 @@ function JobDetail({ )} + {canCancel && (job.status === "running" || job.status === "pausing") && ( + + )} + {canCancel && job.status === "paused" && ( + + )} {canCancel && isActive && ( + )} ); @@ -874,6 +971,7 @@ function ProgressRow({ label, active, reached, + paused, value, suffix, loading, @@ -882,13 +980,14 @@ function ProgressRow({ label: string; active: boolean; reached: boolean; + paused?: boolean; value?: React.ReactNode; suffix?: React.ReactNode; loading?: boolean; children?: React.ReactNode; }) { const [open, setOpen] = useState(false); - const done = reached && !active; + const done = reached && !active && !paused; const expandable = reached; return ( @@ -905,6 +1004,8 @@ function ProgressRow({ {active ? ( + ) : paused ? ( + ) : done ? ( ) : ( diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index bc0180e..a6292ef 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -1,9 +1,9 @@ -"use client" +"use client"; -import { useCallback, useEffect, useMemo, useRef, useState } from "react" -import { Upload, Trash2 } from "lucide-react" +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { Upload, Trash2 } from "lucide-react"; -import { useCurrentUser } from "@/hooks/use-current-user" +import { useCurrentUser } from "@/hooks/use-current-user"; import { getSettings, getDbInfo, @@ -13,11 +13,11 @@ import { deleteLogo, type SettingEntry, type DbInfo, -} from "@/lib/api" -import { SiteHeader } from "@/components/site-header" -import { Button } from "@/components/ui/button" -import { Input } from "@/components/ui/input" -import { Label } from "@/components/ui/label" +} from "@/lib/api"; +import { SiteHeader } from "@/components/site-header"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; const SETTING_KEYS = [ "app_name", @@ -29,16 +29,16 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", -] as const +] as const; -type FieldKey = (typeof SETTING_KEYS)[number] +type FieldKey = (typeof SETTING_KEYS)[number]; type FieldConfig = { - key: FieldKey - label: string - placeholder: string - description: string -} + key: FieldKey; + label: string; + placeholder: string; + description: string; +}; const FIELDS: FieldConfig[] = [ { @@ -74,11 +74,11 @@ const FIELDS: FieldConfig[] = [ placeholder: "https://example.com/privacy", description: "Link to your privacy policy. Optional.", }, -] +]; export default function GeneralSettingsPage() { - const { hasPermission } = useCurrentUser() - const canManage = hasPermission("settings:manage") + const { hasPermission } = useCurrentUser(); + const canManage = hasPermission("settings:manage"); const [values, setValues] = useState>({ app_name: "", @@ -90,8 +90,10 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", - }) - const [sources, setSources] = useState>({ + }); + const [sources, setSources] = useState< + Record + >({ app_name: "unset", backfill_concurrent_dids_per_pds: "unset", backfill_concurrent_pds: "unset", @@ -101,34 +103,46 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", - }) - const [logoUploaded, setLogoUploaded] = useState(false) - const [dbInfo, setDbInfo] = useState(null) - const [error, setError] = useState(null) - const [saving, setSaving] = useState(false) - const [notice, setNotice] = useState(null) - const fileInputRef = useRef(null) + }); + const [logoUploaded, setLogoUploaded] = useState(false); + const [dbInfo, setDbInfo] = useState(null); + const [error, setError] = useState(null); + const [saving, setSaving] = useState(false); + const [notice, setNotice] = useState(null); + const fileInputRef = useRef(null); const load = useCallback(async () => { try { - const entries = await getSettings() - const byKey = new Map(entries.map((e) => [e.key, e])) - const val = (key: string, fallback: string) => byKey.get(key)?.value ?? fallback - const src = (key: string) => (byKey.get(key)?.source as "database" | "env" | undefined) ?? "unset" + const entries = await getSettings(); + const byKey = new Map( + entries.map((e) => [e.key, e]), + ); + const val = (key: string, fallback: string) => + byKey.get(key)?.value ?? fallback; + const src = (key: string) => + (byKey.get(key)?.source as "database" | "env" | undefined) ?? "unset"; setValues({ app_name: val("app_name", ""), - backfill_concurrent_dids_per_pds: val("backfill_concurrent_dids_per_pds", "3"), + backfill_concurrent_dids_per_pds: val( + "backfill_concurrent_dids_per_pds", + "3", + ), backfill_concurrent_pds: val("backfill_concurrent_pds", "10"), - backfill_concurrent_resolution: val("backfill_concurrent_resolution", "100"), + backfill_concurrent_resolution: val( + "backfill_concurrent_resolution", + "100", + ), backfill_retention_days: val("backfill_retention_days", "28"), client_uri: val("client_uri", ""), logo_uri: val("logo_uri", ""), tos_uri: val("tos_uri", ""), policy_uri: val("policy_uri", ""), - }) + }); setSources({ app_name: src("app_name"), - backfill_concurrent_dids_per_pds: src("backfill_concurrent_dids_per_pds"), + backfill_concurrent_dids_per_pds: src( + "backfill_concurrent_dids_per_pds", + ), backfill_concurrent_pds: src("backfill_concurrent_pds"), backfill_concurrent_resolution: src("backfill_concurrent_resolution"), backfill_retention_days: src("backfill_retention_days"), @@ -136,35 +150,35 @@ export default function GeneralSettingsPage() { logo_uri: src("logo_uri"), tos_uri: src("tos_uri"), policy_uri: src("policy_uri"), - }) - setLogoUploaded(byKey.has("logo_data")) + }); + setLogoUploaded(byKey.has("logo_data")); try { - setDbInfo(await getDbInfo()) + setDbInfo(await getDbInfo()); } catch { // non-critical } } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } - }, []) + }, []); useEffect(() => { - load() - }, [load]) + load(); + }, [load]); async function handleSave() { - setError(null) - setNotice(null) - setSaving(true) + setError(null); + setNotice(null); + setSaving(true); try { for (const field of FIELDS) { - const value = values[field.key] + const value = values[field.key]; if (value === "") { if (sources[field.key] === "database") { - await deleteSetting(field.key) + await deleteSetting(field.key); } } else { - await upsertSetting(field.key, value) + await upsertSetting(field.key, value); } } const backfillKeys = [ @@ -172,77 +186,79 @@ export default function GeneralSettingsPage() { "backfill_concurrent_pds", "backfill_concurrent_resolution", "backfill_retention_days", - ] as const + ] as const; for (const key of backfillKeys) { - const value = values[key] + const value = values[key]; if (value === "") { if (sources[key] === "database") { - await deleteSetting(key) + await deleteSetting(key); } } else { - await upsertSetting(key, value) + await upsertSetting(key, value); } } - setNotice("Settings saved.") - await load() + setNotice("Settings saved."); + await load(); } catch (e: unknown) { - setError(e instanceof Error ? e.message : String(e)) + setError(e instanceof Error ? e.message : String(e)); } finally { - setSaving(false) + setSaving(false); } } async function handleLogoUpload(e: React.ChangeEvent) { - const file = e.target.files?.[0] - if (!file) return - setError(null) + const file = e.target.files?.[0]; + if (!file) return; + setError(null); try { - await uploadLogo(file) - setNotice("Logo uploaded.") - await load() + await uploadLogo(file); + setNotice("Logo uploaded."); + await load(); } catch (err: unknown) { - setError(err instanceof Error ? err.message : String(err)) + setError(err instanceof Error ? err.message : String(err)); } finally { - if (fileInputRef.current) fileInputRef.current.value = "" + if (fileInputRef.current) fileInputRef.current.value = ""; } } async function handleLogoDelete() { - setError(null) + setError(null); try { - await deleteLogo() - setNotice("Logo removed.") - await load() + await deleteLogo(); + setNotice("Logo removed."); + await load(); } catch (err: unknown) { - setError(err instanceof Error ? err.message : String(err)) + setError(err instanceof Error ? err.message : String(err)); } } const connectionEstimate = useMemo(() => { - const pds = parseInt(values.backfill_concurrent_pds) || 10 - const dids = parseInt(values.backfill_concurrent_dids_per_pds) || 3 - const resolution = parseInt(values.backfill_concurrent_resolution) || 100 - const needed = pds * dids + resolution + 4 - const mainPool = dbInfo?.main_pool_size ?? 32 - const total = needed + mainPool - const serverMax = dbInfo?.server_max_connections ?? null - return { needed, mainPool, total, serverMax } - }, [values, dbInfo]) + const pds = parseInt(values.backfill_concurrent_pds) || 10; + const dids = parseInt(values.backfill_concurrent_dids_per_pds) || 3; + const resolution = parseInt(values.backfill_concurrent_resolution) || 100; + const needed = pds * dids + resolution + 4; + const mainPool = dbInfo?.main_pool_size ?? 32; + const total = needed + mainPool; + const serverMax = dbInfo?.server_max_connections ?? null; + return { needed, mainPool, total, serverMax }; + }, [values, dbInfo]); const connectionWarning = useMemo(() => { - if (!connectionEstimate.serverMax) return null + if (!connectionEstimate.serverMax) return null; if (connectionEstimate.total > connectionEstimate.serverMax) { - return `These settings need ~${connectionEstimate.total} connections (${connectionEstimate.needed} backfill + ${connectionEstimate.mainPool} main), but the database allows ${connectionEstimate.serverMax}. Reduce concurrency or increase the database's max_connections.` + return `These settings need ~${connectionEstimate.total} connections (${connectionEstimate.needed} backfill + ${connectionEstimate.mainPool} main), but the database allows ${connectionEstimate.serverMax}. Reduce concurrency or increase the database's max_connections.`; } - return null - }, [connectionEstimate]) + return null; + }, [connectionEstimate]); return ( <>
{error &&

{error}

} - {notice &&

{notice}

} + {notice && ( +

{notice}

+ )}

Instance Identity

@@ -320,13 +336,16 @@ export default function GeneralSettingsPage() {

Data Retention

- Configure how long HappyView retains detailed data from completed backfill jobs. + Configure how long HappyView retains detailed data from completed + backfill jobs.

- + {sources["backfill_retention_days"] === "env" && ( from env var @@ -340,25 +359,31 @@ export default function GeneralSettingsPage() { step={1} value={values["backfill_retention_days"]} onChange={(e) => - setValues((v) => ({ ...v, backfill_retention_days: e.target.value })) + setValues((v) => ({ + ...v, + backfill_retention_days: e.target.value, + })) } placeholder="28" disabled={!canManage} />

- How long to keep per-repo detail data from completed backfill jobs. Set to 0 to keep indefinitely. + How long to keep per-repo detail data from completed backfill jobs. + Set to 0 to keep indefinitely.

Backfill Performance

- Tune concurrency limits for backfill jobs. Changes apply to the next job started. - The backfill connection pool is auto-sized on startup based on these values. + Tune concurrency limits for backfill jobs. Changes only apply to new + or resumed jobs.

{dbInfo?.server_max_connections && (

- Database limit: {dbInfo.server_max_connections} connections · Main pool: {connectionEstimate.mainPool} · Backfill estimate: {connectionEstimate.needed} + Database limit: {dbInfo.server_max_connections} connections · Main + pool: {connectionEstimate.mainPool} · Backfill estimate:{" "} + {connectionEstimate.needed}

)} {connectionWarning && ( @@ -366,16 +391,39 @@ export default function GeneralSettingsPage() { )}
- {([ - { key: "backfill_concurrent_resolution" as const, id: "backfill_concurrent_resolution", label: "Concurrent PLC Resolutions", placeholder: "100", description: "How many DID document lookups to run in parallel during PDS resolution." }, - { key: "backfill_concurrent_pds" as const, id: "backfill_concurrent_pds", label: "Concurrent PDS Hosts", placeholder: "10", description: "How many PDS servers to fetch records from simultaneously." }, - { key: "backfill_concurrent_dids_per_pds" as const, id: "backfill_concurrent_dids_per_pds", label: "Concurrent DIDs per PDS", placeholder: "3", description: "How many repos to fetch concurrently from each PDS host." }, - ]).map((field) => ( + {[ + { + key: "backfill_concurrent_resolution" as const, + id: "backfill_concurrent_resolution", + label: "Concurrent PLC Resolutions", + placeholder: "100", + description: + "How many DID document lookups to run in parallel during PDS resolution.", + }, + { + key: "backfill_concurrent_pds" as const, + id: "backfill_concurrent_pds", + label: "Concurrent PDS Hosts", + placeholder: "10", + description: + "How many PDS servers to fetch records from simultaneously.", + }, + { + key: "backfill_concurrent_dids_per_pds" as const, + id: "backfill_concurrent_dids_per_pds", + label: "Concurrent DIDs per PDS", + placeholder: "3", + description: + "How many repos to fetch concurrently from each PDS host.", + }, + ].map((field) => (
{sources[field.key] === "env" && ( - from env var + + from env var + )}
setValues((v) => ({ ...v, [field.key]: e.target.value }))} + onChange={(e) => + setValues((v) => ({ ...v, [field.key]: e.target.value })) + } placeholder={field.placeholder} disabled={!canManage} /> @@ -402,5 +452,5 @@ export default function GeneralSettingsPage() {
- ) + ); } diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 1c43284..1d93562 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -189,6 +189,20 @@ export function cancelBackfillJob(id: string) { ); } +export function pauseBackfillJob(id: string) { + return apiFetch<{ id: string; status: string }>( + `/admin/backfill/${id}/pause`, + { method: "POST" }, + ); +} + +export function resumeBackfillJob(id: string) { + return apiFetch<{ id: string; status: string }>( + `/admin/backfill/${id}/resume`, + { method: "POST" }, + ); +} + export function getBackfillRepos( jobId: string, params: { phase?: string; cursor?: string; limit?: number } = {}, -- 2.51.2 From b0d382b2979a2d1f0fbc4976e06db8011bc7bfd8 Mon Sep 17 00:00:00 2001 From: Trezy Date: Fri, 22 May 2026 20:16:18 -0500 Subject: [PATCH 22/64] feat: recommend restart when backfill concurrency settings dont match existing connection pool size Signed-off-by: Trezy --- web/src/app/dashboard/layout.tsx | 25 ++++++----- web/src/components/app-sidebar.tsx | 22 ++++++++++ web/src/components/site-header.tsx | 67 ++++++++++++++++++++++-------- web/src/lib/api.ts | 1 + web/src/lib/restart-context.tsx | 47 +++++++++++++++++++++ 5 files changed, 133 insertions(+), 29 deletions(-) create mode 100644 web/src/lib/restart-context.tsx diff --git a/web/src/app/dashboard/layout.tsx b/web/src/app/dashboard/layout.tsx index 9e5488f..be0b4e3 100644 --- a/web/src/app/dashboard/layout.tsx +++ b/web/src/app/dashboard/layout.tsx @@ -7,6 +7,7 @@ import { useAuth } from "@/lib/auth-context" import { useConfig } from "@/lib/config-context" import { AppSidebar } from "@/components/app-sidebar" import { PluginUpdateProvider } from "@/components/plugin-update-provider" +import { RestartProvider } from "@/lib/restart-context" import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar" import { Toaster } from "@/components/ui/sonner" @@ -33,17 +34,19 @@ export default function DashboardLayout({ return ( - - - {children} - + + + + {children} + + ) diff --git a/web/src/components/app-sidebar.tsx b/web/src/components/app-sidebar.tsx index cc54c77..e9521d8 100644 --- a/web/src/components/app-sidebar.tsx +++ b/web/src/components/app-sidebar.tsx @@ -29,6 +29,8 @@ import { useAuth } from "@/lib/auth-context"; import { useConfig } from "@/lib/config-context"; import { useCurrentUser } from "@/hooks/use-current-user"; import { usePluginUpdates } from "@/components/plugin-update-provider"; +import { useRestart } from "@/lib/restart-context"; +import { getDbInfo } from "@/lib/api"; import { Scroller } from "@/components/ui/scroller"; import { Sidebar, @@ -138,6 +140,7 @@ export function AppSidebar({ ...props }: React.ComponentProps) { const { app_name, logo_url } = useConfig(); const { hasPermission } = useCurrentUser(); const { hasUpdates } = usePluginUpdates(); + const { addReason, removeReason } = useRestart(); const [deadLetterCount, setDeadLetterCount] = useState(0); @@ -160,6 +163,25 @@ export function AppSidebar({ ...props }: React.ComponentProps) { }; }, []); + useEffect(() => { + if (!hasPermission("settings:manage")) return; + let cancelled = false; + getDbInfo() + .then((info) => { + if (!cancelled) { + if (info.restart_recommended) { + addReason("backfill-pool", "Backfill pool is undersized for current concurrency settings."); + } else { + removeReason("backfill-pool"); + } + } + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, [hasPermission, addReason, removeReason]); + function filterByPermission(items: NavItem[]) { return items.filter( (item) => diff --git a/web/src/components/site-header.tsx b/web/src/components/site-header.tsx index 741edf2..c5a7eaf 100644 --- a/web/src/components/site-header.tsx +++ b/web/src/components/site-header.tsx @@ -1,8 +1,11 @@ "use client" +import { useState } from "react" import { IconArrowLeft } from "@tabler/icons-react" +import { AlertTriangle, ChevronDown } from "lucide-react" import Link from "next/link" +import { useRestart } from "@/lib/restart-context" import { Button } from "@/components/ui/button" import { Separator } from "@/components/ui/separator" import { SidebarTrigger } from "@/components/ui/sidebar" @@ -15,27 +18,55 @@ export function SiteHeader({ title: string backHref?: string }) { + const { reasons } = useRestart() + const [expanded, setExpanded] = useState(false) + return ( -
-
- - - {backHref && ( - - )} -

{title}

-
- +
+
+
+ + + {backHref && ( + + )} +

{title}

+
+ +
+ {reasons.length > 0 && ( +
+ + {expanded && ( +
    + {reasons.map((reason, i) => ( +
  • + + {reason} +
    • + ))} +
+ )} +
+ )}
) } diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 1d93562..9612788 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -384,6 +384,7 @@ export type DbInfo = { server_max_connections: number | null; main_pool_size: number; backfill_pool_size: number; + restart_recommended: boolean; }; export function getDbInfo() { diff --git a/web/src/lib/restart-context.tsx b/web/src/lib/restart-context.tsx new file mode 100644 index 0000000..a503e61 --- /dev/null +++ b/web/src/lib/restart-context.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { createContext, useCallback, useContext, useMemo, useRef, useState } from "react"; + +interface RestartContextType { + reasons: string[]; + addReason: (key: string, reason: string) => void; + removeReason: (key: string) => void; +} + +const RestartContext = createContext({ + reasons: [], + addReason: () => {}, + removeReason: () => {}, +}); + +export function RestartProvider({ children }: { children: React.ReactNode }) { + const [reasonMap, setReasonMap] = useState>({}); + const reasonMapRef = useRef(reasonMap); + reasonMapRef.current = reasonMap; + + const addReason = useCallback((key: string, reason: string) => { + if (reasonMapRef.current[key] === reason) return; + setReasonMap((prev) => ({ ...prev, [key]: reason })); + }, []); + + const removeReason = useCallback((key: string) => { + setReasonMap((prev) => { + if (!(key in prev)) return prev; + const next = { ...prev }; + delete next[key]; + return next; + }); + }, []); + + const reasons = useMemo(() => Object.values(reasonMap), [reasonMap]); + + return ( + + {children} + + ); +} + +export function useRestart() { + return useContext(RestartContext); +} -- 2.51.2 From 485657997b901f08725385921798b26c253b6e70 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 12:08:23 -0500 Subject: [PATCH 23/64] feat: make verbose event logs configurable Signed-off-by: Trezy --- src/admin/settings.rs | 1 + src/lib.rs | 1 + src/lua/atproto_api.rs | 1 + src/lua/db_api.rs | 1 + src/lua/execute.rs | 38 ++++---- src/lua/http_api.rs | 1 + src/lua/xrpc_api.rs | 1 + src/main.rs | 34 ++++++- src/record_handler.rs | 96 +++++-------------- tests/common/app.rs | 1 + tests/lua_atproto_api.rs | 1 + tests/lua_db_api.rs | 1 + .../app/dashboard/settings/general/page.tsx | 50 +++++++++- web/src/lib/api.ts | 4 + 14 files changed, 136 insertions(+), 95 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index f730f7b..c3c1a1b 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -31,6 +31,7 @@ const ENV_FALLBACKS: &[(&str, &str)] = &[ ("logo_uri", "LOGO_URI"), ("tos_uri", "TOS_URI"), ("policy_uri", "POLICY_URI"), + ("verbose_event_logging", "VERBOSE_EVENT_LOGGING"), ]; /// Resolve a setting value: check the DB first, then fall back to env var. diff --git a/src/lib.rs b/src/lib.rs index 3b3a061..39a44ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -80,6 +80,7 @@ pub struct AppState { pub official_registry_config: RegistryConfig, pub proxy_config: Arc>, pub backfill_events_tx: tokio::sync::broadcast::Sender, + pub verbose_event_logging: std::sync::Arc, } impl axum::extract::FromRef for axum_extra::extract::cookie::Key { diff --git a/src/lua/atproto_api.rs b/src/lua/atproto_api.rs index 40af1b0..aeb047c 100644 --- a/src/lua/atproto_api.rs +++ b/src/lua/atproto_api.rs @@ -584,6 +584,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index e8d95b8..5e7f062 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -792,6 +792,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/execute.rs b/src/lua/execute.rs index bf50cf3..0e0037d 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -4,6 +4,7 @@ use mlua::LuaSerdeExt; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::Instant; use crate::AppState; @@ -913,23 +914,25 @@ pub async fn execute_hook_script(event: &HookEvent<'_>) -> Option { match run_hook_once(event).await { Ok(hook_result) => { - log_event( - &event.state.db, - EventLog { - event_type: "hook.executed".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(event.uri.to_string()), - detail: serde_json::json!({ - "lexicon_id": event.lexicon_id, - "action": event.action, - "collection": event.collection, - "attempts": attempt + 1, - }), - }, - backend, - ) - .await; + if event.state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + &event.state.db, + EventLog { + event_type: "hook.executed".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(event.uri.to_string()), + detail: serde_json::json!({ + "lexicon_id": event.lexicon_id, + "action": event.action, + "collection": event.collection, + "attempts": attempt + 1, + }), + }, + backend, + ) + .await; + } return hook_result; } Err(e) => { @@ -1182,6 +1185,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/http_api.rs b/src/lua/http_api.rs index 12b3716..cce9b54 100644 --- a/src/lua/http_api.rs +++ b/src/lua/http_api.rs @@ -190,6 +190,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 82c6ed7..6a5b930 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -294,6 +294,7 @@ mod tests { crate::proxy_config::ProxyConfig::default(), ))), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/src/main.rs b/src/main.rs index 9a3bf7c..f6a78b2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -28,8 +28,11 @@ async fn main() { tracing_subscriber::fmt() .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| "happyview=debug,tower_http=debug".parse().unwrap()), + tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| { + "happyview=debug,tower_http=debug,sqlx=warn" + .parse() + .unwrap() + }), ) .init(); @@ -615,6 +618,15 @@ async fn main() { let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let verbose_event_logging = { + let enabled = + happyview::admin::settings::get_setting(&db_pool, "verbose_event_logging", db_backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(enabled)) + }; + let state = AppState { config: config.clone(), http, @@ -636,6 +648,7 @@ async fn main() { official_registry_config, proxy_config, backfill_events_tx, + verbose_event_logging, }; jetstream::spawn(state.clone(), collections_rx); @@ -649,6 +662,23 @@ async fn main() { state.db_backend, )); + { + let db = state.db.clone(); + let flag = state.verbose_event_logging.clone(); + let backend = state.db_backend; + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(30)).await; + let enabled = + happyview::admin::settings::get_setting(&db, "verbose_event_logging", backend) + .await + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + flag.store(enabled, std::sync::atomic::Ordering::Relaxed); + } + }); + } + happyview::admin::backfill::resume_backfill_jobs(&state).await; { diff --git a/src/record_handler.rs b/src/record_handler.rs index e59f75f..34dacee 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::atomic::Ordering; use serde_json::Value; @@ -74,27 +75,7 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; match hook_result { - None => { - // Hook returned nil — skip indexing this record. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - state.db_backend, - ) - .await; - return; - } + None => return, Some(v) => v, } } else { @@ -138,23 +119,6 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { ) .await; - log_event( - db, - EventLog { - event_type: "record.created".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), uri.clone()); } Err(e) => { @@ -198,48 +162,32 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; if hook_result.is_none() { - // Hook returned nil — skip the delete. - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - backend, - ) - .await; + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "hook returned nil", + }), + }, + backend, + ) + .await; + } return; } } let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); match sqlx::query(&delete_sql).bind(&uri).execute(db).await { - Ok(_) => { - log_event( - db, - EventLog { - event_type: "record.deleted".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - }), - }, - backend, - ) - .await; - } + Ok(_) => {} Err(e) => { tracing::warn!(uri = %uri, "failed to delete record: {e}"); log_event( diff --git a/tests/common/app.rs b/tests/common/app.rs index bac7ff8..cd08de7 100644 --- a/tests/common/app.rs +++ b/tests/common/app.rs @@ -169,6 +169,7 @@ impl TestApp { ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let router = server::router(state.clone()).layer(axum::middleware::from_fn( diff --git a/tests/lua_atproto_api.rs b/tests/lua_atproto_api.rs index f0a24c1..396a7ba 100644 --- a/tests/lua_atproto_api.rs +++ b/tests/lua_atproto_api.rs @@ -103,6 +103,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index bbf17f2..d32bb90 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -106,6 +106,7 @@ async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> ))), backfill_db: pool.clone(), backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index a6292ef..47ebe58 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -18,6 +18,7 @@ import { SiteHeader } from "@/components/site-header"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; const SETTING_KEYS = [ "app_name", @@ -29,6 +30,7 @@ const SETTING_KEYS = [ "logo_uri", "tos_uri", "policy_uri", + "verbose_event_logging", ] as const; type FieldKey = (typeof SETTING_KEYS)[number]; @@ -90,6 +92,7 @@ export default function GeneralSettingsPage() { logo_uri: "", tos_uri: "", policy_uri: "", + verbose_event_logging: "", }); const [sources, setSources] = useState< Record @@ -103,6 +106,7 @@ export default function GeneralSettingsPage() { logo_uri: "unset", tos_uri: "unset", policy_uri: "unset", + verbose_event_logging: "unset", }); const [logoUploaded, setLogoUploaded] = useState(false); const [dbInfo, setDbInfo] = useState(null); @@ -137,6 +141,7 @@ export default function GeneralSettingsPage() { logo_uri: val("logo_uri", ""), tos_uri: val("tos_uri", ""), policy_uri: val("policy_uri", ""), + verbose_event_logging: val("verbose_event_logging", ""), }); setSources({ app_name: src("app_name"), @@ -150,6 +155,7 @@ export default function GeneralSettingsPage() { logo_uri: src("logo_uri"), tos_uri: src("tos_uri"), policy_uri: src("policy_uri"), + verbose_event_logging: src("verbose_event_logging"), }); setLogoUploaded(byKey.has("logo_data")); try { @@ -181,13 +187,14 @@ export default function GeneralSettingsPage() { await upsertSetting(field.key, value); } } - const backfillKeys = [ + const extraKeys = [ "backfill_concurrent_dids_per_pds", "backfill_concurrent_pds", "backfill_concurrent_resolution", "backfill_retention_days", + "verbose_event_logging", ] as const; - for (const key of backfillKeys) { + for (const key of extraKeys) { const value = values[key]; if (value === "") { if (sources[key] === "database") { @@ -442,6 +449,45 @@ export default function GeneralSettingsPage() {
))} +
+

Logging

+

+ Configure event log verbosity. +

+
+ +
+
+
+ + {sources["verbose_event_logging"] === "env" && ( + + from env var + + )} +
+

+ Log every record index, hook execution, and hook skip to the event + log. Generates high write volume and will cause issues if + you're indexing high-traffic collections. Recommended to only use + for debugging. +

+
+ + setValues((v) => ({ + ...v, + verbose_event_logging: checked ? "true" : "", + })) + } + disabled={!canManage} + /> +
+
- - - - - ); - } if (!table.getIsAllPageRowsSelected()) { return ( -- 2.51.2 From 3a0fb9cb18b6afd901e428c3e70976978706e673 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 12:30:44 -0500 Subject: [PATCH 26/64] fix: better pool size accounting when using sqlite Signed-off-by: Trezy --- src/admin/settings.rs | 11 ++++---- src/db.rs | 60 ++++++++++++++++++++++++++----------------- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/admin/settings.rs b/src/admin/settings.rs index c3c1a1b..dd9d3d3 100644 --- a/src/admin/settings.rs +++ b/src/admin/settings.rs @@ -204,11 +204,11 @@ pub(super) async fn db_info( let main_pool_size = state.db.options().get_max_connections() as i64; let backfill_pool_size = state.backfill_db.options().get_max_connections() as i64; - let pds: i64 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) + let pds: u32 = get_setting(&state.db, "backfill_concurrent_pds", state.db_backend) .await .and_then(|v| v.parse().ok()) .unwrap_or(10); - let dids: i64 = get_setting( + let dids: u32 = get_setting( &state.db, "backfill_concurrent_dids_per_pds", state.db_backend, @@ -216,7 +216,7 @@ pub(super) async fn db_info( .await .and_then(|v| v.parse().ok()) .unwrap_or(3); - let resolution: i64 = get_setting( + let resolution: u32 = get_setting( &state.db, "backfill_concurrent_resolution", state.db_backend, @@ -224,8 +224,9 @@ pub(super) async fn db_info( .await .and_then(|v| v.parse().ok()) .unwrap_or(100); - let needed_backfill_pool = (pds * dids) + resolution + 4; - let restart_recommended = needed_backfill_pool > backfill_pool_size; + let would_be_pool_size = + crate::db::compute_backfill_pool_size(state.db_backend, pds, dids, resolution); + let restart_recommended = would_be_pool_size as i64 > backfill_pool_size; Ok(Json(serde_json::json!({ "backend": match state.db_backend { diff --git a/src/db.rs b/src/db.rs index 6eb5cbc..9572d69 100644 --- a/src/db.rs +++ b/src/db.rs @@ -283,33 +283,47 @@ pub async fn connect(url: &str, backend: DatabaseBackend) -> AnyPool { pool } -pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { - let max_connections: u32 = std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") +pub fn backfill_pool_ceiling(backend: DatabaseBackend) -> u32 { + match backend { + DatabaseBackend::Sqlite => 64, + DatabaseBackend::Postgres => 256, + } +} + +pub fn needed_backfill_connections(pds: u32, dids_per_pds: u32, resolution: u32) -> u32 { + (pds * dids_per_pds) + resolution + 4 +} + +pub fn compute_backfill_pool_size( + backend: DatabaseBackend, + pds: u32, + dids_per_pds: u32, + resolution: u32, +) -> u32 { + std::env::var("BACKFILL_DATABASE_MAX_CONNECTIONS") .ok() .and_then(|v| v.parse::().ok()) .unwrap_or_else(|| { - let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let dids: u32 = std::env::var("BACKFILL_CONCURRENT_DIDS_PER_PDS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(3); - let resolution: u32 = std::env::var("BACKFILL_CONCURRENT_RESOLUTION") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(100); - // Each concurrent worker may need a connection: PDS×DIDs for fetching + - // resolution concurrency + a few for bookkeeping queries. - let needed = (pds * dids) + resolution + 4; - let ceiling = match backend { - DatabaseBackend::Sqlite => 64, - DatabaseBackend::Postgres => 256, - }; - needed.min(ceiling) + needed_backfill_connections(pds, dids_per_pds, resolution) + .min(backfill_pool_ceiling(backend)) }) - .max(1); + .max(1) +} + +pub async fn connect_backfill_pool(url: &str, backend: DatabaseBackend) -> AnyPool { + let pds: u32 = std::env::var("BACKFILL_CONCURRENT_PDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10); + let dids: u32 = std::env::var("BACKFILL_CONCURRENT_DIDS_PER_PDS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + let resolution: u32 = std::env::var("BACKFILL_CONCURRENT_RESOLUTION") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(100); + let max_connections = compute_backfill_pool_size(backend, pds, dids, resolution); tracing::info!(max_connections, "backfill pool sized"); -- 2.51.2 From d38d05d672e0f692c013565cc49a32e7dd4cbd8f Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 14:53:01 -0500 Subject: [PATCH 27/64] fix: restore missing log events for verbose logging Signed-off-by: Trezy --- src/record_handler.rs | 63 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/record_handler.rs b/src/record_handler.rs index 34dacee..9631941 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -75,7 +75,28 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { .await; match hook_result { - None => return, + None => { + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "hook returned nil", + }), + }, + state.db_backend, + ) + .await; + } + return; + } Some(v) => v, } } else { @@ -119,6 +140,25 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { ) .await; + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.created".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + }), + }, + backend, + ) + .await; + } + crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), uri.clone()); } Err(e) => { @@ -187,7 +227,26 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); match sqlx::query(&delete_sql).bind(&uri).execute(db).await { - Ok(_) => {} + Ok(_) => { + if state.verbose_event_logging.load(Ordering::Relaxed) { + log_event( + db, + EventLog { + event_type: "record.deleted".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + }), + }, + backend, + ) + .await; + } + } Err(e) => { tracing::warn!(uri = %uri, "failed to delete record: {e}"); log_event( -- 2.51.2 From 7fb418411f760b6c60db3708e2324fc76b4c758d Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 14:57:36 -0500 Subject: [PATCH 28/64] test: add tests for backfill concurrency Signed-off-by: Trezy --- src/db.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/src/db.rs b/src/db.rs index 9572d69..a4a6938 100644 --- a/src/db.rs +++ b/src/db.rs @@ -637,6 +637,57 @@ mod tests { // parse_dt // ----------------------------------------------------------------------- + // ----------------------------------------------------------------------- + // backfill pool sizing + // ----------------------------------------------------------------------- + + #[test] + fn needed_backfill_connections_formula() { + assert_eq!(needed_backfill_connections(10, 3, 100), 134); + assert_eq!(needed_backfill_connections(1, 1, 1), 6); + assert_eq!(needed_backfill_connections(0, 0, 0), 4); + } + + #[test] + fn backfill_pool_ceiling_values() { + assert_eq!(backfill_pool_ceiling(DatabaseBackend::Sqlite), 64); + assert_eq!(backfill_pool_ceiling(DatabaseBackend::Postgres), 256); + } + + #[test] + fn needed_connections_capped_by_sqlite_ceiling() { + let needed = needed_backfill_connections(10, 3, 100); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Sqlite)); + assert_eq!(needed, 134); + assert_eq!(capped, 64); + } + + #[test] + fn needed_connections_capped_by_postgres_ceiling() { + let needed = needed_backfill_connections(50, 10, 200); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Postgres)); + assert_eq!(needed, 704); + assert_eq!(capped, 256); + } + + #[test] + fn needed_connections_below_ceiling_unchanged() { + let needed = needed_backfill_connections(2, 2, 10); + let capped = needed.min(backfill_pool_ceiling(DatabaseBackend::Postgres)); + assert_eq!(needed, 18); + assert_eq!(capped, 18); + } + + #[test] + fn needed_connections_minimum_is_overhead() { + let needed = needed_backfill_connections(0, 0, 0); + assert_eq!(needed, 4); + } + + // ----------------------------------------------------------------------- + // parse_dt + // ----------------------------------------------------------------------- + #[test] fn parse_dt_rfc3339() { let dt = parse_dt("2025-03-16T12:34:56Z"); -- 2.51.2 From 62adab05522156e4146fc32e6d36182033908127 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:36:15 -0500 Subject: [PATCH 29/64] fix: prevent backfill jobs from locking locking out pause/cancellation Signed-off-by: Trezy --- .../20260522000000_add_records_collection_indexed_at_index.sql | 2 ++ src/admin/backfill.rs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql diff --git a/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql b/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql new file mode 100644 index 0000000..6ac0175 --- /dev/null +++ b/migrations/postgres/20260522000000_add_records_collection_indexed_at_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_records_collection_indexed_at +ON records (collection, indexed_at DESC); diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 4e1165d..0418ea4 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -681,7 +681,6 @@ async fn run_pipelined_resolve_and_fetch( let poll_job_id = Arc::clone(&job_id_arc); let poll_cancelled = Arc::clone(&cancelled); let pair = tokio::select! { - biased; result = rx.recv() => result, _ = async move { loop { -- 2.51.2 From 1cf8f751eb20365e0a92ad48044d364a4f0d88ea Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:37:32 -0500 Subject: [PATCH 30/64] fix: reset backfill ui when flushing details Signed-off-by: Trezy --- web/src/app/dashboard/backfill/page.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 347feba..8ab0cb2 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -423,6 +423,8 @@ export default function BackfillPage() { Cancel { await flushAllBackfillDetails(); + setSelectedJobId(null); + load(); }}>Clear -- 2.51.2 From 455fbeccfff2bee26b1ac2e1ca51620c87c94e66 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 15:39:24 -0500 Subject: [PATCH 31/64] docs: update docs to cover backfill updates Signed-off-by: Trezy --- .../docs/api-reference/admin/admin-api.md | 16 +- .../docs/api-reference/admin/backfill.md | 150 +++++++++++++++++- .../docs/api-reference/admin/meta.json | 1 + .../docs/api-reference/admin/records.md | 117 ++++++++++++++ .../docs/api-reference/admin/settings.md | 49 +++++- .../docs/getting-started/configuration.md | 15 +- packages/docs/content/docs/guides/backfill.md | 34 +++- .../content/docs/reference/architecture.md | 57 ++++--- 8 files changed, 409 insertions(+), 30 deletions(-) create mode 100644 packages/docs/content/docs/api-reference/admin/records.md diff --git a/packages/docs/content/docs/api-reference/admin/admin-api.md b/packages/docs/content/docs/api-reference/admin/admin-api.md index 706bf14..409c601 100644 --- a/packages/docs/content/docs/api-reference/admin/admin-api.md +++ b/packages/docs/content/docs/api-reference/admin/admin-api.md @@ -50,7 +50,8 @@ AUTH="Authorization: Bearer $TOKEN" | [API Keys](api-keys.md) | Create, list, and revoke API keys | | [Users](users.md) | Create, list, update, and delete admin users | | [Labelers](labelers.md) | Manage external labeler subscriptions | -| [Instance Settings](settings.md) | Configure app name, logo, and policy URLs | +| [Records](records.md) | List and delete indexed records | +| [Instance Settings](settings.md) | Configure app name, logo, policy URLs, and concurrency settings | | [Domains](domains.md) | Manage domains and their OAuth client identities | | [Script Variables](script-variables.md) | Encrypted key/value pairs for Lua scripts | | [API Clients](api-clients.md) | Register and manage third-party XRPC clients | @@ -72,6 +73,14 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/stats` | `stats:read` | | `POST /admin/backfill` | `backfill:create` | | `GET /admin/backfill/status` | `backfill:read` | +| `POST /admin/backfill/{id}/cancel` | `backfill:create` | +| `POST /admin/backfill/{id}/pause` | `backfill:create` | +| `POST /admin/backfill/{id}/resume` | `backfill:create` | +| `GET /admin/backfill/{id}/events` | `backfill:read` | +| `GET /admin/backfill/{id}/repos` | `backfill:read` | +| `GET /admin/backfill/{id}/pds-summary` | `backfill:read` | +| `DELETE /admin/backfill/{id}/details` | `backfill:create` | +| `DELETE /admin/backfill/details` | `backfill:create` | | `GET /admin/events` | `events:read` | | `POST /admin/api-keys` | `api-keys:create` | | `GET /admin/api-keys` | `api-keys:read` | @@ -89,7 +98,12 @@ Each admin API endpoint requires a specific permission. See the [Permissions gui | `GET /admin/labelers` | `labelers:read` | | `PATCH /admin/labelers/{did}` | `labelers:create` | | `DELETE /admin/labelers/{did}` | `labelers:delete` | +| `GET /admin/records` | `records:read` | +| `GET /admin/records/collections` | `records:read` | +| `DELETE /admin/records` | `records:delete` | +| `DELETE /admin/records/collection` | `records:delete-collection`| | `GET /admin/settings` | `settings:manage` | +| `GET /admin/settings/db-info` | `settings:manage` | | `PUT /admin/settings/{key}` | `settings:manage` | | `DELETE /admin/settings/{key}` | `settings:manage` | | `PUT /admin/settings/logo` | `settings:manage` | diff --git a/packages/docs/content/docs/api-reference/admin/backfill.md b/packages/docs/content/docs/api-reference/admin/backfill.md index b4a5a2e..2749488 100644 --- a/packages/docs/content/docs/api-reference/admin/backfill.md +++ b/packages/docs/content/docs/api-reference/admin/backfill.md @@ -151,6 +151,52 @@ curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/cancel" -H "$AUTH" Returns `400` if the job is not currently running, or `404` if the job ID is not found. +## Pause a backfill job + +``` +POST /admin/backfill/{id}/pause +``` + +Requests a running backfill job to pause. The job status transitions to `pausing` immediately; the background worker will stop at its next checkpoint and set the status to `paused`. Paused jobs retain all progress and can be resumed later. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pause" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "pausing" +} +``` + +Returns `400` if the job is not currently running, or `404` if the job ID is not found. + +## Resume a backfill job + +``` +POST /admin/backfill/{id}/resume +``` + +Resume a paused backfill job. The job status transitions back to `running` and processing continues from where it left off. + +```sh tab="cURL" tab-group="language" +curl -X POST "http://127.0.0.1:3000/admin/backfill/$JOB_ID/resume" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "running" +} +``` + +Returns `400` if the job is not currently paused, or `404` if the job ID is not found. + ## List backfill jobs ``` @@ -165,6 +211,7 @@ interface BackfillJob { status: string; stage: string; total_repos: number | null; + resolved_repos: number | null; processed_repos: number | null; total_records: number | null; error: string | null; @@ -212,6 +259,7 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" "status": "completed", "stage": "completed", "total_repos": 42, + "resolved_repos": 42, "processed_repos": 42, "total_records": 1000, "error": null, @@ -222,4 +270,104 @@ curl http://127.0.0.1:3000/admin/backfill/status -H "$AUTH" ] ``` -The `status` field tracks the overall job state (`running`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, `cancelled`). +The `status` field tracks the overall job state (`running`, `pausing`, `paused`, `cancelling`, `cancelled`, `completed`, `failed`). The `stage` field tracks the current processing phase (`pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, `cancelled`). The `resolved_repos` counter tracks PDS resolution progress during the pipelined phase, while `processed_repos` tracks record fetching progress. + +## List repos for a job + +``` +GET /admin/backfill/{id}/repos +``` + +Paginated list of per-DID tracking rows for a backfill job. Requires `BackfillRead`. + +| Param | Type | Required | Description | +| -------- | ------ | -------- | ---------------------------------------------------------------------------------- | +| `phase` | string | no | Filter: `discovered` (all), `resolved` (PDS known), `fetched` (completed) | +| `cursor` | string | no | Keyset cursor (DID) for pagination | +| `limit` | number | no | Max results per page (default 50, max 100) | + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/repos?phase=fetched&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "repos": [ + { "did": "did:plc:abc", "pds_endpoint": "https://pds.example.com", "status": "completed", "records_fetched": 42 } + ], + "cursor": "did:plc:def" +} +``` + +`cursor` is `null` when there are no more results. + +## PDS summary for a job + +``` +GET /admin/backfill/{id}/pds-summary +``` + +Aggregated PDS breakdown for a backfill job. Requires `BackfillRead`. No pagination — returns all PDS endpoints in one response, sorted by repo count descending. + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/backfill/$JOB_ID/pds-summary" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "pds_endpoints": [ + { "pds_endpoint": "https://morel.us-east.host.bsky.network", "total_repos": 1200, "completed_repos": 800, "total_records": 5000 } + ] +} +``` + +## Stream backfill events (SSE) + +``` +GET /admin/backfill/{id}/events +``` + +Server-Sent Events stream of real-time backfill progress. Requires `BackfillRead`. The connection stays open until the job completes or the client disconnects. A keepalive comment is sent periodically to prevent timeouts. + +Events are sent with `event: event` and a JSON `data` payload. Each event has a `type` field: + +| Event type | Description | +| ------------------ | ------------------------------------------------- | +| `repo_discovered` | A new DID was found during the discovery phase | +| `repo_resolved` | A DID's PDS endpoint was resolved | +| `repo_fetched` | Record fetching completed for a DID | +| `job_counters` | Updated progress counters | +| `job_stage_changed`| The job moved to a new processing stage | +| `job_completed` | The job finished (completed, failed, or cancelled) | + +## Flush job details + +``` +DELETE /admin/backfill/{id}/details +``` + +Delete all per-repo tracking rows for a single backfill job. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/$JOB_ID/details" -H "$AUTH" +``` + +**Response**: `204 No Content` + +## Flush all job details + +``` +DELETE /admin/backfill/details +``` + +Delete per-repo tracking rows for all completed, cancelled, and failed backfill jobs. Requires `BackfillCreate`. + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/backfill/details" -H "$AUTH" +``` + +**Response**: `204 No Content` diff --git a/packages/docs/content/docs/api-reference/admin/meta.json b/packages/docs/content/docs/api-reference/admin/meta.json index 5582234..d5851f1 100644 --- a/packages/docs/content/docs/api-reference/admin/meta.json +++ b/packages/docs/content/docs/api-reference/admin/meta.json @@ -3,6 +3,7 @@ "pages": [ "admin-api", "lexicons", + "records", "stats", "backfill", "events", diff --git a/packages/docs/content/docs/api-reference/admin/records.md b/packages/docs/content/docs/api-reference/admin/records.md new file mode 100644 index 0000000..1a874aa --- /dev/null +++ b/packages/docs/content/docs/api-reference/admin/records.md @@ -0,0 +1,117 @@ +--- +title: "Records" +--- + +Browse and manage indexed records. All endpoints require the appropriate `records:*` permission. + +```sh tab="cURL" tab-group="language" +# All examples assume $TOKEN is an API key (hv_...) +AUTH="Authorization: Bearer $TOKEN" +``` + +## List records + +``` +GET /admin/records +``` + +Paginated list of records in a collection, ordered by `indexed_at` descending. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------- | +| `collection` | string | yes | Collection NSID to list records from | +| `limit` | number | no | Max results per page (default 20, max 100)| +| `cursor` | string | no | Pagination cursor from a previous response| + +```sh tab="cURL" tab-group="language" +curl "http://127.0.0.1:3000/admin/records?collection=xyz.statusphere.status&limit=10" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "records": [ + { + "uri": "at://did:plc:abc/xyz.statusphere.status/3k...", + "did": "did:plc:abc", + "collection": "xyz.statusphere.status", + "rkey": "3k...", + "cid": "bafyrei...", + "indexed_at": "2025-01-01T00:00:00Z", + "record": { "...": "..." }, + "labels": [] + } + ], + "cursor": "20" +} +``` + +`cursor` is omitted when there are no more results. + +## List collections + +``` +GET /admin/records/collections +``` + +Returns the list of collection NSIDs from registered record-type lexicons. This is a fast lookup (no record counting). + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/records/collections -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "collections": [ + "xyz.statusphere.status", + "app.bsky.feed.post" + ] +} +``` + +## Delete a record + +``` +DELETE /admin/records +``` + +Delete a single record by AT URI. + +| Param | Type | Required | Description | +| ----- | ------ | -------- | ------------------------------------ | +| `uri` | string | yes | AT URI of the record to delete | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records?uri=at://did:plc:abc/xyz.statusphere.status/3k..." -H "$AUTH" +``` + +**Response**: `204 No Content` + +Returns `404` if the record is not found. + +## Delete all records in a collection + +``` +DELETE /admin/records/collection +``` + +Delete all indexed records for a given collection. Requires both `records:delete` and `records:delete-collection` permissions. + +| Param | Type | Required | Description | +| ------------ | ------ | -------- | ------------------------------------ | +| `collection` | string | yes | Collection NSID to delete from | + +```sh tab="cURL" tab-group="language" +curl -X DELETE "http://127.0.0.1:3000/admin/records/collection?collection=xyz.statusphere.status" -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "deleted": 42 +} +``` diff --git a/packages/docs/content/docs/api-reference/admin/settings.md b/packages/docs/content/docs/api-reference/admin/settings.md index 189e40d..d4ecb49 100644 --- a/packages/docs/content/docs/api-reference/admin/settings.md +++ b/packages/docs/content/docs/api-reference/admin/settings.md @@ -58,7 +58,22 @@ resp, err := http.DefaultClient.Do(req) curl http://127.0.0.1:3000/admin/settings -H "$AUTH" ``` -Returns all key/value pairs stored in the `instance_settings` table. +Returns all key/value pairs stored in the `instance_settings` table, plus any env-var fallback values for keys not stored in the database. Each entry includes a `source` field: `"database"` for stored values, `"env"` for env-var fallbacks. + +### Known settings + +| Key | Env var | Default | Description | +|-----|---------|---------|-------------| +| `app_name` | `APP_NAME` | --- | Application name shown in sidebar and OAuth consent screen | +| `client_uri` | `CLIENT_URI` | --- | Public URL for this instance, linked from OAuth consent screen | +| `logo_uri` | `LOGO_URI` | --- | External URL to a logo image | +| `tos_uri` | `TOS_URI` | --- | Link to terms of service | +| `policy_uri` | `POLICY_URI` | --- | Link to privacy policy | +| `backfill_concurrent_pds` | `BACKFILL_CONCURRENT_PDS` | `10` | How many PDS servers to fetch from simultaneously during backfill | +| `backfill_concurrent_dids_per_pds` | `BACKFILL_CONCURRENT_DIDS_PER_PDS` | `3` | How many repos to fetch concurrently from each PDS | +| `backfill_concurrent_resolution` | `BACKFILL_CONCURRENT_RESOLUTION` | `100` | How many DID document lookups to run in parallel during PDS resolution | +| `backfill_retention_days` | `BACKFILL_RETENTION_DAYS` | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` = keep indefinitely | +| `verbose_event_logging` | `VERBOSE_EVENT_LOGGING` | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging | ## Upsert a setting @@ -116,6 +131,38 @@ DELETE /admin/settings/{key} Removes the override; the corresponding environment variable (if any) takes effect again. +## Database info + +``` +GET /admin/settings/db-info +``` + +Returns database backend, connection pool sizes, and whether a server restart is recommended to resize the backfill pool for current concurrency settings. + +```sh tab="cURL" tab-group="language" +curl http://127.0.0.1:3000/admin/settings/db-info -H "$AUTH" +``` + +**Response**: `200 OK` + +```json +{ + "backend": "sqlite", + "server_max_connections": null, + "main_pool_size": 32, + "backfill_pool_size": 64, + "restart_recommended": false +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `backend` | string | `"sqlite"` or `"postgres"` | +| `server_max_connections` | number \| null | Postgres `max_connections` setting. `null` for SQLite | +| `main_pool_size` | number | Current main connection pool size | +| `backfill_pool_size` | number | Current backfill connection pool size | +| `restart_recommended` | boolean | `true` if concurrency settings have changed and a restart would resize the pool | + ## Upload / delete logo ``` diff --git a/packages/docs/content/docs/getting-started/configuration.md b/packages/docs/content/docs/getting-started/configuration.md index 5b40060..207a888 100644 --- a/packages/docs/content/docs/getting-started/configuration.md +++ b/packages/docs/content/docs/getting-started/configuration.md @@ -26,7 +26,13 @@ HappyView is configured via environment variables. A `.env` file in the project | `ATTESTATION_PRIVATE_KEY` | no | auto-generated | Hex-encoded 32-byte secp256k1 private key for [attestation signing](../guides/attestation-signing.md). Auto-generated and persisted to database on first run | | `ATTESTATION_KEY_ID` | no | `did:web:{host}#attestation` | Key identifier included in attestation signatures. Derived from `PUBLIC_URL` by default | | `ATTESTATION_SIG_TYPE` | no | app-specific NSID | `$type` value used in attestation signature objects | -| `RUST_LOG` | no | `happyview=debug,tower_http=debug` | Log filter (uses `tracing_subscriber::EnvFilter`) | +| `BACKFILL_CONCURRENT_PDS` | no | `10` | How many PDS servers to fetch from simultaneously during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_DIDS_PER_PDS` | no | `3` | How many repos to fetch concurrently from each PDS during backfill. Overridden by database setting if set via admin API | +| `BACKFILL_CONCURRENT_RESOLUTION` | no | `100` | How many DID document lookups to run in parallel during PDS resolution. Overridden by database setting if set via admin API | +| `BACKFILL_RETENTION_DAYS` | no | `28` | Days to keep per-repo detail data from completed backfill jobs. `0` to keep indefinitely. Overridden by database setting if set via admin API | +| `BACKFILL_DATABASE_MAX_CONNECTIONS` | no | auto-calculated | Override the backfill connection pool size. Auto-calculated from concurrency settings if not set | +| `VERBOSE_EVENT_LOGGING` | no | `false` | Log every record index, hook execution, and hook skip to the event log. High write volume — recommended only for debugging. Overridden by database setting if set via admin API | +| `RUST_LOG` | no | `happyview=debug,tower_http=debug,sqlx=warn` | Log filter (uses `tracing_subscriber::EnvFilter`) | | `APP_NAME` | no | --- | Application name shown on OAuth authorization screens. Overridden by database setting if set via admin API | | `LOGO_URI` | no | --- | URL to application logo for OAuth screens. Overridden by database setting or logo upload | | `TOS_URI` | no | --- | URL to terms of service. Overridden by database setting if set via admin API | @@ -59,7 +65,12 @@ SESSION_SECRET=change-me-in-production # TOKEN_ENCRYPTION_KEY=base64-encoded-32-byte-key # DEFAULT_RATE_LIMIT_CAPACITY=100 # DEFAULT_RATE_LIMIT_REFILL_RATE=2.0 -# RUST_LOG=happyview=debug,tower_http=debug +# BACKFILL_CONCURRENT_PDS=10 +# BACKFILL_CONCURRENT_DIDS_PER_PDS=3 +# BACKFILL_CONCURRENT_RESOLUTION=100 +# BACKFILL_RETENTION_DAYS=28 +# VERBOSE_EVENT_LOGGING=false +# RUST_LOG=happyview=debug,tower_http=debug,sqlx=warn # APP_NAME=My App # LOGO_URI=https://example.com/logo.png # TOS_URI=https://example.com/tos diff --git a/packages/docs/content/docs/guides/backfill.md b/packages/docs/content/docs/guides/backfill.md index 642343a..414ebc0 100644 --- a/packages/docs/content/docs/guides/backfill.md +++ b/packages/docs/content/docs/guides/backfill.md @@ -13,13 +13,12 @@ See the [admin API](../api-reference/admin/backfill.md) for endpoint details. ## How it works -A backfill job runs through three sequential phases: +A backfill job starts with a discovery phase and then pipelines resolution and fetching concurrently: 1. **Discovering repos** — HappyView calls the relay's `com.atproto.sync.listReposByCollection` to find repos that contain records for each target collection. Discovered DIDs are stored in a tracking table so progress can be resumed. -2. **Resolving PDS** — For each discovered DID, HappyView resolves the DID document (via PLC directory or `did:web`) to find the user's PDS endpoint. -3. **Fetching records** — HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). +2. **Resolving PDS + Fetching records** (pipelined) — Resolution and fetching run concurrently. As each DID is resolved (via PLC directory or `did:web`), it's immediately handed off for record fetching — there's no need to wait for all DIDs to resolve before fetching begins. HappyView calls `com.atproto.repo.listRecords` on each PDS for the target collection(s), upserting each record into the local database. PDS endpoints are processed concurrently (up to 10 PDS hosts, 3 DIDs per host). -Progress counters (`total_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. +Progress counters (`total_repos`, `resolved_repos`, `processed_repos`, `total_records`) and the current `stage` are updated in real time. The dashboard's Backfill page shows live progress, and clicking a job opens a detail sheet with a stage-by-stage progress log. ### Rate limiting @@ -37,7 +36,7 @@ A backfill job has both a `status` (overall state) and a `stage` (current phase) | `completed` | Worker finished processing all resolvable repos | | `failed` | An error occurred | -The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_pds`, `fetching_records`, `completed`, `failed`, or `cancelled`. +The `stage` field tracks which phase the job is in: `pending`, `discovering_repos`, `resolving_and_fetching`, `completed`, `failed`, or `cancelled`. ## Cancelling a job @@ -65,6 +64,31 @@ Re-running a backfill for a collection that's already been backfilled is safe. E Deleting records from HappyView (via the dashboard or API) only removes them from the local database — the records still exist on the atproto network. To restore deleted records, create a backfill job for the affected collection. The backfill will re-discover the repos and re-fetch all records from each PDS, restoring any that were previously deleted. +## Diagnostics + +The dashboard's backfill detail panel includes expandable sections for each processing phase. Clicking a phase row reveals per-repo and per-PDS detail data in real time. + +### Per-repo tracking + +Every DID discovered during a backfill job is tracked in the database with its PDS endpoint, processing status, and record count. This data powers three expandable sections: + +- **Discovering repos** — lists all DIDs discovered for the job, with profile avatars and handles resolved from the Bluesky API. +- **Resolving PDS** — summarises PDS endpoints involved in the job, showing how many repos each PDS is responsible for and how many have been processed. +- **Fetching records** — lists completed repos with their record counts and PDS hostnames. + +All three sections update in real time via SSE (Server-Sent Events) while the job is running. + +### Data retention + +Per-repo detail data is retained after job completion to support post-mortem analysis. A background task runs daily and deletes detail rows for jobs completed more than 28 days ago (configurable via the `backfill_retention_days` setting in **Settings > General**, or the `BACKFILL_RETENTION_DAYS` environment variable). Set to `0` to keep data indefinitely. + +You can also manually clear detail data: + +- **Per-job**: "Clear details" button in the job detail panel footer. +- **All completed jobs**: "Clear all details" button on the Backfill page header. + +Both actions require the `backfill:create` permission. + ## Next steps - [Lexicons](lexicons.md#backfill-flag): Control whether lexicons trigger backfill on upload diff --git a/packages/docs/content/docs/reference/architecture.md b/packages/docs/content/docs/reference/architecture.md index ab58e21..0563cef 100644 --- a/packages/docs/content/docs/reference/architecture.md +++ b/packages/docs/content/docs/reference/architecture.md @@ -154,16 +154,31 @@ sequenceDiagram A->>H: POST /admin/backfill H->>D: Create backfill_jobs record (status = running) + + rect rgb(240, 248, 255) + note over H,Relay: Phase 1: Discovery H->>Relay: listReposByCollection (paginated) Relay-->>H: List of DIDs - loop For each DID - H->>PLC: Resolve DID document - PLC-->>H: PDS endpoint - H->>PDS: listRecords (paginated) - PDS-->>H: Records - H->>D: UPSERT each record - H->>D: Update processed_repos / total_records + H->>D: Insert backfill_repos rows end + + rect rgb(240, 255, 240) + note over H,PDS: Phase 2: Pipelined resolve + fetch (concurrent) + par Resolver task + loop Unresolved DIDs + H->>PLC: Resolve DID document + PLC-->>H: PDS endpoint + H->>D: Update backfill_repos.pds_endpoint + end + and Fetcher task + loop Resolved DIDs (as they arrive) + H->>PDS: listRecords (paginated) + PDS-->>H: Records + H->>D: UPSERT each record + end + end + end + H->>D: Mark job completed (or failed) ``` @@ -274,19 +289,21 @@ sequenceDiagram ### `backfill_jobs` -| Column | Type | Description | -| ----------------- | ----------- | ----------------------------------- | -| `id` | uuid (PK) | | -| `collection` | text | Target collection (null = all) | -| `did` | text | Target DID (null = all) | -| `status` | text | pending, running, completed, failed | -| `total_repos` | integer | | -| `processed_repos` | integer | | -| `total_records` | integer | | -| `error` | text | Error message if failed | -| `started_at` | timestamptz | | -| `completed_at` | timestamptz | | -| `created_at` | timestamptz | | +| Column | Type | Description | +| ----------------- | ----------- | -------------------------------------------------------- | +| `id` | uuid (PK) | | +| `collection` | text | Target collection (null = all) | +| `did` | text | Target DID (null = all) | +| `status` | text | pending, running, pausing, paused, cancelling, cancelled, completed, failed | +| `stage` | text | pending, discovering_repos, resolving_and_fetching, completed, failed, cancelled | +| `total_repos` | integer | Total DIDs discovered | +| `resolved_repos` | integer | DIDs with PDS endpoint resolved | +| `processed_repos` | integer | DIDs with records fetched | +| `total_records` | integer | Total records indexed | +| `error` | text | Error message if failed | +| `started_at` | timestamptz | | +| `completed_at` | timestamptz | | +| `created_at` | timestamptz | | ## Testing -- 2.51.2 From e8268c4038d3724492895253aa0686aa06b8202b Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 18:05:55 -0500 Subject: [PATCH 32/64] fix: allow log verbosity toggle to override env var Signed-off-by: Trezy --- web/src/app/dashboard/settings/general/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/src/app/dashboard/settings/general/page.tsx b/web/src/app/dashboard/settings/general/page.tsx index 47ebe58..1319841 100644 --- a/web/src/app/dashboard/settings/general/page.tsx +++ b/web/src/app/dashboard/settings/general/page.tsx @@ -481,7 +481,7 @@ export default function GeneralSettingsPage() { onCheckedChange={(checked) => setValues((v) => ({ ...v, - verbose_event_logging: checked ? "true" : "", + verbose_event_logging: checked ? "true" : "false", })) } disabled={!canManage} -- 2.51.2 From 016b0e529445031bc2481285b2c534eb9da4cd2a Mon Sep 17 00:00:00 2001 From: Trezy Date: Sat, 23 May 2026 20:51:50 -0500 Subject: [PATCH 33/64] fix: prevent opening "discovering trepos stage" from locking the ui thread Signed-off-by: Trezy --- src/admin/backfill.rs | 20 +++- web/src/app/dashboard/backfill/page.tsx | 129 ++++++++++++++---------- web/src/workers/backfill-sse.worker.ts | 8 +- 3 files changed, 93 insertions(+), 64 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 0418ea4..24d48ee 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -325,7 +325,7 @@ async fn run_discovery_phase( ); } else { stream::iter(collections.iter()) - .for_each_concurrent(collections.len(), |collection| async move { + .for_each_concurrent(5, |collection| async move { if should_stop_worker(state, job_id).await { return; } @@ -432,6 +432,16 @@ async fn discover_repos_from_relay( } update_job_counter(state, job_id, "total_repos", running_total).await; + publish_event( + state, + super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: Some(running_total), + resolved_repos: None, + processed_repos: None, + total_records: None, + }, + ); if should_stop_worker(state, job_id).await { return Ok(()); @@ -1283,6 +1293,7 @@ async fn fetch_records_from_pds( let base = pds_endpoint.trim_end_matches('/'); let mut cursor: Option = None; let mut count: u32 = 0; + let index_hook = state.lexicons.get_index_hook(collection).await; loop { if cancelled.load(Ordering::Relaxed) { @@ -1326,12 +1337,11 @@ async fn fetch_records_from_pds( let rkey = entry.uri.rsplit('/').next().unwrap_or_default().to_string(); let uri = format!("at://{did}/{collection}/{rkey}"); - let rec_to_store = if let Some(script) = state.lexicons.get_index_hook(collection).await - { + let rec_to_store = if let Some(ref script) = index_hook { let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { state, lexicon_id: collection, - script: &script, + script, action: "create", uri: &uri, did, @@ -1781,7 +1791,7 @@ pub(super) async fn backfill_status( Option, String, )> = sqlx::query_as(&sql) - .fetch_all(&state.db) + .fetch_all(&state.backfill_db) .await .map_err(|e| AppError::Internal(format!("failed to list backfill jobs: {e}")))?; diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 8ab0cb2..640c532 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -161,7 +161,7 @@ function useBackfillSSE( }); const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ""; - worker.postMessage({ type: "connect", jobId, basePath }); + worker.postMessage({ type: "connect", jobId, baseUrl: `${location.origin}${basePath}` }); return () => { worker.postMessage({ type: "disconnect" }); @@ -390,12 +390,14 @@ export default function BackfillPage() { load(); }, [load]); + const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + const sseActive = selectedJob != null && (selectedJob.status === "running" || selectedJob.status === "cancelling" || selectedJob.status === "pausing"); + useEffect(() => { + if (sseActive) return; const interval = setInterval(load, 5000); return () => clearInterval(interval); - }, [load]); - - const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + }, [load, sseActive]); const canFlush = hasPermission("backfill:create"); return ( @@ -496,7 +498,10 @@ export default function BackfillPage() { { - if (!open) setSelectedJobId(null); + if (!open) { + setSelectedJobId(null); + load(); + } }} > @@ -505,6 +510,13 @@ export default function BackfillPage() { job={selectedJob} canCancel={hasPermission("backfill:create")} canFlush={canFlush} + onJobUpdate={(updater) => { + setJobs((prev) => + prev.map((j) => + j.id === selectedJob.id ? updater(j) : j, + ), + ); + }} onCancel={async () => { await cancelBackfillJob(selectedJob.id); load(); @@ -530,6 +542,7 @@ function JobDetail({ job, canCancel, canFlush, + onJobUpdate, onCancel, onPause, onResume, @@ -537,6 +550,7 @@ function JobDetail({ job: BackfillJob; canCancel: boolean; canFlush: boolean; + onJobUpdate: (updater: (job: BackfillJob) => BackfillJob) => void; onCancel: () => Promise; onPause: () => Promise; onResume: () => Promise; @@ -561,10 +575,11 @@ function JobDetail({ const [fetchedCursor, setFetchedCursor] = useState(null); const [fetchedLoaded, setFetchedLoaded] = useState(false); - // Refs for open state so the SSE callback doesn't need to re-bind on toggle - const discoveredOpenRef = useRef(false); + // Refs for open state and callbacks so the SSE callback doesn't need to re-bind on toggle const pdsOpenRef = useRef(false); const fetchedOpenRef = useRef(false); + const onJobUpdateRef = useRef(onJobUpdate); + onJobUpdateRef.current = onJobUpdate; function hasReached(phase: (typeof PROGRESS_PHASES)[number]): boolean { if (allDone) return true; @@ -626,7 +641,6 @@ function JobDetail({ const [discoveredOpen, setDiscoveredOpen] = useState(false); const [pdsOpen, setPdsOpen] = useState(false); const [fetchedOpen, setFetchedOpen] = useState(false); - discoveredOpenRef.current = discoveredOpen; pdsOpenRef.current = pdsOpen; fetchedOpenRef.current = fetchedOpen; @@ -677,52 +691,33 @@ function JobDetail({ // Uses refs for open state so the callback identity is stable and doesn't // cause the worker to reconnect when sections are toggled. const handleSSEBatch = useCallback((events: BackfillEvent[]) => { - const dOpen = discoveredOpenRef.current; const pOpen = pdsOpenRef.current; const fOpen = fetchedOpenRef.current; - if (dOpen) { - const discovered = events.filter((e) => e.type === "repo_discovered" && e.did); - const resolved = events.filter((e) => e.type === "repo_resolved" && e.did); - if (discovered.length > 0 || resolved.length > 0) { - setDiscoveredRepos((prev) => { - let next = prev; - for (const e of discovered) { - if (!next.some((r) => r.did === e.did)) { - next = [{ did: e.did!, pds_endpoint: null, status: "pending", records_fetched: 0 }, ...next]; - } - } - for (const e of resolved) { - next = next.map((r) => r.did === e.did ? { ...r, pds_endpoint: e.pds_endpoint! } : r); - } - return next; - }); - } - } - if (pOpen) { const pdsEvents = events.filter( (e) => (e.type === "repo_resolved" || e.type === "repo_fetched") && e.pds_endpoint, ); if (pdsEvents.length > 0) { setPdsSummary((prev) => { - let next = [...prev]; + const byEndpoint = new Map(prev.map((p, i) => [p.pds_endpoint, i])); + const next = [...prev]; for (const e of pdsEvents) { - const idx = next.findIndex((p) => p.pds_endpoint === e.pds_endpoint); + const idx = byEndpoint.get(e.pds_endpoint!); if (e.type === "repo_resolved") { - if (idx >= 0) { + if (idx != null) { next[idx] = { ...next[idx], total_repos: next[idx].total_repos + 1 }; } else { + const newIdx = next.length; next.push({ pds_endpoint: e.pds_endpoint!, total_repos: 1, completed_repos: 0, total_records: 0 }); + byEndpoint.set(e.pds_endpoint!, newIdx); } - } else if (e.type === "repo_fetched") { - if (idx >= 0) { - next[idx] = { - ...next[idx], - completed_repos: next[idx].completed_repos + 1, - total_records: next[idx].total_records + (e.records_fetched ?? 0), - }; - } + } else if (e.type === "repo_fetched" && idx != null) { + next[idx] = { + ...next[idx], + completed_repos: next[idx].completed_repos + 1, + total_records: next[idx].total_records + (e.records_fetched ?? 0), + }; } } return next; @@ -734,16 +729,44 @@ function JobDetail({ const fetched = events.filter((e) => e.type === "repo_fetched" && e.did); if (fetched.length > 0) { setFetchedRepos((prev) => { - let next = prev; - for (const e of fetched) { - if (!next.some((r) => r.did === e.did)) { - next = [{ did: e.did!, pds_endpoint: e.pds_endpoint ?? null, status: "completed", records_fetched: e.records_fetched ?? 0 }, ...next]; - } - } - return next; + const existing = new Set(prev.map((r) => r.did)); + const newItems = fetched + .filter((e) => !existing.has(e.did!)) + .map((e) => ({ did: e.did!, pds_endpoint: e.pds_endpoint ?? null, status: "completed" as const, records_fetched: e.records_fetched ?? 0 })); + + if (newItems.length === 0) return prev; + return [...newItems, ...prev]; }); } } + + // Update job counters, stage, and status from SSE + const update = onJobUpdateRef.current; + + // Increment total_repos from repo_discovered events + const discoveredCount = events.filter((e) => e.type === "repo_discovered").length; + const resolvedCount = events.filter((e) => e.type === "repo_resolved").length; + const fetchedEvents = events.filter((e) => e.type === "repo_fetched"); + const fetchedCount = fetchedEvents.length; + const fetchedRecords = fetchedEvents.reduce((sum, e) => sum + (e.records_fetched ?? 0), 0); + + if (discoveredCount > 0 || resolvedCount > 0 || fetchedCount > 0) { + update((j) => ({ + ...j, + total_repos: (j.total_repos ?? 0) + discoveredCount, + resolved_repos: (j.resolved_repos ?? 0) + resolvedCount, + processed_repos: (j.processed_repos ?? 0) + fetchedCount, + total_records: (j.total_records ?? 0) + fetchedRecords, + })); + } + + for (const e of events) { + if (e.type === "job_stage_changed" && e.stage) { + update((j) => ({ ...j, stage: e.stage! })); + } else if (e.type === "job_completed" && e.status) { + update((j) => ({ ...j, status: e.status!, error: e.error ?? null })); + } + } }, []); useBackfillSSE(job.id, isActive, handleSSEBatch); @@ -837,7 +860,6 @@ function JobDetail({ {discoveredRepos.length > 0 ? ( 0 ? ( - {job.status === "pausing" ? "Pausing…" : "Pause Job"} + {pausing || job.status === "pausing" ? "Pausing…" : "Pause Job"} )} {canCancel && job.status === "paused" && ( @@ -1015,7 +1036,6 @@ function JobDetail({ function VirtualRepoList({ repos, - profiles, onVisibleDidsChange, hasMore, onLoadMore, @@ -1023,7 +1043,6 @@ function VirtualRepoList({ renderRow, }: { repos: BackfillRepoEntry[]; - profiles: Map; onVisibleDidsChange: (dids: string[]) => void; hasMore: boolean; onLoadMore: () => void; @@ -1042,10 +1061,10 @@ function VirtualRepoList({ const virtualItems = virtualizer.getVirtualItems(); + const visibleDidsKey = virtualItems.map((item) => repos[item.index]?.did).filter(Boolean).join(","); useEffect(() => { - const dids = virtualItems.map((item) => repos[item.index]?.did).filter(Boolean) as string[]; - onVisibleDidsChange(dids); - }, [virtualItems, repos, onVisibleDidsChange]); + onVisibleDidsChange(visibleDidsKey.split(",").filter(Boolean)); + }, [visibleDidsKey, onVisibleDidsChange]); useEffect(() => { if (!hasMore) return; diff --git a/web/src/workers/backfill-sse.worker.ts b/web/src/workers/backfill-sse.worker.ts index cfdcc82..36d4040 100644 --- a/web/src/workers/backfill-sse.worker.ts +++ b/web/src/workers/backfill-sse.worker.ts @@ -8,7 +8,7 @@ interface BackfillEvent { interface ConnectMessage { type: "connect"; jobId: string; - basePath: string; + baseUrl: string; } interface DisconnectMessage { @@ -30,10 +30,10 @@ function flush() { self.postMessage({ type: "batch", events: batch }); } -function connect(jobId: string, basePath: string) { +function connect(jobId: string, baseUrl: string) { disconnect(); - eventSource = new EventSource(`${basePath}/admin/backfill/${jobId}/events`, { + eventSource = new EventSource(`${baseUrl}/admin/backfill/${jobId}/events`, { withCredentials: true, }); @@ -68,7 +68,7 @@ function disconnect() { self.addEventListener("message", (e: MessageEvent) => { const msg = e.data; if (msg.type === "connect") { - connect(msg.jobId, msg.basePath); + connect(msg.jobId, msg.baseUrl); } else if (msg.type === "disconnect") { disconnect(); } -- 2.51.2 From e12f526835555e9126e4657446cdc4cec9945fbf Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 11:33:33 -0500 Subject: [PATCH 34/64] fix(sdk): support the `kid` param from `@atproto/jwk-webcrypto` Signed-off-by: Trezy --- packages/oauth-client/src/__tests__/session.test.ts | 2 +- packages/oauth-client/src/import-jwk.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/oauth-client/src/__tests__/session.test.ts b/packages/oauth-client/src/__tests__/session.test.ts index ef73e26..9177a28 100644 --- a/packages/oauth-client/src/__tests__/session.test.ts +++ b/packages/oauth-client/src/__tests__/session.test.ts @@ -8,7 +8,7 @@ async function generateTestKey(): Promise { true, ["sign", "verify"], ); - return WebcryptoKey.fromKeypair(keyPair); + return WebcryptoKey.fromKeypair(keyPair, crypto.randomUUID()); } function createSession(overrides?: { diff --git a/packages/oauth-client/src/import-jwk.ts b/packages/oauth-client/src/import-jwk.ts index 764b349..be7edb7 100644 --- a/packages/oauth-client/src/import-jwk.ts +++ b/packages/oauth-client/src/import-jwk.ts @@ -23,5 +23,6 @@ export async function importJwk(jwk: JsonWebKey): Promise { ["verify"], ); - return WebcryptoKey.fromKeypair({ privateKey, publicKey }); + const kid = (jwk as JsonWebKey & { kid?: string }).kid ?? crypto.randomUUID(); + return WebcryptoKey.fromKeypair({ privateKey, publicKey }, kid); } -- 2.51.2 From ec860a528ce455f3c743913dbbf0a1d01b1f9c95 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 11:36:42 -0500 Subject: [PATCH 35/64] fix: reduce backfill db contention and sse event flooding for pds discovery and record fetching during backfill Signed-off-by: Trezy --- src/admin/backfill.rs | 38 +++-- src/admin/dead_letters.rs | 1 + src/lua/db_api.rs | 195 ++++++++++++++++++++++++ src/lua/execute.rs | 43 +++++- src/lua/mod.rs | 3 +- src/main.rs | 2 +- src/record_handler.rs | 2 + web/src/app/dashboard/backfill/page.tsx | 167 +++++++------------- 8 files changed, 327 insertions(+), 124 deletions(-) diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 24d48ee..5d6cb36 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -338,6 +338,16 @@ async fn run_discovery_phase( let total = count_repos(state, job_id).await; update_job_counter(state, job_id, "total_repos", total).await; + publish_event( + state, + super::types::BackfillEvent::JobCounters { + job_id: job_id.to_string(), + total_repos: Some(total), + resolved_repos: None, + processed_repos: None, + total_records: None, + }, + ); } async fn discover_repos_from_relay( @@ -419,15 +429,6 @@ async fn discover_repos_from_relay( if let Ok(result) = query.execute(&state.backfill_db).await { running_total += result.rows_affected() as i32; } - for repo in chunk { - publish_event( - state, - super::types::BackfillEvent::RepoDiscovered { - job_id: job_id.to_string(), - did: repo.did.clone(), - }, - ); - } } } @@ -1275,9 +1276,20 @@ async fn batch_upsert_records(state: &AppState, batch: &[PreparedRecord]) { let _ = ref_query.execute(&state.backfill_db).await; } - // Queue label backfill for each record - for rec in batch { - crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), rec.uri.clone()); + // Queue label backfill only if there are active labeler subscriptions. + // Check once per batch instead of spawning a task per record. + let has_subscriptions: bool = sqlx::query_as::<_, (i64,)>( + "SELECT COUNT(*) FROM labeler_subscriptions WHERE status = 'active'", + ) + .fetch_one(&state.db) + .await + .map(|(c,)| c > 0) + .unwrap_or(false); + + if has_subscriptions { + for rec in batch { + crate::labeler::backfill_labels_for_uri(Arc::new(state.clone()), rec.uri.clone()); + } } } @@ -1294,6 +1306,7 @@ async fn fetch_records_from_pds( let mut cursor: Option = None; let mut count: u32 = 0; let index_hook = state.lexicons.get_index_hook(collection).await; + let env_vars = crate::lua::load_env_vars_cached(&state.db, state.db_backend).await; loop { if cancelled.load(Ordering::Relaxed) { @@ -1348,6 +1361,7 @@ async fn fetch_records_from_pds( collection, rkey: &rkey, record: Some(&entry.value), + cached_env_vars: Some(&env_vars), }) .await; diff --git a/src/admin/dead_letters.rs b/src/admin/dead_letters.rs index 7a79d0f..1ddf68d 100644 --- a/src/admin/dead_letters.rs +++ b/src/admin/dead_letters.rs @@ -527,6 +527,7 @@ async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { collection: &dl.collection, rkey: &dl.rkey, record: record.as_ref(), + cached_env_vars: None, }; match run_hook_once(&event).await { diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 5e7f062..3afea15 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -42,6 +42,7 @@ fn is_valid_json_field_path(path: &str) -> bool { true } +#[derive(Debug)] enum FilterNode { Condition { field: String, @@ -945,6 +946,200 @@ mod tests { assert!(super::decode_cursor(&encoded).is_none()); } + // ----------------------------------------------------------------------- + // parse_filter_node / build_filter_sql + // ----------------------------------------------------------------------- + + fn make_condition_table(lua: &Lua, field: &str, op: &str, value: &str) -> mlua::Table { + let t = lua.create_table().unwrap(); + t.set("field", field).unwrap(); + t.set("op", op).unwrap(); + t.set("value", value).unwrap(); + t + } + + #[test] + fn filter_simple_condition() { + let lua = Lua::new(); + let t = make_condition_table(&lua, "name", "=", "alice"); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!(sql, "json_extract(record, '$.value.name') = ?"); + assert_eq!(binds, vec!["alice"]); + } + + #[test] + fn filter_defaults_op_to_equals() { + let lua = Lua::new(); + let t = lua.create_table().unwrap(); + t.set("field", "status").unwrap(); + t.set("value", "active").unwrap(); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!(sql, "json_extract(record, '$.value.status') = ?"); + } + + #[test] + fn filter_rejects_invalid_op() { + let lua = Lua::new(); + let t = make_condition_table(&lua, "name", "DROP", "x"); + let err = parse_filter_node(&t, 0).unwrap_err(); + assert!(err.to_string().contains("invalid filter op")); + } + + #[test] + fn filter_rejects_invalid_field() { + let lua = Lua::new(); + let t = make_condition_table(&lua, "name; DROP TABLE", "=", "x"); + let err = parse_filter_node(&t, 0).unwrap_err(); + assert!(err.to_string().contains("invalid filter field")); + } + + #[test] + fn filter_and_group() { + let lua = Lua::new(); + let group = lua.create_table().unwrap(); + group.set("combine", "AND").unwrap(); + let c1 = make_condition_table(&lua, "status", "=", "active"); + let c2 = make_condition_table(&lua, "age", ">", "18"); + group.set(1, c1).unwrap(); + group.set(2, c2).unwrap(); + let node = parse_filter_node(&group, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!( + sql, + "(json_extract(record, '$.value.status') = ? AND json_extract(record, '$.value.age') > ?)" + ); + assert_eq!(binds, vec!["active", "18"]); + } + + #[test] + fn filter_or_group() { + let lua = Lua::new(); + let group = lua.create_table().unwrap(); + group.set("combine", "OR").unwrap(); + let c1 = make_condition_table(&lua, "role", "=", "admin"); + let c2 = make_condition_table(&lua, "role", "=", "mod"); + group.set(1, c1).unwrap(); + group.set(2, c2).unwrap(); + let node = parse_filter_node(&group, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!( + sql, + "(json_extract(record, '$.value.role') = ? OR json_extract(record, '$.value.role') = ?)" + ); + assert_eq!(binds, vec!["admin", "mod"]); + } + + #[test] + fn filter_single_child_group_unwraps() { + let lua = Lua::new(); + let group = lua.create_table().unwrap(); + group.set("combine", "AND").unwrap(); + let c1 = make_condition_table(&lua, "x", "=", "1"); + group.set(1, c1).unwrap(); + let node = parse_filter_node(&group, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!(sql, "json_extract(record, '$.value.x') = ?"); + } + + #[test] + fn filter_rejects_invalid_combine() { + let lua = Lua::new(); + let group = lua.create_table().unwrap(); + group.set("combine", "XOR").unwrap(); + let c1 = make_condition_table(&lua, "x", "=", "1"); + group.set(1, c1).unwrap(); + let err = parse_filter_node(&group, 0).unwrap_err(); + assert!(err.to_string().contains("invalid filter combine")); + } + + #[test] + fn filter_rejects_empty_group() { + let lua = Lua::new(); + let group = lua.create_table().unwrap(); + group.set("combine", "AND").unwrap(); + let err = parse_filter_node(&group, 0).unwrap_err(); + assert!(err.to_string().contains("filter group has no conditions")); + } + + #[test] + fn filter_rejects_excessive_depth() { + let lua = Lua::new(); + let c = make_condition_table(&lua, "x", "=", "1"); + let err = parse_filter_node(&c, MAX_FILTER_DEPTH).unwrap_err(); + assert!(err.to_string().contains("filter nesting too deep")); + } + + #[test] + fn filter_accepts_all_ops() { + let lua = Lua::new(); + for op in ALLOWED_OPS { + let t = make_condition_table(&lua, "field", op, "val"); + assert!( + parse_filter_node(&t, 0).is_ok(), + "op '{op}' should be accepted" + ); + } + } + + #[test] + fn filter_op_case_insensitive() { + let lua = Lua::new(); + let t = make_condition_table(&lua, "name", "like", "alice%"); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!(sql, "json_extract(record, '$.value.name') LIKE ?"); + } + + #[test] + fn filter_integer_value() { + let lua = Lua::new(); + let t = lua.create_table().unwrap(); + t.set("field", "count").unwrap(); + t.set("op", ">").unwrap(); + t.set("value", 42).unwrap(); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + build_filter_sql(&node, &mut binds); + assert_eq!(binds, vec!["42"]); + } + + #[test] + fn filter_boolean_value() { + let lua = Lua::new(); + let t = lua.create_table().unwrap(); + t.set("field", "active").unwrap(); + t.set("value", true).unwrap(); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + build_filter_sql(&node, &mut binds); + assert_eq!(binds, vec!["true"]); + } + + #[test] + fn filter_nested_field_path() { + let lua = Lua::new(); + let t = make_condition_table(&lua, "author.websites[0].url", "=", "https://example.com"); + let node = parse_filter_node(&t, 0).unwrap(); + let mut binds = Vec::new(); + let sql = build_filter_sql(&node, &mut binds); + assert_eq!( + sql, + "json_extract(record, '$.value.author.websites[0].url') = ?" + ); + } + + // ----------------------------------------------------------------------- + // query sort direction + // ----------------------------------------------------------------------- + #[tokio::test] async fn query_accepts_valid_sort_direction() { let state = test_state(); diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 0e0037d..a4f7977 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -33,6 +33,38 @@ async fn load_env_vars(db: &sqlx::AnyPool, backend: DatabaseBackend) -> HashMap< .collect() } +/// Load env vars, reusing a cached result if less than 30 seconds old. +/// Avoids per-record DB queries during backfill. +pub(crate) async fn load_env_vars_cached( + db: &sqlx::AnyPool, + backend: DatabaseBackend, +) -> HashMap { + use std::sync::Mutex; + + static CACHE: std::sync::OnceLock)>> = + std::sync::OnceLock::new(); + + let cache = CACHE.get_or_init(|| { + Mutex::new(( + Instant::now() - std::time::Duration::from_secs(60), + HashMap::new(), + )) + }); + { + let guard = cache.lock().unwrap(); + if guard.0.elapsed() < std::time::Duration::from_secs(30) { + return guard.1.clone(); + } + } + + let vars = load_env_vars(db, backend).await; + { + let mut guard = cache.lock().unwrap(); + *guard = (Instant::now(), vars.clone()); + } + vars +} + /// Execute a Lua script for a procedure endpoint. #[allow(clippy::too_many_arguments)] pub async fn execute_procedure_script( @@ -889,6 +921,7 @@ pub struct HookEvent<'a> { pub collection: &'a str, pub rkey: &'a str, pub record: Option<&'a Value>, + pub cached_env_vars: Option<&'a HashMap>, } /// Execute a Lua hook script triggered by a record index event. @@ -1040,7 +1073,14 @@ pub async fn run_hook_once(event: &HookEvent<'_>) -> Result, Strin ) .map_err(|e| format!("failed to set hook context: {e}"))?; - context::set_env_context(&lua, &load_env_vars(&event.state.db, backend).await) + let owned_env_vars; + let env_vars = if let Some(cached) = event.cached_env_vars { + cached + } else { + owned_env_vars = load_env_vars(&event.state.db, backend).await; + &owned_env_vars + }; + context::set_env_context(&lua, env_vars) .map_err(|e| format!("failed to set env context: {e}"))?; lua.load(event.script) @@ -1205,6 +1245,7 @@ mod tests { collection: "test.collection", rkey: "rkey1", record, + cached_env_vars: None, } } diff --git a/src/lua/mod.rs b/src/lua/mod.rs index 58733eb..a535d21 100644 --- a/src/lua/mod.rs +++ b/src/lua/mod.rs @@ -11,6 +11,7 @@ mod xrpc_api; #[allow(unused_imports)] pub(crate) use context::SpaceContext; pub(crate) use execute::{ - HookEvent, execute_hook_script, execute_procedure_script, execute_query_script, run_hook_once, + HookEvent, execute_hook_script, execute_procedure_script, execute_query_script, + load_env_vars_cached, run_hook_once, }; pub(crate) use sandbox::validate_script; diff --git a/src/main.rs b/src/main.rs index f6a78b2..aae42b4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -616,7 +616,7 @@ async fn main() { std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new(config))) }; - let (backfill_events_tx, _) = tokio::sync::broadcast::channel(1024); + let (backfill_events_tx, _) = tokio::sync::broadcast::channel(16384); let verbose_event_logging = { let enabled = diff --git a/src/record_handler.rs b/src/record_handler.rs index 9631941..bc3704e 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -71,6 +71,7 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { collection: &record.collection, rkey: &record.rkey, record: Some(rec), + cached_env_vars: None, }) .await; @@ -198,6 +199,7 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { collection: &record.collection, rkey: &record.rkey, record: None, + cached_env_vars: None, }) .await; diff --git a/web/src/app/dashboard/backfill/page.tsx b/web/src/app/dashboard/backfill/page.tsx index 640c532..27e007b 100644 --- a/web/src/app/dashboard/backfill/page.tsx +++ b/web/src/app/dashboard/backfill/page.tsx @@ -576,8 +576,6 @@ function JobDetail({ const [fetchedLoaded, setFetchedLoaded] = useState(false); // Refs for open state and callbacks so the SSE callback doesn't need to re-bind on toggle - const pdsOpenRef = useRef(false); - const fetchedOpenRef = useRef(false); const onJobUpdateRef = useRef(onJobUpdate); onJobUpdateRef.current = onJobUpdate; @@ -641,8 +639,6 @@ function JobDetail({ const [discoveredOpen, setDiscoveredOpen] = useState(false); const [pdsOpen, setPdsOpen] = useState(false); const [fetchedOpen, setFetchedOpen] = useState(false); - pdsOpenRef.current = pdsOpen; - fetchedOpenRef.current = fetchedOpen; // Lazy-load detail data only when sections are expanded useEffect(() => { @@ -691,77 +687,18 @@ function JobDetail({ // Uses refs for open state so the callback identity is stable and doesn't // cause the worker to reconnect when sections are toggled. const handleSSEBatch = useCallback((events: BackfillEvent[]) => { - const pOpen = pdsOpenRef.current; - const fOpen = fetchedOpenRef.current; - - if (pOpen) { - const pdsEvents = events.filter( - (e) => (e.type === "repo_resolved" || e.type === "repo_fetched") && e.pds_endpoint, - ); - if (pdsEvents.length > 0) { - setPdsSummary((prev) => { - const byEndpoint = new Map(prev.map((p, i) => [p.pds_endpoint, i])); - const next = [...prev]; - for (const e of pdsEvents) { - const idx = byEndpoint.get(e.pds_endpoint!); - if (e.type === "repo_resolved") { - if (idx != null) { - next[idx] = { ...next[idx], total_repos: next[idx].total_repos + 1 }; - } else { - const newIdx = next.length; - next.push({ pds_endpoint: e.pds_endpoint!, total_repos: 1, completed_repos: 0, total_records: 0 }); - byEndpoint.set(e.pds_endpoint!, newIdx); - } - } else if (e.type === "repo_fetched" && idx != null) { - next[idx] = { - ...next[idx], - completed_repos: next[idx].completed_repos + 1, - total_records: next[idx].total_records + (e.records_fetched ?? 0), - }; - } - } - return next; - }); - } - } - - if (fOpen) { - const fetched = events.filter((e) => e.type === "repo_fetched" && e.did); - if (fetched.length > 0) { - setFetchedRepos((prev) => { - const existing = new Set(prev.map((r) => r.did)); - const newItems = fetched - .filter((e) => !existing.has(e.did!)) - .map((e) => ({ did: e.did!, pds_endpoint: e.pds_endpoint ?? null, status: "completed" as const, records_fetched: e.records_fetched ?? 0 })); - - if (newItems.length === 0) return prev; - return [...newItems, ...prev]; - }); - } - } - - // Update job counters, stage, and status from SSE const update = onJobUpdateRef.current; - // Increment total_repos from repo_discovered events - const discoveredCount = events.filter((e) => e.type === "repo_discovered").length; - const resolvedCount = events.filter((e) => e.type === "repo_resolved").length; - const fetchedEvents = events.filter((e) => e.type === "repo_fetched"); - const fetchedCount = fetchedEvents.length; - const fetchedRecords = fetchedEvents.reduce((sum, e) => sum + (e.records_fetched ?? 0), 0); - - if (discoveredCount > 0 || resolvedCount > 0 || fetchedCount > 0) { - update((j) => ({ - ...j, - total_repos: (j.total_repos ?? 0) + discoveredCount, - resolved_repos: (j.resolved_repos ?? 0) + resolvedCount, - processed_repos: (j.processed_repos ?? 0) + fetchedCount, - total_records: (j.total_records ?? 0) + fetchedRecords, - })); - } - for (const e of events) { - if (e.type === "job_stage_changed" && e.stage) { + if (e.type === "job_counters") { + update((j) => ({ + ...j, + ...(e.total_repos != null && { total_repos: e.total_repos }), + ...(e.resolved_repos != null && { resolved_repos: e.resolved_repos }), + ...(e.processed_repos != null && { processed_repos: e.processed_repos }), + ...(e.total_records != null && { total_records: e.total_records }), + })); + } else if (e.type === "job_stage_changed" && e.stage) { update((j) => ({ ...j, stage: e.stage! })); } else if (e.type === "job_completed" && e.status) { update((j) => ({ ...j, status: e.status!, error: e.error ?? null })); @@ -784,6 +721,11 @@ function JobDetail({ const profiles = useBlueskyProfiles(allVisibleDids); + const sortedPdsSummary = useMemo( + () => [...pdsSummary].sort((a, b) => b.total_repos - a.total_repos), + [pdsSummary], + ); + const fetchedWithRecords = useMemo( () => fetchedRepos.filter((r) => r.records_fetched > 0), [fetchedRepos], @@ -858,9 +800,10 @@ function JobDetail({ onOpenChange={setDiscoveredOpen} > {discoveredRepos.length > 0 ? ( - r.did} + onVisibleKeysChange={setVisibleDiscoveredDids} hasMore={!!discoveredCursor} onLoadMore={loadMoreDiscovered} rowHeight={28} @@ -894,20 +837,23 @@ function JobDetail({ open={pdsOpen} onOpenChange={setPdsOpen} > - {pdsSummary.length > 0 ? ( -
- {pdsSummary - .sort((a, b) => b.total_repos - a.total_repos) - .map((pds) => ( -
- - {new URL(pds.pds_endpoint).hostname} - - / repos · records - -
- ))} -
+ {sortedPdsSummary.length > 0 ? ( + p.pds_endpoint} + hasMore={false} + onLoadMore={() => {}} + rowHeight={32} + renderRow={(pds) => ( +
+ + {new URL(pds.pds_endpoint).hostname} + + / repos · records + +
+ )} + /> ) : pdsLoaded ? (

No PDS data yet.

) : null} @@ -938,9 +884,10 @@ function JobDetail({ onOpenChange={setFetchedOpen} > {fetchedWithRecords.length > 0 ? ( - r.did} + onVisibleKeysChange={setVisibleFetchedDids} hasMore={!!fetchedCursor} onLoadMore={loadMoreFetched} rowHeight={40} @@ -1034,26 +981,28 @@ function JobDetail({ ); } -function VirtualRepoList({ - repos, - onVisibleDidsChange, +function VirtualList({ + items, + getKey, hasMore, onLoadMore, rowHeight, renderRow, + onVisibleKeysChange, }: { - repos: BackfillRepoEntry[]; - onVisibleDidsChange: (dids: string[]) => void; + items: T[]; + getKey: (item: T) => string; hasMore: boolean; onLoadMore: () => void; rowHeight: number; - renderRow: (repo: BackfillRepoEntry) => React.ReactNode; + renderRow: (item: T) => React.ReactNode; + onVisibleKeysChange?: (keys: string[]) => void; }) { const parentRef = useRef(null); const loadMoreTriggered = useRef(false); const virtualizer = useVirtualizer({ - count: repos.length, + count: items.length, getScrollElement: () => parentRef.current, estimateSize: () => rowHeight, overscan: 5, @@ -1061,22 +1010,22 @@ function VirtualRepoList({ const virtualItems = virtualizer.getVirtualItems(); - const visibleDidsKey = virtualItems.map((item) => repos[item.index]?.did).filter(Boolean).join(","); + const visibleKeysStr = virtualItems.map((item) => getKey(items[item.index])).filter(Boolean).join(","); useEffect(() => { - onVisibleDidsChange(visibleDidsKey.split(",").filter(Boolean)); - }, [visibleDidsKey, onVisibleDidsChange]); + onVisibleKeysChange?.(visibleKeysStr.split(",").filter(Boolean)); + }, [visibleKeysStr, onVisibleKeysChange]); useEffect(() => { if (!hasMore) return; const lastItem = virtualItems[virtualItems.length - 1]; - if (lastItem && lastItem.index >= repos.length - 5 && !loadMoreTriggered.current) { + if (lastItem && lastItem.index >= items.length - 5 && !loadMoreTriggered.current) { loadMoreTriggered.current = true; onLoadMore(); } - if (lastItem && lastItem.index < repos.length - 5) { + if (lastItem && lastItem.index < items.length - 5) { loadMoreTriggered.current = false; } - }, [virtualItems, repos.length, hasMore, onLoadMore]); + }, [virtualItems, items.length, hasMore, onLoadMore]); return (
@@ -1084,11 +1033,11 @@ function VirtualRepoList({ style={{ height: virtualizer.getTotalSize(), width: "100%", position: "relative" }} > {virtualItems.map((virtualRow) => { - const repo = repos[virtualRow.index]; - if (!repo) return null; + const item = items[virtualRow.index]; + if (!item) return null; return (
- {renderRow(repo)} + {renderRow(item)}
); })} -- 2.51.2 From 454d23956d9bac6ad559eba71124f86642c576f5 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 13:22:40 -0500 Subject: [PATCH 36/64] fix: use dpop thumbprints to enable multi-device auth Signed-off-by: Trezy --- ...20260526000000_dpop_session_per_device.sql | 2 + .../20260526000001_drop_access_token_hash.sql | 2 + ...20260526000000_dpop_session_per_device.sql | 2 + .../20260526000001_drop_access_token_hash.sql | 2 + src/auth/middleware.rs | 41 +- src/delegation/unlink_account.rs | 6 +- src/lua/execute.rs | 5 + src/oauth/dpop_proof.rs | 72 +++ src/oauth/keys.rs | 23 + src/oauth/pds_write.rs | 20 +- src/oauth/routes.rs | 472 ++++++++++++------ src/oauth/sessions.rs | 244 +++++++-- src/repo/pds.rs | 3 + src/repo/upload_blob.rs | 4 + src/xrpc/procedure.rs | 78 +-- tests/dpop_auth.rs | 446 ++++++++++++++++- 16 files changed, 1179 insertions(+), 243 deletions(-) create mode 100644 migrations/postgres/20260526000000_dpop_session_per_device.sql create mode 100644 migrations/postgres/20260526000001_drop_access_token_hash.sql create mode 100644 migrations/sqlite/20260526000000_dpop_session_per_device.sql create mode 100644 migrations/sqlite/20260526000001_drop_access_token_hash.sql diff --git a/migrations/postgres/20260526000000_dpop_session_per_device.sql b/migrations/postgres/20260526000000_dpop_session_per_device.sql new file mode 100644 index 0000000..c865644 --- /dev/null +++ b/migrations/postgres/20260526000000_dpop_session_per_device.sql @@ -0,0 +1,2 @@ +DROP INDEX idx_dpop_sessions_client_user; +CREATE UNIQUE INDEX idx_dpop_sessions_client_user_key ON dpop_sessions(api_client_id, user_did, dpop_key_id); diff --git a/migrations/postgres/20260526000001_drop_access_token_hash.sql b/migrations/postgres/20260526000001_drop_access_token_hash.sql new file mode 100644 index 0000000..1dc42ac --- /dev/null +++ b/migrations/postgres/20260526000001_drop_access_token_hash.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_dpop_sessions_token_hash; +ALTER TABLE dpop_sessions DROP COLUMN access_token_hash; diff --git a/migrations/sqlite/20260526000000_dpop_session_per_device.sql b/migrations/sqlite/20260526000000_dpop_session_per_device.sql new file mode 100644 index 0000000..c865644 --- /dev/null +++ b/migrations/sqlite/20260526000000_dpop_session_per_device.sql @@ -0,0 +1,2 @@ +DROP INDEX idx_dpop_sessions_client_user; +CREATE UNIQUE INDEX idx_dpop_sessions_client_user_key ON dpop_sessions(api_client_id, user_did, dpop_key_id); diff --git a/migrations/sqlite/20260526000001_drop_access_token_hash.sql b/migrations/sqlite/20260526000001_drop_access_token_hash.sql new file mode 100644 index 0000000..1dc42ac --- /dev/null +++ b/migrations/sqlite/20260526000001_drop_access_token_hash.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_dpop_sessions_token_hash; +ALTER TABLE dpop_sessions DROP COLUMN access_token_hash; diff --git a/src/auth/middleware.rs b/src/auth/middleware.rs index f4cc387..4715311 100644 --- a/src/auth/middleware.rs +++ b/src/auth/middleware.rs @@ -17,6 +17,8 @@ pub struct Claims { did: String, /// The API client key (e.g. "hvc_...") if the user authenticated via an API client. client_key: Option, + /// The DPoP key ID identifying the specific device session. + dpop_key_id: Option, } /// Separator used to encode `did` and `client_key` in a single cookie value. @@ -34,11 +36,17 @@ impl Claims { self.client_key.as_deref() } + /// The DPoP key ID, if the user authenticated via a DPoP session. + pub fn dpop_key_id(&self) -> Option<&str> { + self.dpop_key_id.as_deref() + } + /// Create claims for an internal call (e.g. Lua xrpc lib) with no client key. pub fn internal(did: String) -> Self { Self { did, client_key: None, + dpop_key_id: None, } } @@ -68,7 +76,11 @@ impl FromRequestParts for Claims { } else { (value, None) }; - return Ok(Claims { did, client_key }); + return Ok(Claims { + did, + client_key, + dpop_key_id: None, + }); } // Path 2: Authorization header @@ -90,6 +102,7 @@ impl FromRequestParts for Claims { return Ok(Claims { did, client_key: None, + dpop_key_id: None, }); } @@ -98,6 +111,7 @@ impl FromRequestParts for Claims { return Ok(Claims { did: service_auth.did, client_key: None, + dpop_key_id: None, }); } @@ -163,13 +177,23 @@ pub async fn resolve_dpop_claims( crate::oauth::client_auth::resolve_client_by_key(&state.db, state.db_backend, client_key) .await?; - // Look up the session by token - let session = crate::oauth::sessions::get_dpop_session_by_token_hash( + // Extract JWK thumbprint from the DPoP proof and resolve the key ID + let thumbprint = crate::oauth::dpop_proof::extract_proof_thumbprint(dpop_proof)?; + let dpop_key_id = crate::oauth::keys::get_dpop_key_id_by_thumbprint( + &state.db, + state.db_backend, + &client.id, + &thumbprint, + ) + .await?; + + // Look up the session by key ID (stable across token rotations) + let session = crate::oauth::sessions::get_dpop_session_by_key_id( &state.db, state.db_backend, encryption_key, &client.id, - access_token, + &dpop_key_id, ) .await?; @@ -181,14 +205,6 @@ pub async fn resolve_dpop_claims( return Err(AppError::Auth("token_expired".into())); } - // Get the DPoP key thumbprint for proof validation - let thumbprint = crate::oauth::keys::get_dpop_key_thumbprint( - &state.db, - state.db_backend, - &session.dpop_key_id, - ) - .await?; - // Build the request URL for htu validation let scheme = if state.config.public_url.starts_with("https") { "https" @@ -215,6 +231,7 @@ pub async fn resolve_dpop_claims( Ok(Claims { did: session.user_did, client_key: Some(client_key.to_string()), + dpop_key_id: Some(dpop_key_id), }) } diff --git a/src/delegation/unlink_account.rs b/src/delegation/unlink_account.rs index 1a1e6e6..1feb4c8 100644 --- a/src/delegation/unlink_account.rs +++ b/src/delegation/unlink_account.rs @@ -54,9 +54,9 @@ pub async fn unlink_account( // Delete delegated account (CASCADE deletes all delegates) db::delete_delegated_account(&state.db, state.db_backend, account_did).await?; - // Delete the DPoP session for the target account using the stored api_client_id + // Delete all DPoP sessions for the target account using the stored api_client_id if let Some(api_client_id) = stored_api_client_id - && let Err(e) = crate::oauth::sessions::delete_dpop_session( + && let Err(e) = crate::oauth::sessions::delete_all_dpop_sessions( &state.db, state.db_backend, &api_client_id, @@ -64,7 +64,7 @@ pub async fn unlink_account( ) .await { - tracing::warn!(account_did, %e, "failed to clean up DPoP session on unlink"); + tracing::warn!(account_did, %e, "failed to clean up DPoP sessions on unlink"); } log_event( diff --git a/src/lua/execute.rs b/src/lua/execute.rs index a4f7977..5f54131 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -125,8 +125,13 @@ pub async fn execute_procedure_script( return Err(e); } }; + let dpop_key_id = claims + .dpop_key_id() + .ok_or_else(|| AppError::Internal("DPoP key ID not available in claims".into()))? + .to_string(); repo::PdsAuth::Dpop { api_client_id, + dpop_key_id, encryption_key: *encryption_key, } } else { diff --git a/src/oauth/dpop_proof.rs b/src/oauth/dpop_proof.rs index 6ff4422..376680d 100644 --- a/src/oauth/dpop_proof.rs +++ b/src/oauth/dpop_proof.rs @@ -23,6 +23,23 @@ struct DpopPayload { jti: String, } +/// Extract the JWK thumbprint from a DPoP proof JWT header without full validation. +pub fn extract_proof_thumbprint(proof_jwt: &str) -> Result { + let header_b64 = proof_jwt + .split('.') + .next() + .ok_or_else(|| AppError::Auth("invalid DPoP proof format".into()))?; + + let header_bytes = URL_SAFE_NO_PAD + .decode(header_b64) + .map_err(|_| AppError::Auth("invalid DPoP proof header encoding".into()))?; + + let header: DpopHeader = serde_json::from_slice(&header_bytes) + .map_err(|_| AppError::Auth("invalid DPoP proof header".into()))?; + + super::keys::compute_jwk_thumbprint(&header.jwk) +} + /// Validate a DPoP proof JWT. /// /// Checks: @@ -216,4 +233,59 @@ mod tests { assert!(result.is_err()); assert!(result.unwrap_err().to_string().contains("dpop+jwt")); } + + #[test] + fn extract_proof_thumbprint_from_real_proof() { + let keypair = crate::oauth::keys::generate_dpop_keypair().unwrap(); + + let proof = crate::oauth::pds_write::generate_dpop_proof( + &keypair.private_jwk, + "POST", + "https://pds.example.com/xrpc/test", + "token", + None, + ) + .unwrap(); + + let thumbprint = extract_proof_thumbprint(&proof).unwrap(); + assert_eq!(thumbprint, keypair.thumbprint); + } + + #[test] + fn extract_proof_thumbprint_rejects_garbage() { + assert!(extract_proof_thumbprint("not-a-jwt").is_err()); + } + + #[test] + fn extract_proof_thumbprint_rejects_bad_base64() { + assert!(extract_proof_thumbprint("!!!.payload.sig").is_err()); + } + + #[test] + fn extract_proof_thumbprint_different_keys_differ() { + let kp1 = crate::oauth::keys::generate_dpop_keypair().unwrap(); + let kp2 = crate::oauth::keys::generate_dpop_keypair().unwrap(); + + let proof1 = crate::oauth::pds_write::generate_dpop_proof( + &kp1.private_jwk, + "GET", + "https://example.com", + "t", + None, + ) + .unwrap(); + + let proof2 = crate::oauth::pds_write::generate_dpop_proof( + &kp2.private_jwk, + "GET", + "https://example.com", + "t", + None, + ) + .unwrap(); + + let t1 = extract_proof_thumbprint(&proof1).unwrap(); + let t2 = extract_proof_thumbprint(&proof2).unwrap(); + assert_ne!(t1, t2); + } } diff --git a/src/oauth/keys.rs b/src/oauth/keys.rs index 7990efd..96559e0 100644 --- a/src/oauth/keys.rs +++ b/src/oauth/keys.rs @@ -186,6 +186,29 @@ pub async fn get_dpop_key_thumbprint( .ok_or_else(|| AppError::NotFound("DPoP key not found".into())) } +/// Look up a DPoP key ID by api_client_id and JWK thumbprint. +pub async fn get_dpop_key_id_by_thumbprint( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + api_client_id: &str, + thumbprint: &str, +) -> Result { + let sql = adapt_sql( + "SELECT id FROM dpop_keys WHERE api_client_id = ? AND jwk_thumbprint = ?", + backend, + ); + + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(api_client_id) + .bind(thumbprint) + .fetch_optional(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to look up DPoP key: {e}")))?; + + row.map(|(id,)| id) + .ok_or_else(|| AppError::Auth("no DPoP key matching proof thumbprint".into())) +} + /// Delete a DPoP key and its associated session. pub async fn delete_dpop_key( pool: &sqlx::AnyPool, diff --git a/src/oauth/pds_write.rs b/src/oauth/pds_write.rs index 3da90b4..c31fb36 100644 --- a/src/oauth/pds_write.rs +++ b/src/oauth/pds_write.rs @@ -20,6 +20,7 @@ struct DpopCredentials { } /// Resolve DPoP credentials: session, PDS URL, and decrypted private key. +#[allow(clippy::too_many_arguments)] async fn resolve_credentials( http: &reqwest::Client, pool: &sqlx::AnyPool, @@ -28,10 +29,17 @@ async fn resolve_credentials( plc_url: &str, api_client_id: &str, user_did: &str, + dpop_key_id: &str, ) -> Result { - let session = - super::sessions::get_dpop_session(pool, backend, encryption_key, api_client_id, user_did) - .await?; + let session = super::sessions::get_dpop_session( + pool, + backend, + encryption_key, + api_client_id, + user_did, + dpop_key_id, + ) + .await?; let pds_url = match session.pds_url { Some(ref url) => url.clone(), @@ -172,6 +180,7 @@ async fn retry_after_refresh( encryption_key, &creds.session.api_client_id, &creds.session.user_did, + &creds.session.dpop_key_id, ) .await?; @@ -195,6 +204,7 @@ async fn retry_after_refresh( backend, &creds.session.api_client_id, &creds.session.user_did, + &creds.session.dpop_key_id, ) .await { @@ -258,6 +268,7 @@ pub async fn dpop_pds_post( plc_url: &str, api_client_id: &str, user_did: &str, + dpop_key_id: &str, xrpc_method: &str, body: &serde_json::Value, ) -> Result { @@ -269,6 +280,7 @@ pub async fn dpop_pds_post( plc_url, api_client_id, user_did, + dpop_key_id, ) .await?; @@ -310,6 +322,7 @@ pub async fn dpop_pds_post_blob( plc_url: &str, api_client_id: &str, user_did: &str, + dpop_key_id: &str, content_type: &str, blob: bytes::Bytes, ) -> Result { @@ -321,6 +334,7 @@ pub async fn dpop_pds_post_blob( plc_url, api_client_id, user_did, + dpop_key_id, ) .await?; diff --git a/src/oauth/routes.rs b/src/oauth/routes.rs index 4f85a4e..5ec26ca 100644 --- a/src/oauth/routes.rs +++ b/src/oauth/routes.rs @@ -18,6 +18,11 @@ pub fn routes() -> Router { .route("/dpop-keys", post(provision_dpop_key)) .route("/sessions", post(register_session)) .route("/sessions/{did}", get(get_session).delete(delete_session)) + .route("/sessions/{did}/devices", get(list_device_sessions)) + .route( + "/sessions/{did}/devices/{session_id}", + axum::routing::delete(delete_device_session), + ) } // --- Request / response types --- @@ -59,6 +64,15 @@ struct GetSessionResponse { scopes: Vec, } +#[derive(Serialize)] +struct DeviceSessionInfo { + id: String, + dpop_key_id: String, + scopes: Vec, + created_at: String, + updated_at: String, +} + // --- Handlers --- /// POST /oauth/dpop-keys — provision a new DPoP keypair. @@ -233,29 +247,6 @@ async fn register_session( // Validate scopes client_auth::validate_scopes(&body.scopes, &client.scopes, &state.lexicons).await?; - // Clean up any existing session's DPoP key before upserting - // (the ON CONFLICT upsert would orphan the old key otherwise) - { - let lookup_sql = crate::db::adapt_sql( - "SELECT dpop_key_id FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", - state.db_backend, - ); - if let Ok(Some((old_key_id,))) = sqlx::query_as::<_, (String,)>(&lookup_sql) - .bind(&client.id) - .bind(&body.did) - .fetch_optional(&state.db) - .await - && old_key_id != dpop_key_id - { - let del_sql = - crate::db::adapt_sql("DELETE FROM dpop_keys WHERE id = ?", state.db_backend); - let _ = sqlx::query(&del_sql) - .bind(&old_key_id) - .execute(&state.db) - .await; - } - } - // Store the session let session_id = Uuid::new_v4().to_string(); sessions::store_dpop_session( @@ -330,78 +321,90 @@ async fn get_session( .as_ref() .ok_or_else(|| AppError::Internal("TOKEN_ENCRYPTION_KEY not configured".into()))?; - let client = if let Some(ref secret) = client_secret { - client_auth::authenticate_confidential(&state.db, state.db_backend, &client_key, secret) - .await? + let session = if let Some(ref secret) = client_secret { + let c = client_auth::authenticate_confidential( + &state.db, + state.db_backend, + &client_key, + secret, + ) + .await?; + // Confidential clients: look up by (client, user) — no DPoP proof needed + sessions::get_dpop_session_for_user( + &state.db, + state.db_backend, + encryption_key, + &c.id, + &did, + ) + .await? } else { let resolved = client_auth::resolve_client_by_key(&state.db, state.db_backend, &client_key).await?; - if resolved.client_type == "public" { - let auth_header = req - .headers() - .get("authorization") - .and_then(|v| v.to_str().ok()) - .ok_or_else(|| { - AppError::Auth("public clients must provide Authorization: DPoP ".into()) - })?; - let access_token = auth_header.strip_prefix("DPoP ").ok_or_else(|| { - AppError::Auth("public clients must use DPoP authorization scheme".into()) - })?; - let dpop_proof = req - .headers() - .get("dpop") - .and_then(|v| v.to_str().ok()) - .ok_or_else(|| { - AppError::Auth("public clients must provide DPoP proof header".into()) - })?; - - let session = sessions::get_dpop_session_by_token_hash( - &state.db, - state.db_backend, - encryption_key, - &resolved.id, - access_token, - ) - .await?; + if resolved.client_type != "public" { + return Err(AppError::Auth( + "non-public clients must provide X-Client-Secret".into(), + )); + } - let thumbprint = - keys::get_dpop_key_thumbprint(&state.db, state.db_backend, &session.dpop_key_id) - .await?; + let auth_header = req + .headers() + .get("authorization") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + AppError::Auth("public clients must provide Authorization: DPoP ".into()) + })?; + let access_token = auth_header.strip_prefix("DPoP ").ok_or_else(|| { + AppError::Auth("public clients must use DPoP authorization scheme".into()) + })?; + let dpop_proof = req + .headers() + .get("dpop") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + AppError::Auth("public clients must provide DPoP proof header".into()) + })?; - let scheme = if state.config.public_url.starts_with("https") { - "https" - } else { - "http" - }; - let host = req - .headers() - .get("host") - .and_then(|v| v.to_str().ok()) - .unwrap_or("localhost"); - let request_url = format!("{}://{}/oauth/sessions/{}", scheme, host, did); - - crate::oauth::dpop_proof::validate_dpop_proof( - dpop_proof, - "GET", - &request_url, - access_token, - &thumbprint, - )?; - } + let thumbprint = crate::oauth::dpop_proof::extract_proof_thumbprint(dpop_proof)?; + let dpop_key_id = keys::get_dpop_key_id_by_thumbprint( + &state.db, + state.db_backend, + &resolved.id, + &thumbprint, + ) + .await?; - resolved + let scheme = if state.config.public_url.starts_with("https") { + "https" + } else { + "http" + }; + let host = req + .headers() + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or("localhost"); + let request_url = format!("{}://{}/oauth/sessions/{}", scheme, host, did); + + crate::oauth::dpop_proof::validate_dpop_proof( + dpop_proof, + "GET", + &request_url, + access_token, + &thumbprint, + )?; + + sessions::get_dpop_session_by_key_id( + &state.db, + state.db_backend, + encryption_key, + &resolved.id, + &dpop_key_id, + ) + .await? }; - let session = sessions::get_dpop_session( - &state.db, - state.db_backend, - encryption_key, - &client.id, - &did, - ) - .await?; - let scopes: Vec = session .scopes .split_whitespace() @@ -433,87 +436,203 @@ async fn delete_session( .and_then(|v| v.to_str().ok()) .map(|s| s.to_string()); - let client = if let Some(ref secret) = client_secret { - client_auth::authenticate_confidential(&state.db, state.db_backend, &client_key, secret) - .await? + if let Some(ref secret) = client_secret { + let client = client_auth::authenticate_confidential( + &state.db, + state.db_backend, + &client_key, + secret, + ) + .await?; + // Confidential clients: delete all sessions for this user+client + sessions::delete_all_dpop_sessions(&state.db, state.db_backend, &client.id, &did).await?; + + log_event( + &state.db, + EventLog { + event_type: "dpop_session.deleted".to_string(), + severity: Severity::Info, + actor_did: Some(did), + subject: Some(client.client_key), + detail: serde_json::json!({}), + }, + state.db_backend, + ) + .await; } else { let resolved = client_auth::resolve_client_by_key(&state.db, state.db_backend, &client_key).await?; - // Public clients must prove they hold the DPoP key + token - if resolved.client_type == "public" { - let auth_header = req + if resolved.client_type != "public" { + return Err(AppError::Auth( + "non-public clients must provide X-Client-Secret".into(), + )); + } + + let auth_header = req + .headers() + .get("authorization") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + AppError::Auth("public clients must provide Authorization: DPoP ".into()) + })?; + let access_token = auth_header.strip_prefix("DPoP ").ok_or_else(|| { + AppError::Auth("public clients must use DPoP authorization scheme".into()) + })?; + let dpop_proof = req + .headers() + .get("dpop") + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| { + AppError::Auth("public clients must provide DPoP proof header".into()) + })?; + + let thumbprint = crate::oauth::dpop_proof::extract_proof_thumbprint(dpop_proof)?; + let dpop_key_id = keys::get_dpop_key_id_by_thumbprint( + &state.db, + state.db_backend, + &resolved.id, + &thumbprint, + ) + .await?; + + let scheme = if state.config.public_url.starts_with("https") { + "https" + } else { + "http" + }; + let host = req + .headers() + .get("host") + .and_then(|v| v.to_str().ok()) + .unwrap_or("localhost"); + let request_url = format!("{}://{}/oauth/sessions/{}", scheme, host, did); + + crate::oauth::dpop_proof::validate_dpop_proof( + dpop_proof, + "DELETE", + &request_url, + access_token, + &thumbprint, + )?; + + sessions::delete_dpop_session( + &state.db, + state.db_backend, + &resolved.id, + &did, + &dpop_key_id, + ) + .await?; + + log_event( + &state.db, + EventLog { + event_type: "dpop_session.deleted".to_string(), + severity: Severity::Info, + actor_did: Some(did), + subject: Some(resolved.client_key), + detail: serde_json::json!({}), + }, + state.db_backend, + ) + .await; + } + + Ok(StatusCode::NO_CONTENT) +} + +/// Extracted headers for session endpoint authentication. +struct SessionAuthHeaders { + client_key: String, + client_secret: Option, + auth_header: Option, + dpop_proof: Option, + host: String, +} + +impl SessionAuthHeaders { + fn from_request(req: &axum::extract::Request) -> Self { + Self { + client_key: req + .headers() + .get("x-client-key") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(), + client_secret: req + .headers() + .get("x-client-secret") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()), + auth_header: req .headers() .get("authorization") .and_then(|v| v.to_str().ok()) - .ok_or_else(|| { - AppError::Auth("public clients must provide Authorization: DPoP ".into()) - })?; - let access_token = auth_header.strip_prefix("DPoP ").ok_or_else(|| { - AppError::Auth("public clients must use DPoP authorization scheme".into()) - })?; - let dpop_proof = req + .map(|s| s.to_string()), + dpop_proof: req .headers() .get("dpop") .and_then(|v| v.to_str().ok()) - .ok_or_else(|| { - AppError::Auth("public clients must provide DPoP proof header".into()) - })?; - - let encryption_key = - state.config.token_encryption_key.as_ref().ok_or_else(|| { - AppError::Internal("TOKEN_ENCRYPTION_KEY not configured".into()) - })?; - - // Look up the session to get the DPoP key thumbprint - let session = sessions::get_dpop_session_by_token_hash( - &state.db, - state.db_backend, - encryption_key, - &resolved.id, - access_token, - ) - .await?; - - let thumbprint = - keys::get_dpop_key_thumbprint(&state.db, state.db_backend, &session.dpop_key_id) - .await?; - - // Build request URL for htu validation - let scheme = if state.config.public_url.starts_with("https") { - "https" - } else { - "http" - }; - let host = req + .map(|s| s.to_string()), + host: req .headers() .get("host") .and_then(|v| v.to_str().ok()) - .unwrap_or("localhost"); - let request_url = format!("{}://{}/oauth/sessions/{}", scheme, host, did); - - crate::oauth::dpop_proof::validate_dpop_proof( - dpop_proof, - "DELETE", - &request_url, - access_token, - &thumbprint, - )?; + .unwrap_or("localhost") + .to_string(), } + } +} - resolved - }; +/// GET /oauth/sessions/:did/devices — list all device sessions for a user. +async fn list_device_sessions( + State(state): State, + Path(did): Path, + req: axum::extract::Request, +) -> Result>, AppError> { + let request_path = req.uri().path().to_string(); + let headers = SessionAuthHeaders::from_request(&req); + let client = resolve_session_client(&state, &headers, &request_path, "GET").await?; + + let sessions = + sessions::list_dpop_sessions(&state.db, state.db_backend, &client.id, &did).await?; + + let result: Vec = sessions + .into_iter() + .map(|s| DeviceSessionInfo { + id: s.id, + dpop_key_id: s.dpop_key_id, + scopes: s.scopes.split_whitespace().map(String::from).collect(), + created_at: s.created_at, + updated_at: s.updated_at, + }) + .collect(); - sessions::delete_dpop_session(&state.db, state.db_backend, &client.id, &did).await?; + Ok(Json(result)) +} + +/// DELETE /oauth/sessions/:did/devices/:session_id — revoke a specific device session. +async fn delete_device_session( + State(state): State, + Path((did, session_id)): Path<(String, String)>, + req: axum::extract::Request, +) -> Result { + let request_path = req.uri().path().to_string(); + let headers = SessionAuthHeaders::from_request(&req); + let client = resolve_session_client(&state, &headers, &request_path, "DELETE").await?; + + sessions::delete_dpop_session_by_id(&state.db, state.db_backend, &session_id, &client.id, &did) + .await?; log_event( &state.db, EventLog { - event_type: "dpop_session.deleted".to_string(), + event_type: "dpop_session.device_deleted".to_string(), severity: Severity::Info, actor_did: Some(did), subject: Some(client.client_key), - detail: serde_json::json!({}), + detail: serde_json::json!({ "session_id": session_id }), }, state.db_backend, ) @@ -521,3 +640,68 @@ async fn delete_session( Ok(StatusCode::NO_CONTENT) } + +/// Shared client authentication for session endpoints. +async fn resolve_session_client( + state: &AppState, + headers: &SessionAuthHeaders, + request_path: &str, + method: &str, +) -> Result { + if headers.client_key.is_empty() { + return Err(AppError::Auth("X-Client-Key header required".into())); + } + + if let Some(ref secret) = headers.client_secret { + return client_auth::authenticate_confidential( + &state.db, + state.db_backend, + &headers.client_key, + secret, + ) + .await; + } + + let resolved = + client_auth::resolve_client_by_key(&state.db, state.db_backend, &headers.client_key) + .await?; + + if resolved.client_type != "public" { + return Err(AppError::Auth( + "non-public clients must provide X-Client-Secret".into(), + )); + } + + let auth_header = headers.auth_header.as_deref().ok_or_else(|| { + AppError::Auth("public clients must provide Authorization: DPoP ".into()) + })?; + let access_token = auth_header.strip_prefix("DPoP ").ok_or_else(|| { + AppError::Auth("public clients must use DPoP authorization scheme".into()) + })?; + let dpop_proof = headers + .dpop_proof + .as_deref() + .ok_or_else(|| AppError::Auth("public clients must provide DPoP proof header".into()))?; + + let thumbprint = crate::oauth::dpop_proof::extract_proof_thumbprint(dpop_proof)?; + let _dpop_key_id = + keys::get_dpop_key_id_by_thumbprint(&state.db, state.db_backend, &resolved.id, &thumbprint) + .await?; + + let scheme = if state.config.public_url.starts_with("https") { + "https" + } else { + "http" + }; + let request_url = format!("{}://{}{}", scheme, headers.host, request_path); + + crate::oauth::dpop_proof::validate_dpop_proof( + dpop_proof, + method, + &request_url, + access_token, + &thumbprint, + )?; + + Ok(resolved) +} diff --git a/src/oauth/sessions.rs b/src/oauth/sessions.rs index 9258d29..7f5b788 100644 --- a/src/oauth/sessions.rs +++ b/src/oauth/sessions.rs @@ -1,14 +1,7 @@ -use sha2::{Digest, Sha256}; - use crate::db::{DatabaseBackend, adapt_sql, now_rfc3339}; use crate::error::AppError; use crate::plugin::encryption::{decrypt, encrypt}; -/// Compute a hex-encoded SHA-256 hash of a token for indexed lookup. -fn token_hash(token: &str) -> String { - hex::encode(Sha256::digest(token.as_bytes())) -} - /// Stored DPoP session data (decrypted). pub struct DpopSession { pub id: String, @@ -23,10 +16,19 @@ pub struct DpopSession { pub issuer: Option, } +/// Session metadata returned by list_dpop_sessions (no decrypted tokens). +pub struct DpopSessionInfo { + pub id: String, + pub dpop_key_id: String, + pub scopes: String, + pub created_at: String, + pub updated_at: String, +} + /// Store or update a DPoP session. /// /// Uses ON CONFLICT to upsert — if a session already exists for this -/// (api_client_id, user_did), it updates the token data. +/// (api_client_id, user_did, dpop_key_id), it updates the token data. #[allow(clippy::too_many_arguments)] pub async fn store_dpop_session( pool: &sqlx::AnyPool, @@ -46,8 +48,6 @@ pub async fn store_dpop_session( let access_enc = encrypt(encryption_key, access_token.as_bytes()) .map_err(|e| AppError::Internal(format!("failed to encrypt access token: {e}")))?; - let access_hash = token_hash(access_token); - let refresh_enc = refresh_token .map(|t| { encrypt(encryption_key, t.as_bytes()) @@ -57,12 +57,10 @@ pub async fn store_dpop_session( let now = now_rfc3339(); let sql = adapt_sql( - r#"INSERT INTO dpop_sessions (id, api_client_id, dpop_key_id, user_did, access_token_enc, access_token_hash, refresh_token_enc, token_expires_at, scopes, pds_url, issuer, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (api_client_id, user_did) DO UPDATE SET - dpop_key_id = EXCLUDED.dpop_key_id, + r#"INSERT INTO dpop_sessions (id, api_client_id, dpop_key_id, user_did, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (api_client_id, user_did, dpop_key_id) DO UPDATE SET access_token_enc = EXCLUDED.access_token_enc, - access_token_hash = EXCLUDED.access_token_hash, refresh_token_enc = EXCLUDED.refresh_token_enc, token_expires_at = EXCLUDED.token_expires_at, scopes = EXCLUDED.scopes, @@ -78,7 +76,6 @@ pub async fn store_dpop_session( .bind(dpop_key_id) .bind(user_did) .bind(&access_enc) - .bind(&access_hash) .bind(&refresh_enc) .bind(token_expires_at) .bind(scopes) @@ -93,22 +90,22 @@ pub async fn store_dpop_session( Ok(()) } -/// Look up a DPoP session by api_client_id and user_did, decrypting tokens. +/// Look up a DPoP session by api_client_id, user_did, and dpop_key_id, decrypting tokens. pub async fn get_dpop_session( pool: &sqlx::AnyPool, backend: DatabaseBackend, encryption_key: &[u8; 32], api_client_id: &str, user_did: &str, + dpop_key_id: &str, ) -> Result { let sql = adapt_sql( - "SELECT id, dpop_key_id, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", + "SELECT id, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer FROM dpop_sessions WHERE api_client_id = ? AND user_did = ? AND dpop_key_id = ?", backend, ); #[allow(clippy::type_complexity)] let row: Option<( - String, String, Vec, Option>, @@ -119,11 +116,12 @@ pub async fn get_dpop_session( )> = sqlx::query_as(&sql) .bind(api_client_id) .bind(user_did) + .bind(dpop_key_id) .fetch_optional(pool) .await .map_err(|e| AppError::Internal(format!("failed to look up DPoP session: {e}")))?; - let (id, dpop_key_id, access_enc, refresh_enc, token_expires_at, scopes, pds_url, issuer) = + let (id, access_enc, refresh_enc, token_expires_at, scopes, pds_url, issuer) = row.ok_or_else(|| AppError::NotFound("DPoP session not found".into()))?; let access_token = String::from_utf8( @@ -144,7 +142,7 @@ pub async fn get_dpop_session( Ok(DpopSession { id, api_client_id: api_client_id.to_string(), - dpop_key_id, + dpop_key_id: dpop_key_id.to_string(), user_did: user_did.to_string(), access_token, refresh_token, @@ -155,25 +153,22 @@ pub async fn get_dpop_session( }) } -/// Look up a DPoP session by api_client_id and access token. -/// Uses the `access_token_hash` column for indexed lookup instead of -/// decrypting every session. -pub async fn get_dpop_session_by_token_hash( +/// Look up a DPoP session by api_client_id and dpop_key_id, decrypting tokens. +/// Used by the auth middleware where the key ID is derived from the DPoP proof thumbprint. +pub async fn get_dpop_session_by_key_id( pool: &sqlx::AnyPool, backend: DatabaseBackend, encryption_key: &[u8; 32], api_client_id: &str, - access_token: &str, + dpop_key_id: &str, ) -> Result { - let hash = token_hash(access_token); let sql = adapt_sql( - "SELECT id, dpop_key_id, user_did, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer FROM dpop_sessions WHERE api_client_id = ? AND access_token_hash = ?", + "SELECT id, user_did, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer FROM dpop_sessions WHERE api_client_id = ? AND dpop_key_id = ?", backend, ); #[allow(clippy::type_complexity)] let row: Option<( - String, String, String, Vec, @@ -184,24 +179,179 @@ pub async fn get_dpop_session_by_token_hash( Option, )> = sqlx::query_as(&sql) .bind(api_client_id) - .bind(&hash) + .bind(dpop_key_id) .fetch_optional(pool) .await .map_err(|e| AppError::Internal(format!("failed to look up DPoP session: {e}")))?; - let ( + let (id, user_did, access_enc, refresh_enc, token_expires_at, scopes, pds_url, issuer) = + row.ok_or_else(|| AppError::Auth("no matching DPoP session".into()))?; + + let access_token = String::from_utf8( + decrypt(encryption_key, &access_enc) + .map_err(|e| AppError::Internal(format!("failed to decrypt access token: {e}")))?, + ) + .map_err(|e| AppError::Internal(format!("invalid access token bytes: {e}")))?; + + let refresh_token = refresh_enc + .map(|enc| { + let bytes = decrypt(encryption_key, &enc) + .map_err(|e| AppError::Internal(format!("failed to decrypt refresh token: {e}")))?; + String::from_utf8(bytes) + .map_err(|e| AppError::Internal(format!("invalid refresh token bytes: {e}"))) + }) + .transpose()?; + + Ok(DpopSession { id, - dpop_key_id, + api_client_id: api_client_id.to_string(), + dpop_key_id: dpop_key_id.to_string(), user_did, - access_enc, - refresh_enc, + access_token, + refresh_token, token_expires_at, scopes, pds_url, issuer, - ) = row.ok_or_else(|| AppError::Auth("no matching DPoP session".into()))?; + }) +} + +/// Delete a DPoP session by api_client_id, user_did, and dpop_key_id (device-specific). +pub async fn delete_dpop_session( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + api_client_id: &str, + user_did: &str, + dpop_key_id: &str, +) -> Result { + let del_session_sql = adapt_sql( + "DELETE FROM dpop_sessions WHERE api_client_id = ? AND user_did = ? AND dpop_key_id = ?", + backend, + ); + sqlx::query(&del_session_sql) + .bind(api_client_id) + .bind(user_did) + .bind(dpop_key_id) + .execute(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to delete DPoP session: {e}")))?; + + let del_key_sql = adapt_sql("DELETE FROM dpop_keys WHERE id = ?", backend); + sqlx::query(&del_key_sql) + .bind(dpop_key_id) + .execute(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to delete DPoP key: {e}")))?; + + Ok(dpop_key_id.to_string()) +} + +/// Delete all DPoP sessions for a user+client pair (e.g. on account unlink). +pub async fn delete_all_dpop_sessions( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + api_client_id: &str, + user_did: &str, +) -> Result<(), AppError> { + let key_ids_sql = adapt_sql( + "SELECT dpop_key_id FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", + backend, + ); + let key_ids: Vec<(String,)> = sqlx::query_as(&key_ids_sql) + .bind(api_client_id) + .bind(user_did) + .fetch_all(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to list DPoP sessions: {e}")))?; + + let del_sessions_sql = adapt_sql( + "DELETE FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", + backend, + ); + sqlx::query(&del_sessions_sql) + .bind(api_client_id) + .bind(user_did) + .execute(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to delete DPoP sessions: {e}")))?; + + let del_key_sql = adapt_sql("DELETE FROM dpop_keys WHERE id = ?", backend); + for (key_id,) in key_ids { + let _ = sqlx::query(&del_key_sql).bind(&key_id).execute(pool).await; + } + + Ok(()) +} + +/// List all DPoP sessions for a user+client pair (metadata only, no decrypted tokens). +pub async fn list_dpop_sessions( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + api_client_id: &str, + user_did: &str, +) -> Result, AppError> { + let sql = adapt_sql( + "SELECT id, dpop_key_id, scopes, created_at, updated_at FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", + backend, + ); - let access_token_dec = String::from_utf8( + let rows: Vec<(String, String, String, String, String)> = sqlx::query_as(&sql) + .bind(api_client_id) + .bind(user_did) + .fetch_all(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to list DPoP sessions: {e}")))?; + + Ok(rows + .into_iter() + .map( + |(id, dpop_key_id, scopes, created_at, updated_at)| DpopSessionInfo { + id, + dpop_key_id, + scopes, + created_at, + updated_at, + }, + ) + .collect()) +} + +/// Look up a DPoP session by api_client_id and user_did only (without dpop_key_id). +/// Used when the caller doesn't know the specific device key — delegation writes +/// and confidential client session lookups. Returns the first matching session. +pub async fn get_dpop_session_for_user( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + encryption_key: &[u8; 32], + api_client_id: &str, + user_did: &str, +) -> Result { + let sql = adapt_sql( + "SELECT id, dpop_key_id, access_token_enc, refresh_token_enc, token_expires_at, scopes, pds_url, issuer FROM dpop_sessions WHERE api_client_id = ? AND user_did = ? LIMIT 1", + backend, + ); + + #[allow(clippy::type_complexity)] + let row: Option<( + String, + String, + Vec, + Option>, + Option, + String, + Option, + Option, + )> = sqlx::query_as(&sql) + .bind(api_client_id) + .bind(user_did) + .fetch_optional(pool) + .await + .map_err(|e| AppError::Internal(format!("failed to look up DPoP session: {e}")))?; + + let (id, dpop_key_id, access_enc, refresh_enc, token_expires_at, scopes, pds_url, issuer) = + row.ok_or_else(|| AppError::NotFound("DPoP session not found".into()))?; + + let access_token = String::from_utf8( decrypt(encryption_key, &access_enc) .map_err(|e| AppError::Internal(format!("failed to decrypt access token: {e}")))?, ) @@ -220,8 +370,8 @@ pub async fn get_dpop_session_by_token_hash( id, api_client_id: api_client_id.to_string(), dpop_key_id, - user_did, - access_token: access_token_dec, + user_did: user_did.to_string(), + access_token, refresh_token, token_expires_at, scopes, @@ -230,20 +380,20 @@ pub async fn get_dpop_session_by_token_hash( }) } -/// Delete a DPoP session by api_client_id and user_did. -pub async fn delete_dpop_session( +/// Delete a specific DPoP session by its ID, verifying it belongs to the given client and user. +pub async fn delete_dpop_session_by_id( pool: &sqlx::AnyPool, backend: DatabaseBackend, + session_id: &str, api_client_id: &str, user_did: &str, ) -> Result { - // Look up the dpop_key_id before deleting so we can clean up the key too let lookup_sql = adapt_sql( - "SELECT dpop_key_id FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", + "SELECT dpop_key_id FROM dpop_sessions WHERE id = ? AND api_client_id = ? AND user_did = ?", backend, ); - let row: Option<(String,)> = sqlx::query_as(&lookup_sql) + .bind(session_id) .bind(api_client_id) .bind(user_did) .fetch_optional(pool) @@ -252,13 +402,9 @@ pub async fn delete_dpop_session( let (dpop_key_id,) = row.ok_or_else(|| AppError::NotFound("DPoP session not found".into()))?; - let del_session_sql = adapt_sql( - "DELETE FROM dpop_sessions WHERE api_client_id = ? AND user_did = ?", - backend, - ); + let del_session_sql = adapt_sql("DELETE FROM dpop_sessions WHERE id = ?", backend); sqlx::query(&del_session_sql) - .bind(api_client_id) - .bind(user_did) + .bind(session_id) .execute(pool) .await .map_err(|e| AppError::Internal(format!("failed to delete DPoP session: {e}")))?; diff --git a/src/repo/pds.rs b/src/repo/pds.rs index db7c7bd..8bba3c7 100644 --- a/src/repo/pds.rs +++ b/src/repo/pds.rs @@ -19,6 +19,7 @@ pub(crate) enum PdsAuth { OAuth(Arc), Dpop { api_client_id: String, + dpop_key_id: String, encryption_key: [u8; 32], }, } @@ -35,6 +36,7 @@ impl PdsAuth { PdsAuth::OAuth(session) => pds_post_json_raw(state, session, xrpc_method, body).await, PdsAuth::Dpop { api_client_id, + dpop_key_id, encryption_key, } => { crate::oauth::pds_write::dpop_pds_post( @@ -46,6 +48,7 @@ impl PdsAuth { &state.config.plc_url, api_client_id, user_did, + dpop_key_id, xrpc_method, body, ) diff --git a/src/repo/upload_blob.rs b/src/repo/upload_blob.rs index 452d1fd..4af810f 100644 --- a/src/repo/upload_blob.rs +++ b/src/repo/upload_blob.rs @@ -55,6 +55,9 @@ pub async fn upload_blob( .ok_or_else(|| AppError::Internal("TOKEN_ENCRYPTION_KEY not configured".into()))?; let api_client_id = crate::repo::get_dpop_client_id(&state, client_key).await?; + let dpop_key_id = claims + .dpop_key_id() + .ok_or_else(|| AppError::Internal("DPoP key ID not available in claims".into()))?; let resp = crate::oauth::pds_write::dpop_pds_post_blob( &state.http, @@ -65,6 +68,7 @@ pub async fn upload_blob( &state.config.plc_url, &api_client_id, claims.did(), + dpop_key_id, content_type, body, ) diff --git a/src/xrpc/procedure.rs b/src/xrpc/procedure.rs index ab1d618..55d5523 100644 --- a/src/xrpc/procedure.rs +++ b/src/xrpc/procedure.rs @@ -359,39 +359,56 @@ async fn handle_dpop_procedure( ) -> Result { // If delegating, verify the caller has write access and resolve the // api_client_id that owns the delegated session. - let (target_did, effective_api_client_id) = if let Some(did) = delegate_did { - let role = crate::delegation::db::get_delegate_role( - &state.db, - state.db_backend, - did, - claims.did(), - ) - .await? - .ok_or_else(|| AppError::Forbidden("you are not a delegate of this account".into()))?; + let (target_did, effective_api_client_id, effective_dpop_key_id) = + if let Some(did) = delegate_did { + let role = crate::delegation::db::get_delegate_role( + &state.db, + state.db_backend, + did, + claims.did(), + ) + .await? + .ok_or_else(|| AppError::Forbidden("you are not a delegate of this account".into()))?; - if !role.can_write() { - return Err(AppError::Forbidden( - "your role does not have write access to this account".into(), - )); - } + if !role.can_write() { + return Err(AppError::Forbidden( + "your role does not have write access to this account".into(), + )); + } - let stored_client_id = - crate::delegation::db::get_api_client_id(&state.db, state.db_backend, did) - .await? - .ok_or_else(|| { - AppError::Internal("delegated account missing api_client_id".into()) - })?; - - if api_client_id != stored_client_id { - return Err(AppError::Forbidden( - "delegation is scoped to a different application".into(), - )); - } + let stored_client_id = + crate::delegation::db::get_api_client_id(&state.db, state.db_backend, did) + .await? + .ok_or_else(|| { + AppError::Internal("delegated account missing api_client_id".into()) + })?; - (did, stored_client_id) - } else { - (claims.did(), api_client_id.to_string()) - }; + if api_client_id != stored_client_id { + return Err(AppError::Forbidden( + "delegation is scoped to a different application".into(), + )); + } + + // TODO: delegated_accounts needs a dpop_key_id column to identify + // which session to use for PDS writes. For now, look up by + // (api_client_id, user_did) which works when there's one session. + let target_session = crate::oauth::sessions::get_dpop_session_for_user( + &state.db, + state.db_backend, + encryption_key, + &stored_client_id, + did, + ) + .await?; + + (did, stored_client_id, target_session.dpop_key_id) + } else { + let dpop_key_id = claims + .dpop_key_id() + .ok_or_else(|| AppError::Internal("DPoP key ID not available in claims".into()))? + .to_string(); + (claims.did(), api_client_id.to_string(), dpop_key_id) + }; // Strip delegateDid from input — it's a control field, not record data let mut input = input.clone(); @@ -512,6 +529,7 @@ async fn handle_dpop_procedure( &state.config.plc_url, &effective_api_client_id, target_did, + &effective_dpop_key_id, xrpc_method, &pds_body, ) diff --git a/tests/dpop_auth.rs b/tests/dpop_auth.rs index 3eda9a6..98aa721 100644 --- a/tests/dpop_auth.rs +++ b/tests/dpop_auth.rs @@ -27,6 +27,18 @@ fn post_json_with_headers( .unwrap() } +/// Helper to make a GET request with headers +fn get_with_headers(uri: &str, headers: Vec<(&str, &str)>) -> Request { + let mut builder = Request::builder() + .method("GET") + .uri(uri) + .header("host", "127.0.0.1"); + for (name, value) in headers { + builder = builder.header(name, value); + } + builder.body(Body::empty()).unwrap() +} + /// Helper to make a DELETE request with headers fn delete_with_headers(uri: &str, headers: Vec<(&str, &str)>) -> Request { let mut builder = Request::builder() @@ -256,7 +268,7 @@ async fn test_full_flow_provision_register_delete() { let delete_resp = app.router.clone().oneshot(delete_req).await.unwrap(); assert_eq!(delete_resp.status(), StatusCode::NO_CONTENT); - // 4. Verify session is gone (try to delete again) + // 4. Verify session is gone (delete is idempotent for confidential clients) let delete_req2 = delete_with_headers( "/oauth/sessions/did:plc:testuser", vec![ @@ -265,7 +277,7 @@ async fn test_full_flow_provision_register_delete() { ], ); let delete_resp2 = app.router.clone().oneshot(delete_req2).await.unwrap(); - assert_eq!(delete_resp2.status(), StatusCode::NOT_FOUND); + assert_eq!(delete_resp2.status(), StatusCode::NO_CONTENT); } #[tokio::test] @@ -411,3 +423,433 @@ async fn test_xrpc_dpop_auth_accepted() { "DPoP-authenticated XRPC request should not get 401" ); } + +/// Helper: provision a DPoP key and register a session. Returns (provision_id, dpop_key, session_id). +async fn provision_and_register( + app: &common::app::TestApp, + client_key: &str, + client_secret: &str, + did: &str, + access_token: &str, +) -> (String, serde_json::Value, String) { + let key_req = post_json_with_headers( + "/oauth/dpop-keys", + &json!({}), + vec![ + ("x-client-key", client_key), + ("x-client-secret", client_secret), + ], + ); + let key_resp = app.router.clone().oneshot(key_req).await.unwrap(); + assert_eq!(key_resp.status(), StatusCode::CREATED); + let key_body = response_json(key_resp).await; + let provision_id = key_body["provision_id"].as_str().unwrap().to_string(); + let dpop_key = key_body["dpop_key"].clone(); + + let session_req = post_json_with_headers( + "/oauth/sessions", + &json!({ + "provision_id": provision_id, + "did": did, + "access_token": access_token, + "scopes": "atproto", + "pds_url": "https://pds.example.com", + }), + vec![ + ("x-client-key", client_key), + ("x-client-secret", client_secret), + ], + ); + let session_resp = app.router.clone().oneshot(session_req).await.unwrap(); + assert_eq!(session_resp.status(), StatusCode::CREATED); + let session_body = response_json(session_resp).await; + let session_id = session_body["session_id"].as_str().unwrap().to_string(); + + (provision_id, dpop_key, session_id) +} + +#[tokio::test] +#[serial] +async fn test_multi_device_sessions_coexist() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + let did = "did:plc:multidevice"; + + let (_prov1, _key1, session_id_1) = + provision_and_register(&app, &client_key, &client_secret, did, "token-device-1").await; + let (_prov2, _key2, session_id_2) = + provision_and_register(&app, &client_key, &client_secret, did, "token-device-2").await; + + assert_ne!(session_id_1, session_id_2); + + // Both sessions should appear in the device list + let list_req = get_with_headers( + &format!("/oauth/sessions/{}/devices", did), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let list_resp = app.router.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_resp.status(), StatusCode::OK); + let devices: Vec = + serde_json::from_value(response_json(list_resp).await).unwrap(); + assert_eq!(devices.len(), 2); + + let ids: Vec<&str> = devices.iter().map(|d| d["id"].as_str().unwrap()).collect(); + assert!(ids.contains(&session_id_1.as_str())); + assert!(ids.contains(&session_id_2.as_str())); +} + +#[tokio::test] +#[serial] +async fn test_list_device_sessions_empty() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + + let list_req = get_with_headers( + "/oauth/sessions/did:plc:nobody/devices", + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let list_resp = app.router.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_resp.status(), StatusCode::OK); + let devices: Vec = + serde_json::from_value(response_json(list_resp).await).unwrap(); + assert!(devices.is_empty()); +} + +#[tokio::test] +#[serial] +async fn test_delete_device_session_by_id() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + let did = "did:plc:deletedevice"; + + let (_prov1, _key1, session_id_1) = + provision_and_register(&app, &client_key, &client_secret, did, "token-a").await; + let (_prov2, _key2, session_id_2) = + provision_and_register(&app, &client_key, &client_secret, did, "token-b").await; + + // Delete session 1 + let del_req = delete_with_headers( + &format!("/oauth/sessions/{}/devices/{}", did, session_id_1), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let del_resp = app.router.clone().oneshot(del_req).await.unwrap(); + assert_eq!(del_resp.status(), StatusCode::NO_CONTENT); + + // Only session 2 should remain + let list_req = get_with_headers( + &format!("/oauth/sessions/{}/devices", did), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let list_resp = app.router.clone().oneshot(list_req).await.unwrap(); + let devices: Vec = + serde_json::from_value(response_json(list_resp).await).unwrap(); + assert_eq!(devices.len(), 1); + assert_eq!(devices[0]["id"].as_str().unwrap(), session_id_2); +} + +#[tokio::test] +#[serial] +async fn test_delete_device_session_not_found() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + + let del_req = delete_with_headers( + "/oauth/sessions/did:plc:nobody/devices/nonexistent-id", + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let del_resp = app.router.clone().oneshot(del_req).await.unwrap(); + assert_eq!(del_resp.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +#[serial] +async fn test_session_upsert_same_device() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + let did = "did:plc:upsertuser"; + + // Provision one key + let key_req = post_json_with_headers( + "/oauth/dpop-keys", + &json!({}), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let key_resp = app.router.clone().oneshot(key_req).await.unwrap(); + let key_body = response_json(key_resp).await; + let provision_id = key_body["provision_id"].as_str().unwrap(); + + // Register session with token-v1 + let reg1 = post_json_with_headers( + "/oauth/sessions", + &json!({ + "provision_id": provision_id, + "did": did, + "access_token": "token-v1", + "scopes": "atproto", + }), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let resp1 = app.router.clone().oneshot(reg1).await.unwrap(); + assert_eq!(resp1.status(), StatusCode::CREATED); + + // Re-register with same provision_id (same device key) but new token + let reg2 = post_json_with_headers( + "/oauth/sessions", + &json!({ + "provision_id": provision_id, + "did": did, + "access_token": "token-v2", + "scopes": "atproto", + }), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let resp2 = app.router.clone().oneshot(reg2).await.unwrap(); + assert_eq!(resp2.status(), StatusCode::CREATED); + + // Should still be exactly one device session (upsert, not duplicate) + let list_req = get_with_headers( + &format!("/oauth/sessions/{}/devices", did), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let list_resp = app.router.clone().oneshot(list_req).await.unwrap(); + let devices: Vec = + serde_json::from_value(response_json(list_resp).await).unwrap(); + assert_eq!(devices.len(), 1); +} + +#[tokio::test] +#[serial] +async fn test_get_session_with_confidential_client() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + let did = "did:plc:getsession"; + + provision_and_register(&app, &client_key, &client_secret, did, "some-token").await; + + let get_req = get_with_headers( + &format!("/oauth/sessions/{}", did), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let get_resp = app.router.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_resp.status(), StatusCode::OK); + let body = response_json(get_resp).await; + assert_eq!(body["did"], did); + assert!(body["scopes"].is_array()); +} + +#[tokio::test] +#[serial] +async fn test_device_list_response_format() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, client_secret, _id) = app.create_api_client("confidential", None).await; + let did = "did:plc:formatcheck"; + + provision_and_register(&app, &client_key, &client_secret, did, "token-fmt").await; + + let list_req = get_with_headers( + &format!("/oauth/sessions/{}/devices", did), + vec![ + ("x-client-key", &client_key), + ("x-client-secret", &client_secret), + ], + ); + let list_resp = app.router.clone().oneshot(list_req).await.unwrap(); + assert_eq!(list_resp.status(), StatusCode::OK); + let devices: Vec = + serde_json::from_value(response_json(list_resp).await).unwrap(); + assert_eq!(devices.len(), 1); + + let device = &devices[0]; + assert!(device["id"].is_string()); + assert!(device["dpop_key_id"].is_string()); + assert!(device["scopes"].is_array()); + assert!(device["created_at"].is_string()); + assert!(device["updated_at"].is_string()); +} + +#[tokio::test] +#[serial] +async fn test_public_client_dpop_get_session() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, _secret, _id) = app + .create_api_client("public", Some(vec!["http://localhost:3000".to_string()])) + .await; + + use base64::Engine; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use sha2::{Digest, Sha256}; + + let verifier = "test-verifier-for-public-client-session"; + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + + // Provision key with PKCE + let key_req = post_json_with_headers( + "/oauth/dpop-keys", + &json!({ "pkce_challenge": challenge }), + vec![ + ("x-client-key", &client_key), + ("origin", "http://localhost:3000"), + ], + ); + let key_resp = app.router.clone().oneshot(key_req).await.unwrap(); + assert_eq!(key_resp.status(), StatusCode::CREATED); + let key_body = response_json(key_resp).await; + let provision_id = key_body["provision_id"].as_str().unwrap(); + let dpop_key = &key_body["dpop_key"]; + + let did = "did:plc:publicuser"; + let access_token = "public-client-access-token"; + + // Register session with PKCE verifier + let session_req = post_json_with_headers( + "/oauth/sessions", + &json!({ + "provision_id": provision_id, + "pkce_verifier": verifier, + "did": did, + "access_token": access_token, + "scopes": "atproto", + "pds_url": "https://pds.example.com", + }), + vec![("x-client-key", &client_key)], + ); + let session_resp = app.router.clone().oneshot(session_req).await.unwrap(); + assert_eq!(session_resp.status(), StatusCode::CREATED); + + // GET session with DPoP proof + let request_url = format!("http://127.0.0.1/oauth/sessions/{}", did); + let proof = generate_dpop_proof(dpop_key, "GET", &request_url, access_token, None) + .expect("failed to generate DPoP proof"); + + let get_req = get_with_headers( + &format!("/oauth/sessions/{}", did), + vec![ + ("x-client-key", &client_key), + ("authorization", &format!("DPoP {}", access_token)), + ("dpop", &proof), + ], + ); + let get_resp = app.router.clone().oneshot(get_req).await.unwrap(); + assert_eq!(get_resp.status(), StatusCode::OK); + let body = response_json(get_resp).await; + assert_eq!(body["did"], did); +} + +#[tokio::test] +#[serial] +async fn test_public_client_dpop_delete_session() { + common::require_db!(); + let app = common::app::TestApp::new_with_encryption().await; + let (client_key, _secret, _id) = app + .create_api_client("public", Some(vec!["http://localhost:3000".to_string()])) + .await; + + use base64::Engine; + use base64::engine::general_purpose::URL_SAFE_NO_PAD; + use sha2::{Digest, Sha256}; + + let verifier = "test-verifier-for-public-delete"; + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + + let key_req = post_json_with_headers( + "/oauth/dpop-keys", + &json!({ "pkce_challenge": challenge }), + vec![ + ("x-client-key", &client_key), + ("origin", "http://localhost:3000"), + ], + ); + let key_resp = app.router.clone().oneshot(key_req).await.unwrap(); + assert_eq!(key_resp.status(), StatusCode::CREATED); + let key_body = response_json(key_resp).await; + let provision_id = key_body["provision_id"].as_str().unwrap(); + let dpop_key = &key_body["dpop_key"]; + + let did = "did:plc:publicdelete"; + let access_token = "public-delete-token"; + + let session_req = post_json_with_headers( + "/oauth/sessions", + &json!({ + "provision_id": provision_id, + "pkce_verifier": verifier, + "did": did, + "access_token": access_token, + "scopes": "atproto", + "pds_url": "https://pds.example.com", + }), + vec![("x-client-key", &client_key)], + ); + let session_resp = app.router.clone().oneshot(session_req).await.unwrap(); + assert_eq!(session_resp.status(), StatusCode::CREATED); + + // DELETE session with DPoP proof + let request_url = format!("http://127.0.0.1/oauth/sessions/{}", did); + let proof = generate_dpop_proof(dpop_key, "DELETE", &request_url, access_token, None) + .expect("failed to generate DPoP proof"); + + let del_req = delete_with_headers( + &format!("/oauth/sessions/{}", did), + vec![ + ("x-client-key", &client_key), + ("authorization", &format!("DPoP {}", access_token)), + ("dpop", &proof), + ], + ); + let del_resp = app.router.clone().oneshot(del_req).await.unwrap(); + assert_eq!(del_resp.status(), StatusCode::NO_CONTENT); + + // Verify session is gone — GET should fail + let request_url2 = format!("http://127.0.0.1/oauth/sessions/{}", did); + let proof2 = generate_dpop_proof(dpop_key, "GET", &request_url2, access_token, None) + .expect("failed to generate DPoP proof"); + + let get_req = get_with_headers( + &format!("/oauth/sessions/{}", did), + vec![ + ("x-client-key", &client_key), + ("authorization", &format!("DPoP {}", access_token)), + ("dpop", &proof2), + ], + ); + let get_resp = app.router.clone().oneshot(get_req).await.unwrap(); + assert_ne!(get_resp.status(), StatusCode::OK); +} -- 2.51.2 From c5f31e786f844479d4fa86647ffb0978a3506ef8 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 13:35:30 -0500 Subject: [PATCH 37/64] ci: fix npm releases Signed-off-by: Trezy --- .github/workflows/ci.yml | 3 +++ packages/lex-agent/.releaserc.json | 2 +- packages/oauth-client-browser/.releaserc.json | 2 +- packages/oauth-client-node/.releaserc.json | 2 +- packages/oauth-client/.releaserc.json | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ee8050..e6ea1b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,9 @@ on: env: SQLX_OFFLINE: true +permissions: + id-token: write # Required for OIDC + jobs: # --------------------------------------------------------------------------- # Path detection — determines which job groups should run. diff --git a/packages/lex-agent/.releaserc.json b/packages/lex-agent/.releaserc.json index f52c72e..667edc0 100644 --- a/packages/lex-agent/.releaserc.json +++ b/packages/lex-agent/.releaserc.json @@ -9,7 +9,7 @@ "@semantic-release/release-notes-generator", ["@semantic-release/exec", { "prepareCmd": "node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.version='${nextRelease.version}';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"", - "publishCmd": "bun publish --access public --tag ${nextRelease.channel || 'latest'}" + "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || npm publish --access public --tag ${nextRelease.channel || 'latest'}" }], "@semantic-release/github" ] diff --git a/packages/oauth-client-browser/.releaserc.json b/packages/oauth-client-browser/.releaserc.json index 02e61cd..667edc0 100644 --- a/packages/oauth-client-browser/.releaserc.json +++ b/packages/oauth-client-browser/.releaserc.json @@ -9,7 +9,7 @@ "@semantic-release/release-notes-generator", ["@semantic-release/exec", { "prepareCmd": "node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.version='${nextRelease.version}';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"", - "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || bun publish --access public --tag ${nextRelease.channel || 'latest'}" + "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || npm publish --access public --tag ${nextRelease.channel || 'latest'}" }], "@semantic-release/github" ] diff --git a/packages/oauth-client-node/.releaserc.json b/packages/oauth-client-node/.releaserc.json index 02e61cd..667edc0 100644 --- a/packages/oauth-client-node/.releaserc.json +++ b/packages/oauth-client-node/.releaserc.json @@ -9,7 +9,7 @@ "@semantic-release/release-notes-generator", ["@semantic-release/exec", { "prepareCmd": "node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.version='${nextRelease.version}';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"", - "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || bun publish --access public --tag ${nextRelease.channel || 'latest'}" + "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || npm publish --access public --tag ${nextRelease.channel || 'latest'}" }], "@semantic-release/github" ] diff --git a/packages/oauth-client/.releaserc.json b/packages/oauth-client/.releaserc.json index 02e61cd..667edc0 100644 --- a/packages/oauth-client/.releaserc.json +++ b/packages/oauth-client/.releaserc.json @@ -9,7 +9,7 @@ "@semantic-release/release-notes-generator", ["@semantic-release/exec", { "prepareCmd": "node -e \"const fs=require('fs');const p=JSON.parse(fs.readFileSync('package.json','utf8'));p.version='${nextRelease.version}';fs.writeFileSync('package.json',JSON.stringify(p,null,2)+'\\n')\"", - "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || bun publish --access public --tag ${nextRelease.channel || 'latest'}" + "publishCmd": "npm view $(node -p \"require('./package.json').name\")@${nextRelease.version} version 2>/dev/null && echo 'Version already published, skipping' || npm publish --access public --tag ${nextRelease.channel || 'latest'}" }], "@semantic-release/github" ] -- 2.51.2 From cd411d6aeec0919729901c12258349a4292400f8 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 16:01:41 -0500 Subject: [PATCH 38/64] fix: remove prefix typo from db.query filter generation Signed-off-by: Trezy --- src/lua/db_api.rs | 21 +++---- tests/lua_db_api.rs | 143 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 12 deletions(-) diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 3afea15..e97fa7f 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -126,7 +126,7 @@ fn build_filter_sql(node: &FilterNode, binds: &mut Vec) -> String { match node { FilterNode::Condition { field, op, value } => { binds.push(value.clone()); - format!("json_extract(record, '$.value.{field}') {op} ?") + format!("json_extract(record, '$.{field}') {op} ?") } FilterNode::Group { combine, children } => { let parts: Vec = children @@ -205,7 +205,7 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { let order_expr = if top_level_columns.contains(&sort_field.as_str()) { format!("{sort_field} {direction}") } else { - format!("json_extract(record, '$.value.{sort_field}') {direction}") + format!("json_extract(record, '$.{sort_field}') {direction}") }; let did_clause = if did.is_some() { " AND did = ?" } else { "" }; @@ -965,7 +965,7 @@ mod tests { let node = parse_filter_node(&t, 0).unwrap(); let mut binds = Vec::new(); let sql = build_filter_sql(&node, &mut binds); - assert_eq!(sql, "json_extract(record, '$.value.name') = ?"); + assert_eq!(sql, "json_extract(record, '$.name') = ?"); assert_eq!(binds, vec!["alice"]); } @@ -978,7 +978,7 @@ mod tests { let node = parse_filter_node(&t, 0).unwrap(); let mut binds = Vec::new(); let sql = build_filter_sql(&node, &mut binds); - assert_eq!(sql, "json_extract(record, '$.value.status') = ?"); + assert_eq!(sql, "json_extract(record, '$.status') = ?"); } #[test] @@ -1011,7 +1011,7 @@ mod tests { let sql = build_filter_sql(&node, &mut binds); assert_eq!( sql, - "(json_extract(record, '$.value.status') = ? AND json_extract(record, '$.value.age') > ?)" + "(json_extract(record, '$.status') = ? AND json_extract(record, '$.age') > ?)" ); assert_eq!(binds, vec!["active", "18"]); } @@ -1030,7 +1030,7 @@ mod tests { let sql = build_filter_sql(&node, &mut binds); assert_eq!( sql, - "(json_extract(record, '$.value.role') = ? OR json_extract(record, '$.value.role') = ?)" + "(json_extract(record, '$.role') = ? OR json_extract(record, '$.role') = ?)" ); assert_eq!(binds, vec!["admin", "mod"]); } @@ -1045,7 +1045,7 @@ mod tests { let node = parse_filter_node(&group, 0).unwrap(); let mut binds = Vec::new(); let sql = build_filter_sql(&node, &mut binds); - assert_eq!(sql, "json_extract(record, '$.value.x') = ?"); + assert_eq!(sql, "json_extract(record, '$.x') = ?"); } #[test] @@ -1095,7 +1095,7 @@ mod tests { let node = parse_filter_node(&t, 0).unwrap(); let mut binds = Vec::new(); let sql = build_filter_sql(&node, &mut binds); - assert_eq!(sql, "json_extract(record, '$.value.name') LIKE ?"); + assert_eq!(sql, "json_extract(record, '$.name') LIKE ?"); } #[test] @@ -1130,10 +1130,7 @@ mod tests { let node = parse_filter_node(&t, 0).unwrap(); let mut binds = Vec::new(); let sql = build_filter_sql(&node, &mut binds); - assert_eq!( - sql, - "json_extract(record, '$.value.author.websites[0].url') = ?" - ); + assert_eq!(sql, "json_extract(record, '$.author.websites[0].url') = ?"); } // ----------------------------------------------------------------------- diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index d32bb90..3fa11aa 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -345,3 +345,146 @@ async fn db_raw_select_works() { let cnt: i64 = first_row.get("cnt").unwrap(); assert_eq!(cnt, 3); } + +#[tokio::test] +#[serial] +async fn db_query_filter_equals() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + seed_records(&pool, backend).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: mlua::Table = lua + .load( + r#"return db.query({ + collection = "test.collection", + filter = { field = "name", value = "Test One" } + })"#, + ) + .eval_async() + .await + .unwrap(); + + let records: mlua::Table = result.get("records").unwrap(); + assert_eq!(records.raw_len(), 1); + let first: mlua::Table = records.get(1).unwrap(); + assert_eq!(first.get::("name").unwrap(), "Test One"); +} + +#[tokio::test] +#[serial] +async fn db_query_filter_not_equals() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + seed_records(&pool, backend).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: mlua::Table = lua + .load( + r#"return db.query({ + collection = "test.collection", + filter = { field = "name", op = "!=", value = "Test One" } + })"#, + ) + .eval_async() + .await + .unwrap(); + + let records: mlua::Table = result.get("records").unwrap(); + assert_eq!(records.raw_len(), 2); +} + +#[tokio::test] +#[serial] +async fn db_query_filter_no_match_returns_empty() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + seed_records(&pool, backend).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: mlua::Table = lua + .load( + r#"return db.query({ + collection = "test.collection", + filter = { field = "name", value = "Nonexistent" } + })"#, + ) + .eval_async() + .await + .unwrap(); + + let records: mlua::Table = result.get("records").unwrap(); + assert_eq!(records.raw_len(), 0); +} + +#[tokio::test] +#[serial] +async fn db_query_filter_and_group() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + seed_records(&pool, backend).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: mlua::Table = lua + .load( + r#"return db.query({ + collection = "test.collection", + filter = { + combine = "AND", + { field = "name", op = "LIKE", value = "Test%" }, + { field = "value", op = ">", value = "1" } + } + })"#, + ) + .eval_async() + .await + .unwrap(); + + let records: mlua::Table = result.get("records").unwrap(); + assert_eq!(records.raw_len(), 1); + let first: mlua::Table = records.get(1).unwrap(); + assert_eq!(first.get::("name").unwrap(), "Test Two"); +} + +#[tokio::test] +#[serial] +async fn db_query_sort_by_json_field() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + seed_records(&pool, backend).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: mlua::Table = lua + .load( + r#"return db.query({ + collection = "test.collection", + sort = "name", + sortDirection = "asc" + })"#, + ) + .eval_async() + .await + .unwrap(); + + let records: mlua::Table = result.get("records").unwrap(); + assert_eq!(records.raw_len(), 3); + let first: mlua::Table = records.get(1).unwrap(); + let last: mlua::Table = records.get(3).unwrap(); + assert_eq!(first.get::("name").unwrap(), "Other Record"); + assert_eq!(last.get::("name").unwrap(), "Test Two"); +} -- 2.51.2 From ad99c40463268b80514454d37109923019c5a976 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 16:31:08 -0500 Subject: [PATCH 39/64] docs: update auth docs to cover new dpop requirements Signed-off-by: Trezy --- .../docs/getting-started/authentication.md | 62 +++++++++++++++++-- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/packages/docs/content/docs/getting-started/authentication.md b/packages/docs/content/docs/getting-started/authentication.md index b46e894..3251c48 100644 --- a/packages/docs/content/docs/getting-started/authentication.md +++ b/packages/docs/content/docs/getting-started/authentication.md @@ -241,7 +241,7 @@ A request that only carries an `X-Client-Key` header (no DPoP token) can hit que Third-party apps that want HappyView to make PDS writes on behalf of their users use the **DPoP key provisioning** flow. This avoids browser-based redirects through HappyView's domain, which can be blocked by Firefox's Bounce Tracker Protection. -The idea: the app gets a DPoP keypair from HappyView, uses that keypair during its own OAuth flow with the user's PDS, then registers the resulting tokens back with HappyView. From that point on, XRPC requests authenticated with `Authorization: DPoP ` plus a `DPoP` proof header and `X-Client-Key` will have HappyView proxy writes using the stored session. +The idea: for each device, the app gets a DPoP keypair from HappyView, uses that keypair during its own OAuth flow with the user's PDS, then registers the resulting tokens back with HappyView. Each device gets its own keypair and session, so a user can be signed in on multiple devices simultaneously. From that point on, XRPC requests authenticated with `Authorization: DPoP ` plus a `DPoP` proof header and `X-Client-Key` will have HappyView proxy writes using the stored session that matches the request's DPoP key. The client app and HappyView share the same DPoP keypair, so both can generate valid proofs that the PDS will accept. The PDS binds tokens to a key's thumbprint but it doesn't care who signs the proof, only that it was signed by the right key. @@ -466,7 +466,7 @@ HappyView validates the DPoP proof, looks up the stored session, and proxies the #### 5. Logout -Confidential clients authenticate with `X-Client-Key` + `X-Client-Secret`: +Confidential clients authenticate with `X-Client-Key` + `X-Client-Secret`. This revokes **all** device sessions for the user under this client — useful for a full sign-out: ``` DELETE /oauth/sessions/did:plc:user123 @@ -474,7 +474,7 @@ X-Client-Key: hvc_... X-Client-Secret: hvs_... ``` -Public clients must provide a valid DPoP proof to prove they hold the key: +Public clients must provide a valid DPoP proof to prove they hold the key. This revokes only the session that matches the DPoP key used in the proof — other device sessions for the same user are unaffected: ``` DELETE /oauth/sessions/did:plc:user123 @@ -483,7 +483,61 @@ Authorization: DPoP DPoP: ``` -This deletes the stored session and the associated DPoP key. +To revoke a specific device session (for either client type), use the [device management endpoints](#6-managing-device-sessions) instead. + +#### 6. Managing device sessions + +When a user registers sessions from multiple devices (each with its own DPoP keypair), each session is tracked separately. You can list and revoke individual device sessions without affecting the others. + +**List device sessions:** + +Confidential clients authenticate with `X-Client-Key` + `X-Client-Secret`. Public clients authenticate with DPoP proof. + +``` +GET /oauth/sessions/did:plc:user123/devices +X-Client-Key: hvc_... +X-Client-Secret: hvs_... +``` + +Response: + +```json +[ + { + "id": "uuid-session-1", + "dpop_key_id": "uuid-key-1", + "scopes": ["atproto", "transition:generic"], + "created_at": "2026-05-20T12:00:00Z", + "updated_at": "2026-05-20T12:00:00Z" + }, + { + "id": "uuid-session-2", + "dpop_key_id": "uuid-key-2", + "scopes": ["atproto", "transition:generic"], + "created_at": "2026-05-21T08:30:00Z", + "updated_at": "2026-05-21T08:30:00Z" + } +] +``` + +**Delete a specific device session:** + +``` +DELETE /oauth/sessions/did:plc:user123/devices/uuid-session-1 +X-Client-Key: hvc_... +X-Client-Secret: hvs_... +``` + +For public clients, use DPoP auth instead of `X-Client-Secret`: + +``` +DELETE /oauth/sessions/did:plc:user123/devices/uuid-session-1 +X-Client-Key: hvc_... +Authorization: DPoP +DPoP: +``` + +Returns `204 No Content` on success, `404 Not Found` if the session doesn't exist or doesn't belong to the client/user. ### Security notes -- 2.51.2 From 3e799628c5104064fcda0861e252eb130a8b7785 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 16:38:15 -0500 Subject: [PATCH 40/64] ci: prevent pr-builds from running when merging `dev` (taken care of by regular release process) Signed-off-by: Trezy --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6ea1b3..65f7bb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -169,6 +169,7 @@ jobs: needs: changes if: >- github.event_name == 'pull_request' + && github.head_ref != 'dev' && needs.changes.outputs.server == 'true' runs-on: ${{ matrix.runs-on }} strategy: -- 2.51.2 From 926564103417a45390f7f99c46ba07ed2a4c06c2 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 17:15:32 -0500 Subject: [PATCH 41/64] fix: allow docker containers to execute HappyView binaries Signed-off-by: Trezy --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 97108be..ad103be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /app COPY --from=builder /app/target/release/happyview /usr/local/bin/happyview +RUN chmod +x /usr/local/bin/happyview COPY migrations/ /app/migrations COPY --from=frontend /app/web/out /srv/static COPY entrypoint.sh /entrypoint.sh -- 2.51.2 From da23717e1609015a3aff68b0207e2d968485a181 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 17:33:54 -0500 Subject: [PATCH 42/64] ci: merge binary and docker builds for better concurrency Signed-off-by: Trezy --- .github/workflows/ci.yml | 299 ++++++++++++++++----------------------- 1 file changed, 125 insertions(+), 174 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65f7bb2..3b2d737 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,23 +163,30 @@ jobs: run: cargo clippy --all-targets -- -D warnings # --------------------------------------------------------------------------- - # PR binaries — build on every PR push so reviewers can test + # PR builds — compile + Docker on every PR push so reviewers can test # --------------------------------------------------------------------------- - pr-binary: + pr-build: needs: changes if: >- github.event_name == 'pull_request' && github.head_ref != 'dev' && needs.changes.outputs.server == 'true' runs-on: ${{ matrix.runs-on }} + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: include: - - runs-on: depot-ubuntu-24.04 + - platform: linux/amd64 + runs-on: depot-ubuntu-24.04 arch: amd64 - - runs-on: depot-ubuntu-24.04-arm + - platform: linux/arm64 + runs-on: depot-ubuntu-24.04-arm arch: arm64 + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -211,38 +218,10 @@ jobs: name: happyview-linux-${{ matrix.arch }} path: target/release/happyview - pr-docker: - needs: pr-binary - runs-on: ${{ matrix.runs-on }} - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - include: - - platform: linux/amd64 - runs-on: depot-ubuntu-24.04 - arch: amd64 - - platform: linux/arm64 - runs-on: depot-ubuntu-24.04-arm - arch: arm64 - env: - GHCR_IMAGE: ghcr.io/${{ github.repository }} - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Download pre-built binary - uses: actions/download-artifact@v4 - with: - name: happyview-linux-${{ matrix.arch }} - path: .binary - - name: Prepare binary override run: | mkdir -p .builder-override/app/target/release - cp .binary/happyview .builder-override/app/target/release/happyview + cp target/release/happyview .builder-override/app/target/release/happyview - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -284,7 +263,7 @@ jobs: ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} pr-build-comment: - needs: pr-docker + needs: pr-build runs-on: depot-ubuntu-24.04 permissions: pull-requests: write @@ -370,14 +349,16 @@ jobs: run: npx -p semantic-release -p semantic-release-gha-output semantic-release # --------------------------------------------------------------------------- - # Build binaries — compile per-platform and upload to GitHub release + # Build + Docker — compile per-platform, upload to GitHub release, push image # --------------------------------------------------------------------------- - build-binary: + build: needs: release if: needs.release.outputs.version != '' runs-on: ${{ matrix.runs-on }} permissions: contents: write + packages: write + id-token: write strategy: fail-fast: false matrix: @@ -388,6 +369,9 @@ jobs: - platform: linux/arm64 runs-on: depot-ubuntu-24.04-arm arch: arm64 + env: + GHCR_IMAGE: ghcr.io/${{ github.repository }} + ATCR_IMAGE: atcr.io/${{ secrets.ATCR_NAMESPACE }}/happyview steps: - name: Checkout repository uses: actions/checkout@v6 @@ -423,6 +407,111 @@ jobs: happyview-linux-${{ matrix.arch }} \ --clobber + - name: Prepare binary override + run: | + mkdir -p .builder-override/app/target/release + cp target/release/happyview .builder-override/app/target/release/happyview + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # TODO: Re-enable once ATCR rate limit issue is resolved + # - name: Log in to ATCR + # uses: docker/login-action@v3 + # with: + # registry: atcr.io + # username: ${{ secrets.ATCR_USERNAME }} + # password: ${{ secrets.ATCR_PASSWORD }} + + - name: Parse version + id: version + run: | + VERSION="${{ needs.release.outputs.version }}" + MAJOR="${VERSION%%.*}" + MINOR="${VERSION#*.}" + MINOR="${MINOR%%.*}" + echo "full=${VERSION}" >> "$GITHUB_OUTPUT" + echo "major=${MAJOR}" >> "$GITHUB_OUTPUT" + echo "major_minor=${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT" + + - name: Extract GHCR metadata + id: meta-ghcr + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_IMAGE }} + tags: | + type=raw,value=${{ steps.version.outputs.full }} + type=raw,value=${{ steps.version.outputs.major_minor }} + type=raw,value=${{ steps.version.outputs.major }} + type=sha + + # TODO: Re-enable once ATCR rate limit issue is resolved + # - name: Extract ATCR metadata + # id: meta-atcr + # uses: docker/metadata-action@v5 + # with: + # images: ${{ env.ATCR_IMAGE }} + # tags: | + # type=semver,pattern={{version}} + # type=semver,pattern={{major}}.{{minor}} + # type=semver,pattern={{major}} + # type=sha + + - name: Build and push to GHCR by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta-ghcr.outputs.labels }} + build-contexts: | + builder=.builder-override + cache-from: type=gha,scope=build-${{ matrix.platform }} + cache-to: type=gha,scope=build-${{ matrix.platform }},mode=max,ignore-error=true + outputs: type=image,"name=${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + # TODO: Re-enable once ATCR rate limit issue is resolved + # - name: Push to ATCR by digest + # id: build-atcr + # continue-on-error: true + # uses: docker/build-push-action@v5 + # with: + # context: . + # platforms: ${{ matrix.platform }} + # labels: ${{ steps.meta-atcr.outputs.labels }} + # build-contexts: | + # builder=.builder-override + # cache-from: type=gha,scope=build-${{ matrix.platform }} + # outputs: type=image,"name=${{ env.ATCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true + + - name: Create GHCR manifest + run: | + GHCR_TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-ghcr.outputs.json }}') + docker buildx imagetools create --append $GHCR_TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} 2>/dev/null || \ + docker buildx imagetools create $GHCR_TAGS \ + ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} + + # TODO: Re-enable once ATCR rate limit issue is resolved + # - name: Create ATCR manifest + # if: steps.build-atcr.outcome == 'success' + # continue-on-error: true + # run: | + # ATCR_TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-atcr.outputs.json }}') + # if [ -n "$ATCR_TAGS" ]; then + # docker buildx imagetools create --append $ATCR_TAGS \ + # ${{ env.ATCR_IMAGE }}@${{ steps.build-atcr.outputs.digest }} 2>/dev/null || \ + # docker buildx imagetools create $ATCR_TAGS \ + # ${{ env.ATCR_IMAGE }}@${{ steps.build-atcr.outputs.digest }} + # fi + # --------------------------------------------------------------------------- # SDK — per-package tests + releases # --------------------------------------------------------------------------- @@ -739,144 +828,6 @@ jobs: working-directory: packages/lex-agent run: npx semantic-release - # --------------------------------------------------------------------------- - # Docker — main app image (triggered by GitHub release) - # --------------------------------------------------------------------------- - docker: - needs: [release, build-binary] - if: needs.release.outputs.version != '' - runs-on: ${{ matrix.runs-on }} - permissions: - contents: read - packages: write - id-token: write - strategy: - fail-fast: false - matrix: - include: - - platform: linux/amd64 - runs-on: depot-ubuntu-24.04 - arch: amd64 - - platform: linux/arm64 - runs-on: depot-ubuntu-24.04-arm - arch: arm64 - env: - GHCR_IMAGE: ghcr.io/${{ github.repository }} - ATCR_IMAGE: atcr.io/${{ secrets.ATCR_NAMESPACE }}/happyview - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Download pre-built binary - env: - GH_TOKEN: ${{ secrets.DISPATCH_GH_TOKEN }} - run: | - gh release download "v${{ needs.release.outputs.version }}" \ - --pattern "happyview-linux-${{ matrix.arch }}" \ - --dir . - mkdir -p .builder-override/app/target/release - cp happyview-linux-${{ matrix.arch }} .builder-override/app/target/release/happyview - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # TODO: Re-enable once ATCR rate limit issue is resolved - # - name: Log in to ATCR - # uses: docker/login-action@v3 - # with: - # registry: atcr.io - # username: ${{ secrets.ATCR_USERNAME }} - # password: ${{ secrets.ATCR_PASSWORD }} - - - name: Parse version - id: version - run: | - VERSION="${{ needs.release.outputs.version }}" - MAJOR="${VERSION%%.*}" - MINOR="${VERSION#*.}" - MINOR="${MINOR%%.*}" - echo "full=${VERSION}" >> "$GITHUB_OUTPUT" - echo "major=${MAJOR}" >> "$GITHUB_OUTPUT" - echo "major_minor=${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT" - - - name: Extract GHCR metadata - id: meta-ghcr - uses: docker/metadata-action@v5 - with: - images: ${{ env.GHCR_IMAGE }} - tags: | - type=raw,value=${{ steps.version.outputs.full }} - type=raw,value=${{ steps.version.outputs.major_minor }} - type=raw,value=${{ steps.version.outputs.major }} - type=sha - - # TODO: Re-enable once ATCR rate limit issue is resolved - # - name: Extract ATCR metadata - # id: meta-atcr - # uses: docker/metadata-action@v5 - # with: - # images: ${{ env.ATCR_IMAGE }} - # tags: | - # type=semver,pattern={{version}} - # type=semver,pattern={{major}}.{{minor}} - # type=semver,pattern={{major}} - # type=sha - - - name: Build and push to GHCR by digest - id: build - uses: docker/build-push-action@v5 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta-ghcr.outputs.labels }} - build-contexts: | - builder=.builder-override - cache-from: type=gha,scope=build-${{ matrix.platform }} - cache-to: type=gha,scope=build-${{ matrix.platform }},mode=max,ignore-error=true - outputs: type=image,"name=${{ env.GHCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true - - # TODO: Re-enable once ATCR rate limit issue is resolved - # - name: Push to ATCR by digest - # id: build-atcr - # continue-on-error: true - # uses: docker/build-push-action@v5 - # with: - # context: . - # platforms: ${{ matrix.platform }} - # labels: ${{ steps.meta-atcr.outputs.labels }} - # build-contexts: | - # builder=.builder-override - # cache-from: type=gha,scope=build-${{ matrix.platform }} - # outputs: type=image,"name=${{ env.ATCR_IMAGE }}",push-by-digest=true,name-canonical=true,push=true - - - name: Create GHCR manifest - run: | - GHCR_TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-ghcr.outputs.json }}') - docker buildx imagetools create --append $GHCR_TAGS \ - ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} 2>/dev/null || \ - docker buildx imagetools create $GHCR_TAGS \ - ${{ env.GHCR_IMAGE }}@${{ steps.build.outputs.digest }} - - # TODO: Re-enable once ATCR rate limit issue is resolved - # - name: Create ATCR manifest - # if: steps.build-atcr.outcome == 'success' - # continue-on-error: true - # run: | - # ATCR_TAGS=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta-atcr.outputs.json }}') - # if [ -n "$ATCR_TAGS" ]; then - # docker buildx imagetools create --append $ATCR_TAGS \ - # ${{ env.ATCR_IMAGE }}@${{ steps.build-atcr.outputs.digest }} 2>/dev/null || \ - # docker buildx imagetools create $ATCR_TAGS \ - # ${{ env.ATCR_IMAGE }}@${{ steps.build-atcr.outputs.digest }} - # fi - # --------------------------------------------------------------------------- # Mirror to Tangled # --------------------------------------------------------------------------- -- 2.51.2 From bc06766fc9760ee6a61457dd3b433f907bc4ea4d Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 21:34:57 -0500 Subject: [PATCH 43/64] fix: remove superfluous early exit on invalid session token (pds concern, not happyview concern) Signed-off-by: Trezy --- src/auth/middleware.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/auth/middleware.rs b/src/auth/middleware.rs index 4715311..250d98c 100644 --- a/src/auth/middleware.rs +++ b/src/auth/middleware.rs @@ -197,14 +197,6 @@ pub async fn resolve_dpop_claims( ) .await?; - // Check token expiry - if let Some(ref expires_at) = session.token_expires_at - && let Ok(exp) = chrono::DateTime::parse_from_rfc3339(expires_at) - && exp < chrono::Utc::now() - { - return Err(AppError::Auth("token_expired".into())); - } - // Build the request URL for htu validation let scheme = if state.config.public_url.starts_with("https") { "https" -- 2.51.2 From 346f2e09c31ade286fcc0f3234920cdff3f0005d Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 26 May 2026 23:30:15 -0500 Subject: [PATCH 44/64] docs: blog for 2.9 release Signed-off-by: Trezy --- packages/docs/content/blog/happyview-2.9.md | 81 +++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 packages/docs/content/blog/happyview-2.9.md diff --git a/packages/docs/content/blog/happyview-2.9.md b/packages/docs/content/blog/happyview-2.9.md new file mode 100644 index 0000000..5421367 --- /dev/null +++ b/packages/docs/content/blog/happyview-2.9.md @@ -0,0 +1,81 @@ +--- +title: "HappyView v2.9" +description: "Backfill concurrency, db.query filters, multi-device DPoP sessions, and a mountain of performance fixes." +date: 2026-05-26 +author: + name: "Trezy" + avatar: "/authors/trezy.webp" +tags: + - announcements +--- + +While there's not a lot of big, shiny new features this time around, 2.9 is chock full of performance improvements and bug fixes to make everybody's life better. + +## Backfill, but make it concurrent + +The biggest change is that PDS resolution and record fetching now run concurrently. Previously, HappyView resolved every DID's PDS endpoint before it started fetching any records. For large backfills with hundreds of thousands of DIDs, that meant the fetcher sat idle for potentially hours. Now fetching starts as soon as the first DIDs are resolved and runs alongside resolution for the rest of the job. + +On top of that: + +- **Pause and resume** — you can now manually pause a running backfill and pick it back up later. No lost progress. +- **Concurrency settings** — new settings in the dashboard let you tune PDS concurrency, DID concurrency per PDS, and PLC directory concurrency. HappyView will recommend a restart if the settings require a larger connection pool than the one currently running. +- **Concurrent collection discovery** — the repo discovery phase now runs multiple collection queries in parallel instead of sequentially. +- **Batch record inserts** — record inserts are now batched, significantly reducing database round trips during the fetch phase. +- **Separate connection pool** — backfill jobs now use their own database connection pool so they can't starve the main app of connections during heavy backfills. + +The backfill details view got a significant overhaul: progress indicators are more detailed and more accessible. + +## `db.query` filters + +You can now filter records directly in `db.query` without writing raw SQL or post-processing in Lua: + +```lua +local result = db.query({ + collection = "com.example.post", + filter = { field = "status", value = "published" }, +}) +``` + +Filters support comparison operators (`=`, `!=`, `>`, `<`, `>=`, `<=`), `AND`/`OR` groups, and nesting up to 5 levels deep. Field paths use the same dot notation and array indices as `sort` (e.g. `author.handle`, `scores[0]`). + +```lua +local result = db.query({ + collection = "com.example.post", + filter = { + op = "AND", + conditions = { + { field = "status", value = "published" }, + { field = "views", op = ">", value = 100 }, + }, + }, +}) +``` + +Full docs are in the [Database API reference](/docs/api-reference/lua/database-api). + +## Auth fixes + +There were actually bugs that have been making my life hard, but I _finally_ figured them out. + +First, users were basically limited to one auth session per client. If you signed into [Cartridge](https://cartridge.dev) from a second device, it would kill your other auth session. Whoops. + +Second, there were scenarios where the PDS may refresh the auth session while a HappyView XRPC was in-progress. IF that happened, HappyView would handle it internally so any other requests in that XRPC worked, but _it didn't return the refreshed tokens to the client._ Follow up requests from the client would break. Double whoops. + +Both of these are fixed properly now, AND I added a couple new endpoints so clients can allow users to see and manage their active sessions: + +- `GET /oauth/sessions/{did}/devices` — list all active sessions +- `DELETE /oauth/sessions/{did}/devices/{session_id}` — revoke a session + +The existing `DELETE /oauth/sessions/{did}` endpoint still works: confidential clients revoke all device sessions for the user, and public clients revoke the session matching their DPoP key. Full details in the [Authentication guide](/docs/getting-started/authentication#6-managing-device-sessions). + +## SDK fix + +If you tried to use `@happyview/oauth-client` with the latest versions of the `@atproto/*` SDKs, things would break because of a missing parameter. `@happyview/oauth-client` now provides that parameter and should also be backwards-compatible. + +## CI & infrastructure + +- **Binary releases** — Rust binaries are now published to GitHub Releases alongside Docker images, so you can grab a prebuilt binary directly. + +## Go play + +Full changelog is on [GitHub](https://github.com/gamesgamesgamesgamesgames/happyview/releases/tag/v2.9.0). If you have questions, feature requests, or just need a little help, join the [Cartridge](https://cartridge.dev) [Discord Server](https://discord.gg/BUPnjaBwRZ) and hop into the `#happyview` channel. -- 2.51.2 From 0fd76010212e3ce87057b3a32268b0ae5822326b Mon Sep 17 00:00:00 2001 From: Chris Pardy Date: Fri, 1 May 2026 07:46:55 -0400 Subject: [PATCH 45/64] feat(scripts): trigger-keyed scripts subsystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the legacy `index_hook` lookup with a trigger-keyed scripts table where each row's PK IS the trigger string the dispatcher resolves on. There's no separate "name" or "host column" — bindings, names, and inline-vs-named all collapse into one column. Trigger grammar: record.index: — wildcard for any record event record.create: — specifically a create event record.update: — specifically an update event record.delete: — specifically a delete event xrpc.query: — XRPC query handler xrpc.procedure: — XRPC procedure handler labeler.apply: — label whose uri is at://// labeler.apply:_actor — label whose uri is a bare DID Cascade for record events ONLY: the dispatcher tries `record.:` first, falls back to `record.index:`. No cascade for XRPC or labeler triggers. Schema (one new migration, no data copy — origin/dev had no host columns to migrate from): - New `scripts` table (id PK + body / description / script_type + timestamps). - New `dead_letter_scripts` table (script_ref carries the trigger id). - Re-grants `scripts:read` to lexicon-readers, `scripts:manage` to lexicon-managers. Code surface: - New `src/lua/scripts.rs` dispatcher: ParsedTrigger grammar + validator, ScriptRow / ResolvedScript, `resolve` / `resolve_record_event` (with cascade), `run_record_event_script` (fail-open + retry + dead-letter), `run_label_applied_script`, `trigger_for_label_uri` (at:// → nsid, bare DID → _actor), and `run_record_event_once` for the dead-letter retry path. - New `src/admin/scripts.rs` CRUD (list / get / upsert / patch / delete) with trigger-id grammar validation at write-time and Lua body validation via `crate::lua::validate_script`. - New `Permission::ScriptsRead` / `Permission::ScriptsManage`. - `src/labeler.rs::apply_label` computes the trigger from `label.uri` and runs the script chain before persisting; rewritten labels persist; nil-returning scripts skip. - `src/record_handler.rs::handle_record_event` calls `run_record_event_script(state, &collection, &action, ...)` for both create/update and delete actions. - `src/xrpc/procedure.rs` and `src/xrpc/query.rs` look up via the new dispatcher (trigger `xrpc.procedure:` / `xrpc.query:`) before falling through to the default PDS-write / list flows. - `src/admin/dead_letters.rs::retry_single` resolves via the new dispatcher's cascade (404s if no script matches now). - Legacy `execute_hook_script`, `HookEvent`, and `run_hook_once` removed from `src/lua/execute.rs`. - `register_record_api` refactored to take Optional Claims / PdsAuth; new `register_record_api_no_auth` public wrapper for the no-auth contexts (record / labeler / query). PDS-touching methods (`r:save`, `r:delete`, `Record.save_all`) error cleanly with "no PDS auth in this script context" when registered without auth. - `r:save_local()` / `r:delete_local()` instance methods + `Record.delete_local(uri)` static for local-only mutation in any script context. - `r:delete()` proceeds with the local delete even on PDS failure (operator's logical action is removal regardless of PDS state). Legacy `lexicons.index_hook` column is retained but inert — operators with existing data manually port into a `record.index:` script row via the dashboard. Tests: - 12 new lib tests in `src/lua/scripts.rs` covering trigger grammar, NSID validation, label URI routing, and the `_actor` special case. - 8 integration tests in `tests/lua_record_api.rs` covering the Record local API + the no-PDS-auth boundary. - 18 e2e tests in `tests/e2e_scripts.rs` covering admin CRUD, trigger validation, the cascade rule, label-script + record-event Record local mutation, and the dead-letter behavior when a label script reaches for `r:save()`. - The 4 query-script lib tests in `src/lua/xrpc_api.rs` updated to seed scripts via the new table (plus a single-connection test pool so the in-memory sqlite is shared). Signed-off-by: Chris Pardy --- .../20260501000000_scripts_by_trigger.sql | 58 ++ .../20260501000000_scripts_by_trigger.sql | 52 ++ src/admin/backfill.rs | 37 +- src/admin/dead_letters.rs | 57 +- src/admin/mod.rs | 8 + src/admin/permissions.rs | 23 + src/admin/scripts.rs | 373 ++++++++ src/labeler.rs | 59 +- src/lua/execute.rs | 517 +---------- src/lua/mod.rs | 13 +- src/lua/record.rs | 262 +++++- src/lua/scripts.rs | 855 ++++++++++++++++++ src/lua/xrpc_api.rs | 90 +- src/record_handler.rs | 151 ++-- src/xrpc/procedure.rs | 12 +- src/xrpc/query.rs | 14 +- tests/common/db.rs | 4 +- tests/common/mod.rs | 2 + tests/e2e_scripts.rs | 833 +++++++++++++++++ tests/lua_record_api.rs | 478 ++++++++++ 20 files changed, 3210 insertions(+), 688 deletions(-) create mode 100644 migrations/postgres/20260501000000_scripts_by_trigger.sql create mode 100644 migrations/sqlite/20260501000000_scripts_by_trigger.sql create mode 100644 src/admin/scripts.rs create mode 100644 src/lua/scripts.rs create mode 100644 tests/e2e_scripts.rs create mode 100644 tests/lua_record_api.rs diff --git a/migrations/postgres/20260501000000_scripts_by_trigger.sql b/migrations/postgres/20260501000000_scripts_by_trigger.sql new file mode 100644 index 0000000..a277839 --- /dev/null +++ b/migrations/postgres/20260501000000_scripts_by_trigger.sql @@ -0,0 +1,58 @@ +-- Trigger-keyed scripts subsystem. +-- +-- Each row's `id` IS the trigger string the dispatcher resolves on: +-- record.index: — wildcard for any record event +-- record.create: — specifically a create event (cascades to wildcard) +-- record.update: — specifically an update event +-- record.delete: — specifically a delete event +-- xrpc.query: — XRPC query handler +-- xrpc.procedure: — XRPC procedure handler +-- labeler.apply: — label arrives whose subject is at://// +-- labeler.apply:_actor — label arrives whose subject is a bare DID +-- +-- Cascade rule (record events ONLY): the dispatcher tries +-- `record.:` first, falls back to `record.index:` if no +-- specific row exists. No cascade for XRPC or labeler triggers — those +-- resolve directly. Operators express per-action surgical control by +-- creating action-specific rows; the wildcard `record.index:` covers +-- "one body for everything" with branching on `event.action`. +CREATE TABLE scripts ( + id TEXT PRIMARY KEY, + body TEXT NOT NULL, + description TEXT, + script_type TEXT NOT NULL DEFAULT 'lua', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- Permanently-failed runs from the firehose-driven runners (record / label +-- events). XRPC scripts fail-closed and never land here. +CREATE TABLE dead_letter_scripts ( + id BIGSERIAL PRIMARY KEY, + script_ref TEXT NOT NULL, -- = the trigger id whose script failed + host_kind TEXT NOT NULL, -- 'record' | 'label' + host_id TEXT NOT NULL, -- e.g. ':' for record, '' for label + payload JSONB NOT NULL, -- event payload for re-run + error TEXT NOT NULL, + attempts INT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + resolved_at TIMESTAMPTZ +); + +CREATE INDEX idx_dead_letter_scripts_host + ON dead_letter_scripts (host_kind, host_id, created_at DESC); +CREATE INDEX idx_dead_letter_scripts_resolved_at + ON dead_letter_scripts (resolved_at); + +-- Permissions: management for users who can manage lexicons; read for those who can read. +INSERT INTO user_permissions (user_id, permission) +SELECT user_id, 'scripts:manage' + FROM user_permissions + WHERE permission = 'lexicons:create' +ON CONFLICT (user_id, permission) DO NOTHING; + +INSERT INTO user_permissions (user_id, permission) +SELECT user_id, 'scripts:read' + FROM user_permissions + WHERE permission = 'lexicons:read' +ON CONFLICT (user_id, permission) DO NOTHING; diff --git a/migrations/sqlite/20260501000000_scripts_by_trigger.sql b/migrations/sqlite/20260501000000_scripts_by_trigger.sql new file mode 100644 index 0000000..c1420c1 --- /dev/null +++ b/migrations/sqlite/20260501000000_scripts_by_trigger.sql @@ -0,0 +1,52 @@ +-- Trigger-keyed scripts subsystem. +-- +-- Each row's `id` IS the trigger string the dispatcher resolves on: +-- record.index: — wildcard for any record event +-- record.create: — specifically a create event (cascades to wildcard) +-- record.update: — specifically an update event +-- record.delete: — specifically a delete event +-- xrpc.query: — XRPC query handler +-- xrpc.procedure: — XRPC procedure handler +-- labeler.apply: — label arrives whose subject is at://// +-- labeler.apply:_actor — label arrives whose subject is a bare DID +-- +-- See migrations/postgres/20260501000000_scripts_by_trigger.sql for design notes. +-- SQLite mirror. +CREATE TABLE scripts ( + id TEXT PRIMARY KEY, + body TEXT NOT NULL, + description TEXT, + script_type TEXT NOT NULL DEFAULT 'lua', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE TABLE dead_letter_scripts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + script_ref TEXT NOT NULL, -- = the trigger id whose script failed + host_kind TEXT NOT NULL, -- 'record' | 'label' (xrpc fails-closed; never dead-letters) + host_id TEXT NOT NULL, -- e.g. ':' for record, '' for label + payload TEXT NOT NULL, -- JSON-serialized event for re-run + error TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + resolved_at TEXT +); + +CREATE INDEX idx_dead_letter_scripts_host + ON dead_letter_scripts (host_kind, host_id, created_at DESC); +CREATE INDEX idx_dead_letter_scripts_resolved_at + ON dead_letter_scripts (resolved_at); + +-- Permissions: management for users who can manage lexicons; read for those who can read. +INSERT INTO user_permissions (user_id, permission, granted_at) +SELECT user_id, 'scripts:manage', datetime('now') + FROM user_permissions + WHERE permission = 'lexicons:create' +ON CONFLICT (user_id, permission) DO NOTHING; + +INSERT INTO user_permissions (user_id, permission, granted_at) +SELECT user_id, 'scripts:read', datetime('now') + FROM user_permissions + WHERE permission = 'lexicons:read' +ON CONFLICT (user_id, permission) DO NOTHING; diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 5d6cb36..4300ae1 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1305,9 +1305,6 @@ async fn fetch_records_from_pds( let base = pds_endpoint.trim_end_matches('/'); let mut cursor: Option = None; let mut count: u32 = 0; - let index_hook = state.lexicons.get_index_hook(collection).await; - let env_vars = crate::lua::load_env_vars_cached(&state.db, state.db_backend).await; - loop { if cancelled.load(Ordering::Relaxed) { break; @@ -1350,27 +1347,19 @@ async fn fetch_records_from_pds( let rkey = entry.uri.rsplit('/').next().unwrap_or_default().to_string(); let uri = format!("at://{did}/{collection}/{rkey}"); - let rec_to_store = if let Some(ref script) = index_hook { - let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { - state, - lexicon_id: collection, - script, - action: "create", - uri: &uri, - did, - collection, - rkey: &rkey, - record: Some(&entry.value), - cached_env_vars: Some(&env_vars), - }) - .await; - - match hook_result { - None => continue, - Some(v) => v, - } - } else { - entry.value.clone() + let rec_to_store = match crate::lua::run_record_event_script( + state, + collection, + "create", + &uri, + did, + &rkey, + Some(&entry.value), + ) + .await + { + None => continue, + Some(v) => v, }; batch.push(PreparedRecord { diff --git a/src/admin/dead_letters.rs b/src/admin/dead_letters.rs index 1ddf68d..c5a1018 100644 --- a/src/admin/dead_letters.rs +++ b/src/admin/dead_letters.rs @@ -10,7 +10,7 @@ use super::permissions::Permission; use crate::AppState; use crate::db::{adapt_sql, now_rfc3339, parse_dt}; use crate::error::AppError; -use crate::lua::{HookEvent, run_hook_once}; +use crate::lua::{resolve_record_event, run_record_event_once}; use crate::record_handler::RecordEvent; // --------------------------------------------------------------------------- @@ -487,29 +487,22 @@ async fn resolve_bulk_ids(state: &AppState, body: &BulkRequest) -> Result Result, AppError> { - let backend = state.db_backend; - let sql = adapt_sql("SELECT index_hook FROM lexicons WHERE id = ?", backend); - let row: Option<(Option,)> = sqlx::query_as(&sql) - .bind(lexicon_id) - .fetch_optional(&state.db) - .await - .map_err(|e| AppError::Internal(format!("failed to fetch index hook: {e}")))?; - Ok(row.and_then(|r| r.0)) -} - -/// Retry a single dead letter by re-running its hook script. +/// Retry a single dead letter by re-running its trigger-keyed script. +/// +/// Resolves the script via the new dispatcher's cascade +/// (`record.:` → `record.index:`). If no script is +/// bound for the cascade now, returns 404 — the operator either deleted +/// the script or never re-bound it under the new naming. async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { let dl = fetch_dead_letter_for_action(state, id).await?; - let script = get_index_hook_from_db(state, &dl.lexicon_id) - .await? + let resolved = resolve_record_event(state, &dl.collection, &dl.action) + .await .ok_or_else(|| { - AppError::NotFound(format!("no index hook found for lexicon {}", dl.lexicon_id)) + AppError::NotFound(format!( + "no script bound for record.{}:{} (or record.index:{})", + dl.action, dl.collection, dl.collection + )) })?; let record: Option = dl @@ -517,20 +510,18 @@ async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { .as_deref() .and_then(|r| serde_json::from_str(r).ok()); - let event = HookEvent { + match run_record_event_once( state, - lexicon_id: &dl.lexicon_id, - script: &script, - action: &dl.action, - uri: &dl.uri, - did: &dl.did, - collection: &dl.collection, - rkey: &dl.rkey, - record: record.as_ref(), - cached_env_vars: None, - }; - - match run_hook_once(&event).await { + &resolved, + &dl.action, + &dl.uri, + &dl.did, + &dl.collection, + &dl.rkey, + record.as_ref(), + ) + .await + { Ok(_) => { mark_resolved(state, id).await?; Ok(()) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f168637..1446af2 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -14,6 +14,7 @@ mod plugins; mod proxy_config; mod records; mod script_variables; +mod scripts; pub mod settings; mod stats; pub(crate) mod types; @@ -90,6 +91,13 @@ pub fn admin_routes(_state: AppState) -> Router { post(script_variables::upsert).get(script_variables::list), ) .route("/script-variables/{key}", delete(script_variables::delete)) + .route("/scripts", get(scripts::list).post(scripts::upsert)) + .route( + "/scripts/{id}", + get(scripts::get) + .patch(scripts::patch) + .delete(scripts::delete), + ) .route("/labelers", post(labelers::add).get(labelers::list)) .route( "/labelers/{did}", diff --git a/src/admin/permissions.rs b/src/admin/permissions.rs index 1bc0977..9105490 100644 --- a/src/admin/permissions.rs +++ b/src/admin/permissions.rs @@ -108,6 +108,11 @@ pub enum Permission { SpacesManageRecords, #[serde(rename = "spaces:manage-credentials")] SpacesManageCredentials, + + #[serde(rename = "scripts:read")] + ScriptsRead, + #[serde(rename = "scripts:manage")] + ScriptsManage, } impl Permission { @@ -155,6 +160,8 @@ impl Permission { Self::SpacesManageInvites => "spaces:manage-invites", Self::SpacesManageRecords => "spaces:manage-records", Self::SpacesManageCredentials => "spaces:manage-credentials", + Self::ScriptsRead => "scripts:read", + Self::ScriptsManage => "scripts:manage", } } @@ -406,6 +413,18 @@ impl Permission { description: "Issue and revoke space access credentials", category: "Spaces", }, + Self::ScriptsRead => PermissionInfo { + key: "scripts:read", + name: "View Scripts", + description: "View trigger-keyed scripts", + category: "Scripts", + }, + Self::ScriptsManage => PermissionInfo { + key: "scripts:manage", + name: "Manage Scripts", + description: "Create, update, and delete trigger-keyed scripts", + category: "Scripts", + }, } } @@ -453,6 +472,8 @@ impl Permission { Self::SpacesManageInvites, Self::SpacesManageRecords, Self::SpacesManageCredentials, + Self::ScriptsRead, + Self::ScriptsManage, ]) } } @@ -571,6 +592,7 @@ impl Template { Permission::LexiconsRead, Permission::RecordsRead, Permission::ScriptVariablesRead, + Permission::ScriptsRead, Permission::UsersRead, Permission::ApiKeysRead, Permission::BackfillRead, @@ -592,6 +614,7 @@ impl Template { perms.insert(Permission::LexiconsDelete); perms.insert(Permission::ScriptVariablesCreate); perms.insert(Permission::ScriptVariablesDelete); + perms.insert(Permission::ScriptsManage); perms.insert(Permission::RecordsDelete); perms.insert(Permission::LabelersCreate); perms.insert(Permission::LabelersRead); diff --git a/src/admin/scripts.rs b/src/admin/scripts.rs new file mode 100644 index 0000000..76a1279 --- /dev/null +++ b/src/admin/scripts.rs @@ -0,0 +1,373 @@ +//! `/admin/scripts` CRUD — trigger-keyed scripts. +//! +//! Each script row's `id` IS its trigger string (e.g. +//! `record.create:com.example.thing`, `xrpc.query:com.foo.list`, +//! `labeler.apply:_actor`). The dispatcher in [`crate::lua::scripts`] +//! looks up scripts by id at firing time; this admin surface lets +//! operators CRUD those rows. +//! +//! Validation: +//! - On create / patch the body is parsed against the script_type +//! (lua → [`crate::lua::validate_script`]). Invalid bodies are +//! rejected at write-time with a 400. +//! - The trigger id is parsed against +//! [`crate::lua::ParsedTrigger::parse`]; unknown prefixes / invalid +//! NSIDs are rejected at write-time with a 400. +//! +//! Permissions: `scripts:read` for GETs; `scripts:manage` for the +//! mutating endpoints. + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use serde::{Deserialize, Serialize}; + +use crate::AppState; +use crate::db::{adapt_sql, now_rfc3339}; +use crate::error::AppError; +use crate::event_log::{EventLog, Severity, log_event}; +use crate::lua::{ParsedTrigger, ScriptLanguage}; + +use super::auth::UserAuth; +use super::permissions::Permission; + +// --------------------------------------------------------------------------- +// Wire types +// --------------------------------------------------------------------------- + +/// One row from the `scripts` table — what GET endpoints return. +#[derive(Debug, Clone, Serialize)] +pub(super) struct ScriptResponse { + /// The trigger id; identifies the row. + pub id: String, + pub script_type: String, + pub body: String, + pub description: Option, + pub created_at: String, + pub updated_at: String, +} + +/// Body for `POST /admin/scripts` (create or replace by `id`). +#[derive(Debug, Deserialize)] +pub(super) struct UpsertBody { + pub id: String, + /// Defaults to `"lua"` server-side if omitted. + #[serde(default)] + pub script_type: Option, + pub body: String, + #[serde(default)] + pub description: Option, +} + +/// Body for `PATCH /admin/scripts/{id}`. All fields optional. +#[derive(Debug, Deserialize)] +pub(super) struct PatchBody { + #[serde(default)] + pub script_type: Option, + #[serde(default)] + pub body: Option, + /// Set to `Some(None)` to clear via JSON `null`. + #[serde(default, deserialize_with = "deserialize_optional_field")] + pub description: Option>, +} + +/// Three-state field deserializer: missing → `None`, `null` → `Some(None)`, +/// string → `Some(Some(s))`. Lets PATCH distinguish "leave as-is" from +/// "clear to NULL". +fn deserialize_optional_field<'de, D>(d: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + let v: Option = Option::deserialize(d)?; + Ok(Some(v)) +} + +// --------------------------------------------------------------------------- +// Routes +// --------------------------------------------------------------------------- + +/// `GET /admin/scripts` — list all rows. Clients group by trigger family +/// in the UI. +pub(super) async fn list( + State(state): State, + auth: UserAuth, +) -> Result>, AppError> { + auth.require(Permission::ScriptsRead).await?; + + let backend = state.db_backend; + let sql = adapt_sql( + "SELECT id, script_type, body, description, created_at, updated_at + FROM scripts + ORDER BY id", + backend, + ); + #[allow(clippy::type_complexity)] + let rows: Vec<(String, String, String, Option, String, String)> = sqlx::query_as(&sql) + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to list scripts: {e}")))?; + + let scripts: Vec = rows + .into_iter() + .map( + |(id, script_type, body, description, created_at, updated_at)| ScriptResponse { + id, + script_type, + body, + description, + created_at, + updated_at, + }, + ) + .collect(); + + Ok(Json(scripts)) +} + +/// `GET /admin/scripts/{id}` — fetch one row. +pub(super) async fn get( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result, AppError> { + auth.require(Permission::ScriptsRead).await?; + fetch_one(&state, &id).await.map(Json) +} + +/// `POST /admin/scripts` — create or replace a row by `id`. Returns the +/// upserted row. Status `201 Created` for a new row, `200 OK` for an +/// update. +pub(super) async fn upsert( + State(state): State, + auth: UserAuth, + Json(body): Json, +) -> Result<(StatusCode, Json), AppError> { + auth.require(Permission::ScriptsManage).await?; + + // Validate the trigger id grammar up-front (400 with a clear message). + let _trigger = ParsedTrigger::parse(&body.id).map_err(AppError::BadRequest)?; + + let script_type = body.script_type.unwrap_or_default(); + validate_body_for_type(&body.body, script_type)?; + + let backend = state.db_backend; + let now = now_rfc3339(); + let description = body.description.as_deref().filter(|s| !s.is_empty()); + + // Distinguish create vs update so we can return 201 vs 200. + let pre_exists: Option<(String,)> = + sqlx::query_as(&adapt_sql("SELECT id FROM scripts WHERE id = ?", backend)) + .bind(&body.id) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to check script existence: {e}")))?; + let was_new = pre_exists.is_none(); + + let sql = adapt_sql( + r#" + INSERT INTO scripts (id, script_type, body, description, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (id) DO UPDATE SET + script_type = EXCLUDED.script_type, + body = EXCLUDED.body, + description = EXCLUDED.description, + updated_at = EXCLUDED.updated_at + "#, + backend, + ); + sqlx::query(&sql) + .bind(&body.id) + .bind(script_type.as_str()) + .bind(&body.body) + .bind(description) + .bind(&now) + .bind(&now) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to upsert script: {e}")))?; + + log_event( + &state.db, + EventLog { + event_type: if was_new { + "script.created".to_string() + } else { + "script.updated".to_string() + }, + severity: Severity::Info, + actor_did: Some(auth.did.clone()), + subject: Some(body.id.clone()), + detail: serde_json::json!({ + "script_type": script_type.as_str(), + }), + }, + backend, + ) + .await; + + let row = fetch_one(&state, &body.id).await?; + let status = if was_new { + StatusCode::CREATED + } else { + StatusCode::OK + }; + Ok((status, Json(row))) +} + +/// `PATCH /admin/scripts/{id}` — partial update. At least one of +/// `script_type` / `body` / `description` must be present. +pub(super) async fn patch( + State(state): State, + auth: UserAuth, + Path(id): Path, + Json(body): Json, +) -> Result, AppError> { + auth.require(Permission::ScriptsManage).await?; + + if body.script_type.is_none() && body.body.is_none() && body.description.is_none() { + return Err(AppError::BadRequest( + "patch requires at least one of: script_type, body, description".into(), + )); + } + // Patching a body or script_type? We need a body to validate against + // the (possibly new) language. Patching script_type alone is + // ambiguous (we'd be validating the existing body against the new + // language without re-checking it makes sense), so reject it. + if body.script_type.is_some() && body.body.is_none() { + return Err(AppError::BadRequest( + "patching script_type requires body alongside (so the server can re-validate)".into(), + )); + } + if let Some(ref new_body) = body.body { + let lang = body.script_type.unwrap_or_default(); + validate_body_for_type(new_body, lang)?; + } + + // Existence check + fetch current values. + let existing = fetch_one(&state, &id).await?; + + let backend = state.db_backend; + let now = now_rfc3339(); + let new_script_type = body + .script_type + .map(|s| s.as_str().to_string()) + .unwrap_or(existing.script_type); + let new_body = body.body.unwrap_or(existing.body); + let new_description = match body.description { + Some(desc_opt) => desc_opt, + None => existing.description, + }; + + let sql = adapt_sql( + r#" + UPDATE scripts + SET script_type = ?, + body = ?, + description = ?, + updated_at = ? + WHERE id = ? + "#, + backend, + ); + sqlx::query(&sql) + .bind(&new_script_type) + .bind(&new_body) + .bind(new_description.as_deref()) + .bind(&now) + .bind(&id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to patch script: {e}")))?; + + log_event( + &state.db, + EventLog { + event_type: "script.updated".to_string(), + severity: Severity::Info, + actor_did: Some(auth.did.clone()), + subject: Some(id.clone()), + detail: serde_json::json!({ + "script_type": new_script_type, + }), + }, + backend, + ) + .await; + + let row = fetch_one(&state, &id).await?; + Ok(Json(row)) +} + +/// `DELETE /admin/scripts/{id}` — remove a row. 204 on success, 404 if +/// no row matched. +pub(super) async fn delete( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result { + auth.require(Permission::ScriptsManage).await?; + + let backend = state.db_backend; + let sql = adapt_sql("DELETE FROM scripts WHERE id = ?", backend); + let result = sqlx::query(&sql) + .bind(&id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to delete script: {e}")))?; + if result.rows_affected() == 0 { + return Err(AppError::NotFound(format!("script '{id}' not found"))); + } + + log_event( + &state.db, + EventLog { + event_type: "script.deleted".to_string(), + severity: Severity::Info, + actor_did: Some(auth.did.clone()), + subject: Some(id), + detail: serde_json::json!({}), + }, + backend, + ) + .await; + Ok(StatusCode::NO_CONTENT) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Look up a single script row; 404 if missing. +async fn fetch_one(state: &AppState, id: &str) -> Result { + let backend = state.db_backend; + let sql = adapt_sql( + "SELECT id, script_type, body, description, created_at, updated_at + FROM scripts WHERE id = ?", + backend, + ); + #[allow(clippy::type_complexity)] + let row: Option<(String, String, String, Option, String, String)> = + sqlx::query_as(&sql) + .bind(id) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to fetch script: {e}")))?; + let (id, script_type, body, description, created_at, updated_at) = + row.ok_or_else(|| AppError::NotFound(format!("script '{id}' not found")))?; + Ok(ScriptResponse { + id, + script_type, + body, + description, + created_at, + updated_at, + }) +} + +/// Validate the script body against its declared language. Rejects +/// invalid bodies with a 400 at write-time. +fn validate_body_for_type(body: &str, lang: ScriptLanguage) -> Result<(), AppError> { + match lang { + ScriptLanguage::Lua => crate::lua::validate_script(body).map_err(AppError::BadRequest), + } +} diff --git a/src/labeler.rs b/src/labeler.rs index 52fe0ab..5e79cc0 100644 --- a/src/labeler.rs +++ b/src/labeler.rs @@ -253,7 +253,7 @@ async fn run_subscription_once( last_seq = message.seq; for label in &message.labels { - apply_label(&state.db, label, state.db_backend).await; + apply_label(state, label).await; } events_since_cursor_save += 1; @@ -323,22 +323,53 @@ fn http_to_ws(url: &str) -> String { } } -async fn apply_label(db: &sqlx::AnyPool, label: &Label, backend: DatabaseBackend) { - if label.neg { +/// Persist a label received from a subscribed upstream labeler. +/// +/// Before touching the DB we run the trigger-keyed script chain (computed +/// from `label.uri` — `labeler.apply:` for at-uri subjects, +/// `labeler.apply:_actor` for bare DIDs). The script can rewrite any field +/// of the label (including `val` or `neg`) or return nil to skip +/// persistence. Failure is fail-open: a dead-lettered script proceeds with +/// the original label, so a buggy script can't permanently break the +/// firehose. +async fn apply_label(state: &AppState, label: &Label) { + let event = crate::lua::LabelAppliedEvent { + src: label.src.clone(), + uri: label.uri.clone(), + val: label.val.clone(), + neg: label.neg, + cts: label.cts.clone(), + exp: label.exp.clone(), + }; + let final_label = match crate::lua::run_label_applied_script(state, event).await { + crate::lua::LabelHookOutcome::Continue(next) => next, + crate::lua::LabelHookOutcome::Skip => { + tracing::debug!( + src = %label.src, uri = %label.uri, val = %label.val, + "label.applied script skipped persistence" + ); + return; + } + }; + + let db = &state.db; + let backend = state.db_backend; + + if final_label.neg { // Negation label — remove it. let delete_sql = adapt_sql( "DELETE FROM labels WHERE src = ? AND uri = ? AND val = ?", backend, ); if let Err(e) = sqlx::query(&delete_sql) - .bind(&label.src) - .bind(&label.uri) - .bind(&label.val) + .bind(&final_label.src) + .bind(&final_label.uri) + .bind(&final_label.val) .execute(db) .await { tracing::warn!( - src = %label.src, uri = %label.uri, val = %label.val, + src = %final_label.src, uri = %final_label.uri, val = %final_label.val, "failed to delete negated label: {e}" ); } @@ -356,16 +387,16 @@ async fn apply_label(db: &sqlx::AnyPool, label: &Label, backend: DatabaseBackend ); if let Err(e) = sqlx::query(&insert_sql) - .bind(&label.src) - .bind(&label.uri) - .bind(&label.val) - .bind(&label.cts) - .bind(&label.exp) + .bind(&final_label.src) + .bind(&final_label.uri) + .bind(&final_label.val) + .bind(&final_label.cts) + .bind(&final_label.exp) .execute(db) .await { tracing::warn!( - src = %label.src, uri = %label.uri, val = %label.val, + src = %final_label.src, uri = %final_label.uri, val = %final_label.val, "failed to upsert label: {e}" ); } @@ -452,7 +483,7 @@ async fn backfill_from_labeler( let response: QueryLabelsResponse = resp.json().await?; for label in &response.labels { - apply_label(&state.db, label, state.db_backend).await; + apply_label(state, label).await; } Ok(()) diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 5f54131..3ed5a23 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -4,12 +4,11 @@ use mlua::LuaSerdeExt; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; -use std::sync::atomic::Ordering; use std::time::Instant; use crate::AppState; use crate::auth::Claims; -use crate::db::{DatabaseBackend, adapt_sql, now_rfc3339}; +use crate::db::{DatabaseBackend, adapt_sql}; use crate::error::{AppError, LUA_AUTH_ERROR_PREFIX, ScriptErrorType, parse_lua_line}; use crate::event_log::{EventLog, Severity, log_event}; use crate::lexicon::ParsedLexicon; @@ -33,38 +32,6 @@ async fn load_env_vars(db: &sqlx::AnyPool, backend: DatabaseBackend) -> HashMap< .collect() } -/// Load env vars, reusing a cached result if less than 30 seconds old. -/// Avoids per-record DB queries during backfill. -pub(crate) async fn load_env_vars_cached( - db: &sqlx::AnyPool, - backend: DatabaseBackend, -) -> HashMap { - use std::sync::Mutex; - - static CACHE: std::sync::OnceLock)>> = - std::sync::OnceLock::new(); - - let cache = CACHE.get_or_init(|| { - Mutex::new(( - Instant::now() - std::time::Duration::from_secs(60), - HashMap::new(), - )) - }); - { - let guard = cache.lock().unwrap(); - if guard.0.elapsed() < std::time::Duration::from_secs(30) { - return guard.1.clone(); - } - } - - let vars = load_env_vars(db, backend).await; - { - let mut guard = cache.lock().unwrap(); - *guard = (Instant::now(), vars.clone()); - } - vars -} - /// Execute a Lua script for a procedure endpoint. #[allow(clippy::too_many_arguments)] pub async fn execute_procedure_script( @@ -294,9 +261,9 @@ pub async fn execute_procedure_script( if let Err(e) = record::register_record_api( &lua, - state_arc, - claims_arc, - pds_auth_arc, + state_arc.clone(), + Some(claims_arc), + Some(pds_auth_arc), delegate_did.map(|s| s.to_string()), ) { let error_message = format!("failed to register Record API: {e}"); @@ -673,7 +640,9 @@ pub async fn execute_query_script( return Err(AppError::Internal(error_message)); } - if let Err(e) = atproto_api::register_atproto_api(&lua, state_arc, claims.map(|c| c.did())) { + if let Err(e) = + atproto_api::register_atproto_api(&lua, state_arc.clone(), claims.map(|c| c.did())) + { let error_message = format!("failed to register atproto API: {e}"); log_event( &state.db, @@ -695,6 +664,32 @@ pub async fn execute_query_script( return Err(AppError::Internal(error_message)); } + // Register the Record API in no-auth mode. Queries don't have a PDS + // auth context — the local-only methods (Record.load, :save_local, + // :delete_local, Record.delete_local) work; PDS-touching variants + // error with the no-PDS-auth message. + if let Err(e) = record::register_record_api_no_auth(&lua, state_arc) { + let error_message = format!("failed to register Record API: {e}"); + log_event( + &state.db, + EventLog { + event_type: "script.error".to_string(), + severity: Severity::Error, + actor_did: None, + subject: Some(method.to_string()), + detail: serde_json::json!({ + "error": error_message, + "script_source": script_source, + "method": method, + "duration_ms": start.elapsed().as_millis() as u64, + }), + }, + backend, + ) + .await; + return Err(AppError::Internal(error_message)); + } + if let Err(e) = context::set_query_context( &lua, method, @@ -914,449 +909,3 @@ pub async fn execute_query_script( Ok(Json(json_value).into_response()) } - -/// Context for a hook execution triggered by a record index event. -pub struct HookEvent<'a> { - pub state: &'a AppState, - pub lexicon_id: &'a str, - pub script: &'a str, - pub action: &'a str, - pub uri: &'a str, - pub did: &'a str, - pub collection: &'a str, - pub rkey: &'a str, - pub record: Option<&'a Value>, - pub cached_env_vars: Option<&'a HashMap>, -} - -/// Execute a Lua hook script triggered by a record index event. -/// -/// Runs **before** the record is indexed. The return value determines what -/// gets stored: -/// - `None` → skip the DB operation entirely -/// - `Some(value)` → use that value for the insert/update -/// -/// Retries up to 3 times with exponential backoff (1s, 2s, 4s). -/// On final failure, dead-letters the event and returns `Some(original_record)` -/// (fail-open so indexing is not permanently blocked). -pub async fn execute_hook_script(event: &HookEvent<'_>) -> Option { - let max_attempts: i32 = 4; // 1 initial + 3 retries - let mut last_error = String::new(); - let backend = event.state.db_backend; - - for attempt in 0..max_attempts { - if attempt > 0 { - let delay = std::time::Duration::from_secs(1 << (attempt - 1)); // 1s, 2s, 4s - tokio::time::sleep(delay).await; - } - - match run_hook_once(event).await { - Ok(hook_result) => { - if event.state.verbose_event_logging.load(Ordering::Relaxed) { - log_event( - &event.state.db, - EventLog { - event_type: "hook.executed".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(event.uri.to_string()), - detail: serde_json::json!({ - "lexicon_id": event.lexicon_id, - "action": event.action, - "collection": event.collection, - "attempts": attempt + 1, - }), - }, - backend, - ) - .await; - } - return hook_result; - } - Err(e) => { - last_error = e; - tracing::warn!( - uri = event.uri, - lexicon_id = event.lexicon_id, - attempt = attempt + 1, - "hook execution failed: {last_error}" - ); - } - } - } - - // All retries exhausted — dead-letter the event and fail-open with the - // original record so indexing is not permanently blocked. - tracing::error!( - uri = event.uri, - lexicon_id = event.lexicon_id, - "hook dead-lettered after {max_attempts} attempts" - ); - - let record_str = event - .record - .map(|r| serde_json::to_string(r).unwrap_or_default()); - let dead_letter_sql = adapt_sql( - r#" - INSERT INTO dead_letter_hooks (id, lexicon_id, uri, did, collection, rkey, action, record, error, attempts, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - "#, - backend, - ); - if let Err(e) = sqlx::query(&dead_letter_sql) - .bind(uuid::Uuid::new_v4().to_string()) - .bind(event.lexicon_id) - .bind(event.uri) - .bind(event.did) - .bind(event.collection) - .bind(event.rkey) - .bind(event.action) - .bind(&record_str) - .bind(&last_error) - .bind(max_attempts) - .bind(now_rfc3339()) - .execute(&event.state.db) - .await - { - tracing::error!(uri = event.uri, "failed to insert dead letter hook: {e}"); - } - - log_event( - &event.state.db, - EventLog { - event_type: "hook.dead_lettered".to_string(), - severity: Severity::Error, - actor_did: None, - subject: Some(event.uri.to_string()), - detail: serde_json::json!({ - "lexicon_id": event.lexicon_id, - "action": event.action, - "collection": event.collection, - "error": last_error, - "attempts": max_attempts, - }), - }, - backend, - ) - .await; - - // Fail-open: return the original record so indexing proceeds. - event.record.cloned() -} - -/// Execute a hook script once. -/// -/// Returns `Ok(None)` when `handle()` returns nil (meaning "skip indexing"), -/// `Ok(Some(value))` when it returns a table (use that as the record), or -/// `Ok(Some(original))` for other non-nil types. -pub async fn run_hook_once(event: &HookEvent<'_>) -> Result, String> { - let lua = sandbox::create_sandbox().map_err(|e| format!("failed to create Lua VM: {e}"))?; - let backend = event.state.db_backend; - - let state_arc = Arc::new(event.state.clone()); - - db_api::register_db_api(&lua, state_arc.clone()) - .map_err(|e| format!("failed to register db API: {e}"))?; - - http_api::register_http_api(&lua, state_arc.clone()) - .map_err(|e| format!("failed to register http API: {e}"))?; - - super::xrpc_api::register_xrpc_api(&lua, state_arc.clone(), Some(event.did.to_string())) - .map_err(|e| format!("failed to register xrpc API: {e}"))?; - - atproto_api::register_atproto_api(&lua, state_arc, None) - .map_err(|e| format!("failed to register atproto API: {e}"))?; - - context::set_hook_context( - &lua, - event.action, - event.uri, - event.did, - event.collection, - event.rkey, - event.record, - ) - .map_err(|e| format!("failed to set hook context: {e}"))?; - - let owned_env_vars; - let env_vars = if let Some(cached) = event.cached_env_vars { - cached - } else { - owned_env_vars = load_env_vars(&event.state.db, backend).await; - &owned_env_vars - }; - context::set_env_context(&lua, env_vars) - .map_err(|e| format!("failed to set env context: {e}"))?; - - lua.load(event.script) - .exec() - .map_err(|e| format!("script load failed: {e}"))?; - - let handle: mlua::Function = lua - .globals() - .get("handle") - .map_err(|e| format!("script missing handle function: {e}"))?; - - let result: mlua::Value = handle - .call_async::(()) - .await - .map_err(|e| e.to_string())?; - - match result { - mlua::Value::Nil => Ok(None), - mlua::Value::Table(_) => { - let json_value: Value = lua - .from_value(result) - .map_err(|e| format!("failed to convert lua table to JSON: {e}"))?; - Ok(Some(json_value)) - } - _ => { - // Non-nil, non-table return — proceed with the original record. - Ok(event.record.cloned()) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::config::Config; - use crate::db::DatabaseBackend; - use crate::lexicon::LexiconRegistry; - use serde_json::json; - use tokio::sync::watch; - - fn test_state() -> AppState { - let config = Config { - host: "127.0.0.1".into(), - port: 3000, - database_url: String::new(), - database_backend: crate::db::DatabaseBackend::Sqlite, - public_url: String::new(), - session_secret: "test-secret".into(), - jetstream_url: String::new(), - relay_url: String::new(), - plc_url: String::new(), - static_dir: String::new(), - base_path: None, - event_log_retention_days: 30, - app_name: None, - logo_uri: None, - tos_uri: None, - policy_uri: None, - token_encryption_key: None, - default_rate_limit_capacity: 100, - default_rate_limit_refill_rate: 2.0, - }; - let (tx, _) = watch::channel(vec![]); - let (labeler_tx, _) = watch::channel(()); - sqlx::any::install_default_drivers(); - let test_db = sqlx::AnyPool::connect_lazy("sqlite::memory:").unwrap(); - let atrium_http = std::sync::Arc::new(atrium_oauth::DefaultHttpClient::default()); - let did_resolver = atrium_identity::did::CommonDidResolver::new( - atrium_identity::did::CommonDidResolverConfig { - plc_directory_url: "https://plc.directory".into(), - http_client: std::sync::Arc::clone(&atrium_http), - }, - ); - let handle_resolver = atrium_identity::handle::AtprotoHandleResolver::new( - atrium_identity::handle::AtprotoHandleResolverConfig { - dns_txt_resolver: crate::dns::NativeDnsResolver::new(), - http_client: atrium_http, - }, - ); - let oauth = atrium_oauth::OAuthClient::new(atrium_oauth::OAuthClientConfig { - client_metadata: atrium_oauth::AtprotoLocalhostClientMetadata { - redirect_uris: Some(vec!["http://127.0.0.1:0/auth/callback".into()]), - scopes: Some(vec![atrium_oauth::Scope::Known( - atrium_oauth::KnownScope::Atproto, - )]), - }, - keys: None, - state_store: crate::auth::oauth_store::DbStateStore::new( - test_db.clone(), - crate::db::DatabaseBackend::Sqlite, - ), - session_store: crate::auth::oauth_store::DbSessionStore::new( - test_db.clone(), - crate::db::DatabaseBackend::Sqlite, - ), - resolver: atrium_oauth::OAuthResolverConfig { - did_resolver, - handle_resolver, - authorization_server_metadata: Default::default(), - protected_resource_metadata: Default::default(), - }, - }) - .expect("Failed to create test OAuth client"); - AppState { - config, - http: reqwest::Client::new(), - db: test_db.clone(), - backfill_db: test_db.clone(), - db_backend: DatabaseBackend::Sqlite, - domain_cache: crate::domain::DomainCache::new(), - lexicons: LexiconRegistry::new(), - collections_tx: tx, - labeler_subscriptions_tx: labeler_tx, - rate_limiter: crate::rate_limit::RateLimiter::new( - crate::rate_limit::RateLimitDefaults { - query_cost: 1, - procedure_cost: 1, - proxy_cost: 1, - }, - ), - oauth: std::sync::Arc::new(crate::auth::OAuthClientRegistry::new(std::sync::Arc::new( - oauth, - ))), - oauth_state_store: crate::auth::oauth_store::DbStateStore::new( - test_db.clone(), - crate::db::DatabaseBackend::Sqlite, - ), - cookie_key: axum_extra::extract::cookie::Key::derive_from( - b"test-secret-for-tests-only-not-production", - ), - plugin_registry: std::sync::Arc::new(crate::plugin::PluginRegistry::new()), - wasm_runtime: std::sync::Arc::new( - crate::plugin::WasmRuntime::new().expect("wasm runtime"), - ), - attestation_signer: None, - official_registry: std::sync::Arc::new(tokio::sync::RwLock::new( - crate::plugin::official_registry::OfficialRegistryState::default(), - )), - official_registry_config: crate::plugin::official_registry::RegistryConfig::production( - ), - proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( - crate::proxy_config::ProxyConfig::default(), - ))), - backfill_events_tx: tokio::sync::broadcast::channel(16).0, - verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), - } - } - - fn make_event<'a>( - state: &'a AppState, - script: &'a str, - action: &'a str, - record: Option<&'a Value>, - ) -> HookEvent<'a> { - HookEvent { - state, - lexicon_id: "test.lexicon", - script, - action, - uri: "at://did:plc:test/test.collection/rkey1", - did: "did:plc:test", - collection: "test.collection", - rkey: "rkey1", - record, - cached_env_vars: None, - } - } - - #[tokio::test] - async fn hook_runs_simple_script() { - let state = test_state(); - let event = make_event(&state, "function handle() end", "create", None); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - // handle() returns nil implicitly, so result should be None (skip). - assert!(result.unwrap().is_none()); - } - - #[tokio::test] - async fn hook_returns_nil_to_skip() { - let state = test_state(); - let record = json!({"name": "Test"}); - let event = make_event( - &state, - "function handle() return nil end", - "create", - Some(&record), - ); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - assert!(result.unwrap().is_none(), "nil return should produce None"); - } - - #[tokio::test] - async fn hook_returns_modified_record() { - let state = test_state(); - let record = json!({"name": "Original"}); - let script = r#" - function handle() - return { name = "Modified", extra = true } - end - "#; - let event = make_event(&state, script, "create", Some(&record)); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - let value = result.unwrap(); - assert!(value.is_some(), "table return should produce Some"); - let v = value.unwrap(); - assert_eq!(v["name"], "Modified"); - assert_eq!(v["extra"], true); - } - - #[tokio::test] - async fn hook_fails_on_missing_handle() { - let state = test_state(); - let event = make_event(&state, "function other() end", "create", None); - let result = run_hook_once(&event).await; - assert!(result.is_err()); - let err = result.unwrap_err(); - assert!(err.contains("handle"), "expected handle error, got: {err}"); - } - - #[tokio::test] - async fn hook_fails_on_syntax_error() { - let state = test_state(); - let event = make_event(&state, "function handle(", "create", None); - let result = run_hook_once(&event).await; - assert!(result.is_err()); - } - - #[tokio::test] - async fn hook_has_access_to_context_globals() { - let state = test_state(); - let script = r#" - function handle() - if action ~= "create" then error("wrong action: " .. tostring(action)) end - if uri ~= "at://did:plc:test/test.collection/rkey1" then error("wrong uri") end - if did ~= "did:plc:test" then error("wrong did") end - if collection ~= "test.collection" then error("wrong collection") end - if rkey ~= "rkey1" then error("wrong rkey") end - end - "#; - let event = make_event(&state, script, "create", None); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - } - - #[tokio::test] - async fn hook_has_access_to_record() { - let state = test_state(); - let record = json!({"name": "Test"}); - let script = r#" - function handle() - if record.name ~= "Test" then error("wrong name: " .. tostring(record.name)) end - end - "#; - let event = make_event(&state, script, "create", Some(&record)); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - } - - #[tokio::test] - async fn hook_record_nil_on_delete() { - let state = test_state(); - let script = r#" - function handle() - if record ~= nil then error("expected nil record") end - end - "#; - let event = make_event(&state, script, "delete", None); - let result = run_hook_once(&event).await; - assert!(result.is_ok(), "expected Ok, got: {:?}", result); - } -} diff --git a/src/lua/mod.rs b/src/lua/mod.rs index a535d21..7177414 100644 --- a/src/lua/mod.rs +++ b/src/lua/mod.rs @@ -3,15 +3,18 @@ mod context; pub mod db_api; mod execute; mod http_api; -mod record; +pub mod record; pub(crate) mod sandbox; +pub mod scripts; pub(crate) mod tid; mod xrpc_api; #[allow(unused_imports)] pub(crate) use context::SpaceContext; -pub(crate) use execute::{ - HookEvent, execute_hook_script, execute_procedure_script, execute_query_script, - load_env_vars_cached, run_hook_once, -}; +pub(crate) use execute::{execute_procedure_script, execute_query_script}; pub(crate) use sandbox::validate_script; +pub use scripts::{ + LabelAppliedEvent, LabelHookOutcome, ParsedTrigger, ResolvedScript, ScriptLanguage, ScriptRow, + TriggerKind, resolve, resolve_record_event, run_label_applied_script, run_record_event_once, + run_record_event_script, trigger_for_label_uri, +}; diff --git a/src/lua/record.rs b/src/lua/record.rs index 16c7e3d..71bea79 100644 --- a/src/lua/record.rs +++ b/src/lua/record.rs @@ -30,16 +30,44 @@ const INTERNAL_FIELDS: &[&str] = &[ "_repo_override", ]; +/// Error message returned when a script calls a PDS-touching method +/// (`:save()` / `:delete()` / `Record.save_all`) from a context without +/// caller credentials — e.g. label scripts, record-event scripts, or +/// query handlers. +const NO_PDS_AUTH_MSG: &str = "no PDS auth in this script context — \ + use :save_local() / :delete_local() / Record.delete_local(uri) for local-only mutation"; + +/// Register the `Record` global with only the local-only surface +/// (`Record.load`, `:save_local`, `:delete_local`, `Record.delete_local`). +/// PDS-touching methods (`:save`, `:delete`, `Record.save_all`) are still +/// exposed but error with [`NO_PDS_AUTH_MSG`] when called. +/// +/// This is the entry point for label scripts, record-event scripts, and +/// query handlers — contexts that have no caller credentials to round-trip +/// records through a PDS. +pub fn register_record_api_no_auth(lua: &Lua, state: Arc) -> LuaResult<()> { + register_record_api(lua, state, None, None, None) +} + /// Register the `Record` global constructor and static methods. -/// Only registered for procedure scripts (not queries). /// -/// When `delegate_did` is `Some`, record writes default to the delegate's repo -/// instead of the caller's DID. Scripts can still override via `record:set_repo()`. -pub fn register_record_api( +/// `claims` / `pds_auth` are optional: when both are `Some`, the full +/// surface (`:save()`, `:delete()`, `Record.save_all`) round-trips through +/// the PDS. When either is `None` (label scripts, record-event scripts, +/// query handlers), only the local-only methods are usable +/// (`:save_local()`, `:delete_local()`, `Record.delete_local(uri)`); the +/// PDS-touching methods error with [`NO_PDS_AUTH_MSG`]. Most callers want +/// [`register_record_api_no_auth`] instead — this lower-level entry point +/// exposes the internal `PdsAuth` type and is only public to the crate. +/// +/// When `delegate_did` is `Some`, record writes default to the delegate's +/// repo instead of the caller's DID. Scripts can still override via +/// `record:set_repo()`. +pub(crate) fn register_record_api( lua: &Lua, state: Arc, - claims: Arc, - pds_auth: Arc, + claims: Option>, + pds_auth: Option>, delegate_did: Option, ) -> LuaResult<()> { // -- methods table (shared by all Record instances) -- @@ -57,6 +85,8 @@ pub fn register_record_api( let pds_auth = pds_auth.clone(); let delegate_did = delegate_did.clone(); async move { + let claims = claims.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; + let pds_auth = pds_auth.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; let backend = state.db_backend; let collection: String = this.raw_get("_collection")?; let schema: mlua::Value = this.raw_get("_schema")?; @@ -232,6 +262,8 @@ pub fn register_record_api( let pds_auth = pds_auth.clone(); let delegate_did = delegate_did.clone(); async move { + let claims = claims.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; + let pds_auth = pds_auth.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; let backend = state.db_backend; let uri: String = this.raw_get::>("_uri")?.ok_or_else(|| { mlua::Error::runtime("cannot delete a Record that has no _uri") @@ -255,24 +287,39 @@ pub fn register_record_api( "rkey": rkey, }); - let resp = pds_auth + // Try the PDS delete. We log-and-continue on failure so the + // operator's intent ("remove this record") is still + // reflected in the local DB even when the PDS is down or + // refuses the call. The local row is the source of truth + // for the index. + match pds_auth .post_json(&state, repo, "com.atproto.repo.deleteRecord", &pds_body) .await - .map_err(|e| pds_error("deleteRecord", e))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(mlua::Error::runtime(format!( - "PDS deleteRecord returned {status}: {body}" - ))); + { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + tracing::warn!( + uri = %uri, + "PDS deleteRecord returned {status}: {body} \ + — proceeding with local delete anyway" + ); + } + Err(e) => { + tracing::warn!( + uri = %uri, + "PDS deleteRecord failed: {e} \ + — proceeding with local delete anyway" + ); + } } - // Delete from local DB + // Always delete locally — operator's logical action is + // "remove this record from view" regardless of PDS outcome. let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); let _ = sqlx::query(&delete_sql).bind(&uri).execute(&state.db).await; - // Clear _uri and _cid this.raw_set("_uri", mlua::Value::Nil)?; this.raw_set("_cid", mlua::Value::Nil)?; @@ -282,6 +329,156 @@ pub fn register_record_api( methods.set("delete", delete_fn)?; } + // :save_local() — upsert into local DB only, never touches a PDS. + // Works in any script context (no auth required). + // + // For records loaded via `Record.load(uri)` or saved via `:save()` + // (i.e. those with an `_uri`), the repo+rkey are parsed back out of + // the URI. For brand-new records (no `_uri`), we fall back to + // `_repo_override` / `claims.did()` for the repo, and generate an + // rkey via `_key_type` if `_rkey` isn't set. Errors clearly when no + // DID can be determined. + { + let state = state.clone(); + let claims = claims.clone(); + let save_local_fn = lua.create_async_function(move |lua, this: mlua::Table| { + let state = state.clone(); + let claims = claims.clone(); + async move { + let backend = state.db_backend; + let collection: String = this.raw_get("_collection")?; + let schema: mlua::Value = this.raw_get("_schema")?; + + if let mlua::Value::Table(ref schema_table) = schema { + validate_required_fields(&this, schema_table)?; + } + + let data = extract_record_data(&lua, &this, &collection)?; + let data_str = serde_json::to_string(&data).unwrap_or_default(); + let now = now_rfc3339(); + + let existing_uri: Option = this.raw_get("_uri")?; + let (uri, repo, rkey) = if let Some(uri) = existing_uri { + // Parse repo (DID) and rkey out of the URI: + // at://// + let trimmed = uri + .strip_prefix("at://") + .ok_or_else(|| mlua::Error::runtime(format!("invalid AT URI: {uri}")))?; + let mut parts = trimmed.splitn(3, '/'); + let repo = parts + .next() + .ok_or_else(|| mlua::Error::runtime(format!("invalid AT URI: {uri}")))? + .to_string(); + let _col = parts.next(); + let rkey = parts + .next() + .ok_or_else(|| mlua::Error::runtime(format!("invalid AT URI: {uri}")))? + .to_string(); + (uri, repo, rkey) + } else { + // CREATE path — no URI yet. Compute repo + rkey, build URI. + let repo_override: Option = this.raw_get("_repo_override")?; + let repo = repo_override + .or_else(|| claims.as_ref().map(|c| c.did().to_string())) + .ok_or_else(|| { + mlua::Error::runtime( + "save_local() needs a DID — call :set_repo(\"did:plc:...\") \ + or load the record first", + ) + })?; + + let rkey: Option = this.raw_get("_rkey")?; + let rkey = if let Some(rk) = rkey { + rk + } else { + let key_type: Option = this.raw_get("_key_type")?; + match key_type.as_deref() { + Some("tid") | Some("any") | None => generate_tid(), + Some(s) if s.starts_with("literal:") => { + s["literal:".len()..].to_string() + } + Some("nsid") => { + return Err(mlua::Error::runtime( + "cannot auto-generate rkey for nsid key type — \ + call set_rkey() first", + )); + } + Some(other) => { + return Err(mlua::Error::runtime(format!( + "unknown key type '{other}'" + ))); + } + } + }; + let uri = format!("at://{repo}/{collection}/{rkey}"); + (uri, repo, rkey) + }; + + // Upsert. Sentinel CID `""` — no PDS round-trip means we + // have no real CID to record; consumers reading the row + // should treat empty CID as "local-only write". + let upsert_sql = adapt_sql( + r#"INSERT INTO records (uri, did, collection, rkey, record, cid, indexed_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (uri) DO UPDATE + SET record = EXCLUDED.record, + cid = EXCLUDED.cid, + indexed_at = ?"#, + backend, + ); + sqlx::query(&upsert_sql) + .bind(&uri) + .bind(&repo) + .bind(&collection) + .bind(&rkey) + .bind(&data_str) + .bind("") + .bind(&now) + .bind(&now) + .bind(&now) + .execute(&state.db) + .await + .map_err(|e| mlua::Error::runtime(format!("save_local upsert failed: {e}")))?; + + let _ = sync_refs(&state.db, &uri, &collection, &data, backend).await; + + this.raw_set("_uri", uri.as_str())?; + this.raw_set("_cid", "")?; + + Ok(this) + } + })?; + methods.set("save_local", save_local_fn)?; + } + + // :delete_local() — local DB delete only, never touches a PDS. + // Idempotent: succeeds whether or not a row existed at the URI. + { + let state = state.clone(); + let delete_local_fn = lua.create_async_function(move |_lua, this: mlua::Table| { + let state = state.clone(); + async move { + let backend = state.db_backend; + let uri: String = this.raw_get::>("_uri")?.ok_or_else(|| { + mlua::Error::runtime("cannot delete_local a Record that has no _uri") + })?; + + let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); + sqlx::query(&delete_sql) + .bind(&uri) + .execute(&state.db) + .await + .map_err(|e| mlua::Error::runtime(format!("delete_local failed: {e}")))?; + + this.raw_set("_uri", mlua::Value::Nil)?; + this.raw_set("_cid", mlua::Value::Nil)?; + + Ok(this) + } + })?; + methods.set("delete_local", delete_local_fn)?; + } + // :set_key_type(type) { let set_key_type_fn = @@ -470,6 +667,8 @@ pub fn register_record_api( let pds_auth = pds_auth.clone(); let delegate_did = delegate_did.clone(); async move { + let claims = claims.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; + let pds_auth = pds_auth.ok_or_else(|| mlua::Error::runtime(NO_PDS_AUTH_MSG))?; let backend = state.db_backend; // Extract save data from each record (sync) type SaveItem = (mlua::Table, String, Option, Option, Option, Value); @@ -729,8 +928,8 @@ pub fn register_record_api( // Record.load_all(uris) { - let state = state; - let metatable_c = metatable; + let state = state.clone(); + let metatable_c = metatable.clone(); let load_all_fn = lua.create_async_function(move |lua, uris_table: mlua::Table| { let state = state.clone(); let metatable = metatable_c.clone(); @@ -811,6 +1010,31 @@ pub fn register_record_api( record_table.set("load_all", load_all_fn)?; } + // Record.delete_local(uri) — fire-and-forget local-only delete by URI. + // The common one-liner for label-script reactions like: + // if event.val == "spam" then Record.delete_local(event.uri) end + // Returns true if a row was deleted, false if no row matched. + // Always succeeds (no error) regardless of whether the row existed. + { + let state = state; + let delete_local_static_fn = lua.create_async_function(move |_lua, uri: String| { + let state = state.clone(); + async move { + let backend = state.db_backend; + let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); + let res = sqlx::query(&delete_sql) + .bind(&uri) + .execute(&state.db) + .await + .map_err(|e| { + mlua::Error::runtime(format!("Record.delete_local failed: {e}")) + })?; + Ok(res.rows_affected() > 0) + } + })?; + record_table.set("delete_local", delete_local_static_fn)?; + } + // -- Make Record callable via __call metamethod -- let record_mt = lua.create_table()?; { diff --git a/src/lua/scripts.rs b/src/lua/scripts.rs new file mode 100644 index 0000000..43970dc --- /dev/null +++ b/src/lua/scripts.rs @@ -0,0 +1,855 @@ +//! Trigger-keyed scripts dispatcher. +//! +//! Each row in the `scripts` table is identified by a TRIGGER STRING — the +//! `id` column IS the binding. There's no separate "name" or "host column." +//! +//! Trigger grammar: +//! +//! - `record.index:` — fires for any record event on `` +//! (wildcard fallback). +//! - `record.create:` / `record.update:` / +//! `record.delete:` — fires only for that specific action. +//! - `xrpc.query:` / `xrpc.procedure:` — fires when the +//! matching XRPC method is invoked. +//! - `labeler.apply:` — fires when a label arrives whose `uri` +//! is `at:////`. +//! - `labeler.apply:_actor` — fires when a label arrives whose `uri` +//! is a bare DID (actor-level label). +//! +//! **Cascade for record events ONLY**: the dispatcher tries +//! `record.:` first, falls back to `record.index:` +//! if no specific row exists. No cascade for XRPC or labeler triggers. +//! +//! Fail mode varies by host: +//! +//! - **Record / label events**: fail-OPEN — a buggy script eats its +//! retry budget then dead-letters; the upstream operation proceeds +//! with whatever the dispatcher returns (original record / original +//! label). The firehose has no caller to surface errors to. +//! - **XRPC procedures / queries**: fail-CLOSED, single-shot — a script +//! error becomes a 5xx response. The XRPC dispatchers in +//! [`crate::xrpc`] resolve the script via [`resolve`] and call +//! [`super::execute::execute_procedure_script`] / +//! [`super::execute::execute_query_script`] directly. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::sync::Arc; + +use crate::AppState; +use crate::db::{DatabaseBackend, adapt_sql, now_rfc3339}; +use crate::event_log::{EventLog, Severity, log_event}; + +use super::{atproto_api, context, db_api, http_api, record, sandbox, xrpc_api}; + +/// Number of attempts (1 initial + 3 retries) before dead-lettering. +const MAX_ATTEMPTS: u32 = 4; + +// --------------------------------------------------------------------------- +// Trigger grammar +// --------------------------------------------------------------------------- + +/// Which family a trigger belongs to. Determines auth context, fail mode, +/// and which event payload shape the script expects. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TriggerKind { + RecordIndex, + RecordCreate, + RecordUpdate, + RecordDelete, + XrpcQuery, + XrpcProcedure, + LabelerApply, +} + +/// A trigger id parsed into `(kind, suffix)`. The suffix is either an NSID +/// (`record.*`, `xrpc.*`, `labeler.apply:`) or the literal `"_actor"` +/// for `labeler.apply:_actor`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParsedTrigger { + pub kind: TriggerKind, + pub suffix: String, +} + +impl ParsedTrigger { + /// Reconstruct the canonical trigger id from `(kind, suffix)`. + pub fn id(&self) -> String { + match self.kind { + TriggerKind::RecordIndex => format!("record.index:{}", self.suffix), + TriggerKind::RecordCreate => format!("record.create:{}", self.suffix), + TriggerKind::RecordUpdate => format!("record.update:{}", self.suffix), + TriggerKind::RecordDelete => format!("record.delete:{}", self.suffix), + TriggerKind::XrpcQuery => format!("xrpc.query:{}", self.suffix), + TriggerKind::XrpcProcedure => format!("xrpc.procedure:{}", self.suffix), + TriggerKind::LabelerApply => format!("labeler.apply:{}", self.suffix), + } + } + + /// Parse a trigger id. Returns a structured error message naming the + /// valid prefixes when the input doesn't match the grammar. + pub fn parse(id: &str) -> Result { + let (prefix, suffix) = id.split_once(':').ok_or_else(|| { + format!( + "trigger id '{id}' must contain a ':' separator; \ + valid prefixes: record.{{index,create,update,delete}}:, \ + xrpc.{{query,procedure}}:, labeler.apply:" + ) + })?; + + if suffix.is_empty() { + return Err(format!("trigger id '{id}' has empty suffix")); + } + + let kind = match prefix { + "record.index" => TriggerKind::RecordIndex, + "record.create" => TriggerKind::RecordCreate, + "record.update" => TriggerKind::RecordUpdate, + "record.delete" => TriggerKind::RecordDelete, + "xrpc.query" => TriggerKind::XrpcQuery, + "xrpc.procedure" => TriggerKind::XrpcProcedure, + "labeler.apply" => TriggerKind::LabelerApply, + other => { + return Err(format!( + "unknown trigger prefix '{other}'; valid prefixes: \ + record.{{index,create,update,delete}}, xrpc.{{query,procedure}}, \ + labeler.apply" + )); + } + }; + + // Suffix validation: NSID for everything except `labeler.apply:_actor`. + match (kind, suffix) { + (TriggerKind::LabelerApply, "_actor") => {} + _ => validate_nsid(suffix)?, + } + + Ok(Self { + kind, + suffix: suffix.to_string(), + }) + } +} + +/// Minimal NSID validation: at least two dot-separated segments, each +/// non-empty and matching `[a-zA-Z][a-zA-Z0-9-]*`. Mirrors the AT Protocol +/// spec's character class for everyday use; full Unicode strictness lives +/// in atrium downstream. +fn validate_nsid(nsid: &str) -> Result<(), String> { + let segments: Vec<&str> = nsid.split('.').collect(); + if segments.len() < 2 { + return Err(format!( + "invalid NSID '{nsid}': need at least 2 dot-separated segments" + )); + } + for (idx, seg) in segments.iter().enumerate() { + if seg.is_empty() { + return Err(format!("invalid NSID '{nsid}': empty segment")); + } + let mut chars = seg.chars(); + let first = chars.next().unwrap(); + if !first.is_ascii_alphabetic() { + return Err(format!( + "invalid NSID '{nsid}': segment {idx} must start with a letter" + )); + } + for c in chars { + if !c.is_ascii_alphanumeric() && c != '-' { + return Err(format!( + "invalid NSID '{nsid}': segment {idx} contains invalid character '{c}'" + )); + } + } + } + Ok(()) +} + +// --------------------------------------------------------------------------- +// ScriptLanguage +// --------------------------------------------------------------------------- + +/// Runtime a script is written for. Today only [`ScriptLanguage::Lua`] ships; +/// the column is stamped per row so a future runtime (e.g. TypeScript) can +/// land without a schema migration. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ScriptLanguage { + #[default] + Lua, +} + +impl ScriptLanguage { + pub fn as_str(&self) -> &'static str { + match self { + Self::Lua => "lua", + } + } + + pub fn parse_str(s: &str) -> Option { + match s { + "lua" => Some(Self::Lua), + _ => None, + } + } + + pub fn supported() -> &'static [&'static str] { + &["lua"] + } +} + +// --------------------------------------------------------------------------- +// Script row + resolution +// --------------------------------------------------------------------------- + +/// A row from the `scripts` table — the wire shape the admin API returns. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ScriptRow { + pub id: String, + pub body: String, + pub description: Option, + pub script_type: String, + pub created_at: String, + pub updated_at: String, +} + +/// A script ready to execute. +#[derive(Clone, Debug)] +pub struct ResolvedScript { + pub id: String, + pub language: ScriptLanguage, + pub body: String, +} + +/// Look up a single trigger id. Returns `None` when no row matches OR when +/// the row's `script_type` is unknown to this binary (logged at warn). +pub async fn resolve(state: &AppState, trigger_id: &str) -> Option { + let sql = adapt_sql( + "SELECT id, body, script_type FROM scripts WHERE id = ?", + state.db_backend, + ); + let row: Option<(String, String, String)> = match sqlx::query_as(&sql) + .bind(trigger_id) + .fetch_optional(&state.db) + .await + { + Ok(r) => r, + Err(e) => { + tracing::warn!(trigger_id, "scripts lookup failed: {e}"); + return None; + } + }; + let (id, body, script_type) = row?; + let language = match ScriptLanguage::parse_str(&script_type) { + Some(l) => l, + None => { + tracing::warn!( + id, + script_type, + "unknown script_type; this binary supports: {}", + ScriptLanguage::supported().join(", ") + ); + return None; + } + }; + Some(ResolvedScript { id, language, body }) +} + +/// Resolve a record-event trigger with the cascade rule: +/// `record.:` first, then `record.index:`. +pub async fn resolve_record_event( + state: &AppState, + nsid: &str, + action: &str, +) -> Option { + let action_trigger = match action { + "create" => Some(format!("record.create:{nsid}")), + "update" => Some(format!("record.update:{nsid}")), + "delete" => Some(format!("record.delete:{nsid}")), + _ => None, + }; + if let Some(t) = action_trigger + && let Some(s) = resolve(state, &t).await + { + return Some(s); + } + resolve(state, &format!("record.index:{nsid}")).await +} + +// --------------------------------------------------------------------------- +// Record-event runner (fail-open, retry + dead-letter) +// --------------------------------------------------------------------------- + +/// Run the record-event script (if any) for a given event. Returns the +/// record body the indexer should store: `Some(record)` to proceed, +/// `None` to skip indexing. +/// +/// Failure mode is fail-open: a script that exhausts its retry budget is +/// dead-lettered and the indexer proceeds with the original record. +#[allow(clippy::too_many_arguments)] +pub async fn run_record_event_script( + state: &AppState, + nsid: &str, + action: &str, + uri: &str, + did: &str, + rkey: &str, + record: Option<&Value>, +) -> Option { + let resolved = match resolve_record_event(state, nsid, action).await { + Some(s) => s, + // No script for this trigger → indexer keeps the original record. + None => return record.cloned(), + }; + + let host_id = format!("{nsid}:{action}"); + let payload = serde_json::json!({ + "trigger": resolved.id, + "action": action, + "uri": uri, + "did": did, + "collection": nsid, + "rkey": rkey, + "record": record, + }); + + let mut last_error = String::new(); + for attempt in 0..MAX_ATTEMPTS { + if attempt > 0 { + let delay = std::time::Duration::from_secs(1 << (attempt - 1)); + tokio::time::sleep(delay).await; + } + match run_record_event_once(state, &resolved, action, uri, did, nsid, rkey, record).await { + Ok(outcome) => { + log_event( + &state.db, + EventLog { + event_type: "script.executed".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.to_string()), + detail: serde_json::json!({ + "host_kind": "record", + "host_id": host_id, + "trigger": resolved.id, + "attempts": attempt + 1, + }), + }, + state.db_backend, + ) + .await; + return outcome; + } + Err(e) => { + last_error = e; + tracing::warn!( + %uri, + trigger = %resolved.id, + attempt = attempt + 1, + "record script attempt failed: {last_error}" + ); + } + } + } + + write_dead_letter( + state, + &resolved, + "record", + &host_id, + &payload, + &last_error, + MAX_ATTEMPTS, + ) + .await; + log_event( + &state.db, + EventLog { + event_type: "script.dead_lettered".to_string(), + severity: Severity::Error, + actor_did: None, + subject: Some(uri.to_string()), + detail: serde_json::json!({ + "host_kind": "record", + "host_id": host_id, + "trigger": resolved.id, + "error": last_error, + }), + }, + state.db_backend, + ) + .await; + + // Fail-open: indexer proceeds with the original record. + record.cloned() +} + +/// Single attempt at the record-event Lua script. Used internally by the +/// retry loop and externally by admin retry endpoints. +/// +/// Returns `Ok(Some(value))` to continue indexing with `value`, +/// `Ok(None)` when the script returned `nil` (skip), or `Err(msg)` on +/// any execution failure. +#[allow(clippy::too_many_arguments)] +pub async fn run_record_event_once( + state: &AppState, + script: &ResolvedScript, + action: &str, + uri: &str, + did: &str, + collection: &str, + rkey: &str, + record: Option<&Value>, +) -> Result, String> { + if script.language != ScriptLanguage::Lua { + return Err(format!( + "this binary cannot run {} scripts", + script.language.as_str() + )); + } + let lua = sandbox::create_sandbox().map_err(|e| format!("create sandbox: {e}"))?; + let state_arc = Arc::new(state.clone()); + register_default_apis(&lua, &state_arc, Some(did))?; + + // Legacy globals (action, uri, did, collection, rkey, record) for + // back-compat with scripts written against the old `index_hook` + // surface. + context::set_hook_context(&lua, action, uri, did, collection, rkey, record) + .map_err(|e| format!("set hook context: {e}"))?; + + // Also expose an `event` table — same fields, different idiom. New + // scripts can read `event.action` / `event.record.title` instead of + // the bare globals; both styles work. + use mlua::LuaSerdeExt; + let event_value = serde_json::json!({ + "action": action, + "uri": uri, + "did": did, + "collection": collection, + "rkey": rkey, + "record": record, + }); + lua.globals() + .set( + "event", + lua.to_value(&event_value) + .map_err(|e| format!("event lua-conv: {e}"))?, + ) + .map_err(|e| format!("set event global: {e}"))?; + + context::set_env_context(&lua, &load_env_vars(&state.db, state.db_backend).await) + .map_err(|e| format!("set env: {e}"))?; + + lua.load(script.body.as_str()) + .exec() + .map_err(|e| format!("script load: {e}"))?; + let handle: mlua::Function = lua + .globals() + .get("handle") + .map_err(|e| format!("missing handle(): {e}"))?; + let result: mlua::Value = handle + .call_async::(()) + .await + .map_err(|e| e.to_string())?; + + match result { + mlua::Value::Nil => Ok(None), + mlua::Value::Table(_) => { + let v: Value = lua + .from_value(result) + .map_err(|e| format!("convert lua return to JSON: {e}"))?; + Ok(Some(v)) + } + // Non-nil, non-table return — pass-through: keep the original record. + _ => Ok(record.cloned()), + } +} + +// --------------------------------------------------------------------------- +// Label-applied dispatcher (fail-open, retry + dead-letter) +// --------------------------------------------------------------------------- + +/// Payload passed to `labeler.apply:*` scripts. Mirrors the AT Proto label +/// shape from `com.atproto.label.subscribeLabels`. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LabelAppliedEvent { + pub src: String, + pub uri: String, + pub val: String, + #[serde(default)] + pub neg: bool, + pub cts: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exp: Option, +} + +/// What an `on_label_applied` script chain decided. +#[derive(Debug)] +pub enum LabelHookOutcome { + /// Persist the (possibly rewritten) label. + Continue(LabelAppliedEvent), + /// Skip persistence — the script returned `nil`. + Skip, +} + +/// Compute the trigger string for a given label. `at://` URIs route to +/// `labeler.apply:` (using the second path segment); everything else +/// (bare DIDs, malformed) routes to `labeler.apply:_actor`. +pub fn trigger_for_label_uri(uri: &str) -> String { + if let Some(rest) = uri.strip_prefix("at://") { + match rest.split('/').nth(1) { + Some(nsid) if !nsid.is_empty() => format!("labeler.apply:{nsid}"), + _ => "labeler.apply:_actor".to_string(), + } + } else { + "labeler.apply:_actor".to_string() + } +} + +/// Run the label-applied script (if any) for an inbound label. Fail-open: +/// dead-lettered failures fall through with the original label. +pub async fn run_label_applied_script( + state: &AppState, + event: LabelAppliedEvent, +) -> LabelHookOutcome { + let trigger = trigger_for_label_uri(&event.uri); + let resolved = match resolve(state, &trigger).await { + Some(s) => s, + None => return LabelHookOutcome::Continue(event), + }; + + let payload = serde_json::to_value(&event).unwrap_or(Value::Null); + let host_id = event.src.clone(); + let original = event.clone(); + + let mut last_error = String::new(); + for attempt in 0..MAX_ATTEMPTS { + if attempt > 0 { + let delay = std::time::Duration::from_secs(1 << (attempt - 1)); + tokio::time::sleep(delay).await; + } + match run_label_lua_once(state, &resolved, &event).await { + Ok(outcome) => return outcome, + Err(e) => { + last_error = e; + tracing::warn!( + src = %event.src, uri = %event.uri, + trigger = %resolved.id, + attempt = attempt + 1, + "label script attempt failed: {last_error}" + ); + } + } + } + write_dead_letter( + state, + &resolved, + "label", + &host_id, + &payload, + &last_error, + MAX_ATTEMPTS, + ) + .await; + LabelHookOutcome::Continue(original) +} + +async fn run_label_lua_once( + state: &AppState, + script: &ResolvedScript, + event: &LabelAppliedEvent, +) -> Result { + if script.language != ScriptLanguage::Lua { + return Err(format!( + "this binary cannot run {} scripts", + script.language.as_str() + )); + } + let lua = sandbox::create_sandbox().map_err(|e| format!("create sandbox: {e}"))?; + let state_arc = Arc::new(state.clone()); + register_default_apis(&lua, &state_arc, None)?; + + use mlua::LuaSerdeExt; + let globals = lua.globals(); + globals + .set("src", event.src.clone()) + .map_err(|e| format!("set src: {e}"))?; + globals + .set("uri", event.uri.clone()) + .map_err(|e| format!("set uri: {e}"))?; + globals + .set("val", event.val.clone()) + .map_err(|e| format!("set val: {e}"))?; + globals + .set("neg", event.neg) + .map_err(|e| format!("set neg: {e}"))?; + globals + .set("cts", event.cts.clone()) + .map_err(|e| format!("set cts: {e}"))?; + match &event.exp { + Some(exp) => globals.set("exp", exp.clone()), + None => globals.set("exp", mlua::Value::Nil), + } + .map_err(|e| format!("set exp: {e}"))?; + let event_value = serde_json::to_value(event).map_err(|e| format!("encode event: {e}"))?; + globals + .set( + "event", + lua.to_value(&event_value) + .map_err(|e| format!("event lua-conv: {e}"))?, + ) + .map_err(|e| format!("set event: {e}"))?; + context::set_env_context(&lua, &load_env_vars(&state.db, state.db_backend).await) + .map_err(|e| format!("set env: {e}"))?; + + lua.load(script.body.as_str()) + .exec() + .map_err(|e| format!("script load: {e}"))?; + let handle: mlua::Function = lua + .globals() + .get("handle") + .map_err(|e| format!("missing handle(): {e}"))?; + let result: mlua::Value = handle + .call_async::(()) + .await + .map_err(|e| e.to_string())?; + + match result { + mlua::Value::Nil => Ok(LabelHookOutcome::Skip), + mlua::Value::Table(_) => { + let v: Value = lua + .from_value(result) + .map_err(|e| format!("convert lua return: {e}"))?; + // Merge: any field the script omitted falls back to the + // original. This makes "filter only" scripts (return `event`) + // and "rewrite val" scripts (return `{ val = "..." }`) both + // ergonomic. + let next = LabelAppliedEvent { + src: extract_string(&v, "src").unwrap_or_else(|| event.src.clone()), + uri: extract_string(&v, "uri").unwrap_or_else(|| event.uri.clone()), + val: extract_string(&v, "val").unwrap_or_else(|| event.val.clone()), + neg: extract_bool(&v, "neg").unwrap_or(event.neg), + cts: extract_string(&v, "cts").unwrap_or_else(|| event.cts.clone()), + exp: extract_string(&v, "exp").or_else(|| event.exp.clone()), + }; + Ok(LabelHookOutcome::Continue(next)) + } + _ => Ok(LabelHookOutcome::Continue(event.clone())), + } +} + +fn extract_string(v: &Value, key: &str) -> Option { + v.get(key).and_then(|x| x.as_str()).map(String::from) +} + +fn extract_bool(v: &Value, key: &str) -> Option { + v.get(key).and_then(|x| x.as_bool()) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/// Register the default API surface on a fresh sandbox: db / http / xrpc / +/// atproto / Record. `caller_did` flows into xrpc so authenticated calls +/// work; pass `None` for unauthenticated contexts. +/// +/// The Record API is registered in **no-auth mode** here — fine for +/// record-event and label scripts which have no caller credentials. +/// Calling `:save()` / `:delete()` (the PDS-touching variants) errors +/// clearly with the no-PDS-auth message; the local-only variants +/// (`:save_local`, `:delete_local`, `Record.delete_local`) work. +fn register_default_apis( + lua: &mlua::Lua, + state: &Arc, + caller_did: Option<&str>, +) -> Result<(), String> { + db_api::register_db_api(lua, state.clone()).map_err(|e| format!("db api: {e}"))?; + http_api::register_http_api(lua, state.clone()).map_err(|e| format!("http api: {e}"))?; + xrpc_api::register_xrpc_api(lua, state.clone(), caller_did.map(String::from)) + .map_err(|e| format!("xrpc api: {e}"))?; + atproto_api::register_atproto_api(lua, state.clone(), None) + .map_err(|e| format!("atproto api: {e}"))?; + record::register_record_api_no_auth(lua, state.clone()) + .map_err(|e| format!("record api: {e}"))?; + Ok(()) +} + +/// Load `script_variables` as a flat key→value map for the `env` global. +async fn load_env_vars( + db: &sqlx::AnyPool, + backend: DatabaseBackend, +) -> std::collections::HashMap { + let sql = adapt_sql("SELECT key, value FROM script_variables", backend); + sqlx::query_as::<_, (String, String)>(&sql) + .fetch_all(db) + .await + .unwrap_or_default() + .into_iter() + .collect() +} + +/// Persist a permanently-failed run for later admin triage. +async fn write_dead_letter( + state: &AppState, + script: &ResolvedScript, + host_kind: &str, + host_id: &str, + payload: &Value, + error: &str, + attempts: u32, +) { + let payload_str = serde_json::to_string(payload).unwrap_or_else(|_| "{}".to_string()); + let sql = adapt_sql( + "INSERT INTO dead_letter_scripts + (script_ref, host_kind, host_id, payload, error, attempts, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)", + state.db_backend, + ); + if let Err(e) = sqlx::query(&sql) + .bind(script.id.as_str()) + .bind(host_kind) + .bind(host_id) + .bind(&payload_str) + .bind(error) + .bind(attempts as i64) + .bind(now_rfc3339()) + .execute(&state.db) + .await + { + tracing::error!( + host_kind, + host_id, + trigger = %script.id, + "failed to write dead_letter_scripts: {e}" + ); + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_record_index_trigger() { + let t = ParsedTrigger::parse("record.index:com.example.thing").unwrap(); + assert_eq!(t.kind, TriggerKind::RecordIndex); + assert_eq!(t.suffix, "com.example.thing"); + assert_eq!(t.id(), "record.index:com.example.thing"); + } + + #[test] + fn parse_record_action_triggers() { + for (prefix, kind) in [ + ("record.create", TriggerKind::RecordCreate), + ("record.update", TriggerKind::RecordUpdate), + ("record.delete", TriggerKind::RecordDelete), + ] { + let id = format!("{prefix}:com.example.thing"); + let t = ParsedTrigger::parse(&id).unwrap(); + assert_eq!(t.kind, kind); + assert_eq!(t.id(), id); + } + } + + #[test] + fn parse_xrpc_triggers() { + let q = ParsedTrigger::parse("xrpc.query:com.example.list").unwrap(); + assert_eq!(q.kind, TriggerKind::XrpcQuery); + let p = ParsedTrigger::parse("xrpc.procedure:com.example.create").unwrap(); + assert_eq!(p.kind, TriggerKind::XrpcProcedure); + } + + #[test] + fn parse_labeler_apply_with_nsid() { + let t = ParsedTrigger::parse("labeler.apply:app.bsky.feed.post").unwrap(); + assert_eq!(t.kind, TriggerKind::LabelerApply); + assert_eq!(t.suffix, "app.bsky.feed.post"); + } + + #[test] + fn parse_labeler_apply_actor_special_case() { + let t = ParsedTrigger::parse("labeler.apply:_actor").unwrap(); + assert_eq!(t.kind, TriggerKind::LabelerApply); + assert_eq!(t.suffix, "_actor"); + } + + #[test] + fn rejects_no_colon() { + let err = ParsedTrigger::parse("record.index").unwrap_err(); + assert!(err.contains("must contain a ':' separator")); + assert!(err.contains("valid prefixes")); + } + + #[test] + fn rejects_unknown_prefix() { + let err = ParsedTrigger::parse("garbage:com.example.thing").unwrap_err(); + assert!(err.contains("unknown trigger prefix 'garbage'")); + } + + #[test] + fn rejects_bad_nsid() { + // single segment + assert!(ParsedTrigger::parse("record.index:foo").is_err()); + // empty suffix + assert!(ParsedTrigger::parse("record.index:").is_err()); + // non-letter start + assert!(ParsedTrigger::parse("record.index:1.foo").is_err()); + // invalid char + assert!(ParsedTrigger::parse("record.index:com.foo!bar").is_err()); + } + + #[test] + fn allows_only_actor_special_case_for_labeler() { + // _actor is not a valid NSID, but it's the literal special case. + assert!(ParsedTrigger::parse("labeler.apply:_actor").is_ok()); + // Other prefixes don't get the _actor escape hatch. + assert!(ParsedTrigger::parse("record.index:_actor").is_err()); + } + + #[test] + fn label_uri_routes_at_uri_to_nsid() { + assert_eq!( + trigger_for_label_uri("at://did:plc:abc/app.bsky.feed.post/rkey1"), + "labeler.apply:app.bsky.feed.post" + ); + } + + #[test] + fn label_uri_routes_bare_did_to_actor() { + assert_eq!(trigger_for_label_uri("did:plc:abc"), "labeler.apply:_actor"); + } + + #[test] + fn label_uri_routes_malformed_at_uri_to_actor() { + // `at://` with no path → no second segment → actor. + assert_eq!( + trigger_for_label_uri("at://did:plc:abc"), + "labeler.apply:_actor" + ); + // `at:///` → second segment exists but is empty → actor. + assert_eq!( + trigger_for_label_uri("at://did:plc:abc/"), + "labeler.apply:_actor" + ); + } + + #[test] + fn script_language_round_trip() { + assert_eq!(ScriptLanguage::Lua.as_str(), "lua"); + assert_eq!(ScriptLanguage::parse_str("lua"), Some(ScriptLanguage::Lua)); + assert_eq!(ScriptLanguage::parse_str("typescript"), None); + assert_eq!(ScriptLanguage::default(), ScriptLanguage::Lua); + } + + #[test] + fn extract_helpers() { + let v = serde_json::json!({"a": "x", "b": true, "c": null}); + assert_eq!(extract_string(&v, "a"), Some("x".into())); + assert_eq!(extract_string(&v, "missing"), None); + assert_eq!(extract_bool(&v, "b"), Some(true)); + assert_eq!(extract_bool(&v, "a"), None); + } +} diff --git a/src/lua/xrpc_api.rs b/src/lua/xrpc_api.rs index 6a5b930..11a8f23 100644 --- a/src/lua/xrpc_api.rs +++ b/src/lua/xrpc_api.rs @@ -215,7 +215,13 @@ mod tests { let (tx, _) = watch::channel(vec![]); let (labeler_tx, _) = watch::channel(()); sqlx::any::install_default_drivers(); - let test_db = sqlx::AnyPool::connect_lazy("sqlite::memory:").unwrap(); + // Single-connection pool so the in-memory DB is shared across the + // pool's queries (separate connections to `sqlite::memory:` get + // independent DBs otherwise). + let test_db = sqlx::pool::PoolOptions::::new() + .max_connections(1) + .connect_lazy("sqlite::memory:") + .unwrap(); let atrium_http = std::sync::Arc::new(atrium_oauth::DefaultHttpClient::default()); let did_resolver = atrium_identity::did::CommonDidResolver::new( atrium_identity::did::CommonDidResolverConfig { @@ -298,6 +304,38 @@ mod tests { } } + /// Create the `scripts` table on the in-memory test DB and insert one + /// row keyed by trigger id. The trigger-keyed dispatcher reads from + /// here at firing time; `make_*_lexicon`'s `script` field is now + /// inert (kept on the struct for forward-compat with row loaders but + /// not consulted by dispatch). + async fn seed_script(state: &AppState, trigger: &str, body: &str) { + sqlx::query( + r#" + CREATE TABLE IF NOT EXISTS scripts ( + id TEXT PRIMARY KEY, + body TEXT NOT NULL, + description TEXT, + script_type TEXT NOT NULL DEFAULT 'lua', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ) + "#, + ) + .execute(&state.db) + .await + .unwrap(); + sqlx::query( + "INSERT OR REPLACE INTO scripts (id, body, script_type, created_at, updated_at) + VALUES (?, ?, 'lua', datetime('now'), datetime('now'))", + ) + .bind(trigger) + .bind(body) + .execute(&state.db) + .await + .unwrap(); + } + fn make_query_lexicon(id: &str, script: Option<&str>) -> ParsedLexicon { ParsedLexicon { id: id.to_string(), @@ -425,11 +463,14 @@ mod tests { let state = test_state(); // Register a scripted query that returns a static response - let lexicon = make_query_lexicon( - "test.echo", - Some(r#"function handle() return { greeting = "hello" } end"#), - ); + let lexicon = make_query_lexicon("test.echo", None); state.lexicons.upsert(lexicon).await; + seed_script( + &state, + "xrpc.query:test.echo", + r#"function handle() return { greeting = "hello" } end"#, + ) + .await; let mut params = HashMap::new(); let result = execute_local_query(&state, "test.echo", &mut params, None).await; @@ -447,11 +488,14 @@ mod tests { async fn query_local_script_receives_params() { let state = test_state(); - let lexicon = make_query_lexicon( - "test.greet", - Some(r#"function handle() return { greeting = "hello " .. params.name } end"#), - ); + let lexicon = make_query_lexicon("test.greet", None); state.lexicons.upsert(lexicon).await; + seed_script( + &state, + "xrpc.query:test.greet", + r#"function handle() return { greeting = "hello " .. params.name } end"#, + ) + .await; let mut params = HashMap::new(); params.insert("name".into(), Value::String("world".into())); @@ -473,15 +517,16 @@ mod tests { async fn query_local_script_receives_caller_did() { let state = test_state(); - let lexicon = make_query_lexicon( - "test.whoami", - Some( - r#"function handle() - return { did = caller_did or "anonymous" } - end"#, - ), - ); + let lexicon = make_query_lexicon("test.whoami", None); state.lexicons.upsert(lexicon).await; + seed_script( + &state, + "xrpc.query:test.whoami", + r#"function handle() + return { did = caller_did or "anonymous" } + end"#, + ) + .await; // With caller_did let claims = Claims::internal("did:plc:testuser".into()); @@ -552,11 +597,14 @@ mod tests { let state = test_state(); // Register a simple query that the outer script will call - let inner_lexicon = make_query_lexicon( - "test.inner", - Some(r#"function handle() return { value = 42 } end"#), - ); + let inner_lexicon = make_query_lexicon("test.inner", None); state.lexicons.upsert(inner_lexicon).await; + seed_script( + &state, + "xrpc.query:test.inner", + r#"function handle() return { value = 42 } end"#, + ) + .await; let state_arc = Arc::new(state); let lua = sandbox::create_sandbox().unwrap(); diff --git a/src/record_handler.rs b/src/record_handler.rs index bc3704e..e3c5173 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -57,53 +57,45 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { }; let cid = record.cid.as_deref().unwrap_or_default(); - // Run index hook before storing, if configured. The hook's return - // value determines what (if anything) gets written to the DB. - let rec_to_store = - if let Some(script) = state.lexicons.get_index_hook(&record.collection).await { - let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { - state, - lexicon_id: &record.collection, - script: &script, - action: &record.action, - uri: &uri, - did: &record.did, - collection: &record.collection, - rkey: &record.rkey, - record: Some(rec), - cached_env_vars: None, - }) + // Run record-event script (if any) before storing. The script's + // return value determines what gets written: + // None → skip indexing entirely + // Some(record) → upsert with that record body + // The dispatcher cascades `record.:` → + // `record.index:`; failures are dead-lettered fail-open. + let hook_result = crate::lua::run_record_event_script( + state, + &record.collection, + &record.action, + &uri, + &record.did, + &record.rkey, + Some(rec), + ) + .await; + let rec_to_store = match hook_result { + None => { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "script returned nil", + }), + }, + state.db_backend, + ) .await; - - match hook_result { - None => { - if state.verbose_event_logging.load(Ordering::Relaxed) { - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - state.db_backend, - ) - .await; - } - return; - } - Some(v) => v, - } - } else { - // No hook — store the original record as-is. - rec.clone() - }; + return; + } + Some(v) => v, + }; let now = now_rfc3339(); let backend = state.db_backend; @@ -187,44 +179,37 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { "delete" => { let backend = state.db_backend; - // Run index hook before deleting, if configured. - if let Some(script) = state.lexicons.get_index_hook(&record.collection).await { - let hook_result = crate::lua::execute_hook_script(&crate::lua::HookEvent { - state, - lexicon_id: &record.collection, - script: &script, - action: "delete", - uri: &uri, - did: &record.did, - collection: &record.collection, - rkey: &record.rkey, - record: None, - cached_env_vars: None, - }) + // Run record-event script (if any) before deleting. A nil + // return aborts the delete; any other return continues. + let hook_result = crate::lua::run_record_event_script( + state, + &record.collection, + "delete", + &uri, + &record.did, + &record.rkey, + None, + ) + .await; + if hook_result.is_none() { + log_event( + db, + EventLog { + event_type: "record.skipped".to_string(), + severity: Severity::Info, + actor_did: None, + subject: Some(uri.clone()), + detail: serde_json::json!({ + "collection": record.collection, + "did": record.did, + "rkey": record.rkey, + "reason": "script returned nil", + }), + }, + backend, + ) .await; - - if hook_result.is_none() { - if state.verbose_event_logging.load(Ordering::Relaxed) { - log_event( - db, - EventLog { - event_type: "record.skipped".to_string(), - severity: Severity::Info, - actor_did: None, - subject: Some(uri.clone()), - detail: serde_json::json!({ - "collection": record.collection, - "did": record.did, - "rkey": record.rkey, - "reason": "hook returned nil", - }), - }, - backend, - ) - .await; - } - return; - } + return; } let delete_sql = adapt_sql("DELETE FROM records WHERE uri = ?", backend); diff --git a/src/xrpc/procedure.rs b/src/xrpc/procedure.rs index 55d5523..a652dd7 100644 --- a/src/xrpc/procedure.rs +++ b/src/xrpc/procedure.rs @@ -18,7 +18,15 @@ pub(crate) async fn handle_procedure( params: &std::collections::HashMap, lexicon: &crate::lexicon::ParsedLexicon, ) -> Result { - if let Some(ref script) = lexicon.script { + // Trigger-keyed dispatch: a script bound at `xrpc.procedure:` + // overrides the default PDS-write flow. The legacy `lexicon.script` + // column is no longer read. + let trigger = format!("xrpc.procedure:{}", lexicon.id); + if let Some(resolved) = crate::lua::resolve(state, &trigger).await { + // Delegation guard preserved from origin/dev: scripts that run + // under a `delegateDid` must come from a caller who is an + // active write-capable delegate of that account, scoped to the + // calling api_client. let delegate_did = input .get("delegateDid") .and_then(|v| v.as_str()) @@ -71,7 +79,7 @@ pub(crate) async fn handle_procedure( &script_input, params, lexicon, - script, + &resolved.body, None, delegate_did.as_deref(), ) diff --git a/src/xrpc/query.rs b/src/xrpc/query.rs index 1c2a126..14bd9ad 100644 --- a/src/xrpc/query.rs +++ b/src/xrpc/query.rs @@ -15,9 +15,19 @@ pub(crate) async fn handle_query( lexicon: &crate::lexicon::ParsedLexicon, claims: Option<&Claims>, ) -> Result { - if let Some(ref script) = lexicon.script { + // Trigger-keyed dispatch: a script bound at `xrpc.query:` + // overrides the default list / get-record flow. The legacy + // `lexicon.script` column is no longer read. + let trigger = format!("xrpc.query:{}", lexicon.id); + if let Some(resolved) = crate::lua::resolve(state, &trigger).await { return crate::lua::execute_query_script( - state, method, params, lexicon, script, claims, None, + state, + method, + params, + lexicon, + &resolved.body, + claims, + None, ) .await; } diff --git a/tests/common/db.rs b/tests/common/db.rs index d4fc0df..15c7244 100644 --- a/tests/common/db.rs +++ b/tests/common/db.rs @@ -20,7 +20,7 @@ pub async fn truncate_all(pool: &AnyPool) { match backend { DatabaseBackend::Postgres => { sqlx::query( - "TRUNCATE records, lexicons, backfill_jobs, users, user_permissions, api_keys, event_logs, script_variables, dead_letter_hooks, record_refs, labeler_subscriptions, labels, instance_settings, domains, dpop_sessions, dpop_keys, api_clients, delegated_accounts, account_delegates RESTART IDENTITY CASCADE", + "TRUNCATE records, lexicons, backfill_jobs, users, user_permissions, api_keys, event_logs, script_variables, scripts, dead_letter_scripts, dead_letter_hooks, record_refs, labeler_subscriptions, labels, instance_settings, domains, dpop_sessions, dpop_keys, api_clients, delegated_accounts, account_delegates RESTART IDENTITY CASCADE", ) .execute(pool) .await @@ -41,6 +41,8 @@ pub async fn truncate_all(pool: &AnyPool) { "api_keys", "event_logs", "script_variables", + "scripts", + "dead_letter_scripts", "dead_letter_hooks", "record_refs", "labeler_subscriptions", diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6c27470..70ada54 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -7,6 +7,7 @@ pub mod db; #[allow(dead_code, unused_imports)] pub mod fixtures; +#[allow(unused_macros)] macro_rules! require_db { () => { if std::env::var("TEST_DATABASE_URL").is_err() { @@ -16,4 +17,5 @@ macro_rules! require_db { }; } +#[allow(unused_imports)] pub(crate) use require_db; diff --git a/tests/e2e_scripts.rs b/tests/e2e_scripts.rs new file mode 100644 index 0000000..78994ec --- /dev/null +++ b/tests/e2e_scripts.rs @@ -0,0 +1,833 @@ +//! End-to-end tests for the trigger-keyed scripts subsystem. +//! +//! Covers: +//! - Admin CRUD on `/admin/scripts` with trigger-id validation. +//! - Dispatcher cascade for record events +//! (`record.:` → `record.index:`). +//! - Label scripts: URI-routed dispatch + Record local mutation +//! (`Record.delete_local`, `:save_local`). +//! - The no-PDS-auth boundary: a label script that calls `r:save()` +//! gets dead-lettered fail-open with the original record untouched. + +mod common; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use happyview::db::{adapt_sql, now_rfc3339}; +use happyview::lua::{LabelAppliedEvent, LabelHookOutcome, run_label_applied_script}; +use happyview::record_handler::{RecordEvent, handle_record_event}; +use http_body_util::BodyExt; +use serde_json::{Value, json}; +use serial_test::serial; +use tower::ServiceExt; + +use common::app::TestApp; +use common::fixtures; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async fn json_body(resp: axum::response::Response) -> Value { + let body = resp.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice(&body).unwrap() +} + +fn admin_get( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), +) -> Request { + Request::builder() + .uri(uri) + .header(cookie.0, cookie.1) + .body(Body::empty()) + .unwrap() +} + +fn admin_post( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), + body: &Value, +) -> Request { + Request::builder() + .method("POST") + .uri(uri) + .header(cookie.0, cookie.1) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(body).unwrap())) + .unwrap() +} + +fn admin_patch( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), + body: &Value, +) -> Request { + Request::builder() + .method("PATCH") + .uri(uri) + .header(cookie.0, cookie.1) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(body).unwrap())) + .unwrap() +} + +fn admin_delete( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), +) -> Request { + Request::builder() + .method("DELETE") + .uri(uri) + .header(cookie.0, cookie.1) + .body(Body::empty()) + .unwrap() +} + +/// Seed a record-type lexicon (no scripts bound — scripts live in their +/// own table now, addressed by trigger id). +async fn seed_lexicon(app: &TestApp, lexicon: Value) { + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/lexicons", + app.admin_cookie(), + &json!({ "lexicon_json": lexicon }), + )) + .await + .unwrap(); + assert!( + resp.status().is_success(), + "seeding lexicon failed: {:?}", + resp.status() + ); +} + +/// Create a script via the admin API. Returns the created row. +async fn create_script(app: &TestApp, id: &str, body: &str) -> Value { + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/scripts", + app.admin_cookie(), + &json!({ "id": id, "body": body }), + )) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::CREATED, + "create '{id}' failed; body: {:?}", + json_body(resp).await + ); + let resp = app + .router + .clone() + .oneshot(admin_get( + &format!("/admin/scripts/{}", urlencoding::encode(id)), + app.admin_cookie(), + )) + .await + .unwrap(); + json_body(resp).await +} + +async fn seed_record_row( + app: &TestApp, + uri: &str, + did: &str, + collection: &str, + rkey: &str, + body: Value, +) { + let sql = adapt_sql( + "INSERT INTO records (uri, did, collection, rkey, record, cid, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)", + app.state.db_backend, + ); + sqlx::query(&sql) + .bind(uri) + .bind(did) + .bind(collection) + .bind(rkey) + .bind(serde_json::to_string(&body).unwrap_or_default()) + .bind("bafyseed") + .bind(now_rfc3339()) + .execute(&app.state.db) + .await + .expect("failed to seed records row"); +} + +async fn count_records(app: &TestApp, uri: &str) -> i64 { + let (count,): (i64,) = sqlx::query_as(&adapt_sql( + "SELECT COUNT(*) FROM records WHERE uri = ?", + app.state.db_backend, + )) + .bind(uri) + .fetch_one(&app.state.db) + .await + .unwrap(); + count +} + +async fn fetch_record_body(app: &TestApp, uri: &str) -> Option { + let row: Option<(String,)> = sqlx::query_as(&adapt_sql( + "SELECT record FROM records WHERE uri = ?", + app.state.db_backend, + )) + .bind(uri) + .fetch_optional(&app.state.db) + .await + .unwrap(); + row.map(|(s,)| serde_json::from_str(&s).unwrap()) +} + +// --------------------------------------------------------------------------- +// Admin CRUD +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn create_then_get_script_round_trips() { + let app = TestApp::new().await; + let id = "record.create:com.example.thing"; + create_script(&app, id, "function handle() return event.record end").await; + + let resp = app + .router + .clone() + .oneshot(admin_get( + &format!("/admin/scripts/{}", urlencoding::encode(id)), + app.admin_cookie(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let row = json_body(resp).await; + assert_eq!(row["id"], id); + assert_eq!(row["script_type"], "lua"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn list_scripts_returns_all_rows() { + let app = TestApp::new().await; + create_script( + &app, + "record.create:com.example.thing", + "function handle() return event.record end", + ) + .await; + create_script( + &app, + "labeler.apply:_actor", + "function handle() return event end", + ) + .await; + + let resp = app + .router + .clone() + .oneshot(admin_get("/admin/scripts", app.admin_cookie())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let rows = json_body(resp).await; + let arr = rows.as_array().unwrap(); + assert_eq!(arr.len(), 2); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn create_rejects_invalid_trigger_prefix() { + let app = TestApp::new().await; + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/scripts", + app.admin_cookie(), + &json!({ + "id": "garbage:com.example.thing", + "body": "function handle() end", + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + let err = json_body(resp).await; + let msg = err["error"].as_str().unwrap_or(""); + assert!( + msg.contains("unknown trigger prefix"), + "expected validation error, got: {msg}" + ); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn create_rejects_invalid_nsid_suffix() { + let app = TestApp::new().await; + // Single-segment NSID — too few segments. + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/scripts", + app.admin_cookie(), + &json!({ + "id": "record.create:foo", + "body": "function handle() end", + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn create_allows_labeler_apply_actor_special_case() { + let app = TestApp::new().await; + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/scripts", + app.admin_cookie(), + &json!({ + "id": "labeler.apply:_actor", + "body": "function handle() return event end", + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::CREATED); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn create_rejects_invalid_lua_body() { + let app = TestApp::new().await; + let resp = app + .router + .clone() + .oneshot(admin_post( + "/admin/scripts", + app.admin_cookie(), + &json!({ + "id": "record.create:com.example.thing", + "body": "function handle(", // syntax error + }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn patch_updates_body() { + let app = TestApp::new().await; + let id = "record.create:com.example.thing"; + create_script(&app, id, "function handle() return event.record end").await; + + let resp = app + .router + .clone() + .oneshot(admin_patch( + &format!("/admin/scripts/{}", urlencoding::encode(id)), + app.admin_cookie(), + &json!({ "body": "function handle() return nil end" }), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let row = json_body(resp).await; + assert!(row["body"].as_str().unwrap().contains("return nil")); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn delete_removes_script() { + let app = TestApp::new().await; + let id = "record.delete:com.example.thing"; + create_script(&app, id, "function handle() return event.record end").await; + + let resp = app + .router + .clone() + .oneshot(admin_delete( + &format!("/admin/scripts/{}", urlencoding::encode(id)), + app.admin_cookie(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NO_CONTENT); + + let resp = app + .router + .clone() + .oneshot(admin_get( + &format!("/admin/scripts/{}", urlencoding::encode(id)), + app.admin_cookie(), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +// --------------------------------------------------------------------------- +// Cascade resolution: action-specific row wins over wildcard +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn cascade_wildcard_runs_when_no_action_specific() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + create_script( + &app, + "record.index:games.gamesgamesgamesgames.game", + // Wildcard — uppercases the title for any action. + "function handle() event.record.title = string.upper(event.record.title); return event.record end", + ) + .await; + + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rkey1".into(), + action: "create".into(), + record: Some(json!({"title": "test game"})), + cid: Some("bafy".into()), + }, + ) + .await; + + let body = fetch_record_body( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/rkey1", + ) + .await + .unwrap(); + assert_eq!(body["title"], "TEST GAME"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn cascade_action_specific_wins_over_wildcard() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + create_script( + &app, + "record.index:games.gamesgamesgamesgames.game", + "function handle() event.record.title = 'WILDCARD'; return event.record end", + ) + .await; + create_script( + &app, + "record.create:games.gamesgamesgamesgames.game", + "function handle() event.record.title = 'CREATE-SPECIFIC'; return event.record end", + ) + .await; + + // Create action — specific should win. + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rk-create".into(), + action: "create".into(), + record: Some(json!({"title": "x"})), + cid: Some("bafy".into()), + }, + ) + .await; + let body = fetch_record_body( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/rk-create", + ) + .await + .unwrap(); + assert_eq!(body["title"], "CREATE-SPECIFIC"); + + // Update action — no record.update binding → cascades to wildcard. + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rk-update".into(), + action: "update".into(), + record: Some(json!({"title": "x"})), + cid: Some("bafy".into()), + }, + ) + .await; + let body = fetch_record_body( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/rk-update", + ) + .await + .unwrap(); + assert_eq!(body["title"], "WILDCARD"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn no_script_passes_record_through_unchanged() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rk1".into(), + action: "create".into(), + record: Some(json!({"title": "untouched"})), + cid: Some("bafy".into()), + }, + ) + .await; + let body = fetch_record_body( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/rk1", + ) + .await + .unwrap(); + assert_eq!(body["title"], "untouched"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_create_returning_nil_skips_indexing() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + create_script( + &app, + "record.create:games.gamesgamesgamesgames.game", + "function handle() return nil end", + ) + .await; + + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rk1".into(), + action: "create".into(), + record: Some(json!({"title": "doomed"})), + cid: Some("bafy".into()), + }, + ) + .await; + assert_eq!( + count_records( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/rk1" + ) + .await, + 0, + "nil return should drop the record" + ); +} + +// --------------------------------------------------------------------------- +// Label scripts via URI routing +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn label_script_can_drop_record_via_record_delete_local() { + let app = TestApp::new().await; + + let uri = "at://did:plc:victim/app.bsky.feed.post/rkey1"; + seed_record_row( + &app, + uri, + "did:plc:victim", + "app.bsky.feed.post", + "rkey1", + json!({"text": "hello"}), + ) + .await; + + create_script( + &app, + "labeler.apply:app.bsky.feed.post", + "function handle() \ + if event.val == 'spam' then Record.delete_local(event.uri) end \ + return event \ + end", + ) + .await; + + let outcome = run_label_applied_script( + &app.state, + LabelAppliedEvent { + src: "did:plc:labeler".into(), + uri: uri.into(), + val: "spam".into(), + neg: false, + cts: now_rfc3339(), + exp: None, + }, + ) + .await; + assert!(matches!(outcome, LabelHookOutcome::Continue(_))); + assert_eq!(count_records(&app, uri).await, 0); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn label_script_can_redact_record_via_save_local() { + let app = TestApp::new().await; + + let uri = "at://did:plc:author/app.bsky.feed.post/rkey1"; + seed_record_row( + &app, + uri, + "did:plc:author", + "app.bsky.feed.post", + "rkey1", + json!({"text": "original content"}), + ) + .await; + + create_script( + &app, + "labeler.apply:app.bsky.feed.post", + "function handle() \ + if event.val == 'redact' then \ + local r = Record.load(event.uri) \ + if r then r.text = '[redacted by ' .. event.src .. ']'; r:save_local() end \ + end; \ + return event \ + end", + ) + .await; + + let outcome = run_label_applied_script( + &app.state, + LabelAppliedEvent { + src: "did:plc:labeler".into(), + uri: uri.into(), + val: "redact".into(), + neg: false, + cts: now_rfc3339(), + exp: None, + }, + ) + .await; + assert!(matches!(outcome, LabelHookOutcome::Continue(_))); + + let body = fetch_record_body(&app, uri).await.unwrap(); + assert_eq!(body["text"], "[redacted by did:plc:labeler]"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn label_script_uri_routes_actor_special_case() { + let app = TestApp::new().await; + + create_script( + &app, + "labeler.apply:_actor", + // Sentinel: write a row into records-table-as-flag so we can + // detect that the script ran. + "function handle() \ + db.raw('INSERT INTO records (uri, did, collection, rkey, record, cid, created_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?)', \ + {'at://did:plc:flag/flag.col/k', 'did:plc:flag', 'flag.col', 'k', '{}', 'b', '2026-05-01'}) \ + return event \ + end", + ) + .await; + + // Bare DID URI should route to `labeler.apply:_actor`. + let outcome = run_label_applied_script( + &app.state, + LabelAppliedEvent { + src: "did:plc:labeler".into(), + uri: "did:plc:somebody".into(), + val: "imposter".into(), + neg: false, + cts: now_rfc3339(), + exp: None, + }, + ) + .await; + assert!(matches!(outcome, LabelHookOutcome::Continue(_))); + + // Sentinel row should exist if the script ran. + assert_eq!( + count_records(&app, "at://did:plc:flag/flag.col/k").await, + 1, + "labeler.apply:_actor should have fired for bare-DID label" + ); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn label_script_calling_record_save_dead_letters_with_clear_message() { + let app = TestApp::new().await; + + let uri = "at://did:plc:author/app.bsky.feed.post/rkey1"; + seed_record_row( + &app, + uri, + "did:plc:author", + "app.bsky.feed.post", + "rkey1", + json!({"text": "untouched"}), + ) + .await; + + create_script( + &app, + "labeler.apply:app.bsky.feed.post", + "function handle() \ + local r = Record.load(event.uri) \ + if r then r.text = 'should fail'; r:save() end \ + return event \ + end", + ) + .await; + + let outcome = run_label_applied_script( + &app.state, + LabelAppliedEvent { + src: "did:plc:labeler".into(), + uri: uri.into(), + val: "anything".into(), + neg: false, + cts: now_rfc3339(), + exp: None, + }, + ) + .await; + // Fail-open: the original label still continues even after the script + // fails its retry budget. + assert!(matches!(outcome, LabelHookOutcome::Continue(_))); + + // The original record is unchanged. + let body = fetch_record_body(&app, uri).await.unwrap(); + assert_eq!(body["text"], "untouched"); + + // A dead-letter row exists with the NO_PDS_AUTH message. + let dl: (String,) = sqlx::query_as(&adapt_sql( + "SELECT error FROM dead_letter_scripts WHERE host_kind = 'label' \ + AND host_id = 'did:plc:labeler' ORDER BY id DESC LIMIT 1", + app.state.db_backend, + )) + .fetch_one(&app.state.db) + .await + .expect("expected a dead_letter_scripts row"); + assert!( + dl.0.contains("no PDS auth"), + "expected NO_PDS_AUTH message in dead-letter, got: {}", + dl.0 + ); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_event_script_can_call_record_delete_local() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + let victim_uri = "at://did:plc:test/games.gamesgamesgamesgames.game/old"; + seed_record_row( + &app, + victim_uri, + "did:plc:test", + "games.gamesgamesgamesgames.game", + "old", + json!({"title": "should-be-gone"}), + ) + .await; + + create_script( + &app, + "record.create:games.gamesgamesgamesgames.game", + "function handle() \ + Record.delete_local('at://did:plc:test/games.gamesgamesgamesgames.game/old') \ + return event.record \ + end", + ) + .await; + + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "new1".into(), + action: "create".into(), + record: Some(json!({"title": "fresh game"})), + cid: Some("bafy".into()), + }, + ) + .await; + + assert_eq!(count_records(&app, victim_uri).await, 0); + assert_eq!( + count_records( + &app, + "at://did:plc:test/games.gamesgamesgamesgames.game/new1" + ) + .await, + 1 + ); +} + +// --------------------------------------------------------------------------- +// Permission gating +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn no_auth_returns_401() { + let app = TestApp::new().await; + let resp = app + .router + .clone() + .oneshot( + Request::builder() + .uri("/admin/scripts") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} diff --git a/tests/lua_record_api.rs b/tests/lua_record_api.rs new file mode 100644 index 0000000..6de7cdb --- /dev/null +++ b/tests/lua_record_api.rs @@ -0,0 +1,478 @@ +//! Integration tests for the local-only Record API surface +//! (`Record.delete_local`, `r:save_local`, `r:delete_local`) and the +//! auth-boundary errors when label / record-event / query scripts reach +//! for PDS-touching methods (`r:save`, `r:delete`). + +mod common; + +use atrium_identity::did::{CommonDidResolver, CommonDidResolverConfig}; +use atrium_identity::handle::{AtprotoHandleResolver, AtprotoHandleResolverConfig}; +use atrium_oauth::{ + AtprotoLocalhostClientMetadata, DefaultHttpClient, KnownScope, OAuthClientConfig, + OAuthResolverConfig, Scope, +}; +use happyview::AppState; +use happyview::config::Config; +use happyview::db::{DatabaseBackend, adapt_sql, now_rfc3339}; +use happyview::lexicon::LexiconRegistry; +use happyview::lua::record::register_record_api_no_auth; +use mlua::Lua; +use serial_test::serial; +use std::sync::Arc; +use tokio::sync::watch; + +use common::db; + +async fn test_state_with_pool(pool: sqlx::AnyPool, backend: DatabaseBackend) -> AppState { + let config = Config { + host: "127.0.0.1".into(), + port: 3000, + database_url: String::new(), + database_backend: backend, + public_url: String::new(), + base_path: None, + session_secret: "test-secret".into(), + jetstream_url: String::new(), + relay_url: String::new(), + plc_url: String::new(), + static_dir: String::new(), + event_log_retention_days: 30, + app_name: None, + logo_uri: None, + tos_uri: None, + policy_uri: None, + token_encryption_key: None, + default_rate_limit_capacity: 100, + default_rate_limit_refill_rate: 2.0, + }; + let (tx, _) = watch::channel(vec![]); + let (labeler_tx, _) = watch::channel(()); + let atrium_http = std::sync::Arc::new(DefaultHttpClient::default()); + let did_resolver = CommonDidResolver::new(CommonDidResolverConfig { + plc_directory_url: "https://plc.directory".into(), + http_client: std::sync::Arc::clone(&atrium_http), + }); + let handle_resolver = AtprotoHandleResolver::new(AtprotoHandleResolverConfig { + dns_txt_resolver: happyview::dns::NativeDnsResolver::new(), + http_client: atrium_http, + }); + let oauth_pool = db::test_pool().await; + let oauth = atrium_oauth::OAuthClient::new(OAuthClientConfig { + client_metadata: AtprotoLocalhostClientMetadata { + redirect_uris: Some(vec!["http://127.0.0.1:0/auth/callback".into()]), + scopes: Some(vec![Scope::Known(KnownScope::Atproto)]), + }, + keys: None, + state_store: happyview::auth::oauth_store::DbStateStore::new(oauth_pool.clone(), backend), + session_store: happyview::auth::oauth_store::DbSessionStore::new(oauth_pool, backend), + resolver: OAuthResolverConfig { + did_resolver, + handle_resolver, + authorization_server_metadata: Default::default(), + protected_resource_metadata: Default::default(), + }, + }) + .expect("Failed to create test OAuth client"); + AppState { + config, + http: reqwest::Client::new(), + db: pool.clone(), + db_backend: backend, + lexicons: LexiconRegistry::new(), + collections_tx: tx, + labeler_subscriptions_tx: labeler_tx, + rate_limiter: happyview::rate_limit::RateLimiter::new( + happyview::rate_limit::RateLimitDefaults { + query_cost: 1, + procedure_cost: 1, + proxy_cost: 1, + }, + ), + oauth: std::sync::Arc::new(happyview::auth::OAuthClientRegistry::new( + std::sync::Arc::new(oauth), + )), + oauth_state_store: happyview::auth::oauth_store::DbStateStore::new(pool.clone(), backend), + cookie_key: axum_extra::extract::cookie::Key::derive_from( + b"test-secret-that-is-at-least-32-bytes-long", + ), + plugin_registry: std::sync::Arc::new(happyview::plugin::PluginRegistry::new()), + wasm_runtime: std::sync::Arc::new( + happyview::plugin::WasmRuntime::new().expect("wasm runtime"), + ), + attestation_signer: None, + official_registry: std::sync::Arc::new(tokio::sync::RwLock::new( + happyview::plugin::official_registry::OfficialRegistryState::default(), + )), + official_registry_config: happyview::plugin::official_registry::RegistryConfig::production( + ), + domain_cache: happyview::domain::DomainCache::new(), + proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( + happyview::proxy_config::ProxyConfig::default(), + ))), + backfill_db: pool.clone(), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + } +} + +async fn seed_record( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + uri: &str, + did: &str, + collection: &str, + rkey: &str, + record: serde_json::Value, +) { + let sql = adapt_sql( + "INSERT INTO records (uri, did, collection, rkey, record, cid, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)", + backend, + ); + sqlx::query(&sql) + .bind(uri) + .bind(did) + .bind(collection) + .bind(rkey) + .bind(serde_json::to_string(&record).unwrap_or_default()) + .bind("bafyseed") + .bind(now_rfc3339()) + .execute(pool) + .await + .expect("failed to seed record"); +} + +async fn count_records(pool: &sqlx::AnyPool, backend: DatabaseBackend, uri: &str) -> i64 { + let sql = adapt_sql("SELECT COUNT(*) FROM records WHERE uri = ?", backend); + let (count,): (i64,) = sqlx::query_as(&sql) + .bind(uri) + .fetch_one(pool) + .await + .expect("count query"); + count +} + +async fn fetch_record_body( + pool: &sqlx::AnyPool, + backend: DatabaseBackend, + uri: &str, +) -> Option { + let sql = adapt_sql("SELECT record FROM records WHERE uri = ?", backend); + let row: Option<(String,)> = sqlx::query_as(&sql) + .bind(uri) + .fetch_optional(pool) + .await + .expect("fetch record"); + row.map(|(s,)| serde_json::from_str(&s).expect("record json")) +} + +/// Build a sandbox with the Record API registered in **no-auth mode** — +/// the same shape label / record-event / query scripts get. +fn setup_no_auth_lua(state: &AppState) -> Lua { + let lua = Lua::new(); + register_record_api_no_auth(&lua, Arc::new(state.clone())).expect("register record api"); + lua +} + +// --------------------------------------------------------------------------- +// Record.delete_local(uri) static +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn record_static_delete_local_returns_true_when_row_existed() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + + let uri = "at://did:plc:test/test.collection/rkey1"; + seed_record( + &pool, + backend, + uri, + "did:plc:test", + "test.collection", + "rkey1", + serde_json::json!({"name": "kept"}), + ) + .await; + + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + let deleted: bool = lua + .load(format!(r#"return Record.delete_local("{uri}")"#)) + .eval_async() + .await + .expect("delete_local call"); + assert!(deleted, "expected true (row existed before)"); + + let after = count_records(&pool, backend, uri).await; + assert_eq!(after, 0, "row should be gone"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_static_delete_local_returns_false_when_row_absent() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_no_auth_lua(&state); + + let deleted: bool = lua + .load(r#"return Record.delete_local("at://did:plc:nope/test.collection/none")"#) + .eval_async() + .await + .expect("delete_local call"); + assert!(!deleted, "no row → false (idempotent)"); +} + +// --------------------------------------------------------------------------- +// r:delete_local() instance method +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn record_instance_delete_local_removes_row_and_clears_uri() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + + let uri = "at://did:plc:test/test.collection/rkey1"; + seed_record( + &pool, + backend, + uri, + "did:plc:test", + "test.collection", + "rkey1", + serde_json::json!({"name": "doomed"}), + ) + .await; + + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + // Load via Record.load(), then call :delete_local() — this is the + // primary shape we expect from a label-script reaction. + let uri_after: mlua::Value = lua + .load(format!( + r#" + local r = Record.load("{uri}") + assert(r ~= nil, "record not loaded") + r:delete_local() + return r._uri + "# + )) + .eval_async() + .await + .expect("delete_local instance call"); + + assert!(matches!(uri_after, mlua::Value::Nil), "_uri should be nil"); + assert_eq!(count_records(&pool, backend, uri).await, 0); +} + +// --------------------------------------------------------------------------- +// r:save_local() instance method +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn record_instance_save_local_updates_existing_row() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + + let uri = "at://did:plc:test/test.collection/rkey1"; + seed_record( + &pool, + backend, + uri, + "did:plc:test", + "test.collection", + "rkey1", + serde_json::json!({"text": "original"}), + ) + .await; + + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + // Redact-style flow: load, mutate, save_local. + lua.load(format!( + r#" + local r = Record.load("{uri}") + assert(r ~= nil) + r.text = "[redacted]" + r:save_local() + "# + )) + .exec_async() + .await + .expect("save_local instance call"); + + let body = fetch_record_body(&pool, backend, uri).await.unwrap(); + assert_eq!(body["text"], "[redacted]"); + // $type is injected automatically by the serializer. + assert_eq!(body["$type"], "test.collection"); + // Row count unchanged (upsert). + assert_eq!(count_records(&pool, backend, uri).await, 1); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_save_local_creates_new_row_when_repo_set() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + // Build a fresh record. `:set_repo` provides the DID for the URI; + // without auth there's no fallback. We also manually `:set_rkey` + // since there's no key_type from a (missing) lexicon. + let uri: String = lua + .load( + r#" + local r = Record.new("test.collection", { value = 42 }) + r:set_repo("did:plc:newowner") + r:set_rkey("brandnew") + r:save_local() + return r._uri + "#, + ) + .eval_async() + .await + .expect("save_local creating call"); + + assert_eq!(uri, "at://did:plc:newowner/test.collection/brandnew"); + let body = fetch_record_body(&pool, backend, &uri).await.unwrap(); + assert_eq!(body["value"], 42); + assert_eq!(body["$type"], "test.collection"); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_save_local_errors_without_did_when_no_uri() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_no_auth_lua(&state); + + // No `:set_repo` and no claims → :save_local() must error. + let err = lua + .load( + r#" + local r = Record.new("test.collection", { value = 1 }) + r:save_local() + "#, + ) + .exec_async() + .await + .expect_err("expected error: no DID resolvable"); + let msg = err.to_string(); + assert!( + msg.contains("save_local() needs a DID"), + "expected DID-required message, got: {msg}" + ); +} + +// --------------------------------------------------------------------------- +// PDS-touching methods error cleanly when no auth present +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn record_save_errors_without_pds_auth() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + + let uri = "at://did:plc:test/test.collection/rkey1"; + seed_record( + &pool, + backend, + uri, + "did:plc:test", + "test.collection", + "rkey1", + serde_json::json!({"value": 1}), + ) + .await; + + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + let err = lua + .load(format!( + r#" + local r = Record.load("{uri}") + r:save() + "# + )) + .exec_async() + .await + .expect_err("expected NO_PDS_AUTH error"); + let msg = err.to_string(); + assert!( + msg.contains("no PDS auth"), + "expected NO_PDS_AUTH message, got: {msg}" + ); + + // The original row should be untouched. + let body = fetch_record_body(&pool, backend, uri).await.unwrap(); + assert_eq!(body["value"], 1); +} + +#[tokio::test] +#[serial] +#[ignore] +async fn record_delete_errors_without_pds_auth() { + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + + let uri = "at://did:plc:test/test.collection/rkey1"; + seed_record( + &pool, + backend, + uri, + "did:plc:test", + "test.collection", + "rkey1", + serde_json::json!({"value": 1}), + ) + .await; + + let state = test_state_with_pool(pool.clone(), backend).await; + let lua = setup_no_auth_lua(&state); + + let err = lua + .load(format!( + r#" + local r = Record.load("{uri}") + r:delete() + "# + )) + .exec_async() + .await + .expect_err("expected NO_PDS_AUTH error"); + let msg = err.to_string(); + assert!( + msg.contains("no PDS auth"), + "expected NO_PDS_AUTH message, got: {msg}" + ); + + // No-auth :delete() must NOT touch the local DB either — + // the row should still be there. + assert_eq!(count_records(&pool, backend, uri).await, 1); +} -- 2.51.2 From 673de0972700ad9ece1a4e9dcf899395fd438587 Mon Sep 17 00:00:00 2001 From: Chris Pardy Date: Fri, 1 May 2026 10:02:00 -0400 Subject: [PATCH 46/64] feat(scripts): promote log() to write into event_logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Lua `log("...")` primitive used to call `tracing::debug!` only — fine for ops who tail server stderr, useless for operators who use the dashboard. Wire it into `event_logs` so script output is queryable from `/dashboard/events` without shell access. New `register_log_event_api(lua, state, trigger_id, caller_did)` in `src/lua/scripts.rs` registers a Lua `log(msg)` global that does both: - Emits `tracing::debug!(lua_log = msg, trigger = trigger_id, ...)` for ops who do tail. - Inserts a row into `event_logs` with: event_type = "script.log" severity = Info actor_did = caller_did (XRPC handlers; None for record / label) subject = trigger_id (queryable per script) detail = { trigger, message } This intentionally OVERRIDES the basic `log()` from `sandbox::create_sandbox()`. All four script runners (record-event, label-applied, xrpc.procedure, xrpc.query) now wire it through: - `register_default_apis(lua, state, trigger_id, caller_did)` gains the `trigger_id` parameter and calls `register_log_event_api` after the other registrations. - `execute_procedure_script` and `execute_query_script` register the log API directly with `xrpc.{procedure,query}:` as the trigger. E2E test in `tests/e2e_scripts.rs` verifies a record-event script's `log("hello from script")` produces a `script.log` row whose subject is the trigger id and detail carries `{ trigger, message }`. Signed-off-by: Chris Pardy --- src/lua/execute.rs | 63 +++++++++++++++++++++++++++++++++++++++++++- src/lua/sandbox.rs | 14 ++++++---- src/lua/scripts.rs | 61 ++++++++++++++++++++++++++++++++++++++++-- tests/e2e_scripts.rs | 51 +++++++++++++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 8 deletions(-) diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 3ed5a23..7ee6f3c 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -289,6 +289,37 @@ pub async fn execute_procedure_script( return Err(AppError::Internal(error_message)); } + // Override the sandbox's tracing-only `log()` with a version that + // also writes a `script.log` row to `event_logs` so operators can + // see script output from the dashboard. The xrpc trigger id is + // computed from the lexicon's id + procedure type. + let trigger_id = format!("xrpc.procedure:{}", lexicon.id); + if let Err(e) = + super::scripts::register_log_event_api(&lua, &state_arc, &trigger_id, Some(claims.did())) + { + let error_message = format!("failed to register log API: {e}"); + log_event( + &state.db, + EventLog { + event_type: "script.error".to_string(), + severity: Severity::Error, + actor_did: Some(claims.did().to_string()), + subject: Some(method.to_string()), + detail: serde_json::json!({ + "error": error_message, + "script_source": script_source, + "input": input_json, + "caller_did": claims.did(), + "method": method, + "duration_ms": start.elapsed().as_millis() as u64, + }), + }, + backend, + ) + .await; + return Err(AppError::Internal(error_message)); + } + if let Err(e) = context::set_procedure_context( &lua, method, @@ -668,7 +699,7 @@ pub async fn execute_query_script( // auth context — the local-only methods (Record.load, :save_local, // :delete_local, Record.delete_local) work; PDS-touching variants // error with the no-PDS-auth message. - if let Err(e) = record::register_record_api_no_auth(&lua, state_arc) { + if let Err(e) = record::register_record_api_no_auth(&lua, state_arc.clone()) { let error_message = format!("failed to register Record API: {e}"); log_event( &state.db, @@ -690,6 +721,36 @@ pub async fn execute_query_script( return Err(AppError::Internal(error_message)); } + // Override the sandbox's tracing-only `log()` with a version that + // also writes a `script.log` row to `event_logs`. + let trigger_id = format!("xrpc.query:{}", lexicon.id); + if let Err(e) = super::scripts::register_log_event_api( + &lua, + &state_arc, + &trigger_id, + claims.map(|c| c.did()), + ) { + let error_message = format!("failed to register log API: {e}"); + log_event( + &state.db, + EventLog { + event_type: "script.error".to_string(), + severity: Severity::Error, + actor_did: None, + subject: Some(method.to_string()), + detail: serde_json::json!({ + "error": error_message, + "script_source": script_source, + "method": method, + "duration_ms": start.elapsed().as_millis() as u64, + }), + }, + backend, + ) + .await; + return Err(AppError::Internal(error_message)); + } + if let Err(e) = context::set_query_context( &lua, method, diff --git a/src/lua/sandbox.rs b/src/lua/sandbox.rs index fe7fe3d..833f83d 100644 --- a/src/lua/sandbox.rs +++ b/src/lua/sandbox.rs @@ -55,11 +55,15 @@ pub fn create_sandbox() -> LuaResult { let now_fn = lua.create_function(|_, ()| Ok(chrono::Utc::now().to_rfc3339()))?; globals.set("now", now_fn)?; - // Utility: log(message) logs via tracing::debug - let log_fn = lua.create_function(|_, msg: String| { - tracing::debug!(lua_log = %msg, "lua script log"); - Ok(()) - })?; + // `log(message)` no-op stub. The real implementation lives in + // `super::scripts::register_log_event_api` and is registered by + // every runner so the trigger context can be threaded into each + // `event_logs` row. The stub here exists only so paths that exec + // a script body OUTSIDE a runner — namely `validate_script` + // (admin write-time linting) and the in-process xrpc_api tests — + // don't break on top-level `log("...")` calls in user scripts. + // The runner-level registration always overrides this stub. + let log_fn = lua.create_function(|_, _msg: String| Ok(()))?; globals.set("log", log_fn)?; // Utility: TID table — callable as TID() to generate, plus conversion methods diff --git a/src/lua/scripts.rs b/src/lua/scripts.rs index 43970dc..870d373 100644 --- a/src/lua/scripts.rs +++ b/src/lua/scripts.rs @@ -407,7 +407,7 @@ pub async fn run_record_event_once( } let lua = sandbox::create_sandbox().map_err(|e| format!("create sandbox: {e}"))?; let state_arc = Arc::new(state.clone()); - register_default_apis(&lua, &state_arc, Some(did))?; + register_default_apis(&lua, &state_arc, &script.id, Some(did))?; // Legacy globals (action, uri, did, collection, rkey, record) for // back-compat with scripts written against the old `index_hook` @@ -565,7 +565,7 @@ async fn run_label_lua_once( } let lua = sandbox::create_sandbox().map_err(|e| format!("create sandbox: {e}"))?; let state_arc = Arc::new(state.clone()); - register_default_apis(&lua, &state_arc, None)?; + register_default_apis(&lua, &state_arc, &script.id, None)?; use mlua::LuaSerdeExt; let globals = lua.globals(); @@ -660,6 +660,7 @@ fn extract_bool(v: &Value, key: &str) -> Option { fn register_default_apis( lua: &mlua::Lua, state: &Arc, + trigger_id: &str, caller_did: Option<&str>, ) -> Result<(), String> { db_api::register_db_api(lua, state.clone()).map_err(|e| format!("db api: {e}"))?; @@ -670,6 +671,62 @@ fn register_default_apis( .map_err(|e| format!("atproto api: {e}"))?; record::register_record_api_no_auth(lua, state.clone()) .map_err(|e| format!("record api: {e}"))?; + register_log_event_api(lua, state, trigger_id, caller_did)?; + Ok(()) +} + +/// Register `log(msg)` as a Lua global that writes a `script.log` row to +/// `event_logs` (so operators can inspect script output from +/// `/dashboard/events` without tailing stderr) AND emits a +/// `tracing::debug!` for ops who do tail. +/// +/// `trigger_id` is recorded as the row's `subject` so events for a +/// specific script can be filtered by trigger. `caller_did` is recorded +/// as `actor_did` when the runner has one (XRPC handlers); record / +/// label runners pass `None`. +/// +/// This intentionally **overrides** the basic `log()` helper that +/// `sandbox::create_sandbox()` registers (which only writes to +/// `tracing::debug!`). All script runners call this helper so every +/// trigger family lands its `log()` calls in the event log. +pub(crate) fn register_log_event_api( + lua: &mlua::Lua, + state: &Arc, + trigger_id: &str, + caller_did: Option<&str>, +) -> Result<(), String> { + let state = state.clone(); + let trigger_id = trigger_id.to_string(); + let caller_did = caller_did.map(String::from); + let log_fn = lua + .create_async_function(move |_, msg: String| { + let state = state.clone(); + let trigger_id = trigger_id.clone(); + let caller_did = caller_did.clone(); + async move { + tracing::debug!(lua_log = %msg, trigger = %trigger_id, "lua script log"); + log_event( + &state.db, + EventLog { + event_type: "script.log".to_string(), + severity: Severity::Info, + actor_did: caller_did, + subject: Some(trigger_id.clone()), + detail: serde_json::json!({ + "trigger": trigger_id, + "message": msg, + }), + }, + state.db_backend, + ) + .await; + Ok(()) + } + }) + .map_err(|e| format!("log api: {e}"))?; + lua.globals() + .set("log", log_fn) + .map_err(|e| format!("set log global: {e}"))?; Ok(()) } diff --git a/tests/e2e_scripts.rs b/tests/e2e_scripts.rs index 78994ec..8b8a357 100644 --- a/tests/e2e_scripts.rs +++ b/tests/e2e_scripts.rs @@ -555,6 +555,57 @@ async fn record_create_returning_nil_skips_indexing() { ); } +// --------------------------------------------------------------------------- +// log() in scripts → event_logs +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +#[ignore] +async fn record_event_script_log_writes_event_log_row() { + let app = TestApp::new().await; + seed_lexicon(&app, fixtures::game_record_lexicon()).await; + + create_script( + &app, + "record.create:games.gamesgamesgamesgames.game", + "function handle() log('hello from script'); return event.record end", + ) + .await; + + handle_record_event( + &app.state, + &RecordEvent { + did: "did:plc:test".into(), + collection: "games.gamesgamesgamesgames.game".into(), + rkey: "rk1".into(), + action: "create".into(), + record: Some(json!({"title": "anything"})), + cid: Some("bafy".into()), + }, + ) + .await; + + // The script's log("hello from script") should land in event_logs + // as a `script.log` row whose subject is the trigger id. + let row: (String, String) = sqlx::query_as(&adapt_sql( + "SELECT subject, detail FROM event_logs + WHERE event_type = 'script.log' + ORDER BY id DESC LIMIT 1", + app.state.db_backend, + )) + .fetch_one(&app.state.db) + .await + .expect("expected a script.log row"); + assert_eq!(row.0, "record.create:games.gamesgamesgamesgames.game"); + let detail: Value = serde_json::from_str(&row.1).unwrap(); + assert_eq!(detail["message"], "hello from script"); + assert_eq!( + detail["trigger"], + "record.create:games.gamesgamesgamesgames.game" + ); +} + // --------------------------------------------------------------------------- // Label scripts via URI routing // --------------------------------------------------------------------------- -- 2.51.2 From 64b699cf0bda87a3f2c3ed3d7c3eaee04b154b13 Mon Sep 17 00:00:00 2001 From: Chris Pardy Date: Fri, 1 May 2026 11:22:11 -0400 Subject: [PATCH 47/64] refactor(scripts): bundle record-event runner args into RecordEventPayload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `run_record_event_script` and `run_record_event_once` each took the record-event context as 6 positional arguments (nsid, action, uri, did, rkey, record). Six `&str`s in a row that the type system can't help you order — easy to swap `did` and `uri` and have it compile. Introduce a `RecordEventPayload<'a>` struct so the runner reads as run_record_event_script(state, payload).await run_record_event_once(state, &resolved, payload).await Both call sites (record_handler::handle_record_event and admin::dead_letters::retry_single) updated to construct the struct literal — names instead of positions. No behavior change. Signed-off-by: Chris Pardy --- src/admin/dead_letters.rs | 16 ++++--- src/lua/mod.rs | 7 +-- src/lua/scripts.rs | 90 ++++++++++++++++++++++----------------- src/record_handler.rs | 28 ++++++------ 4 files changed, 79 insertions(+), 62 deletions(-) diff --git a/src/admin/dead_letters.rs b/src/admin/dead_letters.rs index c5a1018..42c1d33 100644 --- a/src/admin/dead_letters.rs +++ b/src/admin/dead_letters.rs @@ -10,7 +10,7 @@ use super::permissions::Permission; use crate::AppState; use crate::db::{adapt_sql, now_rfc3339, parse_dt}; use crate::error::AppError; -use crate::lua::{resolve_record_event, run_record_event_once}; +use crate::lua::{RecordEventPayload, resolve_record_event, run_record_event_once}; use crate::record_handler::RecordEvent; // --------------------------------------------------------------------------- @@ -513,12 +513,14 @@ async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { match run_record_event_once( state, &resolved, - &dl.action, - &dl.uri, - &dl.did, - &dl.collection, - &dl.rkey, - record.as_ref(), + RecordEventPayload { + nsid: &dl.collection, + action: &dl.action, + uri: &dl.uri, + did: &dl.did, + rkey: &dl.rkey, + record: record.as_ref(), + }, ) .await { diff --git a/src/lua/mod.rs b/src/lua/mod.rs index 7177414..bed07b3 100644 --- a/src/lua/mod.rs +++ b/src/lua/mod.rs @@ -14,7 +14,8 @@ pub(crate) use context::SpaceContext; pub(crate) use execute::{execute_procedure_script, execute_query_script}; pub(crate) use sandbox::validate_script; pub use scripts::{ - LabelAppliedEvent, LabelHookOutcome, ParsedTrigger, ResolvedScript, ScriptLanguage, ScriptRow, - TriggerKind, resolve, resolve_record_event, run_label_applied_script, run_record_event_once, - run_record_event_script, trigger_for_label_uri, + LabelAppliedEvent, LabelHookOutcome, ParsedTrigger, RecordEventPayload, ResolvedScript, + ScriptLanguage, ScriptRow, TriggerKind, resolve, resolve_record_event, + run_label_applied_script, run_record_event_once, run_record_event_script, + trigger_for_label_uri, }; diff --git a/src/lua/scripts.rs b/src/lua/scripts.rs index 870d373..f3f8ef6 100644 --- a/src/lua/scripts.rs +++ b/src/lua/scripts.rs @@ -278,37 +278,45 @@ pub async fn resolve_record_event( // Record-event runner (fail-open, retry + dead-letter) // --------------------------------------------------------------------------- +/// All the contextual fields a record-event script needs at execution +/// time. Bundled into a struct so the runner doesn't take 6+ `&str` +/// positional arguments — easy to swap `did` and `uri` and have the +/// type checker shrug. +#[derive(Clone, Copy, Debug)] +pub struct RecordEventPayload<'a> { + pub nsid: &'a str, + pub action: &'a str, + pub uri: &'a str, + pub did: &'a str, + pub rkey: &'a str, + pub record: Option<&'a Value>, +} + /// Run the record-event script (if any) for a given event. Returns the /// record body the indexer should store: `Some(record)` to proceed, /// `None` to skip indexing. /// /// Failure mode is fail-open: a script that exhausts its retry budget is /// dead-lettered and the indexer proceeds with the original record. -#[allow(clippy::too_many_arguments)] pub async fn run_record_event_script( state: &AppState, - nsid: &str, - action: &str, - uri: &str, - did: &str, - rkey: &str, - record: Option<&Value>, + payload: RecordEventPayload<'_>, ) -> Option { - let resolved = match resolve_record_event(state, nsid, action).await { + let resolved = match resolve_record_event(state, payload.nsid, payload.action).await { Some(s) => s, // No script for this trigger → indexer keeps the original record. - None => return record.cloned(), + None => return payload.record.cloned(), }; - let host_id = format!("{nsid}:{action}"); - let payload = serde_json::json!({ + let host_id = format!("{}:{}", payload.nsid, payload.action); + let event_payload = serde_json::json!({ "trigger": resolved.id, - "action": action, - "uri": uri, - "did": did, - "collection": nsid, - "rkey": rkey, - "record": record, + "action": payload.action, + "uri": payload.uri, + "did": payload.did, + "collection": payload.nsid, + "rkey": payload.rkey, + "record": payload.record, }); let mut last_error = String::new(); @@ -317,7 +325,7 @@ pub async fn run_record_event_script( let delay = std::time::Duration::from_secs(1 << (attempt - 1)); tokio::time::sleep(delay).await; } - match run_record_event_once(state, &resolved, action, uri, did, nsid, rkey, record).await { + match run_record_event_once(state, &resolved, payload).await { Ok(outcome) => { log_event( &state.db, @@ -325,7 +333,7 @@ pub async fn run_record_event_script( event_type: "script.executed".to_string(), severity: Severity::Info, actor_did: None, - subject: Some(uri.to_string()), + subject: Some(payload.uri.to_string()), detail: serde_json::json!({ "host_kind": "record", "host_id": host_id, @@ -341,7 +349,7 @@ pub async fn run_record_event_script( Err(e) => { last_error = e; tracing::warn!( - %uri, + uri = %payload.uri, trigger = %resolved.id, attempt = attempt + 1, "record script attempt failed: {last_error}" @@ -355,7 +363,7 @@ pub async fn run_record_event_script( &resolved, "record", &host_id, - &payload, + &event_payload, &last_error, MAX_ATTEMPTS, ) @@ -366,7 +374,7 @@ pub async fn run_record_event_script( event_type: "script.dead_lettered".to_string(), severity: Severity::Error, actor_did: None, - subject: Some(uri.to_string()), + subject: Some(payload.uri.to_string()), detail: serde_json::json!({ "host_kind": "record", "host_id": host_id, @@ -379,7 +387,7 @@ pub async fn run_record_event_script( .await; // Fail-open: indexer proceeds with the original record. - record.cloned() + payload.record.cloned() } /// Single attempt at the record-event Lua script. Used internally by the @@ -388,16 +396,10 @@ pub async fn run_record_event_script( /// Returns `Ok(Some(value))` to continue indexing with `value`, /// `Ok(None)` when the script returned `nil` (skip), or `Err(msg)` on /// any execution failure. -#[allow(clippy::too_many_arguments)] pub async fn run_record_event_once( state: &AppState, script: &ResolvedScript, - action: &str, - uri: &str, - did: &str, - collection: &str, - rkey: &str, - record: Option<&Value>, + payload: RecordEventPayload<'_>, ) -> Result, String> { if script.language != ScriptLanguage::Lua { return Err(format!( @@ -407,25 +409,33 @@ pub async fn run_record_event_once( } let lua = sandbox::create_sandbox().map_err(|e| format!("create sandbox: {e}"))?; let state_arc = Arc::new(state.clone()); - register_default_apis(&lua, &state_arc, &script.id, Some(did))?; + register_default_apis(&lua, &state_arc, &script.id, Some(payload.did))?; // Legacy globals (action, uri, did, collection, rkey, record) for // back-compat with scripts written against the old `index_hook` // surface. - context::set_hook_context(&lua, action, uri, did, collection, rkey, record) - .map_err(|e| format!("set hook context: {e}"))?; + context::set_hook_context( + &lua, + payload.action, + payload.uri, + payload.did, + payload.nsid, + payload.rkey, + payload.record, + ) + .map_err(|e| format!("set hook context: {e}"))?; // Also expose an `event` table — same fields, different idiom. New // scripts can read `event.action` / `event.record.title` instead of // the bare globals; both styles work. use mlua::LuaSerdeExt; let event_value = serde_json::json!({ - "action": action, - "uri": uri, - "did": did, - "collection": collection, - "rkey": rkey, - "record": record, + "action": payload.action, + "uri": payload.uri, + "did": payload.did, + "collection": payload.nsid, + "rkey": payload.rkey, + "record": payload.record, }); lua.globals() .set( @@ -459,7 +469,7 @@ pub async fn run_record_event_once( Ok(Some(v)) } // Non-nil, non-table return — pass-through: keep the original record. - _ => Ok(record.cloned()), + _ => Ok(payload.record.cloned()), } } diff --git a/src/record_handler.rs b/src/record_handler.rs index e3c5173..c35dd27 100644 --- a/src/record_handler.rs +++ b/src/record_handler.rs @@ -65,12 +65,14 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { // `record.index:`; failures are dead-lettered fail-open. let hook_result = crate::lua::run_record_event_script( state, - &record.collection, - &record.action, - &uri, - &record.did, - &record.rkey, - Some(rec), + crate::lua::RecordEventPayload { + nsid: &record.collection, + action: &record.action, + uri: &uri, + did: &record.did, + rkey: &record.rkey, + record: Some(rec), + }, ) .await; let rec_to_store = match hook_result { @@ -183,12 +185,14 @@ pub async fn handle_record_event(state: &AppState, record: &RecordEvent) { // return aborts the delete; any other return continues. let hook_result = crate::lua::run_record_event_script( state, - &record.collection, - "delete", - &uri, - &record.did, - &record.rkey, - None, + crate::lua::RecordEventPayload { + nsid: &record.collection, + action: "delete", + uri: &uri, + did: &record.did, + rkey: &record.rkey, + record: None, + }, ) .await; if hook_result.is_none() { -- 2.51.2 From 4a576ea814834f185c7a5df95d7ab34f55266bff Mon Sep 17 00:00:00 2001 From: Chris Pardy Date: Fri, 1 May 2026 11:30:25 -0400 Subject: [PATCH 48/64] fix(dead-letters): admin endpoints span both legacy + new tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trigger-keyed dispatcher writes failures to `dead_letter_scripts` (introduced in a086830). The admin endpoints under `/admin/dead-letters/*` were still hardcoded to read from the legacy `dead_letter_hooks` table, so: - The dashboard list / count silently dropped any failure from the new dispatcher (record-event scripts AND label scripts). - Per-id `retry` / `dismiss` / `reindex` returned 404 for any new id. Make the surface span both tables. Per-id operations route by id format (id parses as integer → `dead_letter_scripts`; anything else → `dead_letter_hooks`); the two id namespaces are disjoint (BIGSERIAL/INTEGER vs UUID/TEXT) so this dispatch is unambiguous. List / count run two queries and merge in Rust — `dead_letter_hooks` has flat columns while `dead_letter_scripts` carries event fields inside `payload` JSON, so a portable SQL UNION isn't worth the backend-specific JSON-extract dance. Schema mapping for `dead_letter_scripts` rows onto the existing `DeadLetterSummary` shape: - **Record events** (`host_kind = "record"`): collection / did / uri / rkey / action come from `payload.{collection,did,uri,rkey,action}`; `lexicon_id` reuses the collection. - **Label arrivals** (`host_kind = "label"`): `lexicon_id` is the trigger id (e.g. `labeler.apply:app.bsky.feed.post`); `did` is the labeler DID (`host_id`); `action` is the literal `"label"`; `rkey` is `payload.val`. `retry_single` / `reindex_single` reject label dead letters with a clear 400 — the upstream label event is gone, so re-running the script with the original payload is meaningful but reindexing isn't, and the user-visible error spells out which. Helpers `mark_resolved` / `update_error` now take a `DeadLetterSource` and bind the id as `i64` for the new table (sqlite implicit-conversion otherwise quietly fails to match). Bulk operations: - `all: true` updates / collects ids from both tables; the optional `collection` filter only constrains the legacy table (the new table doesn't have a collection column — filtering by collection there would require JSON-parsing the payload in SQL). - `ids: [...]` routes each id by source. No new tests added — the existing `e2e_admin` suite still passes (22/23; the 1 failure is `backfill_create_job` which is flaky on origin/dev too). Signed-off-by: Chris Pardy --- src/admin/dead_letters.rs | 819 ++++++++++++++++++++++++++++---------- 1 file changed, 606 insertions(+), 213 deletions(-) diff --git a/src/admin/dead_letters.rs b/src/admin/dead_letters.rs index 42c1d33..a28482c 100644 --- a/src/admin/dead_letters.rs +++ b/src/admin/dead_letters.rs @@ -1,3 +1,22 @@ +//! Admin surface for dead-lettered events. +//! +//! Two tables back this: +//! +//! - **`dead_letter_hooks`** (legacy) — written by the pre-trigger-keyed +//! indexer when a hook script exhausted retries. Columns are +//! per-event-field (lexicon_id, uri, did, collection, rkey, action, +//! record). UUID / TEXT primary keys. +//! - **`dead_letter_scripts`** (current) — written by the trigger-keyed +//! dispatcher in `crate::lua::scripts`. The event-specific fields are +//! inside `payload` (JSON). INTEGER primary keys. Carries both record +//! and label dead letters via the `host_kind` discriminator. +//! +//! Both tables are kept readable + manageable through this admin +//! surface. Per-id operations route by id format: an id that parses as +//! an integer routes to `dead_letter_scripts`, anything else (UUIDs, +//! sqlite NULL-stringified primary keys) routes to `dead_letter_hooks`. +//! The two id namespaces are disjoint so this dispatch is unambiguous. + use axum::{ Json, extract::{Path, Query, State}, @@ -13,6 +32,37 @@ use crate::error::AppError; use crate::lua::{RecordEventPayload, resolve_record_event, run_record_event_once}; use crate::record_handler::RecordEvent; +// --------------------------------------------------------------------------- +// Source enum — which table backs a given dead-letter id +// --------------------------------------------------------------------------- + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum DeadLetterSource { + /// Pre-trigger-keyed legacy hooks table. + LegacyHooks, + /// New trigger-keyed scripts table. + Scripts, +} + +impl DeadLetterSource { + /// Pick the table by id format. Integer-parseable → Scripts; + /// anything else → LegacyHooks. + fn from_id(id: &str) -> Self { + if id.parse::().is_ok() { + Self::Scripts + } else { + Self::LegacyHooks + } + } + + fn table(self) -> &'static str { + match self { + Self::LegacyHooks => "dead_letter_hooks", + Self::Scripts => "dead_letter_scripts", + } + } +} + // --------------------------------------------------------------------------- // Query / request / response types // --------------------------------------------------------------------------- @@ -73,118 +123,77 @@ pub struct BulkRequest { } /// Internal row type for fetching action data needed by retry/reindex. -#[allow(dead_code)] +/// Populated from either table by `fetch_dead_letter_for_action`. struct DeadLetterRow { id: String, - lexicon_id: String, + source: DeadLetterSource, + /// Discriminator for new-table rows: `"record"` or `"label"`. + /// Always `"record"` for legacy rows. Retries are only supported + /// for record dead letters. + host_kind: String, uri: String, did: String, collection: String, rkey: String, action: String, record: Option, - error: String, - attempts: i64, } // --------------------------------------------------------------------------- // Handlers // --------------------------------------------------------------------------- -/// GET /admin/dead-letters +/// `GET /admin/dead-letters` — list rows from both tables, merge by +/// created_at, paginate via cursor. pub(super) async fn list( auth: UserAuth, State(state): State, Query(query): Query, ) -> Result, AppError> { auth.require(Permission::DeadLettersRead).await?; - let backend = state.db_backend; let limit = query.limit.unwrap_or(50).clamp(1, 100); - - let mut sql = String::from( - "SELECT id, lexicon_id, uri, did, collection, rkey, action, error, attempts, created_at, resolved_at - FROM dead_letter_hooks WHERE 1=1", + let resolved = query.resolved.as_deref().unwrap_or("false"); + + // Fetch up to `limit` rows from each table, then merge + slice. + // Two queries instead of a SQL UNION because the schemas differ + // (legacy has columns; scripts has payload JSON we parse in Rust). + let mut rows = list_legacy( + &state, + resolved, + query.collection.as_deref(), + &query.cursor, + limit, + ) + .await?; + rows.extend( + list_scripts( + &state, + resolved, + query.collection.as_deref(), + &query.cursor, + limit, + ) + .await?, ); - let resolved_filter = query.resolved.as_deref().unwrap_or("false"); - match resolved_filter { - "false" => sql.push_str(" AND resolved_at IS NULL"), - "true" => sql.push_str(" AND resolved_at IS NOT NULL"), - _ => {} // no filter - } - - if query.collection.is_some() { - sql.push_str(" AND collection = ?"); - } - if query.cursor.is_some() { - sql.push_str(" AND created_at < ?"); - } - - sql.push_str(" ORDER BY created_at DESC LIMIT ?"); - - let sql = adapt_sql(&sql, backend); - - #[allow(clippy::type_complexity)] - let mut q = sqlx::query_as::< - _, - ( - String, - String, - String, - String, - String, - String, - String, - String, - i64, - String, - Option, - ), - >(&sql); - - if let Some(ref collection) = query.collection { - q = q.bind(collection); - } - if let Some(ref cursor) = query.cursor { - q = q.bind(cursor); - } - q = q.bind(limit); - - let rows = q - .fetch_all(&state.db) - .await - .map_err(|e| AppError::Internal(format!("failed to query dead letters: {e}")))?; + // Newest first, then truncate. + rows.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + let truncated = rows.len() as i64 > limit; + rows.truncate(limit as usize); - let dead_letters: Vec = rows - .into_iter() - .map(|row| DeadLetterSummary { - id: row.0, - lexicon_id: row.1, - uri: row.2, - did: row.3, - collection: row.4, - rkey: row.5, - action: row.6, - error: row.7, - attempts: row.8, - created_at: parse_dt(&row.9), - resolved_at: row.10.as_deref().map(parse_dt), - }) - .collect(); - - let cursor = if dead_letters.len() as i64 >= limit { - dead_letters.last().map(|dl| dl.created_at.to_rfc3339()) + let cursor = if truncated { + rows.last().map(|r| r.created_at.to_rfc3339()) } else { None }; Ok(Json(ListResponse { - dead_letters, + dead_letters: rows, cursor, })) } -/// GET /admin/dead-letters/count +/// `GET /admin/dead-letters/count` — sum of unresolved across both tables. pub(super) async fn count( auth: UserAuth, State(state): State, @@ -192,96 +201,58 @@ pub(super) async fn count( ) -> Result, AppError> { auth.require(Permission::DeadLettersRead).await?; let backend = state.db_backend; + let resolved = query.resolved.as_deref().unwrap_or("false"); + let resolved_clause = match resolved { + "false" => " AND resolved_at IS NULL", + "true" => " AND resolved_at IS NOT NULL", + _ => "", + }; - let mut sql = String::from("SELECT COUNT(*) FROM dead_letter_hooks WHERE 1=1"); - - let resolved_filter = query.resolved.as_deref().unwrap_or("false"); - match resolved_filter { - "false" => sql.push_str(" AND resolved_at IS NULL"), - "true" => sql.push_str(" AND resolved_at IS NOT NULL"), - _ => {} + let mut total: i64 = 0; + for table in [ + DeadLetterSource::LegacyHooks.table(), + DeadLetterSource::Scripts.table(), + ] { + let sql = adapt_sql( + &format!("SELECT COUNT(*) FROM {table} WHERE 1=1{resolved_clause}"), + backend, + ); + let (n,): (i64,) = sqlx::query_as(&sql) + .fetch_one(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to count dead letters: {e}")))?; + total += n; } - let sql = adapt_sql(&sql, backend); - let (count,): (i64,) = sqlx::query_as(&sql) - .fetch_one(&state.db) - .await - .map_err(|e| AppError::Internal(format!("failed to count dead letters: {e}")))?; - - Ok(Json(CountResponse { count })) + Ok(Json(CountResponse { count: total })) } -/// GET /admin/dead-letters/{id} +/// `GET /admin/dead-letters/{id}` — detail view. Routes by id format. pub(super) async fn detail( auth: UserAuth, State(state): State, Path(id): Path, ) -> Result, AppError> { auth.require(Permission::DeadLettersRead).await?; - let backend = state.db_backend; - - let sql = adapt_sql( - "SELECT id, lexicon_id, uri, did, collection, rkey, action, error, attempts, created_at, resolved_at, record - FROM dead_letter_hooks WHERE id = ?", - backend, - ); - - #[allow(clippy::type_complexity)] - let row: ( - String, - String, - String, - String, - String, - String, - String, - String, - i64, - String, - Option, - Option, - ) = sqlx::query_as(&sql) - .bind(&id) - .fetch_optional(&state.db) - .await - .map_err(|e| AppError::Internal(format!("failed to fetch dead letter: {e}")))? - .ok_or_else(|| AppError::NotFound(format!("dead letter {id} not found")))?; - - let summary = DeadLetterSummary { - id: row.0, - lexicon_id: row.1, - uri: row.2, - did: row.3, - collection: row.4, - rkey: row.5, - action: row.6, - error: row.7, - attempts: row.8, - created_at: parse_dt(&row.9), - resolved_at: row.10.as_deref().map(parse_dt), - }; - - let record = row.11.as_deref().and_then(|r| serde_json::from_str(r).ok()); - - Ok(Json(DeadLetterDetail { summary, record })) + match DeadLetterSource::from_id(&id) { + DeadLetterSource::LegacyHooks => detail_legacy(&state, &id).await.map(Json), + DeadLetterSource::Scripts => detail_scripts(&state, &id).await.map(Json), + } } -/// POST /admin/dead-letters/{id}/dismiss +/// `POST /admin/dead-letters/{id}/dismiss` pub(super) async fn dismiss( auth: UserAuth, State(state): State, Path(id): Path, ) -> Result, AppError> { auth.require(Permission::DeadLettersManage).await?; - let dl = fetch_dead_letter_for_action(&state, &id).await?; - if dl.id.is_empty() { - return Err(AppError::NotFound(format!("dead letter {id} not found"))); - } - mark_resolved(&state, &id).await?; + let source = DeadLetterSource::from_id(&id); + mark_resolved(&state, &id, source).await?; Ok(Json(serde_json::json!({ "ok": true }))) } -/// POST /admin/dead-letters/{id}/retry +/// `POST /admin/dead-letters/{id}/retry` pub(super) async fn retry( auth: UserAuth, State(state): State, @@ -292,7 +263,7 @@ pub(super) async fn retry( Ok(Json(serde_json::json!({ "ok": true }))) } -/// POST /admin/dead-letters/{id}/reindex +/// `POST /admin/dead-letters/{id}/reindex` pub(super) async fn reindex( auth: UserAuth, State(state): State, @@ -303,7 +274,7 @@ pub(super) async fn reindex( Ok(Json(serde_json::json!({ "ok": true }))) } -/// POST /admin/dead-letters/bulk/dismiss +/// `POST /admin/dead-letters/bulk/dismiss` pub(super) async fn bulk_dismiss( auth: UserAuth, State(state): State, @@ -314,31 +285,32 @@ pub(super) async fn bulk_dismiss( let now = now_rfc3339(); if body.all == Some(true) { - let mut sql = - String::from("UPDATE dead_letter_hooks SET resolved_at = ? WHERE resolved_at IS NULL"); - if body.collection.is_some() { - sql.push_str(" AND collection = ?"); - } - let sql = adapt_sql(&sql, backend); - let mut q = sqlx::query(&sql).bind(&now); - if let Some(ref collection) = body.collection { - q = q.bind(collection); + // Operate against both tables. Collection filter only applies + // to legacy rows (the new `dead_letter_scripts` doesn't have a + // collection column — to filter by collection there we'd need + // to JSON-parse `payload`, which is portable-SQL pain. Good + // enough: legacy table is the one with bulk-by-collection + // history anyway.). + for source in [DeadLetterSource::LegacyHooks, DeadLetterSource::Scripts] { + let table = source.table(); + let mut sql = format!("UPDATE {table} SET resolved_at = ? WHERE resolved_at IS NULL"); + let collection_filter = + source == DeadLetterSource::LegacyHooks && body.collection.is_some(); + if collection_filter { + sql.push_str(" AND collection = ?"); + } + let sql = adapt_sql(&sql, backend); + let mut q = sqlx::query(&sql).bind(&now); + if collection_filter && let Some(ref c) = body.collection { + q = q.bind(c); + } + q.execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("bulk dismiss failed: {e}")))?; } - q.execute(&state.db) - .await - .map_err(|e| AppError::Internal(format!("bulk dismiss failed: {e}")))?; } else if let Some(ref ids) = body.ids { for id in ids { - let sql = adapt_sql( - "UPDATE dead_letter_hooks SET resolved_at = ? WHERE id = ? AND resolved_at IS NULL", - backend, - ); - sqlx::query(&sql) - .bind(&now) - .bind(id) - .execute(&state.db) - .await - .map_err(|e| AppError::Internal(format!("bulk dismiss failed for {id}: {e}")))?; + mark_resolved(&state, id, DeadLetterSource::from_id(id)).await?; } } else { return Err(AppError::BadRequest( @@ -349,7 +321,7 @@ pub(super) async fn bulk_dismiss( Ok(Json(serde_json::json!({ "ok": true }))) } -/// POST /admin/dead-letters/bulk/retry +/// `POST /admin/dead-letters/bulk/retry` pub(super) async fn bulk_retry( auth: UserAuth, State(state): State, @@ -363,7 +335,7 @@ pub(super) async fn bulk_retry( Ok(Json(serde_json::json!({ "ok": true }))) } -/// POST /admin/dead-letters/bulk/reindex +/// `POST /admin/dead-letters/bulk/reindex` pub(super) async fn bulk_reindex( auth: UserAuth, State(state): State, @@ -378,21 +350,234 @@ pub(super) async fn bulk_reindex( } // --------------------------------------------------------------------------- -// Helper functions +// Per-table list / detail // --------------------------------------------------------------------------- -/// Fetch an unresolved dead letter by ID, returning an error if not found or already resolved. -async fn fetch_dead_letter_for_action( +async fn list_legacy( + state: &AppState, + resolved: &str, + collection: Option<&str>, + cursor: &Option, + limit: i64, +) -> Result, AppError> { + let backend = state.db_backend; + let mut sql = String::from( + "SELECT id, lexicon_id, uri, did, collection, rkey, action, error, attempts, created_at, resolved_at + FROM dead_letter_hooks WHERE 1=1", + ); + match resolved { + "false" => sql.push_str(" AND resolved_at IS NULL"), + "true" => sql.push_str(" AND resolved_at IS NOT NULL"), + _ => {} + } + if collection.is_some() { + sql.push_str(" AND collection = ?"); + } + if cursor.is_some() { + sql.push_str(" AND created_at < ?"); + } + sql.push_str(" ORDER BY created_at DESC LIMIT ?"); + + let sql = adapt_sql(&sql, backend); + #[allow(clippy::type_complexity)] + let mut q = sqlx::query_as::< + _, + ( + String, + String, + String, + String, + String, + String, + String, + String, + i64, + String, + Option, + ), + >(&sql); + if let Some(c) = collection { + q = q.bind(c); + } + if let Some(cur) = cursor { + q = q.bind(cur); + } + q = q.bind(limit); + + let rows = q + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query legacy dead letters: {e}")))?; + + Ok(rows + .into_iter() + .map(|row| DeadLetterSummary { + id: row.0, + lexicon_id: row.1, + uri: row.2, + did: row.3, + collection: row.4, + rkey: row.5, + action: row.6, + error: row.7, + attempts: row.8, + created_at: parse_dt(&row.9), + resolved_at: row.10.as_deref().map(parse_dt), + }) + .collect()) +} + +async fn list_scripts( state: &AppState, + resolved: &str, + collection: Option<&str>, + cursor: &Option, + limit: i64, +) -> Result, AppError> { + let backend = state.db_backend; + let mut sql = String::from( + "SELECT id, script_ref, host_kind, host_id, payload, error, attempts, created_at, resolved_at + FROM dead_letter_scripts WHERE 1=1", + ); + match resolved { + "false" => sql.push_str(" AND resolved_at IS NULL"), + "true" => sql.push_str(" AND resolved_at IS NOT NULL"), + _ => {} + } + if cursor.is_some() { + sql.push_str(" AND created_at < ?"); + } + sql.push_str(" ORDER BY created_at DESC LIMIT ?"); + + let sql = adapt_sql(&sql, backend); + #[allow(clippy::type_complexity)] + let mut q = sqlx::query_as::< + _, + ( + i64, + String, + String, + String, + String, + String, + i64, + String, + Option, + ), + >(&sql); + if let Some(cur) = cursor { + q = q.bind(cur); + } + q = q.bind(limit); + + let rows = q + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to query scripts dead letters: {e}")))?; + + let summaries: Vec = rows + .into_iter() + .map(|row| { + summary_from_scripts_row( + &row.0.to_string(), + &row.1, + &row.2, + &row.3, + &row.4, + &row.5, + row.6, + &row.7, + row.8.as_deref(), + ) + }) + .collect(); + + // Filter by collection in Rust since the column lives inside + // `payload`. Negligible cost — already client-side after the + // unfiltered DB fetch. + Ok(if let Some(want) = collection { + summaries + .into_iter() + .filter(|s| s.collection == want) + .collect() + } else { + summaries + }) +} + +#[allow(clippy::too_many_arguments)] +fn summary_from_scripts_row( id: &str, -) -> Result { + script_ref: &str, + host_kind: &str, + host_id: &str, + payload: &str, + error: &str, + attempts: i64, + created_at: &str, + resolved_at: Option<&str>, +) -> DeadLetterSummary { + let payload_v: Value = serde_json::from_str(payload).unwrap_or(Value::Null); + let s = |key: &str| { + payload_v + .get(key) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + + // Map the trigger-keyed payload onto the legacy DeadLetterSummary + // shape. Record-event dead letters fit cleanly. Label-arrival dead + // letters reuse the slots: did = labeler DID (host_id), action = + // "label", rkey = label.val, collection extracted from the trigger + // suffix when possible. + let (lexicon_id, collection, did, rkey, action) = match host_kind { + "label" => { + let collection_from_trigger = script_ref + .split_once(':') + .map(|(_, suf)| suf.to_string()) + .unwrap_or_default(); + ( + script_ref.to_string(), + collection_from_trigger, + host_id.to_string(), + s("val"), + "label".to_string(), + ) + } + _ => ( + s("collection"), + s("collection"), + s("did"), + s("rkey"), + s("action"), + ), + }; + + DeadLetterSummary { + id: id.to_string(), + lexicon_id, + uri: s("uri"), + did, + collection, + rkey, + action, + error: error.to_string(), + attempts, + created_at: parse_dt(created_at), + resolved_at: resolved_at.map(parse_dt), + } +} + +async fn detail_legacy(state: &AppState, id: &str) -> Result { let backend = state.db_backend; let sql = adapt_sql( - "SELECT id, lexicon_id, uri, did, collection, rkey, action, record, error, attempts - FROM dead_letter_hooks WHERE id = ? AND resolved_at IS NULL", + "SELECT id, lexicon_id, uri, did, collection, rkey, action, error, attempts, created_at, resolved_at, record + FROM dead_letter_hooks WHERE id = ?", backend, ); + #[allow(clippy::type_complexity)] let row: ( String, String, @@ -401,19 +586,19 @@ async fn fetch_dead_letter_for_action( String, String, String, - Option, String, i64, + String, + Option, + Option, ) = sqlx::query_as(&sql) .bind(id) .fetch_optional(&state.db) .await .map_err(|e| AppError::Internal(format!("failed to fetch dead letter: {e}")))? - .ok_or_else(|| { - AppError::NotFound(format!("dead letter {id} not found or already resolved")) - })?; + .ok_or_else(|| AppError::NotFound(format!("dead letter {id} not found")))?; - Ok(DeadLetterRow { + let summary = DeadLetterSummary { id: row.0, lexicon_id: row.1, uri: row.2, @@ -421,63 +606,253 @@ async fn fetch_dead_letter_for_action( collection: row.4, rkey: row.5, action: row.6, - record: row.7, - error: row.8, - attempts: row.9, - }) + error: row.7, + attempts: row.8, + created_at: parse_dt(&row.9), + resolved_at: row.10.as_deref().map(parse_dt), + }; + + let record = row.11.as_deref().and_then(|r| serde_json::from_str(r).ok()); + Ok(DeadLetterDetail { summary, record }) +} + +async fn detail_scripts(state: &AppState, id: &str) -> Result { + let backend = state.db_backend; + let sql = adapt_sql( + "SELECT id, script_ref, host_kind, host_id, payload, error, attempts, created_at, resolved_at + FROM dead_letter_scripts WHERE id = ?", + backend, + ); + let id_int: i64 = id + .parse() + .map_err(|_| AppError::NotFound(format!("dead letter {id} not found")))?; + + #[allow(clippy::type_complexity)] + let row: ( + i64, + String, + String, + String, + String, + String, + i64, + String, + Option, + ) = sqlx::query_as(&sql) + .bind(id_int) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to fetch dead letter: {e}")))? + .ok_or_else(|| AppError::NotFound(format!("dead letter {id} not found")))?; + + let summary = summary_from_scripts_row( + &row.0.to_string(), + &row.1, + &row.2, + &row.3, + &row.4, + &row.5, + row.6, + &row.7, + row.8.as_deref(), + ); + + let payload_v: Value = serde_json::from_str(&row.4).unwrap_or(Value::Null); + // For record events the original record body lives at `payload.record`. + // For label events the entire payload is the event; surface it whole. + let record = if row.2 == "label" { + Some(payload_v.clone()) + } else { + payload_v.get("record").cloned() + }; + + Ok(DeadLetterDetail { summary, record }) +} + +// --------------------------------------------------------------------------- +// Per-row helpers (retry, reindex, mark resolved, update error) +// --------------------------------------------------------------------------- + +/// Fetch an unresolved dead letter from whichever table holds it. +async fn fetch_dead_letter_for_action( + state: &AppState, + id: &str, +) -> Result { + let source = DeadLetterSource::from_id(id); + match source { + DeadLetterSource::LegacyHooks => { + let backend = state.db_backend; + let sql = adapt_sql( + "SELECT id, lexicon_id, uri, did, collection, rkey, action, record, error, attempts + FROM dead_letter_hooks WHERE id = ? AND resolved_at IS NULL", + backend, + ); + #[allow(clippy::type_complexity)] + let row: ( + String, + String, + String, + String, + String, + String, + String, + Option, + String, + i64, + ) = sqlx::query_as(&sql) + .bind(id) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to fetch dead letter: {e}")))? + .ok_or_else(|| { + AppError::NotFound(format!("dead letter {id} not found or already resolved")) + })?; + Ok(DeadLetterRow { + id: row.0, + source, + host_kind: "record".to_string(), + uri: row.2, + did: row.3, + collection: row.4, + rkey: row.5, + action: row.6, + record: row.7, + }) + } + DeadLetterSource::Scripts => { + let backend = state.db_backend; + let id_int: i64 = id.parse().unwrap_or_default(); + let sql = adapt_sql( + "SELECT id, host_kind, payload FROM dead_letter_scripts + WHERE id = ? AND resolved_at IS NULL", + backend, + ); + let row: (i64, String, String) = sqlx::query_as(&sql) + .bind(id_int) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to fetch dead letter: {e}")))? + .ok_or_else(|| { + AppError::NotFound(format!("dead letter {id} not found or already resolved")) + })?; + let payload_v: Value = serde_json::from_str(&row.2).unwrap_or(Value::Null); + let s = |k: &str| { + payload_v + .get(k) + .and_then(|v| v.as_str()) + .unwrap_or_default() + .to_string() + }; + // The record body is nested under `payload.record` for + // record events; serialize it back to a string for the + // retry call site. + let record = payload_v + .get("record") + .filter(|v| !v.is_null()) + .map(|v| v.to_string()); + Ok(DeadLetterRow { + id: row.0.to_string(), + source, + host_kind: row.1.clone(), + uri: s("uri"), + did: s("did"), + collection: s("collection"), + rkey: s("rkey"), + action: s("action"), + record, + }) + } + } } -/// Mark a dead letter as resolved. -async fn mark_resolved(state: &AppState, id: &str) -> Result<(), AppError> { +async fn mark_resolved( + state: &AppState, + id: &str, + source: DeadLetterSource, +) -> Result<(), AppError> { let backend = state.db_backend; let now = now_rfc3339(); + let table = source.table(); let sql = adapt_sql( - "UPDATE dead_letter_hooks SET resolved_at = ? WHERE id = ?", + &format!("UPDATE {table} SET resolved_at = ? WHERE id = ?"), backend, ); - sqlx::query(&sql) - .bind(&now) - .bind(id) - .execute(&state.db) + let q = sqlx::query(&sql).bind(&now); + let q = match source { + // Scripts table has INTEGER ids; bind as i64 to avoid sqlite's + // implicit-conversion quirks. + DeadLetterSource::Scripts => q.bind(id.parse::().unwrap_or(0)), + DeadLetterSource::LegacyHooks => q.bind(id), + }; + q.execute(&state.db) .await .map_err(|e| AppError::Internal(format!("failed to mark dead letter resolved: {e}")))?; Ok(()) } -/// Update the error message and increment attempts. -async fn update_error(state: &AppState, id: &str, error: &str) -> Result<(), AppError> { +async fn update_error( + state: &AppState, + id: &str, + source: DeadLetterSource, + error: &str, +) -> Result<(), AppError> { let backend = state.db_backend; + let table = source.table(); let sql = adapt_sql( - "UPDATE dead_letter_hooks SET error = ?, attempts = attempts + 1 WHERE id = ?", + &format!("UPDATE {table} SET error = ?, attempts = attempts + 1 WHERE id = ?"), backend, ); - sqlx::query(&sql) - .bind(error) - .bind(id) - .execute(&state.db) + let q = sqlx::query(&sql).bind(error); + let q = match source { + DeadLetterSource::Scripts => q.bind(id.parse::().unwrap_or(0)), + DeadLetterSource::LegacyHooks => q.bind(id), + }; + q.execute(&state.db) .await .map_err(|e| AppError::Internal(format!("failed to update dead letter error: {e}")))?; Ok(()) } -/// Resolve a BulkRequest into a list of dead letter IDs. async fn resolve_bulk_ids(state: &AppState, body: &BulkRequest) -> Result, AppError> { if body.all == Some(true) { let backend = state.db_backend; + let mut ids: Vec = Vec::new(); + + // Legacy table — supports the optional collection filter. let mut sql = String::from("SELECT id FROM dead_letter_hooks WHERE resolved_at IS NULL"); if body.collection.is_some() { sql.push_str(" AND collection = ?"); } let sql = adapt_sql(&sql, backend); let mut q = sqlx::query_as::<_, (String,)>(&sql); - if let Some(ref collection) = body.collection { - q = q.bind(collection); + if let Some(ref c) = body.collection { + q = q.bind(c); } let rows = q .fetch_all(&state.db) .await .map_err(|e| AppError::Internal(format!("failed to resolve bulk ids: {e}")))?; - Ok(rows.into_iter().map(|r| r.0).collect()) + ids.extend(rows.into_iter().map(|r| r.0)); + + // New table — collection isn't a column; if a filter was + // requested, only include rows whose payload.collection matches. + // Simpler to just include all unresolved when no collection + // filter is set, and skip the new table entirely when one is + // (the legacy table is the one historically pinned to a + // collection anyway). + if body.collection.is_none() { + let sql = adapt_sql( + "SELECT id FROM dead_letter_scripts WHERE resolved_at IS NULL", + backend, + ); + let rows = sqlx::query_as::<_, (i64,)>(&sql) + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to resolve bulk ids: {e}")))?; + ids.extend(rows.into_iter().map(|r| r.0.to_string())); + } + + Ok(ids) } else if let Some(ref ids) = body.ids { Ok(ids.clone()) } else { @@ -490,12 +865,21 @@ async fn resolve_bulk_ids(state: &AppState, body: &BulkRequest) -> Result:` → `record.index:`). If no script is -/// bound for the cascade now, returns 404 — the operator either deleted -/// the script or never re-bound it under the new naming. +/// (`record.:` → `record.index:`). Label-arrival +/// dead letters are not retried (the upstream label is gone; there's +/// nothing to feed back into the runner). Caller gets a 400. async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { let dl = fetch_dead_letter_for_action(state, id).await?; + if dl.host_kind == "label" { + return Err(AppError::BadRequest( + "label-arrival dead letters can't be retried — the upstream label \ + event is gone. Dismiss this row and let the labeler subscription \ + redeliver if needed." + .into(), + )); + } + let resolved = resolve_record_event(state, &dl.collection, &dl.action) .await .ok_or_else(|| { @@ -525,11 +909,11 @@ async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { .await { Ok(_) => { - mark_resolved(state, id).await?; + mark_resolved(state, &dl.id, dl.source).await?; Ok(()) } Err(e) => { - update_error(state, id, &e).await?; + update_error(state, &dl.id, dl.source, &e).await?; Err(AppError::Internal(format!( "retry failed for dead letter {id}: {e}" ))) @@ -537,10 +921,19 @@ async fn retry_single(state: &AppState, id: &str) -> Result<(), AppError> { } } -/// Reindex a single dead letter by fetching the record fresh from the PDS. +/// Reindex by fetching the record fresh from the PDS. Only applies to +/// record-event dead letters. async fn reindex_single(state: &AppState, id: &str) -> Result<(), AppError> { let dl = fetch_dead_letter_for_action(state, id).await?; + if dl.host_kind == "label" { + return Err(AppError::BadRequest( + "label-arrival dead letters can't be reindexed — they don't have \ + a record to fetch." + .into(), + )); + } + let pds_endpoint = crate::profile::resolve_pds_endpoint(&state.http, &state.config.plc_url, &dl.did).await?; @@ -585,7 +978,7 @@ async fn reindex_single(state: &AppState, id: &str) -> Result<(), AppError> { }; crate::record_handler::handle_record_event(state, &event).await; - mark_resolved(state, id).await?; + mark_resolved(state, &dl.id, dl.source).await?; Ok(()) } -- 2.51.2 From 735e1047c42db5fa550dbe6fd330b54c181a43d0 Mon Sep 17 00:00:00 2001 From: Chris Pardy Date: Fri, 1 May 2026 07:56:09 -0400 Subject: [PATCH 49/64] feat(dashboard): scripts grouped by trigger family + lexicon targeting panel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Frontend for the trigger-keyed scripts subsystem (commit a086830). The dashboard now exposes: - `/dashboard/settings/scripts` — grouped list of all scripts by trigger family (Record events, XRPC handlers, Label arrivals). The script's `id` IS its trigger string; the table makes that visible in monospace and links to the detail page. - `/dashboard/settings/scripts/new` — create form. The trigger id is composed via a `(kind, suffix)` picker (kind dropdown for the prefix family + free-text suffix for the NSID). The Lua body prefills with a `handle()` skeleton. The form accepts a `?id=` URL param so the lexicon detail page can deep-link with a candidate id pre-filled. - `/dashboard/settings/scripts/[id]` — detail/edit. The id is the PK so it's locked here; rename = delete + recreate. The header shows the cascade hint for record events ("`record.create:` fires only on this action; cascades to `record.index:` if absent."). The lexicon detail page gets a new "Scripts targeting this lexicon" panel that lists the trigger ids relevant to the lexicon's type: - Record-type lexicons show the four `record.*` slots (`record.index` / `record.create` / `record.update` / `record.delete`) with Edit/Create links. - XRPC lexicons show the matching `xrpc.{query,procedure}` slot. Existing scripts link straight to the detail page; missing rows link to the New Script page pre-filled with the trigger id. Operators get a one-click path from a lexicon to the scripts that affect it. The legacy `index_hook` editor on the lexicon page is left in place for now — operators with existing data can still see and edit the column, even though the runtime no longer reads it. A follow-up commit can drop the editor once we're confident no one relies on it. New: `web/src/types/scripts.ts` (Script, UpsertScriptBody, PatchScriptBody, TriggerKind / TriggerFamily helpers, parseTriggerId, DEFAULT_SCRIPT_BODY). Sidebar gets a Scripts nav item under Integrations (`scripts:read`-gated). Signed-off-by: Chris Pardy --- src/admin/backfill.rs | 14 +- .../lexicons/[id]/lexicon-detail.tsx | 273 ++++++++++----- .../dashboard/settings/scripts/[id]/page.tsx | 11 + .../settings/scripts/[id]/script-detail.tsx | 188 +++++++++++ .../dashboard/settings/scripts/new/page.tsx | 125 +++++++ .../app/dashboard/settings/scripts/page.tsx | 181 ++++++++++ .../settings/scripts/script-form.tsx | 315 ++++++++++++++++++ web/src/components/app-sidebar.tsx | 7 + web/src/lib/api.ts | 45 +++ web/src/types/scripts.ts | 135 ++++++++ 10 files changed, 1196 insertions(+), 98 deletions(-) create mode 100644 web/src/app/dashboard/settings/scripts/[id]/page.tsx create mode 100644 web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx create mode 100644 web/src/app/dashboard/settings/scripts/new/page.tsx create mode 100644 web/src/app/dashboard/settings/scripts/page.tsx create mode 100644 web/src/app/dashboard/settings/scripts/script-form.tsx create mode 100644 web/src/types/scripts.ts diff --git a/src/admin/backfill.rs b/src/admin/backfill.rs index 4300ae1..5199c9c 100644 --- a/src/admin/backfill.rs +++ b/src/admin/backfill.rs @@ -1349,12 +1349,14 @@ async fn fetch_records_from_pds( let rec_to_store = match crate::lua::run_record_event_script( state, - collection, - "create", - &uri, - did, - &rkey, - Some(&entry.value), + crate::lua::RecordEventPayload { + nsid: collection, + action: "create", + uri: &uri, + did, + rkey: &rkey, + record: Some(&entry.value), + }, ) .await { diff --git a/web/src/app/dashboard/lexicons/[id]/lexicon-detail.tsx b/web/src/app/dashboard/lexicons/[id]/lexicon-detail.tsx index a8452cd..bb20a21 100644 --- a/web/src/app/dashboard/lexicons/[id]/lexicon-detail.tsx +++ b/web/src/app/dashboard/lexicons/[id]/lexicon-detail.tsx @@ -1,6 +1,8 @@ "use client"; import { useCallback, useEffect, useState } from "react"; +import { IconPlus } from "@tabler/icons-react"; +import Link from "next/link"; import { usePathname, useRouter } from "next/navigation"; import { useCurrentUser } from "@/hooks/use-current-user"; @@ -9,26 +11,22 @@ import { deleteLexicon, deleteNetworkLexicon, getLexicon, + getScripts, uploadLexicon, } from "@/lib/api"; import type { LexiconDetail } from "@/types/lexicons"; -import { - indexHookScript, - procedureScript, - queryScript, -} from "@/lib/lua-templates"; -import { useLuaCompletions } from "@/hooks/use-lua-completions"; +import type { Script, TriggerKind } from "@/types/scripts"; import { SiteHeader } from "@/components/site-header"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; -import { - Tooltip, - TooltipContent, - TooltipProvider, - TooltipTrigger, -} from "@/components/ui/tooltip"; export default function LexiconDetailPage() { const pathname = usePathname(); @@ -38,51 +36,39 @@ export default function LexiconDetailPage() { const { hasPermission } = useCurrentUser(); const router = useRouter(); const [lexicon, setLexicon] = useState(null); + // All scripts in the system — we filter to those targeting this + // lexicon's id below. Best-effort: render an empty panel if the + // scripts call fails so the rest of the page still works. + const [scripts, setScripts] = useState([]); const [error, setError] = useState(null); const [deleting, setDeleting] = useState(false); const [saving, setSaving] = useState(false); - // Editable text state + // Editable text state. The lexicon page used to edit `script` and + // `index_hook` columns inline via lua editors; those columns are + // now managed via the Scripts subsystem (see "Scripts targeting + // this lexicon" panel below). We pass the existing values through + // unchanged on save so legacy data isn't accidentally NULLed. const [jsonText, setJsonText] = useState(""); - const [luaText, setLuaText] = useState(""); const [originalJson, setOriginalJson] = useState(""); - const [originalLua, setOriginalLua] = useState(""); - const [hookText, setHookText] = useState(""); - const [originalHook, setOriginalHook] = useState(""); - const [showHookEditor, setShowHookEditor] = useState(false); const [tokenCost, setTokenCost] = useState(""); const [originalTokenCost, setOriginalTokenCost] = useState(""); - const { luaCompletions, collections } = useLuaCompletions(jsonText); const load = useCallback(() => { + // Fire both requests in parallel; the scripts list is best-effort. + getScripts() + .then(setScripts) + .catch(() => setScripts([])); getLexicon(id) .then((lex) => { setLexicon(lex); const json = JSON.stringify(lex.lexicon_json, null, 2); setJsonText(json); setOriginalJson(json); - - // If lexicon has no script but is a query/procedure, auto-generate one - if ( - !lex.script && - (lex.lexicon_type === "query" || lex.lexicon_type === "procedure") - ) { - const generated = - lex.lexicon_type === "procedure" - ? procedureScript(lex.target_collection ?? "") - : queryScript(lex.target_collection ?? ""); - setLuaText(generated); - // Set originalLua to "" so isDirty becomes true, prompting user to save - setOriginalLua(""); - } else { - setLuaText(lex.script ?? ""); - setOriginalLua(lex.script ?? ""); - } - setHookText(lex.index_hook ?? ""); - setOriginalHook(lex.index_hook ?? ""); - setShowHookEditor(!!lex.index_hook); setTokenCost(lex.token_cost != null ? String(lex.token_cost) : ""); - setOriginalTokenCost(lex.token_cost != null ? String(lex.token_cost) : ""); + setOriginalTokenCost( + lex.token_cost != null ? String(lex.token_cost) : "", + ); }) .catch((e) => setError(e instanceof Error ? e.message : String(e))); }, [id]); @@ -92,10 +78,7 @@ export default function LexiconDetailPage() { }, [load]); const isDirty = - jsonText !== originalJson || - luaText !== originalLua || - hookText !== originalHook || - tokenCost !== originalTokenCost; + jsonText !== originalJson || tokenCost !== originalTokenCost; async function handleSave() { if (!lexicon) return; @@ -106,8 +89,11 @@ export default function LexiconDetailPage() { await uploadLexicon({ lexicon_json: lexiconJson, backfill: lexicon.backfill, - script: luaText || undefined, - index_hook: hookText || undefined, + // Preserve any legacy script / index_hook values verbatim — we + // no longer edit them here, but leaving them out of the body + // would NULL the columns on upsert and lose data. + script: lexicon.script ?? undefined, + index_hook: lexicon.index_hook ?? undefined, token_cost: tokenCost ? Number(tokenCost) : null, }); load(); @@ -157,12 +143,28 @@ export default function LexiconDetailPage() { } const isNetwork = lexicon.source === "network"; - const showLua = - lexicon.has_script || - lexicon.lexicon_type === "query" || - lexicon.lexicon_type === "procedure"; const isRecord = lexicon.lexicon_type === "record"; - const showHook = isRecord && (showHookEditor || !!lexicon.index_hook); + + // Triggers that target this lexicon's id, given its type. + // + // Record-type lexicons get the four `record.*` slots (the cascade + // wildcard `record.index` listed first as the most common starting + // point, then the three action-specific slots), plus `labeler.apply` + // for "react to labels arriving on records of this type." + // XRPC lexicons get the matching `xrpc.{query,procedure}` slot only. + const targetingTriggers: { kind: TriggerKind; label: string }[] = isRecord + ? [ + { kind: "record.index", label: "Default handler (any action)" }, + { kind: "record.create", label: "On create" }, + { kind: "record.update", label: "On update" }, + { kind: "record.delete", label: "On delete" }, + { kind: "labeler.apply", label: "On label applied" }, + ] + : lexicon.lexicon_type === "query" + ? [{ kind: "xrpc.query", label: "Query handler" }] + : lexicon.lexicon_type === "procedure" + ? [{ kind: "xrpc.procedure", label: "Procedure handler" }] + : []; return (
@@ -244,20 +246,30 @@ export default function LexiconDetailPage() {
)}
+ + {/* Trigger-keyed scripts that target this lexicon. Each row + either links to the existing script or to the New Script + page pre-filled with the trigger id. */} + {targetingTriggers.length > 0 && ( + + )}
- {/* Code Panels */} + {/* JSON editor only — scripts (record-event handlers, XRPC + handlers, label-arrival handlers) are managed via the + "Scripts targeting this lexicon" panel above. The legacy + `script` / `index_hook` columns on the lexicons table are + preserved as-is on save but no longer edited here. */} {/* Actions */} @@ -273,40 +285,6 @@ export default function LexiconDetailPage() { )}
- {hasPermission("lexicons:create") && isRecord && !showHook && ( - - - - - - - { - "An Index Hook is a Lua script that runs automatically whenever a record in this collection is created, updated, or deleted on the network." - } - - - - )} - {hasPermission("lexicons:create") && isRecord && showHook && ( - - )} - {hasPermission("lexicons:create") && (
); } + +/** + * Lists scripts targeting this lexicon and offers a "+ New" dropdown + * for the slots that don't yet have a script. + * + * Existing scripts (those whose trigger id matches one of `entries`) + * appear as rows linking to the Scripts detail page. Missing slots + * are surfaced via a single "+ New script" dropdown so the operator + * picks the kind of handler they want without seeing four "Create" + * buttons stacked. The dropdown hides when every slot is taken. + */ +function ScriptsTargetingPanel({ + lexiconId, + scripts, + entries, + canManage, +}: { + lexiconId: string; + scripts: Script[]; + entries: { kind: TriggerKind; label: string }[]; + canManage: boolean; +}) { + const byId = new Map(scripts.map((s) => [s.id, s])); + const existing = entries + .map((e) => ({ ...e, triggerId: `${e.kind}:${lexiconId}` })) + .filter((e) => byId.has(e.triggerId)); + const available = entries + .map((e) => ({ ...e, triggerId: `${e.kind}:${lexiconId}` })) + .filter((e) => !byId.has(e.triggerId)); + + return ( +
+
+
+

+ Scripts targeting this lexicon +

+

+ Each row is a{" "} + + trigger + {" "} + the dispatcher resolves at firing time. +

+
+ {canManage && available.length > 0 && ( + + + + + + {available.map(({ kind, label, triggerId }) => ( + + + {label} + + {triggerId} + + + + ))} + + + )} +
+ {existing.length === 0 ? ( +

+ No scripts yet. + {canManage && available.length > 0 && ( + <> Use the “New script” menu to add one. + )} +

+ ) : ( +
    + {existing.map(({ kind, label, triggerId }) => ( +
  • + + {label} + + {triggerId} + + + +
    • + ))} +
+ )} +
+ ); +} diff --git a/web/src/app/dashboard/settings/scripts/[id]/page.tsx b/web/src/app/dashboard/settings/scripts/[id]/page.tsx new file mode 100644 index 0000000..6dd8586 --- /dev/null +++ b/web/src/app/dashboard/settings/scripts/[id]/page.tsx @@ -0,0 +1,11 @@ +import ScriptDetail from "./script-detail"; + +// https://github.com/vercel/next.js/issues/71862 +// Returning [] fails with output:"export", so provide a dummy param. +export async function generateStaticParams() { + return [{ id: "_" }]; +} + +export default function ScriptDetailPage() { + return ; +} diff --git a/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx new file mode 100644 index 0000000..cfd207a --- /dev/null +++ b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx @@ -0,0 +1,188 @@ +"use client"; + +import { useCallback, useEffect, useMemo, useState } from "react"; +import { usePathname, useRouter } from "next/navigation"; + +import { useCurrentUser } from "@/hooks/use-current-user"; +import { deleteScript, getScript, patchScript } from "@/lib/api"; +import type { Script, TriggerFamily } from "@/types/scripts"; +import { + TRIGGER_KIND_LABELS, + familyOf, + parseTriggerId, +} from "@/types/scripts"; +import { SiteHeader } from "@/components/site-header"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; + +import { + ScriptForm, + type ScriptFormState, + stateFromScript, +} from "../script-form"; + +export default function ScriptDetail() { + const pathname = usePathname(); + // The `[id]` route segment carries the URL-encoded trigger id. + // Decode once so all downstream calls see the canonical id (which + // contains `:` and `.`). + const id = decodeURIComponent( + pathname.split("/").filter(Boolean).pop() ?? "", + ); + const { hasPermission } = useCurrentUser(); + const router = useRouter(); + const [script, setScript] = useState