From 815843f681736aeee628532e93fe31df8597f073 Mon Sep 17 00:00:00 2001 From: Trezy Date: Tue, 7 Jul 2026 12:32:43 -0500 Subject: [PATCH] fix: prevent db.raw access to sensitive tables Signed-off-by: Trezy --- Cargo.lock | 44 ++++ Cargo.toml | 1 + .../docs/api-reference/lua/database-api.md | 27 ++- src/lua/db_api.rs | 192 +++++++++++++++++- tests/e2e_scripts.rs | 16 +- tests/lua_db_api.rs | 35 +++- web/src/lib/lua-hover.ts | 2 +- 7 files changed, 294 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e244f8b..c1db34b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1740,6 +1740,7 @@ dependencies = [ "serde_json", "serial_test", "sha2", + "sqlparser", "sqlx", "thiserror 2.0.18", "tokio", @@ -3173,6 +3174,26 @@ dependencies = [ "yasna", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3799,6 +3820,16 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", +] + [[package]] name = "sqlx" version = "0.8.6" @@ -3999,6 +4030,19 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + [[package]] name = "stringprep" version = "0.1.5" diff --git a/Cargo.toml b/Cargo.toml index 9bb091e..c4f3023 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ async-stream = "0.3.6" blake3 = "1" hkdf = "0.12" hmac = "0.12" +sqlparser = "0.62.0" [[bin]] name = "migrate-lua-sql" diff --git a/packages/docs/content/docs/api-reference/lua/database-api.md b/packages/docs/content/docs/api-reference/lua/database-api.md index 83ec672..cc6a9ce 100644 --- a/packages/docs/content/docs/api-reference/lua/database-api.md +++ b/packages/docs/content/docs/api-reference/lua/database-api.md @@ -148,10 +148,10 @@ local n = db.count("xyz.statusphere.status", "did:plc:abc") -- filter by DID ## db.raw -Run a raw SQL query against the database. Supports `SELECT`, `INSERT`, `UPDATE`, `DELETE`, and `CREATE TABLE` statements. +Run a raw SQL query against the database. Supports `SELECT`, `INSERT`, `UPDATE`, `DELETE`, and `CREATE TABLE` statements — use it for your own tables and to reach the record index directly. ```lua --- Read query +-- Read the record index local rows = db.raw( "SELECT uri, did, record FROM happyview_records WHERE collection = $1 AND did = $2 LIMIT $3", { "xyz.statusphere.status", "did:plc:abc", 10 } @@ -161,7 +161,7 @@ for _, row in ipairs(rows) do -- row.uri, row.did, row.record (JSONB is returned as a Lua table) end --- Write query (returns affected rows, if any) +-- Create and use your own tables db.raw("CREATE TABLE IF NOT EXISTS my_table (id TEXT PRIMARY KEY, value TEXT NOT NULL)") db.raw("INSERT INTO my_table (id, value) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET value = $2", { "key1", "hello" }) @@ -169,6 +169,23 @@ db.raw("INSERT INTO my_table (id, value) VALUES ($1, $2) ON CONFLICT (id) DO UPD Parameters are passed as an array and bound to `$1`, `$2`, etc. Supported parameter types: strings, integers, numbers, booleans, and nil. +### Protected tables + +`db.raw` blocks HappyView's **sensitive internal tables** — a statement that references one is rejected before it runs. Blocked tables cover instance secrets and tokens (OAuth/DPoP keys and sessions, API keys/clients, `happyview_script_variables`), auth and privilege state (users, permissions, delegation), trust config (domains, instance settings), and cryptographic material (space credentials and repo state). Internal tables are blocked **by default**, so anything not on the allowlist below is protected. + +Available internal tables: + +| Table | Contents | +| --- | --- | +| `happyview_records` | indexed AT Protocol records | +| `happyview_record_refs` | backlink index | +| `happyview_labels` | applied labels | +| `happyview_lexicons` | uploaded lexicons | +| `happyview_jobs` | background job queue | +| `happyview_spaces`, `happyview_space_members`, `happyview_space_records`, `happyview_space_record_oplog`, `happyview_space_notify_registrations`, `happyview_space_dids` | space membership and data | + +Space data is available because a space defines *access*, not confidentiality; if you need record data without exposing internals, the structured accessors [`db.query`](#dbquery), [`db.get`](#dbget), and [`db.count`](#dbcount) are the backend-portable option. + ### SQL dialect Unlike the structured API methods (`db.query`, `db.get`, etc.), `db.raw` does **not** translate SQL between backends. Write native SQL for the database you're running against — `$1`/`$2` placeholders for Postgres, `?` for SQLite. Use `db.backend()` to branch when you need to support both. @@ -196,9 +213,9 @@ Returns `"sqlite"` or `"postgres"`. Useful when you need database-specific SQL t ```lua if db.backend() == "postgres" then - db.raw("SELECT * FROM happyview_records WHERE record @> $1::jsonb", { json.encode({ status = "active" }) }) + db.raw("SELECT * FROM my_events WHERE payload @> $1::jsonb", { json.encode({ status = "active" }) }) else -- SQLite fallback - db.raw("SELECT * FROM happyview_records WHERE json_extract(record, '$.status') = $1", { "active" }) + db.raw("SELECT * FROM my_events WHERE json_extract(payload, '$.status') = $1", { "active" }) end ``` diff --git a/src/lua/db_api.rs b/src/lua/db_api.rs index 6c4872f..2c44261 100644 --- a/src/lua/db_api.rs +++ b/src/lua/db_api.rs @@ -10,6 +10,88 @@ use crate::db::{DatabaseBackend, adapt_sql, decode_cursor, encode_cursor}; const MAX_FILTER_DEPTH: u8 = 5; const ALLOWED_OPS: &[&str] = &["=", "!=", "<", ">", "<=", ">=", "LIKE", "NOT LIKE"]; +/// Table-name prefix reserved for HappyView's own internal tables. `db.raw` +/// blocks these **by default** — so a table added in a future migration is +/// protected until it is deliberately allowed — except for the data tables in +/// [`ALLOWED_INTERNAL_TABLES`]. +const PROTECTED_TABLE_PREFIX: &str = "happyview_"; + +/// Internal tables that don't carry the `happyview_` prefix but are still +/// off-limits (SQLx's migration bookkeeping). +const PROTECTED_EXACT_TABLES: &[&str] = &["_sqlx_migrations"]; + +/// Internal tables `db.raw` is allowed to read and write despite the reserved +/// prefix: public AppView data and space *data*. Everything else `happyview_*` +/// stays blocked — secrets and tokens (`happyview_dpop_keys`/`_sessions`, +/// `happyview_api_keys`, `happyview_oauth_*`, `happyview_script_variables`), +/// auth/privilege state (`happyview_users`/`_user_permissions`, the delegation +/// tables), trust config (`happyview_domains`, `happyview_instance_settings`), +/// and cryptographic material (`happyview_space_credentials`, and +/// `happyview_space_repo_state` which holds commit-signature key material). +/// +/// Space membership/records are exposed because a space defines *access*, not +/// confidentiality — whether to expose otherwise-private space data through the +/// AppView is left to the admin. +const ALLOWED_INTERNAL_TABLES: &[&str] = &[ + // Public AppView data. + "happyview_records", + "happyview_record_refs", + "happyview_labels", + "happyview_lexicons", + // Background jobs. + "happyview_jobs", + // Space data (not the credential/key-material tables). + "happyview_spaces", + "happyview_space_members", + "happyview_space_records", + "happyview_space_record_oplog", + "happyview_space_notify_registrations", + "happyview_space_dids", +]; + +/// Reject a `db.raw` SQL string that references a protected internal table. +/// +/// Tokenizes the SQL (so string literals and comments containing the prefix are +/// ignored, and quoted / schema-qualified identifiers are still caught) and +/// blocks any `happyview_*` (or `_sqlx_migrations`) identifier that is not in +/// [`ALLOWED_INTERNAL_TABLES`]. Unicode-escaped identifiers (`U&"…"`) are refused +/// outright as an evasion vector, and SQL that cannot be tokenized fails closed. +fn check_raw_sql_tables(sql: &str) -> Result<(), String> { + use sqlparser::dialect::GenericDialect; + use sqlparser::tokenizer::{Token, Tokenizer}; + + // `U&'…'` / `U&"…"` unicode-escaped literals could smuggle a protected + // identifier past tokenization (the escapes decode to letters); there is no + // legitimate need for them in `db.raw`, so refuse them outright. + let lowered = sql.to_ascii_lowercase(); + if lowered.contains("u&\"") || lowered.contains("u&'") { + return Err("db.raw does not allow unicode-escaped identifiers".into()); + } + + // Tokenizing (rather than substring matching) means the prefix inside string + // literals or comments is ignored, while quoted and schema-qualified + // identifiers are still seen. SQL we cannot tokenize fails closed. + let tokens = Tokenizer::new(&GenericDialect {}, sql) + .tokenize() + .map_err(|e| format!("db.raw could not parse SQL: {e}"))?; + + for token in tokens { + if let Token::Word(word) = token { + let name = word.value.to_ascii_lowercase(); + let is_internal = name.starts_with(PROTECTED_TABLE_PREFIX) + || PROTECTED_EXACT_TABLES.contains(&name.as_str()); + if is_internal && !ALLOWED_INTERNAL_TABLES.contains(&name.as_str()) { + return Err(format!( + "db.raw cannot reference the protected internal HappyView table '{}'", + word.value + )); + } + } + } + + Ok(()) +} + fn is_valid_json_field_path(path: &str) -> bool { if path.is_empty() { return false; @@ -613,6 +695,10 @@ pub fn register_db_api(lua: &Lua, state: Arc) -> LuaResult<()> { lua.create_async_function(move |lua, (sql, params): (String, Option)| { let state = state_raw.clone(); async move { + // Protect HappyView's internal tables (secrets, auth, config, + // AppView bookkeeping) from raw access; own tables are fine. + check_raw_sql_tables(&sql).map_err(mlua::Error::runtime)?; + let mut query = sqlx::query(&sql); if let Some(ref params_table) = params { for value in params_table.sequence_values::() { @@ -809,19 +895,36 @@ mod tests { } #[tokio::test] - async fn raw_allows_non_select() { + async fn raw_allows_non_select_on_own_tables() { let state = test_state(); let lua = setup(&state); + // Non-SELECT statements are allowed against non-internal tables. Passes + // table validation; may then fail on the (empty in-memory) DB. let result: Result = lua - .load(r#"return db.raw("DELETE FROM happyview_records")"#) + .load(r#"return db.raw("DELETE FROM my_table")"#) .eval_async() .await; - // Should fail with a DB connection error, NOT a validation error - assert!(result.is_err()); - let err = result.unwrap_err().to_string(); + if let Err(e) = &result { + let err = e.to_string(); + assert!( + !err.contains("internal HappyView table"), + "should have passed table validation but got: {err}" + ); + } + } + + #[tokio::test] + async fn raw_blocks_internal_tables() { + let state = test_state(); + let lua = setup(&state); + let result: Result = lua + .load(r#"return db.raw("SELECT * FROM happyview_dpop_keys")"#) + .eval_async() + .await; + let err = result.expect_err("querying an internal table must be rejected"); assert!( - !err.contains("only supports SELECT"), - "should have passed validation but got: {err}" + err.to_string().contains("internal HappyView table"), + "expected an internal-table error, got: {err}" ); } @@ -842,6 +945,81 @@ mod tests { } } + #[test] + fn raw_sql_allows_non_protected_tables() { + // Admins can get wild with their own tables. + assert!(super::check_raw_sql_tables("SELECT * FROM my_table").is_ok()); + assert!(super::check_raw_sql_tables("CREATE TABLE analytics (id INT)").is_ok()); + assert!(super::check_raw_sql_tables("INSERT INTO analytics VALUES (1)").is_ok()); + assert!(super::check_raw_sql_tables("UPDATE analytics SET id = 2").is_ok()); + assert!(super::check_raw_sql_tables("DELETE FROM analytics WHERE id = 1").is_ok()); + assert!(super::check_raw_sql_tables("DROP TABLE analytics").is_ok()); + // A table that merely *contains* the prefix mid-name is fine. + assert!(super::check_raw_sql_tables("SELECT * FROM myhappyview_data").is_ok()); + // The prefix appearing inside a string literal is not a table reference. + assert!( + super::check_raw_sql_tables("INSERT INTO logs (msg) VALUES ('happyview_started')") + .is_ok() + ); + } + + #[test] + fn raw_sql_allows_allowlisted_internal_tables() { + // Public AppView data and background jobs are readable/writable. + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_records").is_ok()); + assert!( + super::check_raw_sql_tables("DELETE FROM happyview_records WHERE uri = $1").is_ok() + ); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_record_refs").is_ok()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_labels").is_ok()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_lexicons").is_ok()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_jobs").is_ok()); + // Space data (access, not confidentiality). + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_space_records").is_ok()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_space_members").is_ok()); + } + + #[test] + fn raw_sql_blocks_protected_tables() { + // Secrets / tokens / keys. + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_dpop_keys").is_err()); + assert!(super::check_raw_sql_tables("DROP TABLE happyview_api_keys").is_err()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_script_variables").is_err()); + // Auth / privilege / trust config. + assert!(super::check_raw_sql_tables("UPDATE happyview_users SET is_super = true").is_err()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_domains").is_err()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_instance_settings").is_err()); + // Space credential / key-material tables stay blocked even though other + // space tables are allowed. + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_space_credentials").is_err()); + assert!(super::check_raw_sql_tables("SELECT * FROM happyview_space_repo_state").is_err()); + // The migration bookkeeping table is off-limits too. + assert!(super::check_raw_sql_tables("SELECT * FROM _sqlx_migrations").is_err()); + } + + #[test] + fn raw_sql_blocks_protected_tables_evasion() { + // Case-insensitive. + assert!(super::check_raw_sql_tables("SELECT * FROM HAPPYVIEW_USERS").is_err()); + // Double-quoted identifier. + assert!(super::check_raw_sql_tables(r#"SELECT * FROM "happyview_api_keys""#).is_err()); + // Schema-qualified. + assert!( + super::check_raw_sql_tables("SELECT * FROM public.happyview_dpop_sessions").is_err() + ); + // Second statement in a batch. + assert!(super::check_raw_sql_tables("SELECT 1; SELECT * FROM happyview_users").is_err()); + // JOIN / subquery position. + assert!( + super::check_raw_sql_tables( + "SELECT * FROM my_table JOIN happyview_api_clients USING (id)" + ) + .is_err() + ); + // Unicode-escaped identifier evasion is refused outright. + assert!(super::check_raw_sql_tables(r#"SELECT * FROM U&"happyview_dpop_keys""#).is_err()); + } + #[test] fn valid_json_field_paths() { assert!(super::is_valid_json_field_path("name")); diff --git a/tests/e2e_scripts.rs b/tests/e2e_scripts.rs index 2535f00..654821a 100644 --- a/tests/e2e_scripts.rs +++ b/tests/e2e_scripts.rs @@ -710,12 +710,11 @@ async fn label_script_uri_routes_actor_special_case() { create_script( &app, "labeler.apply:_actor", - // Sentinel: write a row into records-table-as-flag so we can - // detect that the script ran. + // Sentinel: write a row into a caller-owned table (db.raw cannot touch + // internal HappyView tables) so we can detect that the script ran. "function handle() \ - db.raw('INSERT INTO happyview_records (uri, did, collection, rkey, record, cid, created_at) \ - VALUES (?, ?, ?, ?, ?, ?, ?)', \ - {'at://did:plc:flag/flag.col/k', 'did:plc:flag', 'flag.col', 'k', '{}', 'b', '2026-05-01'}) \ + db.raw('CREATE TABLE IF NOT EXISTS script_sentinel (k TEXT)') \ + db.raw('INSERT INTO script_sentinel (k) VALUES (?)', {'fired'}) \ return event \ end", ) @@ -737,9 +736,12 @@ async fn label_script_uri_routes_actor_special_case() { assert!(matches!(outcome, LabelHookOutcome::Continue(_))); // Sentinel row should exist if the script ran. + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM script_sentinel WHERE k = 'fired'") + .fetch_one(&app.state.db) + .await + .unwrap(); assert_eq!( - count_records(&app, "at://did:plc:flag/flag.col/k").await, - 1, + count, 1, "labeler.apply:_actor should have fired for bare-DID label" ); } diff --git a/tests/lua_db_api.rs b/tests/lua_db_api.rs index f9045a2..d0d4cda 100644 --- a/tests/lua_db_api.rs +++ b/tests/lua_db_api.rs @@ -328,13 +328,21 @@ async fn db_raw_select_works() { let pool = db::test_pool().await; let backend = db::test_backend(); db::truncate_all(&pool).await; - seed_records(&pool, backend).await; let state = test_state_with_pool(pool, backend).await; let lua = setup_lua(&state); + // db.raw operates freely on the caller's own tables (internal HappyView + // tables are protected — see raw_blocks_internal_table). let result: mlua::Table = lua .load( - r#"return db.raw("SELECT COUNT(*) as cnt FROM happyview_records WHERE collection = $1", {"test.collection"})"#, + r#" + db.raw("DROP TABLE IF EXISTS raw_probe") + db.raw("CREATE TABLE raw_probe (n INT)") + db.raw("INSERT INTO raw_probe (n) VALUES (1), (2), (3)") + local rows = db.raw("SELECT COUNT(*) as cnt FROM raw_probe WHERE n >= $1", {2}) + db.raw("DROP TABLE raw_probe") + return rows + "#, ) .eval_async() .await @@ -343,7 +351,28 @@ async fn db_raw_select_works() { // Result is an array of row tables let first_row: mlua::Table = result.get(1).unwrap(); let cnt: i64 = first_row.get("cnt").unwrap(); - assert_eq!(cnt, 3); + assert_eq!(cnt, 2); +} + +#[tokio::test] +#[serial] +async fn db_raw_blocks_internal_table() { + common::require_db!(); + let pool = db::test_pool().await; + let backend = db::test_backend(); + db::truncate_all(&pool).await; + let state = test_state_with_pool(pool, backend).await; + let lua = setup_lua(&state); + + let result: Result = lua + .load(r#"return db.raw("SELECT * FROM happyview_dpop_keys")"#) + .eval_async() + .await; + let err = result.expect_err("db.raw must reject internal HappyView tables"); + assert!( + err.to_string().contains("internal HappyView table"), + "unexpected error: {err}" + ); } #[tokio::test] diff --git a/web/src/lib/lua-hover.ts b/web/src/lib/lua-hover.ts index 6f85916..b2cc7eb 100644 --- a/web/src/lib/lua-hover.ts +++ b/web/src/lib/lua-hover.ts @@ -157,7 +157,7 @@ export const HOVER_DOCS = new Map([ ["db.count", { signature: "db.count(collection [, did])", description: "Count records in a collection", module: "db" }], ["db.search", { signature: "db.search({collection, field, query, limit?})", description: "Search records by field value — returns {records}", module: "db" }], ["db.backlinks", { signature: "db.backlinks({collection, uri, did?, limit?, cursor?})", description: "Find records that reference a URI via record_refs — returns {records, cursor?}", module: "db" }], - ["db.raw", { signature: "db.raw(sql [, params])", description: "Execute a raw SQL query — returns array of row tables", module: "db" }], + ["db.raw", { signature: "db.raw(sql [, params])", description: "Execute a raw SQL query — returns array of row tables. Can reach your own tables plus the record index and space data; HappyView's sensitive internal tables (secrets, auth, config) are blocked.", module: "db" }], ["db.backend", { signature: "db.backend()", description: "Returns the database backend — \"sqlite\" or \"postgres\"", module: "db" }], // ── HappyView HTTP API ─────────────────────────────────────────────── -- 2.51.2