From 408359f365838f0393d91db640cf8903071369c4 Mon Sep 17 00:00:00 2001 From: Trezy Date: Wed, 1 Jul 2026 13:46:02 -0500 Subject: [PATCH 1/3] feat: add support for async jobs Signed-off-by: Trezy --- .../postgres/20260701000000_create_jobs.sql | 17 + ...0260702000000_add_inherit_auth_to_jobs.sql | 1 + .../sqlite/20260701000000_create_jobs.sql | 17 + ...0260702000000_add_inherit_auth_to_jobs.sql | 1 + packages/docs/content/blog/happyview-2.10.md | 6 +- packages/docs/content/blog/happyview-2.9.md | 8 +- src/admin/jobs.rs | 124 +++++ src/admin/mod.rs | 6 + src/admin/permissions.rs | 34 ++ src/jobs/db.rs | 299 ++++++++++ src/jobs/mod.rs | 20 + src/jobs/worker.rs | 340 ++++++++++++ src/lib.rs | 1 + src/lua/execute.rs | 26 + src/lua/jobs_api.rs | 297 ++++++++++ src/lua/mod.rs | 9 +- src/lua/scripts.rs | 2 + src/main.rs | 9 + tests/common/db.rs | 3 +- tests/e2e_jobs.rs | 491 +++++++++++++++++ web/playwright.config.ts | 2 + web/src/app/dashboard/jobs/page.tsx | 515 ++++++++++++++++++ .../settings/scripts/[id]/script-detail.tsx | 32 +- .../dashboard/settings/scripts/new/page.tsx | 147 ++++- .../settings/scripts/script-form.tsx | 196 +++++-- web/src/components/app-sidebar.tsx | 7 + web/src/lib/api.ts | 33 ++ web/src/types/jobs.ts | 18 + web/src/types/scripts.ts | 32 +- web/tests/e2e/jobs.spec.ts | 216 ++++++++ web/tests/e2e/script-job.spec.ts | 90 +++ 31 files changed, 2919 insertions(+), 80 deletions(-) create mode 100644 migrations/postgres/20260701000000_create_jobs.sql create mode 100644 migrations/postgres/20260702000000_add_inherit_auth_to_jobs.sql create mode 100644 migrations/sqlite/20260701000000_create_jobs.sql create mode 100644 migrations/sqlite/20260702000000_add_inherit_auth_to_jobs.sql create mode 100644 src/admin/jobs.rs create mode 100644 src/jobs/db.rs create mode 100644 src/jobs/mod.rs create mode 100644 src/jobs/worker.rs create mode 100644 src/lua/jobs_api.rs create mode 100644 tests/e2e_jobs.rs create mode 100644 web/src/app/dashboard/jobs/page.tsx create mode 100644 web/src/types/jobs.ts create mode 100644 web/tests/e2e/jobs.spec.ts create mode 100644 web/tests/e2e/script-job.spec.ts diff --git a/migrations/postgres/20260701000000_create_jobs.sql b/migrations/postgres/20260701000000_create_jobs.sql new file mode 100644 index 0000000..1a704ee --- /dev/null +++ b/migrations/postgres/20260701000000_create_jobs.sql @@ -0,0 +1,17 @@ +CREATE TABLE happyview_jobs ( + id TEXT PRIMARY KEY, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + input TEXT NOT NULL DEFAULT '{}', + progress TEXT NOT NULL DEFAULT '{}', + result TEXT, + error TEXT, + created_by TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + created_at TEXT NOT NULL +); + +CREATE INDEX idx_happyview_jobs_status ON happyview_jobs (status); +CREATE INDEX idx_happyview_jobs_job_type ON happyview_jobs (job_type); +CREATE INDEX idx_happyview_jobs_created_by ON happyview_jobs (created_by); diff --git a/migrations/postgres/20260702000000_add_inherit_auth_to_jobs.sql b/migrations/postgres/20260702000000_add_inherit_auth_to_jobs.sql new file mode 100644 index 0000000..88887f1 --- /dev/null +++ b/migrations/postgres/20260702000000_add_inherit_auth_to_jobs.sql @@ -0,0 +1 @@ +ALTER TABLE happyview_jobs ADD COLUMN inherit_auth BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/migrations/sqlite/20260701000000_create_jobs.sql b/migrations/sqlite/20260701000000_create_jobs.sql new file mode 100644 index 0000000..681889a --- /dev/null +++ b/migrations/sqlite/20260701000000_create_jobs.sql @@ -0,0 +1,17 @@ +CREATE TABLE happyview_jobs ( + id TEXT PRIMARY KEY, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + input TEXT NOT NULL DEFAULT '{}', + progress TEXT NOT NULL DEFAULT '{}', + result TEXT, + error TEXT, + created_by TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX idx_happyview_jobs_status ON happyview_jobs (status); +CREATE INDEX idx_happyview_jobs_job_type ON happyview_jobs (job_type); +CREATE INDEX idx_happyview_jobs_created_by ON happyview_jobs (created_by); diff --git a/migrations/sqlite/20260702000000_add_inherit_auth_to_jobs.sql b/migrations/sqlite/20260702000000_add_inherit_auth_to_jobs.sql new file mode 100644 index 0000000..ff842d4 --- /dev/null +++ b/migrations/sqlite/20260702000000_add_inherit_auth_to_jobs.sql @@ -0,0 +1 @@ +ALTER TABLE happyview_jobs ADD COLUMN inherit_auth BOOLEAN NOT NULL DEFAULT 0; diff --git a/packages/docs/content/blog/happyview-2.10.md b/packages/docs/content/blog/happyview-2.10.md index 2e4a3e3..9201751 100644 --- a/packages/docs/content/blog/happyview-2.10.md +++ b/packages/docs/content/blog/happyview-2.10.md @@ -23,7 +23,7 @@ HappyView offers three modes: With a service identity in place, HappyView can act as a service proxy. A PDS sends a request with an `atproto-proxy` header pointing at your AppView, HappyView verifies the caller via service auth, runs your XRPC handler, and responds. This is how atproto apps are _supposed_ to work! Up to this point HappyView only supported direct connections via DPoP. -Full docs: [Service Identity](/docs/getting-started/service-identity). +Full docs: [Service Identity](/getting-started/service-identity). ## Permissioned spaces alignment @@ -63,7 +63,7 @@ Also, `getMemberGrant` is now `getDelegationToken` (and it's a `GET`, not a `POS - **Record operation log** - `listRepoOps` returns the oplog for sync - **Write notifications** - `registerNotify`, `notifyWrite`, `notifySpaceDeleted` -Full docs: [Permissioned Spaces](/docs/experimental/spaces/). +Full docs: [Permissioned Spaces](/experimental/spaces/). ## Blob utilities for Lua @@ -75,7 +75,7 @@ local uploaded = atproto.blob_upload(downloaded.handle, downloaded.mimeType) local new_blob_ref = uploaded.blob ``` -Full docs: [atproto API (`blob_download` / `blob_upload`)](/docs/api-reference/lua/atproto-api#atprotoblob_download). +Full docs: [atproto API (`blob_download` / `blob_upload`)](/api-reference/lua/atproto-api#atprotoblob_download). ## Prefixed database tables diff --git a/packages/docs/content/blog/happyview-2.9.md b/packages/docs/content/blog/happyview-2.9.md index 0f26140..2195b49 100644 --- a/packages/docs/content/blog/happyview-2.9.md +++ b/packages/docs/content/blog/happyview-2.9.md @@ -28,11 +28,11 @@ The biggest conceptual change in 2.9: scripts are no longer embedded with lexico For record events, the dispatcher tries the action-specific trigger first (e.g. `record.create:com.example.post`), then falls back to the wildcard `record.index:com.example.post`. This means you can have one general-purpose script that handles everything, or surgical scripts for specific actions — or both. -Scripts are managed through the dashboard under **Settings > Scripts**, or via the new [`/admin/scripts`](/docs/api-reference/admin/scripts) API endpoints. The lexicon detail page also shows which scripts target each lexicon, with links to create or edit them. +Scripts are managed through the dashboard under **Settings > Scripts**, or via the new [`/admin/scripts`](/api-reference/admin/scripts) API endpoints. The lexicon detail page also shows which scripts target each lexicon, with links to create or edit them. **If you're upgrading from v2.x to v2.9:** existing index hooks and lexicon scripts will be migrated to the new system automatically. -Full docs: [Record & Label Scripts](/docs/guides/label-scripts), [Lua Scripting](/docs/guides/lua-scripting), [Admin API — Scripts](/docs/api-reference/admin/scripts). +Full docs: [Record & Label Scripts](/guides/label-scripts), [Lua Scripting](/guides/lua-scripting), [Admin API — Scripts](/api-reference/admin/scripts). ## Backfill, but concurrent @@ -72,7 +72,7 @@ local result = db.query({ }) ``` -Full docs are in the [Database API reference](/docs/api-reference/lua/database-api). +Full docs are in the [Database API reference](/api-reference/lua/database-api). ## Auth fixes @@ -87,7 +87,7 @@ Both of these are fixed properly now, AND I added a couple new endpoints so clie - `GET /oauth/sessions/{did}/devices` — list all active sessions - `DELETE /oauth/sessions/{did}/devices/{session_id}` — revoke a session -The existing `DELETE /oauth/sessions/{did}` endpoint still works: confidential clients revoke all device sessions for the user, and public clients revoke the session matching their DPoP key. Full details in the [Authentication guide](/docs/getting-started/authentication#6-managing-device-sessions). +The existing `DELETE /oauth/sessions/{did}` endpoint still works: confidential clients revoke all device sessions for the user, and public clients revoke the session matching their DPoP key. Full details in the [Authentication guide](/getting-started/authentication#6-managing-device-sessions). ## SDK fix diff --git a/src/admin/jobs.rs b/src/admin/jobs.rs new file mode 100644 index 0000000..27bca7c --- /dev/null +++ b/src/admin/jobs.rs @@ -0,0 +1,124 @@ +use axum::Json; +use axum::extract::{Path, Query, State}; +use serde::Deserialize; + +use crate::AppState; +use crate::error::AppError; +use crate::jobs; + +use super::auth::UserAuth; +use super::permissions::Permission; + +#[derive(Deserialize)] +pub struct ListJobsQuery { + pub status: Option, + pub limit: Option, + pub cursor: Option, +} + +pub async fn list_jobs( + State(state): State, + auth: UserAuth, + Query(query): Query, +) -> Result, AppError> { + auth.require(Permission::JobsRead).await?; + + let limit = query.limit.unwrap_or(50).min(100); + let (jobs_list, cursor) = jobs::db::list_jobs( + &state, + query.status.as_deref(), + limit, + query.cursor.as_deref(), + ) + .await?; + + Ok(Json(serde_json::json!({ + "jobs": jobs_list, + "cursor": cursor, + }))) +} + +pub async fn get_job( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result, AppError> { + auth.require(Permission::JobsRead).await?; + + let job = jobs::db::get_job(&state, &id) + .await? + .ok_or_else(|| AppError::NotFound("job not found".into()))?; + + Ok(Json(serde_json::to_value(job).unwrap())) +} + +pub async fn cancel_job( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result, AppError> { + auth.require(Permission::JobsManage).await?; + + let job = jobs::db::get_job(&state, &id) + .await? + .ok_or_else(|| AppError::NotFound("job not found".into()))?; + + match job.status.as_str() { + "running" => { + jobs::db::set_status(&state, &id, "cancelling").await?; + Ok(Json(serde_json::json!({ "status": "cancelling" }))) + } + "pending" | "paused" => { + jobs::db::set_status(&state, &id, "cancelled").await?; + Ok(Json(serde_json::json!({ "status": "cancelled" }))) + } + _ => Err(AppError::BadRequest(format!( + "cannot cancel job with status: {}", + job.status + ))), + } +} + +pub async fn pause_job( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result, AppError> { + auth.require(Permission::JobsManage).await?; + + let job = jobs::db::get_job(&state, &id) + .await? + .ok_or_else(|| AppError::NotFound("job not found".into()))?; + + if job.status != "running" { + return Err(AppError::BadRequest(format!( + "cannot pause job with status: {}", + job.status + ))); + } + + jobs::db::set_status(&state, &id, "pausing").await?; + Ok(Json(serde_json::json!({ "status": "pausing" }))) +} + +pub async fn resume_job( + State(state): State, + auth: UserAuth, + Path(id): Path, +) -> Result, AppError> { + auth.require(Permission::JobsManage).await?; + + let job = jobs::db::get_job(&state, &id) + .await? + .ok_or_else(|| AppError::NotFound("job not found".into()))?; + + if job.status != "paused" { + return Err(AppError::BadRequest(format!( + "cannot resume job with status: {}", + job.status + ))); + } + + jobs::db::set_status(&state, &id, "pending").await?; + Ok(Json(serde_json::json!({ "status": "pending" }))) +} diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 9dc9eab..a2e89bc 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -6,6 +6,7 @@ mod dead_letters; mod domains; mod events; mod feature_flags; +mod jobs; mod labelers; mod lexicons; mod network_lexicons; @@ -62,6 +63,11 @@ pub fn admin_routes(_state: AppState) -> Router { "/backfill/{id}/details", delete(backfill::flush_backfill_details), ) + .route("/jobs", get(jobs::list_jobs)) + .route("/jobs/{id}", get(jobs::get_job)) + .route("/jobs/{id}/cancel", post(jobs::cancel_job)) + .route("/jobs/{id}/pause", post(jobs::pause_job)) + .route("/jobs/{id}/resume", post(jobs::resume_job)) .route("/events", get(events::list_events)) .route("/users", post(users::create_user).get(users::list_users)) .route("/users/transfer-super", post(users::transfer_super)) diff --git a/src/admin/permissions.rs b/src/admin/permissions.rs index e82c0ac..f2cb1e7 100644 --- a/src/admin/permissions.rs +++ b/src/admin/permissions.rs @@ -113,6 +113,13 @@ pub enum Permission { ScriptsRead, #[serde(rename = "scripts:manage")] ScriptsManage, + + #[serde(rename = "jobs:read")] + JobsRead, + #[serde(rename = "jobs:create")] + JobsCreate, + #[serde(rename = "jobs:manage")] + JobsManage, } impl Permission { @@ -162,6 +169,9 @@ impl Permission { Self::SpacesManageCredentials => "spaces:manage-credentials", Self::ScriptsRead => "scripts:read", Self::ScriptsManage => "scripts:manage", + Self::JobsRead => "jobs:read", + Self::JobsCreate => "jobs:create", + Self::JobsManage => "jobs:manage", } } @@ -425,6 +435,24 @@ impl Permission { description: "Create, update, and delete trigger-keyed scripts", category: "Scripts", }, + Self::JobsRead => PermissionInfo { + key: "jobs:read", + name: "View Jobs", + description: "View background job status and progress", + category: "Jobs", + }, + Self::JobsCreate => PermissionInfo { + key: "jobs:create", + name: "Create Jobs", + description: "Queue new background jobs", + category: "Jobs", + }, + Self::JobsManage => PermissionInfo { + key: "jobs:manage", + name: "Manage Jobs", + description: "Cancel, pause, and resume background jobs", + category: "Jobs", + }, } } @@ -474,6 +502,9 @@ impl Permission { Self::SpacesManageCredentials, Self::ScriptsRead, Self::ScriptsManage, + Self::JobsRead, + Self::JobsCreate, + Self::JobsManage, ]) } } @@ -525,6 +556,9 @@ pub fn catalog() -> Vec { SpacesManageInvites, SpacesManageRecords, SpacesManageCredentials, + JobsRead, + JobsCreate, + JobsManage, ] .iter() .map(|p| p.info()) diff --git a/src/jobs/db.rs b/src/jobs/db.rs new file mode 100644 index 0000000..10120c8 --- /dev/null +++ b/src/jobs/db.rs @@ -0,0 +1,299 @@ +use serde_json::Value; +use uuid::Uuid; + +use crate::AppState; +use crate::db::{adapt_sql, now_rfc3339}; +use crate::error::AppError; + +use super::Job; + +type JobRow = ( + String, + String, + String, + String, + String, + Option, + Option, + String, + Option, + Option, + String, + bool, +); + +fn row_to_job( + ( + id, + job_type, + status, + input, + progress, + result, + error, + created_by, + started_at, + completed_at, + created_at, + inherit_auth, + ): JobRow, +) -> Job { + Job { + id, + job_type, + status, + input: serde_json::from_str(&input).unwrap_or(Value::Null), + progress: serde_json::from_str(&progress).unwrap_or(Value::Null), + result: result.and_then(|r| serde_json::from_str(&r).ok()), + error, + created_by, + started_at, + completed_at, + created_at, + inherit_auth, + } +} + +pub async fn create_job( + state: &AppState, + job_type: &str, + input: &Value, + created_by: &str, + inherit_auth: bool, +) -> Result { + let id = Uuid::new_v4().to_string(); + let now = now_rfc3339(); + let input_str = serde_json::to_string(input) + .map_err(|e| AppError::Internal(format!("failed to serialize job input: {e}")))?; + + let sql = adapt_sql( + "INSERT INTO happyview_jobs (id, job_type, status, input, created_by, created_at, inherit_auth) VALUES (?, ?, 'pending', ?, ?, ?, ?)", + state.db_backend, + ); + sqlx::query(&sql) + .bind(&id) + .bind(job_type) + .bind(&input_str) + .bind(created_by) + .bind(&now) + .bind(inherit_auth) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to create job: {e}")))?; + + Ok(id) +} + +pub async fn get_job(state: &AppState, id: &str) -> Result, AppError> { + let sql = adapt_sql( + "SELECT * FROM happyview_jobs WHERE id = ?", + state.db_backend, + ); + let row: Option = sqlx::query_as(&sql) + .bind(id) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to fetch job: {e}")))?; + + Ok(row.map(row_to_job)) +} + +pub async fn list_jobs( + state: &AppState, + status_filter: Option<&str>, + limit: i64, + cursor: Option<&str>, +) -> Result<(Vec, Option), AppError> { + let sql = if status_filter.is_some() { + let base = if cursor.is_some() { + "SELECT * FROM happyview_jobs WHERE status = ? AND created_at < ? ORDER BY created_at DESC LIMIT ?" + } else { + "SELECT * FROM happyview_jobs WHERE status = ? ORDER BY created_at DESC LIMIT ?" + }; + adapt_sql(base, state.db_backend) + } else { + let base = if cursor.is_some() { + "SELECT * FROM happyview_jobs WHERE created_at < ? ORDER BY created_at DESC LIMIT ?" + } else { + "SELECT * FROM happyview_jobs ORDER BY created_at DESC LIMIT ?" + }; + adapt_sql(base, state.db_backend) + }; + + let mut query = sqlx::query_as::<_, JobRow>(&sql); + + if let Some(status) = status_filter { + query = query.bind(status); + } + if let Some(cursor) = cursor { + query = query.bind(cursor); + } + query = query.bind(limit + 1); + + let rows = query + .fetch_all(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to list jobs: {e}")))?; + + let has_more = rows.len() as i64 > limit; + let jobs: Vec = rows + .into_iter() + .take(limit as usize) + .map(row_to_job) + .collect(); + + let next_cursor = if has_more { + jobs.last().map(|j| j.created_at.clone()) + } else { + None + }; + + Ok((jobs, next_cursor)) +} + +pub async fn set_status(state: &AppState, id: &str, status: &str) -> Result<(), AppError> { + let now = now_rfc3339(); + let sql = match status { + "running" => adapt_sql( + "UPDATE happyview_jobs SET status = ?, started_at = ? WHERE id = ?", + state.db_backend, + ), + "completed" | "failed" | "cancelled" => adapt_sql( + "UPDATE happyview_jobs SET status = ?, completed_at = ? WHERE id = ?", + state.db_backend, + ), + _ => adapt_sql( + "UPDATE happyview_jobs SET status = ? WHERE id = ? AND 1=1", + state.db_backend, + ), + }; + + match status { + "running" | "completed" | "failed" | "cancelled" => { + sqlx::query(&sql) + .bind(status) + .bind(&now) + .bind(id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to update job status: {e}")))?; + } + _ => { + let sql = adapt_sql( + "UPDATE happyview_jobs SET status = ? WHERE id = ?", + state.db_backend, + ); + sqlx::query(&sql) + .bind(status) + .bind(id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to update job status: {e}")))?; + } + } + + Ok(()) +} + +pub async fn update_progress(state: &AppState, id: &str, progress: &Value) -> Result<(), AppError> { + let progress_str = serde_json::to_string(progress) + .map_err(|e| AppError::Internal(format!("failed to serialize progress: {e}")))?; + let sql = adapt_sql( + "UPDATE happyview_jobs SET progress = ? WHERE id = ?", + state.db_backend, + ); + sqlx::query(&sql) + .bind(&progress_str) + .bind(id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to update job progress: {e}")))?; + Ok(()) +} + +pub async fn set_result(state: &AppState, id: &str, result: &Value) -> Result<(), AppError> { + let result_str = serde_json::to_string(result) + .map_err(|e| AppError::Internal(format!("failed to serialize result: {e}")))?; + let now = now_rfc3339(); + let sql = adapt_sql( + "UPDATE happyview_jobs SET status = 'completed', result = ?, completed_at = ? WHERE id = ?", + state.db_backend, + ); + sqlx::query(&sql) + .bind(&result_str) + .bind(&now) + .bind(id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to set job result: {e}")))?; + Ok(()) +} + +pub async fn set_error(state: &AppState, id: &str, error: &str) -> Result<(), AppError> { + let now = now_rfc3339(); + let sql = adapt_sql( + "UPDATE happyview_jobs SET status = 'failed', error = ?, completed_at = ? WHERE id = ?", + state.db_backend, + ); + sqlx::query(&sql) + .bind(error) + .bind(&now) + .bind(id) + .execute(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to set job error: {e}")))?; + Ok(()) +} + +/// Check if a job should stop (status changed to cancelling or pausing). +/// Same cooperative cancellation pattern as the backfill system. +pub async fn should_stop(state: &AppState, id: &str) -> Option<&'static str> { + let sql = adapt_sql( + "SELECT status FROM happyview_jobs WHERE id = ?", + state.db_backend, + ); + let status = sqlx::query_as::<_, (String,)>(&sql) + .bind(id) + .fetch_optional(&state.db) + .await + .ok() + .flatten() + .map(|(s,)| s); + match status.as_deref() { + Some("cancelling") => Some("cancelling"), + Some("pausing") => Some("pausing"), + _ => None, + } +} + +/// Find jobs that were interrupted by a server restart. +pub async fn find_interrupted_jobs(state: &AppState) -> Vec { + let sql = adapt_sql( + "SELECT * FROM happyview_jobs WHERE status IN ('running', 'cancelling', 'pausing')", + state.db_backend, + ); + let rows: Vec = sqlx::query_as(&sql) + .fetch_all(&state.db) + .await + .unwrap_or_default(); + + rows.into_iter().map(row_to_job).collect() +} + +/// Pick the next pending job and atomically set it to running. +pub async fn claim_next_job(state: &AppState) -> Result, AppError> { + let now = now_rfc3339(); + + let sql = adapt_sql( + "UPDATE happyview_jobs SET status = 'running', started_at = ? WHERE id = (SELECT id FROM happyview_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1) RETURNING *", + state.db_backend, + ); + + let row: Option = sqlx::query_as(&sql) + .bind(&now) + .fetch_optional(&state.db) + .await + .map_err(|e| AppError::Internal(format!("failed to claim job: {e}")))?; + + Ok(row.map(row_to_job)) +} diff --git a/src/jobs/mod.rs b/src/jobs/mod.rs new file mode 100644 index 0000000..d24aa07 --- /dev/null +++ b/src/jobs/mod.rs @@ -0,0 +1,20 @@ +pub(crate) mod db; +pub mod worker; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Job { + pub id: String, + pub job_type: String, + pub status: String, + pub input: serde_json::Value, + pub progress: serde_json::Value, + pub result: Option, + pub error: Option, + pub created_by: String, + pub started_at: Option, + pub completed_at: Option, + pub created_at: String, + pub inherit_auth: bool, +} diff --git a/src/jobs/worker.rs b/src/jobs/worker.rs new file mode 100644 index 0000000..89a8051 --- /dev/null +++ b/src/jobs/worker.rs @@ -0,0 +1,340 @@ +use std::sync::Arc; +use std::time::Duration; + +use mlua::LuaSerdeExt; + +use crate::AppState; +use crate::db::adapt_sql; +use crate::event_log::{EventLog, Severity, log_event}; +use crate::lua::{sandbox, scripts}; +use crate::repo; + +use super::db; + +const POLL_INTERVAL: Duration = Duration::from_secs(5); + +/// Start the background job worker. Polls for pending jobs and +/// executes them one at a time. +pub async fn run_worker(state: AppState) { + tracing::info!("job worker started"); + + loop { + match db::claim_next_job(&state).await { + Ok(Some(job)) => { + tracing::info!(job_id = %job.id, job_type = %job.job_type, "executing job"); + execute_job(&state, &job).await; + } + Ok(None) => { + tokio::time::sleep(POLL_INTERVAL).await; + } + Err(e) => { + tracing::error!(error = %e, "job worker: failed to claim job"); + tokio::time::sleep(POLL_INTERVAL).await; + } + } + } +} + +/// Resume jobs that were interrupted by a server restart. +pub async fn resume_interrupted_jobs(state: &AppState) { + let jobs = db::find_interrupted_jobs(state).await; + + for job in jobs { + match job.status.as_str() { + "cancelling" => { + tracing::info!(job_id = %job.id, "finalising cancelled job from previous run"); + let _ = db::set_status(state, &job.id, "cancelled").await; + } + "pausing" => { + tracing::info!(job_id = %job.id, "finalising paused job from previous run"); + let _ = db::set_status(state, &job.id, "paused").await; + } + "running" => { + tracing::info!(job_id = %job.id, "re-queuing interrupted job"); + let _ = db::set_status(state, &job.id, "pending").await; + } + _ => {} + } + } +} + +async fn execute_job(state: &AppState, job: &super::Job) { + let backend = state.db_backend; + + log_event( + &state.db, + EventLog { + event_type: "job.started".to_string(), + severity: Severity::Info, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ + "job_id": job.id, + "job_type": job.job_type, + }), + }, + backend, + ) + .await; + + let trigger_id = format!("job.run:{}", job.job_type); + let script = match scripts::resolve(state, &trigger_id).await { + Some(s) => s, + None => { + let error = format!("no script found for trigger: {trigger_id}"); + tracing::error!(job_id = %job.id, %error); + let _ = db::set_error(state, &job.id, &error).await; + log_event( + &state.db, + EventLog { + event_type: "job.failed".to_string(), + severity: Severity::Error, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ + "job_id": job.id, + "error": error, + }), + }, + backend, + ) + .await; + return; + } + }; + + let (claims, pds_auth_arc) = if job.inherit_auth { + let pds_auth = match repo::get_oauth_session(state, &job.created_by).await { + Ok(session) => repo::PdsAuth::OAuth(Arc::new(session)), + Err(e) => { + let error = format!("failed to obtain PDS auth for {}: {e}", job.created_by); + tracing::error!(job_id = %job.id, %error); + let _ = db::set_error(state, &job.id, &error).await; + log_event( + &state.db, + EventLog { + event_type: "job.failed".to_string(), + severity: Severity::Error, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ + "job_id": job.id, + "error": error, + }), + }, + backend, + ) + .await; + return; + } + }; + ( + Some(Arc::new(crate::auth::Claims::internal( + job.created_by.clone(), + ))), + Some(Arc::new(pds_auth)), + ) + } else { + (None, None) + }; + + let lua = match sandbox::create_sandbox() { + Ok(l) => l, + Err(e) => { + let error = format!("failed to create Lua VM: {e}"); + let _ = db::set_error(state, &job.id, &error).await; + return; + } + }; + + lua.remove_hook(); + + let state_arc = Arc::new(state.clone()); + + if let Err(e) = crate::lua::db_api::register_db_api(&lua, state_arc.clone()) { + let _ = db::set_error(state, &job.id, &format!("db api: {e}")).await; + return; + } + if let Err(e) = crate::lua::http_api::register_http_api(&lua, state_arc.clone()) { + let _ = db::set_error(state, &job.id, &format!("http api: {e}")).await; + return; + } + if let Err(e) = crate::lua::xrpc_api::register_xrpc_api( + &lua, + state_arc.clone(), + Some(job.created_by.clone()), + ) { + let _ = db::set_error(state, &job.id, &format!("xrpc api: {e}")).await; + return; + } + if let Err(e) = crate::lua::atproto_api::register_atproto_api( + &lua, + state_arc.clone(), + Some(&job.created_by), + ) { + let _ = db::set_error(state, &job.id, &format!("atproto api: {e}")).await; + return; + } + if let (Some(c), Some(p)) = (&claims, &pds_auth_arc) + && let Err(e) = crate::lua::atproto_api::register_atproto_blob_api( + &lua, + state_arc.clone(), + c.clone(), + p.clone(), + ) + { + let _ = db::set_error(state, &job.id, &format!("blob api: {e}")).await; + return; + } + if let Err(e) = crate::lua::jobs_api::register_jobs_api( + &lua, + state_arc.clone(), + Some(job.created_by.clone()), + ) { + let _ = db::set_error(state, &job.id, &format!("jobs api: {e}")).await; + return; + } + if let Err(e) = + crate::lua::record::register_record_api(&lua, state_arc.clone(), claims, pds_auth_arc, None) + { + let _ = db::set_error(state, &job.id, &format!("record api: {e}")).await; + return; + } + if let Err(e) = crate::lua::scripts::register_log_event_api( + &lua, + &state_arc, + &trigger_id, + Some(&job.created_by), + ) { + let _ = db::set_error(state, &job.id, &format!("log api: {e}")).await; + return; + } + if let Err(e) = crate::lua::jobs_api::register_job_context( + &lua, + state_arc.clone(), + job.id.clone(), + job.input.clone(), + ) { + let _ = db::set_error(state, &job.id, &format!("job context: {e}")).await; + return; + } + + let env_vars = load_env_vars(&state.db, backend).await; + if let Err(e) = crate::lua::context::set_env_context(&lua, &env_vars) { + let _ = db::set_error(state, &job.id, &format!("env context: {e}")).await; + return; + } + + if let Err(e) = lua.globals().set("caller_did", job.created_by.as_str()) { + let _ = db::set_error(state, &job.id, &format!("caller_did: {e}")).await; + return; + } + + if let Err(e) = lua.load(script.body.as_str()).exec() { + let error = format!("script load failed: {e}"); + let _ = db::set_error(state, &job.id, &error).await; + return; + } + + let handle: mlua::Function = match lua.globals().get("handle") { + Ok(f) => f, + Err(e) => { + let _ = db::set_error(state, &job.id, &format!("missing handle(): {e}")).await; + return; + } + }; + + match handle.call_async::(()).await { + Ok(result) => { + let json_result: serde_json::Value = + lua.from_value(result).unwrap_or(serde_json::json!(null)); + + match db::should_stop(state, &job.id).await { + Some("pausing") => { + let _ = db::set_status(state, &job.id, "paused").await; + tracing::info!(job_id = %job.id, "job paused"); + log_event( + &state.db, + EventLog { + event_type: "job.paused".to_string(), + severity: Severity::Info, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ "job_id": job.id }), + }, + backend, + ) + .await; + } + Some("cancelling") => { + let _ = db::set_status(state, &job.id, "cancelled").await; + tracing::info!(job_id = %job.id, "job cancelled"); + log_event( + &state.db, + EventLog { + event_type: "job.cancelled".to_string(), + severity: Severity::Info, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ "job_id": job.id }), + }, + backend, + ) + .await; + } + _ => { + let _ = db::set_result(state, &job.id, &json_result).await; + tracing::info!(job_id = %job.id, "job completed"); + log_event( + &state.db, + EventLog { + event_type: "job.completed".to_string(), + severity: Severity::Info, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ + "job_id": job.id, + "result": json_result, + }), + }, + backend, + ) + .await; + } + } + } + Err(e) => { + let error = format!("{e}"); + tracing::error!(job_id = %job.id, %error, "job script failed"); + let _ = db::set_error(state, &job.id, &error).await; + log_event( + &state.db, + EventLog { + event_type: "job.failed".to_string(), + severity: Severity::Error, + actor_did: Some(job.created_by.clone()), + subject: Some(job.job_type.clone()), + detail: serde_json::json!({ + "job_id": job.id, + "error": error, + }), + }, + backend, + ) + .await; + } + } +} + +async fn load_env_vars( + db: &sqlx::AnyPool, + backend: crate::db::DatabaseBackend, +) -> std::collections::HashMap { + let sql = adapt_sql("SELECT key, value FROM happyview_script_variables", backend); + sqlx::query_as::<_, (String, String)>(&sql) + .fetch_all(db) + .await + .unwrap_or_default() + .into_iter() + .collect() +} diff --git a/src/lib.rs b/src/lib.rs index 0b1b6cd..51711f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ pub mod feature_flags; pub mod feature_middleware; pub mod http_retry; pub mod jetstream; +pub mod jobs; pub mod labeler; pub mod lexicon; pub mod lua; diff --git a/src/lua/execute.rs b/src/lua/execute.rs index 706a475..5997665 100644 --- a/src/lua/execute.rs +++ b/src/lua/execute.rs @@ -268,6 +268,32 @@ pub async fn execute_procedure_script( return Err(AppError::Internal(error_message)); } + if let Err(e) = + super::jobs_api::register_jobs_api(&lua, state_arc.clone(), Some(claims.did().to_string())) + { + let error_message = format!("failed to register jobs API: {e}"); + log_event( + &state.db, + EventLog { + event_type: "script.error".to_string(), + severity: Severity::Error, + actor_did: Some(claims.did().to_string()), + subject: Some(method.to_string()), + detail: serde_json::json!({ + "error": error_message, + "script_source": script_source, + "input": input_json, + "caller_did": claims.did(), + "method": method, + "duration_ms": start.elapsed().as_millis() as u64, + }), + }, + backend, + ) + .await; + return Err(AppError::Internal(error_message)); + } + if let Err(e) = record::register_record_api( &lua, state_arc.clone(), diff --git a/src/lua/jobs_api.rs b/src/lua/jobs_api.rs new file mode 100644 index 0000000..959628d --- /dev/null +++ b/src/lua/jobs_api.rs @@ -0,0 +1,297 @@ +use mlua::{Lua, LuaSerdeExt, Result as LuaResult}; +use std::sync::Arc; + +use crate::AppState; +use crate::jobs; + +/// Register the `jobs` table for queuing jobs from scripts. +/// Available in all script contexts (procedure, query, record-event). +pub fn register_jobs_api( + lua: &Lua, + state: Arc, + caller_did: Option, +) -> LuaResult<()> { + let jobs_table = lua.create_table()?; + + // jobs.create(job_type, input[, opts]) -> job_id string + // opts.auth: boolean (default false) — inherit caller's PDS auth + { + let state = state.clone(); + let caller_did = caller_did.clone(); + let create_fn = lua.create_async_function( + move |lua, (job_type, input, opts): (String, mlua::Value, Option)| { + let state = state.clone(); + let caller_did = caller_did.clone(); + + let input_json: serde_json::Value = + lua.from_value(input).unwrap_or(serde_json::json!({})); + + let inherit_auth = opts + .and_then(|t| t.get::("auth").ok()) + .unwrap_or(false); + + async move { + let caller = caller_did.as_deref().ok_or_else(|| { + mlua::Error::runtime("jobs.create requires an authenticated caller") + })?; + + let job_id = + jobs::db::create_job(&state, &job_type, &input_json, caller, inherit_auth) + .await + .map_err(|e| { + mlua::Error::runtime(format!("jobs.create failed: {e}")) + })?; + + Ok(job_id) + } + }, + )?; + jobs_table.set("create", create_fn)?; + } + + lua.globals().set("jobs", jobs_table)?; + Ok(()) +} + +/// Register the `job` context table for use inside job scripts. +/// Provides access to job input, progress reporting, cooperative +/// cancellation, and sleep/wait. +/// +/// Called by the job worker, not by the normal script execution path. +pub fn register_job_context( + lua: &Lua, + state: Arc, + job_id: String, + input: serde_json::Value, +) -> LuaResult<()> { + let job_table = lua.create_table()?; + + // job.input — the JSONB input passed to jobs.create() + let input_value = lua.to_value(&input)?; + job_table.set("input", input_value)?; + + // job.id — the job's UUID + job_table.set("id", job_id.clone())?; + + // job.progress(data) — persist progress to DB + { + let state = state.clone(); + let job_id = job_id.clone(); + let progress_fn = lua.create_async_function(move |lua, data: mlua::Value| { + let state = state.clone(); + let job_id = job_id.clone(); + let json_data: serde_json::Value = + lua.from_value(data).unwrap_or(serde_json::json!({})); + async move { + jobs::db::update_progress(&state, &job_id, &json_data) + .await + .map_err(|e| mlua::Error::runtime(format!("job.progress failed: {e}")))?; + Ok(()) + } + })?; + job_table.set("progress", progress_fn)?; + } + + // job.should_stop() -> boolean + { + let state = state.clone(); + let job_id = job_id.clone(); + let should_stop_fn = lua.create_async_function(move |_lua, ()| { + let state = state.clone(); + let job_id = job_id.clone(); + async move { + let result = jobs::db::should_stop(&state, &job_id).await; + Ok(result.is_some()) + } + })?; + job_table.set("should_stop", should_stop_fn)?; + } + + // job.wait(seconds) — yield execution for the given duration + { + let wait_fn = lua.create_async_function(move |_lua, seconds: f64| async move { + let duration = std::time::Duration::from_secs_f64(seconds.clamp(0.0, 3600.0)); + tokio::time::sleep(duration).await; + Ok(()) + })?; + job_table.set("wait", wait_fn)?; + } + + lua.globals().set("job", job_table)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::Config; + use crate::db::DatabaseBackend; + use crate::lexicon::LexiconRegistry; + use tokio::sync::watch; + + fn test_state() -> AppState { + let config = Config { + host: "127.0.0.1".into(), + port: 3000, + database_url: String::new(), + database_backend: crate::db::DatabaseBackend::Sqlite, + public_url: String::new(), + session_secret: "test-secret".into(), + jetstream_url: String::new(), + relay_url: String::new(), + plc_url: String::new(), + static_dir: String::new(), + base_path: None, + event_log_retention_days: 30, + app_name: None, + logo_uri: None, + tos_uri: None, + policy_uri: None, + token_encryption_key: None, + default_rate_limit_capacity: 100, + default_rate_limit_refill_rate: 2.0, + }; + let (tx, _) = watch::channel(vec![]); + let (labeler_tx, _) = watch::channel(()); + sqlx::any::install_default_drivers(); + let test_db = sqlx::AnyPool::connect_lazy("sqlite::memory:").unwrap(); + let atrium_http = std::sync::Arc::new(atrium_oauth::DefaultHttpClient::default()); + let did_resolver = atrium_identity::did::CommonDidResolver::new( + atrium_identity::did::CommonDidResolverConfig { + plc_directory_url: "https://plc.directory".into(), + http_client: std::sync::Arc::clone(&atrium_http), + }, + ); + let handle_resolver = atrium_identity::handle::AtprotoHandleResolver::new( + atrium_identity::handle::AtprotoHandleResolverConfig { + dns_txt_resolver: crate::dns::NativeDnsResolver::new(), + http_client: atrium_http, + }, + ); + let oauth = atrium_oauth::OAuthClient::new(atrium_oauth::OAuthClientConfig { + client_metadata: atrium_oauth::AtprotoLocalhostClientMetadata { + redirect_uris: Some(vec!["http://127.0.0.1:0/auth/callback".into()]), + scopes: Some(vec![atrium_oauth::Scope::Known( + atrium_oauth::KnownScope::Atproto, + )]), + }, + keys: None, + state_store: crate::auth::oauth_store::DbStateStore::new( + test_db.clone(), + crate::db::DatabaseBackend::Sqlite, + ), + session_store: crate::auth::oauth_store::DbSessionStore::new( + test_db.clone(), + crate::db::DatabaseBackend::Sqlite, + ), + resolver: atrium_oauth::OAuthResolverConfig { + did_resolver, + handle_resolver, + authorization_server_metadata: Default::default(), + protected_resource_metadata: Default::default(), + }, + }) + .expect("Failed to create test OAuth client"); + AppState { + config, + http: reqwest::Client::new(), + db: test_db.clone(), + backfill_db: test_db.clone(), + db_backend: DatabaseBackend::Sqlite, + domain_cache: crate::domain::DomainCache::new(), + lexicons: LexiconRegistry::new(), + collections_tx: tx, + labeler_subscriptions_tx: labeler_tx, + rate_limiter: crate::rate_limit::RateLimiter::new( + crate::rate_limit::RateLimitDefaults { + query_cost: 1, + procedure_cost: 1, + proxy_cost: 1, + }, + ), + oauth: std::sync::Arc::new(crate::auth::OAuthClientRegistry::new(std::sync::Arc::new( + oauth, + ))), + oauth_state_store: crate::auth::oauth_store::DbStateStore::new( + test_db.clone(), + crate::db::DatabaseBackend::Sqlite, + ), + cookie_key: axum_extra::extract::cookie::Key::derive_from( + b"test-secret-for-tests-only-not-production", + ), + plugin_registry: std::sync::Arc::new(crate::plugin::PluginRegistry::new()), + wasm_runtime: std::sync::Arc::new( + crate::plugin::WasmRuntime::new().expect("wasm runtime"), + ), + attestation_signer: None, + official_registry: std::sync::Arc::new(tokio::sync::RwLock::new( + crate::plugin::official_registry::OfficialRegistryState::default(), + )), + official_registry_config: crate::plugin::official_registry::RegistryConfig::production( + ), + proxy_config: std::sync::Arc::new(arc_swap::ArcSwap::new(std::sync::Arc::new( + crate::proxy_config::ProxyConfig::default(), + ))), + backfill_events_tx: tokio::sync::broadcast::channel(16).0, + verbose_event_logging: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + } + } + + #[tokio::test] + async fn jobs_api_is_registered() { + let lua = crate::lua::sandbox::create_sandbox().unwrap(); + let state = test_state(); + register_jobs_api(&lua, Arc::new(state), Some("did:plc:test".into())).unwrap(); + + let has_create: bool = lua + .load("return type(jobs.create) == 'function'") + .eval_async() + .await + .unwrap(); + assert!(has_create); + } + + #[tokio::test] + async fn job_context_exposes_input() { + let lua = crate::lua::sandbox::create_sandbox().unwrap(); + let state = test_state(); + let input = serde_json::json!({ "game_uri": "at://did:plc:test/game/123" }); + register_job_context(&lua, Arc::new(state), "test-job-id".into(), input).unwrap(); + + let game_uri: String = lua + .load("return job.input.game_uri") + .eval_async() + .await + .unwrap(); + assert_eq!(game_uri, "at://did:plc:test/game/123"); + + let job_id: String = lua.load("return job.id").eval_async().await.unwrap(); + assert_eq!(job_id, "test-job-id"); + } + + #[tokio::test] + async fn job_context_has_required_functions() { + let lua = crate::lua::sandbox::create_sandbox().unwrap(); + let state = test_state(); + register_job_context( + &lua, + Arc::new(state), + "test-id".into(), + serde_json::json!({}), + ) + .unwrap(); + + let result: bool = lua + .load( + r#" + return type(job.progress) == 'function' + and type(job.should_stop) == 'function' + and type(job.wait) == 'function' + "#, + ) + .eval_async() + .await + .unwrap(); + assert!(result); + } +} diff --git a/src/lua/mod.rs b/src/lua/mod.rs index bed07b3..5e04b2a 100644 --- a/src/lua/mod.rs +++ b/src/lua/mod.rs @@ -1,13 +1,14 @@ -mod atproto_api; -mod context; +pub(crate) mod atproto_api; +pub(crate) mod context; pub mod db_api; mod execute; -mod http_api; +pub(crate) mod http_api; +pub(crate) mod jobs_api; pub mod record; pub(crate) mod sandbox; pub mod scripts; pub(crate) mod tid; -mod xrpc_api; +pub(crate) mod xrpc_api; #[allow(unused_imports)] pub(crate) use context::SpaceContext; diff --git a/src/lua/scripts.rs b/src/lua/scripts.rs index 324906c..df4ae59 100644 --- a/src/lua/scripts.rs +++ b/src/lua/scripts.rs @@ -725,6 +725,8 @@ fn register_default_apis( .map_err(|e| format!("xrpc api: {e}"))?; atproto_api::register_atproto_api(lua, state.clone(), None) .map_err(|e| format!("atproto api: {e}"))?; + super::jobs_api::register_jobs_api(lua, state.clone(), caller_did.map(String::from)) + .map_err(|e| format!("jobs api: {e}"))?; record::register_record_api_no_auth(lua, state.clone()) .map_err(|e| format!("record api: {e}"))?; register_log_event_api(lua, state, trigger_id, caller_did)?; diff --git a/src/main.rs b/src/main.rs index bbbe6ca..6a1f814 100644 --- a/src/main.rs +++ b/src/main.rs @@ -681,6 +681,15 @@ async fn main() { happyview::admin::backfill::resume_backfill_jobs(&state).await; + // Resume interrupted jobs and start the job worker + happyview::jobs::worker::resume_interrupted_jobs(&state).await; + { + let job_state = state.clone(); + tokio::spawn(async move { + happyview::jobs::worker::run_worker(job_state).await; + }); + } + { let state = state.clone(); tokio::spawn(async move { diff --git a/tests/common/db.rs b/tests/common/db.rs index 3ccb9d1..80f4968 100644 --- a/tests/common/db.rs +++ b/tests/common/db.rs @@ -48,7 +48,7 @@ pub async fn truncate_all(pool: &AnyPool) { match backend { DatabaseBackend::Postgres => { sqlx::query( - "TRUNCATE happyview_records, happyview_lexicons, happyview_backfill_jobs, happyview_users, happyview_user_permissions, happyview_api_keys, happyview_event_logs, happyview_script_variables, happyview_scripts, happyview_dead_letter_scripts, happyview_dead_letter_hooks, happyview_record_refs, happyview_labeler_subscriptions, happyview_labels, happyview_instance_settings, happyview_domains, happyview_dpop_sessions, happyview_dpop_keys, happyview_api_clients, happyview_delegated_accounts, happyview_account_delegates, happyview_service_identity, happyview_service_entries, happyview_service_entry_xrpcs RESTART IDENTITY CASCADE", + "TRUNCATE happyview_records, happyview_lexicons, happyview_backfill_jobs, happyview_users, happyview_user_permissions, happyview_api_keys, happyview_event_logs, happyview_script_variables, happyview_scripts, happyview_dead_letter_scripts, happyview_dead_letter_hooks, happyview_record_refs, happyview_labeler_subscriptions, happyview_labels, happyview_instance_settings, happyview_domains, happyview_dpop_sessions, happyview_dpop_keys, happyview_api_clients, happyview_delegated_accounts, happyview_account_delegates, happyview_service_identity, happyview_service_entries, happyview_service_entry_xrpcs, happyview_jobs RESTART IDENTITY CASCADE", ) .execute(pool) .await @@ -80,6 +80,7 @@ pub async fn truncate_all(pool: &AnyPool) { "happyview_labels", "happyview_instance_settings", "happyview_domains", + "happyview_jobs", ]; for table in tables { sqlx::query(&format!("DELETE FROM {table}")) diff --git a/tests/e2e_jobs.rs b/tests/e2e_jobs.rs new file mode 100644 index 0000000..1ab363c --- /dev/null +++ b/tests/e2e_jobs.rs @@ -0,0 +1,491 @@ +mod common; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use happyview::db::adapt_sql; +use http_body_util::BodyExt; +use serde_json::{Value, json}; +use serial_test::serial; +use tower::ServiceExt; +use uuid::Uuid; + +use common::app::TestApp; + +async fn json_body(resp: axum::response::Response) -> Value { + let body = resp.into_body().collect().await.unwrap().to_bytes(); + serde_json::from_slice(&body).unwrap() +} + +fn admin_get( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), +) -> Request { + Request::builder() + .uri(uri) + .header(cookie.0, cookie.1) + .body(Body::empty()) + .unwrap() +} + +fn admin_post( + uri: &str, + cookie: (axum::http::HeaderName, axum::http::HeaderValue), + body: &Value, +) -> Request { + Request::builder() + .method("POST") + .uri(uri) + .header(cookie.0, cookie.1) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(body).unwrap())) + .unwrap() +} + +async fn seed_job(app: &TestApp, job_type: &str, status: &str) -> String { + let id = Uuid::new_v4().to_string(); + let now = happyview::db::now_rfc3339(); + let input = serde_json::to_string(&json!({"test": true})).unwrap(); + + let sql = adapt_sql( + "INSERT INTO happyview_jobs (id, job_type, status, input, progress, created_by, created_at) VALUES (?, ?, ?, ?, '{}', ?, ?)", + app.state.db_backend, + ); + sqlx::query(&sql) + .bind(&id) + .bind(job_type) + .bind(status) + .bind(&input) + .bind(&app.admin_did) + .bind(&now) + .execute(&app.state.db) + .await + .expect("seed_job: insert failed"); + + id +} + +async fn set_job_status(app: &TestApp, id: &str, status: &str) { + let sql = adapt_sql( + "UPDATE happyview_jobs SET status = ? WHERE id = ?", + app.state.db_backend, + ); + sqlx::query(&sql) + .bind(status) + .bind(id) + .execute(&app.state.db) + .await + .expect("set_job_status failed"); +} + +// --------------------------------------------------------------------------- +// List jobs +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn list_jobs_empty() { + common::require_db!(); + let app = TestApp::new().await; + + let resp = app + .router + .clone() + .oneshot(admin_get("/admin/jobs", app.admin_cookie())) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["jobs"].as_array().unwrap().len(), 0); + assert_eq!(body["cursor"], Value::Null); +} + +#[tokio::test] +#[serial] +async fn list_jobs_returns_seeded_jobs() { + common::require_db!(); + let app = TestApp::new().await; + + seed_job(&app, "test.export", "pending").await; + seed_job(&app, "test.import", "running").await; + + let resp = app + .router + .clone() + .oneshot(admin_get("/admin/jobs", app.admin_cookie())) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + let jobs = body["jobs"].as_array().unwrap(); + assert_eq!(jobs.len(), 2); +} + +#[tokio::test] +#[serial] +async fn list_jobs_filters_by_status() { + common::require_db!(); + let app = TestApp::new().await; + + seed_job(&app, "test.export", "pending").await; + seed_job(&app, "test.import", "running").await; + seed_job(&app, "test.cleanup", "completed").await; + + let resp = app + .router + .clone() + .oneshot(admin_get("/admin/jobs?status=running", app.admin_cookie())) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + let jobs = body["jobs"].as_array().unwrap(); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0]["job_type"], "test.import"); + assert_eq!(jobs[0]["status"], "running"); +} + +// --------------------------------------------------------------------------- +// Get job +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn get_job_returns_details() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "pending").await; + + let resp = app + .router + .clone() + .oneshot(admin_get(&format!("/admin/jobs/{id}"), app.admin_cookie())) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["id"], id); + assert_eq!(body["job_type"], "test.export"); + assert_eq!(body["status"], "pending"); + assert_eq!(body["input"]["test"], true); +} + +#[tokio::test] +#[serial] +async fn get_job_not_found() { + common::require_db!(); + let app = TestApp::new().await; + + let fake_id = Uuid::new_v4(); + let resp = app + .router + .clone() + .oneshot(admin_get( + &format!("/admin/jobs/{fake_id}"), + app.admin_cookie(), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::NOT_FOUND); +} + +// --------------------------------------------------------------------------- +// Cancel job +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn cancel_pending_job_sets_cancelled() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "pending").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/cancel"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["status"], "cancelled"); +} + +#[tokio::test] +#[serial] +async fn cancel_running_job_sets_cancelling() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "running").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/cancel"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["status"], "cancelling"); +} + +#[tokio::test] +#[serial] +async fn cancel_paused_job_sets_cancelled() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "paused").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/cancel"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["status"], "cancelled"); +} + +#[tokio::test] +#[serial] +async fn cancel_completed_job_returns_400() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "completed").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/cancel"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +// --------------------------------------------------------------------------- +// Pause job +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn pause_running_job_sets_pausing() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "running").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/pause"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["status"], "pausing"); +} + +#[tokio::test] +#[serial] +async fn pause_pending_job_returns_400() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "pending").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/pause"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +// --------------------------------------------------------------------------- +// Resume job +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn resume_paused_job_sets_pending() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "paused").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/resume"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = json_body(resp).await; + assert_eq!(body["status"], "pending"); +} + +#[tokio::test] +#[serial] +async fn resume_running_job_returns_400() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.export", "running").await; + + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/resume"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::BAD_REQUEST); +} + +// --------------------------------------------------------------------------- +// Auth: unauthenticated requests +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn list_jobs_without_auth_returns_401() { + common::require_db!(); + let app = TestApp::new().await; + + let resp = app + .router + .clone() + .oneshot( + Request::builder() + .uri("/admin/jobs") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); +} + +// --------------------------------------------------------------------------- +// Full lifecycle: pending → running → pausing → paused → pending → cancel +// --------------------------------------------------------------------------- + +#[tokio::test] +#[serial] +async fn full_job_lifecycle() { + common::require_db!(); + let app = TestApp::new().await; + + let id = seed_job(&app, "test.lifecycle", "pending").await; + + // Simulate worker claiming → running + set_job_status(&app, &id, "running").await; + + // Pause the running job + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/pause"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(json_body(resp).await["status"], "pausing"); + + // Simulate worker acknowledging pause + set_job_status(&app, &id, "paused").await; + + // Resume the paused job + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/resume"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(json_body(resp).await["status"], "pending"); + + // Cancel the pending job + let resp = app + .router + .clone() + .oneshot(admin_post( + &format!("/admin/jobs/{id}/cancel"), + app.admin_cookie(), + &json!({}), + )) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!(json_body(resp).await["status"], "cancelled"); + + // Verify final state + let resp = app + .router + .clone() + .oneshot(admin_get(&format!("/admin/jobs/{id}"), app.admin_cookie())) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + let job = json_body(resp).await; + assert_eq!(job["status"], "cancelled"); + assert!(job["completed_at"].is_string()); +} diff --git a/web/playwright.config.ts b/web/playwright.config.ts index 3f22d07..28f5124 100644 --- a/web/playwright.config.ts +++ b/web/playwright.config.ts @@ -30,9 +30,11 @@ export default defineConfig({ "lexicon-services.spec.ts", "lexicon-delete.spec.ts", "script-delete.spec.ts", + "script-job.spec.ts", "record-delete.spec.ts", "proxy-config.spec.ts", "spaces.spec.ts", + "jobs.spec.ts", ], dependencies: ["setup"], use: { browserName: "chromium" }, diff --git a/web/src/app/dashboard/jobs/page.tsx b/web/src/app/dashboard/jobs/page.tsx new file mode 100644 index 0000000..72a09f6 --- /dev/null +++ b/web/src/app/dashboard/jobs/page.tsx @@ -0,0 +1,515 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { + CheckCircle2, + ChevronDown, + Circle, + Loader2, + PauseCircle, + XCircle, +} from "lucide-react"; + +import { useCurrentUser } from "@/hooks/use-current-user"; +import { toastError } from "@/lib/format"; +import { + cancelJob, + getJobs, + pauseJob, + resumeJob, +} from "@/lib/api"; +import type { Job } from "@/types/jobs"; +import { SiteHeader } from "@/components/site-header"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from "@/components/ui/collapsible"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { + Sheet, + SheetContent, + SheetFooter, + SheetHeader, + SheetTitle, +} from "@/components/ui/sheet"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +const STATUS_OPTIONS = [ + { value: "all", label: "All statuses" }, + { value: "pending", label: "Pending" }, + { value: "running", label: "Running" }, + { value: "paused", label: "Paused" }, + { value: "completed", label: "Completed" }, + { value: "failed", label: "Failed" }, + { value: "cancelled", label: "Cancelled" }, +] as const; + +function statusBadge(status: string) { + switch (status) { + case "completed": + return ( + + completed + + ); + case "failed": + return failed; + case "cancelled": + return ( + + cancelled + + ); + case "cancelling": + return ( + + cancelling + + ); + case "pausing": + return ( + + pausing + + ); + case "paused": + return ( + + paused + + ); + case "running": + return ( + + running + + ); + case "pending": + return pending; + default: + return {status}; + } +} + +function statusIcon(status: string) { + switch (status) { + case "completed": + return ; + case "failed": + return ; + case "cancelled": + return ; + case "cancelling": + return ; + case "pausing": + return ; + case "paused": + return ; + case "running": + return ; + default: + return ; + } +} + +function hasContent(obj: Record | null | undefined): boolean { + if (!obj) return false; + return Object.keys(obj).length > 0; +} + +function relativeTime(dateStr: string): string { + const now = Date.now(); + const then = new Date(dateStr).getTime(); + const diff = now - then; + const seconds = Math.floor(diff / 1000); + if (seconds < 60) return "just now"; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h ago`; + const days = Math.floor(hours / 24); + return `${days}d ago`; +} + +export default function JobsPage() { + const { hasPermission } = useCurrentUser(); + const [jobs, setJobs] = useState([]); + const [statusFilter, setStatusFilter] = useState("all"); + const [selectedJobId, setSelectedJobId] = useState(null); + const [loading, setLoading] = useState(true); + + const load = useCallback(() => { + const params = statusFilter !== "all" ? { status: statusFilter } : {}; + getJobs(params) + .then((resp) => { + setJobs(resp.jobs); + setLoading(false); + }) + .catch((e) => { + toastError("Failed to load jobs", e); + setLoading(false); + }); + }, [statusFilter]); + + useEffect(() => { + setLoading(true); + load(); + }, [load]); + + // Poll every 5 seconds for active jobs + const hasActiveJobs = jobs.some( + (j) => + j.status === "running" || + j.status === "pending" || + j.status === "cancelling" || + j.status === "pausing", + ); + + useEffect(() => { + const interval = setInterval(load, hasActiveJobs ? 3000 : 10000); + return () => clearInterval(interval); + }, [load, hasActiveJobs]); + + const selectedJob = jobs.find((j) => j.id === selectedJobId) ?? null; + const canManage = hasPermission("jobs:manage"); + + return ( + <> + +
+
+

Background Jobs

+ +
+ +
+ + + + + Type + Status + Created by + Created + + + + {loading && jobs.length === 0 && ( + + + + + + )} + {!loading && jobs.length === 0 && ( + + + {statusFilter !== "all" + ? `No ${statusFilter} jobs.` + : "No background jobs yet. Jobs are created by Lua scripts via jobs.create()."} + + + )} + {jobs.map((job) => ( + setSelectedJobId(job.id)} + onKeyDown={(e) => { + if (e.key === "Enter" || e.key === " ") { + e.preventDefault(); + setSelectedJobId(job.id); + } + }} + > + + {statusIcon(job.status)} + + + {job.job_type} + + {statusBadge(job.status)} + + {job.created_by} + + + {relativeTime(job.created_at)} + + + ))} + +
+
+ + { + if (!open) { + setSelectedJobId(null); + load(); + } + }} + > + + {selectedJob && ( + + )} + + +
+ + ); +} + +function JobDetail({ + job, + canManage, + onAction, +}: { + job: Job; + canManage: boolean; + onAction: () => void; +}) { + const [actionLoading, setActionLoading] = useState(null); + const isActive = + job.status === "running" || + job.status === "cancelling" || + job.status === "pausing"; + + async function handleCancel() { + setActionLoading("cancel"); + try { + await cancelJob(job.id); + toast.success("Job cancelled"); + onAction(); + } catch (e) { + toastError("Failed to cancel job", e); + } finally { + setActionLoading(null); + } + } + + async function handlePause() { + setActionLoading("pause"); + try { + await pauseJob(job.id); + toast.success("Job paused"); + onAction(); + } catch (e) { + toastError("Failed to pause job", e); + } finally { + setActionLoading(null); + } + } + + async function handleResume() { + setActionLoading("resume"); + try { + await resumeJob(job.id); + toast.success("Job resumed"); + onAction(); + } catch (e) { + toastError("Failed to resume job", e); + } finally { + setActionLoading(null); + } + } + + return ( + <> + + + Job Details + + +
+
+
+ Job ID +

{job.id}

+
+
+ Type +

{job.job_type}

+
+
+ Status +
{statusBadge(job.status)}
+
+
+ Created by +

{job.created_by}

+
+
+ Created +

+ {new Date(job.created_at).toLocaleString()} +

+
+ {job.started_at && ( +
+ Started +

+ {new Date(job.started_at).toLocaleString()} +

+
+ )} + {job.completed_at && ( +
+ Completed +

+ {new Date(job.completed_at).toLocaleString()} +

+
+ )} +
+ + {job.error && ( +
+ Error +
+ {job.error} +
+
+ )} + + + + {job.result && } +
+ + {canManage && ( + + {(job.status === "running" || job.status === "pausing") && ( + + )} + {job.status === "paused" && ( + + )} + {isActive && ( + + )} + {job.status === "paused" && ( + + )} + + )} + + ); +} + +function JsonSection({ + title, + data, + defaultOpen = false, +}: { + title: string; + data: Record | null; + defaultOpen?: boolean; +}) { + const [open, setOpen] = useState(defaultOpen); + const empty = !hasContent(data); + + if (empty) return null; + + return ( + + + + + + 
+          {JSON.stringify(data, null, 2)}
+
+ + + ); +} diff --git a/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx index cfd207a..1624515 100644 --- a/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx +++ b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx @@ -66,13 +66,20 @@ export default function ScriptDetail() { ); }, [state, original]); - async function handleSave() { - if (!state || !script) return; + useEffect(() => { + if (!isDirty) return; + function onBeforeUnload(e: BeforeUnloadEvent) { + e.preventDefault(); + } + window.addEventListener("beforeunload", onBeforeUnload); + return () => window.removeEventListener("beforeunload", onBeforeUnload); + }, [isDirty]); + + const handleSave = useCallback(async () => { + if (!state || !script || !isDirty || saving) return; setSaving(true); setError(null); try { - // PATCH only the editable fields. Trigger id is the PK — to - // rename, delete and recreate. await patchScript(script.id, { body: state.body, description: state.description.trim() || null, @@ -83,7 +90,18 @@ export default function ScriptDetail() { } finally { setSaving(false); } - } + }, [state, script, isDirty, saving, load]); + + useEffect(() => { + function onKeyDown(e: KeyboardEvent) { + if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { + e.preventDefault(); + handleSave(); + } + } + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [handleSave]); async function handleDelete() { if (!script) return; @@ -126,6 +144,7 @@ export default function ScriptDetail() { record: "Record event", xrpc: "XRPC handler", labeler: "Label arrival", + job: "Job runner", }; return ( @@ -178,6 +197,9 @@ export default function ScriptDetail() { {canManage && ( )} diff --git a/web/src/app/dashboard/settings/scripts/new/page.tsx b/web/src/app/dashboard/settings/scripts/new/page.tsx index 33973b5..dda5e0f 100644 --- a/web/src/app/dashboard/settings/scripts/new/page.tsx +++ b/web/src/app/dashboard/settings/scripts/new/page.tsx @@ -1,20 +1,37 @@ "use client"; -import { Suspense, useEffect, useState } from "react"; +import { Suspense, useCallback, useEffect, useMemo, useState } from "react"; import { useRouter, useSearchParams } from "next/navigation"; import { useCurrentUser } from "@/hooks/use-current-user"; import { getLexicons, upsertScript } from "@/lib/api"; import type { LexiconSummary } from "@/types/lexicons"; import type { TriggerKind } from "@/types/scripts"; -import { DEFAULT_SCRIPT_BODY, parseTriggerId } from "@/types/scripts"; +import { + DEFAULT_JOB_SCRIPT_BODY, + DEFAULT_SCRIPT_BODY, + parseTriggerId, +} from "@/types/scripts"; import { SiteHeader } from "@/components/site-header"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, +} from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { + JOB_SOURCE, ScriptForm, type ScriptFormState, composeTriggerId, + isValidJobType, } from "../script-form"; function NewScriptInner() { @@ -28,6 +45,7 @@ function NewScriptInner() { // form even if the call fails (the operator can still pick "Actor" // and create a labeler.apply:_actor script). const [lexicons, setLexicons] = useState([]); + const [lexiconsLoading, setLexiconsLoading] = useState(true); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); @@ -39,23 +57,36 @@ function NewScriptInner() { useEffect(() => { getLexicons() .then(setLexicons) - .catch(() => setLexicons([])); + .catch(() => setLexicons([])) + .finally(() => setLexiconsLoading(false)); }, []); - if (!hasPermission("scripts:manage")) { + const isDirty = useMemo(() => { + const defaultBody = + state.source === JOB_SOURCE ? DEFAULT_JOB_SCRIPT_BODY : DEFAULT_SCRIPT_BODY; return ( - <> - -
-

- You don't have permission to create scripts. -

-
- + state.suffix !== "" || + state.description !== "" || + state.body !== defaultBody ); - } + }, [state]); + + useEffect(() => { + if (!isDirty) return; + function onBeforeUnload(e: BeforeUnloadEvent) { + e.preventDefault(); + } + window.addEventListener("beforeunload", onBeforeUnload); + return () => window.removeEventListener("beforeunload", onBeforeUnload); + }, [isDirty]); + + const canSave = + !saving && + !!state.suffix && + !(state.source === JOB_SOURCE && !isValidJobType(state.suffix)); - async function handleSave() { + const handleSave = useCallback(async () => { + if (!canSave) return; setSaving(true); setError(null); try { @@ -70,6 +101,30 @@ function NewScriptInner() { setError(e instanceof Error ? e.message : String(e)); setSaving(false); } + }, [canSave, state, router]); + + useEffect(() => { + function onKeyDown(e: KeyboardEvent) { + if ((e.metaKey || e.ctrlKey) && e.key === "Enter") { + e.preventDefault(); + handleSave(); + } + } + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [handleSave]); + + if (!hasPermission("scripts:manage")) { + return ( + <> + +
+

+ You don't have permission to create scripts. +

+
+ + ); } return ( @@ -78,11 +133,48 @@ function NewScriptInner() {
{error &&

{error}

} - +
-
- + + + + Discard changes? + + You have unsaved changes that will be lost. + + + + Keep editing + router.push("/dashboard/settings/scripts")} + > + Discard + + + + + ) : ( + + )} +
@@ -92,7 +184,11 @@ function NewScriptInner() { export default function NewScriptPage() { return ( - + + } + > ); @@ -105,21 +201,26 @@ function initialState(searchParams: URLSearchParams): ScriptFormState { if (presetId) { const parsed = parseTriggerId(presetId); if (parsed) { + const isJob = parsed.kind === "job.run"; return { kind: parsed.kind, suffix: parsed.suffix, + source: isJob ? JOB_SOURCE : parsed.suffix, description: "", - body: DEFAULT_SCRIPT_BODY, + body: isJob ? DEFAULT_JOB_SCRIPT_BODY : DEFAULT_SCRIPT_BODY, }; } } // Fallbacks to a sensible default. Suffix starts empty so the form - // surfaces the "Pick a lexicon to compose the trigger id" hint. + // surfaces the "Pick a source to compose the trigger id" hint. const kind = (searchParams.get("kind") as TriggerKind | null) ?? "record.index"; + const source = searchParams.get("source") ?? searchParams.get("suffix") ?? ""; + const isJob = kind === "job.run" || source === JOB_SOURCE; return { - kind, - suffix: searchParams.get("suffix") ?? "", + kind: isJob ? "job.run" : kind, + suffix: isJob ? "" : (searchParams.get("suffix") ?? ""), + source: isJob ? JOB_SOURCE : source, description: "", - body: DEFAULT_SCRIPT_BODY, + body: isJob ? DEFAULT_JOB_SCRIPT_BODY : DEFAULT_SCRIPT_BODY, }; } diff --git a/web/src/app/dashboard/settings/scripts/script-form.tsx b/web/src/app/dashboard/settings/scripts/script-form.tsx index f4f5862..6f3621f 100644 --- a/web/src/app/dashboard/settings/scripts/script-form.tsx +++ b/web/src/app/dashboard/settings/scripts/script-form.tsx @@ -4,6 +4,7 @@ import { useEffect, useMemo, useRef } from "react"; import { MonacoEditor } from "@/components/monaco-editor"; import { Badge } from "@/components/ui/badge"; +import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { Select, @@ -18,7 +19,12 @@ import { import { Textarea } from "@/components/ui/textarea"; import type { LexiconSummary } from "@/types/lexicons"; import type { TriggerKind } from "@/types/scripts"; -import { TRIGGER_KIND_LABELS, parseTriggerId } from "@/types/scripts"; +import { + DEFAULT_JOB_SCRIPT_BODY, + DEFAULT_SCRIPT_BODY, + TRIGGER_KIND_LABELS, + parseTriggerId, +} from "@/types/scripts"; /** * Sentinel suffix used when the operator picks "Actor" in the lexicon @@ -27,15 +33,27 @@ import { TRIGGER_KIND_LABELS, parseTriggerId } from "@/types/scripts"; */ export const ACTOR_SUFFIX = "_actor"; +/** + * Sentinel value for the source selector when the operator picks "Job". + * The actual suffix is typed into a free-form input (the job type name). + */ +export const JOB_SOURCE = "_job"; + export interface ScriptFormState { /** Trigger kind selector value (e.g. `record.create`). */ kind: TriggerKind; /** * Suffix portion of the trigger id — usually an NSID (= a lexicon id), * or the literal `_actor` when `kind === "labeler.apply"` for - * actor-level labels. + * actor-level labels. For jobs, this is the user-typed job type name. */ suffix: string; + /** + * Which source-selector value was chosen. Usually identical to `suffix` + * (i.e. a lexicon NSID or `_actor`). For jobs this is `_job` while + * `suffix` holds the free-form job type name. + */ + source: string; description: string; body: string; } @@ -51,9 +69,15 @@ export function stateFromScript(args: { body: string; }): ScriptFormState { const parsed = parseTriggerId(args.id); + const kind = parsed?.kind ?? "record.index"; + const suffix = parsed?.suffix ?? ""; + let source = suffix; + if (kind === "job.run") source = JOB_SOURCE; + else if (suffix === ACTOR_SUFFIX) source = ACTOR_SUFFIX; return { - kind: parsed?.kind ?? "record.index", - suffix: parsed?.suffix ?? "", + kind, + suffix, + source, description: args.description ?? "", body: args.body, }; @@ -93,9 +117,23 @@ const PROCEDURE_ACTIONS: ActionOption[] = [ { kind: "xrpc.procedure", label: "Procedure handler" }, ]; -function actionsFor(suffix: string, lexicons: LexiconSummary[]): ActionOption[] { - if (suffix === ACTOR_SUFFIX) return ACTOR_ACTIONS; - const lex = lexicons.find((l) => l.id === suffix); +const JOB_ACTIONS: ActionOption[] = [{ kind: "job.run", label: "Job runner" }]; + +const JOB_TYPE_PATTERN = /^[a-z0-9][a-z0-9._-]*$/; + +export function isValidJobType(value: string): boolean { + return ( + value.length > 0 && value.length <= 128 && JOB_TYPE_PATTERN.test(value) + ); +} + +function actionsFor( + source: string, + lexicons: LexiconSummary[], +): ActionOption[] { + if (source === ACTOR_SUFFIX) return ACTOR_ACTIONS; + if (source === JOB_SOURCE) return JOB_ACTIONS; + const lex = lexicons.find((l) => l.id === source); if (!lex) return []; switch (lex.lexicon_type) { case "record": @@ -129,12 +167,15 @@ export function ScriptForm({ onChange, idLocked, lexicons, + lexiconsLoading, }: { state: ScriptFormState; onChange: (next: ScriptFormState) => void; idLocked?: boolean; /** Required when `idLocked` is false; ignored otherwise. */ lexicons?: LexiconSummary[]; + /** True while the lexicon list is being fetched. */ + lexiconsLoading?: boolean; }) { return (
@@ -145,6 +186,7 @@ export function ScriptForm({ state={state} onChange={onChange} lexicons={lexicons ?? []} + lexiconsLoading={lexiconsLoading} /> )} @@ -193,20 +235,24 @@ function TriggerComposer({ state, onChange, lexicons, + lexiconsLoading, }: { state: ScriptFormState; onChange: (next: ScriptFormState) => void; lexicons: LexiconSummary[]; + lexiconsLoading?: boolean; }) { const sortedLexicons = useMemo( () => [...lexicons].sort((a, b) => a.id.localeCompare(b.id)), [lexicons], ); const actions = useMemo( - () => actionsFor(state.suffix, lexicons), - [state.suffix, lexicons], + () => actionsFor(state.source, lexicons), + [state.source, lexicons], ); + const isJob = state.source === JOB_SOURCE; + const stateRef = useRef(state); stateRef.current = state; @@ -218,14 +264,34 @@ function TriggerComposer({ } }, [actions, onChange]); - function handleSuffixChange(next: string) { - // Pre-snap kind so the resolved trigger id badge updates immediately - // rather than flickering through an invalid state. + function handleSourceChange(next: string) { + const wasJob = state.source === JOB_SOURCE; + const isNowJob = next === JOB_SOURCE; + const bodyIsDefault = + state.body === DEFAULT_SCRIPT_BODY || + state.body === DEFAULT_JOB_SCRIPT_BODY; + + if (isNowJob) { + onChange({ + ...state, + source: JOB_SOURCE, + suffix: "", + kind: "job.run", + body: bodyIsDefault ? DEFAULT_JOB_SCRIPT_BODY : state.body, + }); + return; + } const nextActions = actionsFor(next, lexicons); const nextKind = nextActions.some((a) => a.kind === state.kind) ? state.kind : (nextActions[0]?.kind ?? state.kind); - onChange({ ...state, suffix: next, kind: nextKind }); + onChange({ + ...state, + source: next, + suffix: next, + kind: nextKind, + body: wasJob && bodyIsDefault ? DEFAULT_SCRIPT_BODY : state.body, + }); } const triggerPreview = @@ -235,12 +301,12 @@ function TriggerComposer({ <>
-
- - + {isJob ? ( + <> + + onChange({ ...state, suffix: e.target.value })} + placeholder="e.g. export, migrate, sync" + className="h-8 text-sm font-mono" + aria-invalid={ + state.suffix.length > 0 && !isValidJobType(state.suffix) + } + /> + {state.suffix.length > 0 && !isValidJobType(state.suffix) ? ( +

+ Lowercase letters, numbers, dots, hyphens, and underscores + only. +

+ ) : ( +

+ Must match the type passed to{" "} + + jobs.create() + {" "} + in the queuing script. +

+ )} + + ) : ( + <> + + + + )}
@@ -304,7 +420,9 @@ function TriggerComposer({ {triggerPreview} ) : ( - Pick a lexicon to compose the trigger id. + {isJob + ? "Enter a job type to compose the trigger id." + : "Pick a source to compose the trigger id."} )}

diff --git a/web/src/components/app-sidebar.tsx b/web/src/components/app-sidebar.tsx index 95cab2a..1151d8a 100644 --- a/web/src/components/app-sidebar.tsx +++ b/web/src/components/app-sidebar.tsx @@ -22,6 +22,7 @@ import { IconSkull, IconFlask, IconFingerprint, + IconPlayerPlay, } from "@tabler/icons-react"; import Image from "next/image"; import Link from "next/link"; @@ -59,6 +60,12 @@ const dataItems: NavItem[] = [ { title: "Lexicons", url: "/dashboard/lexicons", icon: IconFileDescription }, { title: "Records", url: "/dashboard/records", icon: IconTable }, { title: "Backfill", url: "/dashboard/backfill", icon: IconDatabase }, + { + title: "Jobs", + url: "/dashboard/jobs", + icon: IconPlayerPlay, + requiredPermissions: ["jobs:read"], + }, { title: "Dead Letters", url: "/dashboard/dead-letters", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 120b3db..378e939 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -3,6 +3,7 @@ import type { StatsResponse } from "@/types/stats"; import type { LexiconSummary, LexiconDetail } from "@/types/lexicons"; import type { NetworkLexiconSummary } from "@/types/network-lexicons"; import type { BackfillJob, BackfillReposResponse, PdsSummaryResponse } from "@/types/backfill"; +import type { Job, JobsListResponse } from "@/types/jobs"; import type { UserSummary } from "@/types/users"; import type { AdminListRecordsResponse } from "@/types/records"; import type { EventsListResponse } from "@/types/events"; @@ -254,6 +255,38 @@ export function flushAllBackfillDetails() { return apiFetch(`/admin/backfill/details`, { method: "DELETE" }); } +// Jobs +export function getJobs(params: { status?: string; limit?: number; cursor?: string } = {}) { + const qs = new URLSearchParams(); + if (params.status) qs.set("status", params.status); + if (params.limit) qs.set("limit", String(params.limit)); + if (params.cursor) qs.set("cursor", params.cursor); + const query = qs.toString(); + return apiFetch(`/admin/jobs${query ? `?${query}` : ""}`); +} + +export function getJob(id: string) { + return apiFetch(`/admin/jobs/${id}`); +} + +export function cancelJob(id: string) { + return apiFetch<{ status: string }>(`/admin/jobs/${id}/cancel`, { + method: "POST", + }); +} + +export function pauseJob(id: string) { + return apiFetch<{ status: string }>(`/admin/jobs/${id}/pause`, { + method: "POST", + }); +} + +export function resumeJob(id: string) { + return apiFetch<{ status: string }>(`/admin/jobs/${id}/resume`, { + method: "POST", + }); +} + // Users export function getUsers() { return apiFetch("/admin/users"); diff --git a/web/src/types/jobs.ts b/web/src/types/jobs.ts new file mode 100644 index 0000000..c457980 --- /dev/null +++ b/web/src/types/jobs.ts @@ -0,0 +1,18 @@ +export interface Job { + id: string; + job_type: string; + status: string; + input: Record; + progress: Record; + result: Record | null; + error: string | null; + created_by: string; + started_at: string | null; + completed_at: string | null; + created_at: string; +} + +export interface JobsListResponse { + jobs: Job[]; + cursor: string | null; +} diff --git a/web/src/types/scripts.ts b/web/src/types/scripts.ts index 614d73f..eef2e6b 100644 --- a/web/src/types/scripts.ts +++ b/web/src/types/scripts.ts @@ -65,6 +65,7 @@ export type TriggerKind = | "xrpc.query" | "xrpc.procedure" | "labeler.apply" + | "job.run" /** Display labels for each trigger kind. */ export const TRIGGER_KIND_LABELS: Record = { @@ -75,21 +76,24 @@ export const TRIGGER_KIND_LABELS: Record = { "xrpc.query": "XRPC query", "xrpc.procedure": "XRPC procedure", "labeler.apply": "Label arrival", + "job.run": "Job runner", } /** Top-level grouping for the Scripts list page. */ -export type TriggerFamily = "record" | "xrpc" | "labeler" +export type TriggerFamily = "record" | "xrpc" | "labeler" | "job" export const TRIGGER_FAMILY_LABELS: Record = { record: "Record events", xrpc: "XRPC handlers", labeler: "Label arrivals", + job: "Job runners", } /** Map a trigger kind to its top-level family. */ export function familyOf(kind: TriggerKind): TriggerFamily { if (kind.startsWith("record.")) return "record" if (kind.startsWith("xrpc.")) return "xrpc" + if (kind.startsWith("job.")) return "job" return "labeler" } @@ -113,6 +117,7 @@ export function parseTriggerId( "xrpc.query", "xrpc.procedure", "labeler.apply", + "job.run", ] as const).find((k) => k === prefix) if (!kind) return null return { kind, suffix } @@ -133,3 +138,28 @@ function handle() return event end ` + +export const DEFAULT_JOB_SCRIPT_BODY = `-- Job runner: executes as a background job. +-- +-- Available globals: +-- job.input — the input table passed to jobs.create() +-- job.id — the job's UUID +-- job.progress() — persist progress (visible in the dashboard) +-- job.should_stop() — check for pause/cancel (cooperative) +-- job.wait(seconds) — sleep (0–3600s) +-- +-- Available APIs: db.*, http.*, xrpc.*, atproto.*, Record.*, env. +-- Return value becomes the job's result. + +function handle() + local input = job.input + + job.progress({ status = "working" }) + + if job.should_stop() then + return { partial = true } + end + + return { done = true } +end +` diff --git a/web/tests/e2e/jobs.spec.ts b/web/tests/e2e/jobs.spec.ts new file mode 100644 index 0000000..581ec97 --- /dev/null +++ b/web/tests/e2e/jobs.spec.ts @@ -0,0 +1,216 @@ +import { test, expect } from "@playwright/test" +import { randomUUID } from "crypto" +import pg from "pg" +import { loginAsTestAdmin } from "./auth-helper" + +const DB_URL = "postgres://happyview:happyview@localhost:5434/happyview_test" +const TEST_DID = "did:plc:e2e-test-admin" + +async function seedJob( + status: string, + jobType = "test.e2e.export", +): Promise { + const client = new pg.Client(DB_URL) + await client.connect() + try { + const id = randomUUID() + const now = new Date().toISOString() + await client.query( + `INSERT INTO happyview_jobs (id, job_type, status, input, progress, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + [ + id, + jobType, + status, + JSON.stringify({ source: "e2e-test" }), + JSON.stringify({}), + TEST_DID, + now, + ], + ) + return id + } finally { + await client.end() + } +} + +async function cleanupJobs(): Promise { + const client = new pg.Client(DB_URL) + await client.connect() + try { + await client.query( + "DELETE FROM happyview_jobs WHERE created_by = $1", + [TEST_DID], + ) + } finally { + await client.end() + } +} + +test.describe("Jobs Dashboard", () => { + test.beforeEach(async ({ page }) => { + await loginAsTestAdmin(page) + }) + + test.afterEach(async () => { + await cleanupJobs() + }) + + test("shows empty state when no jobs exist", async ({ page }) => { + await page.goto("/dashboard/jobs") + + await expect( + page.getByText("No background jobs yet"), + ).toBeVisible({ timeout: 5000 }) + }) + + test("lists seeded jobs in the table", async ({ page }) => { + await seedJob("pending", "test.e2e.alpha") + await seedJob("running", "test.e2e.beta") + + await page.goto("/dashboard/jobs") + + const rows = page.locator("table tbody tr") + await expect(rows).toHaveCount(2, { timeout: 5000 }) + + await expect(page.getByText("test.e2e.alpha")).toBeVisible() + await expect(page.getByText("test.e2e.beta")).toBeVisible() + }) + + test("filters jobs by status", async ({ page }) => { + await seedJob("pending", "test.e2e.pending-job") + await seedJob("completed", "test.e2e.completed-job") + + await page.goto("/dashboard/jobs") + + const rows = page.locator("table tbody tr") + await expect(rows).toHaveCount(2, { timeout: 5000 }) + + await page.getByRole("combobox").click() + await page.getByRole("option", { name: "Completed" }).click() + + await expect(rows).toHaveCount(1, { timeout: 5000 }) + await expect(page.getByText("test.e2e.completed-job")).toBeVisible() + await expect(page.getByText("test.e2e.pending-job")).not.toBeVisible() + }) + + test("opens detail sheet when clicking a job row", async ({ page }) => { + const id = await seedJob("pending", "test.e2e.detail") + + await page.goto("/dashboard/jobs") + + const row = page.locator("table tbody tr", { + hasText: "test.e2e.detail", + }) + await expect(row).toBeVisible({ timeout: 5000 }) + await row.click() + + const sheet = page.locator("[data-state='open'][role='dialog']") + await expect(sheet).toBeVisible({ timeout: 3000 }) + + await expect(sheet.getByText("Job Details")).toBeVisible() + await expect(sheet.getByText(id)).toBeVisible() + await expect(sheet.getByText("test.e2e.detail")).toBeVisible() + await expect(sheet.getByText("pending")).toBeVisible() + }) + + test("shows cancel button for running job and cancels it", async ({ + page, + }) => { + const id = await seedJob("running", "test.e2e.cancel") + + await page.goto("/dashboard/jobs") + + const row = page.locator("table tbody tr", { + hasText: "test.e2e.cancel", + }) + await expect(row).toBeVisible({ timeout: 5000 }) + await row.click() + + const sheet = page.locator("[data-state='open'][role='dialog']") + await expect(sheet).toBeVisible({ timeout: 3000 }) + + const cancelButton = sheet.getByRole("button", { name: "Cancel Job" }) + await expect(cancelButton).toBeVisible() + await cancelButton.click() + + await expect(page.getByText("Job cancelled")).toBeVisible({ + timeout: 5000, + }) + }) + + test("shows pause button for running job", async ({ page }) => { + await seedJob("running", "test.e2e.pause") + + await page.goto("/dashboard/jobs") + + const row = page.locator("table tbody tr", { + hasText: "test.e2e.pause", + }) + await expect(row).toBeVisible({ timeout: 5000 }) + await row.click() + + const sheet = page.locator("[data-state='open'][role='dialog']") + await expect(sheet).toBeVisible({ timeout: 3000 }) + + await expect( + sheet.getByRole("button", { name: "Pause Job" }), + ).toBeVisible() + }) + + test("shows resume button for paused job", async ({ page }) => { + await seedJob("paused", "test.e2e.resume") + + await page.goto("/dashboard/jobs") + + const row = page.locator("table tbody tr", { + hasText: "test.e2e.resume", + }) + await expect(row).toBeVisible({ timeout: 5000 }) + await row.click() + + const sheet = page.locator("[data-state='open'][role='dialog']") + await expect(sheet).toBeVisible({ timeout: 3000 }) + + await expect( + sheet.getByRole("button", { name: "Resume Job" }), + ).toBeVisible() + }) + + test("shows error section for failed job", async ({ page }) => { + const client = new pg.Client(DB_URL) + await client.connect() + try { + const id = randomUUID() + const now = new Date().toISOString() + await client.query( + `INSERT INTO happyview_jobs (id, job_type, status, input, progress, error, created_by, created_at, completed_at) + VALUES ($1, $2, 'failed', $3, $4, $5, $6, $7, $7)`, + [ + id, + "test.e2e.failed", + JSON.stringify({}), + JSON.stringify({}), + "something went wrong", + TEST_DID, + now, + ], + ) + } finally { + await client.end() + } + + await page.goto("/dashboard/jobs") + + const row = page.locator("table tbody tr", { + hasText: "test.e2e.failed", + }) + await expect(row).toBeVisible({ timeout: 5000 }) + await row.click() + + const sheet = page.locator("[data-state='open'][role='dialog']") + await expect(sheet).toBeVisible({ timeout: 3000 }) + + await expect(sheet.getByText("something went wrong")).toBeVisible() + }) +}) diff --git a/web/tests/e2e/script-job.spec.ts b/web/tests/e2e/script-job.spec.ts new file mode 100644 index 0000000..adc1448 --- /dev/null +++ b/web/tests/e2e/script-job.spec.ts @@ -0,0 +1,90 @@ +import { test, expect } from "@playwright/test" +import { loginAsTestAdmin } from "./auth-helper" + +const JOB_TYPE = "test.e2e.myjob" +const TRIGGER_ID = `job.run:${JOB_TYPE}` + +async function cleanupScript( + request: import("@playwright/test").APIRequestContext, +) { + await request.delete(`/admin/scripts/${encodeURIComponent(TRIGGER_ID)}`) +} + +test.describe("Job Script Creation", () => { + test.beforeEach(async ({ page }) => { + await loginAsTestAdmin(page) + }) + + test.afterEach(async ({ page }) => { + await cleanupScript(page.request) + }) + + test("selecting Job source shows job type input and composes trigger id", async ({ + page, + }) => { + await page.goto("/dashboard/settings/scripts/new") + + const sourceSelect = page.locator("#source-pick") + await expect(sourceSelect).toBeVisible({ timeout: 5000 }) + + await sourceSelect.click() + await page.getByRole("option", { name: /Job/ }).click() + + const jobTypeInput = page.locator("#job-type-input") + await expect(jobTypeInput).toBeVisible() + + await expect(page.locator("#action-pick")).not.toBeVisible() + + await jobTypeInput.fill(JOB_TYPE) + + await expect(page.getByText(TRIGGER_ID)).toBeVisible() + }) + + test("creating a job script navigates to detail page", async ({ page }) => { + await page.goto("/dashboard/settings/scripts/new") + + await page.locator("#source-pick").click() + await page.getByRole("option", { name: /Job/ }).click() + + await page.locator("#job-type-input").fill(JOB_TYPE) + + await page.getByRole("button", { name: "Create script" }).click() + + await page.waitForURL( + `**/dashboard/settings/scripts/${encodeURIComponent(TRIGGER_ID)}`, + { timeout: 5000 }, + ) + + await expect(page.getByText("Job runner")).toBeVisible() + await expect(page.getByText(TRIGGER_ID)).toBeVisible() + }) + + test("job script has job-specific template body", async ({ page }) => { + await page.goto("/dashboard/settings/scripts/new") + + await page.locator("#source-pick").click() + await page.getByRole("option", { name: /Job/ }).click() + + await expect(page.getByText("job.input")).toBeVisible({ timeout: 3000 }) + await expect(page.getByText("job.should_stop")).toBeVisible() + }) + + test("job script appears in scripts list with Job runners family", async ({ + page, + }) => { + await page.request.post("/admin/scripts", { + data: { + id: TRIGGER_ID, + body: "function handle()\n return { ok = true }\nend", + }, + }) + + await page.goto("/dashboard/settings/scripts") + + const row = page.locator("table tbody tr", { hasText: JOB_TYPE }) + await expect(row).toBeVisible({ timeout: 5000 }) + + await expect(row.getByText("Job runner")).toBeVisible() + await expect(row.getByText("Job runners")).toBeVisible() + }) +}) -- 2.51.2 From e8914e112225a3bd25effffd1f99560c8b465098 Mon Sep 17 00:00:00 2001 From: Trezy Date: Thu, 2 Jul 2026 11:05:36 -0500 Subject: [PATCH 2/3] docs: add jobs docs Signed-off-by: Trezy --- bun.lock | 38 +-- .../content/docs/api-reference/admin/jobs.md | 151 ++++++++++++ .../docs/api-reference/admin/meta.json | 1 + .../docs/api-reference/lua/jobs-api.md | 156 ++++++++++++ .../content/docs/api-reference/lua/meta.json | 1 + .../content/docs/guides/background-jobs.md | 232 ++++++++++++++++++ .../docs/content/docs/guides/lua-scripting.md | 31 ++- packages/docs/content/docs/guides/meta.json | 1 + .../content/docs/guides/record-scripts.md | 8 + packages/docs/package.json | 2 +- 10 files changed, 581 insertions(+), 40 deletions(-) create mode 100644 packages/docs/content/docs/api-reference/admin/jobs.md create mode 100644 packages/docs/content/docs/api-reference/lua/jobs-api.md create mode 100644 packages/docs/content/docs/guides/background-jobs.md diff --git a/bun.lock b/bun.lock index 78347db..b049d38 100644 --- a/bun.lock +++ b/bun.lock @@ -24,7 +24,7 @@ "fumadocs-mdx": "^15.0.4", "fumadocs-ui": "^16.8.10", "lucide-react": "^1.14.0", - "mermaid": "^11.6.0", + "mermaid": "^11.16.0", "next": "^16.1.6", "next-themes": "^0.4.6", "react": "^19.2.0", @@ -196,15 +196,7 @@ "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.2", "", {}, "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA=="], - "@chevrotain/cst-dts-gen": ["@chevrotain/cst-dts-gen@12.0.0", "", { "dependencies": { "@chevrotain/gast": "12.0.0", "@chevrotain/types": "12.0.0" } }, "sha512-fSL4KXjTl7cDgf0B5Rip9Q05BOrYvkJV/RrBTE/bKDN096E4hN/ySpcBK5B24T76dlQ2i32Zc3PAE27jFnFrKg=="], - - "@chevrotain/gast": ["@chevrotain/gast@12.0.0", "", { "dependencies": { "@chevrotain/types": "12.0.0" } }, "sha512-1ne/m3XsIT8aEdrvT33so0GUC+wkctpUPK6zU9IlOyJLUbR0rg4G7ZiApiJbggpgPir9ERy3FRjT6T7lpgetnQ=="], - - "@chevrotain/regexp-to-ast": ["@chevrotain/regexp-to-ast@12.0.0", "", {}, "sha512-p+EW9MaJwgaHguhoqwOtx/FwuGr+DnNn857sXWOi/mClXIkPGl3rn7hGNWvo31HA3vyeQxjqe+H36yZJwYU8cA=="], - - "@chevrotain/types": ["@chevrotain/types@12.0.0", "", {}, "sha512-S+04vjFQKeuYw0/eW3U52LkAHQsB1ASxsPGsLPUyQgrZ2iNNibQrsidruDzjEX2JYfespXMG0eZmXlhA6z7nWA=="], - - "@chevrotain/utils": ["@chevrotain/utils@12.0.0", "", {}, "sha512-lB59uJoaGIfOOL9knQqQRfhl9g7x8/wqFkp13zTdkRu1huG9kg6IJs1O8hqj9rs6h7orGxHJUKb+mX3rPbWGhA=="], + "@chevrotain/types": ["@chevrotain/types@11.1.2", "", {}, "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw=="], "@colors/colors": ["@colors/colors@1.5.0", "", {}, "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="], @@ -352,7 +344,7 @@ "@mdx-js/mdx": ["@mdx-js/mdx@3.1.1", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdx": "^2.0.0", "acorn": "^8.0.0", "collapse-white-space": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-util-scope": "^1.0.0", "estree-walker": "^3.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "markdown-extensions": "^2.0.0", "recma-build-jsx": "^1.0.0", "recma-jsx": "^1.0.0", "recma-stringify": "^1.0.0", "rehype-recma": "^1.0.0", "remark-mdx": "^3.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "source-map": "^0.7.0", "unified": "^11.0.0", "unist-util-position-from-estree": "^2.0.0", "unist-util-stringify-position": "^4.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ=="], - "@mermaid-js/parser": ["@mermaid-js/parser@1.1.0", "", { "dependencies": { "langium": "^4.0.0" } }, "sha512-gxK9ZX2+Fex5zu8LhRQoMeMPEHbc73UKZ0FQ54YrQtUxE1VVhMwzeNtKRPAu5aXks4FasbMe4xB4bWrmq6Jlxw=="], + "@mermaid-js/parser": ["@mermaid-js/parser@1.2.0", "", { "dependencies": { "@chevrotain/types": "~11.1.2" } }, "sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA=="], "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], @@ -770,10 +762,6 @@ "character-reference-invalid": ["character-reference-invalid@2.0.1", "", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], - "chevrotain": ["chevrotain@12.0.0", "", { "dependencies": { "@chevrotain/cst-dts-gen": "12.0.0", "@chevrotain/gast": "12.0.0", "@chevrotain/regexp-to-ast": "12.0.0", "@chevrotain/types": "12.0.0", "@chevrotain/utils": "12.0.0" } }, "sha512-csJvb+6kEiQaqo1woTdSAuOWdN0WTLIydkKrBnS+V5gZz0oqBrp4kQ35519QgK6TpBThiG3V1vNSHlIkv4AglQ=="], - - "chevrotain-allstar": ["chevrotain-allstar@0.4.1", "", { "dependencies": { "lodash-es": "^4.17.21" }, "peerDependencies": { "chevrotain": "^12.0.0" } }, "sha512-PvVJm3oGqrveUVW2Vt/eZGeiAIsJszYweUcYwcskg9e+IubNYKKD+rHHem7A6XVO22eDAL+inxNIGAzZ/VIWlA=="], - "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], "class-variance-authority": ["class-variance-authority@0.7.1", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="], @@ -838,7 +826,7 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], - "cytoscape": ["cytoscape@3.33.2", "", {}, "sha512-sj4HXd3DokGhzZAdjDejGvTPLqlt84vNFN8m7bGsOzDY5DyVcxIb2ejIXat2Iy7HxWhdT/N1oKyheJ5YdpsGuw=="], + "cytoscape": ["cytoscape@3.34.0", "", {}, "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg=="], "cytoscape-cose-bilkent": ["cytoscape-cose-bilkent@4.1.0", "", { "dependencies": { "cose-base": "^1.0.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ=="], @@ -956,6 +944,8 @@ "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + "es-toolkit": ["es-toolkit@1.49.0", "", {}, "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g=="], + "esast-util-from-estree": ["esast-util-from-estree@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "unist-util-position-from-estree": "^2.0.0" } }, "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ=="], "esast-util-from-js": ["esast-util-from-js@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "acorn": "^8.0.0", "esast-util-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw=="], @@ -1188,8 +1178,6 @@ "khroma": ["khroma@2.1.0", "", {}, "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw=="], - "langium": ["langium@4.2.2", "", { "dependencies": { "@chevrotain/regexp-to-ast": "~12.0.0", "chevrotain": "~12.0.0", "chevrotain-allstar": "~0.4.1", "vscode-languageserver": "~9.0.1", "vscode-languageserver-textdocument": "~1.0.11", "vscode-uri": "~3.1.0" } }, "sha512-JUshTRAfHI4/MF9dH2WupvjSXyn8JBuUEWazB8ZVJUtXutT0doDlAv1XKbZ1Pb5sMexa8FF4CFBc0iiul7gbUQ=="], - "layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="], "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], @@ -1294,7 +1282,7 @@ "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], - "mermaid": ["mermaid@11.14.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.1.0", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.1", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.19", "dompurify": "^3.3.1", "katex": "^0.16.25", "khroma": "^2.1.0", "lodash-es": "^4.17.23", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0" } }, "sha512-GSGloRsBs+JINmmhl0JDwjpuezCsHB4WGI4NASHxL3fHo3o/BRXTxhDLKnln8/Q0lRFRyDdEjmk1/d5Sn1Xz8g=="], + "mermaid": ["mermaid@11.16.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.2", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.2.0", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.3", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.20", "dompurify": "^3.3.3", "es-toolkit": "^1.45.1", "katex": "^0.16.45", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" } }, "sha512-Zvm3kbstgdpvIJPPItlL7fppIZ3kibvc1oZIGxdvk9t6UFz6flv+Jw7FtRGKwfcI8OckmH04LqG6LlS6X4B1pA=="], "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], @@ -1794,18 +1782,6 @@ "vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], - "vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], - - "vscode-languageserver": ["vscode-languageserver@9.0.1", "", { "dependencies": { "vscode-languageserver-protocol": "3.17.5" }, "bin": { "installServerIntoExtension": "bin/installServerIntoExtension" } }, "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g=="], - - "vscode-languageserver-protocol": ["vscode-languageserver-protocol@3.17.5", "", { "dependencies": { "vscode-jsonrpc": "8.2.0", "vscode-languageserver-types": "3.17.5" } }, "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg=="], - - "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.12", "", {}, "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA=="], - - "vscode-languageserver-types": ["vscode-languageserver-types@3.17.5", "", {}, "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg=="], - - "vscode-uri": ["vscode-uri@3.1.0", "", {}, "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ=="], - "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], "web-worker": ["web-worker@1.5.0", "", {}, "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw=="], diff --git a/packages/docs/content/docs/api-reference/admin/jobs.md b/packages/docs/content/docs/api-reference/admin/jobs.md new file mode 100644 index 0000000..ce55a13 --- /dev/null +++ b/packages/docs/content/docs/api-reference/admin/jobs.md @@ -0,0 +1,151 @@ +--- +title: "Jobs" +--- + +Admin API endpoints for managing background jobs. For a conceptual overview, see [Background Jobs](../../guides/background-jobs.md). + +## List jobs + +```http +GET /admin/jobs +``` + +Returns a paginated list of jobs, newest first. + +**Query parameters:** + +| Parameter | Type | Description | +| --------- | ------ | ------------------------------------------------------------------------------------- | +| `status` | string | Filter by status (`pending`, `running`, `completed`, `failed`, `paused`, `cancelled`) | +| `limit` | number | Maximum number of results (default: 50) | +| `cursor` | string | Pagination cursor from a previous response | + +**Permission:** `jobs:read` + +**Response:** + +```json +{ + "jobs": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "job_type": "export", + "status": "completed", + "input": { "collection": "xyz.statusphere.status" }, + "progress": { "processed": 1500 }, + "result": { "processed": 1500 }, + "error": null, + "created_by": "did:plc:abc123", + "inherit_auth": false, + "started_at": "2026-07-01T12:00:05Z", + "completed_at": "2026-07-01T12:02:30Z", + "created_at": "2026-07-01T12:00:00Z" + } + ], + "cursor": "next-page-cursor" +} +``` + +## Get job + +```http +GET /admin/jobs/:id +``` + +Returns a single job by ID. + +**Permission:** `jobs:read` + +**Response:** Same shape as a single item in the list response. + +## Cancel job + +```http +POST /admin/jobs/:id/cancel +``` + +Request cancellation of a job. If the job is `pending` or `paused`, it's immediately set to `cancelled`. If the job is `running`, it's set to `cancelling` — the worker will stop the job when the script next calls `job.should_stop()`. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "cancelling" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ---------------------------------------------- | +| 404 | Job not found | +| 409 | Job is already completed, failed, or cancelled | + +## Pause job + +```http +POST /admin/jobs/:id/pause +``` + +Request pause of a running job. Sets the status to `pausing` — the worker will pause the job when the script next calls `job.should_stop()`. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "pausing" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ------------------ | +| 404 | Job not found | +| 409 | Job is not running | + +## Resume job + +```http +POST /admin/jobs/:id/resume +``` + +Resume a paused job. Sets the status back to `pending` so the worker picks it up again. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "pending" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ----------------- | +| 404 | Job not found | +| 409 | Job is not paused | + +## Job object + +| Field | Type | Description | +| -------------- | ------------ | ------------------------------------------------------------------------------- | +| `id` | string | UUID | +| `job_type` | string | The type name passed to `jobs.create()` | +| `status` | string | Current status (see [lifecycle](../../guides/background-jobs.md#job-lifecycle)) | +| `input` | object | Input data passed to `jobs.create()` | +| `progress` | object | Last progress update from `job.progress()` | +| `result` | object\|null | Return value of the script on completion | +| `error` | string\|null | Error message on failure | +| `created_by` | string | DID of the user who enqueued the job | +| `inherit_auth` | boolean | Whether the job inherits the creator's PDS auth | +| `started_at` | string\|null | ISO 8601 timestamp when the worker started executing | +| `completed_at` | string\|null | ISO 8601 timestamp when the job finished | +| `created_at` | string | ISO 8601 timestamp when the job was enqueued | diff --git a/packages/docs/content/docs/api-reference/admin/meta.json b/packages/docs/content/docs/api-reference/admin/meta.json index 79305b5..e93ac1b 100644 --- a/packages/docs/content/docs/api-reference/admin/meta.json +++ b/packages/docs/content/docs/api-reference/admin/meta.json @@ -7,6 +7,7 @@ "records", "stats", "backfill", + "jobs", "events", "api-keys", "users", diff --git a/packages/docs/content/docs/api-reference/lua/jobs-api.md b/packages/docs/content/docs/api-reference/lua/jobs-api.md new file mode 100644 index 0000000..bd2a81d --- /dev/null +++ b/packages/docs/content/docs/api-reference/lua/jobs-api.md @@ -0,0 +1,156 @@ +--- +title: "Jobs API" +--- + +Lua API for creating and managing background jobs. For a conceptual overview, see [Background Jobs](../../guides/background-jobs.md). + +## `jobs` table + +The `jobs` table is available in **all** script contexts (procedures, queries, record scripts, label scripts, and job scripts). It provides functions for queuing new jobs. + +### `jobs.create(job_type, input[, opts])` + +Enqueue a new background job. + +**Parameters:** + +| Parameter | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| `job_type` | string | The job type name. Must match a `job.run:` script trigger. | +| `input` | table | Input data passed to the job script via `job.input`. | +| `opts` | table? | Optional settings (see below). | + +**Options:** + +| Key | Type | Default | Description | +| ------ | ------- | ------- | -------------------------------------------------------- | +| `auth` | boolean | `false` | Inherit the caller's PDS auth. When `true`, the job script can use `r:save()`, `r:delete()`, and blob uploads as the creating user. When `false`, only local operations (`r:save_local()`, `r:delete_local()`) are available. | + +**Returns:** `string` — the new job's UUID. + +**Requires:** An authenticated caller (`caller_did` must be set). Raises an error in unauthenticated contexts. + +```lua +-- Enqueue a job without PDS auth (default) +function handle() + local job_id = jobs.create("stats.rebuild", { + collection = collection, + }) + return { job_id = job_id } +end +``` + +```lua +-- Enqueue a job that needs to write records on behalf of the caller +function handle() + local job_id = jobs.create("export", { + collection = collection, + format = input.format, + }, { auth = true }) + return { job_id = job_id } +end +``` + +Jobs can enqueue other jobs — a job script can call `jobs.create()` to spawn follow-up work: + +```lua +-- Inside a job script: fan out to per-collection jobs +function handle() + local collections = job.input.collections + local child_ids = {} + for _, col in ipairs(collections) do + table.insert(child_ids, jobs.create("export.collection", { + collection = col, + parent_job = job.id, + })) + end + return { children = child_ids } +end +``` + +## `job` table + +The `job` table is available **only inside job scripts** (trigger `job.run:`). It is `nil` in all other script contexts. + +### `job.id` + +**Type:** `string` + +The job's UUID. + +### `job.input` + +**Type:** `table` + +The input table that was passed to `jobs.create()` when the job was queued. + +### `job.progress(data)` + +Persist a progress snapshot to the database. + +**Parameters:** + +| Parameter | Type | Description | +| --------- | ----- | -------------------------------- | +| `data` | table | Any Lua table — stored as JSONB. | + +Each call overwrites the previous progress value. The snapshot is visible in the job detail panel in the dashboard and via `GET /admin/jobs/:id`. + +```lua +job.progress({ phase = "fetching", fetched = 250, total = 1000 }) +``` + +### `job.should_stop()` + +Check whether the job has been paused or cancelled. + +**Returns:** `boolean` — `true` if the operator requested a pause or cancel. + +Pause and cancel are cooperative. The worker sets a flag when the operator requests it, but the script must call `job.should_stop()` and exit gracefully. If your script never checks, pause and cancel requests wait until the script finishes on its own. + +```lua +for i, item in ipairs(items) do + if job.should_stop() then + return { partial = true, last = i } + end + process(item) +end +``` + +### `job.wait(seconds)` + +Sleep for the specified duration. + +**Parameters:** + +| Parameter | Type | Description | +| --------- | ------ | ---------------------------------------- | +| `seconds` | number | Duration in seconds (clamped to 0–3600). | + +Values below 0 are clamped to 0. Values above 3600 are clamped to 3600. + +```lua +-- Poll an external API with a delay between requests +for _, batch in ipairs(batches) do + local resp = http.post("https://api.example.com/import", { + body = json.encode(batch), + }) + job.wait(1) -- rate limit +end +``` + +## Available APIs + +Job scripts have access to all standard Lua APIs: + +- [`db.*`](database-api.md) — database queries +- [`http.*`](http-api.md) — HTTP client +- [`xrpc.*`](xrpc-lua-api.md) — XRPC calls +- [`atproto.*`](atproto-api.md) — DID resolution, labels, signing +- [`Record.*`](record-api.md) — record operations +- [`json.*`](json-api.md) — JSON encode/decode +- [`jobs.*`](#jobs-table) — queue follow-up jobs +- [`log()`](utility-globals.md), [`now()`](utility-globals.md), [`TID()`](utility-globals.md#tid), [`toarray()`](utility-globals.md) — utility globals +- `env.` — [script variables](../admin/script-variables.md) + +Unlike XRPC and record scripts, job scripts have **no instruction count limit** — they can run arbitrarily long computations. diff --git a/packages/docs/content/docs/api-reference/lua/meta.json b/packages/docs/content/docs/api-reference/lua/meta.json index d65b99a..09bfb19 100644 --- a/packages/docs/content/docs/api-reference/lua/meta.json +++ b/packages/docs/content/docs/api-reference/lua/meta.json @@ -7,6 +7,7 @@ "xrpc-lua-api", "atproto-api", "json-api", + "jobs-api", "utility-globals", "standard-libraries" ] diff --git a/packages/docs/content/docs/guides/background-jobs.md b/packages/docs/content/docs/guides/background-jobs.md new file mode 100644 index 0000000..3b532f0 --- /dev/null +++ b/packages/docs/content/docs/guides/background-jobs.md @@ -0,0 +1,232 @@ +--- +title: "Background Jobs" +--- + +Background jobs let you run long-running Lua scripts outside the request cycle. A script running in any context can queue a job with `jobs.create()`, and a dedicated worker picks it up and executes the matching job script. Jobs are useful for data migrations, batch exports, external API syncs, or any work that's too slow for a synchronous request. + +## How it works + +1. **Queue** - any Lua script calls `jobs.create("my-type", { ... })`, passing a free-form type name and an input table. This inserts a row into the `happyview_jobs` table with status `pending` and returns the job's UUID. +2. **Match** - the worker resolves the job's type to a script by looking up the trigger `job.run:my-type`. If no script exists for that type, the job fails immediately. +3. **Execute** - the worker calls the script's `handle()` function with the `job` global set. The script can report progress, check for cancellation, sleep, and return a result. + +## Creating a job script + +Job scripts are created from the [dashboard](../getting-started/dashboard.md) (Settings > Scripts > New) or via the [admin API](../api-reference/admin/scripts.md). + +In the dashboard, select **Job** as the trigger source, then type a job type name. The type name is a free-form string that must match `/^[a-z0-9][a-z0-9._-]*$/` (max 128 characters). The resulting trigger id is `job.run:`; for example, `job.run:export` or `job.run:data.migrate`. + +### Trigger grammar + +| Trigger | Fires when | +| ---------------- | ------------------------------------------------------- | +| `job.run:` | A job with the matching type is picked up by the worker | + +There is no cascade for job triggers; the type must match exactly. + +### Script structure + +Job scripts follow the same `handle()` convention as all other scripts. The return value becomes the job's `result` field. + +```lua +function handle() + local data = job.input + + for i, item in ipairs(data.items) do + -- process each item + job.progress({ processed = i, total = #data.items }) + + if job.should_stop() then + return { partial = true, processed = i } + end + end + + return { processed = #data.items } +end +``` + +## The `job` global + +Inside a job script, the `job` global provides access to the job's metadata and control functions. This global is only available in job scripts. It's `nil` in all other script contexts. + +| Field / Function | Type | Description | +| -------------------- | -------- | ----------------------------------------------------------- | +| `job.id` | string | The job's UUID | +| `job.input` | table | The input table passed to `jobs.create()` | +| `job.progress(data)` | function | Persist progress to the database (visible in the dashboard) | +| `job.should_stop()` | function | Returns `true` if the job has been paused or cancelled | +| `job.wait(seconds)` | function | Sleep for 0–3600 seconds | + +### `job.progress(data)` + +Call `job.progress()` to persist a progress snapshot. The `data` argument can be any Lua table: it's stored as JSONB and displayed in the job detail panel in the dashboard. Call it as often as you like; each call overwrites the previous progress value. + +```lua +job.progress({ status = "indexing", page = 5, total_pages = 20 }) +``` + +### `job.should_stop()` + +Check `job.should_stop()` at natural checkpoints in your script. It returns `true` when an operator has paused or cancelled the job from the dashboard. Cancellation and pausing are **cooperative**: the worker sets a flag, but it's up to your script to check it and exit gracefully. If your script never checks, pause and cancel requests will wait until the script finishes on its own. + +```lua +for i, repo in ipairs(repos) do + if job.should_stop() then + return { partial = true, last_processed = i } + end + process(repo) +end +``` + +### `job.wait(seconds)` + +Pause execution for up to 3600 seconds (1 hour). Useful for rate-limited external API calls or scheduled delays. Values below 0 are clamped to 0; values above 3600 are clamped to 3600. + +```lua +for _, batch in ipairs(batches) do + push_to_api(batch) + job.wait(2) -- respect rate limits +end +``` + +## Enqueuing jobs + +Any Lua script can enqueue a job using the `jobs` global. This is available in **all** script contexts: procedures, queries, record scripts, label scripts, and even other job scripts (a job can enqueue follow-up jobs). + +```lua +-- In a procedure script +function handle() + local job_id = jobs.create("export", { + collection = collection, + format = input.format, + }) + return { job_id = job_id, status = "queued" } +end +``` + +`jobs.create(type, input[, opts])` returns the new job's UUID as a string. The `type` argument must match a `job.run:` script trigger. If no matching script exists, the job will fail when the worker picks it up. + +See the [Jobs API reference](../api-reference/lua/jobs-api.md#jobscreatejob_type-input-opts) for the full parameter list. + +## Authentication + +By default, jobs run **without PDS auth**. This is intentional. Most jobs don't need to write records on behalf of a user, and granting auth by default would give long-running background scripts access to a user's PDS session unnecessarily. + +### What's available without auth + +Every job script — regardless of auth setting — has access to: + +- `caller_did` - the DID of the user who enqueued the job (always set) +- `db.*` - full database access (queries, raw SQL, search, backlinks) +- `http.*` - outbound HTTP requests +- `xrpc.*` - XRPC calls (local and proxied) +- `atproto.*` - DID resolution, label queries, signature verification +- `json.*` - JSON encode/decode +- `jobs.*` - enqueue follow-up jobs +- `Record.load()` - load records from the local database +- `r:save_local()` / `r:delete_local()` - write or delete records in HappyView's local database only +- `Record.delete_local()` - delete by URI from the local database +- Utility globals: `log()`, `now()`, `TID()`, `toarray()` +- `env.` - script variables + +### What requires auth + +PDS-touching operations need the creator's OAuth session. Without auth, these raise an error: + +- `r:save()` - writes a record to the user's PDS and indexes it locally +- `r:delete()` - deletes a record from the user's PDS and removes it locally +- `Record.save_all()` - batch save to PDS +- `atproto.upload_blob()` - upload a blob to the user's PDS + +### Opting into auth + +To give a job access to the creator's PDS session, pass `{ auth = true }` as the third argument to `jobs.create()`: + +```lua +-- Without auth (default) - local-only operations +jobs.create("stats.rebuild", { collection = collection }) + +-- With auth - can write to the creator's PDS +jobs.create("sync-records", { collection = collection }, { auth = true }) +``` + +When `auth = true`, the worker loads the creator's OAuth session at execution time. If the session is no longer valid (expired, revoked, or the user has no session), the job fails immediately with an error. The creating user must have a valid OAuth session when the job runs, not just when it was enqueued. + +### When to use auth + +Use `{ auth = true }` when the job needs to create, update, or delete records on the AT Protocol network on behalf of the user. For example, batch record creation, cross-collection syncs, or migrations that write back to the user's PDS. + +Leave auth off (the default) for jobs that only read data, compute aggregates, sync to external services, clean up local records, or perform any work that doesn't touch a user's PDS. + +## Job lifecycle + +Jobs move through these statuses: + +| Status | Description | +| ------------ | ------------------------------------------------- | +| `pending` | Queued, waiting for the worker to pick it up | +| `running` | Currently executing | +| `completed` | Script returned successfully | +| `failed` | Script raised an error | +| `pausing` | Pause requested, waiting for the script to check | +| `paused` | Script exited after detecting the pause flag | +| `cancelling` | Cancel requested, waiting for the script to check | +| `cancelled` | Script exited after detecting the cancel flag | + +### Pausing and cancelling + +Pause and cancel are requested via the dashboard or the [admin API](../api-reference/admin/jobs.md). Both are cooperative: + +1. The endpoint sets the job's status to `pausing` or `cancelling`. +2. The worker continues running the script. At its next `job.should_stop()` check, it returns `true`. +3. The script should exit gracefully. Whatever it returns becomes the job's result. +4. The worker sets the final status to `paused` or `cancelled`. + +If the script never calls `job.should_stop()`, the pause or cancel request waits until the script finishes naturally. + +A paused job can be resumed via `POST /admin/jobs/:id/resume` or the Resume button in the dashboard. Resuming sets the status back to `pending`, and the worker picks it up again, but the script runs from the beginning. Use `job.input` or progress data to implement resumable logic. + +### Recovery after restart + +Jobs survive server restarts. On startup, the worker checks for orphaned jobs: + +- **Running** jobs are reset to `pending` and re-queued. +- **Cancelling** jobs are finalised as `cancelled`. +- **Pausing** jobs are finalised as `paused`. + +## Worker + +The job worker runs as a background task inside the HappyView server process. It polls for pending jobs every 5 seconds and executes one job at a time. Job scripts have **no instruction count limit** (unlike XRPC and record scripts, which are capped at 1,000,000 instructions), so they can run arbitrarily long computations. + +Job scripts have access to all standard Lua APIs: `db.*`, `http.*`, `xrpc.*`, `atproto.*`, `Record.*`, `json.*`, `env.`, `log()`, `now()`, `TID()`, `toarray()`, and `jobs.*` (including `jobs.create()` to queue follow-up jobs). + +## Dashboard + +The **Jobs** page in the dashboard (`/dashboard/jobs`) shows all background jobs in a filterable table. You can filter by status using the dropdown at the top. + +Clicking a job row opens a detail sheet showing: + +- Job ID, type, and status +- Input data (the table passed to `jobs.create()`) +- Progress (the last value passed to `job.progress()`) +- Result or error +- Timestamps (created, started, completed) +- Action buttons: **Cancel**, **Pause**, or **Resume** depending on the current status + +## Permissions + +Job management requires specific permissions: + +| Permission | Grants | +| ------------- | ---------------------------------- | +| `jobs:read` | View jobs in the dashboard and API | +| `jobs:manage` | Cancel, pause, and resume jobs | + +Queuing jobs via `jobs.create()` in a script requires an authenticated caller (`caller_did` must be set). + +## Next steps + +- [Admin API - Jobs](../api-reference/admin/jobs.md): Full reference for job endpoints +- [Lua API - Jobs](../api-reference/lua/jobs-api.md): Full reference for the `jobs` and `job` Lua APIs +- [Lua Scripting](lua-scripting.md): General Lua scripting reference +- [Record & Label Scripts](record-scripts.md): Trigger grammar for all script types diff --git a/packages/docs/content/docs/guides/lua-scripting.md b/packages/docs/content/docs/guides/lua-scripting.md index 4887f97..58cb68c 100644 --- a/packages/docs/content/docs/guides/lua-scripting.md +++ b/packages/docs/content/docs/guides/lua-scripting.md @@ -69,14 +69,14 @@ These globals are set automatically before `handle()` is called. When a script handles a space-scoped request, the `space` global is set to a table with the space's metadata. For non-space requests, `space` is `nil`. -| Field | Type | Description | -| ----------- | ------ | -------------------------------------------------------- | -| `space` | string | The full `ats://` space URI | -| `space_id` | string | Internal space identifier | -| `did` | string | The space's DID | -| `authority_did` | string | The space authority's DID | -| `type_nsid` | string | Space type NSID | -| `skey` | string | Space key | +| Field | Type | Description | +| --------------- | ------ | --------------------------- | +| `space` | string | The full `ats://` space URI | +| `space_id` | string | Internal space identifier | +| `did` | string | The space's DID | +| `authority_did` | string | The space authority's DID | +| `type_nsid` | string | Space type NSID | +| `skey` | string | Space key | ```lua function handle() @@ -179,6 +179,21 @@ The `json` global provides JSON serialization and deserialization. See the full [JSON API reference](../api-reference/lua/json-api.md) for `json.encode` and `json.decode`. +## Jobs API + +The `jobs` table lets any script queue background jobs for long-running work. Available in all script contexts. + +See the full [Jobs API reference](../api-reference/lua/jobs-api.md) for `jobs.create()` and the `job` global available inside job scripts. + +Quick example: + +```lua +local job_id = jobs.create("export", { collection = collection }) +return { job_id = job_id } +``` + +For the full guide on background jobs, see [Background Jobs](background-jobs.md). + ## Debugging ### Logging diff --git a/packages/docs/content/docs/guides/meta.json b/packages/docs/content/docs/guides/meta.json index 7bc4499..c06e2b0 100644 --- a/packages/docs/content/docs/guides/meta.json +++ b/packages/docs/content/docs/guides/meta.json @@ -4,6 +4,7 @@ "upgrading-to-v2", "lexicons", "backfill", + "background-jobs", "label-scripts", "lua-scripting", "api-clients", diff --git a/packages/docs/content/docs/guides/record-scripts.md b/packages/docs/content/docs/guides/record-scripts.md index f1ac958..ea96e58 100644 --- a/packages/docs/content/docs/guides/record-scripts.md +++ b/packages/docs/content/docs/guides/record-scripts.md @@ -41,6 +41,14 @@ XRPC scripts handle the request and return the response. Without a script, Happy There is no cascade for label or XRPC triggers -- each trigger string must match exactly. +### Job triggers + +| Trigger | Fires when | +| -------------------------- | --------------------------------------------- | +| `job.run:` | A background job with the matching type is picked up by the worker | + +There is no cascade for job triggers -- the type must match exactly. See [Background Jobs](./background-jobs.md) for the full job scripting guide. + ## Creating scripts You can create scripts through the [dashboard](../getting-started/dashboard.md) (Settings > Scripts > New) or via the [admin API](../api-reference/admin/scripts.md) (`POST /admin/scripts`). diff --git a/packages/docs/package.json b/packages/docs/package.json index e035f81..3f36614 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -17,7 +17,7 @@ "fumadocs-mdx": "^15.0.4", "fumadocs-ui": "^16.8.10", "lucide-react": "^1.14.0", - "mermaid": "^11.6.0", + "mermaid": "^11.16.0", "next": "^16.1.6", "next-themes": "^0.4.6", "react": "^19.2.0", -- 2.51.2 From c9cbd88377fdae30d69ea895df50d2b3f04507e1 Mon Sep 17 00:00:00 2001 From: Trezy Date: Sun, 5 Jul 2026 10:03:07 -0500 Subject: [PATCH 3/3] fix: fix Playwright tests Signed-off-by: Trezy --- src/admin/jobs.rs | 8 +-- src/jobs/db.rs | 16 ++++-- src/lua/jobs_api.rs | 35 ++++++++++++- src/lua/scripts.rs | 51 ++++++++++++++++--- tests/e2e_jobs.rs | 12 ++--- web/src/app/dashboard/jobs/page.tsx | 15 ++++-- .../settings/scripts/[id]/script-detail.tsx | 1 + .../dashboard/settings/scripts/new/page.tsx | 1 + web/src/types/jobs.ts | 27 +++++----- web/tests/e2e/script-job.spec.ts | 48 ++++++++++++----- 10 files changed, 160 insertions(+), 54 deletions(-) diff --git a/src/admin/jobs.rs b/src/admin/jobs.rs index 27bca7c..46d3b1c 100644 --- a/src/admin/jobs.rs +++ b/src/admin/jobs.rs @@ -23,7 +23,7 @@ pub async fn list_jobs( ) -> Result, AppError> { auth.require(Permission::JobsRead).await?; - let limit = query.limit.unwrap_or(50).min(100); + let limit = query.limit.unwrap_or(50).clamp(1, 100); let (jobs_list, cursor) = jobs::db::list_jobs( &state, query.status.as_deref(), @@ -72,7 +72,7 @@ pub async fn cancel_job( jobs::db::set_status(&state, &id, "cancelled").await?; Ok(Json(serde_json::json!({ "status": "cancelled" }))) } - _ => Err(AppError::BadRequest(format!( + _ => Err(AppError::Conflict(format!( "cannot cancel job with status: {}", job.status ))), @@ -91,7 +91,7 @@ pub async fn pause_job( .ok_or_else(|| AppError::NotFound("job not found".into()))?; if job.status != "running" { - return Err(AppError::BadRequest(format!( + return Err(AppError::Conflict(format!( "cannot pause job with status: {}", job.status ))); @@ -113,7 +113,7 @@ pub async fn resume_job( .ok_or_else(|| AppError::NotFound("job not found".into()))?; if job.status != "paused" { - return Err(AppError::BadRequest(format!( + return Err(AppError::Conflict(format!( "cannot resume job with status: {}", job.status ))); diff --git a/src/jobs/db.rs b/src/jobs/db.rs index 10120c8..6d00d6a 100644 --- a/src/jobs/db.rs +++ b/src/jobs/db.rs @@ -2,7 +2,7 @@ use serde_json::Value; use uuid::Uuid; use crate::AppState; -use crate::db::{adapt_sql, now_rfc3339}; +use crate::db::{DatabaseBackend, adapt_sql, now_rfc3339}; use crate::error::AppError; use super::Job; @@ -284,10 +284,16 @@ pub async fn find_interrupted_jobs(state: &AppState) -> Vec { pub async fn claim_next_job(state: &AppState) -> Result, AppError> { let now = now_rfc3339(); - let sql = adapt_sql( - "UPDATE happyview_jobs SET status = 'running', started_at = ? WHERE id = (SELECT id FROM happyview_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1) RETURNING *", - state.db_backend, - ); + let sql = match state.db_backend { + DatabaseBackend::Postgres => adapt_sql( + "UPDATE happyview_jobs SET status = 'running', started_at = ? WHERE id = (SELECT id FROM happyview_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING *", + state.db_backend, + ), + DatabaseBackend::Sqlite => adapt_sql( + "UPDATE happyview_jobs SET status = 'running', started_at = ? WHERE id = (SELECT id FROM happyview_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT 1) AND status = 'pending' RETURNING *", + state.db_backend, + ), + }; let row: Option = sqlx::query_as(&sql) .bind(&now) diff --git a/src/lua/jobs_api.rs b/src/lua/jobs_api.rs index 959628d..f50b397 100644 --- a/src/lua/jobs_api.rs +++ b/src/lua/jobs_api.rs @@ -1,9 +1,13 @@ use mlua::{Lua, LuaSerdeExt, Result as LuaResult}; -use std::sync::Arc; +use regex::Regex; +use std::sync::{Arc, LazyLock}; use crate::AppState; use crate::jobs; +static JOB_TYPE_PATTERN: LazyLock = + LazyLock::new(|| Regex::new(r"^[a-z0-9][a-z0-9._-]*$").unwrap()); + /// Register the `jobs` table for queuing jobs from scripts. /// Available in all script contexts (procedure, query, record-event). pub fn register_jobs_api( @@ -31,6 +35,15 @@ pub fn register_jobs_api( .unwrap_or(false); async move { + if job_type.is_empty() + || job_type.len() > 128 + || !JOB_TYPE_PATTERN.is_match(&job_type) + { + return Err(mlua::Error::runtime( + "job_type must be 1-128 characters matching /^[a-z0-9][a-z0-9._-]*$/", + )); + } + let caller = caller_did.as_deref().ok_or_else(|| { mlua::Error::runtime("jobs.create requires an authenticated caller") })?; @@ -294,4 +307,24 @@ mod tests { .unwrap(); assert!(result); } + + #[tokio::test] + async fn jobs_create_rejects_invalid_job_type() { + let lua = crate::lua::sandbox::create_sandbox().unwrap(); + let state = test_state(); + register_jobs_api(&lua, Arc::new(state), Some("did:plc:test".into())).unwrap(); + + for bad in [ + "", + "UPPER", + "has space", + "has:colon", + "-leading-dash", + ".leading-dot", + ] { + let script = format!(r#"return jobs.create("{bad}", {{}})"#); + let result: mlua::Result = lua.load(&script).eval_async().await; + assert!(result.is_err(), "expected error for job_type={bad:?}"); + } + } } diff --git a/src/lua/scripts.rs b/src/lua/scripts.rs index df4ae59..439c7c0 100644 --- a/src/lua/scripts.rs +++ b/src/lua/scripts.rs @@ -32,9 +32,13 @@ //! [`super::execute::execute_procedure_script`] / //! [`super::execute::execute_query_script`] directly. +use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; + +static JOB_TYPE_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^[a-z0-9][a-z0-9._-]*$").unwrap()); use crate::AppState; use crate::db::{DatabaseBackend, adapt_sql, now_rfc3339}; @@ -60,6 +64,7 @@ pub enum TriggerKind { XrpcQuery, XrpcProcedure, LabelerApply, + JobRun, } /// A trigger id parsed into `(kind, suffix)`. The suffix is either an NSID @@ -82,6 +87,7 @@ impl ParsedTrigger { TriggerKind::XrpcQuery => format!("xrpc.query:{}", self.suffix), TriggerKind::XrpcProcedure => format!("xrpc.procedure:{}", self.suffix), TriggerKind::LabelerApply => format!("labeler.apply:{}", self.suffix), + TriggerKind::JobRun => format!("job.run:{}", self.suffix), } } @@ -92,7 +98,8 @@ impl ParsedTrigger { format!( "trigger id '{id}' must contain a ':' separator; \ valid prefixes: record.{{index,create,update,delete}}:, \ - xrpc.{{query,procedure}}:, labeler.apply:" + xrpc.{{query,procedure}}:, labeler.apply:, \ + job.run:" ) })?; @@ -108,18 +115,21 @@ impl ParsedTrigger { "xrpc.query" => TriggerKind::XrpcQuery, "xrpc.procedure" => TriggerKind::XrpcProcedure, "labeler.apply" => TriggerKind::LabelerApply, + "job.run" => TriggerKind::JobRun, other => { return Err(format!( "unknown trigger prefix '{other}'; valid prefixes: \ record.{{index,create,update,delete}}, xrpc.{{query,procedure}}, \ - labeler.apply" + labeler.apply, job.run" )); } }; - // Suffix validation: NSID for everything except `labeler.apply:_actor`. - match (kind, suffix) { - (TriggerKind::LabelerApply, "_actor") => {} + // Suffix validation: NSID for most triggers, but `labeler.apply:_actor` + // and `job.run:` have their own formats. + match kind { + TriggerKind::JobRun => validate_job_type(suffix)?, + TriggerKind::LabelerApply if suffix == "_actor" => {} _ => validate_nsid(suffix)?, } @@ -130,6 +140,20 @@ impl ParsedTrigger { } } +fn validate_job_type(job_type: &str) -> Result<(), String> { + if job_type.is_empty() || job_type.len() > 128 { + return Err(format!( + "invalid job type '{job_type}': must be 1–128 characters" + )); + } + if !JOB_TYPE_RE.is_match(job_type) { + return Err(format!( + "invalid job type '{job_type}': must match /^[a-z0-9][a-z0-9._-]*$/" + )); + } + Ok(()) +} + /// Minimal NSID validation: at least two dot-separated segments, each /// non-empty and matching `[a-zA-Z][a-zA-Z0-9-]*`. Mirrors the AT Protocol /// spec's character class for everyday use; full Unicode strictness lives @@ -900,6 +924,21 @@ mod tests { assert!(err.contains("valid prefixes")); } + #[test] + fn parse_job_run_trigger() { + let t = ParsedTrigger::parse("job.run:test.export").unwrap(); + assert_eq!(t.kind, TriggerKind::JobRun); + assert_eq!(t.suffix, "test.export"); + assert_eq!(t.id(), "job.run:test.export"); + } + + #[test] + fn rejects_bad_job_type() { + assert!(ParsedTrigger::parse("job.run:UPPER").is_err()); + assert!(ParsedTrigger::parse("job.run:has space").is_err()); + assert!(ParsedTrigger::parse("job.run:").is_err()); + } + #[test] fn rejects_unknown_prefix() { let err = ParsedTrigger::parse("garbage:com.example.thing").unwrap_err(); diff --git a/tests/e2e_jobs.rs b/tests/e2e_jobs.rs index 1ab363c..eead318 100644 --- a/tests/e2e_jobs.rs +++ b/tests/e2e_jobs.rs @@ -272,7 +272,7 @@ async fn cancel_paused_job_sets_cancelled() { #[tokio::test] #[serial] -async fn cancel_completed_job_returns_400() { +async fn cancel_completed_job_returns_409() { common::require_db!(); let app = TestApp::new().await; @@ -289,7 +289,7 @@ async fn cancel_completed_job_returns_400() { .await .unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), StatusCode::CONFLICT); } // --------------------------------------------------------------------------- @@ -322,7 +322,7 @@ async fn pause_running_job_sets_pausing() { #[tokio::test] #[serial] -async fn pause_pending_job_returns_400() { +async fn pause_pending_job_returns_409() { common::require_db!(); let app = TestApp::new().await; @@ -339,7 +339,7 @@ async fn pause_pending_job_returns_400() { .await .unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), StatusCode::CONFLICT); } // --------------------------------------------------------------------------- @@ -372,7 +372,7 @@ async fn resume_paused_job_sets_pending() { #[tokio::test] #[serial] -async fn resume_running_job_returns_400() { +async fn resume_running_job_returns_409() { common::require_db!(); let app = TestApp::new().await; @@ -389,7 +389,7 @@ async fn resume_running_job_returns_400() { .await .unwrap(); - assert_eq!(resp.status(), StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), StatusCode::CONFLICT); } // --------------------------------------------------------------------------- diff --git a/web/src/app/dashboard/jobs/page.tsx b/web/src/app/dashboard/jobs/page.tsx index 72a09f6..43a0026 100644 --- a/web/src/app/dashboard/jobs/page.tsx +++ b/web/src/app/dashboard/jobs/page.tsx @@ -129,9 +129,14 @@ function statusIcon(status: string) { } } -function hasContent(obj: Record | null | undefined): boolean { - if (!obj) return false; - return Object.keys(obj).length > 0; +function hasContent(value: unknown): boolean { + if (value == null) return false; + if (typeof value === "object") { + return Array.isArray(value) + ? value.length > 0 + : Object.keys(value as Record).length > 0; + } + return true; } function relativeTime(dateStr: string): string { @@ -421,7 +426,7 @@ function JobDetail({ - {job.result && } + {hasContent(job.result) && }
{canManage && ( @@ -484,7 +489,7 @@ function JsonSection({ defaultOpen = false, }: { title: string; - data: Record | null; + data: unknown; defaultOpen?: boolean; }) { const [open, setOpen] = useState(defaultOpen); diff --git a/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx index 1624515..b2cac0d 100644 --- a/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx +++ b/web/src/app/dashboard/settings/scripts/[id]/script-detail.tsx @@ -70,6 +70,7 @@ export default function ScriptDetail() { if (!isDirty) return; function onBeforeUnload(e: BeforeUnloadEvent) { e.preventDefault(); + e.returnValue = ""; } window.addEventListener("beforeunload", onBeforeUnload); return () => window.removeEventListener("beforeunload", onBeforeUnload); diff --git a/web/src/app/dashboard/settings/scripts/new/page.tsx b/web/src/app/dashboard/settings/scripts/new/page.tsx index dda5e0f..8ed4071 100644 --- a/web/src/app/dashboard/settings/scripts/new/page.tsx +++ b/web/src/app/dashboard/settings/scripts/new/page.tsx @@ -75,6 +75,7 @@ function NewScriptInner() { if (!isDirty) return; function onBeforeUnload(e: BeforeUnloadEvent) { e.preventDefault(); + e.returnValue = ""; } window.addEventListener("beforeunload", onBeforeUnload); return () => window.removeEventListener("beforeunload", onBeforeUnload); diff --git a/web/src/types/jobs.ts b/web/src/types/jobs.ts index c457980..74dcec1 100644 --- a/web/src/types/jobs.ts +++ b/web/src/types/jobs.ts @@ -1,18 +1,19 @@ export interface Job { - id: string; - job_type: string; - status: string; - input: Record; - progress: Record; - result: Record | null; - error: string | null; - created_by: string; - started_at: string | null; - completed_at: string | null; - created_at: string; + id: string + job_type: string + status: string + input: unknown + progress: unknown + result: unknown | null + error: string | null + created_by: string + inherit_auth: boolean + started_at: string | null + completed_at: string | null + created_at: string } export interface JobsListResponse { - jobs: Job[]; - cursor: string | null; + jobs: Job[] + cursor: string | null } diff --git a/web/tests/e2e/script-job.spec.ts b/web/tests/e2e/script-job.spec.ts index adc1448..d10629c 100644 --- a/web/tests/e2e/script-job.spec.ts +++ b/web/tests/e2e/script-job.spec.ts @@ -4,6 +4,23 @@ import { loginAsTestAdmin } from "./auth-helper" const JOB_TYPE = "test.e2e.myjob" const TRIGGER_ID = `job.run:${JOB_TYPE}` +async function seedScript( + request: import("@playwright/test").APIRequestContext, +) { + const resp = await request.post("/admin/scripts", { + data: { + id: TRIGGER_ID, + body: "function handle()\n return { ok = true }\nend", + }, + }) + if (!resp.ok()) { + const text = await resp.text() + if (!text.includes("already exists")) { + throw new Error(`Failed to seed script: ${resp.status()} ${text}`) + } + } +} + async function cleanupScript( request: import("@playwright/test").APIRequestContext, ) { @@ -48,15 +65,21 @@ test.describe("Job Script Creation", () => { await page.locator("#job-type-input").fill(JOB_TYPE) - await page.getByRole("button", { name: "Create script" }).click() + const createButton = page.getByRole("button", { name: "Create script" }) + await expect(createButton).toBeEnabled({ timeout: 3000 }) + await createButton.click() await page.waitForURL( `**/dashboard/settings/scripts/${encodeURIComponent(TRIGGER_ID)}`, - { timeout: 5000 }, + { timeout: 10000 }, ) - await expect(page.getByText("Job runner")).toBeVisible() - await expect(page.getByText(TRIGGER_ID)).toBeVisible() + await expect( + page.getByText("Job runner", { exact: true }), + ).toBeVisible() + await expect( + page.getByText(TRIGGER_ID, { exact: true }), + ).toBeVisible() }) test("job script has job-specific template body", async ({ page }) => { @@ -65,26 +88,23 @@ test.describe("Job Script Creation", () => { await page.locator("#source-pick").click() await page.getByRole("option", { name: /Job/ }).click() - await expect(page.getByText("job.input")).toBeVisible({ timeout: 3000 }) - await expect(page.getByText("job.should_stop")).toBeVisible() + await expect(page.getByText("job.input").first()).toBeVisible({ + timeout: 3000, + }) + await expect(page.getByText("job.should_stop").first()).toBeVisible() }) test("job script appears in scripts list with Job runners family", async ({ page, }) => { - await page.request.post("/admin/scripts", { - data: { - id: TRIGGER_ID, - body: "function handle()\n return { ok = true }\nend", - }, - }) + await seedScript(page.request) await page.goto("/dashboard/settings/scripts") const row = page.locator("table tbody tr", { hasText: JOB_TYPE }) await expect(row).toBeVisible({ timeout: 5000 }) - await expect(row.getByText("Job runner")).toBeVisible() - await expect(row.getByText("Job runners")).toBeVisible() + await expect(row.getByText("Job runner", { exact: true })).toBeVisible() + await expect(row.getByText("Job runners", { exact: true })).toBeVisible() }) }) -- 2.51.2