From 220ebbc1fe34ffdab7839be274818b478dcb9533 Mon Sep 17 00:00:00 2001 From: Trezy Date: Thu, 2 Jul 2026 11:05:36 -0500 Subject: [PATCH] docs: add jobs docs Signed-off-by: Trezy --- bun.lock | 38 +-- .../content/docs/api-reference/admin/jobs.md | 151 ++++++++++++ .../docs/api-reference/admin/meta.json | 1 + .../docs/api-reference/lua/jobs-api.md | 156 ++++++++++++ .../content/docs/api-reference/lua/meta.json | 1 + .../content/docs/guides/background-jobs.md | 232 ++++++++++++++++++ .../docs/content/docs/guides/lua-scripting.md | 31 ++- packages/docs/content/docs/guides/meta.json | 1 + .../content/docs/guides/record-scripts.md | 8 + packages/docs/package.json | 2 +- 10 files changed, 581 insertions(+), 40 deletions(-) create mode 100644 packages/docs/content/docs/api-reference/admin/jobs.md create mode 100644 packages/docs/content/docs/api-reference/lua/jobs-api.md create mode 100644 packages/docs/content/docs/guides/background-jobs.md diff --git a/bun.lock b/bun.lock index 78347db..b049d38 100644 --- a/bun.lock +++ b/bun.lock @@ -24,7 +24,7 @@ "fumadocs-mdx": "^15.0.4", "fumadocs-ui": "^16.8.10", "lucide-react": "^1.14.0", - "mermaid": "^11.6.0", + "mermaid": "^11.16.0", "next": "^16.1.6", "next-themes": "^0.4.6", "react": "^19.2.0", @@ -196,15 +196,7 @@ "@braintree/sanitize-url": ["@braintree/sanitize-url@7.1.2", "", {}, "sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA=="], - "@chevrotain/cst-dts-gen": ["@chevrotain/cst-dts-gen@12.0.0", "", { "dependencies": { "@chevrotain/gast": "12.0.0", "@chevrotain/types": "12.0.0" } }, "sha512-fSL4KXjTl7cDgf0B5Rip9Q05BOrYvkJV/RrBTE/bKDN096E4hN/ySpcBK5B24T76dlQ2i32Zc3PAE27jFnFrKg=="], - - "@chevrotain/gast": ["@chevrotain/gast@12.0.0", "", { "dependencies": { "@chevrotain/types": "12.0.0" } }, "sha512-1ne/m3XsIT8aEdrvT33so0GUC+wkctpUPK6zU9IlOyJLUbR0rg4G7ZiApiJbggpgPir9ERy3FRjT6T7lpgetnQ=="], - - "@chevrotain/regexp-to-ast": ["@chevrotain/regexp-to-ast@12.0.0", "", {}, "sha512-p+EW9MaJwgaHguhoqwOtx/FwuGr+DnNn857sXWOi/mClXIkPGl3rn7hGNWvo31HA3vyeQxjqe+H36yZJwYU8cA=="], - - "@chevrotain/types": ["@chevrotain/types@12.0.0", "", {}, "sha512-S+04vjFQKeuYw0/eW3U52LkAHQsB1ASxsPGsLPUyQgrZ2iNNibQrsidruDzjEX2JYfespXMG0eZmXlhA6z7nWA=="], - - "@chevrotain/utils": ["@chevrotain/utils@12.0.0", "", {}, "sha512-lB59uJoaGIfOOL9knQqQRfhl9g7x8/wqFkp13zTdkRu1huG9kg6IJs1O8hqj9rs6h7orGxHJUKb+mX3rPbWGhA=="], + "@chevrotain/types": ["@chevrotain/types@11.1.2", "", {}, "sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw=="], "@colors/colors": ["@colors/colors@1.5.0", "", {}, "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ=="], @@ -352,7 +344,7 @@ "@mdx-js/mdx": ["@mdx-js/mdx@3.1.1", "", { "dependencies": { "@types/estree": "^1.0.0", "@types/estree-jsx": "^1.0.0", "@types/hast": "^3.0.0", "@types/mdx": "^2.0.0", "acorn": "^8.0.0", "collapse-white-space": "^2.0.0", "devlop": "^1.0.0", "estree-util-is-identifier-name": "^3.0.0", "estree-util-scope": "^1.0.0", "estree-walker": "^3.0.0", "hast-util-to-jsx-runtime": "^2.0.0", "markdown-extensions": "^2.0.0", "recma-build-jsx": "^1.0.0", "recma-jsx": "^1.0.0", "recma-stringify": "^1.0.0", "rehype-recma": "^1.0.0", "remark-mdx": "^3.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "source-map": "^0.7.0", "unified": "^11.0.0", "unist-util-position-from-estree": "^2.0.0", "unist-util-stringify-position": "^4.0.0", "unist-util-visit": "^5.0.0", "vfile": "^6.0.0" } }, "sha512-f6ZO2ifpwAQIpzGWaBQT2TXxPv6z3RBzQKpVftEWN78Vl/YweF1uwussDx8ECAXVtr3Rs89fKyG9YlzUs9DyGQ=="], - "@mermaid-js/parser": ["@mermaid-js/parser@1.1.0", "", { "dependencies": { "langium": "^4.0.0" } }, "sha512-gxK9ZX2+Fex5zu8LhRQoMeMPEHbc73UKZ0FQ54YrQtUxE1VVhMwzeNtKRPAu5aXks4FasbMe4xB4bWrmq6Jlxw=="], + "@mermaid-js/parser": ["@mermaid-js/parser@1.2.0", "", { "dependencies": { "@chevrotain/types": "~11.1.2" } }, "sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA=="], "@next/env": ["@next/env@16.2.6", "", {}, "sha512-gd8HoHN4ufj73WmR3JmVolrpJR47ILK6LouP5xElPglaVxir6e1a7VzvTvDWkOoPXT9rkkTzyCxBu4yeZfZwcw=="], @@ -770,10 +762,6 @@ "character-reference-invalid": ["character-reference-invalid@2.0.1", "", {}, "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw=="], - "chevrotain": ["chevrotain@12.0.0", "", { "dependencies": { "@chevrotain/cst-dts-gen": "12.0.0", "@chevrotain/gast": "12.0.0", "@chevrotain/regexp-to-ast": "12.0.0", "@chevrotain/types": "12.0.0", "@chevrotain/utils": "12.0.0" } }, "sha512-csJvb+6kEiQaqo1woTdSAuOWdN0WTLIydkKrBnS+V5gZz0oqBrp4kQ35519QgK6TpBThiG3V1vNSHlIkv4AglQ=="], - - "chevrotain-allstar": ["chevrotain-allstar@0.4.1", "", { "dependencies": { "lodash-es": "^4.17.21" }, "peerDependencies": { "chevrotain": "^12.0.0" } }, "sha512-PvVJm3oGqrveUVW2Vt/eZGeiAIsJszYweUcYwcskg9e+IubNYKKD+rHHem7A6XVO22eDAL+inxNIGAzZ/VIWlA=="], - "chokidar": ["chokidar@5.0.0", "", { "dependencies": { "readdirp": "^5.0.0" } }, "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw=="], "class-variance-authority": ["class-variance-authority@0.7.1", "", { "dependencies": { "clsx": "^2.1.1" } }, "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg=="], @@ -838,7 +826,7 @@ "csstype": ["csstype@3.2.3", "", {}, "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ=="], - "cytoscape": ["cytoscape@3.33.2", "", {}, "sha512-sj4HXd3DokGhzZAdjDejGvTPLqlt84vNFN8m7bGsOzDY5DyVcxIb2ejIXat2Iy7HxWhdT/N1oKyheJ5YdpsGuw=="], + "cytoscape": ["cytoscape@3.34.0", "", {}, "sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg=="], "cytoscape-cose-bilkent": ["cytoscape-cose-bilkent@4.1.0", "", { "dependencies": { "cose-base": "^1.0.0" }, "peerDependencies": { "cytoscape": "^3.2.0" } }, "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ=="], @@ -956,6 +944,8 @@ "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + "es-toolkit": ["es-toolkit@1.49.0", "", {}, "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g=="], + "esast-util-from-estree": ["esast-util-from-estree@2.0.0", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "devlop": "^1.0.0", "estree-util-visit": "^2.0.0", "unist-util-position-from-estree": "^2.0.0" } }, "sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ=="], "esast-util-from-js": ["esast-util-from-js@2.0.1", "", { "dependencies": { "@types/estree-jsx": "^1.0.0", "acorn": "^8.0.0", "esast-util-from-estree": "^2.0.0", "vfile-message": "^4.0.0" } }, "sha512-8Ja+rNJ0Lt56Pcf3TAmpBZjmx8ZcK5Ts4cAzIOjsjevg9oSXJnl6SUQ2EevU8tv3h6ZLWmoKL5H4fgWvdvfETw=="], @@ -1188,8 +1178,6 @@ "khroma": ["khroma@2.1.0", "", {}, "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw=="], - "langium": ["langium@4.2.2", "", { "dependencies": { "@chevrotain/regexp-to-ast": "~12.0.0", "chevrotain": "~12.0.0", "chevrotain-allstar": "~0.4.1", "vscode-languageserver": "~9.0.1", "vscode-languageserver-textdocument": "~1.0.11", "vscode-uri": "~3.1.0" } }, "sha512-JUshTRAfHI4/MF9dH2WupvjSXyn8JBuUEWazB8ZVJUtXutT0doDlAv1XKbZ1Pb5sMexa8FF4CFBc0iiul7gbUQ=="], - "layout-base": ["layout-base@1.0.2", "", {}, "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg=="], "lightningcss": ["lightningcss@1.32.0", "", { "dependencies": { "detect-libc": "^2.0.3" }, "optionalDependencies": { "lightningcss-android-arm64": "1.32.0", "lightningcss-darwin-arm64": "1.32.0", "lightningcss-darwin-x64": "1.32.0", "lightningcss-freebsd-x64": "1.32.0", "lightningcss-linux-arm-gnueabihf": "1.32.0", "lightningcss-linux-arm64-gnu": "1.32.0", "lightningcss-linux-arm64-musl": "1.32.0", "lightningcss-linux-x64-gnu": "1.32.0", "lightningcss-linux-x64-musl": "1.32.0", "lightningcss-win32-arm64-msvc": "1.32.0", "lightningcss-win32-x64-msvc": "1.32.0" } }, "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ=="], @@ -1294,7 +1282,7 @@ "merge2": ["merge2@1.4.1", "", {}, "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg=="], - "mermaid": ["mermaid@11.14.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.1", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.1.0", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.1", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.19", "dompurify": "^3.3.1", "katex": "^0.16.25", "khroma": "^2.1.0", "lodash-es": "^4.17.23", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0" } }, "sha512-GSGloRsBs+JINmmhl0JDwjpuezCsHB4WGI4NASHxL3fHo3o/BRXTxhDLKnln8/Q0lRFRyDdEjmk1/d5Sn1Xz8g=="], + "mermaid": ["mermaid@11.16.0", "", { "dependencies": { "@braintree/sanitize-url": "^7.1.2", "@iconify/utils": "^3.0.2", "@mermaid-js/parser": "^1.2.0", "@types/d3": "^7.4.3", "@upsetjs/venn.js": "^2.0.0", "cytoscape": "^3.33.3", "cytoscape-cose-bilkent": "^4.1.0", "cytoscape-fcose": "^2.2.0", "d3": "^7.9.0", "d3-sankey": "^0.12.3", "dagre-d3-es": "7.0.14", "dayjs": "^1.11.20", "dompurify": "^3.3.3", "es-toolkit": "^1.45.1", "katex": "^0.16.45", "khroma": "^2.1.0", "marked": "^16.3.0", "roughjs": "^4.6.6", "stylis": "^4.3.6", "ts-dedent": "^2.2.0", "uuid": "^11.1.0 || ^12 || ^13 || ^14.0.0" } }, "sha512-Zvm3kbstgdpvIJPPItlL7fppIZ3kibvc1oZIGxdvk9t6UFz6flv+Jw7FtRGKwfcI8OckmH04LqG6LlS6X4B1pA=="], "micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="], @@ -1794,18 +1782,6 @@ "vfile-message": ["vfile-message@4.0.3", "", { "dependencies": { "@types/unist": "^3.0.0", "unist-util-stringify-position": "^4.0.0" } }, "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw=="], - "vscode-jsonrpc": ["vscode-jsonrpc@8.2.0", "", {}, "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA=="], - - "vscode-languageserver": ["vscode-languageserver@9.0.1", "", { "dependencies": { "vscode-languageserver-protocol": "3.17.5" }, "bin": { "installServerIntoExtension": "bin/installServerIntoExtension" } }, "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g=="], - - "vscode-languageserver-protocol": ["vscode-languageserver-protocol@3.17.5", "", { "dependencies": { "vscode-jsonrpc": "8.2.0", "vscode-languageserver-types": "3.17.5" } }, "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg=="], - - "vscode-languageserver-textdocument": ["vscode-languageserver-textdocument@1.0.12", "", {}, "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA=="], - - "vscode-languageserver-types": ["vscode-languageserver-types@3.17.5", "", {}, "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg=="], - - "vscode-uri": ["vscode-uri@3.1.0", "", {}, "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ=="], - "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], "web-worker": ["web-worker@1.5.0", "", {}, "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw=="], diff --git a/packages/docs/content/docs/api-reference/admin/jobs.md b/packages/docs/content/docs/api-reference/admin/jobs.md new file mode 100644 index 0000000..ce55a13 --- /dev/null +++ b/packages/docs/content/docs/api-reference/admin/jobs.md @@ -0,0 +1,151 @@ +--- +title: "Jobs" +--- + +Admin API endpoints for managing background jobs. For a conceptual overview, see [Background Jobs](../../guides/background-jobs.md). + +## List jobs + +```http +GET /admin/jobs +``` + +Returns a paginated list of jobs, newest first. + +**Query parameters:** + +| Parameter | Type | Description | +| --------- | ------ | ------------------------------------------------------------------------------------- | +| `status` | string | Filter by status (`pending`, `running`, `completed`, `failed`, `paused`, `cancelled`) | +| `limit` | number | Maximum number of results (default: 50) | +| `cursor` | string | Pagination cursor from a previous response | + +**Permission:** `jobs:read` + +**Response:** + +```json +{ + "jobs": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "job_type": "export", + "status": "completed", + "input": { "collection": "xyz.statusphere.status" }, + "progress": { "processed": 1500 }, + "result": { "processed": 1500 }, + "error": null, + "created_by": "did:plc:abc123", + "inherit_auth": false, + "started_at": "2026-07-01T12:00:05Z", + "completed_at": "2026-07-01T12:02:30Z", + "created_at": "2026-07-01T12:00:00Z" + } + ], + "cursor": "next-page-cursor" +} +``` + +## Get job + +```http +GET /admin/jobs/:id +``` + +Returns a single job by ID. + +**Permission:** `jobs:read` + +**Response:** Same shape as a single item in the list response. + +## Cancel job + +```http +POST /admin/jobs/:id/cancel +``` + +Request cancellation of a job. If the job is `pending` or `paused`, it's immediately set to `cancelled`. If the job is `running`, it's set to `cancelling` — the worker will stop the job when the script next calls `job.should_stop()`. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "cancelling" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ---------------------------------------------- | +| 404 | Job not found | +| 409 | Job is already completed, failed, or cancelled | + +## Pause job + +```http +POST /admin/jobs/:id/pause +``` + +Request pause of a running job. Sets the status to `pausing` — the worker will pause the job when the script next calls `job.should_stop()`. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "pausing" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ------------------ | +| 404 | Job not found | +| 409 | Job is not running | + +## Resume job + +```http +POST /admin/jobs/:id/resume +``` + +Resume a paused job. Sets the status back to `pending` so the worker picks it up again. + +**Permission:** `jobs:manage` + +**Response:** + +```json +{ + "status": "pending" +} +``` + +**Errors:** + +| Status | Condition | +| ------ | ----------------- | +| 404 | Job not found | +| 409 | Job is not paused | + +## Job object + +| Field | Type | Description | +| -------------- | ------------ | ------------------------------------------------------------------------------- | +| `id` | string | UUID | +| `job_type` | string | The type name passed to `jobs.create()` | +| `status` | string | Current status (see [lifecycle](../../guides/background-jobs.md#job-lifecycle)) | +| `input` | object | Input data passed to `jobs.create()` | +| `progress` | object | Last progress update from `job.progress()` | +| `result` | object\|null | Return value of the script on completion | +| `error` | string\|null | Error message on failure | +| `created_by` | string | DID of the user who enqueued the job | +| `inherit_auth` | boolean | Whether the job inherits the creator's PDS auth | +| `started_at` | string\|null | ISO 8601 timestamp when the worker started executing | +| `completed_at` | string\|null | ISO 8601 timestamp when the job finished | +| `created_at` | string | ISO 8601 timestamp when the job was enqueued | diff --git a/packages/docs/content/docs/api-reference/admin/meta.json b/packages/docs/content/docs/api-reference/admin/meta.json index 79305b5..e93ac1b 100644 --- a/packages/docs/content/docs/api-reference/admin/meta.json +++ b/packages/docs/content/docs/api-reference/admin/meta.json @@ -7,6 +7,7 @@ "records", "stats", "backfill", + "jobs", "events", "api-keys", "users", diff --git a/packages/docs/content/docs/api-reference/lua/jobs-api.md b/packages/docs/content/docs/api-reference/lua/jobs-api.md new file mode 100644 index 0000000..bd2a81d --- /dev/null +++ b/packages/docs/content/docs/api-reference/lua/jobs-api.md @@ -0,0 +1,156 @@ +--- +title: "Jobs API" +--- + +Lua API for creating and managing background jobs. For a conceptual overview, see [Background Jobs](../../guides/background-jobs.md). + +## `jobs` table + +The `jobs` table is available in **all** script contexts (procedures, queries, record scripts, label scripts, and job scripts). It provides functions for queuing new jobs. + +### `jobs.create(job_type, input[, opts])` + +Enqueue a new background job. + +**Parameters:** + +| Parameter | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| `job_type` | string | The job type name. Must match a `job.run:` script trigger. | +| `input` | table | Input data passed to the job script via `job.input`. | +| `opts` | table? | Optional settings (see below). | + +**Options:** + +| Key | Type | Default | Description | +| ------ | ------- | ------- | -------------------------------------------------------- | +| `auth` | boolean | `false` | Inherit the caller's PDS auth. When `true`, the job script can use `r:save()`, `r:delete()`, and blob uploads as the creating user. When `false`, only local operations (`r:save_local()`, `r:delete_local()`) are available. | + +**Returns:** `string` — the new job's UUID. + +**Requires:** An authenticated caller (`caller_did` must be set). Raises an error in unauthenticated contexts. + +```lua +-- Enqueue a job without PDS auth (default) +function handle() + local job_id = jobs.create("stats.rebuild", { + collection = collection, + }) + return { job_id = job_id } +end +``` + +```lua +-- Enqueue a job that needs to write records on behalf of the caller +function handle() + local job_id = jobs.create("export", { + collection = collection, + format = input.format, + }, { auth = true }) + return { job_id = job_id } +end +``` + +Jobs can enqueue other jobs — a job script can call `jobs.create()` to spawn follow-up work: + +```lua +-- Inside a job script: fan out to per-collection jobs +function handle() + local collections = job.input.collections + local child_ids = {} + for _, col in ipairs(collections) do + table.insert(child_ids, jobs.create("export.collection", { + collection = col, + parent_job = job.id, + })) + end + return { children = child_ids } +end +``` + +## `job` table + +The `job` table is available **only inside job scripts** (trigger `job.run:`). It is `nil` in all other script contexts. + +### `job.id` + +**Type:** `string` + +The job's UUID. + +### `job.input` + +**Type:** `table` + +The input table that was passed to `jobs.create()` when the job was queued. + +### `job.progress(data)` + +Persist a progress snapshot to the database. + +**Parameters:** + +| Parameter | Type | Description | +| --------- | ----- | -------------------------------- | +| `data` | table | Any Lua table — stored as JSONB. | + +Each call overwrites the previous progress value. The snapshot is visible in the job detail panel in the dashboard and via `GET /admin/jobs/:id`. + +```lua +job.progress({ phase = "fetching", fetched = 250, total = 1000 }) +``` + +### `job.should_stop()` + +Check whether the job has been paused or cancelled. + +**Returns:** `boolean` — `true` if the operator requested a pause or cancel. + +Pause and cancel are cooperative. The worker sets a flag when the operator requests it, but the script must call `job.should_stop()` and exit gracefully. If your script never checks, pause and cancel requests wait until the script finishes on its own. + +```lua +for i, item in ipairs(items) do + if job.should_stop() then + return { partial = true, last = i } + end + process(item) +end +``` + +### `job.wait(seconds)` + +Sleep for the specified duration. + +**Parameters:** + +| Parameter | Type | Description | +| --------- | ------ | ---------------------------------------- | +| `seconds` | number | Duration in seconds (clamped to 0–3600). | + +Values below 0 are clamped to 0. Values above 3600 are clamped to 3600. + +```lua +-- Poll an external API with a delay between requests +for _, batch in ipairs(batches) do + local resp = http.post("https://api.example.com/import", { + body = json.encode(batch), + }) + job.wait(1) -- rate limit +end +``` + +## Available APIs + +Job scripts have access to all standard Lua APIs: + +- [`db.*`](database-api.md) — database queries +- [`http.*`](http-api.md) — HTTP client +- [`xrpc.*`](xrpc-lua-api.md) — XRPC calls +- [`atproto.*`](atproto-api.md) — DID resolution, labels, signing +- [`Record.*`](record-api.md) — record operations +- [`json.*`](json-api.md) — JSON encode/decode +- [`jobs.*`](#jobs-table) — queue follow-up jobs +- [`log()`](utility-globals.md), [`now()`](utility-globals.md), [`TID()`](utility-globals.md#tid), [`toarray()`](utility-globals.md) — utility globals +- `env.` — [script variables](../admin/script-variables.md) + +Unlike XRPC and record scripts, job scripts have **no instruction count limit** — they can run arbitrarily long computations. diff --git a/packages/docs/content/docs/api-reference/lua/meta.json b/packages/docs/content/docs/api-reference/lua/meta.json index d65b99a..09bfb19 100644 --- a/packages/docs/content/docs/api-reference/lua/meta.json +++ b/packages/docs/content/docs/api-reference/lua/meta.json @@ -7,6 +7,7 @@ "xrpc-lua-api", "atproto-api", "json-api", + "jobs-api", "utility-globals", "standard-libraries" ] diff --git a/packages/docs/content/docs/guides/background-jobs.md b/packages/docs/content/docs/guides/background-jobs.md new file mode 100644 index 0000000..3b532f0 --- /dev/null +++ b/packages/docs/content/docs/guides/background-jobs.md @@ -0,0 +1,232 @@ +--- +title: "Background Jobs" +--- + +Background jobs let you run long-running Lua scripts outside the request cycle. A script running in any context can queue a job with `jobs.create()`, and a dedicated worker picks it up and executes the matching job script. Jobs are useful for data migrations, batch exports, external API syncs, or any work that's too slow for a synchronous request. + +## How it works + +1. **Queue** - any Lua script calls `jobs.create("my-type", { ... })`, passing a free-form type name and an input table. This inserts a row into the `happyview_jobs` table with status `pending` and returns the job's UUID. +2. **Match** - the worker resolves the job's type to a script by looking up the trigger `job.run:my-type`. If no script exists for that type, the job fails immediately. +3. **Execute** - the worker calls the script's `handle()` function with the `job` global set. The script can report progress, check for cancellation, sleep, and return a result. + +## Creating a job script + +Job scripts are created from the [dashboard](../getting-started/dashboard.md) (Settings > Scripts > New) or via the [admin API](../api-reference/admin/scripts.md). + +In the dashboard, select **Job** as the trigger source, then type a job type name. The type name is a free-form string that must match `/^[a-z0-9][a-z0-9._-]*$/` (max 128 characters). The resulting trigger id is `job.run:`; for example, `job.run:export` or `job.run:data.migrate`. + +### Trigger grammar + +| Trigger | Fires when | +| ---------------- | ------------------------------------------------------- | +| `job.run:` | A job with the matching type is picked up by the worker | + +There is no cascade for job triggers; the type must match exactly. + +### Script structure + +Job scripts follow the same `handle()` convention as all other scripts. The return value becomes the job's `result` field. + +```lua +function handle() + local data = job.input + + for i, item in ipairs(data.items) do + -- process each item + job.progress({ processed = i, total = #data.items }) + + if job.should_stop() then + return { partial = true, processed = i } + end + end + + return { processed = #data.items } +end +``` + +## The `job` global + +Inside a job script, the `job` global provides access to the job's metadata and control functions. This global is only available in job scripts. It's `nil` in all other script contexts. + +| Field / Function | Type | Description | +| -------------------- | -------- | ----------------------------------------------------------- | +| `job.id` | string | The job's UUID | +| `job.input` | table | The input table passed to `jobs.create()` | +| `job.progress(data)` | function | Persist progress to the database (visible in the dashboard) | +| `job.should_stop()` | function | Returns `true` if the job has been paused or cancelled | +| `job.wait(seconds)` | function | Sleep for 0–3600 seconds | + +### `job.progress(data)` + +Call `job.progress()` to persist a progress snapshot. The `data` argument can be any Lua table: it's stored as JSONB and displayed in the job detail panel in the dashboard. Call it as often as you like; each call overwrites the previous progress value. + +```lua +job.progress({ status = "indexing", page = 5, total_pages = 20 }) +``` + +### `job.should_stop()` + +Check `job.should_stop()` at natural checkpoints in your script. It returns `true` when an operator has paused or cancelled the job from the dashboard. Cancellation and pausing are **cooperative**: the worker sets a flag, but it's up to your script to check it and exit gracefully. If your script never checks, pause and cancel requests will wait until the script finishes on its own. + +```lua +for i, repo in ipairs(repos) do + if job.should_stop() then + return { partial = true, last_processed = i } + end + process(repo) +end +``` + +### `job.wait(seconds)` + +Pause execution for up to 3600 seconds (1 hour). Useful for rate-limited external API calls or scheduled delays. Values below 0 are clamped to 0; values above 3600 are clamped to 3600. + +```lua +for _, batch in ipairs(batches) do + push_to_api(batch) + job.wait(2) -- respect rate limits +end +``` + +## Enqueuing jobs + +Any Lua script can enqueue a job using the `jobs` global. This is available in **all** script contexts: procedures, queries, record scripts, label scripts, and even other job scripts (a job can enqueue follow-up jobs). + +```lua +-- In a procedure script +function handle() + local job_id = jobs.create("export", { + collection = collection, + format = input.format, + }) + return { job_id = job_id, status = "queued" } +end +``` + +`jobs.create(type, input[, opts])` returns the new job's UUID as a string. The `type` argument must match a `job.run:` script trigger. If no matching script exists, the job will fail when the worker picks it up. + +See the [Jobs API reference](../api-reference/lua/jobs-api.md#jobscreatejob_type-input-opts) for the full parameter list. + +## Authentication + +By default, jobs run **without PDS auth**. This is intentional. Most jobs don't need to write records on behalf of a user, and granting auth by default would give long-running background scripts access to a user's PDS session unnecessarily. + +### What's available without auth + +Every job script — regardless of auth setting — has access to: + +- `caller_did` - the DID of the user who enqueued the job (always set) +- `db.*` - full database access (queries, raw SQL, search, backlinks) +- `http.*` - outbound HTTP requests +- `xrpc.*` - XRPC calls (local and proxied) +- `atproto.*` - DID resolution, label queries, signature verification +- `json.*` - JSON encode/decode +- `jobs.*` - enqueue follow-up jobs +- `Record.load()` - load records from the local database +- `r:save_local()` / `r:delete_local()` - write or delete records in HappyView's local database only +- `Record.delete_local()` - delete by URI from the local database +- Utility globals: `log()`, `now()`, `TID()`, `toarray()` +- `env.` - script variables + +### What requires auth + +PDS-touching operations need the creator's OAuth session. Without auth, these raise an error: + +- `r:save()` - writes a record to the user's PDS and indexes it locally +- `r:delete()` - deletes a record from the user's PDS and removes it locally +- `Record.save_all()` - batch save to PDS +- `atproto.upload_blob()` - upload a blob to the user's PDS + +### Opting into auth + +To give a job access to the creator's PDS session, pass `{ auth = true }` as the third argument to `jobs.create()`: + +```lua +-- Without auth (default) - local-only operations +jobs.create("stats.rebuild", { collection = collection }) + +-- With auth - can write to the creator's PDS +jobs.create("sync-records", { collection = collection }, { auth = true }) +``` + +When `auth = true`, the worker loads the creator's OAuth session at execution time. If the session is no longer valid (expired, revoked, or the user has no session), the job fails immediately with an error. The creating user must have a valid OAuth session when the job runs, not just when it was enqueued. + +### When to use auth + +Use `{ auth = true }` when the job needs to create, update, or delete records on the AT Protocol network on behalf of the user. For example, batch record creation, cross-collection syncs, or migrations that write back to the user's PDS. + +Leave auth off (the default) for jobs that only read data, compute aggregates, sync to external services, clean up local records, or perform any work that doesn't touch a user's PDS. + +## Job lifecycle + +Jobs move through these statuses: + +| Status | Description | +| ------------ | ------------------------------------------------- | +| `pending` | Queued, waiting for the worker to pick it up | +| `running` | Currently executing | +| `completed` | Script returned successfully | +| `failed` | Script raised an error | +| `pausing` | Pause requested, waiting for the script to check | +| `paused` | Script exited after detecting the pause flag | +| `cancelling` | Cancel requested, waiting for the script to check | +| `cancelled` | Script exited after detecting the cancel flag | + +### Pausing and cancelling + +Pause and cancel are requested via the dashboard or the [admin API](../api-reference/admin/jobs.md). Both are cooperative: + +1. The endpoint sets the job's status to `pausing` or `cancelling`. +2. The worker continues running the script. At its next `job.should_stop()` check, it returns `true`. +3. The script should exit gracefully. Whatever it returns becomes the job's result. +4. The worker sets the final status to `paused` or `cancelled`. + +If the script never calls `job.should_stop()`, the pause or cancel request waits until the script finishes naturally. + +A paused job can be resumed via `POST /admin/jobs/:id/resume` or the Resume button in the dashboard. Resuming sets the status back to `pending`, and the worker picks it up again, but the script runs from the beginning. Use `job.input` or progress data to implement resumable logic. + +### Recovery after restart + +Jobs survive server restarts. On startup, the worker checks for orphaned jobs: + +- **Running** jobs are reset to `pending` and re-queued. +- **Cancelling** jobs are finalised as `cancelled`. +- **Pausing** jobs are finalised as `paused`. + +## Worker + +The job worker runs as a background task inside the HappyView server process. It polls for pending jobs every 5 seconds and executes one job at a time. Job scripts have **no instruction count limit** (unlike XRPC and record scripts, which are capped at 1,000,000 instructions), so they can run arbitrarily long computations. + +Job scripts have access to all standard Lua APIs: `db.*`, `http.*`, `xrpc.*`, `atproto.*`, `Record.*`, `json.*`, `env.`, `log()`, `now()`, `TID()`, `toarray()`, and `jobs.*` (including `jobs.create()` to queue follow-up jobs). + +## Dashboard + +The **Jobs** page in the dashboard (`/dashboard/jobs`) shows all background jobs in a filterable table. You can filter by status using the dropdown at the top. + +Clicking a job row opens a detail sheet showing: + +- Job ID, type, and status +- Input data (the table passed to `jobs.create()`) +- Progress (the last value passed to `job.progress()`) +- Result or error +- Timestamps (created, started, completed) +- Action buttons: **Cancel**, **Pause**, or **Resume** depending on the current status + +## Permissions + +Job management requires specific permissions: + +| Permission | Grants | +| ------------- | ---------------------------------- | +| `jobs:read` | View jobs in the dashboard and API | +| `jobs:manage` | Cancel, pause, and resume jobs | + +Queuing jobs via `jobs.create()` in a script requires an authenticated caller (`caller_did` must be set). + +## Next steps + +- [Admin API - Jobs](../api-reference/admin/jobs.md): Full reference for job endpoints +- [Lua API - Jobs](../api-reference/lua/jobs-api.md): Full reference for the `jobs` and `job` Lua APIs +- [Lua Scripting](lua-scripting.md): General Lua scripting reference +- [Record & Label Scripts](record-scripts.md): Trigger grammar for all script types diff --git a/packages/docs/content/docs/guides/lua-scripting.md b/packages/docs/content/docs/guides/lua-scripting.md index 4887f97..58cb68c 100644 --- a/packages/docs/content/docs/guides/lua-scripting.md +++ b/packages/docs/content/docs/guides/lua-scripting.md @@ -69,14 +69,14 @@ These globals are set automatically before `handle()` is called. When a script handles a space-scoped request, the `space` global is set to a table with the space's metadata. For non-space requests, `space` is `nil`. -| Field | Type | Description | -| ----------- | ------ | -------------------------------------------------------- | -| `space` | string | The full `ats://` space URI | -| `space_id` | string | Internal space identifier | -| `did` | string | The space's DID | -| `authority_did` | string | The space authority's DID | -| `type_nsid` | string | Space type NSID | -| `skey` | string | Space key | +| Field | Type | Description | +| --------------- | ------ | --------------------------- | +| `space` | string | The full `ats://` space URI | +| `space_id` | string | Internal space identifier | +| `did` | string | The space's DID | +| `authority_did` | string | The space authority's DID | +| `type_nsid` | string | Space type NSID | +| `skey` | string | Space key | ```lua function handle() @@ -179,6 +179,21 @@ The `json` global provides JSON serialization and deserialization. See the full [JSON API reference](../api-reference/lua/json-api.md) for `json.encode` and `json.decode`. +## Jobs API + +The `jobs` table lets any script queue background jobs for long-running work. Available in all script contexts. + +See the full [Jobs API reference](../api-reference/lua/jobs-api.md) for `jobs.create()` and the `job` global available inside job scripts. + +Quick example: + +```lua +local job_id = jobs.create("export", { collection = collection }) +return { job_id = job_id } +``` + +For the full guide on background jobs, see [Background Jobs](background-jobs.md). + ## Debugging ### Logging diff --git a/packages/docs/content/docs/guides/meta.json b/packages/docs/content/docs/guides/meta.json index 7bc4499..c06e2b0 100644 --- a/packages/docs/content/docs/guides/meta.json +++ b/packages/docs/content/docs/guides/meta.json @@ -4,6 +4,7 @@ "upgrading-to-v2", "lexicons", "backfill", + "background-jobs", "label-scripts", "lua-scripting", "api-clients", diff --git a/packages/docs/content/docs/guides/record-scripts.md b/packages/docs/content/docs/guides/record-scripts.md index f1ac958..ea96e58 100644 --- a/packages/docs/content/docs/guides/record-scripts.md +++ b/packages/docs/content/docs/guides/record-scripts.md @@ -41,6 +41,14 @@ XRPC scripts handle the request and return the response. Without a script, Happy There is no cascade for label or XRPC triggers -- each trigger string must match exactly. +### Job triggers + +| Trigger | Fires when | +| -------------------------- | --------------------------------------------- | +| `job.run:` | A background job with the matching type is picked up by the worker | + +There is no cascade for job triggers -- the type must match exactly. See [Background Jobs](./background-jobs.md) for the full job scripting guide. + ## Creating scripts You can create scripts through the [dashboard](../getting-started/dashboard.md) (Settings > Scripts > New) or via the [admin API](../api-reference/admin/scripts.md) (`POST /admin/scripts`). diff --git a/packages/docs/package.json b/packages/docs/package.json index e035f81..3f36614 100644 --- a/packages/docs/package.json +++ b/packages/docs/package.json @@ -17,7 +17,7 @@ "fumadocs-mdx": "^15.0.4", "fumadocs-ui": "^16.8.10", "lucide-react": "^1.14.0", - "mermaid": "^11.6.0", + "mermaid": "^11.16.0", "next": "^16.1.6", "next-themes": "^0.4.6", "react": "^19.2.0", -- 2.51.2