From d1432dd4c7096d3358030b02254a8b6d5f470e9c Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Sat, 13 Jun 2026 14:23:59 -0500 Subject: [PATCH] docs: release & deployment * pbs (personal backup server) plan --- docs/reference/README.md | 2 + docs/reference/budget.md | 5 + docs/reference/deployment-observability.md | 344 +++++++++++++++++++-- docs/reference/deployment.md | 309 ++++++++++++++++++ docs/reference/identity-troubleshooting.md | 3 + docs/reference/interop-testing.md | 26 +- docs/reference/release.md | 205 ++++++++++++ docs/specs/README.md | 2 + docs/specs/personal-backups.md | 288 +++++++++++++++++ docs/tasks/15-deployment-verification.md | 14 +- docs/tasks/18-personal-backups.md | 110 +++++++ docs/tasks/README.md | 1 + test/smoke/README.md | 27 +- test/smoke/deployment.hurl | 87 ++++++ 14 files changed, 1380 insertions(+), 43 deletions(-) create mode 100644 docs/reference/deployment.md create mode 100644 docs/reference/release.md create mode 100644 docs/specs/personal-backups.md create mode 100644 docs/tasks/18-personal-backups.md create mode 100644 test/smoke/deployment.hurl diff --git a/docs/reference/README.md b/docs/reference/README.md index f650696..aa52e68 100644 --- a/docs/reference/README.md +++ b/docs/reference/README.md @@ -19,7 +19,9 @@ Graduated reference docs: - [Migration and Account Lifecycle](./migration-lifecycle.md) - [Security, OAuth, and Delegated Access](./security-oauth.md) - [Admin and Operator Operations](./admin-operations.md) +- [Deployment Guide](./deployment.md) - [Deployment and Observability](./deployment-observability.md) +- [Initial Release Readiness](./release.md) - [PDS Compatibility Matrix](./pds-compatibility.md) - [Lexicon Schemas](./lexicon-schemas.md) - [Interop and Integration Testing](./interop-testing.md) diff --git a/docs/reference/budget.md b/docs/reference/budget.md index bb4b2d1..4bcefe7 100644 --- a/docs/reference/budget.md +++ b/docs/reference/budget.md @@ -7,6 +7,8 @@ This page describes the lowest-cost hosted shape for a single-user Tempest PDS: Railway Hobby for compute and durable SQLite state, plus Cloudflare R2 Standard storage for blob objects and backup uploads. The target is \< $10 a month. +For the step-by-step deploy flow, use [Deployment Guide](./deployment.md). + Provider prices and limits change. The numbers below were checked on 2026-06-12 against the public Railway and @@ -199,6 +201,9 @@ Do not count R2 blob storage as a complete backup by itself. The SQLite files and repo databases are the authoritative state tying accounts, repo commits, and blob metadata together. +The full deployment restore runbook lives in +[`deployment observability`](./deployment-observability.md#restore-drill). + ## Cost controls Set alerts or calendar checks for: diff --git a/docs/reference/deployment-observability.md b/docs/reference/deployment-observability.md index b05180b..f947229 100644 --- a/docs/reference/deployment-observability.md +++ b/docs/reference/deployment-observability.md @@ -1,13 +1,31 @@ --- title: Deployment and Observability -updated: 2026-06-03 +updated: 2026-06-13 --- Tempest's first production shape is a Phoenix release behind HTTPS with one durable data volume. SQLite databases, repo data, signing keys, OAuth keys, blob state, and backup workspaces must live on durable storage. -## Local deployment shape +For a step-by-step Railway deployment path, use +[Deployment Guide](./deployment.md). + +## Deployment profiles + +Tempest supports three deployment profiles for the current single-node target: + +- local-only with all state under `TEMPEST_DATA_DIR`; +- local release or container behind a reverse proxy, usually Caddy; +- managed PaaS, currently documented as Railway plus optional Cloudflare R2. + +All profiles use one running Tempest instance. Do not run multiple writers +against the same data directory. SQLite, repo storage, sequencer state, OAuth +keys, signing keys, and backup workspaces must move together. + +## Local-only profile + +Use this profile for local proving, private LAN testing, or a single host where +Phoenix terminates HTTP directly. ```yaml services: @@ -24,6 +42,104 @@ services: No external database service is required for the SQLite-first profile. +For a release or container, the mounted data path must be durable: + +```text +TEMPEST_DATA_DIR=/var/lib/tempest +``` + +Local-only verification: + +```bash +curl -fsS http://localhost:4000/xrpc/_health +curl -fsS http://localhost:4000/xrpc/com.atproto.server.describeServer +``` + +This profile does not prove public DID, handle, relay, AppView, OAuth, or +WebSocket behavior unless the host is reachable from the public internet. + +## S3/R2-backed profile + +Use this profile when blob bytes and backup archives should live outside the +local volume. R2 or another S3-compatible store does not replace the durable +Tempest data directory. Account state, sessions, repo SQLite files, the +sequencer, metadata, and key material still live under `TEMPEST_DATA_DIR`. + +Set blob storage: + +```text +TEMPEST_BLOB_STORE=s3 +TEMPEST_BLOB_S3_ENDPOINT=https://.r2.cloudflarestorage.com +TEMPEST_BLOB_S3_BUCKET=tempest-blobs +TEMPEST_BLOB_S3_REGION=auto +TEMPEST_BLOB_S3_ACCESS_KEY_ID=... +TEMPEST_BLOB_S3_SECRET_ACCESS_KEY=... +``` + +Set backup uploads: + +```text +TEMPEST_BACKUP_STORE=s3 +TEMPEST_BACKUP_S3_ENDPOINT=https://.r2.cloudflarestorage.com +TEMPEST_BACKUP_S3_BUCKET=tempest-backups +TEMPEST_BACKUP_S3_REGION=auto +TEMPEST_BACKUP_S3_ACCESS_KEY_ID=... +TEMPEST_BACKUP_S3_SECRET_ACCESS_KEY=... +``` + +Use separate buckets or separate scoped credentials when possible. The blob +bucket needs object read/write for normal operation. The backup bucket needs +write for backup creation and read for restore drills. + +## Reverse-proxy profile + +A reverse proxy terminates HTTPS and forwards HTTP/WebSocket traffic to Phoenix. +The proxy must preserve host headers and pass WebSocket upgrades for: + +```text +/xrpc/com.atproto.sync.subscribeRepos +``` + +A local Caddy deployment can use: + +```caddyfile +tempest.example.com { + reverse_proxy 127.0.0.1:4000 +} +``` + +`TEMPEST_HOSTNAME` must be the bare external hostname. `TEMPEST_PUBLIC_URL` must +match the external HTTPS origin because DID documents and OAuth metadata use it +as the service boundary. + +```text +TEMPEST_HOSTNAME=tempest.example.com +TEMPEST_PUBLIC_URL=https://tempest.example.com +``` + +Do not use a temporary host for account migration. It can prove boot, but DID +documents, handles, OAuth clients, relays, and AppViews should be verified +against the final hostname. + +## Railway profile + +For the single-user Railway plus Cloudflare R2 budget profile, see +[`budget`](./budget.md). + +Railway-specific rules: + +- use one Railway service; +- keep replicas at 1; +- attach one volume at `/var/lib/tempest`; +- set `TEMPEST_DATA_DIR=/var/lib/tempest`; +- let Railway provide `PORT`; +- use the final custom domain for `TEMPEST_HOSTNAME`. + +Railway volumes are persistent, but Railway documents several caveats relevant +to Tempest: each service can only have one volume, replicas cannot be used with +volumes, and deployments with an attached volume may have a short downtime window +because multiple active deployments cannot mount the same service volume safely. + ## Required environment ```text @@ -31,7 +147,6 @@ TEMPEST_HOSTNAME=tempest.example.com TEMPEST_PUBLIC_URL=https://tempest.example.com TEMPEST_DATA_DIR=/var/lib/tempest SECRET_KEY_BASE=... -TEMPEST_JWT_SECRET=... TEMPEST_ADMIN_TOKEN_HASH=... TEMPEST_BLOB_STORE=local TEMPEST_BLOB_MAX_BYTES=10000000 @@ -44,9 +159,6 @@ Optional adapters add SMTP, S3/R2 blob storage, and S3/R2 backup uploads. Those profiles also need endpoint, bucket, region, and credential variables for the chosen object store. -For the single-user Railway plus Cloudflare R2 budget profile, see -[`budget`](./budget.md). - ## Durable paths A deployment must preserve these paths together: @@ -65,25 +177,6 @@ Losing only one of these can make accounts unverifiable. For example, repo data without signing keys cannot safely emit new commits, and blobs without repo state cannot prove which blobs are public. -## HTTPS and WebSockets - -A local Caddy deployment can terminate HTTPS and proxy Phoenix: - -```caddyfile -tempest.example.com { - reverse_proxy 127.0.0.1:4000 -} -``` - -The reverse proxy must pass WebSocket upgrades for: - -```text -/xrpc/com.atproto.sync.subscribeRepos -``` - -`TEMPEST_PUBLIC_URL` must match the externally reachable URL because DID -documents and OAuth metadata use it as the service boundary. - ## Observability Tempest emits telemetry and structured logs around the public protocol surface. @@ -115,7 +208,198 @@ GET /admin/storage `/_health` is public and minimal. Admin status requires the admin token. -## Verification +## Deployed smoke profile + +Run these checks against the final external hostname. Do not count localhost, +private DNS, or a temporary Railway domain as deployment verification. + +Set variables: + +```bash +export BASE_URL=https://tempest.example.com +export HOSTNAME=tempest.example.com +export ADMIN_TOKEN=... +``` + +Health and metadata: + +```bash +curl -fsS "$BASE_URL/xrpc/_health" +curl -fsS "$BASE_URL/xrpc/com.atproto.server.describeServer" +curl -fsS "$BASE_URL/.well-known/oauth-protected-resource" +curl -fsS "$BASE_URL/.well-known/oauth-authorization-server" +``` + +Admin status: + +```bash +curl -fsS \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + "$BASE_URL/xrpc/_admin/status" +``` + +Non-destructive Hurl smoke: + +```bash +hurl --test --jobs 1 \ + --variable base_url="$BASE_URL" \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` + +WebSocket firehose: + +```bash +websocat "wss://$HOSTNAME/xrpc/com.atproto.sync.subscribeRepos?cursor=0" +``` + +If `websocat` is not available, use any WebSocket client that reports connection +failure and frame output clearly. An idle connection is acceptable immediately +after deploy; the important first check is that the HTTPS upgrade succeeds. + +## Restore drill + +A managed deployment is not proven until a restore has booted from separate +storage. For Railway plus R2, use a separate test service or stop the live +service before attaching an empty volume. + +1. Create a backup from the running release. + + ```bash + bin/tempest eval 'case Tempest.Admin.Backup.create(upload?: true) do {:ok, result} -> IO.inspect(result); {:error, reason} -> raise inspect(reason) end' + ``` + + If running from a Mix environment instead of a release: + + ```bash + mix pds.backup.create --upload-s3 + ``` + +2. Confirm the backup archive exists in the configured backup bucket. +3. Download and extract the backup archive into a temporary workspace. +4. Attach an empty Railway volume to a test service, mounted at + `/var/lib/tempest`. +5. Restore into the empty mounted directory: + + ```bash + bin/tempest eval 'case Tempest.Admin.Backup.restore("/path/to/extracted-backup", target: "/var/lib/tempest") do {:ok, result} -> IO.inspect(result); {:error, reason} -> raise inspect(reason) end' + ``` + + Or, from a Mix environment: + + ```bash + mix pds.backup.restore \ + --input /path/to/extracted-backup \ + --target /var/lib/tempest + ``` + +6. Start the restored service with the same runtime variables, except use a test + hostname if the original identity must not move. +7. Re-run health, describeServer, admin status, blob read, repo read, DID/handle, + and WebSocket checks. + +Do not treat R2 blob storage as a complete backup. The SQLite files, repo +databases, sequencer, keys, and metadata are the state that explains which blobs +belong to which account. + +## Public identity verification + +For each hosted account used in deployment testing, record: + +```text +HANDLE=alice.example.com +DID=did:plc:... +BASE_URL=https://tempest.example.com +``` + +Verify handle resolution through Tempest: + +```bash +curl -fsS \ + "$BASE_URL/xrpc/com.atproto.identity.resolveHandle?handle=$HANDLE" +``` + +Verify handle well-known resolution when the handle host is controlled by this +deployment: + +```bash +curl -fsS "https://$HANDLE/.well-known/atproto-did" +``` + +Verify the DID document with the appropriate resolver. For `did:plc`, fetch the +PLC document: + +```bash +curl -fsS "https://plc.directory/$DID" +``` + +For `did:web`, fetch the DID document from its well-known URL. + +The DID document must include: + +```text +alsoKnownAs: at:// +service id: #atproto_pds +serviceEndpoint: +``` + +Only activate or migrate an account after the public DID document points at the +final `TEMPEST_PUBLIC_URL`. Existing services may cache identity; after a change, +repeat the checks from a network outside the deployment provider. + +## Relay and AppView crawl verification + +Configure public crawlers: + +```text +TEMPEST_CRAWLERS=https://bsky.network,https://vsky.network +``` + +Run the deployed crawler smoke test: + +```bash +hurl --test --jobs 1 \ + --variable base_url="$BASE_URL" \ + --variable crawler_hostname="$HOSTNAME" \ + test/smoke/deployed/crawlers.hurl +``` + +Then create a small repo-visible event from a real or smoke account, such as a +profile update or post, and verify: + +- `com.atproto.sync.getLatestCommit` returns the new rev; +- `com.atproto.sync.getRepo` returns a CAR for the account DID; +- `com.atproto.sync.subscribeRepos` emits a commit frame for new writes; +- a relay or AppView that can crawl the hostname no longer reports the PDS as + unreachable. + +If a relay appears stale, first verify that DNS, TLS, `TEMPEST_PUBLIC_URL`, and +WebSocket upgrades are correct. Then run `com.atproto.sync.requestCrawl` again +against the deployed hostname. + +## Real-client checklist + +Use at least one real client against the final hostname before migrating the +admin account. Record the client name and date. + +- Add the custom service URL in the client. +- Log in with the test account. +- Refresh the session or close/reopen the client and confirm the session still + works. +- Read the profile. +- Update the profile display name or description. +- Create a post. +- Upload an image blob and create a record that references it. +- Read the post and blob back through the client. +- Revoke or rotate an app password if the client used one. +- Confirm admin routes still reject normal account tokens. +- Confirm the same writes appear through repo reads and the firehose. + +Do not use the admin account as the first client test. Use a disposable account +or an inactive migration test account until the service has passed restore and +public identity checks. + +## Verification summary Local: @@ -135,3 +419,11 @@ curl --no-buffer \ A deployment is not verified until HTTPS, DID/handle resolution, WebSockets, blob reads, and restore drills all work from outside the host. + +## Sources checked + +- Railway Phoenix guide: +- Railway volumes reference: +- AT Protocol account migration guide: + +- AT Protocol handle spec: diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md new file mode 100644 index 0000000..7eb8eee --- /dev/null +++ b/docs/reference/deployment.md @@ -0,0 +1,309 @@ +--- +title: Deployment Guide +updated: 2026-06-13 +--- + +This guide deploys Tempest as a single-user PDS on Railway with one persistent +volume and optional Cloudflare R2 storage for blobs and backup archives. + +Use this guide with: + +- [Initial Release Readiness](./release.md) +- [Deployment and Observability](./deployment-observability.md) +- [Budget Deployment](./budget.md) + +## Before You Start + +Choose the final public hostname first. + +```text +TEMPEST_HOSTNAME=tempest.example.com +TEMPEST_PUBLIC_URL=https://tempest.example.com +``` + +`TEMPEST_HOSTNAME` is the bare host only. Do not include scheme, path, or port. +Do not migrate a real account to a temporary Railway hostname. DID documents, +handles, OAuth metadata, relays, and AppViews should be verified against the +final HTTPS hostname. + +Required local tools: + +```bash +mix phx.gen.secret +hurl --version +docker --version +``` + +Generate secrets from a trusted local machine: + +```bash +mix phx.gen.secret +ADMIN_TOKEN="$(openssl rand -base64 48)" +ADMIN_TOKEN="$ADMIN_TOKEN" \ + mix run -e 'IO.puts Tempest.AdminAuth.hash_token(System.fetch_env!("ADMIN_TOKEN"))' +``` + +Store the raw `ADMIN_TOKEN` in a password manager. Railway gets only +`TEMPEST_ADMIN_TOKEN_HASH`. + +## Build Check + +Before deploying, prove the release image builds: + +```bash +docker build -f conf/Dockerfile -t tempest:release-check . +docker rmi tempest:release-check +``` + +Run the normal project gate: + +```bash +mix precommit +``` + +## Create R2 Buckets + +R2 is recommended for the first hosted deployment. It keeps blob bytes and backup +archives off the Railway volume, but it does not replace the volume. + +Create two buckets or two separately scoped prefixes: + +```text +tempest-blobs +tempest-backups +``` + +Create scoped R2 tokens: + +- blob bucket: Object Read and Write +- backup bucket: Object Read and Write + +Record the account endpoint: + +```text +https://.r2.cloudflarestorage.com +``` + +If the bucket uses an R2 jurisdiction, use the jurisdiction endpoint instead. + +## Create Railway Service + +Create one Railway service for Tempest. + +Required service shape: + +```text +replicas=1 +volume mount=/var/lib/tempest +``` + +Do not run multiple replicas. Railway volumes do not support active replicas, +and Tempest's SQLite profile expects one writer. + +Set the custom domain in Railway before migrating an account. Wait for DNS and +TLS to become healthy. + +## Set Railway Variables + +Minimum variables: + +```text +PHX_SERVER=true +POOL_SIZE=5 +SECRET_KEY_BASE= +TEMPEST_HOSTNAME=tempest.example.com +TEMPEST_PUBLIC_URL=https://tempest.example.com +TEMPEST_DATA_DIR=/var/lib/tempest +TEMPEST_HOSTED_DID_METHOD=plc +TEMPEST_ADMIN_TOKEN_HASH= +TEMPEST_BLOB_MAX_BYTES=10000000 +TEMPEST_CRAWLERS=https://bsky.network,https://vsky.network +``` + +Railway supplies `PORT`; leave it unset unless you have a specific reason to +override Railway's value. + +R2 blob storage: + +```text +TEMPEST_BLOB_STORE=s3 +TEMPEST_BLOB_S3_ENDPOINT=https://.r2.cloudflarestorage.com +TEMPEST_BLOB_S3_BUCKET=tempest-blobs +TEMPEST_BLOB_S3_REGION=auto +TEMPEST_BLOB_S3_ACCESS_KEY_ID=... +TEMPEST_BLOB_S3_SECRET_ACCESS_KEY=... +``` + +R2 backup uploads: + +```text +TEMPEST_BACKUP_STORE=s3 +TEMPEST_BACKUP_S3_ENDPOINT=https://.r2.cloudflarestorage.com +TEMPEST_BACKUP_S3_BUCKET=tempest-backups +TEMPEST_BACKUP_S3_REGION=auto +TEMPEST_BACKUP_S3_ACCESS_KEY_ID=... +TEMPEST_BACKUP_S3_SECRET_ACCESS_KEY=... +``` + +Optional SMTP: + +```text +TEMPEST_SMTP_ENABLED=false +``` + +Leave SMTP disabled for the first deployment unless password reset and email +confirmation delivery have been configured and tested. + +## First Boot + +Deploy the service. The Docker entrypoint creates the storage layout, bootstraps +SQLite, runs migrations, and starts the Phoenix release. + +Check Railway logs for startup errors. Then verify externally: + +```bash +export BASE_URL=https://tempest.example.com +export HOSTNAME=tempest.example.com +export ADMIN_TOKEN= + +curl -fsS "$BASE_URL/xrpc/_health" +curl -fsS "$BASE_URL/xrpc/com.atproto.server.describeServer" +curl -fsS \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + "$BASE_URL/xrpc/_admin/status" +``` + +Run the deployed HTTPS smoke test: + +```bash +hurl --test --jobs 1 \ + --variable base_url="$BASE_URL" \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` + +## WebSocket Check + +Verify the firehose WebSocket upgrades over HTTPS: + +```bash +websocat "wss://$HOSTNAME/xrpc/com.atproto.sync.subscribeRepos?cursor=0" +``` + +An idle connection is acceptable before any account writes. The first check is +that the connection upgrades successfully and does not fail at the proxy or TLS +layer. + +## Relay Crawl Check + +Run the crawler smoke test against the public hostname: + +```bash +hurl --test --jobs 1 \ + --variable base_url="$BASE_URL" \ + --variable crawler_hostname="$HOSTNAME" \ + test/smoke/deployed/crawlers.hurl +``` + +This only proves that Tempest accepts the crawl request shape. Full federation +proof also requires a repo-visible write and checking that external relays or +AppViews can fetch the repo. + +## Backup and Restore Drill + +Create an uploaded backup from the running release: + +```bash +bin/tempest eval 'case Tempest.Admin.Backup.create(upload?: true) do {:ok, result} -> IO.inspect(result); {:error, reason} -> raise inspect(reason) end' +``` + +Confirm the archive exists in R2. Download and extract it locally or in a restore +workspace. + +Restore into a separate test service or stopped service with an empty volume: + +```bash +bin/tempest eval 'case Tempest.Admin.Backup.restore("/path/to/extracted-backup", target: "/var/lib/tempest") do {:ok, result} -> IO.inspect(result); {:error, reason} -> raise inspect(reason) end' +``` + +Start the restored service and rerun: + +```bash +hurl --test --jobs 1 \ + --variable base_url="$BASE_URL" \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` + +R2 blob storage is not a complete backup by itself. The SQLite files, repo +databases, sequencer, OAuth keys, signing keys, and metadata under +`TEMPEST_DATA_DIR` are the authoritative state. + +## Identity Check + +Before migrating any real account, verify identity from outside Railway. + +For a test account: + +```bash +export HANDLE=alice.example.com +export DID=did:plc:... + +curl -fsS "$BASE_URL/xrpc/com.atproto.identity.resolveHandle?handle=$HANDLE" +curl -fsS "https://plc.directory/$DID" +``` + +The public DID document must include: + +```text +alsoKnownAs: at:// +service id: #atproto_pds +serviceEndpoint: +``` + +For `did:web`, fetch the DID document from the handle's well-known URL instead +of PLC. + +## Real Client Check + +Use a disposable account before the admin account. + +- Add the custom service URL in the client. +- Log in. +- Close and reopen the client to prove session refresh. +- Read the profile. +- Update the profile. +- Create a post. +- Upload an image blob and create a post or record that references it. +- Confirm the write appears through `getLatestCommit`, `getRepo`, and the + firehose. +- Confirm admin routes still reject normal account tokens. + +## Admin Account Migration Gate + +Only after the checks above pass: + +1. Export the old PDS repo CAR. +2. Request service auth from the old PDS for account creation on Tempest. +3. Create the inactive account on Tempest. +4. Import the CAR. +5. Upload missing blobs. +6. Run `checkAccountStatus`. +7. Update identity so `#atproto_pds` points at Tempest. +8. Verify public DID and handle resolution again. +9. Activate the account. +10. Keep the old account undeleted through the validation window. + +The migration procedure is described in +[Account Migration](./account-migration.md). The release gate is described in +[Initial Release Readiness](./release.md). + +## Rollback + +Before activation, rollback is deleting or ignoring the Tempest staging account. +The old PDS remains authoritative. + +After identity update, rollback means moving the DID document back to the old PDS +and waiting for caches to settle. Keep the latest Tempest backup archive outside +Railway, and keep old-PDS credentials available until the new deployment has +passed the validation window. diff --git a/docs/reference/identity-troubleshooting.md b/docs/reference/identity-troubleshooting.md index 6fc8844..e1b74c9 100644 --- a/docs/reference/identity-troubleshooting.md +++ b/docs/reference/identity-troubleshooting.md @@ -47,3 +47,6 @@ hurl --test --jobs 1 \ --variable base_url=http://localhost:4000 \ test/smoke/identity-correctness.hurl ``` + +For deployed public verification, use the runbook in +[`deployment-observability`](./deployment-observability.md#public-identity-verification). diff --git a/docs/reference/interop-testing.md b/docs/reference/interop-testing.md index ad4a90d..c4b5288 100644 --- a/docs/reference/interop-testing.md +++ b/docs/reference/interop-testing.md @@ -56,12 +56,15 @@ Tempest uses these layers together: - `test/smoke/operator-account-ux.hurl`: account operator UI checks - `test/smoke/tempest_basic.hurl`: end-to-end baseline PDS flow - `test/smoke/tempest_compat.hurl`: compatibility hardening checks +- `test/smoke/deployment.hurl`: non-destructive deployed HTTPS smoke checks - `test/smoke/deployed/crawlers.hurl`: deployed relay crawler fan-out checks Run suites that create accounts or depend on event order with `--jobs 1`. Use fresh account variables for every run. `accounts.hurl` and `identity.hurl` both use `account_handle`, so run them separately or give the full directory run a fresh database/handle plan if both files create accounts in the same pass. +Do not include `test/smoke/deployment.hurl` in local wildcard runs; it requires +a deployed HTTPS hostname and an admin token. ## Hurl rules @@ -118,21 +121,19 @@ firehose observation. `test/smoke/deployed/crawlers.hurl` covers `requestCrawl` against configured relays. Run it only against a publicly reachable deployment, because real relays such as `bsky.network` and `vsky.network` reject `localhost` and private hostnames. +The full deployed relay/AppView procedure lives in +[`deployment-observability`](./deployment-observability.md#relay-and-appview-crawl-verification). ## Verification ```bash mix test -suffix="$(date +%s)" -hurl --test --jobs 1 \ - --variable base_url=http://localhost:4000 \ - --variable suffix="${suffix}" \ - --variable account_handle="smoke-${suffix}.test" \ - --variable account_email="smoke-${suffix}@example.com" \ - --variable account_password="correct horse battery staple" \ - test/smoke/*.hurl +test/smoke/local-pds-compat.sh http://localhost:4000 ``` +For broader local smoke runs, list local files explicitly. Do not use +`test/smoke/*.hurl`, because `test/smoke/deployment.hurl` is deployed-only. + Deployed crawler check: ```bash @@ -142,6 +143,15 @@ hurl --test --jobs 1 \ test/smoke/deployed/crawlers.hurl ``` +Deployed HTTPS smoke check: + +```bash +hurl --test --jobs 1 \ + --variable base_url=https://tempest.example.com \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` + ## Sources - diff --git a/docs/reference/release.md b/docs/reference/release.md new file mode 100644 index 0000000..1cd8192 --- /dev/null +++ b/docs/reference/release.md @@ -0,0 +1,205 @@ +--- +title: Initial Release Readiness +date: 2026-06-13 +status: draft +--- + +Tempest is ready for an initial Railway staging deployment. It is not yet ready +for an irreversible migration of the main Bluesky account. + +The app has enough release packaging, local protocol coverage, and storage documentation +to deploy it behind a stable HTTPS hostname. It still needs public-network proof before it +can become the authoritative PDS for the admin account. + +## Plan + +Proceed with a controlled Railway deployment using a single service, one mounted +volume at `/var/lib/tempest`, and Cloudflare R2 for blobs and backups. +Use [Deployment Guide](./deployment.md) for the step-by-step deployment path. + +Do not activate the migrated admin account until all release gates pass: + +- HTTPS health and XRPC checks pass against the final custom domain. +- WebSocket firehose works from outside Railway. +- DID and handle resolution point at the final `TEMPEST_PUBLIC_URL`. +- Relay/AppView crawl checks succeed. +- A backup can be created, downloaded, restored into a fresh volume, and used to + boot a separate test service. +- A real client can log in, refresh a session, write a profile/post, upload and + read a blob, and read the migrated repository. + +## Tests + +Local verification on 2026-06-13: + +```text +mix precommit +Result: 292 passed +``` + +The precommit alias compiled with warnings as errors, checked unused +dependencies, formatted the codebase, and ran the test suite. + +Docker Compose config renders cleanly: + +```text +docker compose -f conf/docker-compose.yml config +``` + +The release image builds successfully: + +```text +docker build -f conf/Dockerfile -t tempest:deploy-readiness . +``` + +The deployed HTTPS smoke file exists at `test/smoke/deployment.hurl`. It should +be run after the Railway service is reachable at the final hostname: + +```bash +hurl --test --jobs 1 \ + --variable base_url=https://tempest.example.com \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` + +## Release Surface + +- Phoenix release Dockerfile at `conf/Dockerfile`. +- Docker entrypoint at `conf/docker-entrypoint.sh` that creates the durable data + layout, bootstraps storage, runs Ecto migrations, and starts the release. +- Production runtime config that reads Railway's `PORT`. +- Required production secrets and URL boundary in `conf/.env.example`. +- Budget Railway plus R2 profile in `docs/reference/budget.md`. +- Step-by-step Railway deployment guide in `docs/reference/deployment.md`. +- Deployment and observability notes in + `docs/reference/deployment-observability.md`. +- Local PDS compatibility and migration lifecycle smoke coverage. +- Admin UI and admin JSON status for operator checks. + +## Railway Conf + +Use one Railway service and one volume. + +Required service shape: + +```text +replicas=1 +volume mount=/var/lib/tempest +TEMPEST_DATA_DIR=/var/lib/tempest +``` + +Required variables: + +```text +PHX_SERVER=true +SECRET_KEY_BASE=... +TEMPEST_HOSTNAME= +TEMPEST_PUBLIC_URL=https:// +TEMPEST_DATA_DIR=/var/lib/tempest +TEMPEST_HOSTED_DID_METHOD=plc +TEMPEST_ADMIN_TOKEN_HASH=... +TEMPEST_BLOB_MAX_BYTES=10000000 +TEMPEST_CRAWLERS=https://bsky.network,https://vsky.network +POOL_SIZE=5 +``` + +Recommended for the first deployment: + +```text +TEMPEST_BLOB_STORE=s3 +TEMPEST_BACKUP_STORE=s3 +TEMPEST_SMTP_ENABLED=false +``` + +Add the matching R2 endpoint, bucket, region, access key, and secret variables +for blob and backup storage. + +Do not use a temporary Railway domain for account migration. It can prove the app +boots, but the admin account needs a stable hostname because DID documents, +handles, OAuth metadata, and relay crawlers depend on that URL. + +## Admin Account Migration Plan + +Use the admin account as the first real test bed only after staging passes. + +1. Deploy Tempest to Railway with the final hostname. +2. Confirm `/xrpc/_health` and `com.atproto.server.describeServer` over HTTPS. +3. Confirm WebSocket access to `com.atproto.sync.subscribeRepos`. +4. Export the Bluesky-hosted repository CAR. +5. Request service auth from the old PDS for account creation on Tempest. +6. Create the Tempest account with the existing DID. It should start inactive. +7. Import the CAR. +8. Check `com.atproto.repo.listMissingBlobs`. +9. Upload missing blobs. +10. Check `com.atproto.server.checkAccountStatus`. +11. Update the DID document so `#atproto_pds` points at Tempest. +12. Re-check public DID and handle resolution from outside Railway. +13. Activate the Tempest account. +14. Deactivate the old PDS account only after the new account passes real-client + login, write, blob, sync, and firehose checks. + +This sequence follows the current AT Protocol migration flow: create an inactive +account on the new PDS with service auth, migrate repository and blobs, update +identity, then activate the new account. + +## Release Checklist + +The initial release can be called done when these are true: + +- `mix precommit` passes from a clean worktree. +- `docker build -f conf/Dockerfile -t tempest:release .` passes. +- Railway deploy boots with the mounted volume. +- Railway health check and `test/smoke/deployment.hurl` pass against the final + hostname. +- Admin status can be read with the admin token and rejects normal account + tokens. +- R2 blob write/read works. +- R2 backup upload works. +- Restore drill succeeds against a fresh Railway volume or separate Railway test + service. +- Public deployed checks pass for HTTPS, WebSocket, DID, handle, blob reads, and + relay/AppView crawl. +- Real-client checks pass for login, session refresh, profile/post write, blob + upload/read, and repo reads. + +## Remaining Release Gates + +- Run `test/smoke/deployment.hurl` against the final Railway hostname. +- Run `test/smoke/deployed/crawlers.hurl` against the final Railway hostname. +- Exercise the managed-volume plus R2 restore drill. +- Verify public DID and handle resolution from outside Railway. +- Complete the real-client login, session refresh, write, blob, and repo-read + checklist. + +The account migration reference also notes that full `did:plc` migration still +needs black-box migration-out coverage. + +## Rollback + +Before activation: + +- The old Bluesky-hosted account remains authoritative. +- Tempest can be discarded or redeployed without public identity impact. + +After identity update but before old-account deactivation: + +- Keep both accounts accessible. +- Use `checkAccountStatus` on both sides. +- Keep the old account undeleted while caches settle. + +After activation: + +- Keep the latest Tempest backup archive outside Railway. +- Keep R2 blob and backup credentials recoverable. +- Do not delete the old account during the first validation window. + +## External References Checked + +- Railway Phoenix deployment guide, last updated 2026-06-01: + +- Railway volumes reference, last updated 2026-05-29: + +- Railway pricing page checked 2026-06-13: + +- AT Protocol account migration guide checked 2026-06-13: + diff --git a/docs/specs/README.md b/docs/specs/README.md index 1d09603..52e62f4 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -32,6 +32,7 @@ Subsystem specifications live in this directory. 16. [PDS Compatibility Against Reference Surface](./pds-compatibility.md) 17. [Public Stats Dashboard](./public-stats-dashboard.md) 18. [Documentation Viewer](./doc-viewer.md) +19. [Personal Account Backups](./personal-backups.md) ## Source Baseline @@ -51,6 +52,7 @@ Research was checked on 2026-05-07 against: - AT Protocol OAuth scopes guide: - AT Protocol account migration guide: - Reference PDS implementation: +- Cocoon PDS: ## Documentation Rules diff --git a/docs/specs/personal-backups.md b/docs/specs/personal-backups.md new file mode 100644 index 0000000..709a063 --- /dev/null +++ b/docs/specs/personal-backups.md @@ -0,0 +1,288 @@ +--- +title: Personal Account Backups +updated: 2026-06-13 +status: planned +--- + +Tempest should be able to back up other AT Protocol accounts controlled by the +operator without becoming the active PDS for those accounts. + +This feature is custody and archive work. It must not update identity, submit PLC +operations, activate accounts, write records to source PDS instances, or imply +that Tempest is hosting the backed-up account. + +## Source Baseline + +Research checked on 2026-06-13: + +- AT Protocol Repository spec: +- AT Protocol Sync spec: +- AT Protocol Blob Lifecycle guide: +- AT Protocol Account Migration guide: + +- `com.atproto.sync.listBlobs` Lexicon: + +- `com.atproto.sync.getBlob` Lexicon: + +- `app.bsky.actor.getPreferences` Lexicon: + +- Official Bluesky PDS distribution: + +- Reference PDS implementation: + +- Cocoon PDS: + +- Tranquil PDS local reference: + + +The repository spec defines full repo exports as CAR files suitable for sync, +offline backup, and migration. The sync spec exposes unauthenticated +`com.atproto.sync.getRepo` for public repository export. Blob backup should use +`com.atproto.sync.listBlobs` and `com.atproto.sync.getBlob`; both are PDS +endpoints and do not require auth for public blobs. Private preferences require +auth through `app.bsky.actor.getPreferences`. + +The Tranquil local reference reinforces two design constraints. First, operator +guidance there treats repo CAR export, blob download, preference export, and +separately held rotation keys as distinct backup concerns. Second, Tranquil's +history includes an `account_backups` table with `storage_key`, repo root CID, +rev, block count, size, and created time, followed later by a migration that +dropped the table. Treat that as a warning to keep Tempest's first account +backup format portable and manifest-driven instead of tightly coupling it to +one internal storage engine. + +## Goals + +- Register external accounts by DID and handle. +- Back up public repository state as immutable CAR snapshots. +- Back up public blobs associated with the account. +- Back up private preferences when the operator supplies account credentials. +- Verify each snapshot before marking it complete. +- Export a portable bundle containing CAR, blobs, preferences, manifest, and + verification report. +- Keep this feature separate from account migration and active hosting. + +## Non-goals + +- No PLC updates. +- No identity migration. +- No automatic activation of backed-up accounts on Tempest. +- No writes to the source PDS in the first version. +- No backups of DMs, notifications, AppView-only timelines, label-service state, + moderation decisions stored outside the PDS, or feed-generator state. +- No broad network crawler. The operator must explicitly add each account. +- No whole-PDS disaster recovery in this feature. Tempest's service backup and + restore work remains separate from per-account personal snapshots. + +## Account Registry + +Add a registry for external accounts. Each entry should include: + +```text +id +label +did +handle +source_pds_url +credential_state +last_checked_at +last_success_at +last_snapshot_id +status +status_reason +inserted_at +updated_at +``` + +`did` is the stable account identifier. `handle` is display and discovery +metadata. A backup must verify that the resolved handle still points to the DID, +but a handle change must not orphan existing snapshots. + +`source_pds_url` is the PDS used for backup reads. It can be discovered from the +DID document, but the operator may pin it for an account. If discovery and the +pinned source disagree, the backup should fail closed unless the operator +confirms a source update. + +## Credentials + +Support three credential states: + +```text +none +app_password +access_token +``` + +Public repo and blob backup should work with no credential. Private preference +backup requires auth. + +Credential rules: + +- Store secrets encrypted or through the existing secret-storage approach chosen + for deployment. +- Never display a stored secret after save. +- Allow credential replacement and deletion. +- Record credential kind and last verification time. +- Treat failed auth as a backup warning when public backup succeeds, not as a + failed public backup. + +## Snapshot Model + +Each backup run creates a snapshot. A snapshot is immutable after completion. + +Suggested manifest fields: + +```json +{ + "version": 1, + "account": { "did": "did:plc:...", "handle": "example.com", "sourcePds": "https://bsky.social" }, + "repo": { "carPath": "repo.car", "commit": "bafy...", "rev": "3l...", "byteSize": 12345, "sha256": "..." }, + "blobs": { "count": 10, "complete": true, "missing": [] }, + "preferences": { "included": true, "path": "preferences.json" }, + "verification": { "status": "ok", "checkedAt": "2026-06-13T00:00:00Z" } +} +``` + +Store snapshots under the existing backup storage profile. Local and S3/R2 +storage should share the same logical layout: + +```text +personal-backups/ + / + snapshots/ + -/ + manifest.json + repo.car + blobs/ + + preferences.json + verification.json +``` + +## Backup Flow + +1. Resolve account identity. +2. Determine the source PDS from the DID document or pinned account config. +3. Fetch `com.atproto.sync.getRepo?did=` from the source PDS. +4. Parse and verify the CAR. +5. Extract current commit CID, DID, rev, records, and blob references. +6. Call `com.atproto.sync.listBlobs` with pagination. +7. Fetch each blob with `com.atproto.sync.getBlob`. +8. Verify blob bytes match the expected CID. +9. If credentials are configured, call `app.bsky.actor.getPreferences`. +10. Write the snapshot to a temporary location. +11. Write manifest and verification report. +12. Atomically mark the snapshot complete. + +If blob enumeration fails but repo backup succeeds, keep the snapshot incomplete +and record missing blob state. Do not mark the snapshot complete until every +referenced available blob is either stored or explicitly recorded as missing. + +## Verification + +A completed snapshot must prove: + +- the CAR root points at a commit object; +- the commit DID matches the registered account DID; +- the commit signature verifies against the resolved DID document; +- the repo MST is complete for the exported commit; +- record paths and CIDs pass existing repo-core validation; +- blob CIDs discovered from records and `listBlobs` have matching stored bytes; +- preference JSON was fetched with auth when credentials were enabled; +- the manifest hashes match files on disk or in object storage. + +Verification should be callable without contacting the source PDS, except for an +optional identity freshness check. Offline verification is the point of a backup. + +## Admin UI + +Add an admin-only account backup area: + +```text +/admin/backups/accounts +/admin/backups/accounts/:id +``` + +The UI should show: + +- registered accounts; +- credential state, without secret values; +- latest snapshot status; +- backup now action; +- verification action; +- snapshot list; +- missing blob report; +- export bundle download or object-storage location; +- source PDS and identity mismatch warnings. + +The operator account UI may link to this area, but external account backups are +admin-only in the first version. + +## Scheduling + +Manual backup comes first. Scheduled backup can follow after the manual flow is +stable. + +Scheduling rules: + +- Run one account backup at a time by default. +- Use `Task.async_stream/3` with bounded concurrency only for blob downloads. +- Persist backup run state so an interrupted run can be marked failed or resumed. +- Do not retry auth failures without operator action. +- Use exponential backoff for transient source PDS or object-storage errors. + +## Storage and Retention + +Retention should be explicit per account: + +```text +keep_all +keep_last_n +keep_for_days +``` + +Default to `keep_last_n=3` for scheduled backups. Manual snapshots may be pinned +to prevent deletion. + +Storage reporting should include: + +- repo CAR bytes; +- blob bytes; +- preference bytes; +- manifest and verification bytes; +- total per account; +- total across personal backups. + +The snapshot manifest should stay useful if Tempest later changes database or +object-storage internals. Do not make an internal row ID or storage backend the +only way to understand a backup. + +## Security + +Backups may contain private preference data and deleted public data that still +exists in an older snapshot. Treat bundles as sensitive. + +Required safeguards: + +- admin auth for all personal backup routes and APIs; +- no public snapshot listing; +- no public bundle downloads; +- no credential values in logs or templates; +- redacted error messages for auth headers and tokens; +- rate limits on manual backup triggers; +- clear warning before deleting snapshots. + +## HTTP Verification + +Future smoke test: + +```bash +hurl --test --jobs 1 \ + --variable base_url=http://localhost:4000 \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/personal-backups.hurl +``` + +The smoke test should cover account registration, public repo snapshot creation, +blob backup, credentialed preferences backup against a fixture server, snapshot +verification, export bundle creation, and deletion of an unpinned snapshot. diff --git a/docs/tasks/15-deployment-verification.md b/docs/tasks/15-deployment-verification.md index 704aa77..d3acc58 100644 --- a/docs/tasks/15-deployment-verification.md +++ b/docs/tasks/15-deployment-verification.md @@ -17,20 +17,20 @@ SQLite-first PDS on local Docker or a managed PaaS with optional S3/R2 storage. - [x] T15-03: Add docker-compose example. - [x] T15-04: Add Caddy reverse proxy example. - [x] T15-05: Add production env template. -- [ ] T15-06: Add deployment docs for local-only, S3-backed, and reverse-proxy +- [x] T15-06: Add deployment docs for local-only, S3-backed, and reverse-proxy setups. - [x] T15-07: Add managed PaaS deployment profile for Railway-like hosts. - [x] T15-08: Document persistent volume requirements for SQLite, repos, keys, WAL files, and backup workspaces. - [x] T15-09: Add Cloudflare R2 blob-store configuration docs. - [x] T15-10: Add Cloudflare R2 backup-store configuration docs. -- [ ] T15-11: Add deployed HTTPS/WebSocket smoke test profile. -- [ ] T15-12: Add Hurl smoke test for deployed HTTPS target. -- [ ] T15-13: Add restore drill for managed PaaS volume plus S3/R2 backups. -- [ ] T15-14: Add public DID and handle verification procedure. -- [ ] T15-15: Add public relay/AppView crawl verification procedure for a +- [x] T15-11: Add deployed HTTPS/WebSocket smoke test profile. +- [x] T15-12: Add Hurl smoke test for deployed HTTPS target. +- [x] T15-13: Add restore drill for managed PaaS volume plus S3/R2 backups. +- [x] T15-14: Add public DID and handle verification procedure. +- [x] T15-15: Add public relay/AppView crawl verification procedure for a deployed HTTPS node. -- [ ] T15-16: Add real-client compatibility checklist for deployed login, +- [x] T15-16: Add real-client compatibility checklist for deployed login, profile writes, posts, blobs, and session refresh. - [x] T15-17: Add budget deployment guide for Railway Hobby plus Cloudflare R2 free-tier planning. diff --git a/docs/tasks/18-personal-backups.md b/docs/tasks/18-personal-backups.md new file mode 100644 index 0000000..896f9c1 --- /dev/null +++ b/docs/tasks/18-personal-backups.md @@ -0,0 +1,110 @@ +--- +title: Milestone 18 - Personal Account Backups +specs: + - ../specs/personal-backups.md + - ../specs/admin-operations.md + - ../specs/storage-sqlite.md + - ../specs/security-oauth.md +references: + - ../reference/account-migration.md + - ../reference/blobs.md + - ../reference/repo-core.md + - ../reference/admin-operations.md +--- + +Goal: let the operator back up other AT Protocol accounts they control, without +turning Tempest into the active PDS for those accounts. + +- [ ] T18-01: Add a `Tempest.PersonalBackups` context and migrations for + external backup accounts, backup runs, immutable snapshots, blob records, + and retention settings. +- [ ] T18-02: Add account registration with DID, handle, source PDS URL, label, + and status fields. +- [ ] T18-03: Add identity/source verification that resolves handle and DID + document, verifies `#atproto_pds`, and fails closed on mismatched pinned + source PDS values. +- [ ] T18-04: Add credential storage for no-auth, app-password, and access-token + modes. Store secrets defensively, never render them, and allow rotation and + deletion. +- [ ] T18-05: Add a source PDS client using `Req` for `com.atproto.sync.getRepo`, + `com.atproto.sync.listBlobs`, `com.atproto.sync.getBlob`, and + `app.bsky.actor.getPreferences`. +- [ ] T18-06: Add CAR snapshot creation that stores `repo.car`, commit CID, rev, + byte size, hash, source PDS, handle, and DID. +- [ ] T18-07: Reuse repo-core verification to validate commit DID, commit + signature, MST completeness, record paths, record CIDs, and CAR integrity. +- [ ] T18-08: Extract blob references from repo records and merge them with + paginated `listBlobs` output. +- [ ] T18-09: Add bounded concurrent blob download with CID verification, + missing-blob recording, and retry handling for transient source failures. +- [ ] T18-10: Add credentialed private preference backup through + `app.bsky.actor.getPreferences`, with auth failures reported separately + from public repo/blob backup status. +- [ ] T18-11: Write immutable snapshot manifests and verification reports, first + to a temporary workspace and then atomically mark snapshots complete. +- [ ] T18-12: Store personal backup snapshots through the existing local and + S3/R2 backup storage shape. +- [ ] T18-13: Add retention policies: keep all, keep last N, keep for days, and + pinned snapshots. +- [ ] T18-14: Add portable export bundle creation containing manifest, repo CAR, blobs, + preferences JSON when present, and verification report. +- [ ] T18-15: Add offline snapshot verification that can run without contacting + the source PDS. +- [ ] T18-16: Add tests proving a snapshot can be understood from its manifest + and files without relying on Tempest database rows. +- [ ] T18-17: Add Mix tasks for backup, verify, list snapshots, export bundle, + prune, and show account backup status. +- [ ] T18-18: Add admin-only routes and controllers for external backup account + list, detail, create, edit, delete, backup now, verify, prune, and export. +- [ ] T18-19: Add admin templates under the existing UI language for credential + state, latest backup status, missing blobs, snapshot history, storage + totals, and source identity warnings. +- [ ] T18-20: Add manual backup locking so two runs cannot mutate the same + account snapshot workspace at the same time. +- [ ] T18-21: Add optional scheduled backups after manual backups are stable, + with one-account-at-a-time default scheduling and persisted run state. +- [ ] T18-22: Add unit tests for account registration, credential redaction, + manifest writing, retention pruning, and snapshot state transitions. +- [ ] T18-23: Add fixture-server integration tests for `getRepo`, `listBlobs`, + `getBlob`, preferences auth success, preferences auth failure, missing + blobs, bad CIDs, bad CARs, and source identity mismatch. +- [ ] T18-24: Add admin ConnCase tests for route auth, create/edit forms, + backup-now action, verification action, export action, and deletion + confirmation. +- [ ] T18-25: Add Hurl smoke test `test/smoke/personal-backups.hurl`. +- [ ] T18-26: Add reference documentation after implementation describing the + backup format, restore/export limits, security model, and operational + checks. + +## Integration Tests + +- Public repo backup succeeds without credentials. +- Blob backup stores every available listed or referenced blob. +- CID mismatch marks a snapshot failed. +- Missing blobs are recorded and visible to the operator. +- Private preferences are included only when valid credentials are configured. +- Auth failures do not leak secrets and do not destroy public backup output. +- Offline verification catches corrupted CAR, blob, manifest, and preferences + files. +- Retention pruning never deletes pinned snapshots. +- Admin routes reject unauthenticated, account-token, and normal app-password + requests. + +## HTTP Verification + +```bash +hurl --test --jobs 1 \ + --variable base_url=http://localhost:4000 \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/personal-backups.hurl +``` + +## Implementation Notes + +Keep this feature read-only against source PDS instances in the first version. +The backup client may authenticate to read private preferences, but it must not +write records, submit PLC operations, activate accounts, deactivate accounts, or +call migration endpoints. + +Prefer portable snapshot bundles over direct restore. Direct restore into Tempest +belongs in migration work after backup verification has real use. diff --git a/docs/tasks/README.md b/docs/tasks/README.md index 6bf4cf0..70ff9fe 100644 --- a/docs/tasks/README.md +++ b/docs/tasks/README.md @@ -22,6 +22,7 @@ title: Milestone Tasks 16. [Deployment and Post-deployment Verification](./15-deployment-verification.md) 17. [Public Stats Dashboard](./16-public-stats-dashboard.md) 18. [Doc Viewer](./17-doc-viewer.md) +19. [Personal Account Backups](./18-personal-backups.md) Each file in this directory is a milestone. Each task is intended to be the smallest useful unit of work: one focused implementation change, test, or diff --git a/test/smoke/README.md b/test/smoke/README.md index fe44ef3..b18f02f 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -11,7 +11,7 @@ mix phx.server test/smoke/local-pds-compat.sh http://localhost:4000 ``` -Quick full-directory run: +Quick local run: ```bash mix phx.server @@ -23,8 +23,31 @@ hurl --test --jobs 1 \ --variable account_handle="smoke-${suffix}.test" \ --variable account_email="smoke-${suffix}@example.com" \ --variable account_password="correct horse battery staple" \ - test/smoke/*.hurl + test/smoke/health.hurl \ + test/smoke/xrpc.hurl \ + test/smoke/accounts.hurl \ + test/smoke/identity.hurl \ + test/smoke/records.hurl \ + test/smoke/car-sync.hurl \ + test/smoke/firehose.hurl \ + test/smoke/blobs.hurl \ + test/smoke/lexicon-schemas.hurl \ + test/smoke/migration-lifecycle.hurl \ + test/smoke/oauth-security.hurl \ + test/smoke/operator-account-ux.hurl \ + test/smoke/tempest_basic.hurl \ + test/smoke/tempest_compat.hurl ``` Use fresh account variables for every run. Some files create accounts with the same variables, so run those files separately if the shared handle already exists. + +`test/smoke/deployment.hurl` is deployed-only. Run it against the final HTTPS +hostname with an admin token: + +```bash +hurl --test --jobs 1 \ + --variable base_url=https://tempest.example.com \ + --variable admin_token="$ADMIN_TOKEN" \ + test/smoke/deployment.hurl +``` diff --git a/test/smoke/deployment.hurl b/test/smoke/deployment.hurl new file mode 100644 index 0000000..642a39b --- /dev/null +++ b/test/smoke/deployment.hurl @@ -0,0 +1,87 @@ +# Deployed HTTPS smoke coverage. +# +# Run against a publicly reachable HTTPS deployment with the final hostname: +# +# hurl --test --jobs 1 \ +# --variable base_url=https://tempest.example.com \ +# --variable admin_token="$ADMIN_TOKEN" \ +# test/smoke/deployment.hurl +# +# This file is intentionally non-destructive. Account writes, blob writes, relay +# crawl, and real-client checks are covered by the broader deployment runbook. + +GET {{base_url}}/xrpc/_health +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.status" == "ok" +jsonpath "$.version" exists +jsonpath "$.storage.dataDir" exists +jsonpath "$.storage.writable" == true + +GET {{base_url}}/xrpc/com.atproto.server.describeServer +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.availableUserDomains" exists +jsonpath "$.inviteCodeRequired" == false +jsonpath "$.links.privacyPolicy" exists +jsonpath "$.links.termsOfService" exists + +GET {{base_url}}/.well-known/oauth-protected-resource +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.resource" exists +jsonpath "$.authorization_servers[0]" exists + +GET {{base_url}}/.well-known/oauth-authorization-server +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.issuer" exists +jsonpath "$.pushed_authorization_request_endpoint" exists +jsonpath "$.token_endpoint" exists + +GET {{base_url}}/oauth/jwks +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.keys" isCollection + +GET {{base_url}}/xrpc/_admin/status +HTTP 401 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "AuthenticationRequired" + +GET {{base_url}}/xrpc/_admin/status +Authorization: Bearer not-the-admin-token +HTTP 401 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "InvalidToken" + +GET {{base_url}}/xrpc/_admin/status +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.status" == "ok" +jsonpath "$.database.accountDb.exists" == true +jsonpath "$.database.sequencerDb.exists" == true +jsonpath "$.sequencer.currentSeq" exists +jsonpath "$.blobStore" exists + +GET {{base_url}}/admin/storage +Authorization: Bearer {{admin_token}} +HTTP 200 +[Asserts] +header "content-type" contains "text/html" +body contains "Storage Status" + +GET {{base_url}}/xrpc/com.atproto.unknown.method +HTTP 404 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "UnknownMethod" -- 2.51.2