diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..09c7f7b --- /dev/null +++ b/.env.example @@ -0,0 +1,59 @@ +# (optional) debug, info, warn, or error +# default: info +# PDS_LOG_LEVEL=info + +# (optional) directory where databases and blobs will be stored +# default: ./data [relative to cwd] +# PDS_DATA_DIR=./data + +# (required) hostname where this PDS is running +# for handles to work, you should set A records for *.{PDS_HOSTNAME} and PDS_HOSTNAME to point to this server +PDS_HOSTNAME= + +# (optional) the PDS's DID +# default: did:web:{PDS_HOSTNAME} +# PDS_DID= + +# (optional) whether an invite code is needed to create an account +# default: true +# PDS_INVITE_CODE_REQUIRED=true + +# (required) the rotation key the PDS will use, in multibase format +# tip: run the `gen-keys` binary to generate this (more info in readme) +PDS_ROTATION_KEY= + +# (required) the key used to sign JWKs +# tip: run the `gen-keys` binary to generate this (more info in readme) +PDS_JWK_MULTIBASE= + +# (required) the PDS admin password +# this can be used to access com.atproto.admin.* endpoints as well as the /admin dashboard +PDS_ADMIN_PASSWORD= + +# (optional) relays/crawlers to inform to subscribe to this PDS, comma-separated +# default: https://bsky.network +# PDS_CRAWLERS=https://bsky.network + +# (optional [but recommended]): a secret token used to generate DPoP nonces; 32 bytes, base64url +# tip: run the `gen-keys` binary to generate this (more info in readme) +# PDS_DPOP_NONCE_SECRET= + +# (optional) email config (see readme for details) +# auth uri should look like smtp[s]://user:pass@host[:port] +# sender should look like e@mail.com or Name +# without setting these, emails will be logged to stdout +# PDS_SMTP_STARTTLS=false +# PDS_SMTP_AUTH_URI= +# PDS_SMTP_SENDER= + +# (optional) S3 config (see readme for details) +# set these to back up your databases and/or blobs to S3(-compatible storge) +# PDS_S3_BLOBS_ENABLED=false +# PDS_S3_BACKUPS_ENABLED=false +# PDS_S3_BACKUP_INTERVAL_S=3600 +# PDS_S3_ENDPOINT= +# PDS_S3_REGION= +# PDS_S3_BUCKET= +# PDS_S3_ACCESS_KEY= +# PDS_S3_SECRET_KEY= +# PDS_S3_CDN_URL= diff --git a/.gitignore b/.gitignore index ef4152f..914e3a4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,3 @@ node_modules .zed/ *.env -.env* diff --git a/README.md b/README.md index 478af0b..e09e388 100644 --- a/README.md +++ b/README.md @@ -18,19 +18,22 @@ docker compose run --rm --entrypoint gen-keys pds to generate some of the environment variables you'll need. -Head to [`docker-compose.yaml`](docker-compose.yaml) and fill in the environment variables marked as required. See [Environment](#environment) for further details on configuration. +Copy [`.env.example`](.env.example) to `.env` and fill in the environment variables marked as required. See [Environment](#environment) for further details on configuration. After that, run + ``` docker compose up -d ``` + to start the PDS, then navigate to `https://{PDS_HOSTNAME}/admin` to log in with the admin password you specified and create an invite code or a new account on the PDS. ## environment -Documentation for most environment variables can be found in [`docker-compose.yaml`](docker-compose.yaml). There are two optional categories of environment variables that add functionality: +Documentation for most environment variables can be found in [`.env.example`](.env.example). There are two optional categories of environment variables that add functionality: ### SMTP + The PDS can email users for password changes, identity updates, and account deletion. If these environment variables are not set, emails will instead be logged to the process' stdout. - `PDS_SMTP_AUTH_URI` — The URI to connect to the mail server. This should look like `smtp[s]://user:pass@host[:port]`. @@ -38,10 +41,11 @@ The PDS can email users for password changes, identity updates, and account dele - `PDS_SMTP_SENDER` — The identity to send emails as. Can be an email address (`e@mail.com`) or a mailbox (`Name `). ### S3 + The PDS can be configured to back up server data to and/or store blobs in S3(-compatible storage). - `PDS_S3_BLOBS_ENABLED=false` — Whether to store blobs in S3. By default, blobs are stored locally in `{PDS_DATA_DIR}/blobs/[did]/`. -- `PDS_S3_BACKUPS_ENABLED=false` — Whether to back up data to S3. +- `PDS_S3_BACKUPS_ENABLED=false` — Whether to back up data to S3. - `PDS_S3_BACKUP_INTERVAL_S=3600` — How often to back up to S3, in seconds. - `PDS_S3_ENDPOINT`, `PDS_S3_REGION`, `PDS_S3_BUCKET`, `PDS_S3_ACCESS_KEY`, `PDS_S3_SECRET_KEY` — S3 configuration. - `PDS_S3_CDN_URL` — You may optionally set this to redirect `getBlob` requests to `{PDS_S3_CDN_URL}/blobs/{did}/{cid}`. When unset, blobs will be fetched either from local storage or from S3, depending on `PDS_S3_BLOBS_ENABLED`. @@ -50,45 +54,55 @@ The PDS can be configured to back up server data to and/or store blobs in S3(-co This repo contains several libraries in addition to the `pegasus` PDS: -| library | description | -|----------|-------------| -| frontend | The PDS frontend, containing the admin dashboard and account page. | +| library | description | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| frontend | The PDS frontend, containing the admin dashboard and account page. | | ipld | A mostly [DASL-compliant](https://dasl.ing/) implementation of [CIDs](https://dasl.ing/cid.html), [CAR](https://dasl.ing/car.html), and [DAG-CBOR](https://dasl.ing/drisl.html). | -| kleidos | An atproto-valid interface for secp256k1 and secp256r1 key management, signing/verifying, and encoding/decoding. | -| mist | A [Merkle Search Tree](https://atproto.com/specs/repository#mst-structure) implementation for data repository purposes. | -| pegasus | The PDS implementation. | +| kleidos | An atproto-valid interface for secp256k1 and secp256r1 key management, signing/verifying, and encoding/decoding. | +| mist | A [Merkle Search Tree](https://atproto.com/specs/repository#mst-structure) implementation for data repository purposes. | +| pegasus | The PDS implementation. | To start developing, you'll need: + - [`opam`](https://opam.ocaml.org/doc/Install.html), the OCaml Package Manager - and the following packages, or their equivalents on your operating system: `cmake git libev-dev libffi-dev libgmp-dev libssl-dev libsqlite3-dev libpcre3-dev pkg-config` Start by creating an opam switch; similar to a Python virtual environment, storing the dependencies for this project and a specific compiler version. After that, install [`dune`](https://dune.build), the build system/package manager pegasus uses. + ``` opam switch create . 5.2.1 opam install dune ``` You may need to run `eval $(opam env)` for this to work. Next, run + ``` dune pkg lock ``` + to solve dependencies. -Finally, after setting the required environment variables (see [Environment](#environment)), either run +Set the required environment variables (see [Environment](#environment)), noting that the program won't automatically read from `.env`, then either run + ``` dune exec pegasus ``` + to run the program directly, or + ``` dune build ``` + to produce an executable that you'll likely find in `_build/default/bin`. For development, you'll also want to run + ``` dune tools exec ocamlformat dune tools exec ocamllsp ``` + to download the formatter and LSP services. You can run `dune fmt` to format the project. -The [frontend](frontend) is written in [MLX](https://github.com/ocaml-mlx/mlx), a JSX-ish OCaml dialect. To format it, you'll need to `opam install ocamlformat-mlx`, then `ocamlformat-mlx -i frontend/**/*.mlx`. You'll see a few errors on formatting files containing `[%browser_only]`; I'm waiting on the next release of `mlx` to fix those. \ No newline at end of file +The [frontend](frontend) is written in [MLX](https://github.com/ocaml-mlx/mlx), a JSX-ish OCaml dialect. To format it, you'll need to `opam install ocamlformat-mlx`, then `ocamlformat-mlx -i frontend/**/*.mlx`. You'll see a few errors on formatting files containing `[%browser_only]`; I'm waiting on the next release of `mlx` to fix those. diff --git a/docker-compose.yaml b/docker-compose.yaml index cb3a770..8d042cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,66 +7,30 @@ services: volumes: - pds:/data environment: - # (optional) debug, info, warn, or error - # default: info - # - PDS_LOG_LEVEL=info - - # (optional) directory where databases and blobs will be stored - # default: ./data [relative to cwd] - - PDS_DATA_DIR=./data - - # (required) hostname where this PDS is running - # for handles to work, you should configure *.hostname and hostname to point to this server - # make sure to also set this env var for the caddy container below - - PDS_HOSTNAME= - - # (optional) the PDS's DID - # default: did:web:{hostname} - # - PDS_DID= - - # (optional) whether an invite code is needed to create an account - # default: true - # - PDS_INVITE_CODE_REQUIRED=true - - # (required) the rotation key the PDS will use, in multibase format - # tip: run the `gen-keys` binary to generate this - - PDS_ROTATION_KEY= - - # (required) the key used to sign JWKs - # tip: run the `gen-keys` binary to generate this - - PDS_JWK_MULTIBASE= - - # (required) the PDS admin password - # this can be used to access com.atproto.admin.* endpoints as well as the /admin dashboard - - PDS_ADMIN_PASSWORD= - - # (optional) relays/crawlers to inform to subscribe to this PDS, comma-separated - # default: https://bsky.network - # - PDS_CRAWLERS=https://bsky.network - - # (optional [but recommended]): a secret token used to generate DPoP nonces; 32 bytes, base64url - # tip: run the `gen-keys` binary to generate this - # - PDS_DPOP_NONCE_SECRET= - - # (optional) email config (see readme for details) - # auth uri should look like smtp[s]://user:pass@host[:port] - # sender should look like e@mail.com or Name - # without setting these, emails will be logged to stdout - # - PDS_SMTP_STARTTLS=false - # - PDS_SMTP_AUTH_URI= - # - PDS_SMTP_SENDER= - - # (optional) S3 config (see readme for details) - # set these to back up your databases and/or blobs to S3(-compatible storge) - # - PDS_S3_BLOBS_ENABLED=false - # - PDS_S3_BACKUPS_ENABLED=false - # - PDS_S3_BACKUP_INTERVAL_S=3600 - # - PDS_S3_ENDPOINT= - # - PDS_S3_REGION= - # - PDS_S3_BUCKET= - # - PDS_S3_ACCESS_KEY= - # - PDS_S3_SECRET_KEY= - # - PDS_S3_CDN_URL= + - PDS_LOG_LEVEL=${PDS_LOG_LEVEL:-info} + - PDS_DATA_DIR=${PDS_DATA_DIR:-./data} + - PDS_HOSTNAME=${PDS_HOSTNAME:?} + - PDS_DID=${PDS_DID} + - PDS_INVITE_CODE_REQUIRED=${PDS_INVITE_CODE_REQUIRED:-true} + - PDS_ROTATION_KEY=${PDS_ROTATION_KEY:?} + - PDS_JWK_MULTIBASE=${PDS_JWK_MULTIBASE:?} + - PDS_ADMIN_PASSWORD=${PDS_ADMIN_PASSWORD:?} + - PDS_CRAWLERS=${PDS_CRAWLERS:-https://bsky.network} + - PDS_DPOP_NONCE_SECRET=${PDS_DPOP_NONCE_SECRET} + + - PDS_SMTP_STARTTLS=${PDS_SMTP_STARTTLS:-false} + - PDS_SMTP_AUTH_URI=${PDS_SMTP_AUTH_URI} + - PDS_SMTP_SENDER=${PDS_SMTP_SENDER} + + - PDS_S3_BLOBS_ENABLED=${PDS_S3_BLOBS_ENABLED:-false} + - PDS_S3_BACKUPS_ENABLED=${PDS_S3_BACKUPS_ENABLED:-false} + - PDS_S3_BACKUP_INTERVAL_S=${PDS_S3_BACKUP_INTERVAL_S:-3600} + - PDS_S3_ENDPOINT=${PDS_S3_ENDPOINT} + - PDS_S3_REGION=${PDS_S3_REGION} + - PDS_S3_BUCKET=${PDS_S3_BUCKET} + - PDS_S3_ACCESS_KEY=${PDS_S3_ACCESS_KEY} + - PDS_S3_SECRET_KEY=${PDS_S3_SECRET_KEY} + - PDS_S3_CDN_URL=${PDS_S3_CDN_URL} restart: unless-stopped caddy: @@ -79,7 +43,7 @@ services: - caddy-data:/data - caddy-config:/config environment: - - PDS_HOSTNAME= + - PDS_HOSTNAME=${PDS_HOSTNAME:?} restart: unless-stopped volumes: diff --git a/pegasus/lib/env.ml b/pegasus/lib/env.ml index 5e23a3f..023b1aa 100644 --- a/pegasus/lib/env.ml +++ b/pegasus/lib/env.ml @@ -2,27 +2,31 @@ let getenv name = try Sys.getenv name with Not_found -> failwith ("Missing environment variable " ^ name) +let getenv_opt name ~default = + match Sys.getenv_opt name with + | Some value when value <> "" -> + value + | _ -> + default + let log_level = - match Sys.getenv_opt "PDS_LOG_LEVEL" |> Option.map String.lowercase_ascii with - | Some "debug" -> + match getenv_opt "PDS_LOG_LEVEL" ~default:"info" with + | "debug" -> `Debug - | Some "info" -> - `Info - | Some "warn" | Some "warning" -> + | "warn" | "warning" -> `Warning - | Some "error" -> + | "error" -> `Error | _ -> `Info -let data_dir = Option.value ~default:"./data" @@ Sys.getenv_opt "PDS_DATA_DIR" +let data_dir = getenv_opt "PDS_DATA_DIR" ~default:"./data" let hostname = getenv "PDS_HOSTNAME" let host_endpoint = "https://" ^ hostname -let did = - Option.value ~default:("did:web:" ^ hostname) @@ Sys.getenv_opt "PDS_DID" +let did = getenv_opt "PDS_DID" ~default:("did:web:" ^ hostname) let invite_required = getenv "PDS_INVITE_CODE_REQUIRED" = "true" @@ -36,8 +40,7 @@ let jwt_pubkey = Kleidos.derive_pubkey jwt_key let admin_password = getenv "PDS_ADMIN_PASSWORD" let crawlers = - Sys.getenv_opt "PDS_CRAWLERS" - |> Option.value ~default:"https://bsky.network" + getenv_opt "PDS_CRAWLERS" ~default:"https://bsky.network" |> String.split_on_char ',' |> List.map (fun u -> match Uri.of_string @@ String.trim u with @@ -47,65 +50,65 @@ let crawlers = Uri.make ~scheme:"https" ~host:u () ) let dpop_nonce_secret = - match Sys.getenv_opt "PDS_DPOP_NONCE_SECRET" with - | Some sec -> - let secret = - Base64.(decode_exn ~alphabet:uri_safe_alphabet ~pad:false) sec - |> Bytes.of_string - in - if Bytes.length secret = 32 then secret - else failwith "PDS_DPOP_NONCE_SECRET must be 32 bytes in base64uri" - | None -> + match getenv_opt "PDS_DPOP_NONCE_SECRET" ~default:"" with + | "" -> let secret = Mirage_crypto_rng_unix.getrandom 32 in Dream.warning (fun log -> log "PDS_DPOP_NONCE_SECRET not set; using PDS_DPOP_NONCE_SECRET=%s" ( Base64.(encode ~alphabet:uri_safe_alphabet ~pad:false) secret |> Result.get_ok ) ) ; Bytes.of_string secret + | sec -> + let secret = + Base64.(decode_exn ~alphabet:uri_safe_alphabet ~pad:false) sec + |> Bytes.of_string + in + if Bytes.length secret = 32 then secret + else failwith "PDS_DPOP_NONCE_SECRET must be 32 bytes in base64uri" let smtp_config, smtp_sender = begin let with_starttls = - Option.value ~default:"false" @@ Sys.getenv_opt "PDS_SMTP_STARTTLS" - = "true" + getenv_opt "PDS_SMTP_STARTTLS" ~default:"false" = "true" in match - ( Option.map Uri.of_string (Sys.getenv_opt "PDS_SMTP_AUTH_URI") - , Sys.getenv_opt "PDS_SMTP_SENDER" ) + ( getenv_opt "PDS_SMTP_AUTH_URI" ~default:"" + , getenv_opt "PDS_SMTP_SENDER" ~default:"" ) with - | Some uri, Some sender -> ( - match - ( Uri.scheme uri - , Uri.user uri - , Uri.password uri - , Uri.host uri - , Uri.port uri ) - with - | Some scheme, Some username, Some password, Some hostname, port - when scheme = "smtp" || scheme = "smtps" -> ( - match Emile.of_string sender with - | Ok _ -> - ( Some - Letters.Config.( - create ~username ~password ~hostname ~with_starttls () - |> set_port port ) - , Some sender ) - | Error _ -> - failwith - "PDS_SMTP_SENDER should be a valid mailbox, e.g. `e@mail.com` or \ - `Name `" ) - | _ -> - failwith - "PDS_SMTP_AUTH_URI must be a valid smtp:// or smtps:// URI with \ - username, password, and hostname" ) - | Some _, None -> + | "", "" -> + (None, None) + | _uri, "" -> failwith "PDS_SMTP_SENDER must be set alongside PDS_SMTP_AUTH_URI; it should \ look like `e@mail.com` or `Name `" - | None, Some _ -> + | "", _uri -> failwith "PDS_SMTP_AUTH_URI must be set alongside PDS_SMTP_SENDER" - | None, None -> - (None, None) + | uri, sender -> ( + let uri = Uri.of_string uri in + match + ( Uri.scheme uri + , Uri.user uri + , Uri.password uri + , Uri.host uri + , Uri.port uri ) + with + | Some scheme, Some username, Some password, Some hostname, port + when scheme = "smtp" || scheme = "smtps" -> ( + match Emile.of_string sender with + | Ok _ -> + ( Some + Letters.Config.( + create ~username ~password ~hostname ~with_starttls () + |> set_port port ) + , Some sender ) + | Error _ -> + failwith + "PDS_SMTP_SENDER should be a valid mailbox, e.g. `e@mail.com` \ + or `Name `" ) + | _ -> + failwith + "PDS_SMTP_AUTH_URI must be a valid smtp:// or smtps:// URI with \ + username, password, and hostname" ) end type s3_config = @@ -125,26 +128,27 @@ let s3_config = begin let default_backup_interval = 3600.0 in let blobs_enabled = - Sys.getenv_opt "PDS_S3_BLOBS_ENABLED" |> Option.map (( = ) "true") + getenv_opt "PDS_S3_BLOBS_ENABLED" ~default:"false" = "true" in let backups_enabled = - Sys.getenv_opt "PDS_S3_BACKUPS_ENABLED" |> Option.map (( = ) "true") + getenv_opt "PDS_S3_BACKUPS_ENABLED" ~default:"false" = "true" in let backup_interval_s = - Sys.getenv_opt "PDS_S3_BACKUP_INTERVAL_S" - |> Option.map float_of_string_opt - |> Option.join + getenv_opt "PDS_S3_BACKUP_INTERVAL_S" + ~default:(string_of_float default_backup_interval) + |> float_of_string in let endpoint = Sys.getenv_opt "PDS_S3_ENDPOINT" in match (blobs_enabled, backups_enabled) with - | Some true, _ | _, Some true -> ( + | true, _ | _, true -> ( match ( Sys.getenv_opt "PDS_S3_REGION" , Sys.getenv_opt "PDS_S3_BUCKET" , Sys.getenv_opt "PDS_S3_ACCESS_KEY" - , Sys.getenv_opt "PDS_S3_SECRET_KEY" ) + , Sys.getenv_opt "PDS_S3_SECRET_KEY" + , Sys.getenv_opt "PDS_S3_CDN_URL" ) with - | Some region, Some bucket, Some access_key, Some secret_key -> + | Some region, Some bucket, Some access_key, Some secret_key, cdn_url -> let region_obj = match endpoint with | Some host -> @@ -155,19 +159,20 @@ let s3_config = let endpoint_obj = Aws_s3.Region.endpoint ~inet:`V4 ~scheme:`Https region_obj in + let credentials_obj = + Aws_s3.Credentials.make ~access_key ~secret_key () + in Some - { blobs_enabled= Option.value ~default:false blobs_enabled - ; backups_enabled= Option.value ~default:false backups_enabled - ; backup_interval_s= - Option.value ~default:default_backup_interval backup_interval_s + { blobs_enabled + ; backups_enabled + ; backup_interval_s ; endpoint ; region ; bucket ; access_key ; secret_key - ; cdn_url= Sys.getenv_opt "PDS_S3_CDN_URL" - ; credentials_obj= - Aws_s3.Credentials.make ~access_key ~secret_key () + ; cdn_url + ; credentials_obj ; endpoint_obj } | _ -> failwith