diff --git a/Cargo.lock b/Cargo.lock index 2717680..5f959f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -992,9 +992,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.48" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" dependencies = [ "clap_builder", "clap_derive", @@ -1002,9 +1002,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.48" +version = "4.5.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" dependencies = [ "anstream", "anstyle", @@ -1014,9 +1014,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.47" +version = "4.5.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" dependencies = [ "heck", "proc-macro2", @@ -1375,7 +1375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4fdb82bd54a12e42fb58a800dcae6b9e13982238ce2296dc3570b92148e1f" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.106", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a119dd2..fa58c0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,3 +13,6 @@ members = [ "pocket", "reflector", ] + +[workspace.dependencies] +clap = { version = "4.5.56", features = ["derive", "env"] } diff --git a/constellation/Cargo.toml b/constellation/Cargo.toml index ab8b27a..18cd525 100644 --- a/constellation/Cargo.toml +++ b/constellation/Cargo.toml @@ -11,7 +11,7 @@ axum = "0.8.1" axum-extra = { version = "0.10.0", features = ["query", "typed-header"] } axum-metrics = "0.2" bincode = "1.3.3" -clap = { version = "4.5.26", features = ["derive"] } +clap = { workspace = true } ctrlc = "3.4.5" flume = { version = "0.11.1", default-features = false } fs4 = { version = "0.12.0", features = ["sync"] } diff --git a/constellation/src/bin/main.rs b/constellation/src/bin/main.rs index 52cfc0f..d557fd6 100644 --- a/constellation/src/bin/main.rs +++ b/constellation/src/bin/main.rs @@ -26,12 +26,11 @@ struct Args { #[arg(long)] #[clap(default_value = "0.0.0.0:6789")] bind: SocketAddr, - /// optionally disable the metrics server - #[arg(long)] - #[clap(default_value_t = false)] + /// enable metrics collection and serving + #[arg(long, action)] collect_metrics: bool, /// metrics server's listen address - #[arg(long)] + #[arg(long, requires("collect_metrics"))] #[clap(default_value = "0.0.0.0:8765")] bind_metrics: SocketAddr, /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: @@ -222,58 +221,58 @@ fn run( // only spawn monitoring thread if the metrics server is running if collect_metrics { s.spawn(move || { // monitor thread - let stay_alive = stay_alive.clone(); - let check_alive = stay_alive.clone(); - - let process_collector = metrics_process::Collector::default(); - process_collector.describe(); - metrics::describe_gauge!( - "storage_available", - metrics::Unit::Bytes, - "available to be allocated" - ); - metrics::describe_gauge!( - "storage_free", - metrics::Unit::Bytes, - "unused bytes in filesystem" - ); - if let Some(ref p) = data_dir { - if let Err(e) = fs4::available_space(p) { - eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); - } else { - println!("disk space monitoring should work, watching at {p:?}"); - } - } + let stay_alive = stay_alive.clone(); + let check_alive = stay_alive.clone(); - 'monitor: loop { - match readable.get_stats() { - Ok(StorageStats { dids, targetables, linking_records, .. }) => { - metrics::gauge!("storage.stats.dids").set(dids as f64); - metrics::gauge!("storage.stats.targetables").set(targetables as f64); - metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); + let process_collector = metrics_process::Collector::default(); + process_collector.describe(); + metrics::describe_gauge!( + "storage_available", + metrics::Unit::Bytes, + "available to be allocated" + ); + metrics::describe_gauge!( + "storage_free", + metrics::Unit::Bytes, + "unused bytes in filesystem" + ); + if let Some(ref p) = data_dir { + if let Err(e) = fs4::available_space(p) { + eprintln!("fs4 failed to get available space. may not be supported here? space metrics may be absent. e: {e:?}"); + } else { + println!("disk space monitoring should work, watching at {p:?}"); } - Err(e) => eprintln!("failed to get stats: {e:?}"), } - process_collector.collect(); - if let Some(ref p) = data_dir { - if let Ok(avail) = fs4::available_space(p) { - metrics::gauge!("storage.available").set(avail as f64); + 'monitor: loop { + match readable.get_stats() { + Ok(StorageStats { dids, targetables, linking_records, .. }) => { + metrics::gauge!("storage.stats.dids").set(dids as f64); + metrics::gauge!("storage.stats.targetables").set(targetables as f64); + metrics::gauge!("storage.stats.linking_records").set(linking_records as f64); + } + Err(e) => eprintln!("failed to get stats: {e:?}"), } - if let Ok(free) = fs4::free_space(p) { - metrics::gauge!("storage.free").set(free as f64); + + process_collector.collect(); + if let Some(ref p) = data_dir { + if let Ok(avail) = fs4::available_space(p) { + metrics::gauge!("storage.available").set(avail as f64); + } + if let Ok(free) = fs4::free_space(p) { + metrics::gauge!("storage.free").set(free as f64); + } } - } - let wait = time::Instant::now(); - while wait.elapsed() < MONITOR_INTERVAL { - thread::sleep(time::Duration::from_millis(100)); - if check_alive.is_cancelled() { - break 'monitor + let wait = time::Instant::now(); + while wait.elapsed() < MONITOR_INTERVAL { + thread::sleep(time::Duration::from_millis(100)); + if check_alive.is_cancelled() { + break 'monitor + } } } - } - stay_alive.drop_guard(); - }); + stay_alive.drop_guard(); + }); } }); diff --git a/pocket/Cargo.toml b/pocket/Cargo.toml index 3b785ff..2b6561b 100644 --- a/pocket/Cargo.toml +++ b/pocket/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] atrium-crypto = "0.1.2" -clap = { version = "4.5.41", features = ["derive"] } +clap = { workspace = true } jwt-compact = { git = "https://github.com/fatfingers23/jwt-compact.git", features = ["es256k"] } log = "0.4.27" poem = { version = "3.1.12", features = ["acme", "static-files"] } diff --git a/quasar/Cargo.toml b/quasar/Cargo.toml index 0206b37..57cca0b 100644 --- a/quasar/Cargo.toml +++ b/quasar/Cargo.toml @@ -4,5 +4,5 @@ version = "0.1.0" edition = "2024" [dependencies] -clap = { version = "4.5.46", features = ["derive"] } +clap = { workspace = true } fjall = "2.11.2" diff --git a/reflector/Cargo.toml b/reflector/Cargo.toml index 9911736..e18e65e 100644 --- a/reflector/Cargo.toml +++ b/reflector/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] -clap = { version = "4.5.47", features = ["derive"] } +clap = { workspace = true } log = "0.4.28" poem = "3.1.12" serde = { version = "1.0.219", features = ["derive"] } diff --git a/slingshot/Cargo.toml b/slingshot/Cargo.toml index 9991b65..eed7de8 100644 --- a/slingshot/Cargo.toml +++ b/slingshot/Cargo.toml @@ -8,7 +8,7 @@ atrium-api = { git = "https://github.com/uniphil/atrium.git", branch = "fix/reso atrium-common = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } atrium-identity = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } atrium-oauth = { git = "https://github.com/uniphil/atrium.git", branch = "fix/resolve-handle-https-accept-whitespace" } -clap = { version = "4.5.41", features = ["derive"] } +clap = { workspace = true } ctrlc = "3.4.7" foyer = { version = "0.18.0", features = ["serde"] } hickory-resolver = "0.25.2" diff --git a/slingshot/src/main.rs b/slingshot/src/main.rs index b2cc768..6ef8058 100644 --- a/slingshot/src/main.rs +++ b/slingshot/src/main.rs @@ -15,45 +15,61 @@ use tokio_util::sync::CancellationToken; struct Args { /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] + #[arg(long, env = "SLINGSHOT_JETSTREAM")] jetstream: String, /// don't request zstd-compressed jetstream events /// /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] + #[arg(long, action, env = "SLINGSHOT_JETSTREAM_NO_ZSTD")] jetstream_no_zstd: bool, /// where to keep disk caches - #[arg(long)] + #[arg(long, env = "SLINGSHOT_CACHE_DIR")] cache_dir: PathBuf, + /// where to listen for incomming requests + /// + /// cannot be used with acme -- if you need ipv6 see --acme-ipv6 + #[arg(long, env = "SLINGSHOT_BIND")] + #[clap(default_value = "0.0.0.0:8080")] + bind: std::net::SocketAddr, /// the domain pointing to this server /// /// if present: /// - a did:web document will be served at /.well-known/did.json /// - an HTTPS certs will be automatically configured with Acme/letsencrypt /// - TODO: a rate-limiter will be installed - #[arg(long)] - domain: Option, + #[arg( + long, + conflicts_with("bind"), + requires("acme_cache_path"), + env = "SLINGSHOT_ACME_DOMAIN" + )] + acme_domain: Option, /// email address for letsencrypt contact /// /// recommended in production, i guess? - #[arg(long)] + #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CONTACT")] acme_contact: Option, /// a location to cache acme https certs /// - /// only used if --host is specified. omitting requires re-requesting certs - /// on every restart, and letsencrypt has rate limits that are easy to hit. + /// required when (and only used when) --acme-domain is specified. /// /// recommended in production, but mind the file permissions. - #[arg(long)] - certs: Option, + #[arg(long, requires("acme_domain"), env = "SLINGSHOT_ACME_CACHE_PATH")] + acme_cache_path: Option, + /// listen for ipv6 when using acme + /// + /// you must also configure the relevant DNS records for this to work + #[arg(long, action, requires("acme_domain"), env = "SLINGSHOT_ACME_IPV6")] + acme_ipv6: bool, /// an web address to send healtcheck pings to every ~51s or so - #[arg(long)] + #[arg(long, env = "SLINGSHOT_HEALTHCHECK")] healthcheck: Option, - /// slingshot server's listen address - #[arg(long, default_value = "0.0.0.0:8080")] - bind: std::net::SocketAddr, + /// enable metrics collection and serving + #[arg(long, action, env = "SLINGSHOT_COLLECT_METRICS")] + collect_metrics: bool, /// metrics server's listen address - #[arg(long, default_value = "0.0.0.0:8765")] + #[arg(long, requires("collect_metrics"), env = "SLINGSHOT_BIND_METRICS")] + #[clap(default_value = "[::]:8765")] bind_metrics: std::net::SocketAddr, } @@ -68,10 +84,13 @@ async fn main() -> Result<(), String> { let args = Args::parse(); - if let Err(e) = install_metrics_server(args.bind_metrics) { - log::error!("failed to install metrics server: {e:?}"); - } else { - log::info!("metrics listening at http://{}", args.bind_metrics); + if args.collect_metrics { + log::trace!("installing metrics server..."); + if let Err(e) = install_metrics_server(args.bind_metrics) { + log::error!("failed to install metrics server: {e:?}"); + } else { + log::info!("metrics listening at http://{}", args.bind_metrics); + } } std::fs::create_dir_all(&args.cache_dir).map_err(|e| { @@ -116,9 +135,10 @@ async fn main() -> Result<(), String> { server_cache_handle, identity, repo, - args.domain, + args.acme_domain, args.acme_contact, - args.certs, + args.acme_cache_path, + args.acme_ipv6, server_shutdown, bind, ) @@ -180,7 +200,9 @@ async fn main() -> Result<(), String> { Ok(()) } -fn install_metrics_server(bind_metrics: std::net::SocketAddr) -> Result<(), metrics_exporter_prometheus::BuildError> { +fn install_metrics_server( + bind_metrics: std::net::SocketAddr, +) -> Result<(), metrics_exporter_prometheus::BuildError> { log::info!("installing metrics server..."); PrometheusBuilder::new() .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? diff --git a/slingshot/src/server.rs b/slingshot/src/server.rs index 30fce0a..ad9dd72 100644 --- a/slingshot/src/server.rs +++ b/slingshot/src/server.rs @@ -687,13 +687,15 @@ fn get_did_doc(domain: &str) -> impl Endpoint + use<> { make_sync(move |_| doc.clone()) } +#[allow(clippy::too_many_arguments)] pub async fn serve( cache: HybridCache, identity: Identity, repo: Repo, - domain: Option, + acme_domain: Option, acme_contact: Option, - certs: Option, + acme_cache_path: Option, + acme_ipv6: bool, shutdown: CancellationToken, bind: std::net::SocketAddr, ) -> Result<(), ServerError> { @@ -707,7 +709,7 @@ pub async fn serve( "Slingshot", env!("CARGO_PKG_VERSION"), ) - .server(if let Some(ref h) = domain { + .server(if let Some(ref h) = acme_domain { format!("https://{h}") } else { "http://localhost:3000".to_string() @@ -728,7 +730,7 @@ pub async fn serve( .nest("/openapi", api_service.spec_endpoint()) .nest("/xrpc/", api_service); - if let Some(domain) = domain { + if let Some(domain) = acme_domain { rustls::crypto::aws_lc_rs::default_provider() .install_default() .expect("alskfjalksdjf"); @@ -741,13 +743,13 @@ pub async fn serve( if let Some(contact) = acme_contact { auto_cert = auto_cert.contact(contact); } - if let Some(certs) = certs { - auto_cert = auto_cert.cache_path(certs); + if let Some(cache_path) = acme_cache_path { + auto_cert = auto_cert.cache_path(cache_path); } let auto_cert = auto_cert.build().map_err(ServerError::AcmeBuildError)?; run( - TcpListener::bind("0.0.0.0:443").acme(auto_cert), + TcpListener::bind(if acme_ipv6 { "[::]:443" } else { "0.0.0.0:443" }).acme(auto_cert), app, shutdown, ) diff --git a/spacedust/Cargo.toml b/spacedust/Cargo.toml index d30a08c..15fb9cb 100644 --- a/spacedust/Cargo.toml +++ b/spacedust/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] async-trait = "0.1.88" -clap = { version = "4.5.40", features = ["derive"] } +clap = { workspace = true } ctrlc = "3.4.7" dropshot = "0.16.2" env_logger = "0.11.8" diff --git a/spacedust/src/main.rs b/spacedust/src/main.rs index df99eff..1e26bd8 100644 --- a/spacedust/src/main.rs +++ b/spacedust/src/main.rs @@ -16,18 +16,23 @@ use tokio_util::sync::CancellationToken; struct Args { /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] + #[arg(long, env = "SPACEDUST_JETSTREAM")] jetstream: String, /// don't request zstd-compressed jetstream events /// /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] + #[arg(long, action, env = "SPACEDUST_JETSTREAM_NO_ZSTD")] jetstream_no_zstd: bool, /// spacedust server's listen address - #[arg(long, default_value = "0.0.0.0:9998")] + #[arg(long, env = "SPACEDUST_BIND")] + #[clap(default_value = "[::]:8080")] bind: std::net::SocketAddr, + /// enable metrics collection and serving + #[arg(long, action, env = "SPACEDUST_COLLECT_METRICS")] + collect_metrics: bool, /// metrics server's listen address - #[arg(long, default_value = "0.0.0.0:8765")] + #[arg(long, requires("collect_metrics"), env = "SPACEDUST_BIND_METRICS")] + #[clap(default_value = "[::]:8765")] bind_metrics: std::net::SocketAddr, } @@ -66,9 +71,12 @@ async fn main() -> Result<(), String> { let args = Args::parse(); - if let Err(e) = install_metrics_server(args.bind_metrics) { - log::error!("failed to install metrics server: {e:?}"); - }; + if args.collect_metrics { + log::trace!("installing metrics server..."); + if let Err(e) = install_metrics_server(args.bind_metrics) { + log::error!("failed to install metrics server: {e:?}"); + }; + } let mut tasks: tokio::task::JoinSet> = tokio::task::JoinSet::new(); @@ -129,7 +137,9 @@ async fn main() -> Result<(), String> { Ok(()) } -fn install_metrics_server(bind_metrics: std::net::SocketAddr) -> Result<(), metrics_exporter_prometheus::BuildError> { +fn install_metrics_server( + bind_metrics: std::net::SocketAddr, +) -> Result<(), metrics_exporter_prometheus::BuildError> { log::info!("installing metrics server..."); PrometheusBuilder::new() .set_quantiles(&[0.5, 0.9, 0.99, 1.0])? diff --git a/ufos/Cargo.toml b/ufos/Cargo.toml index 7e99ac6..02c7a62 100644 --- a/ufos/Cargo.toml +++ b/ufos/Cargo.toml @@ -10,7 +10,7 @@ base64 = "0.22.1" bincode = { version = "2.0.1", features = ["serde"] } cardinality-estimator-safe = { version = "4.0.2", features = ["with_serde", "with_digest"] } chrono = { version = "0.4.41", features = ["serde"] } -clap = { version = "4.5.31", features = ["derive"] } +clap = { workspace = true } dropshot = "0.16.0" env_logger = "0.11.7" fjall = { git = "https://github.com/fjall-rs/fjall.git", rev = "fb229572bb7d1d6966a596994dc1708e47ec57d8", features = ["lz4"] } diff --git a/ufos/src/main.rs b/ufos/src/main.rs index 76dd580..9f5cafe 100644 --- a/ufos/src/main.rs +++ b/ufos/src/main.rs @@ -26,40 +26,45 @@ static GLOBAL: Jemalloc = Jemalloc; struct Args { /// Jetstream server to connect to (exclusive with --fixture). Provide either a wss:// URL, or a shorhand value: /// 'us-east-1', 'us-east-2', 'us-west-1', or 'us-west-2' - #[arg(long)] + #[arg(long, env = "UFOS_JETSTREAM")] jetstream: String, /// allow changing jetstream endpoints - #[arg(long, action)] + #[arg(long, action, env = "UFOS_JETSTREAM_FORCE")] jetstream_force: bool, /// don't request zstd-compressed jetstream events /// /// reduces CPU at the expense of more ingress bandwidth - #[arg(long, action)] + #[arg(long, action, env = "UFOS_JETSTREAM_NO_ZSTD")] jetstream_no_zstd: bool, + /// ufos server's listen address + #[arg(long, env = "UFOS_BIND")] + #[clap(default_value = "0.0.0.0:9990")] + bind: std::net::SocketAddr, /// Location to store persist data to disk - #[arg(long)] + #[arg(long, env = "UFOS_DATA")] data: PathBuf, /// DEBUG: don't start the jetstream consumer or its write loop - #[arg(long, action)] + #[arg(long, action, env = "UFOS_PAUSE_WRITER")] pause_writer: bool, /// Adjust runtime settings like background task intervals for efficient backfill - #[arg(long, action)] + #[arg(long, action, env = "UFOS_BACKFILL_MODE")] backfill: bool, /// DEBUG: force the rw loop to fall behind by pausing it /// todo: restore this #[arg(long, action)] pause_rw: bool, /// reset the rollup cursor, scrape through missed things in the past (backfill) - #[arg(long, action)] + #[arg(long, action, env = "UFOS_REROLL")] reroll: bool, /// DEBUG: interpret jetstream as a file fixture - #[arg(long, action)] + #[arg(long, action, env = "UFOS_JETSTREAM_FIXTURE")] jetstream_fixture: bool, - /// ufos server's listen address - #[arg(long, default_value = "0.0.0.0:9990")] - bind: std::net::SocketAddr, + /// enable metrics collection and serving + #[arg(long, action, env = "UFOS_COLLECT_METRICS")] + collect_metrics: bool, /// metrics server's listen address - #[arg(long, default_value = "0.0.0.0:8765")] + #[arg(long, env = "UFOS_BIND_METRICS")] + #[clap(default_value = "0.0.0.0:8765")] bind_metrics: std::net::SocketAddr, } @@ -143,7 +148,10 @@ async fn go( Ok(()) }); - install_metrics_server(args.bind_metrics)?; + if args.collect_metrics { + log::trace!("installing metrics server..."); + install_metrics_server(args.bind_metrics)?; + } for (i, t) in consumer_tasks.join_all().await.iter().enumerate() { log::warn!("task {i} done: {t:?}"); diff --git a/ufos/src/server/mod.rs b/ufos/src/server/mod.rs index 1626ee1..bb9b9be 100644 --- a/ufos/src/server/mod.rs +++ b/ufos/src/server/mod.rs @@ -716,7 +716,10 @@ async fn search_collections( .await } -pub async fn serve(storage: impl StoreReader + 'static, bind: std::net::SocketAddr) -> Result<(), String> { +pub async fn serve( + storage: impl StoreReader + 'static, + bind: std::net::SocketAddr, +) -> Result<(), String> { describe_metrics(); let log = ConfigLogging::StderrTerminal { level: ConfigLoggingLevel::Warn, diff --git a/who-am-i/Cargo.toml b/who-am-i/Cargo.toml index 29f097f..399fc10 100644 --- a/who-am-i/Cargo.toml +++ b/who-am-i/Cargo.toml @@ -11,7 +11,7 @@ atrium-oauth = "0.1.3" axum = "0.8.4" axum-extra = { version = "0.10.1", features = ["cookie-signed", "typed-header"] } axum-template = { version = "3.0.0", features = ["handlebars"] } -clap = { version = "4.5.40", features = ["derive", "env"] } +clap = { workspace = true } ctrlc = "3.4.7" dashmap = "6.1.0" elliptic-curve = "0.13.8"