diff --git a/docs/configuration.md b/docs/configuration.md index fdd2ab9..000c62a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -17,7 +17,7 @@ for what to set for a given workload, see [tuning](tuning.md). - numbers are plain decimal integers. `4GiB`, `1_000` or `1e9` fall back to the default. values aren't trimmed either, so `" 8"` does too. - bools only take exactly `true` or `false`. `1`, `yes` and `TRUE` fall back to the default, so `ENABLE_FIREHOSE=0` leaves the firehose **on**. - durations use [humantime](https://docs.rs/humantime/latest/humantime/fn.parse_duration.html) and need a unit: `3s`, `5min`, `2h`, `3d`. a bare `300` falls back to the default, only a bare `0` works without one. -- lists are comma separated, entries trimmed, empty entries dropped. a bad entry in `RELAY_HOSTS`, `SEED_HOSTS`, `RATE_TIERS` or `TIER_RULES` is dropped with a warning, but config is read before logging starts, so on the first start that warning goes nowhere. a bad `CRAWLER_URLS` entry is dropped with no warning at all. +- lists are comma separated, entries trimmed, empty entries dropped. a bad entry in `RELAY_HOSTS`, `SEED_HOSTS`, `CRAWLER_URLS`, `RATE_TIERS` or `TIER_RULES` is dropped with a warning in the log, and the rest of the list is used. - a few are strict and stop startup with an error instead: `PLC_URL`, `HISTORY_TTL`, `FILTER_EXCLUDES` (every entry has to be a did), `API_BIND`, `API_SOCKET_MODE`, `API_TCP_MANAGEMENT`, and any non-empty `BACKFILL_PROXIES` in an indexer. - a few crash hydrant at startup rather than erroring: `FIREHOSE_WORKERS=0`, `DB_WORKER_THREADS=0` and `DB_MAX_JOURNALING_SIZE_MB` below `64`. @@ -116,7 +116,7 @@ these set the database's stored filter, the same one `PATCH /filter` edits (see | variable | default | description | | :--- | :--- | :--- | -| `CRAWLER_URLS` | relay hosts as `list_repos` (full network), `by_collection::https://lightrail.microcosm.blue` (filter) | comma-separated list of `[mode::]url` crawler sources. mode is `list_repos`, `relay_first`, or `by_collection` (dashes work too), bare URLs use the default mode (`list_repos` with full network, `by_collection` otherwise). `list_repos` pages a relay's repo list, `by_collection` asks for repos holding each `FILTER_SIGNALS` collection and does nothing without signals. `relay_first` is an explicit coverage-over-moderation opt-in. an entry with an unknown mode or bad URL is dropped without a warning. set to empty string to disable the configured sources. sources added through `/crawler/sources` are saved in the database and still start | +| `CRAWLER_URLS` | relay hosts as `list_repos` (full network), `by_collection::https://lightrail.microcosm.blue` (filter) | comma-separated list of `[mode::]url` crawler sources. mode is `list_repos`, `relay_first`, or `by_collection` (dashes work too), bare URLs use the default mode (`list_repos` with full network, `by_collection` otherwise). `list_repos` pages a relay's repo list, `by_collection` asks for repos holding each `FILTER_SIGNALS` collection and does nothing without signals. `relay_first` is an explicit coverage-over-moderation opt-in. an entry with an unknown mode or bad URL is dropped with a warning. set to empty string to disable the configured sources. sources added through `/crawler/sources` are saved in the database and still start | | `ENABLE_CRAWLER` | `true` if full network or sources configured | whether to actively query the network for unknown repositories. since `CRAWLER_URLS` has a default in every mode, unset means on unless `CRAWLER_URLS` is empty. a value other than `true` or `false` counts as unset. with `false` the crawler tasks still exist and wait. can also be toggled at runtime through `/ingestion` | | `CRAWLER_MAX_PENDING_REPOS` | `2000` (`25000` full network) | persistent discovery lookahead retained in the pending queue before the crawler pauses. this does not increase active backfill task concurrency. it's checked before each page of discovered repos is queued, so the queue can go over by one page (up to 2000 repos), and while paused the crawler rechecks every 5 seconds | | `CRAWLER_RESUME_PENDING_REPOS` | `1000` (`12500` full network) | pending-repo count at which discovery resumes. not checked against the max, and a value at or above it just resumes as soon as the queue is back under the max | diff --git a/ffi/src/lib.rs b/ffi/src/lib.rs index 0b58fff..9fec30e 100644 --- a/ffi/src/lib.rs +++ b/ffi/src/lib.rs @@ -210,14 +210,14 @@ fn start(settings_json: &str) -> Result { if !settings.contains_key("HYDRANT_API_BIND") { return Err("HYDRANT_API_BIND is required, set it to `none` for no api".to_string()); } - let binds = ApiBinds::from_lookup(lookup, None).map_err(report)?; - let config = Config::from_lookup(lookup).map_err(report)?; - + // before reading the config, so what that warns about gets printed init_logging( settings .get("RUST_LOG") .map_or(hydrant::DEFAULT_LOG_FILTER, String::as_str), ); + let binds = ApiBinds::from_lookup(lookup, None).map_err(report)?; + let config = Config::from_lookup(lookup).map_err(report)?; // fails when the embedder or an earlier start already installed one, which is fine let _ = hydrant::deps::rustls::crypto::aws_lc_rs::default_provider().install_default(); diff --git a/src/config/env.rs b/src/config/env.rs index 17abad8..076c72a 100644 --- a/src/config/env.rs +++ b/src/config/env.rs @@ -29,7 +29,6 @@ macro_rules! __cfg { } use crate::__cfg as cfg; -/// loads `.env` from the current directory, setting any variables not already in the environment. fn dotenv_assignments(contents: &str) -> impl Iterator { contents.lines().filter_map(|line| { let line = line.trim(); @@ -47,7 +46,9 @@ fn dotenv_assignments(contents: &str) -> impl Iterator { }) } -fn load_dotenv() { +/// loads `.env` from the current directory, setting any variables not already in the +/// environment. +pub fn load_dotenv() { let Ok(contents) = std::fs::read_to_string(".env") else { return; }; @@ -346,15 +347,19 @@ impl Config { if let Some(s) = lookup("HYDRANT_RATE_TIERS") { for entry in s.split(',') { let entry = entry.trim(); - if let Some((name, spec)) = entry.split_once(':') { - match RateTier::parse(spec) { - Some(tier) => { - tiers.insert(SmolStr::new(name.trim()), tier); - } - None => tracing::warn!( - "ignoring invalid rate rate tier '{name}': expected base/mul/hourly/daily format" - ), + if entry.is_empty() { + continue; + } + match entry + .split_once(':') + .and_then(|(name, spec)| Some((name, RateTier::parse(spec)?))) + { + Some((name, tier)) => { + tiers.insert(SmolStr::new(name.trim()), tier); } + None => tracing::warn!( + "ignoring invalid rate tier '{entry}': expected name:base/mul/hourly/daily format" + ), } } } @@ -385,21 +390,23 @@ impl Config { if entry.is_empty() { continue; } - if let Some((pattern_str, tier_name)) = entry.split_once(':') { - let pattern_str = pattern_str.trim(); - let tier_name = tier_name.trim(); - match glob::Pattern::new(pattern_str) { - Ok(pattern) => { - rules.push(TierRule { - pattern, - tier_name: SmolStr::new(tier_name), - }); - tier_rules.push((pattern_str.to_string(), tier_name.to_string())); - } - Err(e) => tracing::warn!( - "ignoring invalid tier rule pattern '{pattern_str}': {e}" - ), + let Some((pattern_str, tier_name)) = entry.split_once(':') else { + tracing::warn!("ignoring invalid tier rule '{entry}': expected pattern:tier format"); + continue; + }; + let pattern_str = pattern_str.trim(); + let tier_name = tier_name.trim(); + match glob::Pattern::new(pattern_str) { + Ok(pattern) => { + rules.push(TierRule { + pattern, + tier_name: SmolStr::new(tier_name), + }); + tier_rules.push((pattern_str.to_string(), tier_name.to_string())); } + Err(e) => tracing::warn!( + "ignoring invalid tier rule pattern '{pattern_str}': {e}" + ), } } } @@ -412,7 +419,12 @@ impl Config { .split(',') .map(|s| s.trim()) .filter(|s| !s.is_empty()) - .filter_map(|s| CrawlerSource::parse(s, default_mode)) + .filter_map(|s| { + CrawlerSource::parse(s, default_mode).or_else(|| { + tracing::warn!("ignoring invalid crawler url: {s}"); + None + }) + }) .collect(), None => match default_mode { CrawlerMode::ListRepos => relay_hosts diff --git a/src/main.rs b/src/main.rs index 04de7ee..3934d60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -105,9 +105,9 @@ fn main() -> miette::Result<()> { .install_default() .ok(); - let cfg = Config::from_env()?; - let app = AppConfig::from_env()?; - + // logging starts before the config is read, so what reading it warns about gets + // printed, and after `.env` is loaded, since RUST_LOG may be set there + hydrant::config::env::load_dotenv(); let directives = std::env::var("RUST_LOG") .ok() .filter(|v| !v.trim().is_empty()); @@ -116,6 +116,9 @@ fn main() -> miette::Result<()> { .parse_lossy(directives.as_deref().unwrap_or(hydrant::DEFAULT_LOG_FILTER)); tracing_subscriber::fmt().with_env_filter(env_filter).init(); + let cfg = Config::from_lookup(|key| std::env::var(key).ok())?; + let app = AppConfig::from_env()?; + let stop = watch_stop_signals()?; let runtime = tokio::runtime::Runtime::new().into_diagnostic()?; let hydrant = runtime.block_on(Hydrant::new(cfg))?; diff --git a/tests/cli.rs b/tests/cli.rs index c34d156..5b0c4d9 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -139,3 +139,22 @@ fn sigterm_and_sigint_stop_cleanly_and_the_database_opens_again() { assert!(stdout.contains("stopped"), "{signal}: {stdout}{stderr}"); } } + +#[test] +fn config_warnings_are_logged() { + let tmp = tempfile::tempdir().unwrap(); + // the bad history ttl makes it exit right after reading the config + let child = offline_hydrant(tmp.path()) + .env("HYDRANT_RELAY_HOSTS", "not a url") + .env("HYDRANT_HISTORY_TTL", "soon") + .spawn() + .unwrap(); + let out = wait_for_exit(child, Duration::from_secs(30)).expect("hydrant booted"); + assert!(!out.status.success(), "{out:?}"); + let stdout = String::from_utf8_lossy(&out.stdout); + assert!( + stdout.contains("invalid relay host URL: not a url"), + "{stdout}{}", + String::from_utf8_lossy(&out.stderr) + ); +}