diff --git a/Cargo.lock b/Cargo.lock index a40be08d..e2e2421b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4829,6 +4829,7 @@ dependencies = [ "knot-events", "knot-git", "knot-index", + "knot-keyfill", "knot-lfs", "knot-maintenance", "knot-messages", diff --git a/knot2/Containerfile b/knot2/Containerfile index 2ec966ec..c6d61521 100644 --- a/knot2/Containerfile +++ b/knot2/Containerfile @@ -5,6 +5,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" WORKDIR /src COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates ./crates COPY bobbin/crates ./bobbin/crates COPY shuttle ./shuttle COPY lexicons ./lexicons diff --git a/knot2/crates/knot-config/src/lib.rs b/knot2/crates/knot-config/src/lib.rs index d1f4cdfe..2c76fb7c 100644 --- a/knot2/crates/knot-config/src/lib.rs +++ b/knot2/crates/knot-config/src/lib.rs @@ -41,6 +41,8 @@ pub struct KnotConfig { #[config(nested)] pub lfs: LfsConfig, #[config(nested)] + pub keyfill: KeyfillConfig, + #[config(nested)] pub resources: ResourcesConfig, #[config(nested)] pub homepage: HomepageConfig, @@ -434,6 +436,21 @@ pub struct LfsConfig { pub max_http_downloads: u32, } +#[derive(Debug, Config)] +pub struct KeyfillConfig { + #[config(env = "KNOT_KEYFILL_KEY_BUDGET_MIB", default = 64)] + pub key_budget_mib: u32, + + #[config(env = "KNOT_KEYFILL_TTL_SECS", default = 3_600)] + pub ttl_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_RETRY_SECS", default = 300)] + pub reprieve_retry_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_BUDGET_SECS", default = 21_600)] + pub reprieve_budget_secs: u64, +} + #[derive(Debug, Config)] pub struct ResourcesConfig { #[config(env = "KNOT_MAX_THREADS", default = 0)] @@ -757,6 +774,26 @@ impl KnotConfig { self.xrpc.events_max_subscribers >= self.xrpc.events_max_per_peer, "xrpc.events_max_subscribers must be at least xrpc.events_max_per_peer", ), + check( + (1..=MAX_KEYFILL_BUDGET_MIB).contains(&self.keyfill.key_budget_mib), + "keyfill.key_budget_mib must be between one mebibyte and one tebibyte", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.ttl_secs), + "keyfill.ttl_secs must be between one second and one year", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.reprieve_retry_secs), + "keyfill.reprieve_retry_secs must be between one second and one year", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.reprieve_budget_secs), + "keyfill.reprieve_budget_secs must be between one second and one year", + ), + check( + self.keyfill.reprieve_budget_secs >= self.keyfill.reprieve_retry_secs, + "keyfill.reprieve_budget_secs must be at least keyfill.reprieve_retry_secs", + ), check( self.maintenance.interval_secs > 0, "maintenance.interval_secs must be greater than zero", @@ -1019,6 +1056,10 @@ pub enum EnvError { const MASTER_KEY_MIN_BYTES: usize = 32; +const MAX_KEYFILL_SPAN_SECS: u64 = 365 * 24 * 60 * 60; + +const MAX_KEYFILL_BUDGET_MIB: u32 = 1024 * 1024; + fn verify_writable_dir(field: &'static str, path: &Path) -> Result<(), EnvError> { let metadata = std::fs::metadata(path).map_err(|source| EnvError::DirInaccessible { field, @@ -1269,6 +1310,12 @@ mod tests { max_ssh_transfers: 16, max_http_downloads: 64, }, + keyfill: KeyfillConfig { + key_budget_mib: 64, + ttl_secs: 3_600, + reprieve_retry_secs: 300, + reprieve_budget_secs: 21_600, + }, resources: ResourcesConfig { max_threads: 0, max_memory_bytes: 0, @@ -1560,6 +1607,39 @@ mod tests { }, "events_max_subscribers must be at least", ), + ( + "a_key_budget_too_small_for_any_key", + |config| config.keyfill.key_budget_mib = 0, + "keyfill.key_budget_mib", + ), + ( + "a_key_budget_past_what_any_machine_has", + |config| config.keyfill.key_budget_mib = u32::MAX, + "keyfill.key_budget_mib", + ), + ( + "a_key_ttl_that_expires_on_the_read", + |config| config.keyfill.ttl_secs = 0, + "keyfill.ttl_secs", + ), + ( + "a_key_ttl_past_what_a_unix_timestamp_can_represent", + |config| config.keyfill.ttl_secs = u64::MAX, + "keyfill.ttl_secs", + ), + ( + "a_zero_second_reprieve_retry", + |config| config.keyfill.reprieve_retry_secs = 0, + "keyfill.reprieve_retry_secs", + ), + ( + "a_reprieve_budget_under_one_retry", + |config| { + config.keyfill.reprieve_retry_secs = 600; + config.keyfill.reprieve_budget_secs = 300; + }, + "keyfill.reprieve_budget_secs must be at least", + ), ( "a_non_https_plc_directory", |config| { diff --git a/knot2/crates/knot-server/Cargo.toml b/knot2/crates/knot-server/Cargo.toml index 741d6db5..31c4a381 100644 --- a/knot2/crates/knot-server/Cargo.toml +++ b/knot2/crates/knot-server/Cargo.toml @@ -14,6 +14,7 @@ knot-pack = { workspace = true } knot-resource = { workspace = true } knot-cache = { workspace = true } knot-index = { workspace = true } +knot-keyfill = { workspace = true } knot-atproto = { workspace = true } knot-runtime = { workspace = true } knot-ssh = { workspace = true } diff --git a/knot2/crates/knot-server/src/main.rs b/knot2/crates/knot-server/src/main.rs index c0243c3d..d062c3bc 100644 --- a/knot2/crates/knot-server/src/main.rs +++ b/knot2/crates/knot-server/src/main.rs @@ -31,6 +31,7 @@ use knot_xrpc::XrpcState; use tower_http::services::ServeFile; const MAINTENANCE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(30); +const KEYFILL_SHUTDOWN_DRAIN: Duration = Duration::from_secs(10); const EDGE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(40); const DEFAULT_HOMEPAGE: &str = include_str!("homepage.html"); @@ -120,6 +121,17 @@ fn validate(args: impl Iterator) -> anyhow::Result<()> { Ok(()) } +fn keyfill_pace(config: &knot_config::KeyfillConfig) -> knot_keyfill::Pace { + knot_keyfill::Pace { + ttl: knot_index::KeyTtl::from_secs(config.ttl_secs), + reprieve: knot_index::KeyReprieve::from_secs( + config.reprieve_retry_secs, + config.reprieve_budget_secs, + ), + ..knot_keyfill::Pace::default() + } +} + fn subcommand(name: &str) -> Option> { match name { "config-template" => { @@ -210,7 +222,12 @@ async fn main() -> anyhow::Result<()> { .meta_path(&knot_did) .context("resolve meta-repo path")?; - let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + let key_pace = keyfill_pace(&config.keyfill); + let index = Arc::new(Index::with_key_budget( + meta_path.clone(), + layout.clone(), + knot_index::KeyBudget::from_mib(config.keyfill.key_budget_mib as usize), + )); index.rebuild().context("rebuild index from meta-repo")?; tracing::info!(coverage = ?index.coverage(), "index ready"); @@ -527,6 +544,7 @@ async fn main() -> anyhow::Result<()> { .with_maintenance(maintenance_handle.clone()) .with_limits(pack_limits) .with_slots(slots.clone()) + .with_key_ttl(key_pace.ttl) .with_catalog(Arc::clone(&catalog)); let ssh_state = Arc::new(match &lfs_handle { Some(handle) => ssh_base.with_lfs(handle.clone(), lfs_max_ssh_transfers), @@ -626,6 +644,13 @@ async fn main() -> anyhow::Result<()> { let shutdown = CancellationToken::new(); tokio::spawn(allocator::govern_decay(shutdown.clone())); + let keyfill_task = knot_keyfill::spawn( + Arc::clone(&index), + Arc::clone(&atproto), + slots.clone(), + key_pace, + shutdown.clone(), + ); let mut edge_task = tokio::spawn(knot_edge::serve( edge_config, app, @@ -669,6 +694,15 @@ async fn main() -> anyhow::Result<()> { "aborting edge drain after timeout" ); } + if tokio::time::timeout(KEYFILL_SHUTDOWN_DRAIN, keyfill_task) + .await + .is_err() + { + tracing::warn!( + timeout_secs = KEYFILL_SHUTDOWN_DRAIN.as_secs(), + "aborting key fill drain after timeout" + ); + } if let Some(shutdown) = maintenance_shutdown { let _ = shutdown.send(true); } diff --git a/knot2/example.toml b/knot2/example.toml index b7ef7965..e2f2eabb 100644 --- a/knot2/example.toml +++ b/knot2/example.toml @@ -415,6 +415,23 @@ # Default value: 64 #max_http_downloads = 64 +[keyfill] +# Can also be specified via environment variable `KNOT_KEYFILL_KEY_BUDGET_MIB`. +# Default value: 64 +#key_budget_mib = 64 + +# Can also be specified via environment variable `KNOT_KEYFILL_TTL_SECS`. +# Default value: 3600 +#ttl_secs = 3600 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_RETRY_SECS`. +# Default value: 300 +#reprieve_retry_secs = 300 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_BUDGET_SECS`. +# Default value: 21600 +#reprieve_budget_secs = 21600 + [resources] # Can also be specified via environment variable `KNOT_MAX_THREADS`. # Default value: 0