diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -4829,6 +4829,7 @@ "knot-edge", "knot-events", "knot-git", "knot-index", + "knot-keyfill", "knot-lfs", "knot-maintenance", "knot-messages", diff --git a/knot2/Containerfile b/knot2/Containerfile --- a/knot2/Containerfile +++ b/knot2/Containerfile @@ -5,6 +5,7 @@ && rm -rf /var/lib/apt/lists/* ENV RUSTFLAGS="-C linker=clang -C link-arg=-fuse-ld=mold" WORKDIR /src COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates ./crates COPY bobbin/crates ./bobbin/crates COPY shuttle ./shuttle COPY lexicons ./lexicons diff --git a/knot2/crates/knot-config/src/lib.rs b/knot2/crates/knot-config/src/lib.rs --- a/knot2/crates/knot-config/src/lib.rs +++ b/knot2/crates/knot-config/src/lib.rs @@ -41,6 +41,8 @@ pub pack: PackConfig, #[config(nested)] pub lfs: LfsConfig, #[config(nested)] + pub keyfill: KeyfillConfig, + #[config(nested)] pub resources: ResourcesConfig, #[config(nested)] pub homepage: HomepageConfig, @@ -435,6 +437,21 @@ pub max_http_downloads: u32, } #[derive(Debug, Config)] +pub struct KeyfillConfig { + #[config(env = "KNOT_KEYFILL_KEY_BUDGET_MIB", default = 64)] + pub key_budget_mib: u32, + + #[config(env = "KNOT_KEYFILL_TTL_SECS", default = 3_600)] + pub ttl_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_RETRY_SECS", default = 300)] + pub reprieve_retry_secs: u64, + + #[config(env = "KNOT_KEYFILL_REPRIEVE_BUDGET_SECS", default = 21_600)] + pub reprieve_budget_secs: u64, +} + +#[derive(Debug, Config)] pub struct ResourcesConfig { #[config(env = "KNOT_MAX_THREADS", default = 0)] pub max_threads: u32, @@ -758,6 +775,26 @@ self.xrpc.events_max_subscribers >= self.xrpc.events_max_per_peer, "xrpc.events_max_subscribers must be at least xrpc.events_max_per_peer", ), check( + (1..=MAX_KEYFILL_BUDGET_MIB).contains(&self.keyfill.key_budget_mib), + "keyfill.key_budget_mib must be between one mebibyte and one tebibyte", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.ttl_secs), + "keyfill.ttl_secs must be between one second and one year", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.reprieve_retry_secs), + "keyfill.reprieve_retry_secs must be between one second and one year", + ), + check( + (1..=MAX_KEYFILL_SPAN_SECS).contains(&self.keyfill.reprieve_budget_secs), + "keyfill.reprieve_budget_secs must be between one second and one year", + ), + check( + self.keyfill.reprieve_budget_secs >= self.keyfill.reprieve_retry_secs, + "keyfill.reprieve_budget_secs must be at least keyfill.reprieve_retry_secs", + ), + check( self.maintenance.interval_secs > 0, "maintenance.interval_secs must be greater than zero", ), @@ -1019,6 +1056,10 @@ } const MASTER_KEY_MIN_BYTES: usize = 32; +const MAX_KEYFILL_SPAN_SECS: u64 = 365 * 24 * 60 * 60; + +const MAX_KEYFILL_BUDGET_MIB: u32 = 1024 * 1024; + fn verify_writable_dir(field: &'static str, path: &Path) -> Result<(), EnvError> { let metadata = std::fs::metadata(path).map_err(|source| EnvError::DirInaccessible { field, @@ -1268,6 +1309,12 @@ gc_grace_secs: 1_209_600, gc_interval_secs: 21_600, max_ssh_transfers: 16, max_http_downloads: 64, + }, + keyfill: KeyfillConfig { + key_budget_mib: 64, + ttl_secs: 3_600, + reprieve_retry_secs: 300, + reprieve_budget_secs: 21_600, }, resources: ResourcesConfig { max_threads: 0, @@ -1559,6 +1606,39 @@ config.xrpc.events_max_subscribers = 4; config.xrpc.events_max_per_peer = 8; }, "events_max_subscribers must be at least", + ), + ( + "a_key_budget_too_small_for_any_key", + |config| config.keyfill.key_budget_mib = 0, + "keyfill.key_budget_mib", + ), + ( + "a_key_budget_past_what_any_machine_has", + |config| config.keyfill.key_budget_mib = u32::MAX, + "keyfill.key_budget_mib", + ), + ( + "a_key_ttl_that_expires_on_the_read", + |config| config.keyfill.ttl_secs = 0, + "keyfill.ttl_secs", + ), + ( + "a_key_ttl_past_what_a_unix_timestamp_can_represent", + |config| config.keyfill.ttl_secs = u64::MAX, + "keyfill.ttl_secs", + ), + ( + "a_zero_second_reprieve_retry", + |config| config.keyfill.reprieve_retry_secs = 0, + "keyfill.reprieve_retry_secs", + ), + ( + "a_reprieve_budget_under_one_retry", + |config| { + config.keyfill.reprieve_retry_secs = 600; + config.keyfill.reprieve_budget_secs = 300; + }, + "keyfill.reprieve_budget_secs must be at least", ), ( "a_non_https_plc_directory", diff --git a/knot2/crates/knot-server/Cargo.toml b/knot2/crates/knot-server/Cargo.toml --- a/knot2/crates/knot-server/Cargo.toml +++ b/knot2/crates/knot-server/Cargo.toml @@ -14,6 +14,7 @@ knot-pack = { workspace = true } knot-resource = { workspace = true } knot-cache = { workspace = true } knot-index = { workspace = true } +knot-keyfill = { workspace = true } knot-atproto = { workspace = true } knot-runtime = { workspace = true } knot-ssh = { workspace = true } diff --git a/knot2/crates/knot-server/src/main.rs b/knot2/crates/knot-server/src/main.rs --- a/knot2/crates/knot-server/src/main.rs +++ b/knot2/crates/knot-server/src/main.rs @@ -31,6 +31,7 @@ use knot_xrpc::XrpcState; use tower_http::services::ServeFile; const MAINTENANCE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(30); +const KEYFILL_SHUTDOWN_DRAIN: Duration = Duration::from_secs(10); const EDGE_SHUTDOWN_DRAIN: Duration = Duration::from_secs(40); const DEFAULT_HOMEPAGE: &str = include_str!("homepage.html"); @@ -120,6 +121,17 @@ println!("configuration is valid"); Ok(()) } +fn keyfill_pace(config: &knot_config::KeyfillConfig) -> knot_keyfill::Pace { + knot_keyfill::Pace { + ttl: knot_index::KeyTtl::from_secs(config.ttl_secs), + reprieve: knot_index::KeyReprieve::from_secs( + config.reprieve_retry_secs, + config.reprieve_budget_secs, + ), + ..knot_keyfill::Pace::default() + } +} + fn subcommand(name: &str) -> Option> { match name { "config-template" => { @@ -210,7 +222,12 @@ let meta_path = layout .meta_path(&knot_did) .context("resolve meta-repo path")?; - let index = Arc::new(Index::new(meta_path.clone(), layout.clone())); + let key_pace = keyfill_pace(&config.keyfill); + let index = Arc::new(Index::with_key_budget( + meta_path.clone(), + layout.clone(), + knot_index::KeyBudget::from_mib(config.keyfill.key_budget_mib as usize), + )); index.rebuild().context("rebuild index from meta-repo")?; tracing::info!(coverage = ?index.coverage(), "index ready"); @@ -527,6 +544,7 @@ }) .with_maintenance(maintenance_handle.clone()) .with_limits(pack_limits) .with_slots(slots.clone()) + .with_key_ttl(key_pace.ttl) .with_catalog(Arc::clone(&catalog)); let ssh_state = Arc::new(match &lfs_handle { Some(handle) => ssh_base.with_lfs(handle.clone(), lfs_max_ssh_transfers), @@ -626,6 +644,13 @@ tracing::info!("listening on {scheme}://{http_addr} and ssh://{ssh_addr}"); let shutdown = CancellationToken::new(); tokio::spawn(allocator::govern_decay(shutdown.clone())); + let keyfill_task = knot_keyfill::spawn( + Arc::clone(&index), + Arc::clone(&atproto), + slots.clone(), + key_pace, + shutdown.clone(), + ); let mut edge_task = tokio::spawn(knot_edge::serve( edge_config, app, @@ -667,6 +692,15 @@ { tracing::warn!( timeout_secs = EDGE_SHUTDOWN_DRAIN.as_secs(), "aborting edge drain after timeout" + ); + } + if tokio::time::timeout(KEYFILL_SHUTDOWN_DRAIN, keyfill_task) + .await + .is_err() + { + tracing::warn!( + timeout_secs = KEYFILL_SHUTDOWN_DRAIN.as_secs(), + "aborting key fill drain after timeout" ); } if let Some(shutdown) = maintenance_shutdown { diff --git a/knot2/example.toml b/knot2/example.toml --- a/knot2/example.toml +++ b/knot2/example.toml @@ -415,6 +415,23 @@ # Can also be specified via environment variable `KNOT_LFS_MAX_HTTP_DOWNLOADS`. # Default value: 64 #max_http_downloads = 64 +[keyfill] +# Can also be specified via environment variable `KNOT_KEYFILL_KEY_BUDGET_MIB`. +# Default value: 64 +#key_budget_mib = 64 + +# Can also be specified via environment variable `KNOT_KEYFILL_TTL_SECS`. +# Default value: 3600 +#ttl_secs = 3600 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_RETRY_SECS`. +# Default value: 300 +#reprieve_retry_secs = 300 + +# Can also be specified via environment variable `KNOT_KEYFILL_REPRIEVE_BUDGET_SECS`. +# Default value: 21600 +#reprieve_budget_secs = 21600 + [resources] # Can also be specified via environment variable `KNOT_MAX_THREADS`. # Default value: 0