diff --git a/stats-backfill/src/backfill_repos.rs b/stats-backfill/src/backfill_repos.rs index 59afd9f..1694609 100644 --- a/stats-backfill/src/backfill_repos.rs +++ b/stats-backfill/src/backfill_repos.rs @@ -178,6 +178,8 @@ pub struct Args { pub max_rss_mb: u64, /// Directory under which each tier-3 disk spill creates a fresh subdir. pub spill_dir: PathBuf, + /// Hosts to skip entirely (never picked as candidates). + pub skip_hosts: Vec, } pub async fn run(db: &Db, args: Args, ua: &str) -> Result<(), Error> { @@ -201,6 +203,16 @@ pub async fn run(db: &Db, args: Args, ua: &str) -> Result<(), Error> { let spill_dir = Arc::new(args.spill_dir); let progress = Arc::new(Progress::default()); + let skip_hosts: std::collections::HashSet = + args.skip_hosts.into_iter().collect(); + if !skip_hosts.is_empty() { + tracing::info!( + count = skip_hosts.len(), + hosts = ?skip_hosts.iter().collect::>(), + "skipping hosts (--skip)" + ); + } + let ticker = spawn_progress_ticker(progress.clone(), args.max_rss_mb); let mut active: JoinSet> = JoinSet::new(); @@ -211,7 +223,7 @@ pub async fn run(db: &Db, args: Args, ua: &str) -> Result<(), Error> { // kick off as many per-host workers as allowed while active.len() < args.host_workers_limit { let candidates = db.eligible_hosts().await?; - let Some(host) = pick_next_host(&candidates, &active_hosts) else { + let Some(host) = pick_next_host(&candidates, &active_hosts, &skip_hosts) else { break; }; @@ -354,10 +366,11 @@ fn ensure_nofile_for_big_repos(big_repos_limit: usize) -> Result<(), Error> { fn pick_next_host( candidates: &[HostCandidate], active: &std::collections::HashSet, + skip: &std::collections::HashSet, ) -> Option { candidates .iter() - .filter(|c| !active.contains(&c.host)) + .filter(|c| !active.contains(&c.host) && !skip.contains(&c.host)) .max_by_key(|c| c.listhosts_account_count.unwrap_or(0)) .map(|c| c.host.clone()) } diff --git a/stats-backfill/src/main.rs b/stats-backfill/src/main.rs index e8e2d83..eb591df 100644 --- a/stats-backfill/src/main.rs +++ b/stats-backfill/src/main.rs @@ -114,6 +114,13 @@ struct BackfillReposArgs { /// it's often tmpfs (RAM-backed), which defeats the purpose. #[arg(long, default_value = "./local/spill")] spill_dir: PathBuf, + + /// skip these PDS hosts entirely — they won't be picked as candidates. + /// Repeatable: `--skip foo.example --skip bar.example`. Useful for hosts + /// that are known-broken or known-unreachable in ways we can't auto- + /// detect (e.g. operator has disabled getRepo network-wide). + #[arg(long)] + skip: Vec, } #[tokio::main] @@ -176,5 +183,6 @@ fn scan_args(args: BackfillReposArgs) -> backfill_repos::Args { big_repos_limit: args.big_repos_limit, max_rss_mb: args.max_rss_mb, spill_dir: args.spill_dir, + skip_hosts: args.skip, } }