From 3312728cf3536fa6a9ad970a69d85831084f55be Mon Sep 17 00:00:00 2001 From: Lewis Date: Mon, 10 Aug 2026 11:11:51 +0300 Subject: [PATCH] knot2/knot-migrate: skip an unreadable source repo, refuse owner divergence Lewis: May this revision serve well! --- knot2/crates/knot-migrate/src/adopt.rs | 23 +- knot2/crates/knot-migrate/src/main.rs | 102 ++++-- knot2/crates/knot-migrate/src/mapping.rs | 170 ++++++---- knot2/crates/knot-migrate/src/report.rs | 18 ++ knot2/crates/knot-migrate/tests/migrate.rs | 341 ++++++++++++++++++++- 5 files changed, 574 insertions(+), 80 deletions(-) diff --git a/knot2/crates/knot-migrate/src/adopt.rs b/knot2/crates/knot-migrate/src/adopt.rs index 36d1de33..bcc571e6 100644 --- a/knot2/crates/knot-migrate/src/adopt.rs +++ b/knot2/crates/knot-migrate/src/adopt.rs @@ -74,6 +74,13 @@ pub enum Transfer { Copy, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceProbe { + Repo, + Absent, + Unreadable(std::io::ErrorKind), +} + impl std::fmt::Display for Transfer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -99,8 +106,20 @@ pub fn source_dir(source_root: &Path, repo_did: &SourceRepoDid) -> PathBuf { source_root.join(repo_did.as_str()) } -pub fn source_is_repo(source_root: &Path, repo_did: &SourceRepoDid) -> bool { - source_dir(source_root, repo_did).join("HEAD").is_file() +pub fn reads_as_absent(error: &std::io::Error) -> bool { + matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) +} + +pub fn probe_source(source_root: &Path, repo_did: &SourceRepoDid) -> SourceProbe { + match std::fs::metadata(source_dir(source_root, repo_did).join("HEAD")) { + Ok(head) if head.is_file() => SourceProbe::Repo, + Ok(_) => SourceProbe::Absent, + Err(error) if reads_as_absent(&error) => SourceProbe::Absent, + Err(error) => SourceProbe::Unreadable(error.kind()), + } } pub fn adopt_all( diff --git a/knot2/crates/knot-migrate/src/main.rs b/knot2/crates/knot-migrate/src/main.rs index 90cb37de..7dc52e92 100644 --- a/knot2/crates/knot-migrate/src/main.rs +++ b/knot2/crates/knot-migrate/src/main.rs @@ -7,12 +7,12 @@ use knot_migrate::adopt::{self, AdoptError, SourcePolicy}; use knot_migrate::casbin::{self, CasbinError}; use knot_migrate::emit::{self, ConfigValues, EmitError, MasterKeyEnv}; use knot_migrate::envfile::{EnvFile, EnvFileError}; -use knot_migrate::mapping::{self, Mapping, MappingError}; +use knot_migrate::mapping::{self, Mapping, MappingError, RepoList}; use knot_migrate::report::Report; use knot_migrate::source::{SourceDb, SourceError, SourceRepoDid, SourceRkey, SourceSchema}; use knot_runtime::OsEntropy; use knot_secrets::{MasterKey, SealedStore, SecretsError}; -use knot_types::{AccountDid, KnotHostname, ObjectFormat}; +use knot_types::{AccountDid, KnotHostname, ObjectFormat, RepoDid}; use url::Url; // TODO: I wanted to see how well I could work without clap. I shoulda just used clap. @@ -35,6 +35,8 @@ options: --master-key-env env var holding the base64 master key, default KNOT_MASTER_KEY --consume-source move the source repos into place instead of copying them, which empties the source tree and needs one filesystem + --skip-unreadable migrate the rest when knot-migrate can't read a source path, + and leave those repos on the old knot --dry-run print the mapping and reconciliation report, write nothing "; @@ -66,6 +68,8 @@ enum MigrateError { MissingMasterKey { name: MasterKeyEnv }, #[error("master key env var {name} isn't base64")] MalformedMasterKey { name: MasterKeyEnv }, + #[error("{0}")] + Refused(Refusals), #[error("{context}: {source}")] Io { context: String, @@ -73,6 +77,55 @@ enum MigrateError { }, } +#[derive(Debug, thiserror::Error)] +enum Refusal { + #[error( + "knot-migrate can't read the source path of {repos}. Each repo is in the skip list above with the error behind it. Re-run as root, or as a user in the group that owns those trees, when that error is permission denied. Pass --skip-unreadable to migrate everything else and leave those repos on the old knot." + )] + UnreadableSources { repos: RepoList }, + #[error( + "the acl and repo_keys are recorded with different owners for {repos}. Both DIDs per repo are in the drift section above. Settle each repo in the old knot's database, by deleting the acl rows for the wrong owner or by correcting repo_keys.owner_did, since knot-migrate won't pick a winner for you." + )] + ConflictingOwners { repos: RepoList }, +} + +#[derive(Debug)] +struct Refusals(Vec); + +impl Refusals { + fn gather(mapping: &Mapping, skip_unreadable: bool) -> Self { + let conflicting = mapping.conflicting_owners(); + let unreadable = mapping.unreadable_sources(); + Self( + [ + (!conflicting.is_empty()) + .then_some(Refusal::ConflictingOwners { repos: conflicting }), + (!unreadable.is_empty() && !skip_unreadable) + .then_some(Refusal::UnreadableSources { repos: unreadable }), + ] + .into_iter() + .flatten() + .collect(), + ) + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl std::fmt::Display for Refusals { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0 + .iter() + .enumerate() + .try_for_each(|(position, refusal)| match position { + 0 => write!(f, "{refusal}"), + _ => write!(f, "\n{refusal}"), + }) + } +} + struct Args { source_db: PathBuf, source_repos: Option, @@ -84,6 +137,7 @@ struct Args { object_format: ObjectFormat, master_key_env: MasterKeyEnv, source_policy: SourcePolicy, + skip_unreadable: bool, dry_run: bool, } @@ -91,6 +145,7 @@ struct Args { struct Switches { dry_run: bool, consume_source: bool, + skip_unreadable: bool, } const KNOWN_FLAGS: [&str; 9] = [ @@ -136,6 +191,14 @@ fn parse_args(args: &[String]) -> Result { }, None, )), + (None, "--skip-unreadable") => Ok(( + flags, + Switches { + skip_unreadable: true, + ..switches + }, + None, + )), (None, flag) => match flag.strip_prefix("--").map(|rest| { rest.split_once('=') .map_or((rest, None), |(key, value)| (key, Some(value))) @@ -187,6 +250,7 @@ fn parse_args(args: &[String]) -> Result { true => SourcePolicy::Consume, false => SourcePolicy::Preserve, }, + skip_unreadable: switches.skip_unreadable, dry_run: switches.dry_run, }) } @@ -288,7 +352,7 @@ fn run(args: &[String]) -> Result<(), MigrateError> { ) })); let acl = casbin::decode(&db.acl()?, &resolver)?; - let exists = |repo_did: &SourceRepoDid| adopt::source_is_repo(&source_repos, repo_did); + let probe = |repo_did: &SourceRepoDid| adopt::probe_source(&source_repos, repo_did); let mapping = match schema { SourceSchema::Tables => mapping::map_tables( &repos, @@ -296,11 +360,9 @@ fn run(args: &[String]) -> Result<(), MigrateError> { &db.members()?, &db.collaborators()?, &acl, - exists, + probe, )?, - SourceSchema::PreFlip => { - mapping::map_preflip(&repos, &rkeys, &db.members()?, &acl, exists)? - } + SourceSchema::PreFlip => mapping::map_preflip(&repos, &rkeys, &db.members()?, &acl, probe)?, }; env.get("KNOT_SERVER_OWNER") .filter(|owner| AccountDid::new(*owner).ok().as_ref() != Some(&mapping.knot_owner)) @@ -312,15 +374,16 @@ fn run(args: &[String]) -> Result<(), MigrateError> { })?; let orphan_alias_count = db.orphan_alias_count()?; - let written = match args.dry_run { - true => None, - false => Some(materialize( + let refusals = Refusals::gather(&mapping, args.skip_unreadable); + let written = match (args.dry_run, refusals.is_empty()) { + (false, true) => Some(materialize( &args, &hostname, &plc_directory, &source_repos, &mapping, )?), + _ => None, }; print!( "{}", @@ -331,14 +394,17 @@ fn run(args: &[String]) -> Result<(), MigrateError> { cobs: written.as_ref().map(|written| &written.cobs), } ); - written.map_or(Ok(()), |written| { - println!(); - println!("knot key identity: {}", written.knot_did); - println!("host key algorithm: {}", written.host_key_algorithm); - println!("config: {}", written.config_file.display()); - println!("key archive: {}", written.archive_file.display()); - Ok(()) - }) + match refusals.is_empty() { + false => Err(MigrateError::Refused(refusals)), + true => written.map_or(Ok(()), |written| { + println!(); + println!("knot key identity: {}", written.knot_did); + println!("host key algorithm: {}", written.host_key_algorithm); + println!("config: {}", written.config_file.display()); + println!("key archive: {}", written.archive_file.display()); + Ok(()) + }), + } } fn timed(phase: &str, work: impl FnOnce() -> Result) -> Result { diff --git a/knot2/crates/knot-migrate/src/mapping.rs b/knot2/crates/knot-migrate/src/mapping.rs index 521cf816..b06407ad 100644 --- a/knot2/crates/knot-migrate/src/mapping.rs +++ b/knot2/crates/knot-migrate/src/mapping.rs @@ -4,6 +4,7 @@ use std::fmt; use knot_types::{AccountDid, OwnerDid, ParseError, RepoDid, RepoName, RepoRkey, UnixSeconds}; use zeroize::{Zeroize, ZeroizeOnDrop}; +use crate::adopt::SourceProbe; use crate::casbin::AclRoster; use crate::source::{ CollabRow, MemberRow, RepoRow, SourceDid, SourceKeyType, SourceRepoDid, SourceRepoName, @@ -47,12 +48,6 @@ pub enum MappingError { }, #[error("acl names no server owner")] MissingServerOwner, - #[error("acl marks {marker} as owner of {repo} while repo_keys names {owner}")] - ConflictingOwnerMarker { - repo: SourceRepoDid, - marker: SourceDid, - owner: SourceDid, - }, #[error("record key {owner}/{rkey} has no single alias-backed holder")] AmbiguousRkey { owner: OwnerDid, rkey: RepoRkey }, } @@ -98,6 +93,7 @@ pub enum SkipReason { Rkey { value: SourceRkey }, RkeyCollision { rkey: RepoRkey, winner: RepoDid }, NoSourceRepo, + UnreadableSource { kind: std::io::ErrorKind }, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -107,6 +103,13 @@ pub struct SkippedRepo { pub lost_collaborators: Vec, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OwnerConflict { + pub repo: SourceRepoDid, + pub acl_owner: SourceDid, + pub key_owner: SourceDid, +} + #[derive(Debug, Default, PartialEq, Eq)] pub struct Drift { pub acl_only_collaborators: Vec<(SourceRepoDid, SourceDid)>, @@ -115,6 +118,7 @@ pub struct Drift { pub orphan_collaborator_pairs: Vec<(SourceRepoDid, SourceDid)>, pub markerless_owner_repos: Vec, pub orphan_owner_markers: Vec, + pub conflicting_owner_markers: Vec, pub extra_owner_markers: Vec<(SourceRepoDid, SourceDid)>, pub acl_only_members: Vec, pub table_only_members: Vec, @@ -132,13 +136,63 @@ pub struct Mapping { pub drift: Drift, } +impl Mapping { + pub fn unreadable_sources(&self) -> RepoList { + RepoList( + self.skipped + .iter() + .filter(|skip| matches!(skip.reason, SkipReason::UnreadableSource { .. })) + .map(|skip| skip.repo_did.clone()) + .collect(), + ) + } + + pub fn conflicting_owners(&self) -> RepoList { + RepoList( + self.drift + .conflicting_owner_markers + .iter() + .map(|conflict| conflict.repo.clone()) + .collect::>() + .into_iter() + .collect(), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepoList(Vec); + +impl RepoList { + const LISTED: usize = 5; + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl fmt::Display for RepoList { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.iter().take(Self::LISTED).enumerate().try_for_each( + |(position, did)| match position { + 0 => write!(f, "{did}"), + _ => write!(f, ", {did}"), + }, + )?; + match self.0.len().saturating_sub(Self::LISTED) { + 0 => Ok(()), + rest => write!(f, ", and {rest} more"), + } + } +} + pub fn map_tables( repos: &[RepoRow], rkeys: &BTreeMap, members: &[MemberRow], collabs: &[CollabRow], acl: &AclRoster, - exists: impl Fn(&SourceRepoDid) -> bool, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, ) -> Result { let knot_owner = server_owner(acl)?; let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = @@ -260,8 +314,8 @@ pub fn map_tables( )?; let collab_grants = owner_attributed_grants(&acl_only, &repo_index, true, table_grants)?; - let owners = owner_drift(repos, &repo_index, acl)?; - let (adopted, skipped) = classify(repos, rkeys, &collab_grants, exists)?; + let owners = owner_drift(repos, &repo_index, acl); + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, probe)?; Ok(Mapping { knot_owner, @@ -284,6 +338,7 @@ pub fn map_tables( orphan_collaborator_pairs: orphan_pairs.into_iter().collect(), markerless_owner_repos: owners.markerless, orphan_owner_markers: owners.orphans, + conflicting_owner_markers: owners.conflicts, extra_owner_markers: owners.extras, acl_only_members, table_only_members, @@ -299,7 +354,7 @@ pub fn map_preflip( rkeys: &BTreeMap, members: &[MemberRow], acl: &AclRoster, - exists: impl Fn(&SourceRepoDid) -> bool, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, ) -> Result { let knot_owner = server_owner(acl)?; let repo_index: BTreeMap<&SourceRepoDid, &RepoRow> = @@ -357,8 +412,8 @@ pub fn map_preflip( live.partition(|(repo, _)| repo_index.contains_key(*repo)); let collab_grants = owner_attributed_grants(&resolvable, &repo_index, false, BTreeMap::new())?; - let owners = owner_drift(repos, &repo_index, acl)?; - let (adopted, skipped) = classify(repos, rkeys, &collab_grants, exists)?; + let owners = owner_drift(repos, &repo_index, acl); + let (adopted, skipped) = classify(repos, rkeys, &collab_grants, probe)?; Ok(Mapping { knot_owner, @@ -372,6 +427,7 @@ pub fn map_preflip( .collect(), markerless_owner_repos: owners.markerless, orphan_owner_markers: owners.orphans, + conflicting_owner_markers: owners.conflicts, extra_owner_markers: owners.extras, table_only_members, slash_owner_markers: acl.slash_owner_markers, @@ -426,6 +482,7 @@ fn server_owner(acl: &AclRoster) -> Result { struct OwnerDrift { markerless: Vec, orphans: Vec, + conflicts: Vec, extras: Vec<(SourceRepoDid, SourceDid)>, } @@ -433,64 +490,59 @@ fn owner_drift( repos: &[RepoRow], repo_index: &BTreeMap<&SourceRepoDid, &RepoRow>, acl: &AclRoster, -) -> Result { - repos.iter().try_for_each(|repo| { - let conflicting = acl.owner_markers.get(&repo.repo_did).and_then(|markers| { - (!markers.contains(&repo.owner_did)) - .then(|| markers.iter().next().cloned()) - .flatten() - }); - match conflicting { - Some(marker) => Err(MappingError::ConflictingOwnerMarker { - repo: repo.repo_did.clone(), - marker, - owner: repo.owner_did.clone(), - }), - None => Ok(()), - } - })?; - let markerless = repos - .iter() - .filter(|repo| !acl.owner_markers.contains_key(&repo.repo_did)) - .map(|repo| repo.repo_did.clone()) - .collect(); - let orphans = acl - .owner_markers - .keys() - .filter(|repo| !repo_index.contains_key(*repo)) - .cloned() - .collect(); - let extras = repos +) -> OwnerDrift { + let (conflicting, agreeing): (Vec<_>, Vec<_>) = repos .iter() .filter_map(|repo| { acl.owner_markers .get(&repo.repo_did) .map(|markers| (repo, markers)) }) - .flat_map(|(repo, markers)| { - markers - .iter() - .filter(move |marker| *marker != &repo.owner_did) - .map(move |marker| (repo.repo_did.clone(), marker.clone())) - }) - .collect(); - Ok(OwnerDrift { - markerless, - orphans, - extras, - }) + .partition(|(repo, markers)| !markers.contains(&repo.owner_did)); + OwnerDrift { + markerless: repos + .iter() + .filter(|repo| !acl.owner_markers.contains_key(&repo.repo_did)) + .map(|repo| repo.repo_did.clone()) + .collect(), + orphans: acl + .owner_markers + .keys() + .filter(|repo| !repo_index.contains_key(*repo)) + .cloned() + .collect(), + conflicts: conflicting + .into_iter() + .flat_map(|(repo, markers)| { + markers.iter().map(move |marker| OwnerConflict { + repo: repo.repo_did.clone(), + acl_owner: marker.clone(), + key_owner: repo.owner_did.clone(), + }) + }) + .collect(), + extras: agreeing + .into_iter() + .flat_map(|(repo, markers)| { + markers + .iter() + .filter(move |marker| *marker != &repo.owner_did) + .map(move |marker| (repo.repo_did.clone(), marker.clone())) + }) + .collect(), + } } fn classify( repos: &[RepoRow], rkeys: &BTreeMap, collab_grants: &BTreeMap>, - exists: impl Fn(&SourceRepoDid) -> bool, + probe: impl Fn(&SourceRepoDid) -> SourceProbe, ) -> Result<(Vec, Vec), MappingError> { let (adopted, skipped) = repos.iter().try_fold( (Vec::new(), Vec::new()), |(mut adopted, mut skipped), row| { - match classify_one(row, rkeys, collab_grants, &exists)? { + match classify_one(row, rkeys, collab_grants, &probe)? { Ok(repo) => adopted.push(repo), Err(skip) => skipped.push(skip), } @@ -580,7 +632,7 @@ fn classify_one( row: &RepoRow, rkeys: &BTreeMap, collab_grants: &BTreeMap>, - exists: &impl Fn(&SourceRepoDid) -> bool, + probe: &impl Fn(&SourceRepoDid) -> SourceProbe, ) -> Result, MappingError> { let did = RepoDid::new(row.repo_did.as_str()).map_err(|source| MappingError::BadRepoDid { value: row.repo_did.clone(), @@ -638,8 +690,12 @@ fn classify_one( }))); } }; - if !exists(&row.repo_did) { - return Ok(Err(skip(SkipReason::NoSourceRepo))); + match probe(&row.repo_did) { + SourceProbe::Absent => return Ok(Err(skip(SkipReason::NoSourceRepo))), + SourceProbe::Unreadable(kind) => { + return Ok(Err(skip(SkipReason::UnreadableSource { kind }))); + } + SourceProbe::Repo => {} } Ok(Ok(AdoptRepo { diff --git a/knot2/crates/knot-migrate/src/report.rs b/knot2/crates/knot-migrate/src/report.rs index e64e46b0..08fa2417 100644 --- a/knot2/crates/knot-migrate/src/report.rs +++ b/knot2/crates/knot-migrate/src/report.rs @@ -61,6 +61,21 @@ impl Display for Report<'_> { "orphan owner markers on unknown repos: {}", drift.orphan_owner_markers.len() )?; + writeln!( + f, + "repos recorded with different owners in the acl and repo_keys: {}", + drift.conflicting_owner_markers.len() + )?; + drift + .conflicting_owner_markers + .iter() + .try_for_each(|conflict| { + writeln!( + f, + "{} acl {}, repo_keys {}", + conflict.repo, conflict.acl_owner, conflict.key_owner + ) + })?; writeln!( f, "extra acl owner markers dropped: {}", @@ -166,5 +181,8 @@ fn describe(reason: &SkipReason) -> String { ) } SkipReason::NoSourceRepo => "no git repository at the source path".to_string(), + SkipReason::UnreadableSource { kind } => { + format!("a source path that this process can't read: {kind}") + } } } diff --git a/knot2/crates/knot-migrate/tests/migrate.rs b/knot2/crates/knot-migrate/tests/migrate.rs index 65747861..f9ec86d4 100644 --- a/knot2/crates/knot-migrate/tests/migrate.rs +++ b/knot2/crates/knot-migrate/tests/migrate.rs @@ -7,6 +7,7 @@ use knot_migrate::casbin; use knot_migrate::emit::MasterKeyEnv; use knot_migrate::emit::{self, ConfigValues}; use knot_migrate::mapping::{self, SkipReason}; +use knot_migrate::report; use knot_migrate::source::{ SourceDb, SourceDid, SourceError, SourceRepoDid, SourceRkey, SourceSchema, }; @@ -168,6 +169,13 @@ fn fixture(with_collaborators_table: bool) -> Fixture { } fn map(fx: &Fixture) -> mapping::Mapping { + map_probing(fx, |did| adopt::probe_source(&fx.source_repos, did)) +} + +fn map_probing( + fx: &Fixture, + probe: impl Fn(&SourceRepoDid) -> adopt::SourceProbe, +) -> mapping::Mapping { let db = SourceDb::open(&fx.db_path).unwrap(); let repos = db.repos().unwrap(); let rkeys: BTreeMap = repos @@ -186,7 +194,6 @@ fn map(fx: &Fixture) -> mapping::Mapping { ) })); let acl = casbin::decode(&db.acl().unwrap(), &resolver).unwrap(); - let exists = |did: &SourceRepoDid| adopt::source_is_repo(&fx.source_repos, did); match db.schema().unwrap() { SourceSchema::Tables => mapping::map_tables( &repos, @@ -194,11 +201,11 @@ fn map(fx: &Fixture) -> mapping::Mapping { &db.members().unwrap(), &db.collaborators().unwrap(), &acl, - exists, + probe, ) .unwrap(), SourceSchema::PreFlip => { - mapping::map_preflip(&repos, &rkeys, &db.members().unwrap(), &acl, exists).unwrap() + mapping::map_preflip(&repos, &rkeys, &db.members().unwrap(), &acl, probe).unwrap() } } } @@ -713,3 +720,331 @@ fn host_key_import_preserves_every_algorithm() { ); }); } + +fn honors_permission_bits(dir: &Path) -> bool { + use std::os::unix::fs::PermissionsExt; + let probe = dir.join("permission-probe"); + std::fs::create_dir(&probe).unwrap(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o000)).unwrap(); + let denied = std::fs::read_dir(&probe).is_err(); + std::fs::set_permissions(&probe, std::fs::Permissions::from_mode(0o755)).unwrap(); + std::fs::remove_dir(&probe).unwrap(); + if !denied { + eprintln!( + "skipping the permission case: this process reads a 0000 directory, so it's running \ + as root" + ); + } + denied +} + +#[test] +fn probing_a_source_separates_a_repo_from_an_absence_and_from_an_unreadable_path() { + let fx = fixture(true); + std::fs::write(fx.source_repos.join("did:plc:mussel"), "").unwrap(); + let looped = fx.source_repos.join("did:plc:cuttle"); + std::os::unix::fs::symlink(&looped, &looped).unwrap(); + let probe = |did: &str| adopt::probe_source(&fx.source_repos, &srepo(did)); + assert_eq!(probe("did:plc:squid"), adopt::SourceProbe::Repo); + assert_eq!( + probe("did:plc:kelp"), + adopt::SourceProbe::Absent, + "a repo whose directory an operator deleted mustn't refuse every future migration" + ); + assert_eq!( + probe("did:plc:whelk"), + adopt::SourceProbe::Absent, + "did:plc:whelk has a readable directory without a HEAD, which the mapping skips on its own" + ); + assert_eq!( + probe("did:plc:mussel"), + adopt::SourceProbe::Absent, + "a regular file where a repo directory belongs isn't a repository either" + ); + assert!( + matches!(probe("did:plc:cuttle"), adopt::SourceProbe::Unreadable(_)), + "root can't step over a symlink loop, so this case covers the unreadable path under any uid" + ); +} + +#[test] +fn a_source_directory_that_the_process_cannot_enter_probes_unreadable() { + let fx = fixture(true); + if !honors_permission_bits(&fx.source_repos) { + return; + } + use std::os::unix::fs::PermissionsExt; + let squid = fx.source_repos.join("did:plc:squid"); + std::fs::set_permissions(&squid, std::fs::Permissions::from_mode(0o000)).unwrap(); + let probed = adopt::probe_source(&fx.source_repos, &srepo("did:plc:squid")); + std::fs::set_permissions(&squid, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!( + probed, + adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + "secure mode leaves repo trees at a per-owner uid, and that's how a wrong-user migration \ + sees them" + ); +} + +#[test] +fn an_unreadable_source_is_skipped_separately_from_an_absent_source() { + let fx = fixture(true); + let mapping = map_probing(&fx, |did| match did.as_str() { + "did:plc:squid" => adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + "did:plc:limpet" => { + adopt::SourceProbe::Unreadable(std::io::ErrorKind::StaleNetworkFileHandle) + } + _ => adopt::SourceProbe::Repo, + }); + let reason = |wanted: &str| { + mapping + .skipped + .iter() + .find(|skip| skip.repo_did.as_str() == wanted) + .map(|skip| skip.reason.clone()) + }; + assert_eq!( + reason("did:plc:squid"), + Some(SkipReason::UnreadableSource { + kind: std::io::ErrorKind::PermissionDenied + }), + "the report mustn't call an unreadable repo missing" + ); + assert_eq!( + reason("did:plc:limpet"), + Some(SkipReason::UnreadableSource { + kind: std::io::ErrorKind::StaleNetworkFileHandle + }), + "an error that isn't a permission error mustn't lose the repo either" + ); + assert_eq!( + reason("did:plc:kelp"), + None, + "did:plc:kelp has an alias and an acl grant without a repo_keys row" + ); + assert_eq!( + mapping.unreadable_sources().to_string(), + "did:plc:squid, did:plc:limpet" + ); + assert!(!mapping.unreadable_sources().is_empty()); + let readable = map(&fx); + assert!(readable.unreadable_sources().is_empty()); + assert!( + readable + .skipped + .iter() + .any(|skip| skip.reason == SkipReason::NoSourceRepo), + "did:plc:whelk has a directory without a HEAD in it, which stays a plain absence" + ); +} + +#[test] +fn naming_unreadable_repos_stops_at_five_and_counts_the_rest() { + let fx = fixture(true); + let mapping = map_probing(&fx, |_| { + adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied) + }); + let listed = mapping.unreadable_sources().to_string(); + assert!( + listed.starts_with( + "did:plc:squid, did:plc:limpet, did:plc:whelk, did:plc:nautilus, did:plc:scallop" + ), + "{listed}" + ); + assert!( + listed.ends_with(", and 1 more"), + "did:plc:conch and did:plc:clam are skipped before the probe, so 6 of the 8 rows are \ + unreadable: {listed}" + ); +} + +fn set_acl_owner(fx: &Fixture, repo: &str, acl_owner: &str) { + rusqlite::Connection::open(&fx.db_path) + .unwrap() + .execute( + "update acl set v0 = ?1 where v2 = ?2 and v3 = 'repo:owner'", + rusqlite::params![acl_owner, repo], + ) + .unwrap(); +} + +#[test] +fn two_owners_for_one_repo_are_drift_that_the_report_can_render() { + let fx = fixture(true); + set_acl_owner(&fx, "did:plc:scallop", "did:plc:teq"); + let mapping = map(&fx); + assert_eq!( + mapping.drift.conflicting_owner_markers, + vec![mapping::OwnerConflict { + repo: SourceRepoDid::from_column("did:plc:scallop"), + acl_owner: SourceDid::from_column("did:plc:teq"), + key_owner: SourceDid::from_column("did:plc:isabel"), + }] + ); + assert_eq!(mapping.conflicting_owners().to_string(), "did:plc:scallop"); + assert!( + !mapping + .drift + .extra_owner_markers + .iter() + .any(|(repo, _)| repo.as_str() == "did:plc:scallop"), + "a repo recorded with a different owner mustn't also appear as an extra marker: {:?}", + mapping.drift.extra_owner_markers + ); + let rendered = report::Report { + mapping: &mapping, + orphan_alias_count: 0, + adoption: None, + cobs: None, + } + .to_string(); + assert!( + rendered.contains("repos recorded with different owners in the acl and repo_keys: 1"), + "{rendered}" + ); + assert!( + rendered.contains("did:plc:scallop acl did:plc:teq, repo_keys did:plc:isabel"), + "{rendered}" + ); +} + +#[test] +fn an_owner_marker_beside_the_repo_keys_owner_is_still_an_extra() { + let mapping = map(&fixture(true)); + assert!( + mapping.drift.conflicting_owner_markers.is_empty(), + "did:plc:limpet is recorded with did:plc:bailey beside its repo_keys owner: {:?}", + mapping.drift.conflicting_owner_markers + ); + assert!(mapping.conflicting_owners().is_empty()); + assert!( + mapping.drift.extra_owner_markers.contains(&( + SourceRepoDid::from_column("did:plc:limpet"), + SourceDid::from_column("did:plc:bailey") + )), + "{:?}", + mapping.drift.extra_owner_markers + ); +} + +fn base64_standard(bytes: &[u8]) -> String { + use base64::Engine; + base64::engine::general_purpose::STANDARD.encode(bytes) +} + +fn host_key_file(dir: &Path) -> PathBuf { + let path = dir.join("ssh_host_ed25519_key"); + std::fs::write(&path, HOST_KEY).unwrap(); + path +} + +fn run_migrate(fx: &Fixture, host_key: &Path, extra: &[&str]) -> std::process::Output { + std::process::Command::new(env!("CARGO_BIN_EXE_knot-migrate")) + .args([ + "--source-db", + fx.db_path.to_str().unwrap(), + "--source-repos", + fx.source_repos.to_str().unwrap(), + "--host-key", + host_key.to_str().unwrap(), + "--hostname", + "knot.oyster.cafe", + "--plc-url", + "https://plc.directory", + "--target", + fx.target.to_str().unwrap(), + ]) + .args(extra) + .env("KNOT_MASTER_KEY", base64_standard(&[7_u8; 32])) + .output() + .unwrap() +} + +fn with_unreadable_repo(fx: &Fixture, work: impl FnOnce() -> T) -> Option { + use std::os::unix::fs::PermissionsExt; + if !honors_permission_bits(&fx.source_repos) { + return None; + } + let limpet = fx.source_repos.join("did:plc:limpet"); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o000)).unwrap(); + let outcome = work(); + std::fs::set_permissions(&limpet, std::fs::Permissions::from_mode(0o755)).unwrap(); + Some(outcome) +} + +#[test] +fn a_real_run_refuses_an_unreadable_source_until_it_is_told_to_skip_it() { + let fx = fixture(true); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + let Some((refused, target_after_refusal, outcome)) = with_unreadable_repo(&fx, || { + let refused = run_migrate(&fx, &host_key, &[]); + let target_after_refusal = fx.target.exists(); + ( + refused, + target_after_refusal, + run_migrate(&fx, &host_key, &["--skip-unreadable"]), + ) + }) else { + return; + }; + let refusal = String::from_utf8_lossy(&refused.stderr).to_string(); + assert!(!refused.status.success(), "{refusal}"); + assert!(refusal.contains("did:plc:limpet"), "{refusal}"); + assert!( + !target_after_refusal, + "a refused run mustn't leave a half-migrated target behind" + ); + let stdout = String::from_utf8_lossy(&outcome.stdout).to_string(); + let stderr = String::from_utf8_lossy(&outcome.stderr).to_string(); + assert!(outcome.status.success(), "{stdout}{stderr}"); + let adopted = map_probing(&fx, |did| match did.as_str() { + "did:plc:limpet" => adopt::SourceProbe::Unreadable(std::io::ErrorKind::PermissionDenied), + _ => adopt::probe_source(&fx.source_repos, did), + }) + .repos + .len(); + assert!( + stdout.contains(&format!("adopted by copy: {adopted} new")), + "{stdout}" + ); + assert!( + stdout.contains("a source path that this process can't read: permission denied"), + "{stdout}" + ); + assert!(fx.target.join("config.toml").is_file(), "{stdout}"); +} + +#[test] +fn a_run_refuses_two_owners_for_one_repo_after_it_reports_them() { + let fx = fixture(true); + set_acl_owner(&fx, "did:plc:scallop", "did:plc:teq"); + let dir = tempfile::tempdir().unwrap(); + let host_key = host_key_file(dir.path()); + [vec![], vec!["--dry-run"], vec!["--skip-unreadable"]] + .into_iter() + .for_each(|extra| { + let outcome = run_migrate(&fx, &host_key, &extra); + let stdout = String::from_utf8_lossy(&outcome.stdout).to_string(); + let stderr = String::from_utf8_lossy(&outcome.stderr).to_string(); + assert!(!outcome.status.success(), "{extra:?}: {stdout}{stderr}"); + assert!( + stdout.contains("did:plc:scallop acl did:plc:teq, repo_keys did:plc:isabel"), + "the report has to be on stdout before the refusal, {extra:?}: {stdout}" + ); + assert!( + stderr.contains( + "the acl and repo_keys are recorded with different owners for did:plc:scallop" + ), + "{extra:?}: {stderr}" + ); + assert!( + stderr.contains("repo_keys.owner_did"), + "the refusal has to point at the column to fix, {extra:?}: {stderr}" + ); + assert!( + !fx.target.exists(), + "a refused run mustn't leave a half-migrated target behind, {extra:?}" + ); + }); +} -- 2.51.2