Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
15 kB · 443 lines
Rust
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444//! repo provisioning: import / create with the exact PLC genesis transaction.
use std::path::{Path, PathBuf};use std::sync::Arc;
use anyhow::Context as _;use base64::Engine as _;use knot_cobs::Registration;use knot_types::{BranchName, KnotServiceUrl, OwnerDid, RepoDid, RepoName, RepoRkey, UnixSeconds};
use crate::cli::{CreateArgs, ImportArgs};use crate::config::ValidatedConfig;use crate::registry::{ PendingGenesis, drop_pending_genesis, enqueue_pending_genesis, load_pending_genesis, verify_did_document,};
enum ProvisionSource { Create, Import(PathBuf),}
pub async fn import(config: Arc<ValidatedConfig>, args: ImportArgs) -> anyhow::Result<()> { let lfs_pointers = detect_lfs_pointers(&args.path)?; if !lfs_pointers.is_empty() { anyhow::ensure!( args.allow_lfs_pointers, "detected {} Git LFS pointer file(s) (e.g. {}). Static knots serve the \ pointers as plain content; standard `git lfs pull` will not work. \ Re-run with --allow-lfs-pointers to import anyway.", lfs_pointers.len(), lfs_pointers.first().map_or("?", String::as_str) ); eprintln!( "warning: importing {} Git LFS pointer file(s); the static knot serves \ pointers as plain content", lfs_pointers.len() ); } let repo_did = provision_repo( config, args.owner, args.rkey, args.name, args.default_branch, args.source, ProvisionSource::Import(args.path), ) .await?; println!("{repo_did}"); Ok(())}
const LFS_POINTER_PREFIX: &str = "version https://git-lfs";const LFS_SCAN_MAX_BLOB_BYTES: u64 = 4096;
/// scan a source repository (worktree or bare) for git LFS pointer files:/// small text blobs starting with the LFS version URL. build-time only.fn detect_lfs_pointers(path: &std::path::Path) -> anyhow::Result<Vec<String>> { let run_git = |git_args: &[&str]| -> anyhow::Result<String> { let output = std::process::Command::new("git") .arg("-C") .arg(path) .args(git_args) .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_SYSTEM", "/dev/null") .env("GIT_TERMINAL_PROMPT", "0") .output() .context("run git for LFS scan")?; if !output.status.success() { anyhow::bail!( "git {} failed: {}", git_args.first().unwrap_or(&""), String::from_utf8_lossy(&output.stderr) ); } Ok(String::from_utf8_lossy(&output.stdout).to_string()) };
// worktree: grep is exact and cheap if path.join(".git").exists() || path.join("HEAD").exists() { if let Ok(out) = run_git(&["grep", "-I", "-l", LFS_POINTER_PREFIX, "--", "."]) { let hits: Vec<String> = out.lines().map(str::to_string).collect(); if !hits.is_empty() { return Ok(hits); } } }
// bare repo or otherwise: scan small blobs by content let mut pointers = Vec::new(); let listing = run_git(&[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype) %(objectsize)", ])?; for line in listing.lines() { let mut parts = line.split_whitespace(); let (Some(oid), Some("blob"), Some(size_str)) = (parts.next(), parts.next(), parts.next()) else { continue; }; let Ok(size) = size_str.parse::<u64>() else { continue; }; if !(1..=LFS_SCAN_MAX_BLOB_BYTES).contains(&size) { continue; } let content = run_git(&["cat-file", "blob", oid])?; if content.starts_with(LFS_POINTER_PREFIX) { pointers.push(oid.to_string()); } } Ok(pointers)}
pub async fn create(config: Arc<ValidatedConfig>, args: CreateArgs) -> anyhow::Result<()> { let repo_did = provision_repo( config, args.owner, args.rkey, args.name, args.default_branch, None, ProvisionSource::Create, ) .await?; println!("{repo_did}"); Ok(())}
/// provision an empty repo for an inbound ssh push (auto-provisioning on/// first push); returns the minted repo DID. resumable via the pending/// genesis journal when the PLC submission fails midway.pub(crate) async fn create_for_push( config: Arc<ValidatedConfig>, raw_rkey: String, raw_name: Option<String>,) -> anyhow::Result<RepoDid> { let owner = config.owner.as_str().to_string(); provision_repo( config, owner, raw_rkey, raw_name, None, None, ProvisionSource::Create, ) .await}
async fn provision_repo( config: Arc<ValidatedConfig>, raw_owner: String, raw_rkey: String, raw_name: Option<String>, default_branch: Option<String>, source_annotation: Option<String>, source: ProvisionSource,) -> anyhow::Result<RepoDid> { let owner = OwnerDid::new(&raw_owner).context("invalid owner DID")?; let rkey = RepoRkey::new(&raw_rkey).context("invalid repo rkey")?; let name = match raw_name { Some(name) => RepoName::new(name).context("invalid repo name")?, None => RepoName::new(raw_rkey.clone()).context("invalid repo name from rkey")?, };
let layout = config.layout()?; let meta_path = layout.meta_path(&config.knot_did)?; let meta_repo = layout .bootstrap_meta(&config.knot_did) .context("bootstrap meta.git")?; let index = knot_index::Index::new(&meta_path, layout.clone()); let _ = index.rebuild();
let existing_resolved = match index.resolve_repo(&owner, &rkey) { knot_index::Resolved::Ready(Some(did)) => Some(did), _ => None, }; let pending_entries = load_pending_genesis(&meta_repo)?; let pending_match = pending_entries.iter().find(|entry| { (entry.owner.as_ref() == Some(&owner) && entry.rkey.as_ref() == Some(&rkey)) || existing_resolved.as_ref() == Some(&entry.did) });
if pending_match.is_none() { if let Some(existing_did) = existing_resolved { anyhow::bail!("repository ({owner}, {rkey}) already hosted as {existing_did}"); } }
let master_key = knot_secrets::MasterKey::new( base64::engine::general_purpose::STANDARD .decode( std::env::var(&config.master_key_env) .with_context(|| format!("read master key from {}", config.master_key_env))? .trim(), ) .context("decode master key as base64")?, ) .context("load master key")?;
let secrets = knot_secrets::SealedStore::open( &config.sealed_key_file, &master_key, Box::new(knot_runtime::OsEntropy), ) .context("open sealed key store")?;
secrets .ensure(&config.knot_did) .context("ensure knot signing key")?;
let knot_signer = secrets .signer(&config.knot_did) .context("get knot signer")?;
let service_url = KnotServiceUrl::new(format!("https://{}", config.hostname)).context("knot service url")?;
let (seed, created_at, pending_op) = match pending_match { Some(entry) => ( entry.seed, entry.created_at, Some((entry.did.clone(), entry.operation.clone())), ), None => { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap(); (now.as_nanos() as u64, now.as_secs() as i64, None) } };
let entropy = knot_runtime::SeededEntropy::new(seed); let nonce = knot_atproto::MintNonce::mint(&entropy, &owner, &rkey); let prepared = knot_atproto::prepare_repo_did(&knot_signer, &service_url, &nonce).context( if pending_match.is_some() { "prepare repo did on resume" } else { "prepare repo did" }, )?;
let (repo_did, operation) = match pending_op { Some((did, op)) => (did, op), None => (prepared.did.clone(), prepared.operation().clone()), };
let guarded_path = layout.guarded_path(&repo_did)?; if guarded_path.exists() { knot_git::Repo::open(&guarded_path).context("open existing repo on resume")?; } else { match source { ProvisionSource::Create => { let repo = layout.create(&repo_did).context("create bare repo")?; if let Some(branch_str) = &default_branch { let branch = BranchName::new(branch_str.clone()) .context("invalid default branch name")?; repo.set_head(&branch.head_ref()) .context("set repo head to default branch")?; } } ProvisionSource::Import(source_path) => { let git_dir = resolve_git_dir(&source_path) .with_context(|| format!("resolve git dir for {}", source_path.display()))?; knot_git::Repo::open(&git_dir).with_context(|| { format!("verify knot_git::Repo opens {}", git_dir.display()) })?;
let temp_dir = tempfile::Builder::new() .prefix(".import-staging-") .tempdir_in(&config.scan_path) .context("create staging tempdir in scan path")?;
copy_dir_recursive(&git_dir, temp_dir.path()).context("stage git repository")?;
let staged_repo = knot_git::Repo::open(temp_dir.path()) .context("verify knot_git::Repo opens staged repository")?;
if let Some(branch_str) = &default_branch { let branch = BranchName::new(branch_str.clone()) .context("invalid default branch name")?; staged_repo .set_head(&branch.head_ref()) .context("set repo head to default branch")?; }
if let Some(parent) = guarded_path.parent() { std::fs::create_dir_all(parent) .context("create parent directory for guarded path")?; }
std::fs::rename(temp_dir.path(), &guarded_path) .context("move staged repository to guarded path")?; let _ = temp_dir.keep(); } } }
let cob_store = knot_cob::CobStore::new(&meta_repo); let cob_home = knot_cob::CobHome::from(&config.knot_did); let now = UnixSeconds::new(created_at); let registration = Registration { owner: owner.clone(), rkey: rkey.clone(), name: name.clone(), repo: repo_did.clone(), created_at: now, policy: knot_types::ContributionPolicy::default(), };
register_in_meta(&cob_store, &cob_home, registration, &knot_signer, now)?;
// persist PendingGenesis journal entry before the network side effect. let entry = PendingGenesis { did: repo_did.clone(), operation: operation.clone(), owner: Some(owner.clone()), rkey: Some(rkey.clone()), name: Some(name.to_string()), source: source_annotation, seed, created_at, }; enqueue_pending_genesis(&meta_repo, entry)?;
let transport = knot_runtime::ReqwestHttp::new(knot_runtime::HttpLimits::default()) .context("init http transport")?;
let atproto = knot_atproto::Atproto::new( transport, knot_runtime::SystemClock, config.knot_did.clone(), knot_atproto::PlcDirectory::new(config.plc_directory.clone())?, );
atproto .submit_plc_operation(&prepared.did.clone(), &operation) .await .context("submit plc operation")?;
let verify_transport = knot_runtime::ReqwestHttp::new(knot_runtime::HttpLimits::default()) .context("init verify http transport")?;
verify_did_document( &verify_transport, &config.plc_directory, &repo_did, &config.hostname, ) .await .context("verify live repo DID document on PLC")?;
drop_pending_genesis(&meta_repo, &repo_did)?;
crate::mirror::trigger_repo_resync(&crate::mirror::triggers_from_config(&config), &repo_did) .await;
let fresh_index = knot_index::Index::new(&meta_path, layout.clone()); fresh_index .rebuild() .context("rebuild fresh index after provisioning")?;
match fresh_index.ownership_of(&repo_did) { knot_index::Resolved::Ready(Some(ref_info)) if ref_info.owner == owner && ref_info.rkey == rkey => {} knot_index::Resolved::Ready(Some(ref_info)) => { anyhow::bail!( "ownership mismatch for {repo_did}: expected ({owner}, {rkey}), found ({}, {})", ref_info.owner, ref_info.rkey ); } other => { anyhow::bail!("failed to resolve ownership of {repo_did} in fresh index: {other:?}"); } }
Ok(repo_did)}
fn register_in_meta( store: &knot_cob::CobStore, home: &knot_cob::CobHome, registration: Registration, signer: &dyn knot_runtime::Signer, now: UnixSeconds,) -> anyhow::Result<()> { match store.list::<knot_cobs::RepoRegistryCob>()?.as_slice() { [] => store .create( home, &knot_cobs::RegistryChange::Register(registration), signer, now, ) .map(|_| ()) .map_err(|e| anyhow::anyhow!("create registry cob: {e}")), [object] => knot_cobs::register_repo(store, home, *object, registration, signer, now) .map(|_| ()) .map_err(|e| anyhow::anyhow!("register repo in cob: {e}")), many => anyhow::bail!("{} repo registry objects share meta-repo", many.len()), }}
fn resolve_git_dir(path: &Path) -> anyhow::Result<PathBuf> { let dot_git = path.join(".git"); if dot_git.is_dir() { Ok(dot_git) } else if dot_git.is_file() { let content = std::fs::read_to_string(&dot_git)?; let rest = content .strip_prefix("gitdir:") .ok_or_else(|| anyhow::anyhow!("invalid .git file at {}", dot_git.display()))?; let p = PathBuf::from(rest.trim()); Ok(if p.is_relative() { path.join(p) } else { p }) } else if path.join("HEAD").is_file() { Ok(path.to_path_buf()) } else { anyhow::bail!("no git repository found at {}", path.display()); }}
fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> { std::fs::create_dir_all(dst)?; for entry in std::fs::read_dir(src)? { let entry = entry?; let ft = entry.file_type()?; let target = dst.join(entry.file_name()); if ft.is_dir() { copy_dir_recursive(&entry.path(), &target)?; } else if ft.is_file() { std::fs::copy(entry.path(), &target)?; } else if ft.is_symlink() { let link_target = std::fs::read_link(entry.path())?; #[cfg(unix)] std::os::unix::fs::symlink(&link_target, &target)?; } } Ok(())}