Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
31 kB · 797 lines
Rust
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798//! sanitized public-only dumb-HTTP git export (packet §17).//!//! produces a bare repository containing only objects reachable from public refs//! advertised for upload. reserved refs (cobs, hidden, atproto, cob-checkpoints)//! and unreachable objects are strictly excluded.
use std::collections::{BTreeSet, HashMap};use std::path::Path;use std::process::{Command, Output};
use anyhow::Context as _;use sha2::{Digest as _, Sha256};
use crate::config::ValidatedConfig;use crate::snapshot::RepoSnapshot;
/// deterministic fingerprint for a repo export.////// H(generator profile || repo identity || HEAD || sorted public refs || object format) via sha2pub fn export_fingerprint(snapshot: &RepoSnapshot, profile: &str) -> String { let mut hasher = Sha256::new(); hasher.update(profile.as_bytes()); hasher.update(b"\0"); hasher.update(snapshot.repo_did.as_str().as_bytes()); hasher.update(b"\0"); hasher.update(snapshot.owner.as_str().as_bytes()); hasher.update(b"\0"); hasher.update(snapshot.rkey.as_str().as_bytes()); hasher.update(b"\0"); if let Some(name) = &snapshot.name { hasher.update(name.as_str().as_bytes()); } hasher.update(b"\0"); if let Some(head) = &snapshot.head { hasher.update(head.as_bytes()); } hasher.update(b"\0"); let mut sorted_refs = snapshot.public_refs.clone(); sorted_refs.sort(); for (name, oid) in sorted_refs { hasher.update(name.as_bytes()); hasher.update(b" "); hasher.update(oid.as_bytes()); hasher.update(b"\n"); } hasher.update(b"\0"); hasher.update(snapshot.object_format.capability().as_bytes()); let hash = hasher.finalize(); hash.iter().map(|byte| format!("{byte:02x}")).collect()}
/// export a sanitized bare git repo for dumb-HTTP serving.////// packet §17 steps:/// 1. only refs from snapshot.public_refs are exported./// 2. initializes bare repo with matching object format./// 3. fetches public refs via file:// without local clone optimization./// 4. sets HEAD to the branch containing snapshot.head if any, else the public default branch, else unsets./// 5. expires reflogs and removes FETCH_HEAD and temporary artifacts./// 6. keeps fetched objects loose (CAS dedup unit) and prunes unreachable objects./// 7. updates server info (info/refs and objects/info/packs)./// 8. asserts info/refs contains exactly the intended refs and no reserved refs./// 9. asserts objects/info/alternates does not exist./// 10. runs git fsck --unreachable --no-reflogs to verify no unreachable objects remain./// writes `export.fingerprint` into export_dir.pub fn export_repo( config: &ValidatedConfig, snapshot: &RepoSnapshot, export_dir: &Path,) -> anyhow::Result<()> { if export_dir.exists() { std::fs::remove_dir_all(export_dir) .with_context(|| format!("clean existing export dir {}", export_dir.display()))?; } std::fs::create_dir_all(export_dir) .with_context(|| format!("create export dir {}", export_dir.display()))?;
if let Some((name, _)) = snapshot .public_refs .iter() .find(|(name, _)| is_reserved_ref(name)) { anyhow::bail!("snapshot public_refs contains reserved ref: {name}"); }
let capability = snapshot.object_format.capability(); let mut init_cmd = git_command(export_dir); init_cmd.args(["init", "--bare", &format!("--object-format={capability}")]); run_cmd(init_cmd, "git init --bare")?;
// copy only captured refs via transport that avoids whole-store cloning. let layout = config.layout().context("open layout for repo export")?; let source_path = layout .guarded_path(&snapshot.repo_did) .context("source repo path")?; if !source_path.exists() { anyhow::bail!("source repository not found at {}", source_path.display()); } let canonical_source = source_path .canonicalize() .with_context(|| format!("canonicalize source repo path {}", source_path.display()))?; let source_url = format!("file://{}", canonical_source.display());
// unpackLimit=1 forces loose objects; object files are the CAS dedup unit. for chunk in snapshot.public_refs.chunks(100) { let mut fetch_cmd = git_command(export_dir); fetch_cmd.args([ "-c", "fetch.unpackLimit=1", "-c", "transfer.unpackLimit=1", "fetch", "--no-tags", &source_url, ]); fetch_cmd.args(chunk.iter().map(|(r, _)| format!("{r}:{r}"))); run_cmd(fetch_cmd, "git fetch public refs")?; }
set_export_head(config, snapshot, export_dir)?;
let mut expire_cmd = git_command(export_dir); expire_cmd.args(["reflog", "expire", "--expire=now", "--all"]); let _ = expire_cmd.output();
let _ = std::fs::remove_dir_all(export_dir.join("logs")); for file in ["FETCH_HEAD", "ORIG_HEAD"] { let _ = std::fs::remove_file(export_dir.join(file)); }
// loose objects are the CAS dedup unit; prune unreachable objects without repacking. let mut prune_cmd = git_command(export_dir); prune_cmd.args(["prune", "--expire=now"]); run_cmd(prune_cmd, "git prune --expire=now")?;
let mut update_cmd = git_command(export_dir); update_cmd.arg("update-server-info"); run_cmd(update_cmd, "git update-server-info")?;
assert_info_refs(snapshot, export_dir)?;
let alternates = export_dir.join("objects/info/alternates"); if alternates.exists() { anyhow::bail!( "objects/info/alternates exists in export at {}", alternates.display() ); }
assert_fsck_clean(export_dir)?;
// precompute smart-HTTP advertisement and want-all pack response for static serving. write_smart_artifacts(snapshot, export_dir)?;
let fingerprint = export_fingerprint(snapshot, crate::manifest::GENERATOR_PROFILE_VERSION); let fp_path = export_dir.join("export.fingerprint"); std::fs::write(&fp_path, format!("{fingerprint}\n")) .with_context(|| format!("write export.fingerprint at {}", fp_path.display()))?;
Ok(())}
fn pkt_line(data: &[u8]) -> Vec<u8> { let mut out = format!("{:04x}", data.len() + 4).into_bytes(); out.extend_from_slice(data); out}
/// capture the deterministic smart-HTTP conversation for this frozen export.////// emits `smart/advertisement` (the `?service=git-upload-pack` response including/// the pkt-line service preamble) and `smart/upload-pack` (response to a want-everything/// fetch with no haves: full history, framed with side-band-64k + no-progress,/// the caps every real git client sends).fn write_smart_artifacts(snapshot: &RepoSnapshot, export_dir: &Path) -> anyhow::Result<()> { anyhow::ensure!( !snapshot.public_refs.is_empty(), "smart export requires at least one public ref" ); let smart_dir = export_dir.join("smart"); std::fs::create_dir_all(&smart_dir) .with_context(|| format!("create {}", smart_dir.display()))?;
// advertisement = service preamble + raw `git upload-pack --advertise-refs` let mut adv_cmd = git_command(export_dir); adv_cmd.args(["upload-pack", "--stateless-rpc", "--advertise-refs", "."]); let adv = run_cmd(adv_cmd, "git upload-pack --advertise-refs")?; let mut advertisement = pkt_line(b"# service=git-upload-pack\n"); advertisement.extend_from_slice(b"0000"); advertisement.extend_from_slice(&adv.stdout); for (name, oid) in &snapshot.public_refs { anyhow::ensure!( advertisement .windows(oid.len()) .any(|window| window == oid.as_bytes()), "smart advertisement is missing ref {name}" ); anyhow::ensure!( advertisement .windows(name.len()) .any(|window| window == name.as_bytes()), "smart advertisement is missing ref name {name}" ); } std::fs::write(smart_dir.join("advertisement"), &advertisement) .with_context(|| format!("write {}/advertisement", smart_dir.display()))?;
// protocol v0: want every public tip (caps on first want line), flush, then done. let mut request: Vec<u8> = snapshot .public_refs .iter() .enumerate() .flat_map(|(idx, (_, oid))| { let line = if idx == 0 { format!("want {oid} ofs-delta side-band-64k no-progress agent=knot-static\n") } else { format!("want {oid}\n") }; pkt_line(line.as_bytes()) }) .collect(); request.extend_from_slice(b"0000"); request.extend_from_slice(&pkt_line(b"done\n"));
// temp file avoids pipe-write timing issues with upload-pack. let request_path = std::env::temp_dir().join(format!( "knot-static-upload-pack-request-{}", std::process::id() )); std::fs::write(&request_path, &request) .with_context(|| format!("write {}", request_path.display()))?;
let req_file = std::fs::File::open(&request_path).context("open request file")?; let mut pack_cmd = git_command(export_dir); pack_cmd.args(["upload-pack", "--stateless-rpc", "."]); pack_cmd.stdin(std::process::Stdio::from(req_file)); let pack_output = pack_cmd.output().context("run git upload-pack fetch")?; anyhow::ensure!( pack_output.status.success(), "git upload-pack fetch failed (code {:?}): {} {}", pack_output.status.code(), String::from_utf8_lossy(&pack_output.stderr), String::from_utf8_lossy(&pack_output.stdout), ); anyhow::ensure!( !pack_output.stdout.is_empty(), "empty smart upload-pack response (code {:?}, stderr={})", pack_output.status.code(), String::from_utf8_lossy(&pack_output.stderr), ); std::fs::write(smart_dir.join("upload-pack"), &pack_output.stdout) .with_context(|| format!("write {}/upload-pack", smart_dir.display()))?;
Ok(())}
fn set_export_head( config: &ValidatedConfig, snapshot: &RepoSnapshot, export_dir: &Path,) -> anyhow::Result<()> { let default_ref = format!("refs/heads/{}", config.default_branch.as_str());
let target_ref = snapshot.head.as_deref().and_then(|head_oid| { if snapshot .public_refs .iter() .any(|(r, oid)| r == &default_ref && oid == head_oid) { Some(default_ref.clone()) } else if let Some((branch_ref, _)) = snapshot .public_refs .iter() .find(|(r, oid)| r.starts_with("refs/heads/") && oid == head_oid) { Some(branch_ref.clone()) } else { let mut cmd = git_command(export_dir); cmd.args(["branch", "--contains", head_oid]); let output = cmd.output().ok()?; output.status.success().then_some(())?; let stdout = String::from_utf8_lossy(&output.stdout); let mut branches = stdout .lines() .map(|l| l.trim().trim_start_matches('*').trim()) .filter(|l| !l.is_empty());
if branches .clone() .any(|b| b == config.default_branch.as_str()) { Some(default_ref.clone()) } else { branches.next().map(|first| format!("refs/heads/{first}")) } } });
let chosen_ref = target_ref.or_else(|| { snapshot .public_refs .iter() .any(|(r, _)| r == &default_ref) .then_some(default_ref) });
if let Some(target) = chosen_ref { let mut symref_cmd = git_command(export_dir); symref_cmd.args(["symbolic-ref", "HEAD", &target]); run_cmd(symref_cmd, "git symbolic-ref HEAD")?; }
Ok(())}
fn is_reserved_ref(raw: &str) -> bool { if !knot_git::RefClass::of_revspec(raw).is_public() { return true; } let stripped = raw.strip_prefix("refs/").unwrap_or(raw); const RESERVED: &[&str] = &["cobs", "cob-checkpoints", "hidden", "atproto"]; RESERVED.iter().any(|&prefix| { stripped == prefix || stripped .strip_prefix(prefix) .is_some_and(|rest| rest.starts_with('/')) })}
fn assert_info_refs(snapshot: &RepoSnapshot, export_dir: &Path) -> anyhow::Result<()> { let info_refs_path = export_dir.join("info/refs");
if snapshot.public_refs.is_empty() { if info_refs_path.exists() { let content = std::fs::read_to_string(&info_refs_path)?; anyhow::ensure!( content.trim().is_empty(), "expected empty info/refs for repo with no public refs" ); } return Ok(()); }
anyhow::ensure!( info_refs_path.exists(), "info/refs was not created at {}", info_refs_path.display() );
let content = std::fs::read_to_string(&info_refs_path) .with_context(|| format!("read {}", info_refs_path.display()))?;
let mut found_refs = BTreeSet::new();
for line in content.lines().map(str::trim).filter(|l| !l.is_empty()) { let Some((oid, raw_ref)) = line.split_once('\t') else { anyhow::bail!("malformed info/refs line: {line:?}"); }; let (base_ref, is_peeled) = raw_ref .strip_suffix("^{}") .map_or((raw_ref, false), |base| (base, true));
anyhow::ensure!( !is_reserved_ref(base_ref), "info/refs contains reserved ref: {raw_ref}" );
let Some((_, expected_oid)) = snapshot .public_refs .iter() .find(|(name, _)| name == base_ref) else { anyhow::bail!("info/refs contains unexpected ref not in snapshot: {base_ref}"); }; if !is_peeled && oid != expected_oid { anyhow::bail!("info/refs entry {base_ref} points to {oid}, expected {expected_oid}"); }
found_refs.insert(base_ref.to_string()); }
if let Some((expected_name, _)) = snapshot .public_refs .iter() .find(|(name, _)| !found_refs.contains(name)) { anyhow::bail!("info/refs missing expected public ref: {expected_name}"); }
Ok(())}
fn assert_fsck_clean(export_dir: &Path) -> anyhow::Result<()> { let mut fsck_cmd = git_command(export_dir); fsck_cmd.args(["fsck", "--unreachable", "--no-reflogs"]); let output = fsck_cmd.output().context("execute git fsck")?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); anyhow::bail!( "git fsck failed with exit code {}: stderr={stderr} stdout={stdout}", output.status ); }
let stdout = String::from_utf8_lossy(&output.stdout); let bad_lines: Vec<_> = stdout .lines() .map(str::trim) .filter(|line| { !line.is_empty() && (line.contains("unreachable") || line.contains("dangling") || line.contains("missing")) }) .collect();
if !bad_lines.is_empty() { anyhow::bail!( "git fsck found unexpected objects in export: {}", bad_lines.join("; ") ); }
Ok(())}
fn git_command(cwd: &Path) -> Command { let mut command = Command::new("git"); command .current_dir(cwd) .env("GIT_CONFIG_GLOBAL", "/dev/null") .env("GIT_CONFIG_SYSTEM", "/dev/null") .env("GIT_CONFIG_NOSYSTEM", "1") .env("GIT_CONFIG_COUNT", "2") .env("GIT_CONFIG_KEY_0", "maintenance.auto") .env("GIT_CONFIG_VALUE_0", "false") .env("GIT_CONFIG_KEY_1", "gc.autoDetach") .env("GIT_CONFIG_VALUE_1", "false") .env("GIT_TERMINAL_PROMPT", "0") .env("GIT_ASKPASS", "true") .env("LC_ALL", "C"); command}
fn run_cmd(mut cmd: Command, desc: &str) -> anyhow::Result<Output> { let output = cmd .output() .with_context(|| format!("failed to spawn {desc}"))?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); let stdout = String::from_utf8_lossy(&output.stdout); anyhow::bail!( "{desc} failed (status {}): {stderr} {stdout}", output.status ); } Ok(output)}
#[cfg(test)]mod smart_tests { use super::*; use std::io::Read as _; use std::net::{TcpListener, TcpStream}; use std::thread;
fn git(dir: &Path, args: &[&str]) -> String { let mut cmd = git_command(dir); cmd.args(args); let output = run_cmd(cmd, &format!("git {}", args.join(" "))).expect("git"); String::from_utf8_lossy(&output.stdout).trim().to_string() }
fn commit(dir: &Path, path: &str, body: &str, message: &str) { if let Some(parent) = std::path::Path::new(path).parent() { std::fs::create_dir_all(dir.join(parent)).unwrap(); } std::fs::write(dir.join(path), body).unwrap(); git(dir, &["add", "."]); let mut cmd = git_command(dir); cmd.args(["commit", "-q", "-m", message]); cmd.env("GIT_AUTHOR_NAME", "t") .env("GIT_AUTHOR_EMAIL", "t@t"); cmd.env("GIT_COMMITTER_NAME", "t") .env("GIT_COMMITTER_EMAIL", "t@t"); run_cmd(cmd, "git commit").unwrap(); }
fn snapshot(source: &Path, refs: Vec<(String, String)>) -> RepoSnapshot { RepoSnapshot { repo_did: knot_types::RepoDid::new("did:plc:smarttest").unwrap(), owner: knot_types::OwnerDid::new("did:plc:owner").unwrap(), rkey: knot_types::RepoRkey::new("smart").unwrap(), name: None, head: Some(git(&source, &["rev-parse", "HEAD"])), public_refs: refs, object_format: knot_types::ObjectFormat::SHA1, } }
/// serves only the two smart artifacts like the generated nginx does, then /// clones through it with real git — that's the knotmirror contract. #[test] fn smart_export_serves_a_real_clone() -> anyhow::Result<()> { // fixed path so failures can be post-mortemed under /tmp/ks-smart-e2e let root: &std::path::Path = std::path::Path::new("/tmp/ks-smart-e2e"); let _ = std::fs::remove_dir_all(root); std::fs::create_dir_all(root).unwrap(); // export_repo resolves via Layout::guarded_path(repo_did): // scan_path/<did-path-segments> = <root>/plc/sm/arttest for did:plc:smarttest. let source = root.join("plc").join("sm").join("arttest"); std::fs::create_dir_all(&source).unwrap(); git(&source, &["init", "-q", "-b", "main"]); commit(&source, "hello.txt", "hi\n", "one"); commit(&source, "src/deep/file.txt", "deep\n", "two"); git(&source, &["branch", "feature"]); let main_tip = git(&source, &["rev-parse", "main"]); let feature_tip = git(&source, &["rev-parse", "feature"]); let refs = vec![ ("refs/heads/feature".to_string(), feature_tip.clone()), ("refs/heads/main".to_string(), main_tip.clone()), ];
let export_dir = root.join("export"); let snap = snapshot(&source, refs); export_repo_test(root, &snap, &export_dir);
let advertisement = std::fs::read(export_dir.join("smart/advertisement")).unwrap(); let pack = std::fs::read(export_dir.join("smart/upload-pack")).unwrap(); assert!( advertisement.starts_with(b"001e# service=git-upload-pack\n0000"), "advertisement must open with the service preamble" ); assert!(windows_contains(&advertisement, main_tip.as_bytes())); assert!( windows_contains(&pack, b"PACK"), "pack response must carry a packfile" );
// loopback HTTP server: GET /x/info/refs?service=git-upload-pack → advertisement, // POST /x/git-upload-pack → pack. anything else: 403. let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let adv = advertisement.clone(); let server = thread::spawn(move || { // git opens one connection per request; serve until the clone finishes for stream in listener.incoming() { let Ok(mut stream) = stream else { break }; let mut buf = [0u8; 8192]; let n = stream.read(&mut buf).unwrap_or(0); let req = String::from_utf8_lossy(&buf[..n]).to_string(); let head_end = req.find("\r\n\r\n").map(|i| i + 4).unwrap_or(n); if req.starts_with("GET /__done ") { break; } if req.starts_with("GET /x/info/refs?service=git-upload-pack ") { let body = adv.clone(); let _ = stream.write_all( format!( "HTTP/1.1 200 OK\r\nContent-Type: application/x-git-upload-pack-advertisement\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() ) .as_bytes(), ); let _ = stream.write_all(&body); } else if req.starts_with("POST /x/git-upload-pack ") { // read to end of body (git sends Content-Length) let cl: usize = req .lines() .find_map(|l| { l.to_ascii_lowercase() .strip_prefix("content-length: ") .and_then(|v| v.trim().parse().ok()) }) .unwrap_or(0); let mut body = buf[head_end..].to_vec(); while body.len() < cl { match stream.read(&mut buf) { Ok(0) => break, Ok(m) => body.extend_from_slice(&buf[..m]), Err(_) => break, } } let _ = stream.write_all( format!( "HTTP/1.1 200 OK\r\nContent-Type: application/x-git-upload-pack-result\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", pack.len() ) .as_bytes(), ); let _ = stream.write_all(&pack); } else { let _ = stream.write_all( b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", ); } } });
let clone_dir = root.join("clone"); let mut clone_cmd = git_command(root); clone_cmd.args([ "clone", "-q", &format!("http://{addr}/x"), clone_dir.to_str().unwrap(), ]); let clone_result = run_cmd(clone_cmd, "git clone over smart http"); // sentinel: stop the accept loop, then reap the thread let mut done_stream = TcpStream::connect(addr).context("connect shutdown sentinel")?; use std::io::Write as _; let _ = done_stream.write_all(b"GET /__done HTTP/1.1\r\nHost: x\r\n\r\n"); server.join().unwrap(); clone_result?; let clone_log = git(&clone_dir, &["log", "--oneline"]); assert_eq!(clone_log.lines().count(), 2, "both commits fetched"); assert!(clone_dir.join("src/deep/file.txt").exists()); let fetched_feature = git(&clone_dir, &["rev-parse", "origin/feature"]); assert_eq!(fetched_feature, feature_tip); let _ = std::fs::remove_dir_all("/tmp/ks-smart-e2e"); Ok(()) }
fn windows_contains(haystack: &[u8], needle: &[u8]) -> bool { haystack .windows(needle.len()) .any(|window| window == needle) }
fn export_repo_test(scan_root: &Path, snap: &RepoSnapshot, export_dir: &Path) { let config = test_config(scan_root); super::export_repo(&config, snap, export_dir).unwrap(); }
fn test_config(scan_root: &Path) -> crate::config::ValidatedConfig { crate::config::ValidatedConfig { config_path: scan_root.join("knot-static.toml"), publish: Vec::new(), hostname: knot_types::KnotHostname::new("knot.example").unwrap(), knot_did: knot_types::KnotId::new("did:web:knot.example").unwrap(), owner: knot_types::AccountDid::new("did:plc:owner").unwrap(), scan_path: scan_root.to_path_buf(), default_branch: knot_types::BranchName::new("main").unwrap(), object_format: knot_types::ObjectFormat::SHA1, sealed_key_file: scan_root.join("nope.sealed"), master_key_env: "X".to_string(), plc_directory: url::Url::parse("http://127.0.0.1:9").unwrap(), output_root: scan_root.to_path_buf(), keep_generations: 1, profile: crate::config::Profile::default(), mirror: None, activate: crate::config::ActivateConfig::default(), } }}
/// incremental export: seeds bare repo from prior generation if available,/// fetches only changed objects, and maintains an `objects.cas` cache so/// unchanged objects are not re-read or re-hashed.pub fn export_repo_incremental( config: &ValidatedConfig, snapshot: &RepoSnapshot, export_dir: &Path, prior_export_dir: Option<&Path>, cas: &crate::cas::Cas,) -> anyhow::Result<Vec<String>> { let mut cas_cache: HashMap<String, String> = HashMap::new();
if let Some(prior) = prior_export_dir && prior.exists() && prior.join("objects").is_dir() { let _ = crate::generate::clone_dir(prior, export_dir); if let Ok(cached) = std::fs::read_to_string(export_dir.join("objects.cas")) { for line in cached.lines() { if let Some((oid, digest)) = line.split_once(' ') { cas_cache.insert(oid.trim().to_string(), digest.trim().to_string()); } } } } else { if export_dir.exists() { std::fs::remove_dir_all(export_dir)?; } std::fs::create_dir_all(export_dir)?; let capability = snapshot.object_format.capability(); let mut init_cmd = git_command(export_dir); init_cmd.args(["init", "--bare", &format!("--object-format={capability}")]); run_cmd(init_cmd, "git init --bare")?; }
if let Some((name, _)) = snapshot .public_refs .iter() .find(|(name, _)| is_reserved_ref(name)) { anyhow::bail!("snapshot public_refs contains reserved ref: {name}"); }
let layout = config.layout().context("open layout for repo export")?; let source_path = layout .guarded_path(&snapshot.repo_did) .context("source repo path")?; if !source_path.exists() { anyhow::bail!("source repository not found at {}", source_path.display()); } let canonical_source = source_path .canonicalize() .with_context(|| format!("canonicalize source repo path {}", source_path.display()))?; let source_url = format!("file://{}", canonical_source.display());
for chunk in snapshot.public_refs.chunks(100) { let mut fetch_cmd = git_command(export_dir); fetch_cmd.args([ "-c", "fetch.unpackLimit=1", "-c", "transfer.unpackLimit=1", "fetch", "--no-tags", &source_url, ]); fetch_cmd.args(chunk.iter().map(|(r, _)| format!("{r}:{r}"))); run_cmd(fetch_cmd, "git fetch public refs")?; }
set_export_head(config, snapshot, export_dir)?;
let mut expire_cmd = git_command(export_dir); expire_cmd.args(["reflog", "expire", "--expire=now", "--all"]); let _ = expire_cmd.output();
let _ = std::fs::remove_dir_all(export_dir.join("logs")); for file in ["FETCH_HEAD", "ORIG_HEAD"] { let _ = std::fs::remove_file(export_dir.join(file)); }
let mut prune_cmd = git_command(export_dir); prune_cmd.args(["prune", "--expire=now"]); run_cmd(prune_cmd, "git prune --expire=now")?;
let mut update_cmd = git_command(export_dir); update_cmd.arg("update-server-info"); run_cmd(update_cmd, "git update-server-info")?;
assert_info_refs(snapshot, export_dir)?; assert_fsck_clean(export_dir)?; write_smart_artifacts(snapshot, export_dir)?;
let fingerprint = export_fingerprint(snapshot, crate::manifest::GENERATOR_PROFILE_VERSION); let fp_path = export_dir.join("export.fingerprint"); std::fs::write(&fp_path, format!("{fingerprint}\n")) .with_context(|| format!("write export.fingerprint at {}", fp_path.display()))?;
// Incremental CAS linkify let objects_dir = export_dir.join("objects"); let mut all_digests = Vec::new(); let mut updated_cache = HashMap::new();
if objects_dir.exists() { for entry in std::fs::read_dir(&objects_dir)? { let entry = entry?; let prefix = entry.file_name().to_string_lossy().to_string(); if prefix.len() != 2 { continue; } if entry.file_type()?.is_dir() { for obj_entry in std::fs::read_dir(entry.path())? { let obj_entry = obj_entry?; let git_oid = format!("{prefix}{}", obj_entry.file_name().to_string_lossy()); let path = obj_entry.path();
let cas_digest = if let Some(known) = cas_cache.get(&git_oid) { known.clone() } else { let digest = cas.put_file(&path)?; cas.link_into(&digest, &path)?; digest.hex() }; all_digests.push(cas_digest.clone()); updated_cache.insert(git_oid, cas_digest); } } } }
let mut cache_file = String::new(); for (oid, digest) in &updated_cache { cache_file.push_str(&format!("{oid} {digest}\n")); } let _ = std::fs::write(export_dir.join("objects.cas"), cache_file);
all_digests.sort(); all_digests.dedup(); Ok(all_digests)}