Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
61 kB · 1759 lines
Rust
at main
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760//! generation compiler (packet §14, §20, §21): walk the finite request//! space over public snapshots, satisfy routes from the prior manifest when inputs are//! immutable (the generated files ARE the cache), and emit the manifest the nginx emitter//! and GC consume.
#[cfg(test)]mod tests;
use std::collections::{BTreeMap, BTreeSet, HashMap};use std::path::Path;use std::sync::{Arc, Mutex};use rayon::prelude::*;
use anyhow::Context as _;use knot_git::Repo;use knot_index::Index;use knot_types::{BranchName, RepoPath, TagName};use sha2::{Digest as _, Sha256};
use crate::cache::ReuseIndex;use crate::cas::{Cas, Digest};use crate::config::ValidatedConfig;use crate::git_export;use crate::identity;use crate::manifest::{ BuildStats, GENERATOR_PROFILE_VERSION, GenerationManifest, RepoEntry, StaticResponse,};use crate::project::bounded;use crate::project::errors::StaticXrpcError;use crate::project::pagination::{CursorMode, Endpoint, shapes_for};use crate::project::read_model::{self, BlobOutput};use crate::request_key::{ArchiveKind, Mutability, RefAlias, RepoAlias, RequestKey};
const JSON: &str = "application/json";
/// known-omitted NSIDs get pre-generated JSON errors instead of nginx HTML.const OMITTED_NSIDS: &[&str] = &[ "sh.tangled.repo.create", "sh.tangled.repo.delete", "sh.tangled.repo.rename", "sh.tangled.repo.reserveKey", "sh.tangled.repo.forkSync", "sh.tangled.repo.hiddenRef", "sh.tangled.repo.merge", "sh.tangled.repo.mergeCheck", "sh.tangled.repo.addCollaborator", "sh.tangled.repo.removeCollaborator", "sh.tangled.repo.listCollaborators", "sh.tangled.repo.setDefaultBranch", "sh.tangled.repo.deleteBranch", "sh.tangled.knot.addMember", "sh.tangled.knot.removeMember", "sh.tangled.knot.listMembers", "sh.tangled.knot.ban", "sh.tangled.knot.unban", "com.atproto.sync.subscribeRepos",];
/// everything one repo's projection needs, resolved once.struct RepoPlan { snapshot: crate::snapshot::RepoSnapshot, aliases: Vec<RepoAlias>, token: String, ref_aliases: Vec<(RefAlias, String)>, tips: BTreeSet<String>, default_branch: String, live_branches: Vec<String>, fingerprint: String, fingerprint_unchanged: bool, prior_routes: Option<BTreeMap<String, StaticResponse>>,}
pub async fn run(config: Arc<ValidatedConfig>, repo_hint: Option<&str>) -> anyhow::Result<String> { let _output_lock = crate::output::lock(&config.output_root)?; let layout = config.layout()?; layout .bootstrap_meta(&config.knot_did) .context("bootstrap meta repo")?; let meta_path = layout.meta_path(&config.knot_did)?; let index = Index::new(meta_path, layout.clone()); index.rebuild().context("rebuild registry index")?;
let mut snapshots = crate::snapshot::snapshot_all(&layout, &index)?; if let Some(hint) = repo_hint { snapshots.retain(|snapshot| snapshot.repo_did.as_str() == hint); if snapshots.is_empty() { anyhow::bail!("repo hint {hint} does not resolve to a hosted repo"); } }
let output_root = config.output_root.clone(); std::fs::create_dir_all(output_root.join("cas/sha256"))?; let generations_root = output_root.join("generations"); std::fs::create_dir_all(&generations_root)?; let cas = Cas::open(output_root.join("cas"));
let prior = load_prior_manifest(&output_root); let prior_is_complete = prior.as_ref().is_some_and(|manifest| { let gen_dir = generations_root.join(&manifest.generation_id); gen_dir.join(COMPLETION_FILE).is_file() }); let reuse = prior .as_ref() .map(ReuseIndex::from_manifest) .unwrap_or_default();
let started = std::time::Instant::now(); let sink = RouteSink::new(&cas, &reuse);
emit_service_routes(&config, &sink); emit_unsupported_routes(&sink);
let plans = snapshots .iter() .enumerate() .map(|(ordinal, snapshot)| { let token = format!("r_{ordinal:03}"); let plan = build_plan(snapshot, token, prior.as_ref())?; for alias in &plan.aliases { sink.declare_repo_alias(alias.to_param(), plan.token.clone()); } Ok(plan) }) .collect::<anyhow::Result<Vec<_>>>()?; let plans = Arc::new(plans);
emit_list_repos(&snapshots, &sink);
let private = PrivateGeneration::new(&generations_root)?; let build_dir = private.path().to_path_buf(); let git_root = build_dir.join("www/_git"); std::fs::create_dir_all(&git_root)?;
// Overlap git_export with route generation on background worker thread let export_config = Arc::clone(&config); let export_plans = Arc::clone(&plans); let export_prior = prior.clone(); let export_generations_root = generations_root.clone(); let export_git_root = git_root.clone(); let export_cas = Cas::open(output_root.join("cas"));
let export_worker = std::thread::spawn(move || -> anyhow::Result<Vec<(String, String, Vec<String>)>> { export_plans .par_iter() .map(|plan| { let export_dir = export_git_root.join(&plan.token); let reused = export_prior .as_ref() .filter(|_| prior_is_complete) .and_then(|manifest| { let (prior_token, hash) = prior_export(manifest, plan.snapshot.repo_did.as_str(), &plan.fingerprint)?; let source = export_generations_root .join(&manifest.generation_id) .join("www/_git") .join(prior_token); clone_dir(&source, &export_dir) .is_ok() .then(|| hash.to_string()) }); let (hash, objects) = match reused { Some(hash) => { let objects = export_prior .as_ref() .and_then(|manifest| manifest.section_objects.get(&plan.token).cloned()) .unwrap_or_else(|| { export_cas .linkify_tree(&export_dir.join("objects")) .unwrap_or_default() }); (hash, objects) } None => { let prior_export_dir = export_prior.as_ref().and_then(|m| { let (prior_token, _) = prior_repo_export(m, plan.snapshot.repo_did.as_str())?; let source = export_generations_root .join(&m.generation_id) .join("www/_git") .join(prior_token); source.exists().then_some(source) }); let objects = git_export::export_repo_incremental( &export_config, &plan.snapshot, &export_dir, prior_export_dir.as_deref(), &export_cas, )?; ( git_export::export_fingerprint( &plan.snapshot, GENERATOR_PROFILE_VERSION, ), objects, ) } }; Ok((plan.token.clone(), hash, objects)) }) .collect::<anyhow::Result<Vec<_>>>() });
// Concurrent route generation across all repos let _t_routes = std::time::Instant::now(); plans.par_iter().try_for_each(|plan| -> anyhow::Result<()> { let repo_sink = sink.for_section(&plan.token);
if plan.fingerprint_unchanged { if let Some(prior_routes) = &plan.prior_routes { for (route_key, response) in prior_routes { repo_sink.record(route_key.clone(), response.clone()); } return Ok(()); } } let repo = layout.open(&plan.snapshot.repo_did)?; let ctx = RepoCtx { plan, repo: &repo }; emit_repo_routes(&config, &repo_sink, &ctx)?; Ok(()) })?;
// Collect background git exports let export_results = export_worker .join() .map_err(|_| anyhow::anyhow!("git export worker panicked"))??;
let (routes, section_routes, repo_aliases) = sink.finish();
let mut manifest = GenerationManifest { generation_id: String::new(), generator_profile: GENERATOR_PROFILE_VERSION.to_string(), conf_template: crate::manifest::NGINX_TEMPLATE_VERSION.to_string(), knot_did: config.knot_did.as_str().to_string(), hostname: config.hostname.as_str().to_string(), owner: config.owner.as_str().to_string(), static_version: identity::STATIC_VERSION.to_string(), capabilities: identity::CAPABILITIES .iter() .map(|c| c.to_string()) .collect(), profile: serde_json::json!({ "compare": "default-both-directions", "blame": config.profile.blame, "archives": ["tar.gz", "zip"], "response_limit": read_model::DEFAULT_RESPONSE_LIMIT, "projection_commits": "tips", }), repos: plans .iter() .map(|plan| RepoEntry { repo_did: plan.snapshot.repo_did.as_str().to_string(), owner: plan.snapshot.owner.as_str().to_string(), rkey: plan.snapshot.rkey.as_str().to_string(), name: plan.snapshot.name.as_ref().map(|n| n.as_str().to_string()), head: plan.snapshot.head.clone(), public_refs: plan.snapshot.public_refs.clone(), object_format: plan.snapshot.object_format.capability().to_string(), fingerprint: plan.fingerprint.clone(), git_export_hash: None, }) .collect(), routes, sections: BTreeMap::new(), section_routes, section_objects: BTreeMap::new(), repo_aliases, ref_aliases: plans .iter() .flat_map(|plan| { plan.ref_aliases.iter().map(|(ref_alias, commit)| { ( format!("{}|{}", plan.token, ref_alias.to_param()), commit.clone(), ) }) }) .collect(), stats: BuildStats::default(), };
let mut export_objects: BTreeMap<String, Vec<String>> = BTreeMap::new(); let export_map: HashMap<String, (String, Vec<String>)> = export_results .into_iter() .map(|(token, hash, objects)| (token, (hash, objects))) .collect();
for (plan, entry) in plans.iter().zip(manifest.repos.iter_mut()) { if let Some((hash, objects)) = export_map.get(&plan.token) { entry.git_export_hash = Some(hash.clone()); export_objects.insert(plan.token.clone(), objects.clone()); } } manifest.section_objects = export_objects;
manifest.stats = BuildStats { repos: plans.len(), public_refs: plans.iter().map(|p| p.snapshot.public_refs.len()).sum(), commits_projected: plans.iter().map(|p| p.tips.len()).sum(), build_seconds: Some(started.elapsed().as_secs_f64()), created_at_unix: std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_secs(), ..BuildStats::default() }; manifest.finalize_id(); let _t_write = std::time::Instant::now();
let responses = build_dir.join("www/_responses"); if responses.exists() { let _ = std::fs::remove_file(&responses); let _ = std::fs::remove_dir_all(&responses); } if let Some(parent) = responses.parent() { std::fs::create_dir_all(parent)?; } #[cfg(unix)] std::os::unix::fs::symlink("../../../cas/sha256", &responses)?;
// materialize oauth client metadata for nginx static serving if let Ok(metadata) = crate::oauth::client_metadata_document(&config) { let oauth_dir = build_dir.join("www/oauth"); if std::fs::create_dir_all(&oauth_dir).is_ok() { let bytes = serde_json::to_vec_pretty(&metadata).unwrap_or_default(); let _ = std::fs::write(oauth_dir.join("client-metadata.json"), bytes); } } manifest.write(&build_dir, &cas)?; let generation_dir = generations_root.join(&manifest.generation_id); crate::nginx::emit_at(&config, &manifest, &build_dir, &generation_dir) .context("emit nginx configuration")?; let _t_pub = std::time::Instant::now(); let publication = private.publish(&generation_dir, &manifest)?;
eprintln!( "{} {} ({} repos, {} routes, {:.1}s) at {}", publication.verb(), &manifest.generation_id[..16], plans.len(), manifest.routes.len(), started.elapsed().as_secs_f64(), generation_dir.display() ); Ok(manifest.generation_id.clone())}
const COMPLETION_FILE: &str = ".complete.json";
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq)]struct FileInventory { path: String, bytes: u64, sha256: String,}
#[derive(Debug, serde::Serialize, serde::Deserialize)]struct Completion { version: u32, generation_id: String, files: Vec<FileInventory>,}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]enum Publication { Published, Reused,}
impl Publication { const fn verb(self) -> &'static str { match self { Self::Published => "generated", Self::Reused => "reused", } }}
struct PrivateGeneration { directory: tempfile::TempDir,}
impl PrivateGeneration { fn new(generations_root: &Path) -> anyhow::Result<Self> { let directory = tempfile::Builder::new() .prefix(".building-") .tempdir_in(generations_root) .context("create private generation directory")?; Ok(Self { directory }) }
fn path(&self) -> &Path { self.directory.path() }
fn publish( self, destination: &Path, expected: &GenerationManifest, ) -> anyhow::Result<Publication> { prepare_completion(self.path(), expected)?; sync_tree(self.path())?;
if destination.exists() { validate_published_generation(destination, expected).with_context(|| { format!( "generation {} already exists but is incomplete or corrupt", expected.generation_id ) })?; return Ok(Publication::Reused); }
std::fs::rename(self.path(), destination).with_context(|| { format!( "publish generation {} to {}", expected.generation_id, destination.display() ) })?; sync_directory( destination .parent() .context("generation destination has no parent")?, )?; Ok(Publication::Published) }}
fn prepare_completion(root: &Path, expected: &GenerationManifest) -> anyhow::Result<()> { validate_generation_payload(root, expected)?; let completion = Completion { version: 1, generation_id: expected.generation_id.clone(), files: payload_inventory(root)?, }; let bytes = serde_json::to_vec_pretty(&completion)?; crate::cas::atomic_write(&root.join(COMPLETION_FILE), &bytes)?; Ok(())}
fn validate_published_generation(root: &Path, expected: &GenerationManifest) -> anyhow::Result<()> { validate_generation_payload(root, expected)?; let bytes = std::fs::read(root.join(COMPLETION_FILE)).context("read generation completion marker")?; let completion: Completion = serde_json::from_slice(&bytes).context("parse generation completion marker")?; anyhow::ensure!( completion.version == 1, "unsupported completion marker version" ); anyhow::ensure!( completion.generation_id == expected.generation_id, "completion marker generation ID mismatch" ); anyhow::ensure!( completion.files == payload_inventory(root)?, "generation file inventory mismatch" ); Ok(())}
fn validate_generation_payload(root: &Path, expected: &GenerationManifest) -> anyhow::Result<()> { let manifest = GenerationManifest::load(root)?; anyhow::ensure!( manifest.generation_id == expected.generation_id, "manifest generation ID mismatch" ); anyhow::ensure!( manifest.compute_id() == manifest.generation_id, "manifest generation ID is not its semantic digest" );
let responses = root.join("www/_responses"); for response in manifest.routes.values() { let path = response_path(&responses, &response.body_hash)?; anyhow::ensure!( path.exists(), "missing response body at {}", path.display() ); }
for repo in &manifest.repos { let token = manifest .repo_aliases .get(&repo.repo_did) .with_context(|| format!("missing repo token for {}", repo.repo_did))?; let export = root.join("www/_git").join(token); anyhow::ensure!( export.is_dir(), "missing git export at {}", export.display() ); if let Some(expected_hash) = &repo.git_export_hash { let actual = std::fs::read_to_string(export.join("export.fingerprint")) .with_context(|| format!("read export fingerprint for {}", repo.repo_did))?; anyhow::ensure!( actual.trim() == expected_hash, "git export fingerprint mismatch for {}", repo.repo_did ); } }
anyhow::ensure!( root.join("nginx/knot-static.conf").is_file(), "missing nginx generation configuration" ); anyhow::ensure!( root.join("nginx/maps").is_dir(), "missing nginx maps directory" ); Ok(())}
fn payload_inventory(root: &Path) -> anyhow::Result<Vec<FileInventory>> { fn visit(root: &Path, directory: &Path, files: &mut Vec<FileInventory>) -> anyhow::Result<()> { for entry in std::fs::read_dir(directory) .with_context(|| format!("read generation directory {}", directory.display()))? { let entry = entry?; let path = entry.path(); let metadata = std::fs::symlink_metadata(&path)?; if metadata.is_symlink() { continue; } else if metadata.is_dir() { visit(root, &path, files)?; } else if metadata.is_file() { let relative = path .strip_prefix(root) .expect("inventory path is below generation root"); if relative == Path::new(COMPLETION_FILE) { continue; } files.push(FileInventory { path: relative .to_str() .context("generation path is not UTF-8")? .to_string(), bytes: metadata.len(), sha256: Digest::of_file(&path)?.hex(), }); } else { anyhow::bail!("unsupported generation entry at {}", path.display()); } } Ok(()) }
let mut files = Vec::new(); visit(root, root, &mut files)?; files.sort_by(|left, right| left.path.cmp(&right.path)); Ok(files)}
fn sync_tree(root: &Path) -> anyhow::Result<()> { fn visit(directory: &Path) -> anyhow::Result<()> { for entry in std::fs::read_dir(directory)? { let path = entry?.path(); let metadata = std::fs::symlink_metadata(&path)?; if metadata.is_symlink() { continue; } else if metadata.is_dir() { visit(&path)?; } else if metadata.is_file() { std::fs::File::open(&path)?.sync_all()?; } } sync_directory(directory)?; Ok(()) }
visit(root)}
fn sync_directory(path: &Path) -> std::io::Result<()> { std::fs::File::open(path)?.sync_all()}
fn response_path(responses: &Path, body_hash: &str) -> anyhow::Result<std::path::PathBuf> { anyhow::ensure!( body_hash.len() == 64, "invalid response digest {body_hash:?}" ); Ok(responses.join(&body_hash[..2]).join(&body_hash[2..]))}
fn prior_repo_export<'a>( manifest: &'a GenerationManifest, repo_did: &str,) -> Option<(&'a str, &'a str)> { let repo = manifest.repos.iter().find(|r| r.repo_did == repo_did)?; let token = manifest.repo_aliases.get(repo_did)?; let hash = repo.git_export_hash.as_deref()?; Some((token.as_str(), hash))}
fn prior_export<'a>( manifest: &'a GenerationManifest, repo_did: &str, fingerprint: &str,) -> Option<(&'a str, &'a str)> { let entry = manifest .repos .iter() .find(|repo| repo.repo_did == repo_did && repo.fingerprint == fingerprint)?; let token = manifest.repo_aliases.get(&entry.repo_did)?; Some((token, entry.git_export_hash.as_deref()?))}
fn build_plan( snapshot: &crate::snapshot::RepoSnapshot, token: String, prior: Option<&GenerationManifest>,) -> anyhow::Result<RepoPlan> { let mut ref_aliases: Vec<(RefAlias, String)> = Vec::new(); let mut tips: BTreeSet<String> = BTreeSet::new(); let mut live_branches: Vec<String> = Vec::new(); let mut default_branch = String::new();
if let Some(head) = &snapshot.head { ref_aliases.push((RefAlias::Head, head.clone())); tips.insert(head.clone()); } for (name, oid) in &snapshot.public_refs { if let Some(branch) = name.strip_prefix("refs/heads/") { if default_branch.is_empty() { default_branch = name.clone(); } live_branches.push(name.clone()); ref_aliases.push(( RefAlias::ShortBranch( BranchName::new(branch.to_string()).expect("branch from public ref"), ), oid.clone(), )); ref_aliases.push((RefAlias::FullBranch(name.clone()), oid.clone())); tips.insert(oid.clone()); } else if let Some(tag) = name.strip_prefix("refs/tags/") { ref_aliases.push(( RefAlias::ShortTag(TagName::new(tag.to_string()).expect("tag from public ref")), oid.clone(), )); ref_aliases.push((RefAlias::FullTag(name.clone()), oid.clone())); tips.insert(oid.clone()); } } for oid in &tips { ref_aliases.push((RefAlias::Commit(oid.clone()), oid.clone())); }
let fingerprint = repo_fingerprint(snapshot); let fingerprint_unchanged = prior.map_or(false, |manifest| { manifest.repos.iter().any(|repo| { repo.repo_did == snapshot.repo_did.as_str() && repo.fingerprint == fingerprint }) }); let prior_routes = prior.and_then(|manifest| { manifest.section_routes.get(&token).cloned().or_else(|| { let patterns: Vec<String> = repo_alias_params(snapshot) .iter() .map(|alias| format!("repo={}", alias.to_param())) .collect(); let routes: BTreeMap<String, StaticResponse> = manifest .routes .iter() .filter(|(route_key, _)| patterns.iter().any(|pattern| route_key.contains(pattern))) .map(|(route_key, response)| (route_key.clone(), response.clone())) .collect(); (!routes.is_empty()).then_some(routes) }) });
Ok(RepoPlan { snapshot: snapshot.clone(), aliases: repo_alias_params(snapshot), token, ref_aliases, tips, default_branch, live_branches, fingerprint, fingerprint_unchanged, prior_routes, })}
fn repo_alias_params(snapshot: &crate::snapshot::RepoSnapshot) -> Vec<RepoAlias> { let mut aliases = vec![ RepoAlias::Did(snapshot.repo_did.as_str().to_string()), RepoAlias::OwnerRkey { owner: snapshot.owner.as_str().to_string(), rkey: snapshot.rkey.as_str().to_string(), }, ]; if let Some(name) = &snapshot.name && name.as_str() != snapshot.rkey.as_str() { aliases.push(RepoAlias::OwnerName { owner: snapshot.owner.as_str().to_string(), name: name.as_str().to_string(), }); } aliases}
fn repo_fingerprint(snapshot: &crate::snapshot::RepoSnapshot) -> String { let mut hasher = Sha256::new(); hasher.update(GENERATOR_PROFILE_VERSION.as_bytes()); hasher.update(snapshot.repo_did.as_str().as_bytes()); if let Some(head) = &snapshot.head { hasher.update(head.as_bytes()); } for (name, oid) in &snapshot.public_refs { hasher.update(name.as_bytes()); hasher.update(oid.as_bytes()); } hasher.update(snapshot.object_format.capability().as_bytes()); Digest::from_raw(hasher.finalize().into()).hex()}
#[derive(Default, Clone)]struct RouteCollector { routes: BTreeMap<String, StaticResponse>, section_routes: BTreeMap<String, BTreeMap<String, StaticResponse>>,}
impl RouteCollector { fn record(&mut self, section: &str, route_key: String, response: StaticResponse) { self.section_routes .entry(section.to_string()) .or_default() .insert(route_key.clone(), response.clone()); self.routes.insert(route_key, response); }}
#[derive(Clone)]struct RouteSink<'a> { cas: &'a Cas, reuse: &'a ReuseIndex, section: String, collector: Arc<Mutex<RouteCollector>>, repo_aliases: Arc<Mutex<BTreeMap<String, String>>>,}
const SERVICE_SECTION: &str = "_service";
impl<'a> RouteSink<'a> { fn new(cas: &'a Cas, reuse: &'a ReuseIndex) -> Self { Self { cas, reuse, section: SERVICE_SECTION.to_string(), collector: Arc::new(Mutex::new(RouteCollector::default())), repo_aliases: Arc::new(Mutex::new(BTreeMap::new())), } }
fn for_section(&self, section: impl Into<String>) -> Self { Self { cas: self.cas, reuse: self.reuse, section: section.into(), collector: Arc::clone(&self.collector), repo_aliases: Arc::clone(&self.repo_aliases), } }
fn declare_repo_alias(&self, param: String, token: String) { self.repo_aliases.lock().unwrap().insert(param, token); }
/// insert a route and attribute it to the section being emitted. fn record(&self, route_key: String, response: StaticResponse) { self.collector .lock() .unwrap() .record(&self.section, route_key, response); }
/// reuse an identical prior entry wholesale — unchanged-repo fast path. fn reuse_prior( &self, repo_prior: Option<&BTreeMap<String, StaticResponse>>, key: &RequestKey, nsid: &str, limit_override: Option<&str>, ) -> Option<StaticResponse> { let prior = repo_prior?; let route_key = key.route_key_spelled(nsid, limit_override); let response = prior.get(&route_key)?; let digest = Digest::from_hex(&response.body_hash)?; self.cas.has(&digest).then(|| response.clone()) }
/// cross-generation reuse for immutable keys only. fn reuse_immutable(&self, key: &RequestKey) -> Option<(String, u16, String)> { if key.mutability() != Mutability::Immutable { return None; } self.reuse .lookup(key, self.cas) .map(|reused| (reused.body_hash, reused.status, reused.content_type)) }
fn try_reuse_prior( &self, repo_prior: Option<&BTreeMap<String, StaticResponse>>, key: &RequestKey, nsid: &str, limit_override: Option<&str>, ) -> bool { if key.mutability() != Mutability::Immutable { return false; } let Some(response) = self.reuse_prior(repo_prior, key, nsid, limit_override) else { return false; }; self.record(key.route_key_spelled(nsid, limit_override), response); true }
fn try_reuse_immutable( &self, key: &RequestKey, nsid: &str, limit_override: Option<&str>, ) -> bool { let Some((body_hash, status, content_type)) = self.reuse_immutable(key) else { return false; }; let route_key = key.route_key_spelled(nsid, limit_override); self.record( route_key, StaticResponse { body_hash, status, content_type, headers: vec![("cache-control".to_string(), key.cache_control().to_string())], cas_key: key.cas_key(), }, ); true }
fn store( &self, key: &RequestKey, nsid: &str, limit_override: Option<&str>, bytes: &[u8], status: u16, content_type: &str, extra_headers: Vec<(String, String)>, ) { let digest = self.cas.put(bytes).expect("cas write"); let mut headers = vec![("cache-control".to_string(), key.cache_control().to_string())]; headers.extend(extra_headers); let route_key = key.route_key_spelled(nsid, limit_override); self.record( route_key, StaticResponse { body_hash: digest.hex(), status, content_type: content_type.to_string(), headers, cas_key: key.cas_key(), }, ); }
fn put_json<T: serde::Serialize>( &self, repo_prior: Option<&BTreeMap<String, StaticResponse>>, key: &RequestKey, nsid: &str, limit_override: Option<&str>, value: &T, ) { if self.try_reuse_prior(repo_prior, key, nsid, limit_override) || self.try_reuse_immutable(key, nsid, limit_override) { return; } match read_model::to_json_bytes(value, read_model::DEFAULT_RESPONSE_LIMIT) { Ok(bytes) => self.store(key, nsid, limit_override, &bytes, 200, JSON, vec![]), Err(error) => self.put_error(key, nsid, limit_override, &error), } }
fn put_bytes( &self, repo_prior: Option<&BTreeMap<String, StaticResponse>>, key: &RequestKey, nsid: &str, limit_override: Option<&str>, bytes: &[u8], content_type: &str, extra_headers: Vec<(String, String)>, ) { if self.try_reuse_prior(repo_prior, key, nsid, limit_override) || (extra_headers.is_empty() && self.try_reuse_immutable(key, nsid, limit_override)) { return; } self.store( key, nsid, limit_override, bytes, 200, content_type, extra_headers, ); }
fn put_error( &self, key: &RequestKey, nsid: &str, limit_override: Option<&str>, error: &StaticXrpcError, ) { let body = error.body().to_string(); self.store( key, nsid, limit_override, body.as_bytes(), error.status, JSON, vec![], ); }
fn finish( self, ) -> ( BTreeMap<String, StaticResponse>, BTreeMap<String, BTreeMap<String, StaticResponse>>, BTreeMap<String, String>, ) { let collector = Arc::try_unwrap(self.collector) .map(|m| m.into_inner().unwrap()) .unwrap_or_else(|arc| arc.lock().unwrap().clone()); let aliases = Arc::try_unwrap(self.repo_aliases) .map(|m| m.into_inner().unwrap()) .unwrap_or_else(|arc| arc.lock().unwrap().clone()); (collector.routes, collector.section_routes, aliases) }}fn emit_service_routes(config: &Arc<ValidatedConfig>, sink: &RouteSink) { let did_doc = identity::knot_did_document(config).expect("static did document"); let bytes = serde_json::to_vec_pretty(&did_doc).expect("did json"); sink.store( &RequestKey::DidDocument, "did.json", None, &bytes, 200, "application/did+json", vec![], ); sink.store( &RequestKey::Version, "version", None, &serde_json::to_vec(&identity::version_body()).expect("version json"), 200, JSON, vec![], ); sink.store( &RequestKey::Owner, "owner", None, &serde_json::to_vec(&identity::owner_body(config)).expect("owner json"), 200, JSON, vec![], ); sink.store( &RequestKey::Health, "health", None, &serde_json::to_vec(&identity::health_body()).expect("health json"), 200, JSON, vec![], );}
fn emit_unsupported_routes(sink: &RouteSink) { let body = serde_json::json!({ "error": "NotFound", "message": "this capability is not served by this static knot", }) .to_string(); for nsid in OMITTED_NSIDS { let key = RequestKey::Unsupported { nsid: nsid.to_string(), }; sink.store(&key, nsid, None, body.as_bytes(), 404, JSON, vec![]); }}
fn emit_list_repos(snapshots: &[crate::snapshot::RepoSnapshot], sink: &RouteSink) { const NSID: &str = "sh.tangled.sync.listRepos"; for descending in [true, false] { for shape in shapes_for(Endpoint::ListRepos) { let mut cursor = 0usize; loop { let key = RequestKey::ListRepos { limit: shape.effective_limit, cursor, descending, }; match read_model::list_repos( snapshots, cursor, Some(shape.effective_limit), descending, ) { Ok(out) => { for spelling in shape.limit_raw_aliases { sink.put_json(None, &key, NSID, Some(spelling), &out); } if out.repos.is_empty() { break; } } Err(error) => { sink.put_error(&key, NSID, None, &error); break; } } match shape.cursor_mode { CursorMode::Zero => break, CursorMode::MultipleOf(step) => cursor += step, CursorMode::AnyOffset => cursor += 1, } } } }}
struct RepoCtx<'a> { plan: &'a RepoPlan, repo: &'a Repo,}
fn emit_repo_routes( config: &Arc<ValidatedConfig>, sink: &RouteSink, ctx: &RepoCtx,) -> anyhow::Result<()> { let repo_prior = ctx.plan.prior_routes.as_ref(); let alias0 = ctx.plan.aliases[0].clone();
sink.put_json( repo_prior, &RequestKey::DescribeRepo { repo: alias0.clone(), }, "sh.tangled.repo.describeRepo", None, &read_model::describe_repo(&ctx.plan.snapshot), ); if let Ok(out) = read_model::get_default_branch(ctx.repo) { for alias in &ctx.plan.aliases { sink.put_json( repo_prior, &RequestKey::GetDefaultBranch { repo: alias.clone(), }, "sh.tangled.repo.getDefaultBranch", None, &out, ); } }
for (ref_alias, _oid) in &ctx.plan.ref_aliases { let branch_name = match ref_alias { RefAlias::ShortBranch(name) => Some(name.clone()), RefAlias::FullBranch(name) => BranchName::new(name.clone()).ok(), _ => None, }; if let Some(name) = &branch_name && let Ok(out) = read_model::branch(ctx.repo, name) { for alias in &ctx.plan.aliases { sink.put_json( repo_prior, &RequestKey::Branch { repo: alias.clone(), name: name.clone(), }, "sh.tangled.repo.branch", None, &out, ); } } let tag_name = match ref_alias { RefAlias::ShortTag(name) => Some(name.clone()), RefAlias::FullTag(name) => TagName::new(name.clone()).ok(), _ => None, }; if let Some(name) = &tag_name && let Ok(out) = read_model::tag(ctx.repo, name) { for alias in &ctx.plan.aliases { sink.put_json( repo_prior, &RequestKey::Tag { repo: alias.clone(), name: name.clone(), }, "sh.tangled.repo.tag", None, &out, ); } } }
let t_page = std::time::Instant::now();
let t_pag = std::time::Instant::now(); for (endpoint, nsid) in [ (Endpoint::Branches, "sh.tangled.repo.branches"), (Endpoint::Tags, "sh.tangled.repo.tags"), (Endpoint::ListRefs, "sh.tangled.git.listRefs"), ] { for shape in shapes_for(endpoint) { let mut cursor = 0usize; loop { let exhausted = (|| -> Result<bool, StaticXrpcError> { match endpoint { Endpoint::Branches => { let out = read_model::branches( ctx.repo, cursor, Some(shape.effective_limit), )?; for spelling in shape.limit_raw_aliases { let key = RequestKey::Branches { repo: alias0.clone(), limit: shape.effective_limit, cursor, }; sink.put_json(repo_prior, &key, nsid, Some(spelling), &out); } Ok(out.branches.is_empty()) } Endpoint::Tags => { let out = read_model::tags(ctx.repo, cursor, Some(shape.effective_limit))?; for spelling in shape.limit_raw_aliases { let key = RequestKey::Tags { repo: alias0.clone(), limit: shape.effective_limit, cursor, }; sink.put_json(repo_prior, &key, nsid, Some(spelling), &out); } Ok(out.tags.is_empty()) } Endpoint::ListRefs => { let out = read_model::list_refs( ctx.repo, cursor, Some(shape.effective_limit), )?; for spelling in shape.limit_raw_aliases { let key = RequestKey::ListRefs { repo: alias0.clone(), limit: shape.effective_limit, cursor, }; sink.put_json(repo_prior, &key, nsid, Some(spelling), &out); } Ok(out.refs.is_empty()) } _ => Ok(true), } })(); match exhausted { Ok(true) => break, Ok(false) => {} Err(error) => { let key = RequestKey::Branches { repo: alias0.clone(), limit: shape.effective_limit, cursor, }; sink.put_error(&key, nsid, None, &error); break; } } match shape.cursor_mode { CursorMode::Zero => break, CursorMode::MultipleOf(step) => cursor += step, CursorMode::AnyOffset => cursor += 1, } } } }
let t_walk = std::time::Instant::now(); let mut tip_files: Vec<String> = Vec::new();
for (alias, oid) in &ctx.plan.ref_aliases { } let mut commit_aliases: BTreeMap<&str, Vec<&RefAlias>> = BTreeMap::new(); for (ref_alias, oid) in &ctx.plan.ref_aliases { commit_aliases.entry(oid.as_str()).or_default().push(ref_alias); }
for (&oid, aliases) in &commit_aliases { let files = walk_commit_unique(sink, repo_prior, &alias0, ctx.repo, &aliases, oid)?; emit_languages_unique(sink, repo_prior, &alias0, ctx.repo, &aliases, oid); emit_diff_unique(sink, repo_prior, &alias0, ctx.repo, &aliases, oid); if aliases.iter().any(|a| matches!(a, RefAlias::Commit(_))) { tip_files = files; } }
let t_cmp = std::time::Instant::now(); emit_compares(sink, ctx); let t_log = std::time::Instant::now();
let mut log_cache: BTreeMap<&str, Vec<(usize, usize, bounded::LogOut)>> = BTreeMap::new(); for (ref_alias, oid) in &ctx.plan.ref_aliases { let key = RequestKey::Log { repo: alias0.clone(), refspec: ref_alias.clone(), limit: 50, cursor: 0, }; if sink.try_reuse_prior(repo_prior, &key, "sh.tangled.repo.log", None) || sink.try_reuse_immutable(&key, "sh.tangled.repo.log", None) { continue; } let description = ref_alias.to_param(); let pages: Vec<(usize, usize, bounded::LogOut)> = if let Some(cached) = log_cache.get(oid.as_str()) { cached .iter() .map(|(limit, cursor, out)| { let mut out = out.clone(); out.refspec = description.clone(); out.description = String::new(); (*limit, *cursor, out) }) .collect() } else { match bounded::generate_log_pages_for_ref( ctx.repo, parse_oid(oid)?, &description, &description, ) { Ok(pages) => { log_cache.insert(oid.as_str(), pages.clone()); pages } Err(_) => Vec::new(), } }; for (limit, cursor, out) in pages { let key = RequestKey::Log { repo: alias0.clone(), refspec: ref_alias.clone(), limit, cursor, }; let limit_str = limit.to_string(); let limit_override = if limit == 50 { "" } else { &limit_str }; sink.put_json( repo_prior, &key, "sh.tangled.repo.log", Some(limit_override), &out, ); } }
let t_arch = std::time::Instant::now(); let rkey = ctx.plan.snapshot.rkey.as_str(); let repo_path = ctx.repo.path().to_path_buf();
struct ArchiveTask { key: RequestKey, ref_param: String, format: knot_git::ArchiveFormat, cache_key: String, }
let mut pending_tasks: Vec<ArchiveTask> = Vec::new(); let mut unique_jobs: BTreeMap<String, (knot_types::Oid, String, knot_git::ArchiveFormat)> = BTreeMap::new();
for kind in [ArchiveKind::TarGz, ArchiveKind::Zip] { let format = match kind { ArchiveKind::TarGz => knot_git::ArchiveFormat::TarGz, ArchiveKind::Zip => knot_git::ArchiveFormat::Zip, }; for (ref_alias, oid) in &ctx.plan.ref_aliases { let key = RequestKey::Archive { repo: alias0.clone(), refspec: ref_alias.clone(), kind, }; if sink.try_reuse_prior(repo_prior, &key, "sh.tangled.repo.archive", None) || sink.try_reuse_immutable(&key, "sh.tangled.repo.archive", None) { continue; } let stem = bounded::default_archive_stem(rkey, &ref_alias.to_param()); let cache_key = format!("{oid}:{format:?}:{}", stem.as_str()); if let Ok(parsed) = parse_oid(oid) { unique_jobs .entry(cache_key.clone()) .or_insert_with(|| (parsed, ref_alias.to_param(), format)); } pending_tasks.push(ArchiveTask { key, ref_param: ref_alias.to_param(), format, cache_key, }); } }
let unique_list: Vec<(String, knot_types::Oid, String, knot_git::ArchiveFormat)> = unique_jobs .into_iter() .map(|(k, (oid, refspec, fmt))| (k, oid, refspec, fmt)) .collect();
let computed_archives: HashMap<String, Result<bounded::ArchiveOutput, StaticXrpcError>> = unique_list .into_par_iter() .map_init( || knot_git::Repo::open(&repo_path).expect("open repo for archive worker"), |worker_repo, (cache_key, commit_oid, refspec, format)| { let res = bounded::execute_archive( worker_repo, commit_oid, rkey, &refspec, format, None, ); (cache_key, res) }, ) .collect();
for task in pending_tasks { match computed_archives.get(&task.cache_key) { Some(Ok(out)) => { sink.put_bytes( repo_prior, &task.key, "sh.tangled.repo.archive", None, &out.bytes, out.content_type, vec![("content-disposition".to_string(), out.content_disposition.clone())], ); } Some(Err(error)) => { sink.put_error(&task.key, "sh.tangled.repo.archive", None, error); } None => {} } }
Ok(())}
fn emit_languages_unique( sink: &RouteSink, repo_prior: Option<&BTreeMap<String, StaticResponse>>, alias: &RepoAlias, repo: &Repo, aliases: &[&RefAlias], oid: &str,) { let _ = oid; for ref_alias in aliases { let key = RequestKey::Languages { repo: alias.clone(), refspec: (*ref_alias).clone(), }; if sink.try_reuse_prior(repo_prior, &key, "sh.tangled.repo.languages", None) || sink.try_reuse_immutable(&key, "sh.tangled.repo.languages", None) { continue; } match read_model::languages(repo, &ref_alias.to_param(), None) { Ok(out) => sink.put_json(repo_prior, &key, "sh.tangled.repo.languages", None, &out), Err(error) => sink.put_error(&key, "sh.tangled.repo.languages", None, &error), } }}
fn emit_diff_unique( sink: &RouteSink, repo_prior: Option<&BTreeMap<String, StaticResponse>>, alias: &RepoAlias, repo: &Repo, aliases: &[&RefAlias], oid: &str,) { let _ = oid; for ref_alias in aliases { let key = RequestKey::Diff { repo: alias.clone(), refspec: (*ref_alias).clone(), }; if sink.try_reuse_prior(repo_prior, &key, "sh.tangled.repo.diff", None) || sink.try_reuse_immutable(&key, "sh.tangled.repo.diff", None) { continue; } match read_model::diff(repo, &ref_alias.to_param()) { Ok(out) => sink.put_json(repo_prior, &key, "sh.tangled.repo.diff", None, &out), Err(error) => sink.put_error(&key, "sh.tangled.repo.diff", None, &error), } }}
fn emit_blame( config: &Arc<ValidatedConfig>, sink: &RouteSink, repo: &Repo, ref_alias: &RefAlias, oid: &str, files: &[String],) { let Ok(commit) = parse_oid(oid) else { return; }; let repo_alias = repo_alias_for_blame(config); for path in files { let Ok(repo_path) = RepoPath::new(path) else { continue; }; let key = RequestKey::Blame { repo: repo_alias.clone(), refspec: ref_alias.clone(), path: path.clone(), }; match bounded::execute_blame(repo, config.profile.blame, commit, true, &repo_path) { Ok(bytes) => sink.put_bytes( None, &key, "sh.tangled.repo.blame", None, &bytes, JSON, vec![], ), Err(error) => sink.put_error(&key, "sh.tangled.repo.blame", None, &error), } }}
fn repo_alias_for_blame(config: &Arc<ValidatedConfig>) -> RepoAlias { // blame always uses aliases[0] (the DID spelling) RepoAlias::Did(config.knot_did.as_str().to_string())}
fn emit_compares(sink: &RouteSink, ctx: &RepoCtx) { let pairs = bounded::bounded_compare_pairs(&ctx.plan.default_branch, &ctx.plan.live_branches); let branch_refs: Vec<&str> = ctx.plan.live_branches.iter().map(String::as_str).collect(); let repo_path = ctx.repo.path().to_path_buf();
pairs.into_par_iter().for_each_init( || knot_git::Repo::open(&repo_path).expect("open repo for compare worker"), |repo, (rev1, rev2)| { let (Ok(base), Ok(head)) = ( read_model::commit_for(repo, rev1), read_model::commit_for(repo, rev2), ) else { return; }; let rev1_alias = revspec_alias(rev1); let rev2_alias = revspec_alias(rev2); let result = bounded::evaluate_compare( repo, &ctx.plan.default_branch, &branch_refs, rev1, rev2, base, head, ); for alias in &ctx.plan.aliases { let key = RequestKey::Compare { repo: alias.clone(), rev1: rev1_alias.clone(), rev2: rev2_alias.clone(), }; match &result { Ok(out) => sink.put_json(None, &key, "sh.tangled.repo.compare", None, out), Err(error) => sink.put_error(&key, "sh.tangled.repo.compare", None, error), } } }, );}
fn revspec_alias(revspec: &str) -> RefAlias { revspec .strip_prefix("refs/heads/") .and_then(|branch| BranchName::new(branch.to_string()).ok()) .map(RefAlias::ShortBranch) .unwrap_or_else(|| RefAlias::Commit(revspec.to_string()))}
/// depth-first directory walk emitting tree + blob (json and raw) routes;/// returns the file paths seen at this commit (used for blame).fn walk_commit_unique( sink: &RouteSink, repo_prior: Option<&BTreeMap<String, StaticResponse>>, alias: &RepoAlias, repo: &Repo, aliases: &[&RefAlias], oid: &str,) -> anyhow::Result<Vec<String>> { let commit_oid = parse_oid(oid)?; let mut files = Vec::new(); let mut queue: Vec<Option<String>> = vec![None]; while let Some(dir) = queue.pop() { let repo_path = dir .as_deref() .map(|p| RepoPath::new(p).expect("walked path"));
let mut tree_cached = None; for ref_alias in aliases { let key = RequestKey::Tree { repo: alias.clone(), refspec: (*ref_alias).clone(), path: dir.clone(), }; if !sink.try_reuse_prior(repo_prior, &key, "sh.tangled.repo.tree", None) && !sink.try_reuse_immutable(&key, "sh.tangled.repo.tree", None) { if tree_cached.is_none() { tree_cached = Some(read_model::tree( repo, oid, repo_path.as_ref(), None, read_model::DEFAULT_RESPONSE_LIMIT, )); } match tree_cached.as_ref().unwrap() { Ok(out) => { let mut out = out.clone(); out.refspec = ref_alias.to_param(); sink.put_json(repo_prior, &key, "sh.tangled.repo.tree", None, &out); } Err(error) => sink.put_error(&key, "sh.tangled.repo.tree", None, error), } } }
if let Ok(Some(entries)) = repo.tree_entries_at(commit_oid, repo_path.as_ref()) { for entry in entries { let child = match &dir { Some(parent) => format!("{parent}/{}", entry.name), None => entry.name.clone(), }; match entry.kind { knot_git::EntryKind::Tree => queue.push(Some(child)), _ => files.push(child), } } } }
let repo_path = repo.path().to_path_buf(); files.par_iter().for_each_init( || knot_git::Repo::open(&repo_path).expect("open repo for blob worker"), |worker_repo, child| { let mut blob_tree_cached = None; for ref_alias in aliases { let blob_tree_key = RequestKey::Tree { repo: alias.clone(), refspec: (*ref_alias).clone(), path: Some(child.clone()), }; if !sink.try_reuse_prior(repo_prior, &blob_tree_key, "sh.tangled.repo.tree", None) && !sink.try_reuse_immutable(&blob_tree_key, "sh.tangled.repo.tree", None) { if blob_tree_cached.is_none() { blob_tree_cached = Some(read_model::tree( worker_repo, oid, Some(&RepoPath::new(child).expect("walked path")), None, read_model::DEFAULT_RESPONSE_LIMIT, )); } match blob_tree_cached.as_ref().unwrap() { Ok(out) => { let mut out = out.clone(); out.refspec = ref_alias.to_param(); sink.put_json( repo_prior, &blob_tree_key, "sh.tangled.repo.tree", None, &out, ); } Err(error) => { sink.put_error(&blob_tree_key, "sh.tangled.repo.tree", None, error); } } } } emit_blob_for_aliases(sink, repo_prior, alias, worker_repo, aliases, oid, child); }, );
Ok(files)}fn emit_blob_for_aliases( sink: &RouteSink, repo_prior: Option<&BTreeMap<String, StaticResponse>>, alias: &RepoAlias, repo: &Repo, aliases: &[&RefAlias], oid: &str, path: &str,) { let mut json_cached = None; let mut raw_cached = None;
for ref_alias in aliases { let json_key = RequestKey::Blob { repo: alias.clone(), refspec: (*ref_alias).clone(), path: path.to_string(), raw: false, }; if !sink.try_reuse_prior(repo_prior, &json_key, "sh.tangled.repo.blob", None) && !sink.try_reuse_immutable(&json_key, "sh.tangled.repo.blob", None) { if json_cached.is_none() { json_cached = Some(read_model::blob( repo, oid, path, false, None, read_model::DEFAULT_RESPONSE_LIMIT, )); } match json_cached.as_ref().unwrap() { Ok(BlobOutput::Json(out)) => { let mut out = out.clone(); out.refspec = ref_alias.to_param(); sink.put_json(repo_prior, &json_key, "sh.tangled.repo.blob", None, &out); } Ok(BlobOutput::Raw(raw)) => sink.put_bytes( repo_prior, &json_key, "sh.tangled.repo.blob", None, &raw.contents, raw.mime, vec![], ), Err(error) => sink.put_error(&json_key, "sh.tangled.repo.blob", None, error), } }
let raw_key = RequestKey::Blob { repo: alias.clone(), refspec: (*ref_alias).clone(), path: path.to_string(), raw: true, }; if !sink.try_reuse_prior(repo_prior, &raw_key, "sh.tangled.repo.blob", None) && !sink.try_reuse_immutable(&raw_key, "sh.tangled.repo.blob", None) { if raw_cached.is_none() { raw_cached = Some(read_model::blob( repo, oid, path, true, None, read_model::DEFAULT_RESPONSE_LIMIT, )); } match raw_cached.as_ref().unwrap() { Ok(BlobOutput::Json(out)) => { let mut out = out.clone(); out.refspec = ref_alias.to_param(); sink.put_json(repo_prior, &raw_key, "sh.tangled.repo.blob", None, &out); } Ok(BlobOutput::Raw(raw)) => { let headers = vec![ ("x-content-type-options".to_string(), "nosniff".to_string()), ( "content-security-policy".to_string(), "default-src 'none'; style-src 'unsafe-inline'; sandbox".to_string(), ), ]; sink.put_bytes( repo_prior, &raw_key, "sh.tangled.repo.blob", None, &raw.contents, raw.mime, headers, ); } Err(error) => sink.put_error(&raw_key, "sh.tangled.repo.blob", None, error), } } }}
fn parse_oid(hex: &str) -> anyhow::Result<knot_types::Oid> { knot_types::Oid::from_hex(hex).context("parse oid")}
pub(crate) fn clone_dir(source: &Path, destination: &Path) -> std::io::Result<()> { std::fs::create_dir_all(destination)?; for entry in std::fs::read_dir(source)? { let entry = entry?; let target = destination.join(entry.file_name()); if entry.file_type()?.is_dir() { clone_dir(&entry.path(), &target)?; } else if std::fs::hard_link(entry.path(), &target).is_err() { std::fs::copy(entry.path(), target)?; } } Ok(())}
fn load_prior_manifest(output_root: &Path) -> Option<GenerationManifest> { GenerationManifest::load(&output_root.join("current")).ok()}