Something went wrong. Try again.
This repository has no description
Something went wrong. Try again.
5.9 kB · 185 lines
Rust
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186//! content-addressed store for response bodies and large artifacts.//!//! entries hold bytes only; per-route metadata (status, content type, headers) lives in//! the generation manifest. entries are shared across generations: writes are atomic and//! idempotent, and GC is mark-and-sweep from retained manifests (never during activation).
use std::io::{Read as _, Write as _};use std::path::{Path, PathBuf};
use sha2::{Digest as _, Sha256};
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct Digest([u8; 32]);
impl Digest { pub fn of_bytes(bytes: &[u8]) -> Self { Self(Sha256::digest(bytes).into()) }
pub fn of_file(path: &Path) -> std::io::Result<Self> { let mut file = std::fs::File::open(path)?; let mut hasher = Sha256::new(); let mut buffer = [0u8; 64 * 1024]; loop { let read = file.read(&mut buffer)?; if read == 0 { break; } hasher.update(&buffer[..read]); } Ok(Self(hasher.finalize().into())) }
pub fn from_raw(bytes: [u8; 32]) -> Self { Self(bytes) }
pub fn hex(&self) -> String { self.0.iter().map(|byte| format!("{byte:02x}")).collect() }
pub fn from_hex(hex: &str) -> Option<Self> { (hex.len() == 64).then_some(())?; let mut bytes = [0u8; 32]; for (i, byte) in bytes.iter_mut().enumerate() { *byte = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16).ok()?; } Some(Self(bytes)) }}
pub struct Cas { root: PathBuf,}
impl Cas { /// `root` is the `cas` directory; objects live under `sha256/ab/abcdef…` pub fn open(root: impl Into<PathBuf>) -> Self { Self { root: root.into() } }
pub fn path_for(&self, digest: &Digest) -> PathBuf { let hex = digest.hex(); self.root.join("sha256").join(&hex[..2]).join(&hex[2..]) }
pub fn has(&self, digest: &Digest) -> bool { self.path_for(digest).exists() }
/// idempotent atomic put; returns the digest. the store is the single dedup point — /// callers that need the bytes elsewhere keep them separately. pub fn put(&self, bytes: &[u8]) -> std::io::Result<Digest> { let digest = Digest::of_bytes(bytes); self.put_with(&digest, |path| atomic_write(path, bytes))?; Ok(digest) }
/// put a large artifact from disk without buffering it twice: hash the file, /// then hardlink-or-copy into place. pub fn put_file(&self, source: &Path) -> std::io::Result<Digest> { let digest = Digest::of_file(source)?; self.put_with(&digest, |path| { if source.is_dir() { copy_dir_recursive(source, path) } else { std::fs::hard_link(source, path) .or_else(|_| std::fs::copy(source, path).map(|_| ())) } })?; Ok(digest) }
pub fn link_into(&self, digest: &Digest, dest: &Path) -> std::io::Result<()> { let source = self.path_for(digest); if dest.exists() { std::fs::remove_file(dest)?; } if let Some(parent) = dest.parent() { std::fs::create_dir_all(parent)?; } std::fs::hard_link(&source, dest).or_else(|_| std::fs::copy(&source, dest).map(|_| ())) }
/// put bytes, then hardlink them to `dest`. pub fn put_and_link(&self, bytes: &[u8], dest: &Path) -> anyhow::Result<Digest> { let digest = self.put(bytes)?; self.link_into(&digest, dest)?; Ok(digest) }
/// replace every file under `root` with a hardlink to its CAS entry; return the sorted /// digests. content stays identical, and cross-generation storage dedupes (git export objects). pub fn linkify_tree(&self, root: &Path) -> anyhow::Result<Vec<String>> { let mut digests = Vec::new(); fn visit(cas: &Cas, directory: &Path, digests: &mut Vec<String>) -> anyhow::Result<()> { for entry in std::fs::read_dir(directory)? { let path = entry?.path(); if path.is_dir() { visit(cas, &path, digests)?; } else { let digest = cas.put_file(&path)?; cas.link_into(&digest, &path)?; digests.push(digest.hex()); } } Ok(()) } if root.exists() { visit(self, root, &mut digests)?; } digests.sort(); digests.dedup(); Ok(digests) }
fn put_with( &self, digest: &Digest, place: impl FnOnce(&Path) -> std::io::Result<()>, ) -> std::io::Result<()> { let path = self.path_for(digest); if path.exists() { return Ok(()); } if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } place(&path) }}
pub fn atomic_write(path: &Path, bytes: &[u8]) -> std::io::Result<()> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent)?; } let tmp = path.with_extension(format!("tmp.{}", std::process::id())); std::fs::write(&tmp, bytes)?; std::fs::rename(&tmp, path)}
fn sync_dir(path: &Path) -> std::io::Result<()> { let dir = if path.is_dir() { path } else { path.parent().unwrap_or(Path::new(".")) }; std::fs::File::open(dir)?.sync_all()}
fn copy_dir_recursive(source: &Path, dest: &Path) -> std::io::Result<()> { std::fs::create_dir_all(dest)?; for entry in std::fs::read_dir(source)? { let entry = entry?; let target = dest.join(entry.file_name()); let path = entry.path(); if entry.file_type()?.is_dir() { copy_dir_recursive(&path, &target)?; } else { std::fs::copy(&path, target)?; } } Ok(())}