//! Local images in a pull request body, uploaded as blobs and rewritten to //! `blob+at://` URIs. //! //! Tangled renders a pull body with its default markdown renderer, and that //! renderer resolves exactly two kinds of image source: an absolute URL, //! which goes out through its camo proxy, and a `blob+at://did/cid` URI, //! which it turns into a `com.atproto.sync.getBlob` request against the //! owning PDS. A relative path is left exactly as written — the //! `/raw/{ref}/` rewriting READMEs get needs a repo-and-ref context the //! pull renderer does not have — so on a pull page it is always a broken //! link. A local path in a pull body can therefore only mean one thing, //! "this file, as it is on my disk right now", and that is how atgc reads //! it: the file is uploaded to the author's PDS and the path swapped for //! the `blob+at://` URI, and a relative path that names no file is refused //! rather than published broken. //! //! The record side is the lexicon's `blobs` array (`image/*`, 1 MB each). //! A PDS keeps a blob only while some record references it, so every image //! uploaded here must be listed there — see `sh.tangled.repo.pull`'s generated `Pull::blobs`. //! Everything the renderer would not touch — absolute URLs, `blob+at://` //! URIs someone already minted, anchors, `data:` — passes through unread. use anyhow::{Context, Result, bail}; use futures_util::stream::{self, StreamExt, TryStreamExt}; use jacquard::client::AgentSessionExt; use jacquard::types::blob::{Blob, MimeType}; use pulldown_cmark::{Event, LinkType, Parser, Tag}; use std::ops::Range; use std::path::{Path, PathBuf}; /// The lexicon's cap on one image: the `blobs` items say `maxSize: 1000000`. /// Enforced here so an oversized file is refused by name before anything is /// uploaded, not as a PDS error after the patch blob already went up. pub(in crate::cmd) const MAX_IMAGE_BYTES: usize = 1_000_000; /// One local file the body references: found, read and validated, not yet /// uploaded. #[derive(Debug)] struct LocalImage { /// The destination exactly as the markdown spells it — the string the /// rewrite has to find again inside each span. dest: String, path: PathBuf, mime: &'static str, bytes: Vec, /// The byte range of every `![…](dest)` in the body that uses this /// destination. One file may be embedded more than once; it is uploaded /// once and every span rewritten. spans: Vec>, } /// Every local image a body references, in the order the body mentions them. #[derive(Debug, Default)] pub(in crate::cmd) struct Images(Vec); impl Images { pub(in crate::cmd) fn none() -> Self { Self::default() } pub(in crate::cmd) fn is_empty(&self) -> bool { self.0.is_empty() } /// One line per file, for the pre-flight summary and `--dry-run`. pub(in crate::cmd) fn describe(&self) -> Vec { self.0 .iter() .map(|i| format!("{} ({} bytes, {})", i.dest, i.bytes.len(), i.mime)) .collect() } /// The same files, as fields rather than as a sentence — what the /// writing commands' `--json` puts in their `images` array. The path is /// the destination exactly as the body spells it, since that is what a /// caller wrote and what it would have to edit. pub(in crate::cmd) fn listed(&self) -> Vec { self.0 .iter() .map(|i| ImageJson { path: i.dest.clone(), bytes: i.bytes.len(), mime: i.mime.to_string(), }) .collect() } } /// One embedded image, as `--json` reports it. #[derive(serde::Serialize, Debug, PartialEq)] pub(in crate::cmd) struct ImageJson { pub path: String, pub bytes: usize, pub mime: String, } /// Find and validate every local image `body` references. /// /// Relative paths are tried against the working directory first and the /// repository root second, so a body written from either vantage point /// resolves. All the reading, sniffing and refusing happens here, before /// any network traffic — a `--dry-run` gets the full plan and the full set /// of refusals for free. pub(in crate::cmd) fn scan(body: &str) -> Result { let mut roots = vec![std::env::current_dir().context("no working directory")?]; let top = crate::clients::git::config::toplevel(Path::new(".")); if let Some(top) = &top && !roots.contains(top) { roots.push(top.clone()); } let images = scan_with_roots(body, &roots)?; for warning in dirty_warnings(&images, top.as_deref()) { crate::term::say::warning!(Image, "{warning}"); } Ok(images) } /// Warning lines for embedded images whose working-tree bytes are not what /// git has committed. /// /// The feature's contract is "this file, as it is on my disk right now" — /// but when the file is tracked and modified, the pull's *patch* delivers /// the committed version while the embedded preview shows the working /// tree's, and nothing on the rendered page says the two differ. That split /// is worth a line before the record is written. Untracked files stay /// silent: they are not in any patch, which is the arbitrary-image case /// working as intended, and warning about every screenshot saved into the /// checkout would teach people to ignore the warning that matters. fn dirty_warnings(images: &Images, repo_root: Option<&Path>) -> Vec { let Some(root) = repo_root else { return Vec::new(); }; let Ok(canon_root) = root.canonicalize() else { return Vec::new(); }; let mut warnings = Vec::new(); for img in &images.0 { let Ok(path) = img.path.canonicalize() else { continue; }; let Ok(rel) = path.strip_prefix(&canon_root) else { continue; // outside the repo: nothing to disagree with }; let rel = rel.to_string_lossy(); let Ok(status) = crate::clients::git::run::git_in(root, &["status", "--porcelain", "--", rel.as_ref()]) else { continue; // best-effort: a warning must never block a submit }; if status.lines().any(|l| !l.starts_with("??")) { warnings.push(format!( "{} has uncommitted changes: the embedded image is the working-tree \ version, not what this pull's patch delivers", img.dest )); } } warnings } /// Could this destination name a file, as opposed to something the renderer /// resolves by itself? Anything with a scheme or an anchor is Tangled's to /// handle; the camo proxy takes `https://`, the appview takes `blob+at://`. fn is_local_candidate(dest: &str) -> bool { !(dest.is_empty() || dest.starts_with('#') || dest.starts_with("//") || dest.contains("://") || dest.starts_with("data:") || dest.starts_with("mailto:")) } /// Where `dest` sits inside its own image span: the offset of the /// destination after the span's final `](`. `None` when the body's spelling /// (escapes, percent-encoding) differs from the destination CommonMark /// reports — the case scan refuses and rewrite treats as a backstop. fn dest_position(slice: &str, dest: &str) -> Option { slice .rfind("](") .and_then(|open| slice[open..].find(dest).map(|i| open + i)) } fn resolve(dest: &str, roots: &[PathBuf]) -> Option { let path = Path::new(dest); if path.is_absolute() { return path.is_file().then(|| path.to_path_buf()); } roots.iter().map(|r| r.join(path)).find(|p| p.is_file()) } fn scan_with_roots(body: &str, roots: &[PathBuf]) -> Result { let mut images: Vec = Vec::new(); for (event, span) in Parser::new(body).into_offset_iter() { let Event::Start(Tag::Image { link_type, dest_url, .. }) = event else { continue; }; let dest = dest_url.to_string(); if !is_local_candidate(&dest) { continue; } let Some(path) = resolve(&dest, roots) else { // An absolute path that names nothing here could still be a // site-absolute URL — `/{did}/{repo}/raw/…` is one Tangled // serves — so it passes through rather than being refused. One // that *exists* and is still no file is a different story: a // directory on this disk is not a URL anywhere, only a mistake. if Path::new(&dest).is_absolute() { if Path::new(&dest).exists() { bail!( "the body embeds ![…]({dest}), which exists here but is not a \ regular file" ); } crate::logging::debug::log(format!( "image {dest}: no such file; left for Tangled to resolve as a site path" )); continue; } // Same non-file honesty for the relative spelling: "no such // file" would be a lie about a directory that plainly exists. if let Some(hit) = roots.iter().map(|r| r.join(&dest)).find(|p| p.exists()) { bail!( "the body embeds ![…]({dest}), and {} is not a regular file", hit.display() ); } bail!( "the body embeds ![…]({dest}), and there is no such file (tried it \ under {})\n\ point it at a local file to upload, or use an https:// URL for \ something already hosted", roots .iter() .map(|r| r.display().to_string()) .collect::>() .join(" and ") ); }; // The rewrite replaces the destination *inside its span*, and a // reference-style image keeps its destination in a definition // somewhere else entirely. if !matches!(link_type, LinkType::Inline | LinkType::Autolink) { bail!( "the body embeds {dest} through a reference-style image\n\ spell it inline instead: ![…]({dest}), so the destination can be \ rewritten in place" ); } // The rewrite must find the destination inside its own span, and // CommonMark unescaping can make that impossible (`shot\.png` names // the file `shot.png`, a string the body never spells). Checked now // rather than at rewrite time, which is after the blobs have already // been uploaded. if dest_position(&body[span.clone()], &dest).is_none() { bail!( "could not find {dest} inside its own image span to rewrite it\n\ if the path uses escapes or percent-encoding, rename the file" ); } if let Some(existing) = images.iter_mut().find(|i| i.dest == dest) { existing.spans.push(span); continue; } let bytes = std::fs::read(&path) .with_context(|| format!("could not read image {}", path.display()))?; let Some(mime) = image_mime(&path, &bytes) else { bail!( "{} is not an image atgc recognizes (png, jpeg, gif, webp or svg)", path.display() ); }; if bytes.len() > MAX_IMAGE_BYTES { bail!( "{} is {} bytes; Tangled caps a pull's image blobs at {MAX_IMAGE_BYTES}\n\ resize it, or re-encode it as jpeg or webp", path.display(), bytes.len() ); } images.push(LocalImage { dest, path, mime, bytes, spans: vec![span], }); } Ok(Images(images)) } /// The MIME type the upload will claim, read off the bytes rather than the /// name — a `.png` that is secretly a JPEG renders fine either way, but the /// blob's stated type should not be a lie. SVG is the one text format, so /// it alone is judged by extension plus a look for its root element. fn image_mime(path: &Path, bytes: &[u8]) -> Option<&'static str> { if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { return Some("image/png"); } if bytes.starts_with(b"\xff\xd8\xff") { return Some("image/jpeg"); } if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { return Some("image/gif"); } if bytes.len() >= 12 && &bytes[0..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { return Some("image/webp"); } if path .extension() .is_some_and(|e| e.eq_ignore_ascii_case("svg")) { let head = &bytes[..bytes.len().min(1024)]; if String::from_utf8_lossy(head).contains(" crate::clients::http::Verdict { use crate::clients::http::Verdict; use jacquard::common::error::ClientErrorKind; let worth_it = match err.client_error() { Some(client) => match client.status() { Some(status) => crate::clients::http::retryable_status(status.as_u16()), None => matches!(client.kind(), ClientErrorKind::Transport), }, None => false, }; if worth_it { Verdict::Again(None) } else { Verdict::Fatal } } /// One image, on the crate's shared retry budget and schedule. /// /// A blob upload is a write, and the writes atgc refuses to retry — /// `putRecord`, `applyWrites`, `deleteRecord` — are refused because a second /// attempt after an answer atgc did not see would be a second *record*. This /// one is different in the way that matters: a blob is addressed by the hash /// of its own bytes, so uploading the same file twice puts the same CID in /// the same place, and the duplicate costs a request rather than a record. async fn upload_one( agent: &jacquard::client::Agent, img: &LocalImage, ) -> Result { let what = format!("image upload for {}", img.path.display()); crate::clients::http::retrying( crate::term::say::Topic::Image, &what, verdict, |attempt| async move { crate::logging::debug::log(format!( "uploading image blob {} ({} bytes, {}), attempt {attempt}", img.path.display(), img.bytes.len(), img.mime )); agent .upload_blob(img.bytes.clone(), MimeType::new(img.mime)) .await }, ) .await .map_err(|e| { let msg = e.to_string(); crate::logging::debug::dump_err("uploadBlob error", &e); anyhow::anyhow!("image upload failed for {}: {msg}", img.path.display()) }) } /// Upload every image to the author's PDS and swap each body reference for /// the `blob+at://` URI Tangled resolves. /// /// Uploads run [`UPLOAD_CONCURRENCY`] at a time, each with its own retries; /// one image failing all of its attempts fails the command, since a body /// with a broken image in it is not worth publishing. /// /// The returned blobs must go into the pull record's `blobs` array by the /// caller — a blob no record references is one the PDS may collect, and the /// pull would then render broken later without anything having been deleted. pub(in crate::cmd) async fn upload_and_rewrite( agent: &jacquard::client::Agent, did: &str, body: &str, images: &Images, ) -> Result<(String, Vec)> { let blobs: Vec = stream::iter(images.0.iter().map(|img| upload_one(agent, img))) .buffered(UPLOAD_CONCURRENCY) .try_collect() .await?; let uris: Vec = blobs .iter() .map(|b| format!("blob+at://{did}/{}", b.cid().as_str())) .collect(); let body = rewritten(body, images, &uris)?; Ok((body, blobs)) } /// The CID a PDS will mint for these bytes: CIDv1, raw codec, sha-256. /// /// That shape is not guessed — it is the only one `uploadBlob` produces in /// the wild, and every blob atgc has uploaded came back exactly so /// (cross-checked live against a real PDS's answer, which the test vector /// below pins). Knowing the CID before the upload is what lets the stack /// commands write a patch's *final* text at planning time, so the bytes /// the reconcile compares are stable across resubmits; [`upload_predicted`] /// holds the PDS to the prediction rather than ever publishing a URI that /// dangles. fn predicted_cid(bytes: &[u8]) -> String { use sha2::{Digest, Sha256}; let digest = Sha256::digest(bytes); let mut raw = Vec::with_capacity(36); raw.extend_from_slice(&[0x01, 0x55, 0x12, 0x20]); raw.extend_from_slice(&digest); format!("b{}", base32_lower(&raw)) } /// RFC 4648 base32, lowercase, unpadded — multibase's `b` alphabet. /// Written out here because the alternative is a dependency for a dozen /// lines. fn base32_lower(bytes: &[u8]) -> String { const ALPHABET: &[u8; 32] = b"abcdefghijklmnopqrstuvwxyz234567"; let mut out = String::with_capacity(bytes.len().div_ceil(5) * 8); for chunk in bytes.chunks(5) { let mut buf = [0u8; 5]; buf[..chunk.len()].copy_from_slice(chunk); let v = u64::from_be_bytes([0, 0, 0, buf[0], buf[1], buf[2], buf[3], buf[4]]); let bits = chunk.len() * 8; let mut taken = 0; while taken < bits { out.push(ALPHABET[((v >> (35 - taken)) & 0x1f) as usize] as char); taken += 5; } } out } impl Images { /// Each destination paired with the `blob+at://` URI its bytes will /// mint, in scan order. pub(in crate::cmd) fn predicted_pairs(&self, did: &str) -> Vec<(String, String)> { self.0 .iter() .map(|i| { ( i.dest.clone(), format!("blob+at://{did}/{}", predicted_cid(&i.bytes)), ) }) .collect() } /// The body with every scanned destination replaced by its predicted /// URI — the same splice [`upload_and_rewrite`] performs, minus the /// uploads. pub(in crate::cmd) fn rewrite_with_predictions(&self, body: &str, did: &str) -> Result { let uris: Vec = self .0 .iter() .map(|i| format!("blob+at://{did}/{}", predicted_cid(&i.bytes))) .collect(); rewritten(body, self, &uris) } } /// Upload every image and hold the PDS to the prediction. /// /// The caller has already written the predicted URIs into text that is /// about to be published, so a blob coming back under any other CID would /// leave that text dangling — the mismatch refuses the publish instead. pub(in crate::cmd) async fn upload_predicted( agent: &jacquard::client::Agent, images: &Images, ) -> Result> { let blobs: Vec = stream::iter(images.0.iter().map(|img| upload_one(agent, img))) .buffered(UPLOAD_CONCURRENCY) .try_collect() .await?; for (img, blob) in images.0.iter().zip(&blobs) { let predicted = predicted_cid(&img.bytes); if blob.cid().as_str() != predicted { bail!( "the PDS stored {} under CID {} instead of the predicted {predicted}\n\ the record text spells the prediction, so publishing would dangle; \ this PDS hashes blobs unlike any seen in the wild, and is worth reporting", img.path.display(), blob.cid().as_str(), ); } } Ok(blobs) } /// A commit-message fragment with every known destination swapped for its /// URI. /// /// Textual and deliberately narrow — the three spellings a CommonMark /// destination has inside `](…)` — because this text is a slice of a /// format-patch, not the body the scan measured, so spans are unavailable /// and precision comes from anchoring on the delimiters instead. pub(in crate::cmd) fn rewrite_message(text: &str, pairs: &[(String, String)]) -> String { let mut out = text.to_string(); for (dest, uri) in pairs { out = out.replace(&format!("]({dest})"), &format!("]({uri})")); out = out.replace(&format!("]({dest} "), &format!("]({uri} ")); out = out.replace(&format!("](<{dest}>"), &format!("](<{uri}>")); } out } /// A record's blob list after an edit: everything it already anchored plus /// everything just uploaded, one entry per CID. /// /// Union rather than replacement, because a blob an older body version /// references must stay listed or the PDS may collect it out from under the /// pull's history; deduplicated, because blobs are content-addressed, so /// re-embedding the same file mints the same CID and a naive append would /// grow the record by one copy per edit. pub(in crate::cmd) fn merge_blobs( existing: Option>, new: Vec, ) -> Option> { let mut all = existing.unwrap_or_default(); for blob in new { if !all.iter().any(|b| b.cid().as_str() == blob.cid().as_str()) { all.push(blob); } } (!all.is_empty()).then_some(all) } /// The body with every scanned destination replaced by its URI. Pure, so the /// splice arithmetic is testable without a PDS; `uris` runs parallel to the /// scan order. fn rewritten(body: &str, images: &Images, uris: &[String]) -> Result { let mut edits: Vec<(Range, &str, &str)> = Vec::new(); for (img, uri) in images.0.iter().zip(uris) { for span in &img.spans { edits.push((span.clone(), &img.dest, uri)); } } // Back to front, so each splice leaves every earlier span's offsets true. edits.sort_by_key(|e| std::cmp::Reverse(e.0.start)); let mut out = body.to_string(); for (span, dest, uri) in edits { let slice = &out[span.clone()]; // Scan already proved this position exists; the error is a backstop. let at = dest_position(slice, dest).with_context(|| { format!( "could not find {dest} inside its own image span to rewrite it\n\ if the path uses escapes or percent-encoding, rename the file" ) })?; let start = span.start + at; out.replace_range(start..start + dest.len(), uri); } Ok(out) } #[cfg(test)] mod tests { use super::{Images, MAX_IMAGE_BYTES, rewritten, scan_with_roots}; use std::path::PathBuf; /// The smallest real PNG: an 8-byte signature is enough for the sniffer, /// but writing a whole minimal file keeps the fixture honest. const PNG: &[u8] = b"\x89PNG\r\n\x1a\n0000"; /// A throwaway directory to plant image files in, named for the test so /// a leftover from a killed run says which one made it. struct TempDir(tempfile::TempDir); impl TempDir { fn new(name: &str) -> Self { Self( tempfile::Builder::new() .prefix(&format!("atgc-images-{name}-")) .tempdir() .unwrap(), ) } fn path(&self) -> PathBuf { self.0.path().to_path_buf() } fn write(&self, name: &str, bytes: &[u8]) -> PathBuf { let path = self.0.path().join(name); std::fs::write(&path, bytes).unwrap(); path } } fn scan(body: &str, root: &TempDir) -> anyhow::Result { scan_with_roots(body, &[root.path()]) } #[test] fn external_destinations_pass_through_unread() { let dir = TempDir::new("external"); let body = "![a](https://example.com/x.png) ![b](blob+at://did:plc:x/bafy) \ ![c](data:image/png;base64,AA==) ![d](#anchor) [not an image](y.png)"; let images = scan(body, &dir).unwrap(); assert!(images.is_empty(), "nothing here names a local file"); } #[test] fn a_code_fence_is_not_a_reference() { let dir = TempDir::new("fence"); // The same path inline and inside a fence: only the inline one counts, // so documentation *about* this feature does not trigger it. dir.write("shot.png", PNG); let body = "![real](shot.png)\n\n```\n![example](missing.png)\n```\n`![inline](also-missing.png)`"; let images = scan(body, &dir).unwrap(); assert_eq!(images.describe().len(), 1); assert!(images.describe()[0].starts_with("shot.png ")); } #[test] fn a_missing_relative_path_is_refused_with_the_roots_named() { let dir = TempDir::new("missing"); let err = scan("![gone](no/such.png)", &dir).unwrap_err().to_string(); assert!(err.contains("no/such.png"), "{err}"); assert!(err.contains("tried it under"), "{err}"); } #[test] fn an_absolute_path_that_is_no_file_is_left_for_tangled() { let dir = TempDir::new("site-absolute"); // Plausibly a site-absolute URL, so it passes through rather than // being refused. let images = scan("![raw](/did:plc:x/repo/raw/main/x.png)", &dir).unwrap(); assert!(images.is_empty()); } #[test] fn an_oversized_image_is_refused_by_name() { let dir = TempDir::new("oversize"); let mut big = PNG.to_vec(); big.resize(MAX_IMAGE_BYTES + 1, 0); dir.write("big.png", &big); let err = scan("![big](big.png)", &dir).unwrap_err().to_string(); assert!(err.contains("big.png"), "{err}"); assert!(err.contains(&MAX_IMAGE_BYTES.to_string()), "{err}"); } #[test] fn a_non_image_is_refused() { let dir = TempDir::new("not-image"); dir.write("notes.txt", b"just text"); let err = scan("![x](notes.txt)", &dir).unwrap_err().to_string(); assert!(err.contains("notes.txt"), "{err}"); } #[test] fn sniffing_trusts_bytes_over_names() { let dir = TempDir::new("sniff"); dir.write("actually-jpeg.png", b"\xff\xd8\xff\xe0rest"); let images = scan("![x](actually-jpeg.png)", &dir).unwrap(); assert!(images.describe()[0].contains("image/jpeg")); } #[test] fn one_file_twice_is_one_upload_and_two_rewrites() { let dir = TempDir::new("dedup"); dir.write("shot.png", PNG); let body = "before: ![a](shot.png)\nafter, same file: ![b](shot.png)"; let images = scan(body, &dir).unwrap(); assert_eq!(images.describe().len(), 1, "one upload"); let out = rewritten(body, &images, &["blob+at://did:plc:x/bafy1".into()]).unwrap(); assert_eq!( out, "before: ![a](blob+at://did:plc:x/bafy1)\nafter, same file: ![b](blob+at://did:plc:x/bafy1)" ); } #[test] fn rewriting_touches_only_the_destination() { let dir = TempDir::new("rewrite"); dir.write("shot.png", PNG); dir.write("other.png", PNG); // An alt text that repeats the path, a title, and an external image // in between: only the two local destinations may change. let body = "![shot.png](shot.png \"shot.png\") ![x](https://e.com/shot.png) ![y](other.png)"; let images = scan(body, &dir).unwrap(); let uris = vec![ "blob+at://did:plc:x/bafy1".to_string(), "blob+at://did:plc:x/bafy2".to_string(), ]; let out = rewritten(body, &images, &uris).unwrap(); assert_eq!( out, "![shot.png](blob+at://did:plc:x/bafy1 \"shot.png\") \ ![x](https://e.com/shot.png) ![y](blob+at://did:plc:x/bafy2)" ); } // ----------------------------------------------------------------------- // Roundtripping: everything that is not a local image destination must // come out of the rewrite byte-for-byte, and markdown that is broken or // merely tricky must confuse nothing. // ----------------------------------------------------------------------- #[test] fn an_image_inside_a_link_is_rewritten_and_the_link_kept() { let dir = TempDir::new("linked"); dir.write("shot.png", PNG); let body = "[![click me](shot.png)](https://example.com/full)"; let images = scan(body, &dir).unwrap(); let out = rewritten(body, &images, &["blob+at://did:plc:x/bafy1".into()]).unwrap(); assert_eq!( out, "[![click me](blob+at://did:plc:x/bafy1)](https://example.com/full)" ); } #[test] fn angle_bracketed_and_titled_destinations_rewrite_cleanly() { let dir = TempDir::new("angled"); dir.write("shot.png", PNG); let body = "![a]()\n\n![b](shot.png 'single quoted')"; let images = scan(body, &dir).unwrap(); assert_eq!(images.describe().len(), 1, "same file both times"); let out = rewritten(body, &images, &["blob+at://did:plc:x/bafy1".into()]).unwrap(); assert_eq!( out, "![a]()\n\n![b](blob+at://did:plc:x/bafy1 'single quoted')" ); } #[test] fn a_reference_style_local_image_is_refused_with_inline_advice() { let dir = TempDir::new("refstyle"); dir.write("shot.png", PNG); let err = scan("![a][cat]\n\n[cat]: shot.png", &dir) .unwrap_err() .to_string(); assert!(err.contains("reference-style"), "{err}"); assert!(err.contains("shot.png"), "{err}"); } #[test] fn raw_html_images_pass_through() { let dir = TempDir::new("rawhtml"); dir.write("shot.png", PNG); // Raw HTML is Tangled's to sanitize and render; atgc reads only // markdown image syntax, so this is found nowhere and changed // nowhere — even though the file exists. let body = "\"raw\""; let images = scan(body, &dir).unwrap(); assert!(images.is_empty()); } #[test] fn indented_code_and_an_unclosed_fence_hide_their_images() { let dir = TempDir::new("hidden"); // Neither file exists, and neither may error: a four-space indent is // an indented code block, and an unclosed fence swallows the rest of // the document. let body = "para\n\n ![indented](missing-a.png)\n\n```\n![fenced](missing-b.png)\n"; let images = scan(body, &dir).unwrap(); assert!(images.is_empty()); } #[test] fn broken_markdown_scans_without_panicking_or_uploading() { let dir = TempDir::new("broken"); // An unclosed destination, a bare shortcut with no definition, a // stray closer, an empty destination: none is an image to CommonMark, // so none may error even though no file here exists. for body in [ "![unclosed](missing.png", "![nodef]", "![nodef][]", "](missing.png)", "![empty]()", "!(missing.png)", ] { let images = scan(body, &dir).unwrap_or_else(|e| panic!("{body:?} errored: {e}")); assert!(images.is_empty(), "{body:?} should reference nothing"); } } #[test] fn an_escaped_destination_is_refused_at_scan_time() { let dir = TempDir::new("escaped"); dir.write("shot.png", PNG); // CommonMark unescapes `shot\.png` to the real filename, so the file // resolves — but the body's spelling no longer contains the // destination string, and a rewrite that guessed could splice into // the wrong bytes. Refused by the scan, so nothing has been uploaded // by the time the answer is no. let err = scan("![a](shot\\.png)", &dir).unwrap_err().to_string(); assert!(err.contains("shot.png"), "{err}"); assert!(err.contains("rename the file"), "{err}"); } #[test] fn an_absolute_path_that_is_a_directory_is_refused() { let dir = TempDir::new("absdir"); let sub = dir.path().join("shots.png"); std::fs::create_dir_all(&sub).unwrap(); let err = scan(&format!("![x]({})", sub.display()), &dir) .unwrap_err() .to_string(); assert!(err.contains("not a regular file"), "{err}"); } #[test] fn a_relative_path_that_is_a_directory_is_refused_as_such() { let dir = TempDir::new("reldir"); std::fs::create_dir_all(dir.path().join("shots.png")).unwrap(); let err = scan("![x](shots.png)", &dir).unwrap_err().to_string(); assert!( err.contains("not a regular file"), "a directory that exists must not be called 'no such file': {err}" ); } #[test] fn transient_errors_retry_and_client_errors_do_not() { use super::verdict; use crate::clients::http::Verdict; use jacquard::client::AgentError; use jacquard::common::error::ClientError; /// An `AgentError` carrying the `ClientError` a status of `code` /// produces, which is how the upload's failures reach `verdict`. fn from_status(code: u16) -> AgentError { let status = http::StatusCode::from_u16(code).expect("a real status"); AgentError::new( jacquard::client::AgentErrorKind::Client, Some(Box::new(ClientError::from( jacquard::common::error::HttpError { status, body: None }, ))), ) } for retryable in [429, 500, 502, 503, 504] { assert_eq!( verdict(&from_status(retryable)), Verdict::Again(None), "{retryable} retries" ); } for hopeless in [403, 404, 413] { assert_eq!( verdict(&from_status(hopeless)), Verdict::Fatal, "{hopeless} fails fast" ); } // No response at all: a connect, a handshake or a stalled read. The // one statusless case that is worth another attempt. let transport = AgentError::new( jacquard::client::AgentErrorKind::Client, Some(Box::new(ClientError::transport(std::io::Error::from( std::io::ErrorKind::ConnectionReset, )))), ); assert_eq!( verdict(&transport), Verdict::Again(None), "a reset connection retries" ); // 400 and 401 are decoded as the endpoint's own typed error, so no // `ClientError` is attached. Both are permanent. let typed = AgentError::sub_operation("upload blob", std::io::Error::other("nope")); assert_eq!( verdict(&typed), Verdict::Fatal, "a typed endpoint error fails fast" ); } /// The regression the typed status closes. Every one of these strings /// carries a substring the old matcher read as a status code, and none of /// them is one. #[test] fn a_status_code_hiding_in_the_prose_no_longer_decides_the_retry() { use super::verdict; use crate::clients::http::Verdict; for status in [500u16, 503] { let err = jacquard::client::AgentError::new( jacquard::client::AgentErrorKind::Client, Some(Box::new(jacquard::common::error::ClientError::from( jacquard::common::error::HttpError { status: http::StatusCode::from_u16(status).expect("a real status"), body: Some("uploading 1404 bytes to :8413 failed".into()), }, ))), ); assert_eq!( verdict(&err), Verdict::Again(None), "{status} is retryable however the body reads" ); } } #[test] fn crlf_and_unicode_do_not_shift_the_spans() { let dir = TempDir::new("offsets"); dir.write("shot.png", PNG); let body = "línea uno: 猫が好き 🐈\r\n\r\n![gato](shot.png)\r\ntail"; let images = scan(body, &dir).unwrap(); let out = rewritten(body, &images, &["blob+at://did:plc:x/bafy1".into()]).unwrap(); assert_eq!( out, "línea uno: 猫が好き 🐈\r\n\r\n![gato](blob+at://did:plc:x/bafy1)\r\ntail" ); } #[test] fn a_complex_document_roundtrips_byte_for_byte_outside_the_destinations() { let dir = TempDir::new("complex"); dir.write("logo.png", PNG); dir.write("other.png", PNG); let body = "\ # Cats ![logo](logo.png) Some *emphasis* and `inline code with ![fake](fake.png)`. > A quote with ![quoted](logo.png \"in a quote\") - item one - item ![listed](other.png) two | ![piped](logo.png) | cell | ```text ![fenced](fake.png) ``` External: ![ext](https://example.com/x.png) and ![blob](blob+at://did:plc:x/bafyold) "; let images = scan(body, &dir).unwrap(); assert_eq!(images.describe().len(), 2, "logo and other, once each"); let uris = vec![ "blob+at://did:plc:x/bafyL".to_string(), "blob+at://did:plc:x/bafyO".to_string(), ]; let out = rewritten(body, &images, &uris).unwrap(); assert_eq!( out, "\ # Cats ![logo](blob+at://did:plc:x/bafyL) Some *emphasis* and `inline code with ![fake](fake.png)`. > A quote with ![quoted](blob+at://did:plc:x/bafyL \"in a quote\") - item one - item ![listed](blob+at://did:plc:x/bafyO) two | ![piped](blob+at://did:plc:x/bafyL) | cell | ```text ![fenced](fake.png) ``` External: ![ext](https://example.com/x.png) and ![blob](blob+at://did:plc:x/bafyold) " ); // Idempotence: the rewritten body references nothing local, so a // `pr edit` that resends it uploads nothing and changes nothing. assert!(scan(&out, &dir).unwrap().is_empty()); } #[test] fn a_tracked_modified_image_warns_and_clean_or_untracked_do_not() { use super::dirty_warnings; use std::path::Path; let repo = crate::testutil::TempRepo::new("dirty-image"); let svg = ""; repo.commit("clean.svg", svg, "add clean"); repo.commit("logo.svg", svg, "add logo"); std::fs::write("logo.svg", format!("{svg}")).unwrap(); std::fs::write("untracked.svg", svg).unwrap(); let images = scan_with_roots( "![a](logo.svg) ![b](clean.svg) ![c](untracked.svg)", &[std::env::current_dir().unwrap()], ) .unwrap(); let warnings = dirty_warnings(&images, Some(Path::new("."))); assert_eq!(warnings.len(), 1, "{warnings:?}"); assert!(warnings[0].contains("logo.svg"), "{warnings:?}"); assert!(warnings[0].contains("uncommitted"), "{warnings:?}"); } #[test] fn a_staged_uncommitted_image_still_warns() { use super::dirty_warnings; use std::path::Path; // `git add` moves bytes into the index, but the patch is built from // commits — staged-and-uncommitted previews wrongly exactly like // unstaged-and-modified does, and must warn the same way. let repo = crate::testutil::TempRepo::new("staged-image"); let svg = ""; repo.commit("logo.svg", svg, "add logo"); std::fs::write("logo.svg", format!("{svg}")).unwrap(); repo.git(&["add", "logo.svg"]); let images = scan_with_roots("![a](logo.svg)", &[std::env::current_dir().unwrap()]).unwrap(); let warnings = dirty_warnings(&images, Some(Path::new("."))); assert_eq!(warnings.len(), 1, "{warnings:?}"); } #[test] fn predicted_cids_match_what_a_real_pds_mints() { use super::{base32_lower, predicted_cid}; // base32 sanity against RFC 4648: b"hello" is NBSWY3DP. assert_eq!(base32_lower(b"hello"), "nbswy3dp"); assert_eq!(base32_lower(b""), ""); // The PNG constant's CID, computed independently (sha-256 wrapped as // CIDv1/raw/base32). The same algorithm was cross-checked live: the // bytes of a stress-test image predicted // bafkreiapqi4khm7fq62rtfpbq5eriav5qrijdmt7jiowghomm3yv4e35f4, and // that is the CID the PDS answered with when they were uploaded. assert_eq!( predicted_cid(PNG), "bafkreidtipjwhvbh6wmekwxf6ebnf57izfppuffg6rklrlgdkleuldfmce" ); } #[test] fn predicted_rewrites_match_the_bytes_not_the_name() { let dir = TempDir::new("predict"); dir.write("shot.png", PNG); dir.write("same-bytes.png", PNG); let images = scan( "![a](shot.png) ![b](same-bytes.png) ![x](https://e.com/x.png)", &dir, ) .unwrap(); let out = images .rewrite_with_predictions("![a](shot.png) ![b](same-bytes.png)", "did:plc:me") .unwrap(); // Two names, one content: both predict the same CID. assert_eq!( out, "![a](blob+at://did:plc:me/bafkreidtipjwhvbh6wmekwxf6ebnf57izfppuffg6rklrlgdkleuldfmce) \ ![b](blob+at://did:plc:me/bafkreidtipjwhvbh6wmekwxf6ebnf57izfppuffg6rklrlgdkleuldfmce)" ); let pairs = images.predicted_pairs("did:plc:me"); assert_eq!(pairs.len(), 2, "external URLs predict nothing"); } #[test] fn message_rewriting_is_anchored_on_the_delimiters() { use super::rewrite_message; let pairs = vec![ ("a.png".to_string(), "blob+at://did/one".to_string()), ("b c.png".to_string(), "blob+at://did/two".to_string()), ]; let text = "plain ![x](a.png) titled ![y](a.png \"t\") angled ![z]()\n\ not this: ![w](a.png2) nor prose mentioning a.png bare"; assert_eq!( rewrite_message(text, &pairs), "plain ![x](blob+at://did/one) titled ![y](blob+at://did/one \"t\") \ angled ![z]()\n\ not this: ![w](a.png2) nor prose mentioning a.png bare" ); } #[test] fn merging_blobs_unions_by_cid_and_keeps_order() { use super::merge_blobs; let blob = |cid: &str| -> jacquard::types::blob::Blob { serde_json::from_value(serde_json::json!({ "$type": "blob", "ref": {"$link": cid}, "mimeType": "image/png", "size": 3 })) .unwrap() }; let a = "bafkreieptvktqly4bb3o2rtqp6tudeyprj4dmtkqqtw5kkqeglvfngdj6m"; let b = "bafkreiawojufuzgbjwo3p4r6xlscbpeptajepg5yyhlrtf72cnqgkngjku"; let merged = merge_blobs(Some(vec![blob(a)]), vec![blob(b), blob(a)]).unwrap(); let cids: Vec<&str> = merged.iter().map(|x| x.cid().as_str()).collect(); assert_eq!(cids, [a, b], "existing first, duplicates dropped"); // The fresh-record shape too: one file embedded under two spellings // uploads the same content-addressed blob twice, and the record's // list still carries the CID once. let fresh = merge_blobs(None, vec![blob(a), blob(a), blob(b)]).unwrap(); assert_eq!(fresh.len(), 2); assert!( merge_blobs(None, Vec::new()).is_none(), "no blobs serializes as no field, not an empty array" ); } #[test] fn adjacent_images_rewrite_without_bleeding_into_each_other() { let dir = TempDir::new("adjacent"); dir.write("a.png", PNG); dir.write("b.png", PNG); let body = "![a](a.png)![b](b.png)"; let images = scan(body, &dir).unwrap(); let uris = vec![ "blob+at://did:plc:x/bafyA".to_string(), "blob+at://did:plc:x/bafyB".to_string(), ]; let out = rewritten(body, &images, &uris).unwrap(); assert_eq!( out, "![a](blob+at://did:plc:x/bafyA)![b](blob+at://did:plc:x/bafyB)" ); } }