//! Workspace automation. //! //! ```sh //! cargo xtask update-lexicons [--repo ] [--rev ] //! cargo xtask codegen [--check] //! ``` //! //! `update-lexicons` refreshes the vendored Tangled lexicon JSON files in //! `lexicons/` from the upstream tangled-core repository. //! //! `codegen` regenerates `crates/tangled-api/src/lexicon/` from the vendored //! lexicon JSON files in `lexicons/`. With `--check`, no files are written; //! instead the command exits non-zero if the committed generated code is out //! of date (or contains stray files). use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{bail, Context, Result}; use xtask::codegen; const DEFAULT_TANGLED_CORE_REPO: &str = "https://tangled.org/tangled.org/core"; const TANGLED_COM_ATPROTO_FILES: &[&str] = &["com/atproto/repo/strongRef.json"]; fn main() -> Result<()> { let args: Vec = std::env::args().skip(1).collect(); match args.first().map(String::as_str) { Some("codegen") => { let check = args.iter().any(|a| a == "--check"); run_codegen(check) } Some("update-lexicons") => { let options = UpdateLexiconsOptions::parse(&args[1..])?; update_lexicons(&options) } _ => { print_usage(); std::process::exit(2); } } } fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() .expect("xtask sits directly under the workspace root") .to_path_buf() } fn print_usage() { eprintln!( "usage:\n cargo xtask update-lexicons [--repo ] [--rev ]\n cargo xtask codegen [--check]" ); } struct UpdateLexiconsOptions { repo: String, rev: Option, } impl UpdateLexiconsOptions { fn parse(args: &[String]) -> Result { let mut repo = DEFAULT_TANGLED_CORE_REPO.to_string(); let mut rev = None; let mut i = 0; while i < args.len() { match args[i].as_str() { "--repo" => { i += 1; repo = args .get(i) .context("--repo requires a URL argument")? .clone(); } "--rev" => { i += 1; rev = Some( args.get(i) .context("--rev requires a revision argument")? .clone(), ); } "-h" | "--help" => { print_usage(); std::process::exit(0); } other => bail!("unknown update-lexicons argument: {other}"), } i += 1; } Ok(Self { repo, rev }) } } fn update_lexicons(options: &UpdateLexiconsOptions) -> Result<()> { let root = workspace_root(); let dst = root.join("lexicons"); let tmp = std::env::temp_dir() .join(format!("tangled-cli-lexicons-{}", std::process::id())); if tmp.exists() { std::fs::remove_dir_all(&tmp).with_context(|| { format!("removing stale temp dir {}", tmp.display()) })?; } std::fs::create_dir_all(&tmp) .with_context(|| format!("creating temp dir {}", tmp.display()))?; let checkout = tmp.join("core"); let result = (|| -> Result<()> { run_git( Command::new("git") .args([ "clone", "--depth", "1", "--filter=blob:none", "--sparse", ]) .arg(&options.repo) .arg(&checkout), "cloning tangled-core", )?; if let Some(rev) = &options.rev { run_git( Command::new("git") .arg("-C") .arg(&checkout) .args(["fetch", "--depth", "1", "origin"]) .arg(rev), "fetching requested tangled-core revision", )?; run_git( Command::new("git").arg("-C").arg(&checkout).args([ "checkout", "--detach", "FETCH_HEAD", ]), "checking out requested tangled-core revision", )?; } run_git( Command::new("git").arg("-C").arg(&checkout).args([ "sparse-checkout", "set", "lexicons", ]), "configuring sparse checkout for lexicons", )?; let commit = git_output( Command::new("git") .arg("-C") .arg(&checkout) .args(["rev-parse", "HEAD"]), "reading tangled-core commit", )?; let src = checkout.join("lexicons"); replace_lexicons(&src, &dst)?; update_lexicons_readme(&dst.join("README.md"), commit.trim())?; println!( "updated Tangled lexicons from {} @ {}", options.repo, commit.trim() ); println!("run `cargo xtask codegen` to refresh generated Rust code"); Ok(()) })(); let cleanup = std::fs::remove_dir_all(&tmp) .with_context(|| format!("removing temp dir {}", tmp.display())); result.and(cleanup) } fn replace_lexicons(src: &Path, dst: &Path) -> Result<()> { if !src.is_dir() { bail!("upstream lexicons directory not found: {}", src.display()); } std::fs::create_dir_all(dst) .with_context(|| format!("creating {}", dst.display()))?; for entry in std::fs::read_dir(dst)? { let path = entry?.path(); let name = file_name(&path)?; if name == "README.md" { continue; } // Keep the local atproto subset; only the files listed in // TANGLED_COM_ATPROTO_FILES come from tangled-core. if name == "com" { continue; } remove_path(&path)?; } for rel in TANGLED_COM_ATPROTO_FILES { let path = dst.join(rel); if path.exists() { remove_path(&path)?; } } copy_dir_contents(src, dst)?; Ok(()) } fn copy_dir_contents(src: &Path, dst: &Path) -> Result<()> { for entry in std::fs::read_dir(src)? { let path = entry?.path(); let name = file_name(&path)?; if name == "README.md" { continue; } let target = dst.join(&name); if path.is_dir() { std::fs::create_dir_all(&target) .with_context(|| format!("creating {}", target.display()))?; copy_dir_contents(&path, &target)?; } else { std::fs::copy(&path, &target).with_context(|| { format!("copying {} to {}", path.display(), target.display()) })?; } } Ok(()) } fn update_lexicons_readme(path: &Path, commit: &str) -> Result<()> { let content = std::fs::read_to_string(path) .with_context(|| format!("reading {}", path.display()))?; let mut lines = Vec::new(); let mut replaced = false; for line in content.lines() { if line.trim_start().starts_with("`lexicons/` @ `") { let indent = &line[..line.len() - line.trim_start().len()]; lines.push(format!("{indent}`lexicons/` @ `{commit}`")); replaced = true; } else { lines.push(line.to_string()); } } if !replaced { bail!( "could not find Tangled lexicon commit line in {}", path.display() ); } let mut updated = lines.join("\n"); updated.push('\n'); if updated != content { std::fs::write(path, updated) .with_context(|| format!("writing {}", path.display()))?; } Ok(()) } fn file_name(path: &Path) -> Result { path.file_name() .and_then(|name| name.to_str()) .map(str::to_string) .with_context(|| format!("invalid UTF-8 file name: {}", path.display())) } fn remove_path(path: &Path) -> Result<()> { if path.is_dir() { std::fs::remove_dir_all(path) } else { std::fs::remove_file(path) } .with_context(|| format!("removing {}", path.display())) } fn run_git(cmd: &mut Command, action: &str) -> Result<()> { let output = cmd.output().with_context(|| action.to_string())?; if !output.status.success() { bail!( "{action} failed with {}\nstdout:\n{}\nstderr:\n{}", output.status, String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); } Ok(()) } fn git_output(cmd: &mut Command, action: &str) -> Result { let output = cmd.output().with_context(|| action.to_string())?; if !output.status.success() { bail!( "{action} failed with {}\nstdout:\n{}\nstderr:\n{}", output.status, String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); } Ok(String::from_utf8(output.stdout)?) } fn run_codegen(check: bool) -> Result<()> { let root = workspace_root(); let lexicons_dir = root.join("lexicons"); let out_dir = root.join("crates/tangled-api/src"); let generated = codegen::generate(&lexicons_dir)?; for w in &generated.warnings { eprintln!("warning: {w}"); } let files = rustfmt_all(generated.files)?; let mut existing: BTreeMap<_, _> = read_tree(&out_dir.join("lexicon"))? .into_iter() .map(|(path, content)| (PathBuf::from("lexicon").join(path), content)) .collect(); let root_module = out_dir.join("lexicon.rs"); if root_module.exists() { existing.insert( PathBuf::from("lexicon.rs"), std::fs::read_to_string(&root_module).with_context(|| { format!("reading {}", root_module.display()) })?, ); } let mut stale = Vec::new(); for (path, content) in &files { if existing.get(path) != Some(content) { stale.push(path.clone()); } } let stray: Vec<_> = existing .keys() .filter(|p| !files.contains_key(*p)) .cloned() .collect(); if check { if stale.is_empty() && stray.is_empty() { println!("generated code is up to date ({} files)", files.len()); return Ok(()); } for p in &stale { eprintln!("out of date: {}", out_dir.join(p).display()); } for p in &stray { eprintln!("stray file: {}", out_dir.join(p).display()); } bail!("generated code is out of date; run `cargo xtask codegen`"); } for (path, content) in &files { let full = out_dir.join(path); if let Some(parent) = full.parent() { std::fs::create_dir_all(parent)?; } if existing.get(path) != Some(content) { std::fs::write(&full, content) .with_context(|| format!("writing {}", full.display()))?; } } for p in &stray { let full = out_dir.join(p); std::fs::remove_file(&full) .with_context(|| format!("removing {}", full.display()))?; } remove_empty_dirs(&out_dir.join("lexicon"))?; println!( "wrote {} files ({} changed, {} removed)", files.len(), stale.len(), stray.len() ); Ok(()) } /// Formats each generated file with rustfmt so output is stable and readable. fn rustfmt_all( files: BTreeMap, ) -> Result> { let mut out = BTreeMap::new(); for (path, content) in files { let formatted = rustfmt(&content).with_context(|| { format!("rustfmt failed on generated file {}", path.display()) })?; out.insert(path, formatted); } Ok(out) } fn rustfmt(source: &str) -> Result { use std::io::Write; let mut child = Command::new("rustfmt") .args(["--edition", "2021", "--emit", "stdout"]) .stdin(std::process::Stdio::piped()) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::piped()) .spawn() .context("spawning rustfmt (is it installed?)")?; child.stdin.as_mut().unwrap().write_all(source.as_bytes())?; let output = child.wait_with_output()?; if !output.status.success() { bail!( "rustfmt exited with {}: {}", output.status, String::from_utf8_lossy(&output.stderr) ); } Ok(String::from_utf8(output.stdout)?) } fn read_tree(dir: &Path) -> Result> { let mut out = BTreeMap::new(); if !dir.exists() { return Ok(out); } fn walk( root: &Path, dir: &Path, out: &mut BTreeMap, ) -> Result<()> { for entry in std::fs::read_dir(dir)? { let path = entry?.path(); if path.is_dir() { walk(root, &path, out)?; } else { let rel = path.strip_prefix(root).unwrap().to_path_buf(); out.insert(rel, std::fs::read_to_string(&path)?); } } Ok(()) } walk(dir, dir, &mut out)?; Ok(out) } fn remove_empty_dirs(dir: &Path) -> Result<()> { if !dir.exists() { return Ok(()); } for entry in std::fs::read_dir(dir)? { let path = entry?.path(); if path.is_dir() { remove_empty_dirs(&path)?; if std::fs::read_dir(&path)?.next().is_none() { std::fs::remove_dir(&path)?; } } } Ok(()) }