From f5f3239abcf5854a4c15e5932e600eb5a1842eb3 Mon Sep 17 00:00:00 2001 From: afterlifepro Date: Tue, 14 Oct 2025 23:02:44 +0100 Subject: [PATCH] upload: form an index of all files to be uploaded todo: detect mime type from file extension --- upload/Cargo.lock | 186 +++++++++++++++++++++++++++++++++++++++++- upload/Cargo.toml | 2 + upload/src/main.rs | 22 ++++- upload/src/sitemap.rs | 102 +++++++++++++++++++++++ 4 files changed, 308 insertions(+), 4 deletions(-) create mode 100644 upload/src/sitemap.rs diff --git a/upload/Cargo.lock b/upload/Cargo.lock index 8f67bd2..0677e9e 100644 --- a/upload/Cargo.lock +++ b/upload/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anstream" version = "0.6.21" @@ -38,7 +47,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -49,7 +58,7 @@ checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.60.2", ] [[package]] @@ -57,6 +66,18 @@ name = "atcities-upload" version = "0.1.0" dependencies = [ "clap", + "ignore", + "regex", +] + +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "serde", ] [[package]] @@ -105,18 +126,84 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "globset" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "log" +version = "0.4.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + [[package]] name = "once_cell_polyfill" version = "1.70.1" @@ -141,6 +228,73 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "strsim" version = "0.11.1" @@ -170,6 +324,25 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -185,6 +358,15 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" version = "0.53.5" diff --git a/upload/Cargo.toml b/upload/Cargo.toml index eb93d34..e0897cd 100644 --- a/upload/Cargo.toml +++ b/upload/Cargo.toml @@ -5,3 +5,5 @@ edition = "2024" [dependencies] clap = { version = "4.5.49", features = ["derive"] } +ignore = "0.4.23" +regex = "1.12.2" diff --git a/upload/src/main.rs b/upload/src/main.rs index d32a818..b270497 100644 --- a/upload/src/main.rs +++ b/upload/src/main.rs @@ -1,6 +1,8 @@ -use clap::Parser; +use clap::{ArgAction, Parser}; use std::path::PathBuf; +mod sitemap; + #[derive(Parser, Debug)] #[command(version, about, long_about = None)] struct Config { @@ -12,6 +14,17 @@ struct Config { #[arg(verbatim_doc_comment, short, long)] password: String, + /// Include dotfiles in upload + /// Default: false + #[arg(verbatim_doc_comment, short = 'a', long = "all")] + all_files: bool, + + /// Respect gitignore files + /// Note: gitignore files are not uploaded unless --all is set + /// Default: true + #[arg(verbatim_doc_comment, short = 'g', long = "no-gitignore", action = ArgAction::SetFalse, )] + git_ignore: bool, + /// Directory to upload #[arg(verbatim_doc_comment)] dir: PathBuf, @@ -24,13 +37,18 @@ fn main() -> Result<(), ()> { println!( "user: {} pword: {} -dir: {:?}", +dir: {:?} +all: {} +gitignore: {}", config.user, config.password, config.dir.to_str(), + config.all_files, + config.git_ignore ); // get local site info + let sitemap = sitemap::local_sitemap(config.dir, config.all_files, config.git_ignore); // create session diff --git a/upload/src/sitemap.rs b/upload/src/sitemap.rs new file mode 100644 index 0000000..8998ba3 --- /dev/null +++ b/upload/src/sitemap.rs @@ -0,0 +1,102 @@ +use ignore::WalkBuilder; +use std::{collections::HashMap, error::Error, fmt, path::PathBuf}; + +pub type Sitemap = HashMap; + +#[derive(Debug)] +pub struct SitemapNode { + pub mime_type: String, + pub blob: BlobRef, +} + +#[derive(Debug)] +pub enum BlobRef { + Local(PathBuf), + Remote(String), +} + +#[derive(Debug)] +pub enum SitemapErr { + IoErr(std::io::Error), + IgnoreErr(ignore::Error), +} + +impl From for SitemapErr { + fn from(value: std::io::Error) -> Self { + SitemapErr::IoErr(value) + } +} + +impl From for SitemapErr { + fn from(value: ignore::Error) -> Self { + SitemapErr::IgnoreErr(value) + } +} + +impl Error for SitemapErr {} + +impl fmt::Display for SitemapErr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SitemapErr::IoErr(err) => write!(f, "IO Error: {}", err), + SitemapErr::IgnoreErr(err) => write!(f, "Ignore Error: {}", err), + } + } +} + +pub fn local_sitemap<'a>( + dir: PathBuf, + include_dotfiles: bool, + use_gitignore: bool, +) -> Result { + let res = WalkBuilder::new(dir) + .hidden(!include_dotfiles) + .git_ignore(use_gitignore) + .git_exclude(use_gitignore) + .git_global(use_gitignore) + .follow_links(true) + .build() + .filter_map(|x| { + let x = x + .or_else(|err| { + println!("Warning: {} ", err); + Err(()) + }) + .ok()?; + + let metadata = x + .metadata() + .or_else(|err| { + println!("Warning on file {}:\n{} ", x.path().display(), err); + Err(()) + }) + .ok()?; + + if !metadata.is_file() { + return None; + }; + + let key = x.path().to_str()?; + let key = String::from(key); + + Some(( + key, + SitemapNode { + mime_type: String::from("application/octet-stream"), + blob: BlobRef::Local(x.path().to_owned()), + }, + )) + }) + .collect::>(); + + println!( + " - {}", + res + .iter() + .map(|(k, v)| format!("{}: {:?}", k, v)) + .collect::>() + .join("\n - ") + ); + + Ok(res) +} -- 2.51.2